I'm trying to webscrape the news from the following URL:

CodePudding user response：

As one of the comments mentioned, you can use

as you can see above, there is no div id="search" element; in such cases, the commented out selector might work.

Sample usage:

# selectors for headerless request in comments

blockSel = '#search div[eid] div[data-hveid][data-ved] > div[data-hveid]'
# blockSel = '#main > div > div > a'

innerSels = {
    'heading': 'div[role="heading"]', # 'h3',
    'link': (None, 'href'), # (None, 'href'),
    'snippet': 'div[role="heading"]   div', # '"parent"a > div   div',
    'date': 'div[role="heading"] ~ span   div', # 'div   div div > span',
    'site_name': 'g-img   span' # 'h3   div'
}  

articles = []

srSectn = soup.select(blockSel)
srsLen = len(srSectn)
for i, s in enumerate(srSectn): 
    if s.select_one('a[jsname]'): s = s.select_one('a[jsname]')
    print('', end=f'\radding article {i 1} of {srsLen}...')

    aData = {}
    for k in innerSels:
        sel = innerSels[k]
        target = '"text"'
        if type(sel) in [tuple, list] and len(sel) > 1:
            target = None if sel[1] is None else str(sel[1])
            sel = None if sel[0] is None else str(sel[0])
        
        if type(sel) == str and sel.startswith('"parent"'): 
            sel = s.parent.select_one(sel.replace('"parent"', '', 1))
        else: 
            sel = s if sel is None else s.select_one(sel)
        if sel is None:
            aData[k] = None
            continue

        if target is None: 
            aData[k] = str(sel)
        elif target == '"text"': 
            aData[k] = sel.get_text(strip=True)
        else: aData[k] = sel.get(target)
        
    articles.append(aData)

print(f'\radded {len(articles)} articles from {srsLen} sections')

Data collected:

CodePudding user response：

Just in addition to simplify the selection a bit - You could use all <a> that has a <h3> as container for your iterations:

soup.select('a:has(h3)')

Example

Uses cookies={'CONSENT':'YES '} cause it is necessary from my location to set, but could be ignored from yours.

from bs4 import BeautifulSoup
import requests
import pandas as pd

url = 'https://www.google.com/search?num=250&q=Apple innovation performance&oq=Apple innovation performance=1600&source=lnt&tbs=cdr:1,cd_min:1/1/2018,cd_max:12/31/2018&tbm=nws&hl=en-US'
response = requests.get(url,headers = {'User-Agent': 'Mozilla/5.0'}, cookies={'CONSENT':'YES '})

data = []

soup = BeautifulSoup(response.text)
for e in soup.select('a:has(h3)'):
    data.append({
        'title': e.h3.get_text(),
        'date': e.span.get_text(),
        'excerpt':e.br.previous,
        'site': e.h3.find_next_sibling('div').get_text() if e.h3.find_next_sibling('div') else None,
        'url': e.get('href').strip('/url?q=')
    })
pd.DataFrame(data)

Output

...