Scrape hrefs from two table in the same page-CodePudding

I would scrape hrefs of the matches, after clicking the button in the page "Show all matches" to load all the matches, from this website url = https://www.soccerstats.com/.

I want to get the stats link

I am building this function

def get_matches(url):
    options = Options()
    options.add_argument('--blink-settings=imagesEnabled=false')
    options.headless = True
    options.add_argument("--start-maximised")
    driver = webdriver.Chrome(executable_path=path options= options)

    # Open the link
    driver.get(url)
    sleep(10)
    driver.refresh()
    
    driver.find_element_by_tag_name('body').send_keys(Keys.END)
    more_matches = WebDriverWait(driver, 2).until(EC.visibility_of_element_located((By.XPATH, '//*[@id="content"]/div[3]/div[2]/table/tbody/tr[2]/td/table[4]/tbody/tr/td[2]/button')))
    driver.execute_script("arguments[0].click();", more_matches)
    
    wrap_ = driver.find_element_by_id("btable")
    trs = wrap_.find_elements_by_tag_name('tr')
    
    _lst = list()
    
    # Loop through and print all tr's 
    for tr in trs:
        href = tr.get_attribute('href')
        print(href)

But I get None as output

CodePudding user response：

Does this not get you all the stats links?

import requests
from bs4 import BeautifulSoup

url = 'https://www.soccerstats.com/'

response = requests.get(url)
soup = BeautifulSoup(response.text, 'html.parser')

a_tags = soup.find_all('a', href=True, text='stats')
links = ['https://www.soccerstats.com/'   x['href'] for x in a_tags]

Output:

print(links)
['https://www.soccerstats.com/latest.asp?league=iran', 'https://www.soccerstats.com/latest.asp?league=italy3', 'https://www.soccerstats.com/latest.asp?league=kosovo', 'https://www.soccerstats.com/latest.asp?league=serbia2', 'https://www.soccerstats.com/latest.asp?league=spain', 'https://www.soccerstats.com/pmatch.asp?league=spain&stats=147-3-19-2022', 'https://www.soccerstats.com/latest.asp?league=portugal', 'https://www.soccerstats.com/pmatch.asp?league=portugal&stats=107-4-12-2022', 'https://www.soccerstats.com/latest.asp?league=england2', 'https://www.soccerstats.com/pmatch.asp?league=england2&stats=511-9-15-2022', 'https://www.soccerstats.com/latest.asp?league=italy2', 'https://www.soccerstats.com/pmatch.asp?league=italy2&stats=149-9-17-2022', 'https://www.soccerstats.com/latest.asp?league=spain2', 'https://www.soccerstats.com/pmatch.asp?league=spain2&stats=197-2-7-2022', 'https://www.soccerstats.com/latest.asp?league=netherlands2', 'https://www.soccerstats.com/pmatch.asp?league=netherlands2&stats=195-10-18-2022', 'https://www.soccerstats.com/latest.asp?league=portugal2', 'https://www.soccerstats.com/pmatch.asp?league=portugal2&stats=108-9-13-2022', 'https://www.soccerstats.com/latest.asp?league=algeria2', 'https://www.soccerstats.com/pmatch.asp?league=algeria2&stats=94-30-8-2022', 'https://www.soccerstats.com/pmatch.asp?league=algeria2&stats=83-13-21-2022', 'https://www.soccerstats.com/pmatch.asp?league=algeria2&stats=88-10-17-2022', 'https://www.soccerstats.com/pmatch.asp?league=algeria2&stats=92-2-4-2022', 'https://www.soccerstats.com/pmatch.asp?league=algeria2&stats=86-14-24-2022', 'https://www.soccerstats.com/pmatch.asp?league=algeria2&stats=87-12-23-2022', 'https://www.soccerstats.com/pmatch.asp?league=algeria2&stats=89-25-3-2022', 'https://www.soccerstats.com/pmatch.asp?league=algeria2&stats=93-32-6-2022', 'https://www.soccerstats.com/pmatch.asp?league=algeria2&stats=95-28-7-2022', 'https://www.soccerstats.com/pmatch.asp?league=algeria2&stats=85-16-22-2022', 'https://www.soccerstats.com/pmatch.asp?league=algeria2&stats=96-26-1-2022', 'https://www.soccerstats.com/pmatch.asp?league=algeria2&stats=91-29-5-2022', 'https://www.soccerstats.com/pmatch.asp?league=algeria2&stats=84-18-20-2022', 'https://www.soccerstats.com/pmatch.asp?league=algeria2&stats=90-27-31-2022', 'https://www.soccerstats.com/pmatch.asp?league=algeria2&stats=82-11-15-2022', 'https://www.soccerstats.com/pmatch.asp?league=algeria2&stats=81-9-19-2022', 'https://www.soccerstats.com/latest.asp?league=argentina', 'https://www.soccerstats.com/pmatch.asp?league=argentina&stats=187-1-10-2022', 'https://www.soccerstats.com/pmatch.asp?league=argentina&stats=190-14-9-2022', 'https://www.soccerstats.com/latest.asp?league=armenia2', 'https://www.soccerstats.com/pmatch.asp?league=armenia2&stats=16-1-5-2022', 'https://www.soccerstats.com/latest.asp?league=azerbaijan', 'https://www.soccerstats.com/pmatch.asp?league=azerbaijan&stats=49-8-6-2022', 'https://www.soccerstats.com/latest.asp?league=brazil', 'https://www.soccerstats.com/pmatch.asp?league=brazil&stats=350-18-3-2021', 'https://www.soccerstats.com/latest.asp?league=bulgaria', 'https://www.soccerstats.com/pmatch.asp?league=bulgaria&stats=112-12-10-2022', 'https://www.soccerstats.com/latest.asp?league=chile3', 'https://www.soccerstats.com/pmatch.asp?league=chile3&stats=57-6-7-2021', 'https://www.soccerstats.com/latest.asp?league=china2', 'https://www.soccerstats.com/latest.asp?league=cyprus', 'https://www.soccerstats.com/pmatch.asp?league=cyprus&stats=94-10-11-2022', 'https://www.soccerstats.com/pmatch.asp?league=cyprus&stats=97-6-3-2022', 'https://www.soccerstats.com/latest.asp?league=denmark', 'https://www.soccerstats.com/pmatch.asp?league=denmark&stats=101-5-1-2022', 'https://www.soccerstats.com/latest.asp?league=ecuador2', 'https://www.soccerstats.com/latest.asp?league=egypt4', 'https://www.soccerstats.com/pmatch.asp?league=egypt4&stats=12-6-15-2022', 'https://www.soccerstats.com/latest.asp?league=england12', 'https://www.soccerstats.com/pmatch.asp?league=england12&stats=108-19-1-2022', 'https://www.soccerstats.com/latest.asp?league=england15', 'https://www.soccerstats.com/pmatch.asp?league=england15&stats=90-2-8-2022', 'https://www.soccerstats.com/latest.asp?league=england16', 'https://www.soccerstats.com/pmatch.asp?league=england16&stats=85-6-8-2022', 'https://www.soccerstats.com/pmatch.asp?league=england16&stats=88-4-12-2022', 'https://www.soccerstats.com/pmatch.asp?league=england16&stats=89-7-14-2022', 'https://www.soccerstats.com/pmatch.asp?league=england16&stats=90-11-2-2022', 'https://www.soccerstats.com/latest.asp?league=england19', 'https://www.soccerstats.com/pmatch.asp?league=england19&stats=93-17-11-2022', 'https://www.soccerstats.com/latest.asp?league=georgia', 'https://www.soccerstats.com/latest.asp?league=germany3', 'https://www.soccerstats.com/pmatch.asp?league=germany3&stats=161-18-11-2022', 'https://www.soccerstats.com/latest.asp?league=germany4', 'https://www.soccerstats.com/pmatch.asp?league=germany4&stats=165-13-15-2022', 'https://www.soccerstats.com/latest.asp?league=germany20', 'https://www.soccerstats.com/pmatch.asp?league=germany20&stats=240-19-16-2022', 'https://www.soccerstats.com/latest.asp?league=hungary2', 'https://www.soccerstats.com/pmatch.asp?league=hungary2&stats=173-13-20-2022', 'https://www.soccerstats.com/latest.asp?league=india', 'https://www.soccerstats.com/h2h.asp?league=india&t1id=4&t2id=10&ly=2022', 'https://www.soccerstats.com/latest.asp?league=indonesia', 'https://www.soccerstats.com/pmatch.asp?league=indonesia&stats=126-13-2-2021', 'https://www.soccerstats.com/latest.asp?league=iran', 'https://www.soccerstats.com/pmatch.asp?league=iran&stats=54-8-14-2022', 'https://www.soccerstats.com/pmatch.asp?league=iran&stats=52-11-13-2022', 'https://www.soccerstats.com/pmatch.asp?league=iran&stats=53-7-1-2022', 'https://www.soccerstats.com/pmatch.asp?league=iran&stats=55-16-6-2022', 'https://www.soccerstats.com/latest.asp?league=israel2', 'https://www.soccerstats.com/pmatch.asp?league=israel2&stats=92-1-6-2022', 'https://www.soccerstats.com/pmatch.asp?league=israel2&stats=89-11-14-2022', 'https://www.soccerstats.com/pmatch.asp?league=israel2&stats=91-12-16-2022', 'https://www.soccerstats.com/latest.asp?league=italy3', 'https://www.soccerstats.com/pmatch.asp?league=italy3&stats=131-18-15-2022', 'https://www.soccerstats.com/latest.asp?league=italy5', 'https://www.soccerstats.com/pmatch.asp?league=italy5&stats=136-14-17-2022', 'https://www.soccerstats.com/latest.asp?league=kosovo', 'https://www.soccerstats.com/pmatch.asp?league=kosovo&stats=76-1-6-2022', 'https://www.soccerstats.com/latest.asp?league=malta2', 'https://www.soccerstats.com/pmatch.asp?league=malta2&stats=79-16-3-2022', 'https://www.soccerstats.com/pmatch.asp?league=malta2&stats=80-22-5-2022', 'https://www.soccerstats.com/latest.asp?league=romania', 'https://www.soccerstats.com/pmatch.asp?league=romania&stats=133-7-12-2022', 'https://www.soccerstats.com/pmatch.asp?league=romania&stats=130-5-4-2022', 'https://www.soccerstats.com/latest.asp?league=russia', 'https://www.soccerstats.com/pmatch.asp?league=russia&stats=126-14-8-2022', 'https://www.soccerstats.com/pmatch.asp?league=russia&stats=125-6-5-2022', 'https://www.soccerstats.com/latest.asp?league=saudiarabia2', 'https://www.soccerstats.com/pmatch.asp?league=saudiarabia2&stats=71-8-12-2022', 'https://www.soccerstats.com/pmatch.asp?league=saudiarabia2&stats=72-9-15-2022', 'https://www.soccerstats.com/pmatch.asp?league=saudiarabia2&stats=73-20-2-2022', 'https://www.soccerstats.com/leagueview.asp?league=cup-scotland1', 'https://www.soccerstats.com/leagueview_team.asp?league=cup-scotland1&team1id=39&team2id=40&fmid=20', 'https://www.soccerstats.com/latest.asp?league=serbia', 'https://www.soccerstats.com/pmatch.asp?league=serbia&stats=142-6-7-2022', 'https://www.soccerstats.com/pmatch.asp?league=serbia&stats=136-4-11-2022', 'https://www.soccerstats.com/latest.asp?league=serbia2', 'https://www.soccerstats.com/pmatch.asp?league=serbia2&stats=160-12-14-2022', 'https://www.soccerstats.com/latest.asp?league=slovakia', 'https://www.soccerstats.com/pmatch.asp?league=slovakia&stats=96-4-10-2022', 'https://www.soccerstats.com/latest.asp?league=sweden', 'https://www.soccerstats.com/pmatch.asp?league=sweden&stats=228-9-13-2021', 'https://www.soccerstats.com/pmatch.asp?league=sweden&stats=232-7-3-2021', 'https://www.soccerstats.com/pmatch.asp?league=sweden&stats=230-12-6-2021', 'https://www.soccerstats.com/latest.asp?league=turkey', 'https://www.soccerstats.com/pmatch.asp?league=turkey&stats=139-12-14-2022', 'https://www.soccerstats.com/latest.asp?league=turkey2', 'https://www.soccerstats.com/pmatch.asp?league=turkey2&stats=126-9-16-2022', 'https://www.soccerstats.com/latest.asp?league=ukraine', 'https://www.soccerstats.com/pmatch.asp?league=ukraine&stats=126-12-11-2022', 'https://www.soccerstats.com/latest.asp?league=ukraine2', 'https://www.soccerstats.com/pmatch.asp?league=ukraine2&stats=155-10-16-2022', 'https://www.soccerstats.com/latest.asp?league=uruguay2', 'https://www.soccerstats.com/pmatch.asp?league=uruguay2&stats=108-5-2-2021', 'https://www.soccerstats.com/pmatch.asp?league=brazil&stats=351-11-13-2021', 'https://www.soccerstats.com/pmatch.asp?league=spain&stats=86-10-20-2022', 'https://www.soccerstats.com/pmatch.asp?league=france&stats=157-10-15-2022', 'https://www.soccerstats.com/pmatch.asp?league=portugal&stats=109-6-1-2022', 'https://www.soccerstats.com/pmatch.asp?league=portugal&stats=116-10-13-2022', 'https://www.soccerstats.com/pmatch.asp?league=england&stats=430-12-16-2022', 'https://www.soccerstats.com/pmatch.asp?league=england&stats=429-18-7-2022', 'https://www.soccerstats.com/pmatch.asp?league=italy&stats=156-13-8-2022', 'https://www.soccerstats.com/pmatch.asp?league=italy&stats=157-15-3-2022', 'https://www.soccerstats.com/pmatch.asp?league=spain&stats=155-14-10-2022', 'https://www.soccerstats.com/pmatch.asp?league=germany&stats=118-13-2-2022', 'https://www.soccerstats.com/pmatch.asp?league=france&stats=162-12-10-2022', 'https://www.soccerstats.com/pmatch.asp?league=netherlands&stats=128-6-15-2022', 'https://www.soccerstats.com/pmatch.asp?league=portugal&stats=110-8-4-2022', 'https://www.soccerstats.com/pmatch.asp?league=germany&stats=131-12-10-2022', 'https://www.soccerstats.com/pmatch.asp?league=germany&stats=127-2-15-2022']