I have a code that scrapes data into a dataframe
import os
import re
import threading
from multiprocessing.pool import ThreadPool
import pandas as pd
from bs4 import BeautifulSoup as bs
from selenium import webdriver
class Driver:
def __init__(self):
options = webdriver.ChromeOptions()
options.add_argument("--headless")
# Un-comment next line to supress logging:
options.add_experimental_option('excludeSwitches', ['enable-logging'])
self.driver = webdriver.Chrome(options=options)
def __del__(self):
self.driver.quit() # clean up driver when we are cleaned up
# print('The driver has been "quitted".')
threadLocal = threading.local()
def create_driver():
the_driver = getattr(threadLocal, 'the_driver', None)
if the_driver is None:
the_driver = Driver()
setattr(threadLocal, 'the_driver', the_driver)
return the_driver.driver
class GameData:
def __init__(self):
self.date = []
self.time = []
self.game = []
self.score = []
self.home_odds = []
self.draw_odds = []
self.away_odds = []
self.country = []
self.league = []
def generate_matches(table):
tr_tags = table.findAll('tr')
for tr_tag in tr_tags:
if 'class' in tr_tag.attrs and 'dark' in tr_tag['class']:
th_tag = tr_tag.find('th', {'class': 'first2 tl'})
a_tags = th_tag.findAll('a')
country = a_tags[0].text
league = a_tags[1].text
else:
td_tags = tr_tag.findAll('td')
if len(td_tags) > 0: # or just if td_tags
yield [td_tags[0].text, td_tags[1].text, td_tags[2].text, td_tags[3].text,
td_tags[4].text, td_tags[5].text, country, league]
def parse_data(url, return_urls=False):
browser = create_driver()
browser.get(url)
soup = bs(browser.page_source, "lxml")
div = soup.find('div', {'id': 'col-content'})
table = div.find('table', {'class': 'table-main'})
h1 = soup.find('h1').text
print(h1)
m = re.search(r'\d \w \d{4}$', h1)
game_date = m[0]
game_data = GameData()
for row in generate_matches(table):
game_data.date.append(game_date)
game_data.time.append(row[0])
game_data.game.append(row[1])
# Score present?
if ':' not in row[2]:
# No, shift a few columns right:
row[5], row[4], row[3], row[2] = row[4], row[3], row[2], nan
game_data.score.append(row[2])
game_data.home_odds.append(nan if row[3] == '-' else row[3])
game_data.draw_odds.append(nan if row[4] == '-' else row[4])
game_data.away_odds.append(nan if row[5] == '-' else row[5])
game_data.country.append(row[6])
game_data.league.append(row[7])
if return_urls:
span = soup.find('span', {'class': 'next-games-date'})
a_tags = span.findAll('a')
urls = ['https://www.oddsportal.com' a_tag['href'] for a_tag in a_tags]
return game_data, urls
return game_data
if __name__ == '__main__':
results = None
pool = ThreadPool(5) # We will be getting, however, 7 URLs
# Get today's data and the Urls for the other days:
game_data_today, urls = pool.apply(parse_data, args=('https://www.oddsportal.com/matches/soccer', True))
urls.pop(1) # Remove url for today: We already have the data for that
game_data_results = pool.imap(parse_data, urls)
for i in range(8):
try:
game_data = game_data_today if i == 1 else next(game_data_results)
result = pd.DataFrame(game_data.__dict__)
if results is None:
results = result
else:
results = results.append(result, ignore_index=True)
print(results)
# ensure all the drivers are "quitted":
del threadLocal
import gc
gc.collect() # a little extra insurance
However I get partial output at:
h1 = soup.find('h1').text
print(h1)
Next Soccer Matches: Today, 18 Dec 2021
Next Soccer Matches: Wednesday, 22 Dec 2021
Next Soccer Matches: Thursday, 23 Dec 2021
Next Soccer Matches: Friday, 24 Dec 2021
Next Soccer Matches: Tuesday, 21 Dec 2021
Next Soccer Matches: Monday, 20 Dec 2021
Next Soccer Matches: Yesterday, 17 Dec 2021
Next Soccer Matches: Tomorrow, 19 Dec 2021
While I checked, for the argument
When I explore further
td_tags = tr_tag.findAll('td')
print(td_tags)
if len(td_tags) > 0: # or just if td_tags
print(len(td_tags))
I get 2 cases
[<td >15:00</td>, <td ><a href="/soccer/england/npl-premier-division/south-shields-witton-albion-COXLb3wr/">South Shields - Witton</a><span onm ouseout="allowHideTootip(true);delayHideTip(200);" onm ouseover="toolTip('Postponed due to Covid-19.', this, event, '4');allowHideTootip(false);delayHideTip(200);return false;"> </span></td>, <td >postp.</td>, <td xodd="1.36" xoid="E-4v5i6xv464x0xd4ur7"><a href="" onclick="globals.ch.togle(this , 'E-4v5i6xv464x0xd4ur7');return false;" xparam="odds_text">1.36</a></td>, <td xodd="4.84" xoid="E-4v5i6xv498x0x0"><a href="" onclick="globals.ch.togle(this , 'E-4v5i6xv498x0x0');return false;" xparam="odds_text">4.84</a></td>, <td xodd="7.05" xoid="E-4v5i6xv464x0xd4ur8"><a href="" onclick="globals.ch.togle(this , 'E-4v5i6xv464x0xd4ur8');return false;" xparam="odds_text">7.05</a></td>, <td >11</td>]
7
[<td >15:00</td>, <td colspan="2"><a href="/soccer/england/npl-premier-division/warrington-town-scarborough-athletic-ObyQcNhl/">Warrington - Scarborough</a></td>, <td xodd="1.8" xoid="E-4v5i7xv464x0xd4ur9"><a href="" onclick="globals.ch.togle(this , 'E-4v5i7xv464x0xd4ur9');return false;" xparam="odds_text">1.80</a></td>, <td xodd="3.59" xoid="E-4v5i7xv498x0x0"><a href="" onclick="globals.ch.togle(this , 'E-4v5i7xv498x0x0');return false;" xparam="odds_text">3.59</a></td>, <td xodd="3.91" xoid="E-4v5i7xv464x0xd4ura"><a href="" onclick="globals.ch.togle(this , 'E-4v5i7xv464x0xd4ura');return false;" xparam="odds_text">3.91</a></td>, <td >12</td>]
6
I am getting an IndexError at:
line 67, in generate_matches
yield [td_tags[0].text, td_tags[1].text, td_tags[2].text, td_tags[3].text,
IndexError: list index out of range
How can I resolve this?
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque pellentesque, ipsum vel tempor suscipit, turpis mauris venenatis leo, nec vestibulum arcu urna et quam
CodePudding user response：
The fact that you get an IndexError when accessing td_tags implies that len(td_tags) is not always 6 or 7. It is <6 at least once.
Look at this bit of code:
if len(td_tags) > 0: # or just if td_tags
yield [td_tags[0].text, td_tags[1].text, td_tags[2].text, td_tags[3].text,
td_tags[4].text, td_tags[5].text, country, league]
You are using list indices 0 up to and including 5.
To not get an IndexError requires len(td_tags) of at least 6!
So change the first line to:
if len(td_tags) > 5:
That should get rid of the IndexError.