Using the below code, I can't pull the College Football matchups from pregame.com in the game center.
I've tried multiple class ids with different elements, and even tried pulling with pandas, but can't get the entire table. Is there another way to scrape it successfully?
from bs4 import BeautifulSoup
import lxml
import requests
header = {'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36'}
pregame = requests.get('https://pregame.com/game-center/?d=1636174800000&t=0&l=2&a=0&s=AwayRot&m=false&b=undefined&o=Current&c=All&k=', 'r').text
soup = BeautifulSoup(pregame, 'lxml')
div = soup.find_all('p', class_ = 'pggc-col-data pggc-away')
print(div)
CodePudding user response：
the problem you're running into is that the data is being loaded dynamically via javascript.
You'll want to check out something like Selenium to work around this. Here's a good overview: How to Scrap Data From JavaScript-Based Website Using Python, Selenium, and Headless Web Driver
CodePudding user response：
You may need to do a little data manipulation and joins depending what you are after. But you can get the data back in json format from the api and parse it.
import requests
import pandas as pd
url = 'https://socket.pregame.com/api/gamecenter/bootstrap'
jsonData = requests.get(url).json()
data = {}
for each, v in jsonData.items():
data[each] = pd.DataFrame(v)
for key, table in data.items():
print(f'\n*** {key} ***')
print(table.head(10).to_string())
*** Leagues ***
Id Name Abbreviation Periods PickTypes
0 1 NFL NFL [{'Id': 1, 'Name': 'Game'}, {'Id': 2, 'Name': '1st Half'}, {'Id': 3, 'Name': '2nd Half'}, {'Id': 4, 'Name': '1st Q'}, {'Id': 5, 'Name': '2nd Q'}, {'Id': 6, 'Name': '3rd Q'}, {'Id': 7, 'Name': '4th Q'}] [{'Id': 3, 'PickTypeId': 1, 'SearchablePickTypeId': 1, 'Name': 'Sides'}, {'Id': 4, 'PickTypeId': 2, 'SearchablePickTypeId': 2, 'Name': 'Totals'}, {'Id': 5, 'PickTypeId': 3, 'SearchablePickTypeId': 3, 'Name': 'Money Lines'}, {'Id': 7, 'PickTypeId': 5, 'SearchablePickTypeId': 5, 'Name': 'Parlay'}, {'Id': 8, 'PickTypeId': 6, 'SearchablePickTypeId': 6, 'Name': 'Teaser'}, {'Id': 9, 'PickTypeId': 7, 'SearchablePickTypeId': 7, 'Name': 'Prop'}]
1 2 College Football CFB [{'Id': 1, 'Name': 'Game'}, {'Id': 2, 'Name': '1st Half'}, {'Id': 3, 'Name': '2nd Half'}] [{'Id': 3, 'PickTypeId': 1, 'SearchablePickTypeId': 1, 'Name': 'Sides'}, {'Id': 4, 'PickTypeId': 2, 'SearchablePickTypeId': 2, 'Name': 'Totals'}, {'Id': 5, 'PickTypeId': 3, 'SearchablePickTypeId': 3, 'Name': 'Money Lines'}, {'Id': 7, 'PickTypeId': 5, 'SearchablePickTypeId': 5, 'Name': 'Parlay'}, {'Id': 8, 'PickTypeId': 6, 'SearchablePickTypeId': 6, 'Name': 'Teaser'}, {'Id': 9, 'PickTypeId': 7, 'SearchablePickTypeId': 7, 'Name': 'Prop'}]
2 3 NBA NBA [{'Id': 1, 'Name': 'Game'}, {'Id': 2, 'Name': '1st Half'}, {'Id': 3, 'Name': '2nd Half'}, {'Id': 4, 'Name': '1st Q'}, {'Id': 5, 'Name': '2nd Q'}, {'Id': 6, 'Name': '3rd Q'}, {'Id': 7, 'Name': '4th Q'}] [{'Id': 3, 'PickTypeId': 1, 'SearchablePickTypeId': 1, 'Name': 'Sides'}, {'Id': 4, 'PickTypeId': 2, 'SearchablePickTypeId': 2, 'Name': 'Totals'}, {'Id': 5, 'PickTypeId': 3, 'SearchablePickTypeId': 3, 'Name': 'Money Lines'}, {'Id': 7, 'PickTypeId': 5, 'SearchablePickTypeId': 5, 'Name': 'Parlay'}, {'Id': 8, 'PickTypeId': 6, 'SearchablePickTypeId': 6, 'Name': 'Teaser'}, {'Id': 9, 'PickTypeId': 7, 'SearchablePickTypeId': 7, 'Name': 'Prop'}]
3 5 MLB MLB [{'Id': 1, 'Name': 'Game'}, {'Id': 2, 'Name': '1st 5'}] [{'Id': 2, 'PickTypeId': 1, 'SearchablePickTypeId': 3, 'Name': 'Run Lines'}, {'Id': 4, 'PickTypeId': 2, 'SearchablePickTypeId': 2, 'Name': 'Totals'}, {'Id': 6, 'PickTypeId': 3, 'SearchablePickTypeId': 1, 'Name': 'Sides'}, {'Id': 7, 'PickTypeId': 5, 'SearchablePickTypeId': 5, 'Name': 'Parlay'}, {'Id': 8, 'PickTypeId': 6, 'SearchablePickTypeId': 6, 'Name': 'Teaser'}, {'Id': 9, 'PickTypeId': 7, 'SearchablePickTypeId': 7, 'Name': 'Prop'}]
4 6 NHL NHL [{'Id': 1, 'Name': 'Game'}, {'Id': 4, 'Name': '1st Period'}, {'Id': 5, 'Name': '2nd Period'}, {'Id': 6, 'Name': '3rd Pariod'}] [{'Id': 1, 'PickTypeId': 1, 'SearchablePickTypeId': 3, 'Name': 'Puck Lines'}, {'Id': 4, 'PickTypeId': 2, 'SearchablePickTypeId': 2, 'Name': 'Totals'}, {'Id': 6, 'PickTypeId': 3, 'SearchablePickTypeId': 1, 'Name': 'Sides'}, {'Id': 7, 'PickTypeId': 5, 'SearchablePickTypeId': 5, 'Name': 'Parlay'}, {'Id': 8, 'PickTypeId': 6, 'SearchablePickTypeId': 6, 'Name': 'Teaser'}, {'Id': 9, 'PickTypeId': 7, 'SearchablePickTypeId': 7, 'Name': 'Prop'}]
5 7 Canadian Football League CFL [{'Id': 1, 'Name': 'Game'}, {'Id': 2, 'Name': '1st Half'}, {'Id': 3, 'Name': '2nd Half'}, {'Id': 4, 'Name': '1st Q'}, {'Id': 5, 'Name': '2nd Q'}, {'Id': 6, 'Name': '3rd Q'}, {'Id': 7, 'Name': '4th Q'}] [{'Id': 3, 'PickTypeId': 1, 'SearchablePickTypeId': 1, 'Name': 'Sides'}, {'Id': 4, 'PickTypeId': 2, 'SearchablePickTypeId': 2, 'Name': 'Totals'}, {'Id': 5, 'PickTypeId': 3, 'SearchablePickTypeId': 3, 'Name': 'Money Lines'}, {'Id': 7, 'PickTypeId': 5, 'SearchablePickTypeId': 5, 'Name': 'Parlay'}, {'Id': 8, 'PickTypeId': 6, 'SearchablePickTypeId': 6, 'Name': 'Teaser'}, {'Id': 9, 'PickTypeId': 7, 'SearchablePickTypeId': 7, 'Name': 'Prop'}]
*** Groups ***
Id LeagueId GroupDate GroupText BackColor ForeColor
0 18945 3 2021-12-25T08:00:00Z NBA - Saturday, December 25th 0 0
1 19376 2 2021-11-02T07:00:00Z COLLEGE FOOTBALL - Tuesday, November 2nd 0 0
2 19377 2 2021-11-03T07:00:00Z COLLEGE FOOTBALL - Wednesday, November 3rd 0 0
3 19378 2 2021-11-04T07:00:00Z COLLEGE FOOTBALL - Thursday, November 4th 0 0
4 19379 2 2021-11-05T07:00:00Z COLLEGE FOOTBALL - Friday, November 5th 0 0
5 19380 2 2021-11-06T07:00:00Z COLLEGE FOOTBALL - Saturday, November 6th 0 0
6 19381 2 2021-11-06T07:00:00Z COLLEGE FOOTBALL - Saturday, November 6th - WRITE-IN GAMES - 0 0
7 19389 7 2021-11-05T07:00:00Z CANADIAN FOOTBALL LEAGUE - WEEK 14 - Friday, November 5th 0 0
8 19390 7 2021-11-06T07:00:00Z CANADIAN FOOTBALL LEAGUE - WEEK 14 - Saturday, November 6th 0 0
9 19392 1 2021-11-04T07:00:00Z NFL WEEK 9 - Thursday, November 4th 0 0
*** Events ***
Id EventGroupId LeagueId LeagueName LeagueAbbr ScheduledDateAndTime StartDateAndTime AwayRotationNumber AwayTeamId AwayTeamName AwayTeamFullName AwayTeamAbbr AwayPitcher AwayHasScheduledPitcherChanged HomeRotationNumber HomeTeamId HomeTeamName HomeTeamFullName HomeTeamAbbr HomePitcher HomeHasScheduledPitcherChanged AtNeutralSite IsCircled Status IsMatchup Location TVBroadcast Weather Umpires
0 172676 19427 1 NFL NFL 2021-11-14T18:00:00Z 2021-11-14T18:00:00Z 247 668 Bills Buffalo Bills BUF False 248 697 Jets NY Jets NYJ False False False 0 True None None None []
1 172427 19393 1 NFL NFL 2021-11-07T18:00:00Z 2021-11-07T18:00:00Z 459 668 Bills Buffalo Bills BUF False 460 701 Jaguars Jacksonville Jaguars JAC False False False 0 True None None None []
2 172434 19393 1 NFL NFL 2021-11-08T01:20:00Z 2021-11-08T01:20:00Z 473 670 Titans Tennessee Titans TEN False 474 771 Rams Los Angeles Rams LOS False False False 0 True None None None []
3 172424 19393 1 NFL NFL 2021-11-07T18:00:00Z 2021-11-07T18:00:00Z 453 671 Broncos Denver Broncos DEN False 454 677 Cowboys Dallas Cowboys DAL False False False 0 True None None None []
4 172684 19427 1 NFL NFL 2021-11-15T01:20:00Z 2021-11-15T01:20:00Z 263 672 Chiefs Kansas City Chiefs KC False 264 854 Raiders Las Vegas Raiders LVR False False False 0 True None None None []
5 172682 19427 1 NFL NFL 2021-11-14T21:25:00Z 2021-11-14T21:25:00Z 259 675 Seahawks Seattle Seahawks SEA False 260 684 Packers Green Bay Packers GB False False False 0 True None None None []
6 172433 19393 1 NFL NFL 2021-11-07T21:25:00Z 2021-11-07T21:25:00Z 471 676 Cardinals Arizona Cardinals ARZ False 472 692 49ers San Francisco 49ers SF False False False 0 True None None None []
7 172683 19427 1 NFL NFL 2021-11-14T21:25:00Z 2021-11-14T21:25:00Z 261 680 Eagles Philadelphia Eagles PHI False 262 671 Broncos Denver Broncos DEN False False False 0 True None None None []
8 172435 19394 1 NFL NFL 2021-11-09T01:15:00Z 2021-11-09T01:15:00Z 475 682 Bears Chicago Bears CHI False 476 669 Steelers Pittsburgh Steelers PIT False False False 0 True None None None []
9 172679 19427 1 NFL NFL 2021-11-14T18:00:00Z 2021-11-14T18:00:00Z 253 683 Lions Detroit Lions DET False 254 669 Steelers Pittsburgh Steelers PIT False False False 0 True None None None []
*** Scores ***
EventId Id AwayScore HomeScore AwayStatus HomeStatus
0 172491 172491 7 0 Final
1 172263 172263 52 49 Final
2 172570 172570 117 89 Final
3 172264 172264 31 25 Final
4 172265 172265 33 35 Final
5 172571 172571 125 110 Final
6 172575 172575 0 3 Final
7 172576 172576 0 4 Final
8 172577 172577 0 3 Final
9 172578 172578 3 4 Final
*** Odds ***
IsSiteOdds EventId LeagueId BookId PeriodTypeId ActionTypeId CurrentAwayValue CurrentAwayPoints CurrentAwayPrice CurrentHomeValue CurrentHomePoints CurrentHomePrice PreviousAwayValue PreviousAwayPoints PreviousAwayPrice PreviousHomeValue PreviousHomePoints PreviousHomePrice OpenerAwayValue OpenerAwayPoints OpenerAwayPrice OpenerHomeValue OpenerHomePoints OpenerHomePrice IsEdited IsUnknown IsLocked Key AwayLineHistory HomeLineHistory
0 False 172272 2 12 1 1 -21 -21.0 -110 21 21.0 -110 -20½ -20.5 -110 20½ 20.5 -110 -21 -21.0 -110 21 21.0 -110 False False False 12~1~1 NaN NaN
1 False 172272 2 12 1 2 64½ 64.5 -110 64½ 64.5 -110 64 64.0 -110 64 64.0 -110 63½ 63.5 -110 63½ 63.5 -110 False False False 12~1~2 NaN NaN
2 False 172272 2 12 1 3 -1300 0.0 -1300 850 0.0 850 -1375 0.0 -1375 900 0.0 900 -1375 0.0 -1375 900 0.0 900 False False False 12~1~3 NaN NaN
3 False 172272 2 3 1 1 -21-120 -21.0 -120 21 ev 21.0 100 -21-115 -21.0 -115 21-105 21.0 -105 -20 -20.0 -110 20 20.0 -110 False False False 3~1~1 NaN NaN
4 False 172272 2 3 1 2 64-112 64.0 -112 64-108 64.0 -108 64 64.0 -110 64-108 64.0 -108 64 64.0 -110 64 64.0 -110 False False False 3~1~2 NaN NaN
5 True 172272 2 8 1 1 -21-120 -21.0 -120 21 ev 21.0 100 -21-115 -21.0 -115 21-105 21.0 -105 -20 -20.0 -110 20 20.0 -110 False False False 8~1~1 [0, 6, 17, 4, 24, 2, 39, 2] [0, 6, 17, 8, 24, 10, 39, 10]
6 True 172272 2 8 1 2 64-112 64.0 -112 64-108 64.0 -108 64 64.0 -110 64-108 64.0 -108 64 64.0 -110 64 64.0 -110 False False False 8~1~2 [0, 6, 39, 6] [0, 6, 39, 6]
7 True 172272 2 8 1 3 -1628 0.0 -1628 787 0.0 787 -1631 0.0 -1631 788 0.0 788 -1473 0.0 -1473 739 0.0 739 False False False 8~1~3 [0, 6, 1, 6, 6, 6, 17, 0, 39, 0] [0, 6, 1, 6, 6, 6, 17, 12, 39, 12]
8 False 172272 2 27 1 1 -21 -21.0 -110 21 21.0 -110 -21-110 -21.0 -110 21 ev 21.0 100 -21 -21.0 -110 21 21.0 -110 False False False 27~1~1 NaN NaN
9 False 172272 2 27 1 2 64-111 64.0 -111 64-109 64.0 -109 64 64.0 -110 64-109 64.0 -109 65 65.0 -110 65 65.0 -110 False False False 27~1~2 NaN NaN
*** Consensus ***
AllCash AllCashRanking AllCashAwayPercentage AllCashHomePercentage AllTicketCount AllTicketRanking AllTicketAwayPercentage AllTicketHomePercentage AllPickCount AllPickRanking AllPickAwayPercentage AllPickHomePercentage AllExpertsAwayPickCount AllExpertsHomePickCount LastHourCash LastHourCashRanking LastHourCashAwayPercentage LastHourCashHomePercentage LastHourTicketCount LastHourTicketRanking LastHourTicketAwayPercentage LastHourTicketHomePercentage LastHourPickCount LastHourPickRanking LastHourPickAwayPercentage LastHourPickHomePercentage LastHourExpertsAwayPickCount LastHourExpertsHomePickCount AtCurrentCash AtCurrentCashRanking AtCurrentCashAwayPercentage AtCurrentCashHomePercentage AtCurrentTicketCount AtCurrentTicketRanking AtCurrentTicketAwayPercentage AtCurrentTicketHomePercentage AtCurrentPickCount AtCurrentPickRanking AtCurrentPickAwayPercentage AtCurrentPickHomePercentage AtCurrentExpertsAwayPickCount AtCurrentExpertsHomePickCount EventId PeriodTypeId ActionTypeId Key
0 0.00 0 0.00 0.00 0 0 0.00 0.00 2 0 0.5 0.5 0 1 0.0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0.0 0 0 169990 1 3 1~3
1 20169.35 0 0.45 0.55 73 0 0.55 0.45 2 0 0.5 0.5 0 0 0.0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0.0 0 0 172267 1 2 1~2
2 6884.43 0 0.82 0.18 18 0 0.83 0.17 1 0 0.0 1.0 0 0 0.0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0.0 0 0 172268 1 2 1~2
3 10983.57 0 0.18 0.82 72 0 0.65 0.35 2 0 1.0 0.0 0 0 0.0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0.0 0 0 172273 1 1 1~1
4 6694.63 0 0.04 0.96 66 0 0.14 0.86 2 0 0.5 0.5 0 0 0.0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0.0 0 0 172274 1 1 1~1
5 553.92 0 1.00 0.00 17 0 1.00 0.00 0 0 0.0 0.0 0 0 0.0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0.0 0 0 172279 1 3 1~3
6 3416.98 0 0.39 0.61 45 0 0.33 0.67 1 0 0.0 1.0 0 0 0.0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0.0 0 0 172288 1 1 1~1
7 4797.38 0 0.85 0.15 90 0 0.87 0.13 3 0 1.0 0.0 0 0 0.0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0.0 0 0 172291 1 1 1~1
8 5018.40 0 0.88 0.12 45 0 0.93 0.07 0 0 0.0 0.0 0 0 0.0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0.0 0 0 172298 1 3 1~3
9 5949.84 0 0.51 0.49 90 0 0.21 0.79 5 0 0.8 0.2 0 0 0.0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0 0.0 0.0 0 0 0.0 0.0 0 0 0.0 0.0 0 0 172305 1 1 1~1