Before web scraping, I'm using the following code to login.
import requests
from bs4 import BeautifulSoup
login_url = 'https://www.footballoutsiders.com/user/login'
data = {
'username': 'username',
'password': 'password'
}
with requests.Session() as s:
response = s.post(login_url , data)
print(response.text)
I then do the following to view the table, but the cells are still locked.
index_page= s.get('https://www.footballoutsiders.com/stats/nfl/historical-lookup-by-
week/2020/1/overall')
soup = BeautifulSoup(index_page.text, 'lxml')
table1 = soup.find('table')
table1
What am I doing wrong?
CodePudding user response：
You are passing in the wrong data. It should be 'user' and 'pass'. Also, don't use BeautifulSoup to parse <table> tags (when you only need the content). Pandas can do that for you (uses bs4 under the hood).
import requests
import pandas as pd
LOGIN_URL = 'https://www.footballoutsiders.com/user/login?destination=home'
login = {
'name': '[email protected]',
'pass': '54321',
'form_id': 'user_login_form',
'op': 'Login'}
s = requests.Session()
s.post(LOGIN_URL, data=login)
index_page= s.get('https://www.footballoutsiders.com/stats/nfl/historical-lookup-by-week/2020/1/overall')
df = pd.read_html(index_page.text)[0]
Output:
print(df.to_markdown())
| | Team | W-L | Total DVOA | Total DVOA.1 | Weighted DVOA | Weighted DVOA.1 | Offense DVOA | Offense DVOA.1 | Offense Weighted DVOA | Offense Weighted DVOA.1 | Defense DVOA | Defense DVOA.1 | Defense Weighted DVOA | Defense Weighted DVOA.1 | Special Teams DVOA | Special Teams DVOA.1 | Special Teams Weighted DVOA | Special Teams Weighted DVOA.1 |
|---:|:-------|:------|-------------:|:---------------|----------------:|:------------------|---------------:|:-----------------|------------------------:|:--------------------------|---------------:|:-----------------|------------------------:|:--------------------------|---------------------:|:-----------------------|------------------------------:|:--------------------------------|
| 0 | BAL | 1-0 | 1 | 88.0% | 1 | 88.0% | 1 | 39.9% | 1 | 39.9% | 3 | -38.8% | 3 | -38.8% | 2 | 9.4% | 2 | 9.4% |
| 1 | NE | 1-0 | 2 | 52.3% | 2 | 52.3% | 3 | 36.4% | 3 | 36.4% | 5 | -23.8% | 5 | -23.8% | 25 | -7.9% | 25 | -7.9% |
| 2 | JAX | 1-0 | 3 | 38.0% | 3 | 38.0% | 4 | 35.8% | 4 | 35.8% | 13 | 0.5% | 13 | 0.5% | 5 | 2.8% | 5 | 2.8% |
| 3 | SEA | 1-0 | 4 | 37.0% | 4 | 37.0% | 2 | 38.6% | 2 | 38.6% | 21 | 9.5% | 21 | 9.5% | 3 | 7.8% | 3 | 7.8% |
| 4 | PIT | 1-0 | 5 | 36.0% | 5 | 36.0% | 14 | 6.5% | 14 | 6.5% | 2 | -39.0% | 2 | -39.0% | 27 | -9.4% | 27 | -9.4% |
| 5 | WAS | 1-0 | 6 | 35.9% | 6 | 35.9% | 28 | -32.7% | 28 | -32.7% | 1 | -69.4% | 1 | -69.4% | 13 | -0.8% | 13 | -0.8% |
| 6 | BUF | 1-0 | 7 | 16.7% | 7 | 16.7% | 17 | 2.6% | 17 | 2.6% | 7 | -19.0% | 7 | -19.0% | 22 | -4.8% | 22 | -4.8% |
| 7 | LV | 1-0 | 8 | 13.7% | 8 | 13.7% | 5 | 31.7% | 5 | 31.7% | 23 | 16.5% | 23 | 16.5% | 15 | -1.4% | 15 | -1.4% |
| 8 | NO | 1-0 | 9 | 10.8% | 9 | 10.8% | 24 | -13.7% | 24 | -13.7% | 9 | -14.3% | 9 | -14.3% | 1 | 10.2% | 1 | 10.2% |
| 9 | MIN | 0-1 | 10 | 10.8% | 10 | 10.8% | 6 | 28.2% | 6 | 28.2% | 26 | 20.1% | 26 | 20.1% | 6 | 2.7% | 6 | 2.7% |
| 10 | LAC | 1-0 | 11 | 4.1% | 11 | 4.1% | 22 | -7.6% | 22 | -7.6% | 6 | -20.4% | 6 | -20.4% | 26 | -8.7% | 26 | -8.7% |
| 11 | CAR | 0-1 | 12 | 2.5% | 12 | 2.5% | 8 | 23.4% | 8 | 23.4% | 27 | 24.0% | 27 | 24.0% | 4 | 3.2% | 4 | 3.2% |
| 12 | CHI | 1-0 | 13 | 0.3% | 13 | 0.3% | 19 | 0.9% | 19 | 0.9% | 12 | -1.6% | 12 | -1.6% | 16 | -2.2% | 16 | -2.2% |
| 13 | DAL | 0-1 | 14 | 0.3% | 14 | 0.3% | 12 | 9.2% | 12 | 9.2% | 18 | 3.9% | 18 | 3.9% | 23 | -5.1% | 23 | -5.1% |
| 14 | DET | 0-1 | 15 | -0.1% | 15 | -0.1% | 21 | -0.3% | 21 | -0.3% | 15 | 1.2% | 15 | 1.2% | 9 | 1.4% | 9 | 1.4% |
| 15 | KC | 1-0 | 16 | -1.3% | 16 | -1.3% | 9 | 17.6% | 9 | 17.6% | 25 | 17.5% | 25 | 17.5% | 14 | -1.3% | 14 | -1.3% |
| 16 | GB | 1-0 | 17 | -5.0% | 17 | -5.0% | 7 | 24.2% | 7 | 24.2% | 30 | 28.7% | 30 | 28.7% | 12 | -0.4% | 12 | -0.4% |
| 17 | ARI | 1-0 | 18 | -5.2% | 18 | -5.2% | 20 | 0.5% | 20 | 0.5% | 17 | 2.9% | 17 | 2.9% | 17 | -2.8% | 17 | -2.8% |
| 18 | SF | 0-1 | 19 | -6.1% | 19 | -6.1% | 18 | 2.3% | 18 | 2.3% | 14 | 0.6% | 14 | 0.6% | 24 | -7.8% | 24 | -7.8% |
| 19 | HOU | 0-1 | 20 | -9.3% | 20 | -9.3% | 11 | 11.1% | 11 | 11.1% | 24 | 16.7% | 24 | 16.7% | 19 | -3.6% | 19 | -3.6% |
| 20 | LAR | 1-0 | 21 | -13.5% | 21 | -13.5% | 13 | 7.5% | 13 | 7.5% | 20 | 9.5% | 20 | 9.5% | 28 | -11.4% | 28 | -11.4% |
| 21 | TEN | 1-0 | 22 | -14.3% | 22 | -14.3% | 16 | 3.4% | 16 | 3.4% | 10 | -11.1% | 10 | -11.1% | 32 | -28.7% | 32 | -28.7% |
| 22 | TB | 0-1 | 23 | -16.3% | 23 | -16.3% | 25 | -16.1% | 25 | -16.1% | 8 | -16.4% | 8 | -16.4% | 30 | -16.7% | 30 | -16.7% |
| 23 | DEN | 0-1 | 24 | -17.1% | 24 | -17.1% | 23 | -13.3% | 23 | -13.3% | 19 | 3.9% | 19 | 3.9% | 11 | 0.1% | 11 | 0.1% |
| 24 | ATL | 0-1 | 25 | -26.0% | 25 | -26.0% | 10 | 11.8% | 10 | 11.8% | 32 | 38.9% | 32 | 38.9% | 10 | 1.1% | 10 | 1.1% |
| 25 | IND | 0-1 | 26 | -27.0% | 26 | -27.0% | 15 | 3.8% | 15 | 3.8% | 28 | 26.0% | 28 | 26.0% | 21 | -4.8% | 21 | -4.8% |
| 26 | CIN | 0-1 | 27 | -28.4% | 27 | -28.4% | 29 | -33.2% | 29 | -33.2% | 11 | -8.5% | 11 | -8.5% | 20 | -3.7% | 20 | -3.7% |
| 27 | NYJ | 0-1 | 28 | -35.2% | 28 | -35.2% | 26 | -21.1% | 26 | -21.1% | 16 | 2.7% | 16 | 2.7% | 29 | -11.4% | 29 | -11.4% |
| 28 | PHI | 0-1 | 29 | -41.1% | 29 | -41.1% | 32 | -71.3% | 32 | -71.3% | 4 | -33.6% | 4 | -33.6% | 18 | -3.4% | 18 | -3.4% |
| 29 | MIA | 0-1 | 30 | -48.9% | 30 | -48.9% | 27 | -23.2% | 27 | -23.2% | 29 | 28.3% | 29 | 28.3% | 8 | 2.6% | 8 | 2.6% |
| 30 | NYG | 0-1 | 31 | -54.7% | 31 | -54.7% | 30 | -45.4% | 30 | -45.4% | 22 | 11.9% | 22 | 11.9% | 7 | 2.6% | 7 | 2.6% |
| 31 | CLE | 0-1 | 32 | -107.6% | 32 | -107.6% | 31 | -57.5% | 31 | -57.5% | 31 | 33.3% | 31 | 33.3% | 31 | -16.8% | 31 | -16.8% |