Scraping multiple website data from a table-CodePudding

I am practicing scraping website and inputting the information into a table. I imported the link, but I seem to be getting an error message based off the url.

from bs4 import BeautifulSoup
import requests
import pandas as pd

eastern_basketball_players = {'mens_basketball':'https://www.espn.com/nba/seasonleaders/_/league/east'}

def scraping_processing(season_leaders, url):
    
    player = []
    teamcity = []
    games_played = []
    minutes_per_game = []
    points = []
    espn_score = []
    
    for link in url:
        
        page = requests.get(link)
        
        soup = BeautifulSoup(page.content, 'html.parser')
        
        raw_names = soup.find_all('td', class_ = 'Player')
        
        teams = soup.find_all('td', class_ = 'Team')
        
        games = soup.find_all('td', class_ = 'Games Played')
        
        minutes = soup.find_all('td', class_ = 'Minutes Per Game')
        
        pts = soup.find_all('td', class_ = 'Points Per Game')
        
        espnscores = soup.find_all('td', class_ = 'EPSN Rating')
        
        for raw_name in raw_names:
            player.append(raw_name.get_text().strip())
            
        for team in teams:
            teamcity.append(team.get_text().strip())
            
        for game in games:
            games_played.append(raw_name.get_text().strip())
            
        for minute in minutes:
            minutes_per_game.append(minute.get_text().strip())
            
        for pt in pts:
            points.append(pt.get_text().strip())
            
        for espnscore in espnscores:
            espn_score.append(espnscore.get_text().strip())
     
    filename = season_leaders   '.csv'
    df = pd.DataFrame()
    df['Names'] = player
    df['Teams'] = teamcity
    df['Games Played'] = games_played
    df['Minutes Per Game'] = minutes_per_game
    df['Points'] = points
    df['ESPN Scores'] = espn_score
    df.to_csv(filename, index = False)
    

east_mens_basketball_df = scraping_processing('mens_basketball', eastern_basketball_players['mens_basketball'])

The error message I received states:

MissingSchema                             Traceback (most recent call last)
<ipython-input-49-ca254e49e854> in <module>
----> 1 east_mens_basketball_df = scraping_processing('mens_basketball', eastern_basketball_players['mens_basketball'])

<ipython-input-48-2f1a85c4b240> in scraping_processing(season_leaders, url)
     10     for link in url:
     11 
---> 12         page = requests.get(link)
     13 
     14         soup = BeautifulSoup(page.content, 'html.parser')

~\anaconda3\Python\lib\site-packages\requests\api.py in get(url, params, **kwargs)
     74 
     75     kwargs.setdefault('allow_redirects', True)
---> 76     return request('get', url, params=params, **kwargs)
     77 
     78 

~\anaconda3\Python\lib\site-packages\requests\api.py in request(method, url, **kwargs)
     59     # cases, and look like a memory leak in others.
     60     with sessions.Session() as session:
---> 61         return session.request(method=method, url=url, **kwargs)
     62 
     63 

~\anaconda3\Python\lib\site-packages\requests\sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
    526             hooks=hooks,
    527         )
--> 528         prep = self.prepare_request(req)
    529 
    530         proxies = proxies or {}

~\anaconda3\Python\lib\site-packages\requests\sessions.py in prepare_request(self, request)
    454 
    455         p = PreparedRequest()
--> 456         p.prepare(
    457             method=request.method.upper(),
    458             url=request.url,

~\anaconda3\Python\lib\site-packages\requests\models.py in prepare(self, method, url, headers, files, data, params, auth, cookies, hooks, json)
    314 
    315         self.prepare_method(method)
--> 316         self.prepare_url(url, params)
    317         self.prepare_headers(headers)
    318         self.prepare_cookies(cookies)

~\anaconda3\Python\lib\site-packages\requests\models.py in prepare_url(self, url, params)
    388             error = error.format(to_native_string(url, 'utf8'))
    389 
--> 390             raise MissingSchema(error)
    391 
    392         if not host:

MissingSchema: Invalid URL 'h': No schema supplied. Perhaps you meant http://h?

I rechecked the url to run the code again, but I am still receiving this error message. Can someone please help me?

CodePudding user response：

You pass this https://www.espn.com/nba/seasonleaders/_/league/east as url But You are accessing it with for loop i.e for link in url: You will get link='h'

Which is invaild.. Don't use for loop here

CodePudding user response：

You need to either get rid of the loop, or your urls need to be in a list.

But more importantly, you need to go and review html and how to identify tags and attributes. There are not <td> tags in the html with , or , etc.

Look at this code, and look at the html to see how it iterates through it.

from bs4 import BeautifulSoup
import requests
import pandas as pd

eastern_basketball_players = {'mens_basketball':['https://www.espn.com/nba/seasonleaders/_/league/east']}

def scraping_processing(season_leaders, url):
    
    rows = []
    
    for link in url:
        page = requests.get(link)
        
        soup = BeautifulSoup(page.content, 'html.parser')
        
        colNames = soup.find('tr', class_='colhead')
        colNames = [x.text for x in colNames.find_all('td')]
        
        trs = soup.find_all('tr')
        for row in trs:
            if row['class'][0] not in ['colhead', 'stathead']:
                rows.append([x.text for x in row.find_all('td')])
            
    filename = season_leaders   '.csv'
    df = pd.DataFrame(rows, columns=colNames)
    df.to_csv(filename, index = False)
    

east_mens_basketball_df = scraping_processing('mens_basketball', eastern_basketball_players['mens_basketball'])