Load csv and pass each value into a url parameter?-CodePudding

I'm trying to aggregate NBA player data from nba.com. I have a list of player ids in a csv. I want to load each player id and pass it into the parameters list. And then use the parameters list in a requests.get. The request.get works when I enter a player id directly as a parameter. And tests of the csv looping through the function appears to work. However, I am having trouble passing the playerids into the parameter list successfully. I've tried looking at similar nba python codes and can't see where I'm going wrong.

'''

import pandas as pd
import requests
import csv

best_db=pd.DataFrame()


def table_Scrape():
    
    global best_db 
    
    with open("SHORT_ID_plyr.csv", "r") as f_urls: 
        f_urls_list = csv.reader(f_urls, delimiter=',') 
        next(f_urls_list)    
        
        ##step5. open 1st url from .csv
        for lines in f_urls_list:        
            u = lines[0]
            print(u) #<loop test
            # import requests
            player_id = u
            
            url = """
            http://stats.nba.com/stats/playergamelogs?DateFrom=&DateTo=&GameSegment=&LastNGames=0&LeagueID=00&Location=&MeasureType=Base&Month=0&OpponentTeamID=0&Outcome=&PORound=0&PaceAdjust=N&PerMode=Totals&Period=0&PlayerID=203932&PlusMinus=N&Rank=N&Season=2021-22&SeasonSegment=&SeasonType=Regular Season&ShotClockRange=&VsConference=&VsDivision=
            """
                #url = """
                #https://stats.nba.com/stats/leaguedashplayerstats?College=&Conference=&Country=&DateFrom=&DateTo=&Division=&DraftPick=&DraftYear=&GameScope=&GameSegment=&Height=&LastNGames=0&LeagueID=00&Location=&MeasureType=Base&Month=0&OpponentTeamID=0&Outcome=&PORound=0&PaceAdjust=N&PerMode=Totals&Period=0&PlayerExperience=&PlayerPosition=&PlusMinus=N&Rank=N&Season=2021-22&SeasonSegment=&SeasonType=Regular Season&ShotClockRange=&StarterBench=&TeamID=0&TwoWay=0&VsConference=&VsDivision=&Weight=
                #"""
            header_dict = {
                'User-Agent': 'Mozilla/5.0',
                'x-nba-stats-origin': 'stats',
                'x-nba-stats-token': 'true',
                'Referer': 'https://stats.nba.com',
                'Connection': 'keep-alive',
                'Pragma': 'no-cache',
                'Cache-Control': 'no-cache',
                'Host': 'stats.nba.com'
            }

            params = {
                'LastNGames': '0',
                'LeagueID': '00',
                'MeasureType': 'Base',
                'Month': '0',
                'OpponentTeamID': '0',
                'PORound': '0',
                'PaceAdjust': 'N',
                'PerMode': 'Totals',
                'Period': '0',
                'PlayerID': u,
                'PlusMinus': 'N',
                'Rank': 'N',
                'Season': '2021-22',
                'SeasonType': 'Regular Season'
                }

            res = requests.get(url, headers=header_dict, params=params)
            json_set = res.json()
            headers = json_set['resultSets'][0]['headers']
            data_set = json_set['resultSets'][0]['rowSet']
            df = pd.DataFrame(columns=headers)
            df.head #test the dataframe NOTE: does not appear to be working either
            
table_Scrape() #call the function

CodePudding user response：

You are passing the payload onto a url that already has the payload with a hardcoded player id. Put the palyer_id variable into the url

import pandas as pd
import requests
import csv

best_db=pd.DataFrame()


def table_Scrape():
    
    global best_db 
    
    with open("SHORT_ID_plyr.csv", "r") as f_urls: 
        f_urls_list = csv.reader(f_urls, delimiter=',') 
        next(f_urls_list)    
        
        ##step5. open 1st url from .csv
        for lines in f_urls_list:        
            u = lines[0]
            print(u) #<loop test
            # import requests
            player_id = u
            
            url = f"""http://stats.nba.com/stats/playergamelogs?DateFrom=&DateTo=&GameSegment=&LastNGames=0&LeagueID=00&Location=&MeasureType=Base&Month=0&OpponentTeamID=0&Outcome=&PORound=0&PaceAdjust=N&PerMode=Totals&Period=0&PlayerID={u}&PlusMinus=N&Rank=N&Season=2021-22&SeasonSegment=&SeasonType=Regular Season&ShotClockRange=&VsConference=&VsDivision="""
                
            header_dict = {
                'User-Agent': 'Mozilla/5.0',
                'x-nba-stats-origin': 'stats',
                'x-nba-stats-token': 'true',
                'Referer': 'https://stats.nba.com',
                'Connection': 'keep-alive',
                'Pragma': 'no-cache',
                'Cache-Control': 'no-cache',
                'Host': 'stats.nba.com'
            }

            res = requests.get(url, headers=header_dict)
            json_set = res.json()
            headers = json_set['resultSets'][0]['headers']
            data_set = json_set['resultSets'][0]['rowSet']
            df = pd.DataFrame(data_set,columns=headers)
            df.head() #test the dataframe NOTE: does not appear to be working either
            
table_Scrape() #call the function