Web scraping error selenium: name 'records' is not defined-CodePudding

I get this result:

line 72, in <module>
    writer.writerows(records)
NameError: name 'records' is not defined

This is my code :

def main(search_term):
    records = []
    url= get_url(search_term)
    
    for page in range(1,21):
        driver.get(url.format(page))
        soup = BeautifulSoup(driver.page_source, 'html.parser')
        results = soup.find_all('div',{'data-component-type': 's-search-result'})
        
        for item in results:
            record = extract_record(item)
            if record: 
                records.append(record)
            
    driver.close()        


#save data to csv file 
with open ('results.csv', 'w', newline='', encoding= 'utf-8') as f:
    writer = csv.writer(f)
    writer.writerow(['Title','Current Price', 'Original Price', 'Rating', 'Reviewscount','Reviews', 'Url'])
    writer.writerows(records)
    
main('Bluetooth earphones')

CodePudding user response：

Make your main() function return the records list, and use the returned value in the writer.writerows() method:

def main(search_term):
    records = []
    url = get_url(search_term)
    
    for page in range(1,21):
        driver.get(url.format(page))
        soup = BeautifulSoup(driver.page_source, 'html.parser')
        results = soup.find_all('div', {'data-component-type': 's-search-result'})
        
        for item in results:
            record = extract_record(item)
            if record: 
                records.append(record)
            
    driver.close()   
    return records # return list
  
records = main('Bluetooth earphones') # Get returned list

#save data to csv file 
with open ('results.csv', 'w', newline='', encoding= 'utf-8') as f:
    writer = csv.writer(f)
    writer.writerow(['Title','Current Price', 'Original Price', 'Rating', 'Reviewscount','Reviews', 'Url'])
    writer.writerows(records) # Use returned list

CodePudding user response：

records = [] is defined within main(search_term) method:

def main(search_term):
    records = []

To access records = [] outside the main():

writer.writerows(records)

you need to define/declare in the global scope as follows:

records = []
    def main(search_term):
        url= get_url(search_term)
        .
        .
        for item in results:
            record = extract_record(item)
            if record: 
                records.append(record)
        
#save data to csv file 
with open ('results.csv', 'w', newline='', encoding= 'utf-8') as f:
        writer = csv.writer(f)
        writer.writerow(['Title','Current Price', 'Original Price', 'Rating', 'Reviewscount','Reviews', 'Url'])
        writer.writerows(records)