how to save scraped date from soup object into Csv-CodePudding

i am looking to save scraped date into a csv file this is the scraped data and code

url = "https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBM-DA0321EN- 
SkillsNetwork/labs/datasets/Programming_Languages.html"
from bs4 import BeautifulSoup 
import requests
data  = requests.get(url).text 
soup = BeautifulSoup(data,"html5lib")
table = soup.find('table')
for row in table.find_all('tr'): 

    cols = row.find_all('td') 
    programing_language = cols[1].getText()
    salary = cols[3].getText() 
    print("{}--->{}".format(programing_language,salary))

I'm looking to only save the scraped data.

CodePudding user response：

Here is the solution.

import pandas as pd
from bs4 import BeautifulSoup
import requests
data=[]
url = "https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBM-DA0321EN-SkillsNetwork/labs/datasets/Programming_Languages.html"
from bs4 import BeautifulSoup 
import requests
data  = requests.get(url).text 
soup = BeautifulSoup(data,"html5lib")
table = soup.find('table')
for row in table.find_all('tr'): 

    cols = row.find_all('td') 

    programing_language = cols[1].getText()

    salary = cols[3].getText() 
    data.append([programing_language,salary])

    #print("{}--->{}".format(programing_language,salary))

cols=['programing_language','salary']
df = pd.DataFrame(data,columns=cols)
df.to_csv("data.csv", index=False)

CodePudding user response：

For a lightweight solution you can just use csv. Ignore headers row by using tr:nth-child(n 2). This nth-child range selector selects from the second tr. Then within a loop over the subsequent rows, select for the second and fourth columns as follows:

from bs4 import BeautifulSoup as bs
import requests, csv

response = requests.get('https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBM-DA0321EN-SkillsNetwork/labs/datasets/Programming_Languages.html',
                        headers={'User-Agent': 'Mozilla/5.0'})
soup = bs(response.content, 'lxml')

with open("programming.csv", "w", encoding="utf-8-sig", newline='') as f:

    w = csv.writer(f, delimiter=",", quoting=csv.QUOTE_MINIMAL)
    w.writerow(["Language", "Average Annual Salary"])

    for item in soup.select('tr:nth-child(n 2)'):
        w.writerow([item.select_one('td:nth-child(2)').text,
                   item.select_one('td:nth-child(4)').text])