how to save the result of webs craping in the out put file text with selenium in python-CodePudding

the code below retrieve all the data from the url below. But I want to save the results in the output file text. like (info.text), Would you mind help me.

url: https://www150.statcan.gc.ca/n1/pub/71-607-x/2021004/exp-eng.htm?r1=(1)&r2=0&r3=0&r4=12&r5=0&r7=0&r8=2022-02-01&r9=2022-02-01

from selenium import webdriver
import time

url = "https://www150.statcan.gc.ca/n1/pub/71-607-x/2021004/exp-eng.htm?r1=(1)&r2=0&r3=0&r4=12&r5=0&r7=0&r8=2022-02-01&r9=2022-02-01"

driver = webdriver.Chrome("C:\Program Files\Python310\chromedriver.exe")
driver.get(url)
table = driver.find_element_by_id('report_table')
body = table.find_element_by_tag_name('tbody')
cells = body.find_elements_by_tag_name('td')

for cell in cells:
    print(cell.text)

# run the loop 26 times
for i in range(26):
    
    # your code
    table = driver.find_element_by_id('report_table')
    body = table.find_element_by_tag_name('tbody')
    cells = body.find_elements_by_tag_name('td')
    for cell in cells:
        print(cell.text)
        
    # click on the Next button    
    driver.find_element_by_xpath('//*[@id="report_results_next"]').click()

CodePudding user response：

If you want to write the results from a variable (seems like cell.text in your example), we just need to open a file and write that data to the file.

with open("info.txt", “w”, encoding = 'utf-8') as f:
   f.write(cell.txt)

This is making an assumption that cell.text is a string, by the way

CodePudding user response：

This may help, further you can read about file handling from here - https://www.w3schools.com/python/python_file_write.asp

from bs4 import BeautifulSoup
from selenium import webdriver
import pandas as pd
import time
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium import webdriver
import time


options = Options()
options.add_experimental_option("w3c", True)
options = webdriver.ChromeOptions()

f = open("datafile.txt", "a")

url = "https://www150.statcan.gc.ca/n1/pub/71-607-x/2021004/exp-eng.htm?r1=(1)&r2=0&r3=0&r4=12&r5=0&r7=0&r8=2022-02-01&r9=2022-02-01"

driver = webdriver.Chrome(ChromeDriverManager().install(),options=options)

driver.get(url)
table = driver.find_element_by_id('report_table')
body = table.find_element_by_tag_name('tbody')
cells = body.find_elements_by_tag_name('td')

for cell in cells:
    #print(cell.text)
    f.write(cell.text)

# run the loop 26 times
for i in range(26):
    
    # your code
    table = driver.find_element_by_id('report_table')
    body = table.find_element_by_tag_name('tbody')
    cells = body.find_elements_by_tag_name('td')
    for cell in cells:
        #print(cell.text)
        f.write(cell.text)
        
    # click on the Next button    
    driver.find_element_by_xpath('//*[@id="report_results_next"]').click()
f.close()