Home > database >  Reading weblink from dataframe throws "stale element reference: element is not attached to the
Reading weblink from dataframe throws "stale element reference: element is not attached to the

Time：11-06

I got a dataframe that contains links to google reviews of two restaurants. I wanted to load all reviews of two restaurants (one by one) into the browser and then save them into a new data frame. I wrote a script that reads and load all reviews into the browser as follow:

from selenium import webdriver
import pandas as pd
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.action_chains import ActionChains
import time

link_df =   Link
0   https://www.google.com/search?q=restaurant in christchurch&biw=1280&bih=614&hotel_occupancy=2&tbm=lcl&sxsrf=AOaemvI4qlEAr3btedb6PCx9U53RtXkI2Q:1635630947742&ei=Y799YaHfLOKZ4-EPoeqjmA4&oq=restaurant in christchurch&gs_l=psy-ab.3...0.0.0.614264.0.0.0.0.0.0.0.0..0.0....0...1c..64.psy-ab..0.0.0....0.7jAOI05vCjI#lrd=0x6d318a3aa3041455:0x5f83f4fae76d8656,1,,,&rlfi=hd:;si:6882614014013965910,l,ChpyZXN0YXVyYW50IGluIGNocmlzdGNodXJjaEiglZKhm6qAgAhaKBAAGAAYAiIacmVzdGF1cmFudCBpbiBjaHJpc3RjaHVyY2gqBAgDEACSARJidXJtZXNlX3Jlc3RhdXJhbnSqAQwQASoIIgRmb29kKAA,y,UB2auy7TMYs;mv:[[-43.4870861,172.6509735],[-43.5490232,172.5976049]]
1   https://www.google.com/search?q=restaurant in christchurch&biw=1280&bih=614&hotel_occupancy=2&tbm=lcl&sxsrf=AOaemvI4qlEAr3btedb6PCx9U53RtXkI2Q:1635630947742&ei=Y799YaHfLOKZ4-EPoeqjmA4&oq=restaurant in christchurch&gs_l=psy-ab.3...0.0.0.614264.0.0.0.0.0.0.0.0..0.0....0...1c..64.psy-ab..0.0.0....0.7jAOI05vCjI#lrd=0x6d318bf82139caaf:0xf115cd7fe794cbcc,1,,,&rlfi=hd:;si:17372017086881385420,l,ChpyZXN0YXVyYW50IGluIGNocmlzdGNodXJjaEjh9auu-q6AgAhaKBAAGAAYAiIacmVzdGF1cmFudCBpbiBjaHJpc3RjaHVyY2gqBAgDEACSAQpyZXN0YXVyYW50qgEMEAEqCCIEZm9vZCgA,y,ZeJbBWd7wDg;mv:[[-43.4870861,172.6509735],[-43.5490232,172.5976049]]

i = 0
driver = webdriver.Chrome()
for index, i in link_df.iterrows():
    base_url = i['Link']   #link_df['Link'][i]
    
    driver.get(base_url)
    WebDriverWait(driver,10).until(EC.element_to_be_clickable((By.XPATH,"//div[./span[text()='Newest']]"))).click()
    print('Restaurant number is ',index)
    
    title = driver.find_element_by_xpath("//div[@class='P5Bobd']").text
    address = driver.find_element_by_xpath("//div[@class='T6pBCe']").text
    overall_rating = driver.find_element_by_xpath("//div[@class='review-score-container']//span[@class='Aq14fc']").text
    
    total_reviews_text =driver.find_element_by_xpath("//div[@class='review-score-container']//div//div//span//span[@class='z5jxId']").text
    num_reviews = int (total_reviews_text.split()[0])
    all_reviews = WebDriverWait(driver, 20).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, 'div.gws-localreviews__google-review')))
    time.sleep(2)
    total_reviews = len(all_reviews)
    
    while total_reviews < num_reviews:
        driver.execute_script('arguments[0].scrollIntoView(true);', all_reviews[-1])
        WebDriverWait(driver, 5, 0.25).until_not(EC.presence_of_element_located((By.CSS_SELECTOR, 'div[class$="activityIndicator"]')))
        time.sleep(5)
        all_reviews = WebDriverWait(driver, 5).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, 'div.gws-localreviews__google-review')))
        print(total_reviews)
        total_reviews  =1
    reviews_info = driver.find_elements_by_xpath("//div[@class='jxjCjc']")
    review_information = pd.DataFrame(columns=["Restaurant title","Restaurant rating","Total reviews","Reviewer Name","Rating", "Review"])
    name= ''
    rating = ''
    text = ''
    
    
    for index,review_info in enumerate(reviews_info):
        name = review_info.find_element_by_xpath("./div/div/a").text
        rating = review_info.find_element_by_xpath(".//div[@class='PuaHbe']//g-review-stars//span").get_attribute('aria-label')
        text = review_info.find_element_by_xpath(".//div[@class='Jtu6Td']//span").text
        review_information.at[len(review_information)] = [title,overall_rating,num_reviews,name,rating,text]
    
    filename = 'Google_reviews'   ' '  pd.to_datetime("now").strftime("%Y_%m_%d") '.csv'
    files_present = glob.glob(filename)
    if files_present:
        review_information.to_csv(filename,index=False,mode='a',header=False)
    else:
        review_information.to_csv(filename,index=False)
    
    driver.get('https:ww.google.com')
    time.sleep(3)

The problem is that script throws an error when it reaches the following line.

driver.execute_script('arguments[0].scrollIntoView(true);', all_reviews[-1])

It throws following error:

StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
  (Session info: chrome=95.0.4638.69)

When I tried the same program without storing google links in dataframe (i.e. no for loop and instead of base_url = i['Link'], I wrote base_url = google review link) it works fine.

I am not sure where I am making the mistake. Any suggestion or help to fix the issue would be highly appreciated?

CodePudding user response：

EDIT

  1. you put the creation of driver outside the for loop
  2. you cant launch the new url with gps data when the first popup is always in front, if you launch it, it stays in backdoor, the easier way is to launch a new url without gps data -> https:ww.google.com and wait 3 dec before to follow your loop:
  3. your count is not good, i have changed your selector and change the total and set some lines in comment
from selenium import webdriver
import pandas as pd
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.firefox.options import Options
import time

link_df =  ["https://www.google.com/search?q=restaurant in christchurch&biw=1280&bih=614&hotel_occupancy=2&tbm=lcl&sxsrf=AOaemvI4qlEAr3btedb6PCx9U53RtXkI2Q:1635630947742&ei=Y799YaHfLOKZ4-EPoeqjmA4&oq=restaurant in christchurch&gs_l=psy-ab.3...0.0.0.614264.0.0.0.0.0.0.0.0..0.0....0...1c..64.psy-ab..0.0.0....0.7jAOI05vCjI#lrd=0x6d318a3aa3041455:0x5f83f4fae76d8656,1,,,&rlfi=hd:;si:6882614014013965910,l,ChpyZXN0YXVyYW50IGluIGNocmlzdGNodXJjaEiglZKhm6qAgAhaKBAAGAAYAiIacmVzdGF1cmFudCBpbiBjaHJpc3RjaHVyY2gqBAgDEACSARJidXJtZXNlX3Jlc3RhdXJhbnSqAQwQASoIIgRmb29kKAA,y,UB2auy7TMYs;mv:[[-43.4870861,172.6509735],[-43.5490232,172.5976049]]",
            "https://www.google.com/search?q=restaurant in christchurch&biw=1280&bih=614&hotel_occupancy=2&tbm=lcl&sxsrf=AOaemvI4qlEAr3btedb6PCx9U53RtXkI2Q:1635630947742&ei=Y799YaHfLOKZ4-EPoeqjmA4&oq=restaurant in christchurch&gs_l=psy-ab.3...0.0.0.614264.0.0.0.0.0.0.0.0..0.0....0...1c..64.psy-ab..0.0.0....0.7jAOI05vCjI#lrd=0x6d318bf82139caaf:0xf115cd7fe794cbcc,1,,,&rlfi=hd:;si:17372017086881385420,l,ChpyZXN0YXVyYW50IGluIGNocmlzdGNodXJjaEjh9auu-q6AgAhaKBAAGAAYAiIacmVzdGF1cmFudCBpbiBjaHJpc3RjaHVyY2gqBAgDEACSAQpyZXN0YXVyYW50qgEMEAEqCCIEZm9vZCgA,y,ZeJbBWd7wDg;mv:[[-43.4870861,172.6509735],[-43.5490232,172.5976049]]"
           ]
i = 0
binary = r'C:\Program Files (x86)\Mozilla Firefox\firefox.exe'
cap = DesiredCapabilities().FIREFOX
cap["marionette"] = True
options = Options()
options.binary = binary
driver = webdriver.Firefox(options=options, capabilities=cap, executable_path="E:\\Téléchargement\\geckodriver.exe")

# i have to launch one time to accept the cookies manually 
#by setting a breakpoint after, but you dont have that i think
#driver.get(link_df[0])  

print ("Headless Firefox Initialized")


print(link_df)
for url in link_df:
    base_url = url    # i['Link']  # link_df['Link'][i]
    print(base_url)
    driver.get(base_url)
    WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//div[./span[text()='Avis les plus récents']]"))).click()

    title = driver.find_element_by_xpath("//div[@class='P5Bobd']").text
    address = driver.find_element_by_xpath("//div[@class='T6pBCe']").text
    overall_rating = driver.find_element_by_xpath("//div[@class='review-score-container']//span[@class='Aq14fc']").text

    total_reviews_text = driver.find_element_by_xpath(
        "//div[@class='review-score-container']//div//div//span//span[@class='z5jxId']").text
    num_reviews = int(total_reviews_text.split()[0])
    all_reviews = WebDriverWait(driver, 20).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, '#reviewSort .gws-localreviews__google-review')))
    # time.sleep(2)
    total_reviews = 0

    while total_reviews < num_reviews:
        driver.execute_script('arguments[0].scrollIntoView(true);', all_reviews[-1])
        WebDriverWait(driver, 5, 0.25).until_not(EC.presence_of_element_located((By.CSS_SELECTOR, 'div[class$="activityIndicator"]')))
        
        all_reviews = WebDriverWait(driver, 5).until(
            EC.presence_of_all_elements_located((By.CSS_SELECTOR, '#reviewSort .gws-localreviews__google-review')))
        total_reviews = len(all_reviews)
        print(total_reviews, len(all_reviews))

    driver.get('https:ww.google.com') # or driver.close() if no bugs
    time.sleep(3)

driver.close()
driver.quit()

it seems the solution for chrome needs some fixes:

org.openqa.selenium.StaleElementReferenceException: stale element reference: element is not attached to the page document

The literal meaning is about , The referenced element is out of date , No longer attached to the current page . Usually , This is because the page has been refreshed or skipped , The solution is , Reuse findElement or findElements Method to locate the element .

so its seems for chrome there is a problem of refreshing, so i suggest to load the number of record before to scroll, to have a fresh copy of DOM items, and i have to add a wait 1sec at the end of while loop

from selenium import webdriver
import pandas as pd
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.action_chains import ActionChains
#from selenium.webdriver.firefox.options import Options
from selenium.webdriver.chrome.options import Options
import time

link_df =  [
    "https://www.google.com/search?q=restaurant in christchurch&biw=1280&bih=614&hotel_occupancy=2&tbm=lcl&sxsrf=AOaemvI4qlEAr3btedb6PCx9U53RtXkI2Q:1635630947742&ei=Y799YaHfLOKZ4-EPoeqjmA4&oq=restaurant in christchurch&gs_l=psy-ab.3...0.0.0.614264.0.0.0.0.0.0.0.0..0.0....0...1c..64.psy-ab..0.0.0....0.7jAOI05vCjI#lrd=0x6d318a3aa3041455:0x5f83f4fae76d8656,1,,,&rlfi=hd:;si:6882614014013965910,l,ChpyZXN0YXVyYW50IGluIGNocmlzdGNodXJjaEiglZKhm6qAgAhaKBAAGAAYAiIacmVzdGF1cmFudCBpbiBjaHJpc3RjaHVyY2gqBAgDEACSARJidXJtZXNlX3Jlc3RhdXJhbnSqAQwQASoIIgRmb29kKAA,y,UB2auy7TMYs;mv:[[-43.4870861,172.6509735],[-43.5490232,172.5976049]]",
    "https://www.google.com/search?q=restaurant in christchurch&biw=1280&bih=614&hotel_occupancy=2&tbm=lcl&sxsrf=AOaemvI4qlEAr3btedb6PCx9U53RtXkI2Q:1635630947742&ei=Y799YaHfLOKZ4-EPoeqjmA4&oq=restaurant in christchurch&gs_l=psy-ab.3...0.0.0.614264.0.0.0.0.0.0.0.0..0.0....0...1c..64.psy-ab..0.0.0....0.7jAOI05vCjI#lrd=0x6d318bf82139caaf:0xf115cd7fe794cbcc,1,,,&rlfi=hd:;si:17372017086881385420,l,ChpyZXN0YXVyYW50IGluIGNocmlzdGNodXJjaEjh9auu-q6AgAhaKBAAGAAYAiIacmVzdGF1cmFudCBpbiBjaHJpc3RjaHVyY2gqBAgDEACSAQpyZXN0YXVyYW50qgEMEAEqCCIEZm9vZCgA,y,ZeJbBWd7wDg;mv:[[-43.4870861,172.6509735],[-43.5490232,172.5976049]]"
]

i = 0
binaryfirefox = r'C:\Program Files (x86)\Mozilla Firefox\firefox.exe'
binarychrome = r'C:\Program Files (x86)\Google\Chrome\Application\chrome.exe'


options = Options()

#cap = DesiredCapabilities().CHROME
#cap["marionette"] = True
#cap = DesiredCapabilities().FIREFOX
#options.binary = binaryfirefox
#driver = webdriver.Firefox(options=options, capabilities=cap, executable_path="E:\\Téléchargement\\geckodriver.exe")

options.binary_location  = binarychrome
driver = webdriver.Chrome(options=options, executable_path="E:\\Téléchargement\\chromedriver.exe" )

# same reason tha Firefox i have to load one time
# an url to accept manually the cookies
#driver.get(link_df[0])   



print(link_df)
for url in link_df:
    base_url = url    # i['Link']  # link_df['Link'][i]
    print(base_url)
    driver.get(base_url)
    WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//div[./span[text()='Newest']]"))).click()

    title = driver.find_element_by_xpath("//div[@class='P5Bobd']").text
    address = driver.find_element_by_xpath("//div[@class='T6pBCe']").text
    overall_rating = driver.find_element_by_xpath("//div[@class='review-score-container']//span[@class='Aq14fc']").text

    total_reviews_text = driver.find_element_by_xpath(
        "//div[@class='review-score-container']//div//div//span//span[@class='z5jxId']").text
    num_reviews = int(total_reviews_text.split()[0])
    all_reviews = WebDriverWait(driver, 20).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, '#reviewSort .gws-localreviews__google-review')))
    # time.sleep(2)
    total_reviews = 0

    while total_reviews < num_reviews:
        #reload to avoid exception, or trap scroll with try/except but more expznsive
        all_reviews = WebDriverWait(driver, 20).until(
            EC.presence_of_all_elements_located((By.CSS_SELECTOR, '#reviewSort .gws-localreviews__google-review')))

        driver.execute_script('arguments[0].scrollIntoView(true);', all_reviews[-1])

        total_reviews = len(all_reviews)
        print(total_reviews, len(all_reviews))
        time.sleep(1)

    driver.get('https:ww.google.com') # or driver.close() if no bugs
    time.sleep(3)

driver.close()
driver.quit()

Page link：https//www.codepudding.com/database/181265.html

Prev:Selenium Python: Wait for backgroundColor of Element to be 'rgb(...)`
Next:How do I select option from a Select Listbox dynamically during runtime ? Selenium_Java
  • Related

About Us:  Contact Us      Terms of Service       Privacy Policy

Copyright © 2010-2023，Powered By CodePudding