Hey everyone i am trying to scrape a website and there is not much problem in that but the main problem i am facing is that there is "load more" button in the site so before scraping the website i just thought to click that button till it doesnt exist any more so i wrote some code to do that but now when i run that code i am getting two error and these two errors are random i mean sometimes it occurs and other times it don't so the first error is Stale element exception and the second one is element is not clickable as it is being overlapped or something , now to stop that overlapping i added a line that whenever load button is clicked page will be scrolled back to top but now it is getting overlapped at there. So i am not able to solve these two problems here is the code:-

#importing webdriver from selenium
from numpy.lib.npyio import load
from selenium import webdriver
from selenium.common.exceptions import ElementClickInterceptedException, ElementNotInteractableException, ElementNotSelectableException, NoSuchElementException, StaleElementReferenceException
import csv
import pandas as pd
from selenium.webdriver.common.keys import Keys


#selecting Firefox as the browser
#in order to select Chrome
# webdriver.Chrome() will be used
path = r"D:\\Chromedriver.exe"
driver = webdriver.Chrome(path)

#URL of the website
url = "https://www.carecredit.com/doctor-locator/results/Any-Profession/Any-Specialty//?Sort=D&Radius=75&Page=1"
driver.maximize_window()
#opening link in the browser
driver.get(url)
driver.implicitly_wait(100)
search_bar = driver.find_element_by_xpath('//*[@id="location-display"]')
search_bar.click()
search_bar.send_keys("00704")

search_button = driver.find_element_by_xpath('//*[@id="dl-locate-button-new"]')
search_button.click()

driver.implicitly_wait(50)

distance = driver.find_element_by_xpath('//*[@id="dl-radius-new"]')
distance.click()

driver.implicitly_wait(100)

cookies = driver.find_element_by_xpath('//*[@id="onetrust-accept-btn-handler"]')
cookies.click()


i = 0
loadMoreButtonExists = True

while loadMoreButtonExists:
    try:
        driver.implicitly_wait(100)
        load_more =  driver.find_element_by_xpath('//*[@id="next-page"]')
        load_more.click()
        driver.find_element_by_tag_name('body').send_keys(Keys.CONTROL   Keys.HOME)
        print("done" str(i))
        i = i 1
        driver.implicitly_wait(30)
    except ElementNotInteractableException:
        loadMoreButtonExists = False

print("done loading button")

practise_name = driver.find_elements_by_class_name('page-results')
try:
    for element in practise_name:

        practise_name = element.find_element_by_class_name('dl-result-header dl-map-hover')
        address = element.find_element_by_class_name('dl-result-address')
        phone_no = element.find_element_by_class_name('no-mobile')
        # l=[]
        # l1 = []
        # temp_list=[]
        # l.append(element.text)
        
        # print(l)
        print(practise_name.text)
        print(address.text)
        print(phone_no.text)
except Exception as e:
    print(e)
    driver.close()
driver.close()

and if my code reaches the place where i actually scrape data then at that point it shows invalid id even when i checked the id and its correct. So Please check that part too. Thank you

CodePudding user response：

Try like below and confirm:

You can catch the respective exception like below:

# Imports Required:
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import ElementClickInterceptedException,StaleElementReferenceException,TimeoutException
import time

...
wait.until(EC.element_to_be_clickable((By.ID,"onetrust-accept-btn-handler"))).click()

while True:
    try:
        load_more = wait.until(EC.visibility_of_element_located((By.ID,"next-page")))
        driver.execute_script("arguments[0].scrollIntoView(true);",load_more)
        driver.execute_script("window.scrollBy(0,-200);")
        load_more.click()
        time.sleep(2)
    except ElementClickInterceptedException: # This might not occur since we are scrolling so that the element is visible
        print("in ElementClickInterceptedException")
        pass
    except StaleElementReferenceException: # This might not occur too, since we are trying to find the Load more element everytime.
        print("in StaleElementReferenceException")
        pass
    except TimeoutException: # TimeoutException means there is no Load more button
        print("in TimeoutException")
        break
...