I am trying to write a scraper and I want the scraper to scroll down until it reaches the button "Load More" and then click it, but it is telling me that no such button exists. Here is the code:

SCROLL_PAUSE_TIME = 5

# Get scroll height
last_height = driver.execute_script("return document.body.scrollHeight")

# Keep scrolling
while True:
    # Scroll to bottom
    driver.execute_script(
        "window.scrollTo(0, document.body.scrollHeight);")

    # Wait to load page
    time.sleep(SCROLL_PAUSE_TIME)

    # Calculate new scroll height and compare with last scroll height
    new_height = driver.execute_script("return document.body.scrollHeight")
    if new_height == last_height:
        break
    last_height = new_height

driver.find_element_by_xpath(
    '//button[normalize-space()="LOAD MORE"]').click()
time.sleep(SCROLL_PAUSE_TIME)
while True:
    # Scroll to bottom
    driver.execute_script(
        "window.scrollTo(0, document.body.scrollHeight);")

    # Wait to load page
    time.sleep(SCROLL_PAUSE_TIME)

    # Calculate new scroll height and compare with last scroll height
    new_height = driver.execute_script("return document.body.scrollHeight")
    if new_height == last_height:
        break
    last_height = new_height

And here is the url: https://messari.io/governor/daos?types=Protocol

CodePudding user response：

Tested working solution

import time
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException 

driver = webdriver.Firefox()

driver.get("https://messari.io/governor/daos?types=Protocol")



def check_exists_by_xpath(xpath):
    try:
        driver.find_element_by_xpath(xpath)
    except NoSuchElementException:
        return False
    return True


xpath="//*[contains(text(), 'Load More')]"

loop to continue scrolling until the button is found with a 3 second pause for the page to load.

while  check_exists_by_xpath(xpath)==False :

    time.sleep(3)
    driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
    print("scrolling")



print('button successfully found , clicking ')
driver.find_element_by_xpath(xpath).click()
print('finish')

Output

scrolling
scrolling
scrolling
scrolling
scrolling
scrolling
scrolling
scrolling
scrolling
button successfully found , clicking 
finish