Web scraping using selenium and beautifulsoup for "load more" button:-CodePudding

I am trying to do web scraping for the the below webpage to collect the scammed ICOs: https://foundico.com/#ico-pan-4 I use execute_script() to press the button "load more" that loads all the list. But when I try to scrape, it reads only the list before the button. The button is clicked because when I try to search for it after execute_script, it doesn't find it.

load more button

WebDriverWait(driver,20).until(expected_conditions.visibility_of_element_located((By.XPATH,"//button[@data-type='SCAM']"))) 
time.sleep(0.5)

python_button = driver.find_element(By.XPATH, "//button[@data-type='SCAM']")
print(python_button.text)
driver.execute_script("arguments[0].scrollIntoView();", python_button)
driver.execute_script("arguments[0].click();", python_button)
soup=BeautifulSoup(driver.page_source)

for a in soup.findAll('div', attrs={'class': "ico-item ii-scm"}):
    name=a.find("span", attrs={'class':"ii-name"})
    ended = a.find("span", attrs={'class':"ii-det"})
    category = a.find("span", attrs={'class':"ii-ico-cat"})
    ICO_n.append(name.text) if name.text is not None else ICO_n.append("--")
    Ended_L.append(ended.text) if ended.text is not None else Ended_L.append("--")
    category_L.append(category.text) if category.text is not None else category_L.append("--")

CodePudding user response：

After click on the load more button it take some time to load. Provide some delay for that.

WebDriverWait(driver,20).until(expected_conditions.visibility_of_element_located((By.XPATH,"//button[@data-type='SCAM']"))) 
time.sleep(0.5)

python_button = driver.find_element(By.XPATH, "//button[@data-type='SCAM']")
print(python_button.text)
driver.execute_script("arguments[0].scrollIntoView();", python_button)
driver.execute_script("arguments[0].click();", python_button)
time.sleep(2) # provide some delay here
soup=BeautifulSoup(driver.page_source)

for a in soup.findAll('div', attrs={'class': "ico-item ii-scm"}):
    name=a.find("span", attrs={'class':"ii-name"})
    ended = a.find("span", attrs={'class':"ii-det"})
    category = a.find("span", attrs={'class':"ii-ico-cat"})
    ICO_n.append(name.text) if name.text is not None else ICO_n.append("--")
    Ended_L.append(ended.text) if ended.text is not None else Ended_L.append("--")
    category_L.append(category.text) if category.text is not None else category_L.append("--")