Element not interactable (clickable) while scraping a website-CodePudding

Can anyone please tell me what's wrong with the code?

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
import csv
import requests
from bs4 import BeautifulSoup

rows, cols = (1000,11)
rows = [[0 for i in range(cols)] for j in range(rows)]
driver_path = 'C:/Users/adith/Downloads/chromedriver_win32/chromedriver.exe'
brave_path = 'C:/Program Files/BraveSoftware/Brave-Browser/Application/brave.exe'
option = webdriver.ChromeOptions()
option.binary_location = brave_path
browser = webdriver.Chrome(executable_path=driver_path, options=option)

browser.get('https://www.dell.com/community/Laptops/ct-p/Laptops')

cookies = {
    'lithiumLogin:vjauj58549': '~2TtiW3OEsenvCn5Ir~fBcCal7YbmhAmxNWLe4LgaSRCss_g69Gqm2CAs-fDA_FtccFLDK3AoWuzXHz72fb'
}


def load():
    count = 0
    while True:
        page = requests.get(browser.current_url)
        soup = BeautifulSoup(page.content)
        button = [z.text for z in soup.find_all(class_='lia-link-navigation')]
        print(button[-1])
        if(button[-1]=='Load more'):
            count = count   1
            browser.find_element(By.XPATH, '//*[@id="btn-load-more"]').click()
            if(count>12):
                break
        else:
            break
            
try:
    load()
except:
    load()

It is able to click the load button 3 times but then I get an error as shown below.

Load more
Load more
Load more
Load more
Load more
Load more
---------------------------------------------------------------------------
ElementClickInterceptedException          Traceback (most recent call last)
<ipython-input-6-ef6cae973b0d> in <module>
     18 try:
---> 19     load()
     20 except:

<ipython-input-6-ef6cae973b0d> in load()
     11             count = count   1
---> 12             browser.find_element(By.XPATH, '//*[@id="btn-load-more"]').click()
     13             if(count>15):

~\anaconda3\lib\site-packages\selenium\webdriver\remote\webelement.py in click(self)
     87         """Clicks the element."""
---> 88         self._execute(Command.CLICK_ELEMENT)
     89 

~\anaconda3\lib\site-packages\selenium\webdriver\remote\webelement.py in _execute(self, command, params)
    395         params['id'] = self._id
--> 396         return self._parent.execute(command, params)
    397 

~\anaconda3\lib\site-packages\selenium\webdriver\remote\webdriver.py in execute(self, driver_command, params)
    434         if response:
--> 435             self.error_handler.check_response(response)
    436             response['value'] = self._unwrap_value(

~\anaconda3\lib\site-packages\selenium\webdriver\remote\errorhandler.py in check_response(self, response)
    246             raise exception_class(message, screen, stacktrace, alert_text)  # type: ignore[call-arg]  # mypy is not smart enough here
--> 247         raise exception_class(message, screen, stacktrace)
    248 

ElementClickInterceptedException: Message: element click intercepted: Element is not clickable at point (470, 1629)
  (Session info: chrome=103.0.5060.114)

Can someone please explain what's wrong with the code? Also, can anyone help me out to eliminate the use of BeautifulSoup library completely to get the required solution?

CodePudding user response：

Try scrolling to the end of the page (where the button is located) before clicking on it:

def load():
    count = 0
    while True:
        page = requests.get(browser.current_url)
        soup = BeautifulSoup(page.content)
        button = [z.text for z in soup.find_all(class_='lia-link-navigation')]
        print(button[-1])
        if(button[-1]=='Load more'):
            count = count   1
            browser.execute_script( "window.scrollTo(0,document.body.scrollHeight);")
            time.sleep(1)
            browser.find_element(By.XPATH, '//*[@id="btn-load-more"]').click()
            if(count>12):
                break
        else:
            break

CodePudding user response：

You're using requests and bs4 when you already have the browser via selenium, you should just use selenium without requests to do the same job:

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
import csv

import requests
from bs4 import BeautifulSoup

rows, cols = (1000,11)
rows = [[0 for i in range(cols)] for j in range(rows)]


driver_path = 'C:/Users/adith/Downloads/chromedriver_win32/chromedriver.exe'
brave_path = 'C:/Program Files/BraveSoftware/Brave-Browser/Application/brave.exe'
option = webdriver.ChromeOptions()
option.binary_location = brave_path
browser = webdriver.Chrome(executable_path=driver_path, options=option)

browser.get('https://www.dell.com/community/Laptops/ct-p/Laptops')

def load():
    count = 0
    while True:
        button = [z.text for z in driver.find_elements(By.CLASS_NAME, 'lia-link-navigation')]
        print(button[-1])

        if(button[-1]=='Load more'):
            count = count   1
            browser.execute_script( "window.scrollTo(0,document.body.scrollHeight);")
            browser.find_element(By.XPATH, '//*[@id="btn-load-more"]').click()
            if(count>12):
                break
        else:
            break
            
try:
    load()
except:
    load()