I try to get the len of all products displayed on this site https://www.bershka.com/ro/femeie/accesorii/șosete-c1010194004.html

Using this code

import time
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains


options = Options()
options = webdriver.ChromeOptions()
options.add_experimental_option("detach", True)
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
driver.get("https://www.bershka.com/ro/femeie/accesorii/șosete-c1010194004.html")


cookies_bttn = driver.find_element(By.ID, "onetrust-accept-btn-handler")
cookies_bttn.click()
driver.implicitly_wait(10)
country_save = driver.find_element(By.CSS_SELECTOR, "#geoblocking > div > div > div.select-country-container > button.button.is-sm.confirm")
country_save.click()
hoover = ActionChains(driver)

product = driver.find_elements(By.CLASS_NAME, "grid-item normal")
z = 0
for sosete in product:
    sth = sosete.find_element(By.XPATH, '//*[@id="main-content"]/div/div/div[2]/section[1]/div/ul/li[" str(z 1) "]/div')
    z = z 1
    print(len(sth))

I don't know why it is saying sosete is not defined while I clearly defined it in for loop. Any help please?

Also if I try to get all products len by using class name only 20 products out of 31 are printed like so:

whole_product = driver.find_elements(By.CLASS_NAME, "grid-card-link")
print(len(whole_product))


i = 0
product = driver.find_element(By.CLASS_NAME, "product-image")
hoover.move_to_element(product)
sosete = driver.find_elements(By.CLASS_NAME, "quick-purchase")
for purchase_bttn in sosete:
    purchase_bttn.click()
    time.sleep(1)
    i = i   1

CodePudding user response：

Page is being loaded dynamically, as you scroll. here is way to (correctly) define the product range, scroll the page, wait for them to load, and print them out:

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
import time as t

chrome_options = Options()
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument('disable-notifications')
chrome_options.add_argument("window-size=1280,720")

webdriver_service = Service("chromedriver/chromedriver") ## path to where you saved chromedriver binary
driver = webdriver.Chrome(service=webdriver_service, options=chrome_options)
wait = WebDriverWait(driver, 25)
url = 'https://www.bershka.com/ro/femeie/accesorii/șosete-c1010194004.html'
driver.get(url)
footer = wait.until(EC.element_to_be_clickable((By.XPATH, '//div[@]')))
pbody = wait.until(EC.presence_of_element_located((By.TAG_NAME, 'body')))
for x in range(5):
    pbody.send_keys(Keys.PAGE_DOWN)
    print('scrolled')
    t.sleep(5)
sosetute = wait.until(EC.presence_of_all_elements_located((By.XPATH, '//div[@]')))

print('How many socks?', len(sosetute))
for ciorapel in sosetute:
    print(ciorapel.text)

Result in terminal:

scrolled
scrolled
scrolled
scrolled
scrolled
How many socks? 31
Set 3 perechi de șosete fluture
55,90 RON
Șosete lungi cu dungi
39,90 RON
Set 3 perechi șosete ciuperci
39,90 RON
Set 3 perechi șosete Snoopy
39,90 RON
Set 3 perechi șosete Fetițele Powerpuff
55,90 RON
Set 3 perechi șosete Lizzie
39,90 RON
Set 3 perechi șosete până la gleznă broderie
39,90 RON
Jambiere denim
109,90 RON
Jambiere blană
129,90 RON
Jambiere tricot
55,90 RON
  1 Culoare
Jambiere tricot
55,90 RON
  1 Culoare
Set 3 perechi șosete print
39,90 RON
Set 4 perechi șosete neon
39,90 RON
Set 4 perechi șosete simple
39,90 RON
Set de 3 perechi de șosete cu text
39,90 RON
Set de 2 perechi de șosete texturate
55,90 RON
Șosete print Art Series
69,90 RON
  1 Culoare
Șosete print Art Series
69,90 RON
  1 Culoare
Set 3 perechi șosete print
39,90 RON
Set 3 perechi șosete curly broderie
39,90 RON
Set de 3 perechi de șosete racing
39,90 RON
Set 3 perechi șosete print
39,90 RON
Set 3 perechi șosete print
39,90 RON
Set de 3 perechi de șosete racing
39,90 RON
Set 3 perechi șosete cu imprimeu varsity
39,90 RON
Set 3 perechi șosete lungi varsity campus
39,90 RON
Șosete text
39,90 RON
Set 3 perechi șosete colorate
39,90 RON
Set 3 perechi șosete print
55,90 RON
Șosete broderie Sponge Bob
39,90 RON
Set 3 perechi șosete scurte broderie
39,90 RON

Selenium documentation is quite good.