how to scrape all the pages of transfermarkt using selenium-CodePudding

from selenium import webdriver
import time
driver = webdriver.Firefox()
f = open("superliga.csv", "w")
driver.get("https://www.transfermarkt.com/")
f.write("Emri,Pozicioni,Mosha,Cmimi\\n")
f = open("superliga.csv", "a")
time.sleep(1)
for page in range(1,4,1):
driver.get("https://www.transfermarkt.com/kategoria-superiore/marktwerte/wettbewerb/ALB1")


     elementet=driver.find_elements("xpath","/html/body/div[2]/main/div[2]/div[1]/div/div[3]/div/table/tbody/tr")

     for element in elementet:
          Emri =element.find_element("xpath", "./td[2]/table/tbody/tr[1]/td[2]/a").get_attribute("innerHTML")
          Pozicioni =element.find_element("xpath", "./td[2]/table/tbody/tr[2]/td").get_attribute("innerHTML")
          Mosha =element.find_element("xpath", "./td[4]").get_attribute("innerHTML")
          Cmimi =element.find_element("xpath", "./td[6]/a").get_attribute("innerHTML")
          f.write("{},{},{},{}\n".format(Emri, Pozicioni,Mosha,Cmimi))
f.close()

This code displays the first page players 4 times where I NEED ALL THE PLAYERS THAT ARE ON THE 4 PAGES

CodePudding user response：

Try the following. I modified the URL slightly.

for page in range(1,4,1):
    driver.get(f"https://www.transfermarkt.com/kategoria-superiore/marktwerte/wettbewerb/ALB1/page/{page}"