loop through two dropdowns and downloading files-CodePudding

I would like to download .csv reports for all states and all compliance periods from this web page.

In other words, the selenium script would select a state (for example, "DC") a reporting period (for example, "Jan 2021 - Dec 2021"), and then click "submit." THEN the script would export the results to a spreadsheet by clicking the image that says "CSV".

Ideally, the spreadsheet would do this for all states and all reporting periods. So at the end, my downloads folder would be full of spreadsheets.

I cannot, for the life of me, figure out how to get this to work!

This is what I have so far. There are no loops like I think there should be.

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import chromedriver_autoinstaller
import time
import glob
import os

chromedriver_autoinstaller.install() 
chromeOptions = webdriver.ChromeOptions() 
driver = webdriver.Chrome()
wait = WebDriverWait(driver, 20)

url = "https://gats.pjm-eis.com/GATS2/PublicReports/RPSRetiredCertificatesReportingYear"

driver.get(url)
driver.find_element(By.CSS_SELECTOR, "table:nth-child(4)").click()
driver.find_element(By.ID, "SelectedState0_B-1").click()
driver.find_element(By.ID, "SelectedState0_DDD_L_LBI5T0").click()
driver.find_element(By.ID, "ReportingYear0_B-1").click()
driver.find_element(By.ID, "ReportingYear0_DDD_L_LBI0T0").click()
driver.find_element(By.CSS_SELECTOR, ".dx-vam:nth-child(2)").click()
driver.find_element(By.ID, "CSV0Img").click()

Thank you very much for your help! I truly appreciate it.

CodePudding user response：

Here is the Solution!

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import time
import glob
import os

chromeOptions = webdriver.ChromeOptions()
driver = webdriver.Chrome()
wait = WebDriverWait(driver, 20)

url = "https://gats.pjm-eis.com/GATS2/PublicReports/RPSRetiredCertificatesReportingYear"

state = 'DC'  # Enter State Name Here
compliance_period = 'Jan 2020 - Dec 2020'   # Enter Compliance Period Here


driver.get(url)
wait.until(EC.element_to_be_clickable((By.XPATH, '(//*[@])[1]'))).click()  # Clicking on Dropdown Arrow Down Icon
wait.until(EC.element_to_be_clickable((By.XPATH, '//tr[@]//td[text()="'   state   '"]'))).click()
wait.until(EC.element_to_be_clickable((By.XPATH, '(//*[@])[2]'))).click()  # Clicking on Dropdown Arrow Down Icon
wait.until(EC.element_to_be_clickable((By.XPATH, '//tr[@]//td[text()="'   compliance_period   '"]'))).click()
driver.find_element(By.XPATH, '//*[text()="Submit"]').click()
wait.until(EC.element_to_be_clickable((By.XPATH, '//*[@id="CSV0Img"]'))).click()
print("Successfully Downloaded!")

time.sleep(10)
driver.quit()

* Updated another Solution below as per the case mentioned in the comments where we've to make it loop through all the states and through all the compliance periods.

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import time

chromeOptions = webdriver.ChromeOptions()
driver = webdriver.Chrome()
wait = WebDriverWait(driver, 20)

url = "https://gats.pjm-eis.com/GATS2/PublicReports/RPSRetiredCertificatesReportingYear"


driver.get(url)
count_state = len(driver.find_elements(By.XPATH, '//table[@id="SelectedState0_DDD_L_LBT"]//tr'))
for i in range(1, count_state   1):
    wait.until(EC.element_to_be_clickable((By.XPATH, '(//*[@])[1]'))).click()  # Clicking on Dropdown Arrow Down Icon
    wait.until(EC.element_to_be_clickable((By.XPATH, '(//table[@id="SelectedState0_DDD_L_LBT"]//tr)['   str(i)   ']'))).click()
    state_name = driver.find_element(By.XPATH, '(//table[@id="SelectedState0_DDD_L_LBT"]//tr/td)['   str(i)   ']').get_attribute("textContent")
    count_period = len(driver.find_elements(By.XPATH, '//table[@id="ReportingYear0_DDD_L_LBT"]//tr'))
    for j in range(1, count_period   1):
        wait.until(EC.element_to_be_clickable((By.XPATH, '(//*[@])[2]'))).click()  # Clicking on Dropdown Arrow Down Icon
        wait.until(EC.element_to_be_clickable((By.XPATH, '(//table[@id="ReportingYear0_DDD_L_LBT"]//tr)['   str(j)   ']'))).click()
        driver.find_element(By.XPATH, '//*[text()="Submit"]').click()
        wait.until(EC.element_to_be_clickable((By.XPATH, '//*[@id="CSV0Img"]'))).click()
        compliance_period_name = driver.find_element(By.XPATH, '(//table[@id="ReportingYear0_DDD_L_LBT"]//tr/td)['   str(j)   ']').get_attribute("textContent")
        print("Successfully Downloaded for State:", state_name, "  and   Compliance Period: ", str(compliance_period_name))
    print("\n")

time.sleep(10)
driver.quit()