Selenium web scraping :

1. Dropdown list change
2. tried to changed results scraping
3. faild
4. please help my problem. thanks in advance.

'''

'''
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.select import Select
import time

driver=webdriver.Chrome(executable_path=r'C:\Program Files\Python39\chromedriver.exe')
driver.maximize_window()
driver.get("https://www.gastite.com/locator/?cats=109")

for i in range(1,3,1):
    state=driver.find_element(By.NAME, 'state')
    stateDD=Select(state)
    stateDD.select_by_index(i)
    driver.find_element(By.XPATH,'//*[@id="content"]/div[3]/form/input[2]')
    time.sleep(2)
    lists=driver.find_elements_by_css_selector("div.repcontent > a")
    #print(lists)
    for list in lists:
        company=list.find_element_by_class_name('namelink company_title').text
        address=list.find_element_by_class_name('address').text
        address1=list.find_element_by_class_name('address2').text
        tel=list.find_element_by_tag_name('span').text
        fax=list.find_element_by_tag_name('span').text
        web=list.get_attribute('href')
        print(company, address, address1, tel, fax, web)
    
'''

CodePudding user response：

I select one state from the dropdownlist for an example and rest of the your attempt.

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait 
from selenium.webdriver.support import expected_conditions as EC

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.select import Select
from bs4 import BeautifulSoup

driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))

url ='https://www.gastite.com/locator/?cats=109' 
driver.maximize_window()

Select(WebDriverWait(driver,20).until(EC.visibility_of_element_located((By.XPATH, "//select[@name='state']")))).select_by_value("AL")

soup = BeautifulSoup(driver.page_source,'lxml')
data=[]
for card in soup.select('#resultscontainer > ul > li > div.repcontent'):
    company = card.select_one('h3.company_title').text
    print(company)
    address = card.select_one('div.address').text
    address2 = card.select_one('div.address2').text
    phone = card.select('span').contents[0]
    fax = card.select('span').contents[1]

    data.append({
        'company':company,
        'address':address,
        'address2':address2,
        'phone':phone,
        'fax':fax
        })
        
print(data)