Good day everyone:
I’d like to get data from the url （sorry its encoded, cause online editor always report its a spam）
after click the basketball of mainmenu and close AD then click the date combobox menu and select the date by date
Now my code try to get all data everyday until yesterday , which include the following fields
- date
- league
- time / status (depend on odd or even line)
- team / score1 ~ score5 / score total
After got all data , will save to SQL DB
Below is my code
import time
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.action_chains import ActionChains
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
driver = webdriver.Chrome()
driver.set_window_size(1500, 1350)
delay = 10
# open url (sorry for the url , cause system always report its a spam)
url1 = "http"
url2 = "888.net/spo"
url3 = "s://web2.sa8"
url4 = "rt/Games.a"
url5 = "evice=pc"
url6 = "spx?lang=3&d"
driver.get(url1 url3 url2 url4 url6 url5)
# jump to basketball
locator = (By.XPATH, '//*[@id="menuList"]/div/ul/li[3]/div[2]/a[1]')
pointer = WebDriverWait(driver, delay).until(EC.presence_of_element_located(locator),"element not found")
actions = ActionChains(driver)
actions.click(pointer).perform()
time.sleep(1)
# close AD
locator = (By.ID, 'btn_close')
pointer = WebDriverWait(driver, delay).until(EC.presence_of_element_located(locator),"element not found")
actions = ActionChains(driver)
actions.click(pointer).perform()
# wait for date
locator = (By.XPATH, './/*[@id="dateOption"]')
pointer = WebDriverWait(driver, delay).until(EC.presence_of_element_located(locator),"element not found")
all_dates = []
_dates = driver.find_elements(By.XPATH, './/*[@id="dateOption"]/a[*]')
for date in _dates:
rel = date.get_attribute("rel")
all_dates.append(rel)
#print(rel)
#print(*all_dates, sep = ", ")
#_dates = driver.find_elements(By.XPATH, './/*[@id="dateOption"]/a[*]')
date_counter = 1
for date in range(len(_dates)-8): # until yesterday
# click pull down date menu
locator = (By.XPATH, './/*[@id="chooseDate"]')
pointer = WebDriverWait(driver, delay).until(EC.presence_of_element_located(locator), "element not found")
actions.move_to_element(pointer).perform()
actions.click(pointer).perform()
# click date#n
temp = './/*[@id="dateOption"]/a[' str(date_counter) ']/span[1]'
#print("XPATH=",temp)
locator = (By.XPATH, temp)
pointer = WebDriverWait(driver, delay).until(EC.presence_of_element_located(locator), "element not found")
actions.move_to_element(pointer).perform()
actions.click(pointer).perform()
date_counter = 1
leagues = driver.find_elements(By.XPATH, './/*[@id="scheduleBottom"]/table[*]')
print(all_dates[date_counter],'\tdaily leagues:', len(leagues))
league_counter = 1
for league in leagues:
# print("\tleague.text=",league.text,"\n")
#_rows = Block.find_elements(By.TAG_NAME, "tr")
league_Title = league.find_element(By.XPATH ,'//*[@id="scheduleBottom"]/table[*]/caption/a')
#time.sleep(2)
_rows = league.find_elements(By.XPATH, ".//*[contains(@id, '_mainRow') or contains(@id, '_secondRow')]")
print("\tleague:", league_Title.text) #, '(', len(_rows), ')')
row_counter = 1
for row in _rows:
#print("\t\trow #",row_counter, row.text.replace('\n', ' | ')) # <- clean text
if row_counter %2 > 0:
_time = row.find_element(By.XPATH, './/*[@id="*_gd"]')
_status = row.find_element(By.XPATH, './/*[@id="*_gs"]')
_team = row.find_element(By.XPATH, './/*//*[@id="_ta"]')
_score1 = row.find_element(By.XPATH, './/*[@id="*_rsa1"]')
_score2 = row.find_element(By.XPATH, './/*[@id="*_rsa2"]')
_score3 = row.find_element(By.XPATH, './/*[@id="*_rsa3"]')
_score4 = row.find_element(By.XPATH, './/*[@id="*_rsa4"]')
_score5 = row.find_element(By.XPATH, './/*[@id="*_rsa5"]')
_scorea = row.find_element(By.XPATH, './/*[@id="*_ra"]"]')
# insert to DB
row_counter = 1
league_counter = 1
print('---')
time.sleep(15)
driver.quit()
and I got the error message
2022/03/10 daily leagues: 45
league: NBA
Traceback (most recent call last):
File "/Users/andy/Python/sa8888 #5.7.py", line 85, in <module>
_time = row.find_element(By.XPATH, './/*[@id="*_gd"]')
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/selenium/webdriver/remote/webelement.py", line 735, in find_element
return self._execute(Command.FIND_CHILD_ELEMENT,
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/selenium/webdriver/remote/webelement.py", line 710, in _execute
return self._parent.execute(command, params)
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/selenium/webdriver/remote/webdriver.py", line 424, in execute
self.error_handler.check_response(response)
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/selenium/webdriver/remote/errorhandler.py", line 247, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":".//*[@id="*_gd"]"}
(Session info: chrome=96.0.4664.110)
What should I do to fix it? Thanks in advance
CodePudding user response：
* will not work in string "*_gd" in @id="*_gd" and other xpaths.
You have to use contains() - './/*[contains(@id, "_gd")]'
BTW:
Selenium probably uses XPath 1.2 because it has also starts-with()
but it doesn't have ends-with() (which exists in XPath 2.0)
Stack Overflow: XPath testing that string ends with substring?