looking for some help with my code. Im running selenium loop keyword searches and using panda to get the table search results, but when an search term doesnt come back with any results I would get an error, if it does have result but no match with oval filter it keep running.
Example once the search reaches "Cedarcrest" it would output pandas.errors.UndefinedVariableError: name 'Description' is not defined.
This is my code, how can I fix so it would skip the search terms that yield no results?
import time
import requests
import pandas as pd
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
class KW_POST_BOT(object):
def __init__(self, browser, search_engine_url, kw_list, package):
self.browser = browser
self.package = package
self.search_engine_url = search_engine_url
self.kw_list = kw_list
def main(self):
self.browser.get("https://www.dnv.org/building-development/look-building-and-trades-permits")
### MAIN LOOP
# INJECT KEY WORD LIST
# LOOPS THROUGH KW_LIST AS KW
for kw in self.kw_list:
print("*" * 30)
print("Searching " kw)
print("*" * 30)
print("Finding Data Table")
print("-" * 30)
# WAIT 3 SECONDS
time.sleep(3)
# LOCATE ELEMENT AND CLEAR
elem = self.browser.find_element(By.XPATH,
"/html/body/div[2]/div[1]/div[2]/div[2]/div/div/div/app-root/div/div[1]/div/div[1]/div/div/ng2-completer/div/input").clear()
# LOCATE ELEMENT AND SEND KEYS : KW VARIABLE
elem = self.browser.find_element(By.XPATH,
"/html/body/div[2]/div[1]/div[2]/div[2]/div/div/div/app-root/div/div[1]/div/div[1]/div/div/ng2-completer/div/input").send_keys(
kw)
# PRESS ENTER
elem = self.browser.find_element(By.XPATH,
"/html/body/div[2]/div[1]/div[2]/div[2]/div/div/div/app-root/div/div[1]/div/div[1]/div/div/ng2-completer/div/input").send_keys(
Keys.ENTER)
# WAIT 3 SECONDS
time.sleep(3)
# PRINT IN TERMINAL
table_trs = self.browser.find_elements(By.XPATH, '//table[@id="case_table"]/tbody/tr')
value_list = []
for row in table_trs[1:]:
value_list.append({
# 'Permit': row.find_elements(By.TAG_NAME, "td")[0].text,
'Address': row.find_elements(By.TAG_NAME, "td")[1].text,
# 'Value': row.find_elements(By.TAG_NAME, "td")[2].text,
# 'Contact': row.find_elements(By.TAG_NAME, "td")[3].text,
'Status': row.find_elements(By.TAG_NAME, "td")[4].text,
'Date': row.find_elements(By.TAG_NAME, "td")[5].text,
'Description': row.find_elements(By.TAG_NAME, "td")[6].text
})
df = pd.DataFrame(value_list)
filtered_list = df[
df.eval("Description.str.contains('New') & Date.str.contains('2018|2019|2020|2021|2022').values")]
# TURN THEWEB OBJECT INTO TEXT AND ENCODE IN UTF-8
# table_data = table_data.text.encode("utf-8")
print(filtered_list)
# DEFINE KEW_WORDS LIST TO INPUT
key_words = ["Eldon", "Ruby", "Bracknell", "Pelly", "Sunset", "Edgewood", "Sycamore", "Lodge",
"Virginia", "Loraine", "Kendal", "Emerald", "Dudley", "Highland", "Highland", "Montroyal", "Ranger",
"Shirley", "Cedarcrest", "Lions", "Sunnycrest", "Beaumont", "Tudor", "Winona", "Canterbury",
"Beaconsfield", "Hampshire", "Devon", "Essex", "Derby", "Belgrave", "Cheviot", "Parliament", "Ruskin",
"Handsworth", "Rialto", "Belvedere", "Marineview", "Mapleridge", "Pheasant", "Ruthina", "Marigold",
"Marigold", "Glenwood", "Timberline", "Ventura", "Monteray", "Greenway", "Valencia", "Hermosa", "Vienna",
"Genoa", "Saville", "Granada", "Lucerne", "Verona", "Croydon", "Silverdale", "Lewister", "Langdale",
"Quinton", "Carolyn", "Wavertree", "Wentworth", "Leovista", "Trenton", "Evergreen", "Chelsea", "Crystal",
"Sylvan", "Alpine", "Bonita", "Palisade", "Blueridge", "Skyline", "Glencanyon", "Delmar", "Dolores",
"Delbrook", "Linnae", "Teviot", "Belvista", "Prospect", "Primrose", "Edgewood", "Patterdale", "Newdale",
"Crestwood", "Montroyal", "Glenview", "Arundel", "Ranger", "Capilano", "Salvador", "Grace", "East", "June",
"Cliffridge", "Glenn"]
# INSTANTIATE KW_POST_BOT AS bot
bot = KW_POST_BOT(webdriver.Firefox(), "https://www.dnv.org/building-development/look-building-and-trades-permits",
key_words, [])
bot.main()
CodePudding user response：
You could use a try/except block like this, that would first run the code and if pandas raise an error it would just print a message and go on the next search term, avoiding stopping the program
try:
//your pandas code here
except pd.errors.UndefinedVariableError:
print("No results")
https://docs.python.org/3/tutorial/errors.html?highlight=try except
CodePudding user response：
It looks like the issue you're experiencing is due to the fact that the search for some keywords is not returning any results, which causes an error when you try to access the Description column in the resulting dataframe.
One way to handle this situation would be to wrap the code that processes the search results in a try-except block, and catch the UndefinedVariableError that is being raised. Here's an example of how you can do this:
import pandas as pd
# Perform the search and get the results
results = search(keyword)
# Try to process the results
try:
# Convert the results to a pandas dataframe
df = pd.read_html(results)[0]
# Filter the dataframe to only include rows with 'Oval' in the 'Shape' column
df = df[df['Shape'] == 'Oval']
# Print the 'Description' column for the filtered rows
print(df['Description'])
# If the results do not contain the 'Description' column, skip this keyword
except pd.errors.UndefinedVariableError:
print(f"No results for keyword: {keyword}")
This will allow you to handle the case where the search does not return any results, and avoid the error being raised. You can then continue processing the search results for the next keyword in the loop.