Home > Software design >  Code throws random target frame detached in selenium/python
Code throws random target frame detached in selenium/python

Time:07-11

As it says in the title, when trying to scrape the site https://www.fragrantica.com/search/ while looping through filters, it throws a WebDriverException: Message: target frame detached sometimes and sometimes not.. and when it does throw it, it happens at random lines and not a specific place so I have no clue how to tackle it.. some people suggested that I use this line:

WebDriverWait(driver, 20).until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR,"iframeCssSelector")))

But where to place it if it just happens randomly throughout the code. Here is my code, it is to extract the perfumes the filtered page has:

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import ElementClickInterceptedException
import time


options = Options()
options.add_argument("--profile-directory=Default")
options.add_argument('--disable-blink-features=AutomationControlled')
options.add_argument("start-maximized")
options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36")



driver = webdriver.Chrome(service = Service(executable_path='C:/Users/armon/Downloads/chromedriver_win32/chromedriver.exe'), options=options)
url = 'https://www.fragrantica.com/'
driver.get(url)

time.sleep(3)

perfumes_btn = driver.find_element(by=By.XPATH, value = '//*[@id="offCanvasLeft"]/ul/li[5]/a')
perfumes_btn.click()

search_btn = WebDriverWait(driver,20).until(EC.element_to_be_clickable((By.XPATH,'/html/body/div[2]/div[2]/div[2]/ul/li[5]/ul/li[1]/a')))
search_btn.click()

time.sleep(3)

Industry_more = '//*[@id="offCanvasLeftOverlap1"]/div/div/div[13]/div[2]/div/p/div/button'
Industry_dict = {'Industry_Fragrances' : '//*[@ and @value="Fragrances"]',
                'Industry_Cosmetics' : '//*[@ and @value="Cosmetics"]', 
                'Industry_Fashion' : '//*[@ and @value="Fashion"]', 
                'Industry_NaturalPerfumery': '//*[@ and @value="Natural Perfumery"]', 
                'Industry_Celebrity' : '//*[@ and @value="Celebrity"]', 
                'Industry_Accessories' : '//*[@ and @value="Accessories"]',
                'Industry_Jewelry': '//*[@ and @value="Jewelry"]', 
                'Industry_Retailer': '//*[@ and @value="Retailer"]',
                'Industry_Lingerie': '//*[@ and @value="Lingerie"]', 
                'Industry_NichePerfumes': '//*[@ and @value="Niche Perfumes"]'
            }


Gender_dict = {'Gender_Female' : '//*[@ and @value="female"]',
                'Gender_Unisex' : '//*[@ and @value="unisex"]',
                'Gender_Male' : '//*[@ and @value="male"]'
            }


Country_more = '//*[@id="offCanvasLeftOverlap1"]/div/div/div[10]/div[2]/div/p/div/button'
Country_dict = {'Country_USA' : '//*[@ and @value="United States"]',
                'Country_France' : '//*[@ and @value="France"]',
                'Country_Italy' : '//*[@ and @value="Italy"]',
                'Country_UK' : '//*[@ and @value="United Kigdom"]',
                'Country_Brazil' : '//*[@ and @value="Brazil"]',
                'Country_UAE' : '//*[@ and @value="United Arab Emirates"]',
                'Country_Russia': '//*[@ and @value="Russia"]',
                'Country_Spain' : '//*[@ and @value="Spain"]',
                'Country_Germany' : '//*[@ and @value="Germany"]',
                'Country_SaudiArabia' : '//*[@ and @value="Saudi Arabia"]',
                'Country_Sweden' : '//*[@ and @value="Sweden"]',
                'Country_Latvia' : '//*[@ and @value="Latvia"]',
                'Country_Poland' : '//*[@ and @value="Poland"]',
                'Country_Japan' : '//*[@ and @value="Japan"]',
                'Country_Netherlands' : '//*[@ and @value="Netherlands"]',
                'Country_Canada' : '//*[@ and @value="Canada"]',
                'Country_Australia' : '//*[@ and @value="Australia"]',
                'Country_Switzerland' : '//*[@ and @value="Switzerland"]',
                'Country_Argentina' : '//*[@ and @value="Argentina"]',
                'Country_Belgium' : '//*[@ and @value="Belgium"]',
                'Country_Pakistan' : '//*[@ and @value="Pakistan"]',
                'Country_Slovakia' : '//*[@ and @value="Slovakia"]',
                'Country_Austria' : '//*[@ and @value="Austria"]',
                'Country_Thailand' : '//*[@ and @value="Thailand"]',
                'Country_Philippines' : '//*[@ and @value="Philippines"]',
                'Country_SouthKorea' : '//*[@ and @value="South Korea"]',
                'Country_Belarus' : '//*[@ and @value="Belarus"]',
                'Country_Israel' : '//*[@ and @value="Israel"]',
                'Country_Korea' : '//*[@ and @value="Korea"]',
                'Country_Greece' : '//*[@ and @value="Greece"]',
                'Country_Denmark' : '//*[@ and @value="Denmark"]',
                'Country_Norway' : '//*[@ and @value="Norway"]',
                'Country_Portugal' : '//*[@ and @value="Portugal"]',
                'Country_Iceland' : '//*[@ and @value="Iceland"]',
                'Country_Newzealand' : '//*[@ and @value="New Zealand"]',
                'Country_Ukraine' : '//*[@ and @value="Ukraine"]',
                'Country_Romania' : '//*[@ and @value="Romania"]'
                
            }


start_year_xpath = '//*[@id="offCanvasLeftOverlap1"]/div/div/div[5]/div[2]/div/p/div/form/input[1]'
end_year_xpath = '//*[@id="offCanvasLeftOverlap1"]/div/div/div[5]/div[2]/div/p/div/form/input[2]'

load_more_button = '//*[@id="main-content"]/div[1]/div[1]/div/div/div/div[2]/div[1]/div/div[3]/div/div/div/div/div/button'
clear_filters_button = '//*[@id="offCanvasLeftOverlap1"]/div/div/div[1]/div[2]/p/div/button'

industry_load = WebDriverWait(driver,20).until(EC.element_to_be_clickable((By.XPATH, Industry_more)))
driver.execute_script("arguments[0].click();", industry_load)

country_load = WebDriverWait(driver,20).until(EC.element_to_be_clickable((By.XPATH, Country_more)))
driver.execute_script("arguments[0].click();", country_load)




def start_scraping():

    elements = driver.find_elements(by = By.XPATH, value = '//*[@]')
    if len(elements) == 0:
        return []
    elif len(elements) < 30:
        return elements
    else:
        prev_count = len(elements)
        loading = WebDriverWait(driver, 100).until(EC.element_to_be_clickable((By.XPATH, load_more_button)))
        driver.execute_script("arguments[0].click();", loading)
        time.sleep(2)
        elements = driver.find_elements(by = By.XPATH, value = '//*[@]')
        now_count = len(elements)
        while now_count > prev_count:
            loading = WebDriverWait(driver, 100).until(EC.element_to_be_clickable((By.XPATH, load_more_button)))
            driver.execute_script("arguments[0].click();", loading)
            time.sleep(2)
            elements = driver.find_elements(by = By.XPATH, value = '//*[@]')
            prev_count = now_count
            now_count = len(elements)
        return elements
            



all_links = []

# def scrape():
#     links = []
    
#     try:
#         loadingButton = WebDriverWait(driver,100).until(EC.element_to_be_clickable((By.XPATH,load_more_button)))

#         maxPerfumes = 1000;

#         cond = True

#         while loadingButton:
#             time.sleep(3)
#             driver.execute_script("arguments[0].click();", loadingButton)
#             loadingButton = WebDriverWait(driver,100).until(EC.element_to_be_clickable((By.XPATH,load_more_button)))
#             loadElems = driver.find_elements(by = By.XPATH, value = '//*[@]')
#             if len(loadElems)>0:
#                 loadingButton = WebDriverWait(driver,100).until(EC.element_to_be_clickable((By.XPATH,load_more_button)))
#             else:
#                 cond = False
#                 break
#             if len(loadElems) >= maxPerfumes:
#                 break
        
#         if cond :

#             card_sections = [] # list of card sections (image section and fragrance home page section)
#             sections = driver.find_elements(by=By.CLASS_NAME, value="card-section")
#             for section in sections:
#                 card_sections.append(section)
#             home_pages = card_sections[1::2] # Extracting fragrance home page link section from all sections (image section and fragrance home page section)

#             links = [] # list of final links when extracted.
#             inc = 0 # Increment value to get each card_seection
#             # Looping through to get all links
    #         for link in home_pages:
    #             link = home_pages[inc] # Getting the each div section
    #             link = link.find_elements(by=By.TAG_NAME, value = "p")
    #             link = link[0] # Selecting first paragraph (the one housing the link) element out of two
    #             link = link.find_elements(by=By.TAG_NAME, value = "a")
    #             link = link[0].get_attribute("href") # Extracting link from list
    #             links.append(link)
    #             inc  =1

    #     WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH,clear_filters_button))).click()

    #     all_links = all_links links
    # except:
    #     pass




all_cards = []

def filter():
    for industry in Industry_dict.values():
        for gender in Gender_dict.values():
            for country in Country_dict.values():
                for year in range(1920, 2023):
                    
                    industry_selected = WebDriverWait(driver, 100).until(EC.element_to_be_clickable((By.XPATH, industry)))
                    driver.execute_script("arguments[0].click();", industry_selected)
                    time.sleep(2)

                    gender_selected = WebDriverWait(driver, 100).until(EC.element_to_be_clickable((By.XPATH, gender)))
                    driver.execute_script("arguments[0].click();", gender_selected)
                    time.sleep(2)
                
                    try:
                        country_selected = WebDriverWait(driver, 100).until(EC.element_to_be_clickable((By.XPATH, country)))
                        driver.execute_script("arguments[0].click();", country_selected)
                        time.sleep(2)
                    except NoSuchElementException:
                        break
                         
                    start = WebDriverWait(driver, 100).until(EC.element_to_be_clickable((By.XPATH, start_year_xpath)))
                    start.clear()
                    driver.execute_script("arguments[0].click();", start)
                    start.send_keys(year)

                    time.sleep(1)

                    end = WebDriverWait(driver, 100).until(EC.element_to_be_clickable((By.XPATH, end_year_xpath)))
                    end.clear()
                    driver.execute_script("arguments[0].click();", end)
                    end.send_keys(year)

                    time.sleep(2) 

                    # start of new code

                    all_cards.append(start_scraping())
                    print(len(all_cards)) # to keep track.
                        
                    
                    # end of new code

                    clear  = WebDriverWait(driver, 50).until(EC.element_to_be_clickable((By.XPATH, clear_filters_button)))
                    driver.execute_script("arguments[0].click();", clear)
                    


filter()
print(len(all_cards))

any recommendations to how to approach this ? TIA

CodePudding user response:

There was an issue that was introduced in Chrome 99 bugs.chromium Issue 4048: target frame detached. See comment #53

Status: Fixed (was: Assigned)

Frame detached issue is fixed in ChromeDriver 103

You are using version 102, update your browser and driver versions.

  • Related