Selenium python : my current_url doesnt update after click-CodePudding

Im scraping a website where I need to retrieve values from the url when i click on a button providing different form values.

I have a problem: when i click the button and retrieve the current_url, the provided values in the forms doesnt reflect in the url which should be updated (it's a search button).

My code to retrieve the url value is :

driver = webdriver.Firefox()
driver.get(url)
arrlist = []
idlist = []
service=value
for i in key_list:
    form = driver.find_elements(by=By.XPATH, value='//input[@id="geo_nav"]')
    form[0].send_keys(i)
    form2=driver.find_elements(by=By.XPATH, value='//input[@id="sev_nav"]')
    form2[0].send_keys(service)
    button=driver.find_elements(by=By.XPATH, value='//button[@data-role="filter-apply"]')
    button[0].click()
    time.sleep(5)
    url=driver.current_url
    print(dept)
    print(i)
    id=re.findall(r"(?<=\[population\]=)(\d{9})",url)[0]
    arrlist.append(i)
    idlist.append(id)

the button html code is :

<button  data-role="filter-apply">
            <p >Appliquer</p>
            <div >
                <span >2</span>
                <svg height="18" viewBox="0 0 16 18" width="16" xmlns="http://www.w3.org/2000/svg"><path d="m10.877 17.457 2.026 1.533v-4.553c0-.166.042-.329.12-.475l4.3-7.962h-10.68l4.122 7.978c.074.142.112.3.112.459zm3.026 4.543c-.213 0-.426-.068-.603-.203l-4.026-3.045c-.25-.189-.397-.484-.397-.797v-3.274l-4.765-9.222c-.161-.31-.148-.681.034-.979.181-.298.505-.48.854-.48h14c.352 0 .678.185.859.488.18.302.188.677.021.987l-4.977 9.215v6.31c0 .379-.214.726-.554.895-.141.07-.294.105-.446.105z" fill="#0579c7" fill-rule="evenodd" transform="translate(-4 -4)"></path></svg>            </div>
        </button>

I've tried to use

driver.switch_to.window(driver.window_handles[-1]);

following this post : Python Selenium Chromedriver - Can't Get current_url of new opened tab after click()

But I dont have tab or new windows issues.

I tried to click autocompletion lists in the 2 forms in inputand one of the form produces a modification of the url but not the other (the one of which effects on the url i need to monitor). The form code that works is :

<form data-component="sev_nav_input" data-no-results="Sans résultats" data-default-pho="Services" data-selected-name="Achat compulsif" data-selected-id="5928" >
<input type="text" placeholder="Services" autocomplete="off" name="sev_nav" id="sev_nav" data-role="js_filter" data-id="5928" > 
<span id="clear-sev-input"  style="display: none;">
<img src="data:image/svg xml;base64,PHN2ZyBoZWlnaHQ9IjI0IiB2aWV3Qm94PSIwIDAgMjQgMjQiIHdpZHRoPSIyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48ZyBmaWxsPSJub25lIiBmaWxsLXJ1bGU9ImV2ZW5vZGQiPjxwYXRoIGQ9Im0wIDBoMjR2MjRoLTI0eiIvPjxwYXRoIGQ9Im0xMS43NSAyMC41YzIuNDIyIDAgNC40ODYtLjg1MyA2LjE5MS0yLjU1OSAxLjcwNi0xLjcwNSAyLjU1OS0zLjc3IDIuNTU5LTYuMTkxIDAtMi40MjItLjg1My00LjQ4Ni0yLjU1OS02LjE5MS0xLjcwNS0xLjcwNi0zLjc3LTIuNTU5LTYuMTkxLTIuNTU5LTIuNDIyIDAtNC40OTIuODYtNi4yMSAyLjU3OC0xLjY5NSAxLjY5My0yLjU0IDMuNzUtMi41NCA2LjE3MnMuODUzIDQuNDg2IDIuNTU5IDYuMTkxYzEuNzA1IDEuNzA2IDMuNzcgMi41NTkgNi4xOTEgMi41NTl6bTMuMTY0LTQuNDE0Yy0uMDUyIDAtLjExNy0uMDQtLjE5NS0uMTE3bC0yLjk2OS0yLjkzLTIuOTMgMi45NjljLS4wNTIuMDUyLS4xMy4wNzgtLjIzNC4wNzhzLS4xODItLjAyNi0uMjM0LS4wNzhsLS44Mi0uODZjLS4wNTMtLjA1Mi0uMDc5LS4xMy0uMDc5LS4yMzRzLjAyNi0uMTcuMDc4LS4xOTVsMi45NjktMi45NjktMi45NjktMi45M2MtLjE1Ni0uMTU2LS4xNTYtLjMxMiAwLS40NjhsLjgyLS44MmMuMDc5LS4wNzkuMTU3LS4xMTguMjM1LS4xMTguMDUyIDAgLjExNy4wNC4xOTUuMTE3bDIuOTY5IDIuODkgMi45NjktMi44OWMuMDc4LS4wNzguMTQzLS4xMTcuMTk1LS4xMTcuMDc4IDAgLjE1Ni4wNC4yMzQuMTE3bC44Ni44MmMuMTU2LjE1Ny4xNTYuMzEzIDAgLjQ3bC0yLjk2OSAyLjkyOSAyLjkzIDIuOTNjLjA3OC4wNzguMTE3LjE1Ni4xMTcuMjM0IDAgLjEwNC0uMDQuMTgyLS4xMTcuMjM0bC0uODIuODJjLS4wNzkuMDc5LS4xNTcuMTE4LS4yMzUuMTE4eiIgZmlsbD0iIzE0OWM5NyIvPjwvZz48L3N2Zz4K"></span> 
<span ></span> 
<span  style="display: none;"></span> 
<div  style="display: none;"></div> 
<div id="services-list"  style="display: none;"><ul data-role="autocomplete-list" > </ul></div></form>

The form code that doesnt work is :

<form data-component="geo_nav_input" data-selected-name="" data-selected-id="" data-selected-neighborhood-id="0" data-selected-type="" data-no-results="Sans résultats" data-pho="Localité" data-default-pho="Localité" >
<div >Chercher des professionnels en/à...</div> 
<span ></span> 
<label for="geo_nav" >Localité</label> 
<input type="text" placeholder="Localité" autocomplete="off" name="geo_nav" id="geo_nav" data-role="js_filter" data-id="" data-neighborhoodid="0" data-type="" > <span id="clear-geo-input"  style="display: none;">
<img src="data:image/svg xml;base64,PHN2ZyBoZWlnaHQ9IjI0IiB2aWV3Qm94PSIwIDAgMjQgMjQiIHdpZHRoPSIyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48ZyBmaWxsPSJub25lIiBmaWxsLXJ1bGU9ImV2ZW5vZGQiPjxwYXRoIGQ9Im0wIDBoMjR2MjRoLTI0eiIvPjxwYXRoIGQ9Im0xMS43NSAyMC41YzIuNDIyIDAgNC40ODYtLjg1MyA2LjE5MS0yLjU1OSAxLjcwNi0xLjcwNSAyLjU1OS0zLjc3IDIuNTU5LTYuMTkxIDAtMi40MjItLjg1My00LjQ4Ni0yLjU1OS02LjE5MS0xLjcwNS0xLjcwNi0zLjc3LTIuNTU5LTYuMTkxLTIuNTU5LTIuNDIyIDAtNC40OTIuODYtNi4yMSAyLjU3OC0xLjY5NSAxLjY5My0yLjU0IDMuNzUtMi41NCA2LjE3MnMuODUzIDQuNDg2IDIuNTU5IDYuMTkxYzEuNzA1IDEuNzA2IDMuNzcgMi41NTkgNi4xOTEgMi41NTl6bTMuMTY0LTQuNDE0Yy0uMDUyIDAtLjExNy0uMDQtLjE5NS0uMTE3bC0yLjk2OS0yLjkzLTIuOTMgMi45NjljLS4wNTIuMDUyLS4xMy4wNzgtLjIzNC4wNzhzLS4xODItLjAyNi0uMjM0LS4wNzhsLS44Mi0uODZjLS4wNTMtLjA1Mi0uMDc5LS4xMy0uMDc5LS4yMzRzLjAyNi0uMTcuMDc4LS4xOTVsMi45NjktMi45NjktMi45NjktMi45M2MtLjE1Ni0uMTU2LS4xNTYtLjMxMiAwLS40NjhsLjgyLS44MmMuMDc5LS4wNzkuMTU3LS4xMTguMjM1LS4xMTguMDUyIDAgLjExNy4wNC4xOTUuMTE3bDIuOTY5IDIuODkgMi45NjktMi44OWMuMDc4LS4wNzguMTQzLS4xMTcuMTk1LS4xMTcuMDc4IDAgLjE1Ni4wNC4yMzQuMTE3bC44Ni44MmMuMTU2LjE1Ny4xNTYuMzEzIDAgLjQ3bC0yLjk2OSAyLjkyOSAyLjkzIDIuOTNjLjA3OC4wNzguMTE3LjE1Ni4xMTcuMjM0IDAgLjEwNC0uMDQuMTgyLS4xMTcuMjM0bC0uODIuODJjLS4wNzkuMDc5LS4xNTcuMTE4LS4yMzUuMTE4eiIgZmlsbD0iIzE0OWM5NyIvPjwvZz48L3N2Zz4K"></span> 
<span ></span> 
<span  style="display: none;"></span> 
<div  style="display: none;"></div> 
<div id="location-list"  style="display: none;"><ul data-role="autocomplete-list" > </ul></div>
</form>

CodePudding user response：

Can you make a function to navigate pages, and on each page do the actions you require. And with each call of the function use driver.switch_to.window to ensure you are on the latest page.

Although based on your edits, it now seems the issue is that you are having trouble locating and following one of the links on the pages.

def navigate(n):
    """ Move through the pages. Select the relevant buttons on each page"""
    window_after = driver.window_handles[0]
    driver.switch_to.window(window_after)
    if n == 0:
        form = driver.find_elements(by=By.XPATH, value='//input[@id="geo_nav"]')
        button = driver.find_elements(by=By.XPATH, value='//button[@data-role="filter-apply"]').click()
    elif n == 1:
        pass
    # Do something
    else:
        pass
    # Do something else

for i in range(3):
    navigate(i)
    time.sleep(3)

CodePudding user response：

The solution was in fact linked to the autocompletion forms. They require you to click on the autocompletion suggestions so the button is actually working.

FYI, here is the full code to autocomplete with clicking the form, deleting the content, adding the content, clicking the list and clicking the button.

def get_city_locations(service):
    url='https://www.psychologue.net/search?therapytype=0&service=&location[population]=&neighborhood=&pricefilter=0&reviewfilter=0'
    #options = Options()
    #options.headless = True
    driver = webdriver.Firefox()#options=options)
    driver.get(url)
    time.sleep(2)
    buttoncookie = driver.find_elements(by=By.XPATH, value='//button[@]')
    buttoncookie[0].click()
    time.sleep(1)
    form2 = driver.find_elements(by=By.XPATH, value='//input[@id="sev_nav"]')
    form2[0].click()
    time.sleep(1)
    Static.clear_text(driver)
    form2[0].send_keys(service)
    time.sleep(1)
    autocompleteservice = driver.find_elements(by=By.XPATH, value='//li[not(@class)]')
    for f in autocompleteservice:
        if f.text == service:
            f.click()
    df_pref=pd.read_csv('arrondissement_2022.csv',sep=',')
    deptlist = []
    arrlist = []
    idlist = []
    for i in df_pref['LIBELLE']:
        df_dep=df_pref[df_pref['LIBELLE']==i]
        dept = df_dep.loc[df_dep.index.values[0], 'DEP']
        form = driver.find_elements(by=By.XPATH, value='//input[@id="geo_nav"]')
        form[0].click()
        time.sleep(1)
        Static.clear_text(driver)
        form[0].send_keys(i)
        time.sleep(3)
        autocompletelocation=driver.find_elements(by=By.XPATH, value='//li[not(@class)]')
        cond=0
        for a in autocompletelocation:
            if a.text==i:
                print ('condition ok')
                cond=1
                a.click()
                break

        time.sleep(3)
        button=driver.find_elements(by=By.XPATH, value='//button[@data-role="filter-apply"]')
        button[0].click()
        time.sleep(3)
        driver.switch_to.window(driver.window_handles[-1]);
        url=driver.current_url
        print(dept)
        print(i)
        print(url)
        if cond==0:
            id=0
        else:
            id=re.findall(r"(?<=\[population\]=)(\d{7,10})",url)[0]
        print(f'id = {id}')
        print('\n')
        deptlist.append(dept)
        arrlist.append(i)
        idlist.append(id)
    df0 = pd.DataFrame({"dept": deptlist, "arrondissement":arrlist,"id":idlist})
    df0.to_csv('arr_id.csv',sep=';',index=False)