How to find element from p tag-CodePudding

I am trying to get text from the p tag go to the page first and then select the language english and click on advanced search and then they will show you results these is the page link https://www.counselingcalifornia.com/Find-a-Therapist I am trying to extract result for p tag they will show me error

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.select import Select
from selenium import webdriver
driver = webdriver.Chrome('C:\Program Files (x86)\chromedriver.exe')
driver.maximize_window()

wait = WebDriverWait(driver, 30)

driver.get("https://www.counselingcalifornia.com/Find-a-Therapist")

wait.until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR, "iframe[id$='IFrame_htmIFrame']")))
select = Select(wait.until(EC.visibility_of_element_located((By.ID, "language_field"))))
select.select_by_value('ENG')

wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a#searchBtn"))).click()
dunk=driver.find_elements(By.XPATH, "//div[@class='row']")
for dun in dunk:
    phone=dun.find_element_by_xpath("//p[@id='phoneDiv_80863']/i[@class='fa fa-phone-square']").get_text()
    print(phone)

CodePudding user response：

After clicking the search button you should wait for the results to be presented.
Also you should fix the locators.
This should work:

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.select import Select
from selenium import webdriver
driver = webdriver.Chrome('C:\Program Files (x86)\chromedriver.exe')
driver.maximize_window()

wait = WebDriverWait(driver, 30)

driver.get("https://www.counselingcalifornia.com/Find-a-Therapist")

wait.until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR, "iframe[id$='IFrame_htmIFrame']")))
select = Select(wait.until(EC.visibility_of_element_located((By.ID, "language_field"))))
select.select_by_value('ENG')

wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a#searchBtn"))).click()
wait.until(EC.visibility_of_element_located((By.XPATH, "//div[@id='searchResults']//div[@class='panel-body']//div[contains(@class,'card')]")))
time.sleep(2)
dunk=driver.find_elements(By.XPATH, "//div[@id='searchResults']//div[@class='panel-body']//div[contains(@class,'card')]")
for dun in dunk:
    phone=dun.find_element_by_xpath(".//i[contains(@class,'phone-square')]").get_text()
    print(phone)

Pay attention for the dot . inside the XPath expression here:

dun.find_element_by_xpath(".//i[contains(@class,'phone-square')]")

It means from here and used for searching inside a specific node (element). Otherwise it would find the first match of //i[contains(@class,'phone-square')] locator from the page top.

CodePudding user response：

No need to selenium here. Just get the request from the url directly. And BeautifulSoup can get the <p> tags that have an id attribute that starts with "phoneDiv". You can also alter the query to return more than 25. So here's the list of 1-100

import requests
from bs4 import BeautifulSoup
import re

limit = 100

url = f'https://www.counselingcalifornia.com/cc/cgi-bin/utilities.dll/customlist?FIRSTNAME=~&LASTNAME=~&ZIP=&DONORCLASSSTT=&_MULTIPLE_INSURANCE=&HASPHOTOFLG=&_MULTIPLE_EMPHASIS=&ETHNIC=&_MULTIPLE_LANGUAGE=ENG&QNAME=THERAPISTLIST&WMT=NONE&WNR=NONE&WHP=therapistHeader.htm&WBP=therapistList.htm&RANGE=1/{limit}&SORT=LASTNAME'
headers = {'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Mobile Safari/537.36'}

response = requests.get(url, headers=headers)
soup = BeautifulSoup(response.text, 'html.parser')
rows = soup.find_all('div', {'class':'row'})

for row in rows:
    p = row.find('p', {'id':re.compile('^phoneDiv')})
    print(p.text)