I want to scrape hotel facilities data on TripAdvisor using Selenium. To have all the facilities of an hotel I need to click on show more. However, I can't do that if i use the click function on the element show more. And if I scrap the information like that (without clicking show more) :
Property_Amenities= driver.find_elements_by_xpath(".//div[@class='bUmsU f ME H3 _c']")
for o in range(len(Property_Amenities)):
Property_Amenities[o]=Property_Amenities[o].text
print(Property_Amenities)
It only retrieve 8 facilities and not everything.
All the data is stored in a data-ssrev-handlers above. How can I retrieve information like facilities name easily ?
<div data-ssrev-handlers="{"load":[3,"R5ISL7","default",{"amenities":{"highlightedAmenities":{"roomFeatures":[{"tagId":18905,"amenityNameLocalized":"Blackout curtains","amenityCategoryName":"Comfort","amenityIcon":"hotels"},{"tagId":19085,"amenityNameLocalized":"Soundproof rooms","amenityCategoryName":"Comfort","amenityIcon":"hotels"},{"tagId":9658,"amenityNameLocalized":"Air conditioning","amenityCategoryName":"RoomAmenities","amenityIcon":"hotels"},{"tagId":12120,"amenityNameLocalized":"Housekeeping","amenityCategoryName":"RoomAmenities","amenityIcon":"hotels"},{"tagId":10589,"amenityNameLocalized":"Safe","amenityCategoryName":"RoomAmenities","amenityIcon":"hotels"},{"tagId":9666,"amenityNameLocalized":"Minibar","amenityCategoryName":"KitchenAmenities","amenityIcon":"restaurants"},{"tagId":11269,"amenityNameLocalized":"Flatscreen TV","amenityCategoryName":"Entertainment","amenityIcon":"play"},{"tagId":18897,"amenityNameLocalized":"Bath / shower","amenityCategoryName":"Bath","amenityIcon":"bathroom-fill"}],"roomTypes":[{"tagId":9184,"amenityNameLocalized":"Non-smoking rooms","amenityCategoryName":"RoomTypes","amenityIcon":"hotels"},{"tagId":9160,"amenityNameLocalized":"Suites","amenityCategoryName":"RoomTypes","amenityIcon":"hotels"},{"tagId":9659,"amenityNameLocalized":"Family rooms","amenityCategoryName":"RoomTypes","amenityIcon":"hotels"}],"propertyAmenities":[{"tagId":9176,"amenityNameLocalized":"Free High Speed Internet (WiFi)","amenityCategoryName":"Internet","amenityIcon":"wifi"},{"tagId":12111,"amenityNameLocalized":"Breakfast available","amenityCategoryName":"FoodAndDrink","amenityIcon":"food-drink"},{"tagId":19103,"amenityNameLocalized":"Taxi service","amenityCategoryName":"Transportation","amenityIcon":"taxi"},{"tagId":19063,"amenityNameLocalized":"24-hour security","amenityCategoryName":"General","amenityIcon":"hotels"},{"tagId":18896,"amenityNameLocalized":"Baggage storage","amenityCategoryName":"General","amenityIcon":"hotels"},{"tagId":18924,"amenityNameLocalized":"24-hour check-in","amenityCategoryName":"ReceptionServices","amenityIcon":"suitcase-fill"},{"tagId":18977,"amenityNameLocalized":"24-hour front desk","amenityCategoryName":"ReceptionServices","amenityIcon":"suitcase-fill"},{"tagId":9180,"amenityNameLocalized":"Dry cleaning","amenityCategoryName":"CleaningServices","amenityIcon":"dry-cleaning"}]},"nonHighlightedAmenities":{"roomFeatures":[{"tagId":19104,"amenityNameLocalized":"Telephone","amenityCategoryName":"RoomAmenities","amenityIcon":"hotels"},{"tagId":18911,"amenityNameLocalized":"Bottled water","amenityCategoryName":"RoomAmenities","amenityIcon":"hotels"},{"tagId":19117,"amenityNameLocalized":"Wake-up service / alarm clock","amenityCategoryName":"RoomAmenities","amenityIcon":"hotels"},{"tagId":18949,"amenityNameLocalized":"Electric kettle","amenityCategoryName":"KitchenAmenities","amenityIcon":"restaurants"},{"tagId":18972,"amenityNameLocalized":"Complimentary toiletries","amenityCategoryName":"Bath","amenityIcon":"bathroom-fill"},{"tagId":18984,"amenityNameLocalized":"Hair dryer","amenityCategoryName":"Bath","amenityIcon":"bathroom-fill"}],"roomTypes":[],"propertyAmenities":[{"tagId":9173,"amenityNameLocalized":"Wifi","amenityCategoryName":"Internet","amenityIcon":"wifi"},{"tagId":18971,"amenityNameLocalized":"Complimentary Instant Coffee","amenityCategoryName":"FoodAndDrink","amenityIcon":"food-drink"},{"tagId":18933,"amenityNameLocalized":"Complimentary tea","amenityCategoryName":"FoodAndDrink","amenityIcon":"food-drink"},{"tagId":19080,"amenityNameLocalized":"Snack bar","amenityCategoryName":"FoodAndDrink","amenityIcon":"food-drink"},{"tagId":19087,"amenityNameLocalized":"Special diet menus","amenityCategoryName":"FoodAndDrink","amenityIcon":"food-drink"},{"tagId":9656,"amenityNameLocalized":"Concierge","amenityCategoryName":"General","amenityIcon":"hotels"},{"tagId":9670,"amenityNameLocalized":"Non-smoking hotel","amenityCategoryName":"General","amenityIcon":"hotels"},{"tagId":18925,"amenityNameLocalized":"Express check-in / check-out","amenityCategoryName":"ReceptionServices","amenityIcon":"suitcase-fill"},{"tagId":9623,"amenityNameLocalized":"Laundry service","amenityCategoryName":"CleaningServices","amenityIcon":"dry-cleaning"},{"tagId":19003,"amenityNameLocalized":"Ironing service","amenityCategoryName":"CleaningServices","amenityIcon":"dry-cleaning"}]},"languagesSpoken":[{"tagId":18950,"amenityNameLocalized":"English"},{"tagId":18976,"amenityNameLocalized":"French"},{"tagId":19086,"amenityNameLocalized":"Spanish"},{"tagId":18890,"amenityNameLocalized":"Arabic"},{"tagId":18928,"amenityNameLocalized":"Chinese"}]},"loading":false,"viewport":"DESKTOP"}]}"><div >Property amenities</div><div ><div data-test-target="amenity_text"><span ></span>Free High Speed Internet (WiFi)</div><div data-test-target="amenity_text"><span ></span>Breakfast available</div><div data-test-target="amenity_text"><span ></span>Taxi service</div><div data-test-target="amenity_text"><span ></span>24-hour security</div><div data-test-target="amenity_text"><span ></span>Baggage storage</div><div data-test-target="amenity_text"><span ></span>24-hour check-in</div><div data-test-target="amenity_text"><span ></span>24-hour front desk</div><div data-test-target="amenity_text"><span ></span>Dry cleaning</div><div ><div data-test-target="amenity_text"><span ></span>Wifi</div><div data-test-target="amenity_text"><span ></span>Complimentary Instant Coffee</div><div data-test-target="amenity_text"><span ></span>Complimentary tea</div><div data-test-target="amenity_text"><span ></span>Snack bar</div><div data-test-target="amenity_text"><span ></span>Special diet menus</div><div data-test-target="amenity_text"><span ></span>Concierge</div><div data-test-target="amenity_text"><span ></span>Non-smoking hotel</div><div data-test-target="amenity_text"><span ></span>Express check-in / check-out</div><div data-test-target="amenity_text"><span ></span>Laundry service</div><div data-test-target="amenity_text"><span ></span>Ironing service</div></div></div><div><div><span >Show more</span><span ></span></div></div><div >Room features</div><div ><div data-test-target="amenity_text"><span ></span>Blackout curtains</div><div data-test-target="amenity_text"><span ></span>Soundproof rooms</div><div data-test-target="amenity_text"><span ></span>Air conditioning</div><div data-test-target="amenity_text"><span ></span>Housekeeping</div><div data-test-target="amenity_text"><span ></span>Safe</div><div data-test-target="amenity_text"><span ></span>Minibar</div><div data-test-target="amenity_text"><span ></span>Flatscreen TV</div><div data-test-target="amenity_text"><span ></span>Bath / shower</div><div ><div data-test-target="amenity_text"><span ></span>Telephone</div><div data-test-target="amenity_text"><span ></span>Bottled water</div><div data-test-target="amenity_text"><span ></span>Wake-up service / alarm clock</div><div data-test-target="amenity_text"><span ></span>Electric kettle</div><div data-test-target="amenity_text"><span ></span>Complimentary toiletries</div><div data-test-target="amenity_text"><span ></span>Hair dryer</div></div></div><div><div><span >Show more</span><span ></span></div></div><div >Room types</div><div ><div data-test-target="amenity_text"><span ></span>Non-smoking rooms</div><div data-test-target="amenity_text"><span ></span>Suites</div><div data-test-target="amenity_text"><span ></span>Family rooms</div><div ></div></div><div></div></div>
CodePudding user response：
to click on show more button, you can use the below XPath:
//div[text()='Show more']
I'd recommend you to use explicit wait to click on it:
Code:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//div[text()='Show more']"))).click()
Imports:
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
CodePudding user response：
To solve the problem :
First, scroll to the link using Action class
element=driver.find_element_by_xpath(".//div[contains(@class, 'dPTxH S4 b _S')]")
actions = ActionChains(driver)
actions.move_to_element(element).perform()
Then click on the "show more" using :
ActionChains(driver, 20).move_to_element(WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//div[text()='Show more']")))).click().perform()
After that you can retrieve all the informations you want !