I am trying to execute this script but I don't know why it is throwing 'Null' and duplicate value at the same time! My goal is to put the necessary value and click the search button, get all the 'href' from the page and collect the data, which is working fine but providing the 'Null' and duplicate value at the same time!. I don't know what I am actually missing here.
import scrapy
from scrapy_selenium import SeleniumRequest
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import time
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
class RightMove2Spider(scrapy.Spider):
name = 'rightmove2'
start_urls = ["https://www.rightmove.co.uk/property-for-sale/search.html?searchLocation=London&useLocationIdentifier=true&locationIdentifier=REGION^87490&buy=For sale"]
def __init__(self, name=None, **kwargs):
chrome_options = Options()
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
driver.set_window_size(1920, 1080)
driver.get("https://www.rightmove.co.uk/property-for-sale/search.html?searchLocation=London&useLocationIdentifier=true&locationIdentifier=REGION^87490&buy=For sale")
price_range = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "(//option[@value='2000000'])[2]")))
price_range.click()
time.sleep(1)
bedroom_range = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "(//option[@value='5'])[1]")))
bedroom_range.click()
time.sleep(1)
tick_box = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//span[@class='tickbox--indicator']")))
tick_box.click()
time.sleep(1)
find_properties_btn = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//button[@id='submit']")))
find_properties_btn.click()
time.sleep(3)
self.property_xpath = driver.find_elements(By.XPATH, "//*[@class='l-searchResult is-list']/div/div/div[4]/div[1]/div[2]/a")
# driver.close()
super().__init__(name, **kwargs)
def parse(self, response):
for el in self.property_xpath:
href= el.get_attribute('href')
time.sleep(1)
yield SeleniumRequest(
url=href,
wait_time=3)
yield {
'Title': response.xpath("//h1[@itemprop='streetAddress']/text()").get(),
'Price': response.xpath("//div[@class='_1gfnqJ3Vtd1z40MlC0MzXu']/span/text()").get(),
'Agent Name': response.xpath("//div[@class='RPNfwwZBarvBLs58-mdN8']/a/text()").get(),
'Agent Address': response.xpath("//div[@class='OojFk4MTxFDKIfqreGNt0']/text()").get(),
'Agent Telephone': response.xpath("//a[@class='_3E1fAHUmQ27HFUFIBdrW0u']/text()").get(),
'Added on': response.xpath("//div[@class='_2nk2x6QhNB1UrxdI5KpvaF']/text()").get(),
'Links': response.url
}
for x in range(24, 1008, 24):
abs_url = f'https://www.rightmove.co.uk/property-for-sale/find.html?locationIdentifier=REGION^87490&minBedrooms=5&maxPrice=2000000&index={x}&propertyTypes=&includeSSTC=true&mustHave=&dontShow=&furnishTypes=&keywords='
yield SeleniumRequest(
url= abs_url,
callback=self.parse
)
output
{"Title": null, "Price": null, "Agent Name": null, "Agent Address": null, "Agent Telephone": null, "Added on": null, "Links": "https://www.rightmove.co.uk/property-for-sale/search.html?searchLocation=London&useLocationIdentifier=true&locationIdentifier=REGION^87490&buy=For sale"},
{"Title": "Combwell Crescent, Abbey Wood, London", "Price": "£450,000", "Agent Name": "Anthony Martin Estate Agents, Bexleyheath", "Agent Address": "2 Pickford Lane,\r\nBexleyheath,\r\nDA7 4QW", "Agent Telephone": "020 8012 7475", "Added on": "Added on 30/11/2021", "Links": "https://www.rightmove.co.uk/properties/117050312"},
{"Title": null, "Price": null, "Agent Name": null, "Agent Address": null, "Agent Telephone": null, "Added on": null, "Links": "https://www.rightmove.co.uk/property-for-sale/search.html?searchLocation=London&useLocationIdentifier=true&locationIdentifier=REGION^87490&buy=For sale"},
{"Title": null, "Price": null, "Agent Name": null, "Agent Address": null, "Agent Telephone": null, "Added on": null, "Links": "https://www.rightmove.co.uk/property-for-sale/search.html?searchLocation=London&useLocationIdentifier=true&locationIdentifier=REGION^87490&buy=For sale"},
{"Title": "Combwell Crescent, Abbey Wood, London", "Price": "£450,000", "Agent Name": "Anthony Martin Estate Agents, Bexleyheath", "Agent Address": "2 Pickford Lane,\r\nBexleyheath,\r\nDA7 4QW", "Agent Telephone": "020 8012 7475", "Added on": "Added on 30/11/2021", "Links": "https://www.rightmove.co.uk/properties/117050312"},
{"Title": null, "Price": null, "Agent Name": null, "Agent Address": null, "Agent Telephone": null, "Added on": null, "Links": "https://www.rightmove.co.uk/property-for-sale/search.html?searchLocation=London&useLocationIdentifier=true&locationIdentifier=REGION^87490&buy=For sale"},
{"Title": "Combwell Crescent, Abbey Wood, London", "Price": "£450,000", "Agent Name": "Anthony Martin Estate Agents, Bexleyheath", "Agent Address": "2 Pickford Lane,\r\nBexleyheath,\r\nDA7 4QW", "Agent Telephone": "020 8012 7475", "Added on": "Added on 30/11/2021", "Links": "https://www.rightmove.co.uk/properties/117050312"},
CodePudding user response：
Before starting webscraping project, success depends on to choose right tool in right way.Data is also generating from api calls json response. Why make web scraping so complex using selenium where you can easily grab data from api?
Script:
import scrapy
#import json
class PropertySpider(scrapy.Spider):
name = 'property'
def start_requests(self):
headers= {
"Content-Type": "application/x-www-form-urlencoded",
"X-Requested-With": "XMLHttpRequest",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.82 Safari/537.36"
}
yield scrapy.Request(
url='https://www.rightmove.co.uk/api/_search?locationIdentifier=STATION^9662&numberOfPropertiesPerPage=24&radius=0.5&sortType=2&index=24&includeSSTC=false&viewType=LIST&channel=BUY&areaSizeUnit=sqft¤cyCode=GBP&isFetching=false',
method="GET",
headers=headers,
callback=self.parse
)
def parse(self,response):
resp=response.json()
for item in resp['properties']:
yield {
"title":item['summary'],
'price':item['price']['amount'],
'url':'https://www.rightmove.co.uk' item['propertyUrl']
}
Output:
{'title': "A stunning two bedroom, two bathroom apartment on the 11th floor set over approx 1,645 sq ft, located in St George's brilliant new river fronted development, One Blackfriars, SE1.", 'price': 2000000, 'url': 'https://www.rightmove.co.uk/properties/118739888#/?channel=RES_BUY'}
2022-03-29 15:54:27 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.rightmove.co.uk/api/_search?locationIdentifier=STATION^9662&numberOfPropertiesPerPage=24&radius=0.5&sortType=2&index=24&includeSSTC=false&viewType=LIST&channel=BUY&areaSizeUnit=sqft¤cyCode=GBP&isFetching=false>
{'title': 'An immaculate four bedroom townhouse arranged over three floors nestled along a peaceful row of pretty houses.', 'price': 2000000, 'url': 'https://www.rightmove.co.uk/properties/118772936#/?channel=RES_BUY'}
2022-03-29 15:54:27 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.rightmove.co.uk/api/_search?locationIdentifier=STATION^9662&numberOfPropertiesPerPage=24&radius=0.5&sortType=2&index=24&includeSSTC=false&viewType=LIST&channel=BUY&areaSizeUnit=sqft¤cyCode=GBP&isFetching=false>
{'title': "With a great view of the River Thames, The Shard and the City of London, this bright and ideally located two bedroom apartment is 'as new' and is available for chain free sale through Prime London. The bright and clean living space, coming in at 1,210 sq ft / 112 sq m presents exceptionally well...", 'price': 2000000, 'url': 'https://www.rightmove.co.uk/properties/113289182#/?channel=RES_BUY'}
2022-03-29 15:54:27 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.rightmove.co.uk/api/_search?locationIdentifier=STATION^9662&numberOfPropertiesPerPage=24&radius=0.5&sortType=2&index=24&includeSSTC=false&viewType=LIST&channel=BUY&areaSizeUnit=sqft¤cyCode=GBP&isFetching=false>
{'title': 'With incredible views from the 19th and 20th floors, this 1,678 sq ft (155.9 sqm) penthouse apartment at The Perspective Building is available for sale exclusively through Prime London. The property features two large double bedrooms (both with en suite), occasional/guest bedroom, large open-pla...', 'price': 1999950, 'url': 'https://www.rightmove.co.uk/properties/73980120#/?channel=RES_BUY'}
2022-03-29 15:54:27 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.rightmove.co.uk/api/_search?locationIdentifier=STATION^9662&numberOfPropertiesPerPage=24&radius=0.5&sortType=2&index=24&includeSSTC=false&viewType=LIST&channel=BUY&areaSizeUnit=sqft¤cyCode=GBP&isFetching=false>
{'title': 'Set in one of London’s most desirable riverside locations, adjacent to Westminster and next to the London Eye, 8 Casson Square celebrates the rich history and heritage of its surroundings. The combination of the intricate architectural design and the impressive location will together create so...', 'price': 1965000, 'url': 'https://www.rightmove.co.uk/properties/79565985#/?channel=RES_BUY'}
2022-03-29 15:54:27 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.rightmove.co.uk/api/_search?locationIdentifier=STATION^9662&numberOfPropertiesPerPage=24&radius=0.5&sortType=2&index=24&includeSSTC=false&viewType=LIST&channel=BUY&areaSizeUnit=sqft¤cyCode=GBP&isFetching=false>
{'title': 'Newly refurbished two bedroom, two bathroom apartment in Whitehall Court, Westminster.', 'price': 1950000, 'url': 'https://www.rightmove.co.uk/properties/116568074#/?channel=RES_BUY'}
2022-03-29 15:54:27 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.rightmove.co.uk/api/_search?locationIdentifier=STATION^9662&numberOfPropertiesPerPage=24&radius=0.5&sortType=2&index=24&includeSSTC=false&viewType=LIST&channel=BUY&areaSizeUnit=sqft¤cyCode=GBP&isFetching=false>
... so on