Scrapy tracking and scraping third page-CodePudding

after trying to add third page to this shenanigas i got an error "You can't mix str and non-str arguments". My goal is to use url from 'website' and scrap data from it. How do i do it? Here is my code:

# -*- coding: utf-8 -*-
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy import Request, Spider


class RynekMainSpider(scrapy.Spider):
    name = "RynekMain"
    start_urls = [
        'https://rynekpierwotny.pl/deweloperzy/?page=1']
    def parse(self, response):
        websites = response.css('div#root')[0]
        PAGETEST = response.xpath('//a[contains(@class,"rp-173nt6g")]/../following-sibling::li').css('a::attr(href)').get()
        for website in websites.css('li.rp-np9kb1'):
            page = website.css('a::attr(href)').get()
            address = website.css('address.rp-o9b83y::text').get()
            name = website.css('h2.rp-69f2r4::text').get()
            params = {
            'address' : address,
            'name' : name,
            'href' : page,
            }
            url  = response.urljoin(page)
            urlem = response.urljoin(website)
            yield Request(url=url, cb_kwargs={'params': params}, callback=self.parseMain)
            yield Request(url=urlem, cb_kwargs={'params': params}, callback=self.parseEmail)
        yield Request(url=response.urljoin(PAGETEST), callback=self.parse)

    def parseMain(self, response, params=None):
        # print(response.url)
        website = response.css('div.rp-l0pkv6 a::attr(href)').get()
        params['website'] = website
        yield params
    
    
    
    
    def parseEmail(self,response, params=None):
        hps = HtmlXPathSelector(response)
        email = hxs.xpath('//body').re('([a-zA-Z0-9_. -] @[a-zA-Z0-9-] \.[a-zA-Z0-9-.] )')      

if __name__ == "__main__":
    process =CrawlerProcess()
    process.crawl(RynekMainSpider)
    process.start()

Thanks for help in advance.

CodePudding user response：

A simple debugging pointed me to the error line:

urlem = response.urljoin(website) # You can't mix str and non-str arguments

website is a Selector, and urljoin needs a string.

Perhaps what you are looking for is this:

urlem = response.urljoin(website.xpath('.//a/@href').get())

CodePudding user response：

Ok i solved it. I just moved yield a bit. Yield can't just take non existent strings, string needs to be created first, that's why i got problems before. Website url was scraped in parseMain not in parse.

# -*- coding: utf-8 -*-
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy import Request, Spider


class RynekMainSpider(scrapy.Spider):
    name = "RynekMain"
    start_urls = [
        'https://rynekpierwotny.pl/deweloperzy/?page=1']
    def parse(self, response):
        websites = response.css('div#root')[0]
        PAGETEST = response.xpath('//a[contains(@class,"rp-173nt6g")]/../following-sibling::li').css('a::attr(href)').get()
        for website in websites.css('li.rp-np9kb1'):
            page = website.css('a::attr(href)').get()
            address = website.css('address.rp-o9b83y::text').get()
            name = website.css('h2.rp-69f2r4::text').get()
            params = {
            'address' : address,
            'name' : name,
            'href' : page,
            }
            url  = response.urljoin(page)
            
            yield Request(url=url, cb_kwargs={'params': params}, callback=self.parseMain)
            
        yield Request(url=response.urljoin(PAGETEST), callback=self.parse)

    def parseMain(self, response, params=None):
        # print(response.url)
        website = response.css('div.rp-l0pkv6 a::attr(href)').get()
        params['website'] = website
        urlem = response.urljoin(website)
        yield Request(url=urlem, cb_kwargs={'params': params}, callback=self.parseEmail)
        
    
    
    
    
    def parseEmail(self,response, params=None):
        email = response.css('div.m-Footer__company a::attr(href)').get()
        params['email'] = email        
        yield params
if __name__ == "__main__":
    process =CrawlerProcess()
    process.crawl(RynekMainSpider)
    process.start()