I'm trying to do web scraping on reclameaqui site (

Wher am I wrong?

CodePudding user response：

You can try the next example

import scrapy
from ..items import ComplaintItem
import json


class ComplaintScraper(scrapy.Spider):
    name = "complaintScraper"  

    start_urls = [f"https://iosearch.reclameaqui.com.br/raichu-io-site-search-v1/query/companyComplains/10/{item}?company=98" for item in range(0,100,10)]

    def start_requests(self):
        for url in self.start_urls:
            yield scrapy.Request(url, self.parse, dont_filter=True)
  
    def parse(self, response):

        complaintItem = ComplaintItem()

        data = json.loads(response.text)
      
        for card in data['complainResult']['complains']['data']:
            complaintItem = ComplaintItem()
            complaintItem['id']               = card['id']
            complaintItem['title']            = card['title']   
            complaintItem['description']      = card['description']
            complaintItem['url']              = response.url
            yield complaintItem