I'm trying to extract name, brand, prices, stock microdata from pages extracted from sitemap.xml But I'm blocked with the following step, thank you for helping me as I'm a newbie I can't understand the blocking element

• Scrape the sitemap.xml to have list of urls : OK
• Extract the metadata : OK
• Extract the product schema : OK
• Extract the products not OK
• Crawl the site and store the products not OK

• Scrape the sitemap.xml to have list of urls : OK

import pandas as pd
import requests
import extruct
from w3lib.html import get_base_url
import urllib.request
from urllib.parse import urlparse
from bs4 import BeautifulSoup
import advertools as adv
proximus_sitemap = adv.sitemap_to_df('https://www.proximus.be/iportal/sitemap.xml')
proximus_sitemap = proximus_sitemap[proximus_sitemap['loc'].str.contains('boutique')]
proximus_sitemap = proximus_sitemap[proximus_sitemap['loc'].str.contains('/fr/')]

• Extract the metadata : OK

def extract_metadata(url):
 
    r = requests.get(url)
    base_url = get_base_url(r.text, r.url)
    metadata = extruct.extract(r.text, 
                               base_url=base_url,
                               uniform=True,
                               syntaxes=['json-ld',
                                         'microdata',
                                         'opengraph'])
    return metadata
metadata = extract_metadata('https://www.proximus.be/fr/id_cr_apple-iphone-13-128gb-blue/particuliers/equipement/boutique/apple-iphone-13-128gb-blue.html')
metadata

• Extract the product schema : OK

def get_dictionary_by_key_value(dictionary, target_key, target_value):
    for key in dictionary:
        if len(dictionary[key]) > 0:
            for item in dictionary[key]:
                if item[target_key] == target_value:
                    return item
Product = get_dictionary_by_key_value(metadata, "@type", "Product")
Product

• Extract the products not OK => errormessage = errorkey 'offers'

def get_products(metadata):
 
    
    Product = get_dictionary_by_key_value(metadata, "@type", "Product")

    if Product:
    
        products = []

        for offer in Product['offers']['offers']:
            product = {
                'product_name': Product.get('name', ''),
                'brand': offer.get('description', ''),
                'availability': offer.get('availability', ''),
                'lowprice': offer.get('lowPrice', ''),
                'highprice': offer.get('highPrice', ''),
                'price': offer.get('price', ''),
                'priceCurrency': offer.get('priceCurrency', ''),
            }

            products.append(product)

        return products

• Crawl the site and store the products not OK as blocked during previous step

def scrape_products(proximus_sitemap, url='url'):
 

    df_products = pd.DataFrame(columns = ['product_name', 'brand', 'name', 'availability', 
                                          'lowprice', 'highprice','price','priceCurrency'])

    for index, row in proximus_sitemap.iterrows(): 
        
        metadata = extract_metadata(row[url])          
        products = get_products(metadata)
        
        if products is not None:
            for product in products:             
                df_products = df_products.append(product, ignore_index=True)

    return df_products
df_products = scrape_products(proximus_sitemap, url='loc')
df_products.to_csv('patch.csv', index=False)
df_products.head()

CodePudding user response：

You can simply continue by using the advertools SEO crawler. It has a crawl function that also extracts structured data by default (JSON-LD, OpenGraph, and Twitter).

I tried to crawl a sample of ten pages, and this what the output looks like:

adv.crawl(proximus_sitemap['loc'], 'proximums.jl') 
proximus_crawl = pd.read_json('proximums.jl', lines=True)
proximus_crawl.filter(regex='jsonld').columns

Index(['jsonld_@context', 'jsonld_@type', 'jsonld_name', 'jsonld_url',
       'jsonld_potentialAction.@type', 'jsonld_potentialAction.target',
       'jsonld_potentialAction.query-input', 'jsonld_1_@context',
       'jsonld_1_@type', 'jsonld_1_name', 'jsonld_1_url', 'jsonld_1_logo',
       'jsonld_1_sameAs', 'jsonld_2_@context', 'jsonld_2_@type',
       'jsonld_2_itemListElement', 'jsonld_2_name', 'jsonld_2_image',
       'jsonld_2_description', 'jsonld_2_sku', 'jsonld_2_review',
       'jsonld_2_brand.@type', 'jsonld_2_brand.name',
       'jsonld_2_aggregateRating.@type',
       'jsonld_2_aggregateRating.ratingValue',
       'jsonld_2_aggregateRating.reviewCount', 'jsonld_2_offers.@type',
       'jsonld_2_offers.priceCurrency', 'jsonld_2_offers.availability',
       'jsonld_2_offers.price', 'jsonld_3_@context', 'jsonld_3_@type',
       'jsonld_3_itemListElement', 'jsonld_image', 'jsonld_description',
       'jsonld_sku', 'jsonld_review', 'jsonld_brand.@type',
       'jsonld_brand.name', 'jsonld_aggregateRating.@type',
       'jsonld_aggregateRating.ratingValue',
       'jsonld_aggregateRating.reviewCount', 'jsonld_offers.@type',
       'jsonld_offers.lowPrice', 'jsonld_offers.highPrice',
       'jsonld_offers.priceCurrency', 'jsonld_offers.availability',
       'jsonld_offers.price', 'jsonld_offers.offerCount',
       'jsonld_1_itemListElement', 'jsonld_2_offers.lowPrice',
       'jsonld_2_offers.highPrice', 'jsonld_2_offers.offerCount',
       'jsonld_itemListElement'],
      dtype='object')

These are some of the columns you might be interested in (containing price, currency, availability, etc.)