this is the website i am trying to scrape: [https://www.jurongpoint.com.sg/store-directory/]

This is my code,as u can see i don't know how to fill both of the {} for the url variable as the 4 category that i want to scrape especially url for service is very different. The comment above url variable shows the link of the 4 category when clicked in. Appreciate any help,thank you!

from bs4 import BeautifulSoup
import requests

def parse():


    cate=["Service","Food & Beverage","Fashion & Accessories","Electronics & Technology"]

    #cate=Food & Beverage
    #cate=Electronics & Technology
    #cate=Fashion & Accessories
    #cate=Services


    url="https://www.jurongpoint.com.sg/store-directory/?level=&cate={} & {}"

    for cat in cate:
    
    
        for page in range(1,14):
            print(page)

            soup = BeautifulSoup(requests.get(url).text ,"html.parser")


            for link in soup.find_all('div',class_='entry-content'):

                try:
                    shops=soup.find_all('div',class_="col-9")
                    names=soup.find_all('tr',class_="clickable")

                    for n, k in zip(names, shops):
                        name = n.find_all('td')[1].text.replace(' ','')
                        desc = k.text.replace(' ','')
                        print(name   "\n")
                        print(desc)

                except AttributeError as e:
                    print(e)


                next_button = soup.select_one('.PagedList-skipToNext a')
                if next_button:
                    url = next_button.get('href')
                else:
                    break


parse() 

CodePudding user response：

Use parameters of your request and avoid to manage escape characters (like &)

url = "https://www.jurongpoint.com.sg/store-directory"

for cat in cate:
    for page in range(1, 14):
        print(f'Scraping category {cat} page {page}')
        payload = {
            'level': '',
            'cate': cat,
            'page': page
        }
        resp = requests.get(url, params=payload)
        soup = BeautifulSoup(resp.text, 'html.parser')
        # your code here

>>> resp.url
'https://www.jurongpoint.com.sg/store-directory/?level=&cate=Electronics & Technology&page=8'