I want to receive information about each lot posted on the site. And later, through pandas, pack it into a table. I ran into a problem in line 20. Began to redo the search from one subject to all. Throws this error.

Thanks)enter image description here

import requests                 
from bs4 import BeautifulSoup


url = "https://steamcommunity.com/market/"
headers = {'User-Agent':"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"}
r = requests.get(url, headers = headers)    

soup = BeautifulSoup(r.text, "lxml")



items = soup.findAll('a', class_= 'market_listing_row_link')

data = []


for item in items:
    quality = items.find("div", class_= "market_listing_row market_recent_listing_row market_listing_searchresult").find('div', class_= "market_listing_price_listings_block").find('span', class_="market_table_value").text
    #print("Кол-во лотов: "  quality)
    normprice = soup.find('div', class_= 'market_listing_row market_recent_listing_row market_listing_searchresult').find('div', class_= "market_listing_price_listings_block").find('span', class_="normal_price").text
    #print("Norm price "  normprice)
    games = soup.find('div', class_= 'market_listing_row market_recent_listing_row market_listing_searchresult').find('div', class_= "market_listing_item_name_block").find('span', class_="market_listing_game_name").text
    #print("Game :"  games)
    data.append([quality, normprice, games])

`

I tried to change findAll to find_all

CodePudding user response：

You have a typo inside your for loop:

...

for item in items:
    # Your code had `items.find`
    quality = item.find("div", class_= "market_listing_row market_recent_listing_row market_listing_searchresult").find('div', class_= "market_listing_price_listings_block").find('span', class_="market_table_value").text

...

CodePudding user response：

In your code you are doing:

for item in items:
    quality = items.find("div", class_= "market_listing_row market_recent_listing_row market_listing_searchresult").find('div', class_= "market_listing_price_listings_block").find('span', class_="market_table_value").text

you are calling items.find() but items is really your list:

items = soup.findAll('a', class_= 'market_listing_row_link')

So instead of:

for item in items:
    quality = items.find("div", class_= "market_listing_row market_recent_listing_row market_listing_searchresult").find('div', class_= "market_listing_price_listings_block").find('span', class_="market_table_value").text

use (notice there's no "s" in "item":

for item in items):
    quality = item.find("div", class_= "market_listing_row market_recent_listing_row market_listing_searchresult").find('div', class_= "market_listing_price_listings_block").find('span', class_="market_table_value").text

CodePudding user response：

Another solution, to load the lots using their Ajax API:

import pandas as pd
import requests
from bs4 import BeautifulSoup

api_url = 'https://steamcommunity.com/market/search/render/'

params = {
    "query": "",
    "start": "10",
    "count": "10",
    "search_descriptions": "0",
    "sort_column": "popular",
    "sort_dir": "desc",
}


all_data = []

for page in range(3):  # <-- increase number of pages here
    params['start'] = page * 10
    data = requests.get(api_url, params=params).json()
    soup = BeautifulSoup(data['results_html'], 'html.parser')

    for row in soup.select('.market_listing_row_link'):
        name = row.select_one('.market_listing_item_name').text
        game = row.select_one('.market_listing_game_name').text
        qty = row.select_one('.market_listing_num_listings_qty').text
        price = row.select_one('.normal_price[data-currency]').text
        url = row['href']
        all_data.append((name, game, qty, price, url))

df = pd.DataFrame(all_data, columns=['Name', 'Game', 'Qty', 'Price', 'URL'])
print(df.head().to_markdown(index=False))

Prints: