Home > Net >  How to scrape information from "preview" of "Fetch/XHR" using Python and Request
How to scrape information from "preview" of "Fetch/XHR" using Python and Request

Time:10-11

I am trying to extract the information of a specific NFT of this enter image description here

CodePudding user response:

To download the Json data you can use next example:

import requests


payload = {
    "fields": {
        "address": 1,
        "animationUrl": 1,
        "collectionName": 1,
        "collectionSymbol": 1,
        "creator": 1,
        "currentBasePrice": 1,
        "decimals": 1,
        "description": 1,
        "ethReserves": 1,
        "externalLink": 1,
        "id": 1,
        "imageUrl": 1,
        "lastSale": 1,
        "market": 1,
        "marketplace": 1,
        "marketUrl": 1,
        "name": 1,
        "openRarityRank": 1,
        "owner": 1,
        "paymentToken": 1,
        "pendingTrxs": 1,
        "perItemEthPrice": 1,
        "priceInfo": 1,
        "rarityScore": 1,
        "sellOrders": 1,
        "smallImageUrl": 1,
        "standard": 1,
        "sudoPoolAddress": 1,
        "tokenId": 1,
        "tokenReserves": 1,
        "traits": 1,
    },
    "filters": {
        "address": "0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d",
        "tokenIds": ["7235"],
    },
    "limit": 1,
    "offset": 0,
    "refreshMetadata": False,
    "status": ["all"],
}

api_url = "https://gem-api-v2-1.herokuapp.com/assets"
headers = {
    "x-api-key": "rLnNH1tdrT09EQjGsjrSS7V3uGonfZLW",
    "Origin": "https://www.gem.xyz",
}

data = requests.post(api_url, headers=headers, json=payload).json()
print(data)

Prints:

{
    "hasNext": False,
    "total": 1,
    "data": [
        {
            "_id": "6195e7139ba05365c3225d42",
            "id": "7235",
            "name": None,
            "address": "0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d",
            "description": None,
            "collectionName": "Bored Ape Yacht Club",
            "collectionSymbol": "BAYC",
            "externalLink": None,
            "imageUrl": "https://lh3.googleusercontent.com/I-9cMQAuJkuXfVDGUGfqTT3SKR18BpeqnGlURszvdbEIdK_5qTrqBq-5YzTQCbq1gYpSRz3YGPeOsAYxjKdMAEZS-r6GxMWtoWRT",
            "smallImageUrl": "https://lh3.googleusercontent.com/I-9cMQAuJkuXfVDGUGfqTT3SKR18BpeqnGlURszvdbEIdK_5qTrqBq-5YzTQCbq1gYpSRz3YGPeOsAYxjKdMAEZS-r6GxMWtoWRT=s250",
            "animationUrl": None,
            "standard": "ERC721",
            "decimals": 0,
            "traits": [
                {
                    "trait_type": "Eyes",
                    "value": "eyepatch",
                    "display_type": None,
                    "max_value": None,
                    "trait_count": 333,
                    "order": None,
                },
                {
                    "trait_type": "Mouth",
                    "value": "bored unshaven",
                    "display_type": None,
                    "max_value": None,
                    "trait_count": 1551,
                    "order": None,
                },
                {
                    "trait_type": "Fur",
                    "value": "tan",
                    "display_type": None,
                    "max_value": None,
                    "trait_count": 626,
                    "order": None,
                },
                {
                    "trait_type": "Hat",
                    "value": "ww2 pilot helm",
                    "display_type": None,
                    "max_value": None,
                    "trait_count": 110,
                    "order": None,
                },
                {
                    "trait_type": "Background",
                    "value": "gray",
                    "display_type": None,
                    "max_value": None,
                    "trait_count": 1170,
                    "order": None,
                },
            ],
            "creator": "0xaba7161a7fb69c88e16ed9f455ce62b791ee4d03",
            "owner": "0xe40abeb1343f4e275f17b1656aad13819f90a90a",
            "currentBasePrice": 82000000000000000000,
            "duration": 253364,
            "endingPrice": None,
            "ethReserves": None,
            "lastSale": None,
            "market": "looksrare",
            "marketUrl": "https://looksrare.org/collections/0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d/7235",
            "openseaOrderCreatedAt": "1665354494",
            "paymentToken": {
                "symbol": "ETH",
                "decimals": 18,
                "address": "0x0000000000000000000000000000000000000000",
            },
            "startingPrice": None,
            "tokenReserves": None,
            "rarityScore": 5040,
            "orderCreatedAt": 1665355096,
            "pendingTrxs": [],
            "sudoPoolAddress": None,
            "supportsWyvern": True,
            "marketplace": "looksrare",
            "tokenId": "7235",
            "priceInfo": {
                "price": "82000000000000000000",
                "asset": "0x0000000000000000000000000000000000000000",
                "quantity": 1,
                "pricePerItem": "82000000000000000000",
                "decimals": 18,
                "startingPrice": None,
                "endingPrice": None,
                "fees": None,
            },
            "url": "https://looksrare.org/collections/0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d/7235",
            "tokenType": "ERC721",
        }
    ],
}

CodePudding user response:

Is this what you're after, by any chance?

import requests
import pandas as pd
headers = {
    'accept-language': 'en-US,en;q=0.9',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36'
}

url= 'https://api.opensea.io/api/v1/asset/0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d/7235?include_orders=true'
df = pd.json_normalize(requests.get(url, headers=headers).json(), record_path=['traits'], meta = ['token_id', ['asset_contract', 'address']])
print(df)

Result in terminal:

    trait_type  value   display_type    max_value   trait_count order   token_id    asset_contract.address
0   Eyes    Eyepatch    None    None    333 None    7235    0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d
1   Mouth   Bored Unshaven  None    None    1551    None    7235    0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d
2   Fur Tan None    None    626 None    7235    0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d
3   Hat Ww2 Pilot Helm  None    None    110 None    7235    0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d
4   Background  Gray    None    None    1170    None    7235    0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d
  • Related