Python Extracting Entire data from Webpage for mentioned dates-CodePudding

I am trying to extract data for various dates ( without selenium ) I have written following program which is not fetching data I also don't know how to enter various dates in date field to fetch data I want guidance ..

import json
import pandas as pd
import requests
import datetime
#from datetime import datetime
import os 
import os.path
import schedule      
import time


dt = str(datetime.date.today())
today = datetime.datetime.now()
date_time = today.strftime("%d-%m-%Y_%H.%M")
print("date and time:",date_time)
file_name = 'Data_'   date_time
save_path = r"C:\Users\Desktop\Python_OP\Data"             
path = os.path.join(save_path, file_name ".csv") 

endpoint = "https://bricsonline.nseindia.com/bondsnew/rest/public?r=sebiannexure1"

headers = {
    "pageToken": "f06c7498-ac12-4def-95d2-f0fb903fff64",
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36",
    "X-Requested-With": "XMLHttpRequest",
}
payload = {
    "columnNames": [
        "Deal Type", "ISIN", "Listed / Unlisted Security", "Issuer Name", "Coupon",
        "Issue Description", "Price", "Yield", "Yield Type",
        "Outside Yield Range", "Put/Call Date", "Trade Value in Rs. Lacs", "Trade Date & Time", "Settlement Date",
        "Reported trade/Trade executed on RFQ platform", "Remarks", "Settlement Status",
    ]
}

response = requests.post(endpoint, data=json.dumps(payload), headers=headers)
df = pd.DataFrame(response.json(), columns=payload["columnNames"])
print(df)



#df.to_csv(path, index=False)

#df.to_csv("your_table.csv", index=False)

CodePudding user response：

You can do it as below, change date in json_data variable for different data

import requests

cookies = {
    '_au': '5D74EE4A55663F6A368505CD4FBA3632',
    'ak_bmsc': '272E767CEFD2BFE86A8B2D591830EB33~000000000000000000000000000000~YAAQpPBuaJhf/sWBAQAAD0iezxChkxULrJvqizHvrgRAPlYdsyj6T2rQYEb9rAe5izZEiLU1AkBYnff0uojtu4ttL/RfosZXFaTmWu21t498lYTHnvB/9Xl4z8bOZfOnJI0c0 EJ1L5RcCCcPb6NB05vKNZlD9nJM5glghtV/H80SIulXrLP2LnTxT94w/gRVdGx/FfiCwIRLVvPs5qCN5hLO0670abPUpFhAsy66bV GWngpAS0sxOGr7w DQFnvpmagENL7Y4o1aB 4ddK4gWJbKYQj5hdy4LLW 377hfEjNp ep7ayBCvefjdGfYFcwstc/dGwtzwD ThZkAsWXwY2yTb/FSbaNBdiQTRYhGC0egZXr1mmdbH7h0BhEknriEBKcK6Obyd5VVaWNN7kGN1gEV2oaw9jybDCVjZFPlp39w=',
    'bm_sv': 'F3E240F1831B5411B570DE55492509FB~YAAQpPBuaJNi/sWBAQAA6LqezxAmS74pOfmgYYRG49hcJv6SNK7iWVoRx84cz93tXpzeZTXvfHj6z N90DQHyJaPMV1l5EEs29n0CiXYUgl1uB83bMRdGjh6tPjFnS8BufizM1QR0mAWdtdhDeHUL3kRtGJNGkMhNR8neClTdEH8PsdaJmzINU72Oy 8zvBRaeW mk5BnpX9WcWUAp9R3hsDkLqlqWQJ4dnvy79ZCLCuk758fn0oGMWfZCJrLiYXZIAW~1',
    'bm_mi': '1E2B4B400A3D8F400F1DFD09AAFFF3C9~YAAQpPBuaNxf/sWBAQAA0G6ezxBWH9cCRXbc0lvozBmymZF2rIxZ6ZYY86kjHrnixSXW8H1qdX9tKhqIre/QbtsOpcnHRy8XIS ngtAG5ZfAcaMIP34dgHi1IVV6Muym04KAjLO6c47cumBUPcWXnL/7jKT1TZQnY4 fXWBQICcxOXAf0MmBgkOmNMh1rJzwIXRP99rAmqr/eSUORLctK9WtlR1lZhV9CcxXHsZWqVXFfWI2oqvkWeYtogsjHCdlpM1zMG673zaoEal23cEB6lEDhy/T4EbyZKhWVt  3BdCRDJszPdlVgBoyT7uO0cc3IhDC9XkrnY6/2mRZ5LopScRWSM7rdNECjP4dUst36Vm~1',
}

headers = {
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:98.0) Gecko/20100101 Firefox/98.0',
    'Accept': 'application/json, text/javascript, */*; q=0.01',
    'Accept-Language': 'en-US,en;q=0.5',
    # 'Accept-Encoding': 'gzip, deflate, br',
    # Already added when you pass json=
    # 'Content-Type': 'application/json',
    'pageToken': '574c08ae-2b39-47e8-8854-6464e1485498',
    'X-Requested-With': 'XMLHttpRequest',
    'Origin': 'https://bricsonline.nseindia.com',
    'DNT': '1',
    'Connection': 'keep-alive',
    'Referer': 'https://bricsonline.nseindia.com/bondsnew/rest/public?r=sebiannexure1',
    # Requests sorts cookies= alphabetically
    # 'Cookie': '_au=5D74EE4A55663F6A368505CD4FBA3632; ak_bmsc=272E767CEFD2BFE86A8B2D591830EB33~000000000000000000000000000000~YAAQpPBuaJhf/sWBAQAAD0iezxChkxULrJvqizHvrgRAPlYdsyj6T2rQYEb9rAe5izZEiLU1AkBYnff0uojtu4ttL/RfosZXFaTmWu21t498lYTHnvB/9Xl4z8bOZfOnJI0c0 EJ1L5RcCCcPb6NB05vKNZlD9nJM5glghtV/H80SIulXrLP2LnTxT94w/gRVdGx/FfiCwIRLVvPs5qCN5hLO0670abPUpFhAsy66bV GWngpAS0sxOGr7w DQFnvpmagENL7Y4o1aB 4ddK4gWJbKYQj5hdy4LLW 377hfEjNp ep7ayBCvefjdGfYFcwstc/dGwtzwD ThZkAsWXwY2yTb/FSbaNBdiQTRYhGC0egZXr1mmdbH7h0BhEknriEBKcK6Obyd5VVaWNN7kGN1gEV2oaw9jybDCVjZFPlp39w=; bm_sv=F3E240F1831B5411B570DE55492509FB~YAAQpPBuaJNi/sWBAQAA6LqezxAmS74pOfmgYYRG49hcJv6SNK7iWVoRx84cz93tXpzeZTXvfHj6z N90DQHyJaPMV1l5EEs29n0CiXYUgl1uB83bMRdGjh6tPjFnS8BufizM1QR0mAWdtdhDeHUL3kRtGJNGkMhNR8neClTdEH8PsdaJmzINU72Oy 8zvBRaeW mk5BnpX9WcWUAp9R3hsDkLqlqWQJ4dnvy79ZCLCuk758fn0oGMWfZCJrLiYXZIAW~1; bm_mi=1E2B4B400A3D8F400F1DFD09AAFFF3C9~YAAQpPBuaNxf/sWBAQAA0G6ezxBWH9cCRXbc0lvozBmymZF2rIxZ6ZYY86kjHrnixSXW8H1qdX9tKhqIre/QbtsOpcnHRy8XIS ngtAG5ZfAcaMIP34dgHi1IVV6Muym04KAjLO6c47cumBUPcWXnL/7jKT1TZQnY4 fXWBQICcxOXAf0MmBgkOmNMh1rJzwIXRP99rAmqr/eSUORLctK9WtlR1lZhV9CcxXHsZWqVXFfWI2oqvkWeYtogsjHCdlpM1zMG673zaoEal23cEB6lEDhy/T4EbyZKhWVt  3BdCRDJszPdlVgBoyT7uO0cc3IhDC9XkrnY6/2mRZ5LopScRWSM7rdNECjP4dUst36Vm~1',
    'Sec-Fetch-Dest': 'empty',
    'Sec-Fetch-Mode': 'cors',
    'Sec-Fetch-Site': 'same-origin',
}

json_data = {
    'filtFromModSettleDate': '01-07-2022',
    'columnNames': [
        'modRemarksBuyer',
        'symbol',
        'secPayinRemarks',
        'field4',
        'issueCouponRate',
        'issueDesc',
        'price',
        'yield',
        'yieldType',
        'refNo',
        'putCallDate',
        'value',
        'reportTime',
        'modSettleDate',
        'modRemarksSeller',
        'filtCustodian',
        'filtCounterParty',
    ],
}

response = requests.post('https://bricsonline.nseindia.com/bondsnew/rest/public/sebiannxone/all', cookies=cookies, headers=headers, json=json_data)
response.text

returns

'[["Direct","INE540P07426","Listed","U.P. POWER CORPORATION LIMITED",9.7000,"U.P. POWER CORPORATION LIMITED SR I STRPP G 9.70 BD 31MR31 FVRS10LAC",103.8500,9.3400,"YTM",null,"31-03-2031",1500.00,"30-06-2022 17:09:00","04-07-2022","Reported",null,"Not Settled"],["Direct","INE0M2307057","Listed","ANDHRA PRADESH STATE BEVERAGES CORPORATION LIMITED",9.6200,"ANDHRA PRADESH STATE BEVERAGES CORPORATION LIMITED SR-I 9.62 BD 31MY27 FVRS10LAC",100.0000,9.9600,"YTM",null,"31-05-2027",1000.00,"30-06-2022 17:09:00","04-07-2022","Reported",null,"Not Settled"],["Direct","INE0M2307040","Listed","ANDHRA PRADESH STATE BEVERAGES CORPORATION LIMITED",9.6200,"ANDHRA PRADESH STATE BEVERAGES CORPORATION LIMITED SR-I 9.62 BD 29MY26 FVRS10LAC",100.0000,9.9600,"YTM",null,"29-05-2026",1000.00,"30-06-2022 17:09:00","04-07-2022","Reported",null,"Not Settled"],["Direct","INE0M2307032","Listed","ANDHRA PRADESH STATE BEVERAGES CORPORATION LIMITED",9.6200,"ANDHRA PRADESH STATE BEVERAGES CORPORATION LIMITED SR-I 9.62 BD 30MY25 FVRS10LAC",100.0000,9.9600,"YTM",null,"30-05-2025",1000.00,"30-06-2022 17:09:00","04-07-2022","Reported",null,"Not Settled"],["Direct","INE540P07376","Listed","U.P. POWER CORPORATION LIMITED",9.7000,"U.P. POWER CORPORATION LIMITED SR I STRPP B 9.70 BD 31MR26 FVRS10LAC",100.9400,9.6900,"YTM",null,"31-03-2026",1500.00,"30-06-2022 17:09:00","04-07-2022","Reported",null,"Not Settled"],["Direct","INE0M2307081","Listed","ANDHRA PRADESH STATE BEVERAGES CORPORATION LIMITED",9.6200,"ANDHRA PRADESH STATE BEVERAGES CORPORATION LIMITED SR-I 9.62 BD 31MY30 FVRS10LAC",100.0000,9.9600,"YTM",null,"31-05-2030",1000.00,"30-06-2022 17:09:00","04-07-2022","Reported",null,"Not Settle ETC

CodePudding user response：

There is no real content on that page. Try 'View page source' to see what I mean. That page is just a template / wrapper for layout, which then makes a javascript-generated POST to a web service to get the data that it wants to display, which it then receives, formats and displays.