Im trying to scrape a webpage, but unable to access the endpoint which is dynamically loaded using the service. I checked the network tab and found the service and tried to mock a request, getting error enter image description here

tried to make a get request using the queryparameters with the same payload, getting http 400 error. please someone help!

I have attached a picture, please check it out and help!

The code is as below

import requests
payload = {'doctorId':'739559',
       'regdNovalue':'3608'}
headers = {'Accept': '*/*',
          'Accept-Encoding': 'gzip, deflate, br',
          'Accept-Language': 'en-US,en;q=0.9',
          'Connection': 'keep-alive',
          'Content-Length': '42',
          'Content-Type': 'application/json',
          'Cookie': 'JSESSIONID=-jU0dgec3PJUA- 
           bjaUtO3aPvIUnX2Zz4I31sRlrX.web3; 
           PHPSESSID=0lmo0b9av7vob0m66c1ugtm840',
          'Host': 'www.nmc.org.in',
          'Origin': 'https://www.nmc.org.in',
          'Referer': 'https://www.nmc.org.in/information-desk/indian- 
           medical-register/',
          'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="101", 
          "Google Chrome";v="101"',
          'sec-ch-ua-mobile': '?0',
          'sec-ch-ua-platform': '"Windows"',
          'Sec-Fetch-Dest': 'empty',
          'Sec-Fetch-Mode': 'cors',
          'Sec-Fetch-Site': 'same-origin',
          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) 
           AppleWebKit/537.36 (KHTML, like Gecko) 
           Chrome/101.0.4951.67 Safari/537.36',
          'X-Requested-With': 'XMLHttpRequest'}

url = "https://www.nmc.org.in/MCIRest/open/getDataFromService?service=getDoctorDetailsByIdImr"
res1 = requests.get(url, params = payload,headers=headers)
res2 = requests.post(url, data = payload,headers=headers)

TIA

CodePudding user response：

To get data from the server, use json= parameter:

import json
import requests

api_url = "https://www.nmc.org.in/MCIRest/open/getDataFromService"
params = {"service": "getDoctorDetailsByIdImr"}
payload = {"doctorId": "17068", "regdNoValue": "3608"}

data = requests.post(api_url, params=params, json=payload).json()

# pretty print the data:
print(json.dumps(data, indent=4))

Prints:

{
    "yearInfo": 1981,
    "regDate": "24/03/1981",
    "doctorId": 17068,
    "salutation": null,
    "firstName": "Jagtap Shailendra",
    "middleName": null,
    "lastName": null,
    "phoneNo": null,
    "emailId": null,
    "gender": null,
    "bloodGroup": null,
    "parentName": "Lata Raghunath Jagtap",
    "birthDate": null,
    "isNewDoctor": false,
    "checkExistingUser": false,
    "birthDateStr": null,
    "birthPlace": null,
    "nationality": null,
    "eligbleToVote": null,
    "adharNo": null,
    "uprnNo": null,
    "doctorEducationId": null,
    "college": null,
    "doctorDegree": "MBBS",
    "university": "U.Indore",
    "otherSubject": null,
    "monthOfPass": null,
    "yearOfPassing": "1980",
    "smcId": 15,
    "registrationDate": null,
    "registrationNo": "3608",
    "smcName": "Madhya Pradesh Medical Council",
    "homeAddress": null,
    "officeAddress": null,
    "address": "1 / 10, New Palasia Indore, M. P. ",
    "officeaddress": null,
    "addressLine1": "1 / 10, New Palasia Indore, M. P. ",
    "addressLine2": null,
    "economicStatus": null,
    "city": null,
    "state": null,
    "country": null,
    "pincode": null,
    "photos": null,
    "doctRegistrationNo": null,
    "universityId_view": null,
    "universityId": null,
    "monthandyearOfPass": "1980",
    "passoutCollege": null,
    "collegeId": null,
    "stateId": null,
    "catagory": null,
    "catagory_view": "N/A",
    "role": null,
    "registrationDatePrevious": null,
    "registrationNoPrevious": null,
    "smcNamePrevious": null,
    "uprnNoPrevious": null,
    "removedStatus": false,
    "removedOn": "01/01/1900",
    "restoredStatus": false,
    "restoredOn": "01/01/1900",
    "remarks": null,
    "regnNo": null,
    "smcIds": null,
    "trasanctionStatus": null,
    "addlqual1": "                    ",
    "addlqualyear1": "    ",
    "addlqualuniv1": "                                        ",
    "addlqual2": "                    ",
    "addlqualyear2": "    ",
    "addlqualuniv2": "                                                  ",
    "addlqual3": "                    ",
    "addlqualyear3": "    ",
    "addlqualuniv3": "                                                  ",
    "patientfirstName": null,
    "patientmiddleName": null,
    "patientlastName": null,
    "patientphoneNo": null,
    "patientemailId": null,
    "appealBy": null,
    "altphone": null,
    "landLineNo": null,
    "patientlandLineNo": null,
    "patientaltphone": null,
    "picName": null,
    "signatureName": null,
    "stateMedicalCouncil": null,
    "countryType": null,
    "dateOfBirth": null,
    "universityName": null,
    "qualification": null,
    "imrNumber": null
}