Need help isolating results from xhr request-CodePudding

When I run the code below, it's giving me a lot of information I don't want. I only want to capture the data circled starting with 4. Does anyone know how to isolate the data in the request to get rid of everything but what is circled? Also, if anyone knows how to bring the results to a csv or xlsx file, that would be even better.

Working code:

   import requests

url = "https://www.stockrover.com/stock_infos/grid?_dc=1644769629231"

data = { "ticker": 4, "rank": "5", }

payload = "state={"sortInfo":{},"columns":[77,32,498,500,31,27,499,30,578,28,29,544,181,185,186],"view":281,"priorPrimaryColumn":170,"filterData":[],"name":"New 1","cType":"Screener","cNode":"s_39","cIsFolder":false,"gridSelection":"BTU","lastActive":1396898415,"primaryColumn":76,"folderDisabledParams":{"filterData":[]},"mainGridDateRange":"ytd","groupState":null,"moversGridDateRange":"1_day","peersGridDateRange":"1_day","lastGridSelections":["BTU"],"lastQuantNode":[],"includeQuotesInTable":false,"includeAllQuotesLastValue":false,"markets":{"panel":"summary"},"researchPanel":"tablePanel","recentSearchTickers":["SPY","AMZN","AAPL","s_32","^NDX","AXP","XOM","AFL","^DJX","AIT","ADVC"],"quotesBoxTickers":["AMZN","AAPL","SPY"],"checkedQuotesBoxTickers":["AMZN","AAPL","SPY"],"dashboard":{"buttonRef":"272"},"tickerSelectedFeeds":["Benzinga News","Yahoo News"],"marketSelectedFeeds":["Google News","Stock Market News - Investing.com"],"bondsSelectedFeeds":["Bonds Strategy - Investing.com"],"commoditiesSelectedFeeds":["Commodities & Futures News - Investing.com","Commodities Fundamental Analysis - Investing.com","Commodities Strategy Analysis - Investing.com"],"stocksSelectedFeeds":["CNNMoney News","Google News","Seeking Alpha Top Stories"],"etfsSelectedFeeds":["Economy News - Investing.com","ETF Analysis - Investing.com","Investing Ideas - Investing.com"],"topPanel":"researchPanel","maxRecordsNode":false,"version":7,"lastGridSelectionsRaw":["BTU"],"lastSelectionScreeners":"s_39","quotesDisabled":true,"lastSelectionPortfolios":"p_2","comparisonPanels":{"Portfolio":"p_2","Index":"^DJX","Watchlist":"Watchlists","Screener":"s_39"},"lastSelectionWatchlists":"w_26","indicesSelectedFeeds":["Google News","Yahoo News"],"newsActive":"tickerNews","recentSearchMetrics":["Price","EPS","Sales"],"editPanel":"positionsPanel","newsType":"marketNews","tableColumns":["ticker","rank","score_rank","filter_score","company","cash","currentassets","netppe","intangibles","totalassets","currentliabilities","longtermdebt","totaldebt","totalliabilities","equity","tangiblebookvalue","cash_short_term_p","net_ppe_p","intangibles_p"],"last_save":1644769379,"panels":{"collapsed":{"chp":true,"ip":true,"mp":false,"qp":false,"conp":true,"fsp":false},"viewportWidth":"1920","viewportHeight":"1069","chartPanelHeight":483,"controlPanelWidth":296,"insightPanelWidth":"485","quoteBoxHeight":200,"navigationPanelWidth":277}}&updateMarket=true&page=1&start=0&limit=250"
headers = {
  'authority': 'www.stockrover.com',
  'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="98", "Google Chrome";v="98"',
  'x-csrf-token': '7yR4pfI0kAArtjJak535 NJrpB0L212PAbXCg0kbyE4SyjFaQ73sMHJLiqAkPb5nGzfC8KvAa3kTADLAEQXyOQ==',
  'sec-ch-ua-mobile': '?0',
  'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.82 Safari/537.36',
  'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
  'accept': 'application/json',
  'x-requested-with': 'XMLHttpRequest',
  'sec-ch-ua-platform': '"Windows"',
  'origin': 'https://www.stockrover.com',
  'sec-fetch-site': 'same-origin',
  'sec-fetch-mode': 'cors',
  'sec-fetch-dest': 'empty',
  'referer': 'https://www.stockrover.com/research/table/281/s_39/BTU',
  'accept-language': 'en-US,en;q=0.9',
  'cookie': 'remember_me_pref=0; user_name=test11964; plan=3; premiumBraintreeKey=MIIBCgKCAQEAzM4LJfrNnBOgRFB1dDJkmqTFCWT2Y+ksOydD8xDH4R033WUzxbffMZb+3dqEyQvOVjLcwFIHByDc4Xwej7enas2E/VRyh7Cvyadn7M5zQeRyLcI9Ys5KCozMwxJPc0x76FlXPwiAo1Qlz3RcLb9wGHBag2R51FuTie+hVDCgzWajqDCREzRhi/qlt3D/xXNo/iwJlpOUr/x1QnkkILxgKlq1dD7KJ767O5ojYKXsO+V2Bfu7sSD3djsOxQJ1/RbaDm2E96EDkWhhOeOpPndQ6IuSl4NmnJg/cq6f8csW8M3Ys+MZPFkdxPC4/fRM1XC9o76PjpVNBIO+yJEELKZedwIDAQAB; lr=1644769628; _Ruby2_session=Q1drcmlhazYvUFZLd0NydnRXUGpoUzArZDlxYWRCcW9sRUx5VDBydWVWRHdGWDZlMnlESURzbldwbFV1L0drbUlKaWt5MXRtaS9iR0ZYZEpPVHQ1N25qRnR1d3FrY0tzQW1qQm9CdTZ3MSs0d2c3MlpuMjRiQWhCOHI1cGNWekZ4cUdJd0ZFcGtpeng3MFlqZjFDUW9RYmpFMU9DeGdGMVZKR1EwMjVhSE9yVHl4VXFtQm9aYVBtNHF5d0pwMjJ1aVlNMUVRUzdnVFZWZ1AxQkY5Q0p6a2RKay9QL05tOWk4cHZiSERtaGRxeTlxTWZnV3Q0cjdwR3RndUtmeUp3QThhMnJaV2dGZjlPUUtjcGRidDhiajRxK2g0RUZTMWNZUDBaeGNCcUVxSDJ1QnZVRlRkWk9tUExJNWN3TDN5T1BQcmhVVGsycStVTzJRaUwvSkk2TnNVZldTOGU3Tm5wQ3RUMy9nazFqbzdrUWtvYzRwQWRpV3dnTVB3YzhodFV2U0FRR3VKdllMY01NZmdOdGtmOEJ4UT09LS1nTXBrYldhQ0pEeWJ3ak9qQjcrTGV3PT0=--1439f36a7f9362aee4b5b666747a2d63d72e81bd'
}

response = requests.request("POST", url, headers=headers, data=payload)
print(response.json())

CodePudding user response：

The data structure in this case is rather unusual but this looks like it might work for you:

import requests
    
url = "https://www.stockrover.com/stock_infos/grid?_dc=1644769629231"

payload = "state={"sortInfo":{},"columns":[77,32,498,500,31,27,499,30,578,28,29,544,181,185,186],"view":281,"priorPrimaryColumn":170,"filterData":[],"name":"New 1","cType":"Screener","cNode":"s_39","cIsFolder":false,"gridSelection":"BTU","lastActive":1396898415,"primaryColumn":76,"folderDisabledParams":{"filterData":[]},"mainGridDateRange":"ytd","groupState":null,"moversGridDateRange":"1_day","peersGridDateRange":"1_day","lastGridSelections":["BTU"],"lastQuantNode":[],"includeQuotesInTable":false,"includeAllQuotesLastValue":false,"markets":{"panel":"summary"},"researchPanel":"tablePanel","recentSearchTickers":["SPY","AMZN","AAPL","s_32","^NDX","AXP","XOM","AFL","^DJX","AIT","ADVC"],"quotesBoxTickers":["AMZN","AAPL","SPY"],"checkedQuotesBoxTickers":["AMZN","AAPL","SPY"],"dashboard":{"buttonRef":"272"},"tickerSelectedFeeds":["Benzinga News","Yahoo News"],"marketSelectedFeeds":["Google News","Stock Market News - Investing.com"],"bondsSelectedFeeds":["Bonds Strategy - Investing.com"],"commoditiesSelectedFeeds":["Commodities & Futures News - Investing.com","Commodities Fundamental Analysis - Investing.com","Commodities Strategy Analysis - Investing.com"],"stocksSelectedFeeds":["CNNMoney News","Google News","Seeking Alpha Top Stories"],"etfsSelectedFeeds":["Economy News - Investing.com","ETF Analysis - Investing.com","Investing Ideas - Investing.com"],"topPanel":"researchPanel","maxRecordsNode":false,"version":7,"lastGridSelectionsRaw":["BTU"],"lastSelectionScreeners":"s_39","quotesDisabled":true,"lastSelectionPortfolios":"p_2","comparisonPanels":{"Portfolio":"p_2","Index":"^DJX","Watchlist":"Watchlists","Screener":"s_39"},"lastSelectionWatchlists":"w_26","indicesSelectedFeeds":["Google News","Yahoo News"],"newsActive":"tickerNews","recentSearchMetrics":["Price","EPS","Sales"],"editPanel":"positionsPanel","newsType":"marketNews","tableColumns":["ticker","rank","score_rank","filter_score","company","cash","currentassets","netppe","intangibles","totalassets","currentliabilities","longtermdebt","totaldebt","totalliabilities","equity","tangiblebookvalue","cash_short_term_p","net_ppe_p","intangibles_p"],"last_save":1644769379,"panels":{"collapsed":{"chp":true,"ip":true,"mp":false,"qp":false,"conp":true,"fsp":false},"viewportWidth":"1920","viewportHeight":"1069","chartPanelHeight":483,"controlPanelWidth":296,"insightPanelWidth":"485","quoteBoxHeight":200,"navigationPanelWidth":277}}&updateMarket=true&page=1&start=0&limit=250"
headers = {
    'authority': 'www.stockrover.com',
    'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="98", "Google Chrome";v="98"',
    'x-csrf-token': '7yR4pfI0kAArtjJak535 NJrpB0L212PAbXCg0kbyE4SyjFaQ73sMHJLiqAkPb5nGzfC8KvAa3kTADLAEQXyOQ==',
    'sec-ch-ua-mobile': '?0',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.82 Safari/537.36',
    'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
    'accept': 'application/json',
    'x-requested-with': 'XMLHttpRequest',
    'sec-ch-ua-platform': '"Windows"',
    'origin': 'https://www.stockrover.com',
    'sec-fetch-site': 'same-origin',
    'sec-fetch-mode': 'cors',
    'sec-fetch-dest': 'empty',
    'referer': 'https://www.stockrover.com/research/table/281/s_39/BTU',
    'accept-language': 'en-US,en;q=0.9',
    'cookie': 'remember_me_pref=0; user_name=test11964; plan=3; premiumBraintreeKey=MIIBCgKCAQEAzM4LJfrNnBOgRFB1dDJkmqTFCWT2Y+ksOydD8xDH4R033WUzxbffMZb+3dqEyQvOVjLcwFIHByDc4Xwej7enas2E/VRyh7Cvyadn7M5zQeRyLcI9Ys5KCozMwxJPc0x76FlXPwiAo1Qlz3RcLb9wGHBag2R51FuTie+hVDCgzWajqDCREzRhi/qlt3D/xXNo/iwJlpOUr/x1QnkkILxgKlq1dD7KJ767O5ojYKXsO+V2Bfu7sSD3djsOxQJ1/RbaDm2E96EDkWhhOeOpPndQ6IuSl4NmnJg/cq6f8csW8M3Ys+MZPFkdxPC4/fRM1XC9o76PjpVNBIO+yJEELKZedwIDAQAB; lr=1644769628; _Ruby2_session=Q1drcmlhazYvUFZLd0NydnRXUGpoUzArZDlxYWRCcW9sRUx5VDBydWVWRHdGWDZlMnlESURzbldwbFV1L0drbUlKaWt5MXRtaS9iR0ZYZEpPVHQ1N25qRnR1d3FrY0tzQW1qQm9CdTZ3MSs0d2c3MlpuMjRiQWhCOHI1cGNWekZ4cUdJd0ZFcGtpeng3MFlqZjFDUW9RYmpFMU9DeGdGMVZKR1EwMjVhSE9yVHl4VXFtQm9aYVBtNHF5d0pwMjJ1aVlNMUVRUzdnVFZWZ1AxQkY5Q0p6a2RKay9QL05tOWk4cHZiSERtaGRxeTlxTWZnV3Q0cjdwR3RndUtmeUp3QThhMnJaV2dGZjlPUUtjcGRidDhiajRxK2g0RUZTMWNZUDBaeGNCcUVxSDJ1QnZVRlRkWk9tUExJNWN3TDN5T1BQcmhVVGsycStVTzJRaUwvSkk2TnNVZldTOGU3Tm5wQ3RUMy9nazFqbzdrUWtvYzRwQWRpV3dnTVB3YzhodFV2U0FRR3VKdllMY01NZmdOdGtmOEJ4UT09LS1nTXBrYldhQ0pEeWJ3ak9qQjcrTGV3PT0=--1439f36a7f9362aee4b5b666747a2d63d72e81bd'
}

response = requests.request("POST", url, headers=headers, data=payload)

stock_info = response.json()['stock_infos']

for info in stock_info:
    key = info[0]
    for i, sub in enumerate(info[1:], 1):
        if sub == key:
            print(info[i:])
            break

Output:

['AA', 1452.0, 4285.0, 6679.0, 0.0, 14197.0, 2929.0, 1724.0, 1725.0, 8736.0, 3878.0, 3878.0, 10.23, 47.05, None]
['ACH', 1773.84, 7909.0, 18758.17, 788.41, 30523.38, 9252.6, 8782.1, 14242.28, 18580.33, 9038.55, 8250.17, 5.81, 61.46, 2.58]

...and the values for all observed tickers