Looping through a Payload to get all pages-CodePudding

I know you can loop through page numbers in a URL but is it possible loop through pages in a Payload? I would guess you need beautiful soup? At the end of the Payload, there is this code.

page=1&start=0&limit=250 Currently, I am just getting page 1 with 250 lines. I need to loop through the code and retrieve page=2&start=251&limit=250 and the subsequent 11 pages. Could anyone show me how to do this?

Working Code for first 250 Lines

import requests
import pandas as pd

def stock_data(stock_info):
    data = pd.DataFrame(stock_info)
    data = data.iloc[:, 4:]
    data.to_csv("data.csv", index=False)


url = "https://www.stockrover.com/stock_infos/grid?_dc=1644876887410"

payload = "state={"sortInfo":{},"columns":[76,77,50,80,547,13,21,771],"view":313,"priorPrimaryColumn":170,"filterData":[],"name":"New 1","cType":"Screener","cNode":"s_42","cIsFolder":false,"gridSelection":"ANDE","lastActive":1396898415,"primaryColumn":76,"folderDisabledParams":{"filterData":[]},"mainGridDateRange":"ytd","groupState":null,"moversGridDateRange":"1_day","peersGridDateRange":"1_day","lastGridSelections":["ANDE"],"lastQuantNode":["s_42","s_42"],"includeQuotesInTable":false,"includeAllQuotesLastValue":false,"markets":{"panel":"summary"},"researchPanel":"comparisonPanel","recentSearchTickers":["SPY","AMZN","AAPL","s_32","^NDX","AXP","XOM","AFL","^DJX","AIT","ADVC"],"quotesBoxTickers":["AMZN","AAPL","SPY"],"checkedQuotesBoxTickers":["AMZN","AAPL","SPY"],"dashboard":{"buttonRef":"272"},"tickerSelectedFeeds":["Benzinga News","Yahoo News"],"marketSelectedFeeds":["Google News","Stock Market News - Investing.com"],"bondsSelectedFeeds":["Bonds Strategy - Investing.com"],"commoditiesSelectedFeeds":["Commodities & Futures News - Investing.com","Commodities Fundamental Analysis - Investing.com","Commodities Strategy Analysis - Investing.com"],"stocksSelectedFeeds":["CNNMoney News","Google News","Seeking Alpha Top Stories"],"etfsSelectedFeeds":["Economy News - Investing.com","ETF Analysis - Investing.com","Investing Ideas - Investing.com"],"topPanel":"researchPanel","maxRecordsNode":false,"version":7,"lastGridSelectionsRaw":["ANDE"],"lastSelectionScreeners":"s_42","quotesDisabled":true,"lastSelectionPortfolios":"p_2","comparisonPanels":{"Portfolio":"p_2","Index":"^DJX","Watchlist":"Watchlists","Screener":"s_42"},"lastSelectionWatchlists":"w_26","indicesSelectedFeeds":["Google News","Yahoo News"],"newsActive":"tickerNews","recentSearchMetrics":["Price","EPS","Sales"],"editPanel":"positionsPanel","newsType":"marketNews","tableColumns":["ticker","rank","score_rank","filter_score","company","cash","currentassets","netppe","intangibles","totalassets","currentliabilities","longtermdebt","totaldebt","totalliabilities","equity","tangiblebookvalue","cash_short_term_p","net_ppe_p","intangibles_p"],"last_save":1644837064,"panels":{"collapsed":{"chp":true,"ip":true,"mp":false,"qp":false,"conp":true,"fsp":false},"viewportWidth":"1920","viewportHeight":"1069","chartPanelHeight":483,"controlPanelWidth":296,"insightPanelWidth":"485","quoteBoxHeight":200,"navigationPanelWidth":277}}&updateMarket=true&page=1&start=0&limit=250"

headers = {
  'authority': 'www.stockrover.com',
  'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="98", "Google Chrome";v="98"',
  'x-csrf-token': 'fAeVScD26lby5MQf5YFI5p3snudo3E rw0TL0h1W3j/vcjsIMvgxAF5Z9DkMjjCU4trT/b4EV0VCCPvmms5VIw==',
  'sec-ch-ua-mobile': '?0',
  'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.82 Safari/537.36',
  'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
  'accept': 'application/json',
  'x-requested-with': 'XMLHttpRequest',
  'sec-ch-ua-platform': '"Windows"',
  'origin': 'https://www.stockrover.com',
  'sec-fetch-site': 'same-origin',
  'sec-fetch-mode': 'cors',
  'sec-fetch-dest': 'empty',
  'referer': 'https://www.stockrover.com/research/all/313/s_42/ANDE',
  'accept-language': 'en-US,en;q=0.9',
  'cookie': 'remember_me_pref=0; user_name=test11964; plan=3; premiumBraintreeKey=MIIBCgKCAQEAzM4LJfrNnBOgRFB1dDJkmqTFCWT2Y+ksOydD8xDH4R033WUzxbffMZb+3dqEyQvOVjLcwFIHByDc4Xwej7enas2E/VRyh7Cvyadn7M5zQeRyLcI9Ys5KCozMwxJPc0x76FlXPwiAo1Qlz3RcLb9wGHBag2R51FuTie+hVDCgzWajqDCREzRhi/qlt3D/xXNo/iwJlpOUr/x1QnkkILxgKlq1dD7KJ767O5ojYKXsO+V2Bfu7sSD3djsOxQJ1/RbaDm2E96EDkWhhOeOpPndQ6IuSl4NmnJg/cq6f8csW8M3Ys+MZPFkdxPC4/fRM1XC9o76PjpVNBIO+yJEELKZedwIDAQAB; lr=1644876886; _Ruby2_session=OEs3djBGVmtrSXhSWjFibnp2ck9vem8vd1VmQ00wUkZCVWlaWmYzOFBQQUJyemc0MFNjMlVmTGRUeFpNSTFIRGliVitnb2M1L2JNcCt5SnQxN2xtZDV5M20waEd0elh3aUU3R0k3YnJiVy9BcUhObGpBUU54c2tQRE9RZHBZWk8wa0NBQXJub2tHU3pHZmUvM3dRNGZvVStsT0ZEbUNEa3ZyVDkxdDA5S1B5d3VZWlZNUERxL01VNlYzdHhBci9uSnduOGxodk8rSDJLaUJIWTVyM25GL3o4RHYva3pGeDdIY1NJODV4WkV4MnRiR2RoNHZDUktPWSsxaElPdXNEU0tSaCtKL1o0K2FsSE1rcVI2TkgxOFo0MktSL1JRWjgzRllBZGFhMjg4bitpTUJzQjR6Nk5OZzhzUFMzVGRpVFNZOUxrTnhWNTB2K0dYNXdDcFFXZnpZdlZleFR2cWY5cmJqaDNiY0JhWVJJT0s1TDEvOHU0UTV6NW5uTjcwZjFzdHpxeFg0cVQ3NTRob2xTMlA2ZDhCQT09LS1LOEljN3laVTlBVUpsVVo3c093Y2NnPT0=--b963330daa985315420ea5893f1cfa3e3a54c9d5; _Ruby2_session=UmE1NVo0cjJ1YmR3RGlQUis0V3JVWUprR0xtbzVaV0k2NTNhUmFUU2VsNVB4WFN6M0Y5M0xIVmpMdGdBTEgzMDZwb3JZNGpJRmFkdjRyWG9iY3NGT1pENTRaaXdZZit3b3FCZzdvMTVxZ3p2RGpUUUtyTGpOcjhZcXVxS0FIcElBTnhtZDlJQ3g3d1o2bUF3Tzl4NnIyZ2ZHYmMyb09vVnMwaGU5SEsvTWRwaHdFeUNGUDJZUDY4eXEwRlcyYzFqVVh2VzVjcEhDSU9jOUw4NmVJcmd6aHEwVUUya25Yb0Y5d21hY3g5VTdxbi81dkdiZ2Qrc0YrdEZtNWU5c3ZlcHFRSXVqcXdBVEM1RnV5bFo3N3hQNzg4Q1lJWTRtakp2MHJyN3gvUEtvN0h2R3lTZEFqZkwrVFlDRlk2czZoaDBQcXltUjdQbUNiTXJWMW42WnlPdUc2ZGxzUXRzY3JuYTN1V1VIMkVvdHVpeE40N0l6SjVLelJhUGh6aXBrZ3B2V1gySG1YRUVLbGZGYzQzRzR3QjJwSTFieDZRQ242QTdKNlNEWVFIQ3lTWHJPQzUzZ1lKVzl5S0FoZWxHQ0xnTy0tODZDejdtSmFDek5GeUVMMnJweTJtUT09--a126f3bcc5b8af0a5a824e6b674d55f1fe9ee12e; lr=1644876939'
}

 
for page in range(3):
    pld = payload.format(page 1, page*250, 250)
    response = requests.request("POST", url, headers=headers, data=pld)
    stock_info = response.json()['stock_infos']
    stock_data(stock_info)

CodePudding user response：

Here's how you do it in a loop. This works; I've tried it here.

for page in range(3):
    pld = payload.format(page 1, page*250, 250)
    response = requests.request("POST", url, headers=headers, data=pld)
    stock_info = response.json()['stock_infos']
    stock_data(stock_info)

You will, of course, need to modify your code so that stock_data doesn't overwrite the CSV file every time. You can either append to one big dataframe, or append to the CSV file.