downloadStart = datetime.now()
while (True):
    requestURL = transactionAPI.format(page = tempPage,limit = 5000)
    response = requests.get(requestURL,headers=headers)
    json_data = json.loads(response.content)
    tempMomosTransactionHistory.extend(json_data["list"])  
    if(datetime.fromtimestamp(json_data["list"][-1]["crtime"]) <  datetime(datetime.today().year,datetime.today().month,datetime.today().day - dateRange)):          
        break                       
    tempPage  = 1
downloadEnd = datetime.now()

Any suggestions please threading or something like that ?

Outputs here

downloadtime 0:00:02.056010

downloadtime 0:00:05.680806

downloadtime 0:00:05.447945

CodePudding user response：

You need to improve it in two ways.

1. Optimise code within loop
2. Parallelize code execution

#1 By looking at your code I can see one improvement ie. create datetime.today object instead of doing 3 times. Check other methods like transactionAPI optimise further.

#2: If you multi core CPU machine then you take advantage of machine by spanning thread per page. Refer to modified code of above.

import threading

def processRequest(tempPage):
    requestURL = transactionAPI.format(page = tempPage,limit = 5000)
    response = requests.get(requestURL,headers=headers)
    json_data = json.loads(response.content)
    tempMomosTransactionHistory.extend(json_data["list"])
    
downloadStart = datetime.now()
while (True):
     #create thread per page
     t1 = threading.Thread(target=processRequest, args=(tempPage, ))
     t1.start()
     #Fetch datetime today object once instaed 3 times
     datetimetoday = datetime()
    if(datetime.fromtimestamp(json_data["list"][-1]["crtime"]) <  datetime(datetimetoday.year,datetimetoday.month,datetimetoday.day - dateRange)):          
        break                       
    tempPage  = 1
downloadEnd = datetime.now()