how to parse response immediately with asyncio.gather?-CodePudding

async def main():
    uuids = await get_uuids_from_text_file()
    tasks = []
    # create a task for each uuid
    # and add it to the list of tasks
    for uuid in uuids:
        task = asyncio.create_task(make_hypixel_request(uuid))
        tasks.append(task)
    # wait for all the tasks to finish
    responses = await asyncio.gather(*tasks)
    # run the functions to process the data
    for response in responses:
        print(response['success'])
        data2 = await anti_sniper_request(response)
        await store_data_in_json_file(response, data2)
        await compare_stats(response)

# loop the main function
async def main_loop():
    for _ in itertools.repeat([]):
        await main()


# run the loop
loop = asyncio.get_event_loop()
loop.run_until_complete(main_loop())
loop.close()

basically this is my code the functions have very clear and explanatory name the make_hypixel_requests part i have no issues there, the requests are executed immediately and in parallel, the problem is after that when "for response in responses" it goes hella slow? how do i get the responses instantly and loop through them very fast? i will try to attach a gif.

basically this is the issue:

CodePudding user response：

You can use asyncio.wait and return when at least a task is completed, then continue awaiting for the pending tasks. asyncio.wait return a tuple with two sets, the first with the completed tasks, the second with the still pending tasks. You can call to the result method of the done tasks and get its return value.

async def main():
    uuids = await get_uuids_from_text_file()
    tasks = []
    # create a task for each uuid
    # and add it to the list of tasks
    for uuid in uuids:
        task = asyncio.create_task(make_hypixel_request(uuid))
        tasks.append(task)
    # wait for all the tasks to finish
    while tasks:
        done_tasks, tasks = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
        for done in done_tasks:
            response = done.result()
            print(response['success'])
            data2 = await anti_sniper_request(response)
            await store_data_in_json_file(response, data2)
            await compare_stats(response)

CodePudding user response：

the reason is because after waiting for all the responses to return, u process them in a loop instead of asynchronously, since none of the requests seem to depend on each other, waiting for them all to finish before processing them doesn't make sense, the best way to handle this is to couple the request and the processing e.g.

async def request_and_process(uuid):
    response = await make_hypixel_request(uuid)
    print(response['success'])
    compare_stats_task = asyncio.create_task(compare_stats(response))
    data2 = await anti_sniper_request(response)
    await asyncio.gather(store_data_in_json_file(response, data2), compare_stats_task)

async def main():
    while True:
        uuids = await get_uuids_from_text_file()
        await asyncio.gather(*map(request_and_process, uuids))

asyncio.run(main())