In my program, if there no data found at page for eg.10, then for the rest of the pages, 11 to 50, it is not going into the else statement (which should happen). I am new to python and need help to sort this problem in the code written below:

import concurrent
import functools
import concurrent.futures
import requests
from urllib3.exceptions import InsecureRequestWarning
import csv

requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
from bs4 import BeautifulSoup as bs

def download_page(session, page_no):
    url = 'https://bidplus.gem.gov.in/bidlists?d=1&page_no='   str(page_no)
    print('URL created: '   url)
    resp = session.get(url, verify=False)
    return resp.text


def scrap_bid_data():
    NUMBER_THREADS =5 # number of concurrent download requests
    with open('GEMconcurrent_1004.csv', 'w', newline='') as out_file:
        f = csv.writer(out_file)
        f.writerow(['Bidnumber', 'Items', 'Quantity', 'Department', 'Enddate','pageNumber'])
        with requests.Session() as session:
            page_downloader = functools.partial(download_page, session)
            with concurrent.futures.ThreadPoolExecutor(max_workers=NUMBER_THREADS) as executor:
                pages = executor.map(page_downloader, range(35, 36 ))
                page_no = 0
                for page in pages:
                    page_no  = 1
                    soup_data = bs(page, 'lxml')
                    extracted_data = soup_data.find('div', {'id': 'pagi_content'})
                    if extracted_data is None or len(extracted_data) == 0:
                        print('No data at page number', page_no)
                        print(page)
                        break
                    else:
                        for idx in range(len(extracted_data)):
                            if (idx % 2 == 1):
                                bid_data = extracted_data.contents[idx].text.strip().split('\n')
                                if (len(bid_data)>1):
                                    print(page_no)
                                    if (len(bid_data[8]) > 1):

                                                            bidno = bid_data[0].split(":")[-1]
                                                            items = bid_data[8].split(":")[-1]
                                                            qnty = int(bid_data[9].split(':')[1].strip())
                                                            dept = (bid_data[10]   bid_data[15].strip()).split(":")[-1]
                                                            edate = bid_data[20].split("End Date:")[-1]
                                                            f.writerow([bidno, items, qnty, dept, edate,page_no])
scrap_bid_data()

CodePudding user response：

You have the code formatted in the following way:

for p in pages:
   do stuff
   if condition:
      do stuff
      break
   else:

but if you want to run the else clause on breaks, you need to format it to be on the same level as the for loop:

for p in pages:
    do stuff
    if condition:
        do stuff
        break
else:
   #i got here because of the break statement

CodePudding user response：

Break is used when you want to move out of the condition (loop containing it).

A more detailed explanation here https://www.programiz.com/python-programming/break-continue.

I do not think you need to use break here.