import requests
from bs4 import BeautifulSoup
import pandas as pd

url='https://bni-india.in/en-IN/chapterdetail?chapterId=P+IWaeuOSfF3Mxm03FLSzA==&name=BNI BNI Alpha'

page=requests.get(url)

soup=BeautifulSoup(page.text, "html.parser")


table=soup.find('table',{'class':'table table-hover listtables dataTable no-footer dtr-inline member_list_class'})
print(table)

I'm trying to extract the HTML table on the members section on the following webpage but it only returns "none"

CodePudding user response：

Try:

import requests
import pandas as pd
from bs4 import BeautifulSoup


url = "https://bni-india.in/bnicms/v3/frontend/chapterdetail/display"

data = {
    "pageMode": "Live_Site",
    "chapterId": "P IWaeuOSfF3Mxm03FLSzA==",
    "languageLocaleCode": "en_IN",
    "website_type": "1",
    "website_id": "13451",
    "mappedWidgetSettings": '[{"key":83,"name":"Chapter Website","value":"Chapter Website"},{"key":84,"name":"View Chapter Gallery","value":"View Chapter Gallery"},{"key":85,"name":"Visit This Chapter","value":"Visit This Chapter"},{"key":86,"name":"Member Names","value":"Member Name"},{"key":87,"name":"Company","value":"Company"},{"key":88,"name":"Profession/Speciality","value":"Profession/Speciality"},{"key":89,"name":"Phone","value":"Phone"},{"key":90,"name":"Send Mail","value":"Send Mail"},{"key":91,"name":"Showing","value":"Showing"},{"key":92,"name":"to","value":"to"},{"key":93,"name":"of","value":"of"},{"key":94,"name":"entries","value":"entries"},{"key":95,"name":"Meeting Details","value":"Meeting Details"},{"key":96,"name":"View Map","value":"View Map"},{"key":97,"name":"Member Count","value":"Member Count"},{"key":98,"name":"Show Members","value":"Show Members"},{"key":99,"name":"Chapter Leadership","value":"Chapter Leadership"},{"key":233,"name":"Directions","value":"Directions"},{"key":307,"name":"Zero Records","value":"Zero Records"},{"key":390,"name":"Apply Now","value":"Apply Now"}]',
    "planyourvisit": "y",
}

soup = BeautifulSoup(requests.post(url, data=data).content, "html.parser")


df = pd.read_html(str(soup.select_one("#DataTables_Table_0")))[0]
cols = ["Member Name", "Company", "Profession/Specialty", "Phone", ""]
df.columns = cols
print(df[cols[:-1]].head().to_markdown(index=False))

Prints: