I want to scrape multiple URLs with 2 different HTML templates. I can scrape each HTML by itself without issue, but I ran into a problem when trying to combine the two scrapers. Below is my code:

import requests
from bs4 import BeautifulSoup
import pandas as pd

page_url1 = 'https://www.vet.upenn.edu/research/centers-laboratories/research-laboratory/research-laboratory/anguera-laboratory'
page_url2 = 'https://www.med.upenn.edu/apps/faculty/index.php/g20001100/p8866286'
page_url_lst = {'url': [page_url1, page_url2], 'template': [1,2]}
page_url_df = pd.DataFrame(page_url_lst)

data = []
if page_url_df['template'] == 1:
    for url in page_url_df['url']:
        r = requests.get(url)
        soup = BeautifulSoup(r.text, 'lxml')
        for e in soup.select('#tabs-publications em'):
            data.append({
                'author':e.previous.get_text(strip=True)[:-1],
                'title':e.get_text(strip=True),
                'journal':e.next_sibling.get_text(strip=True),
                'source': url
            })
else:
    for url_2 in page_url_df['url']:
        r_2 = requests.get(url_2)
        soup_2 = BeautifulSoup(r_2.text, 'lxml')
        for a in soup_2.find_all('span',{'class':'fac_citation'}):
            data.append({
                'author':a.find('b').get_text(),
                'title':a.find('i').get_text(strip=True),
                'journal':a.find('i').next_sibling.get_text(strip=True),
                'source': url_2
            })

The logic here if the column 'template' returns a value of 1, then extract the data using the first template, else extract the data using the second template. However, this code return this error code: The truth value of a Series is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all().

Thank you in advance!

CodePudding user response：

If I understand you right, you want to create new dataframe based on page_url_df:

import requests
import pandas as pd
from bs4 import BeautifulSoup


page_url1 = "https://www.vet.upenn.edu/research/centers-laboratories/research-laboratory/research-laboratory/anguera-laboratory"
page_url2 = (
    "https://www.med.upenn.edu/apps/faculty/index.php/g20001100/p8866286"
)
page_url_lst = {"url": [page_url1, page_url2], "template": [1, 2]}
page_url_df = pd.DataFrame(page_url_lst)


def get_template_1(url):
    data = []
    soup = BeautifulSoup(requests.get(url).content, "lxml")
    for e in soup.select("#tabs-publications em"):
        data.append(
            {
                "author": e.previous.get_text(strip=True)[:-1],
                "title": e.get_text(strip=True),
                "journal": e.next_sibling.get_text(strip=True),
                "source": url,
            }
        )
    return data


def get_template_2(url):
    data = []
    soup = BeautifulSoup(requests.get(url).text, "lxml")
    for a in soup.find_all("span", {"class": "fac_citation"}):
        data.append(
            {
                "author": a.find("b").get_text(),
                "title": a.find("i").get_text(strip=True),
                "journal": a.find("i").next_sibling.get_text(strip=True),
                "source": url,
            }
        )
    return data


all_data = []
for _, row in page_url_df.iterrows():
    print("Getting", row["url"])
    if row["template"] == 1:
        all_data.extend(get_template_1(row["url"]))
    elif row["template"] == 2:
        all_data.extend(get_template_2(row["url"]))


df_out = pd.DataFrame(all_data)

# print sample data
print(df_out.head().to_markdown())

Prints:

CodePudding user response：

You are need an iterable in an outer loop. One way would be to generate a tuple list from your existing dataframe columns and loop that. You can then have your conditional logic, simplified, within the loop.

import requests
from bs4 import BeautifulSoup
import pandas as pd

page_url1 = "https://www.vet.upenn.edu/research/centers-laboratories/research-laboratory/research-laboratory/anguera-laboratory"
page_url2 = "https://www.med.upenn.edu/apps/faculty/index.php/g20001100/p8866286"
page_url_lst = {"url": [page_url1, page_url2], "template": [1, 2]}
page_url_df = pd.DataFrame(page_url_lst)

data = []

with requests.Session() as s:
    for template, url in zip(
        page_url_df["template"].to_list(), page_url_df["url"].to_list()
    ):
        r = s.get(url)
        soup = BeautifulSoup(r.text, "lxml")

        if template == 1:
           
            for e in soup.select("#tabs-publications em"):
                data.append(
                    {
                        "author": e.previous.get_text(strip=True)[:-1],
                        "title": e.get_text(strip=True),
                        "journal": e.next_sibling.get_text(strip=True),
                        "source": url,
                    }
                )
        else:

            for a in soup.find_all("span", {"class": "fac_citation"}):
                data.append(
                    {
                        "author": a.find("b").get_text(),
                        "title": a.find("i").get_text(strip=True),
                        "journal": a.find("i").next_sibling.get_text(strip=True),
                        "source": url,
                    }
                )
print(data)