I have a number of text items contained in span tags that i need to extract. I am able to do this in a list comp in the class table-main__odds as shown. I need to get the same info fronm the table-main__odds coloured tag. The logic below does not return any values. any help is appreciated?

import requests
from bs4 import BeautifulSoup
import pandas as pd

url = 'https://www.betexplorer.com/soccer/england/premier-league/results/'
soup = BeautifulSoup(requests.get(url).content)
odds_raw = soup.find_all("td", class_="table-main__odds")
fav_odds_raw = soup.find_all("td",class_="table-main__odds colored")


odds = [o.get('data-odd') for o in odds_raw]

the desired result is a list with the values contained here in data-odd

CodePudding user response：

Try:

import requests
import pandas as pd
from bs4 import BeautifulSoup


url = "https://www.betexplorer.com/soccer/england/premier-league/results/"
soup = BeautifulSoup(requests.get(url).content, "html.parser")


def get_odd_or_text(td):
    if "data-odd" in td.attrs:
        return td["data-odd"]

    odd = td.select_one("[data-odd]")
    if odd:
        return odd["data-odd"]

    return td.get_text(strip=True)


all_data = []
for row in soup.select(".table-main tr:has(td)"):
    tds = [get_odd_or_text(td) for td in row.select("td")]
    round_ = row.find_previous("th").find_previous("tr").th.text
    all_data.append([round_, *tds])

df = pd.DataFrame(
    all_data, columns=["Round", "Match", "Score", "1", "X", "2", "Date"]
)
print(df.head().to_markdown(index=False))
df.to_csv('data.csv', index=False)

Prints:

and saves data.csv (screenshot from LibreOffice):

CodePudding user response：

The logic below does not return any values.

and yet, I see some values in your screenshot.

Anyway, if you mean that you want the odd attribute value from inner elements even if td itself doesn't have it, you can do this

odds = [
    o.get('data-odd') if o.get('data-odd') else (
        o.select_one('*[data-odd]').get('data-odd')
        if o.select('*[data-odd]') else None
    ) for o in odds_raw
]

output of print(odds):

['1.22', '6.75', '13.19', '1.71', '3.87', '4.97', '4.97', '3.72', '1.74', '2.17', '3.43', '3.41', '3.07', '3.35', '2.38', '2.09', '3.39', '3.71', '2.28', '3.42', '3.20', '7.55', '5.05', '1.40', '1.22', '6.96', '11.93', '1.76', '3.81', '4.65', '1.60', '4.00', '5.88', '2.06', '3.48', '3.66', '2.07', '3.72', '3.41', '6.81', '4.60', '1.47', '2.21', '3.43', '3.33', '2.35', '3.33', '3.13', '2.13', '3.55', '3.42', '2.52', '3.13', '3.05', '1.23', '6.68', '12.94', '8.85', '5.42', '1.34', '2.95', '3.39', '2.43', '2.30', '3.54', '3.06', '2.66', '3.21', '2.79', '4.35', '3.70', '1.83', '1.42', '4.98', '7.11', '2.37', '3.48', '2.98', '1.67', '3.90', '5.32', '1.39', '4.95', '8.17', '2.10', '3.28', '3.79', '3.95', '3.46', '1.99', '5.10', '4.23', '1.63', '3.81', '4.06', '1.87', '2.06', '3.65', '3.51', '2.96', '3.35', '2.45', '1.95', '3.60', '3.93', '2.38', '3.45', '2.98', '1.46', '4.66', '6.94', '1.72', '3.78', '4.98', '3.22', '3.44', '2.26', '3.73', '3.57', '2.01', '2.53', '3.72', '2.65', '2.06', '3.56', '3.59', '3.57', '3.56', '2.07', '1.63', '3.98', '5.53', '3.27', '3.50', '2.20', '2.57', '3.30', '2.83', '1.50', '4.28', '6.81', '1.12', '10.21', '21.18', '1.68', '3.96', '5.10', '1.71', '4.02', '4.73', '2.42', '3.39', '2.97', '1.43', '5.29', '6.58', '1.98', '3.69', '3.74', '3.17', '3.47', '2.27', '4.81', '3.71', '1.77', '2.92', '3.40', '2.46', '1.46', '4.76', '6.74', '2.33', '3.29', '3.22', '2.04', '3.35', '3.94', '4.48', '3.88', '1.78', '2.92', '3.29', '2.52', '1.41', '4.79', '7.85', '1.51', '4.54', '6.36', '9.09', '5.50', '1.33', '2.07', '3.53', '3.61', '2.78', '3.38', '2.58', '4.80', '3.55', '1.86', '4.50', '4.10', '1.80', '1.32', '6.10', '10.00', '2.38', '3.55', '3.10', '1.92', '3.56', '4.21', '2.65', '3.47', '2.67', '9.06', '5.77', '1.32', '2.14', '3.56', '3.40', '1.67', '3.99', '5.28', '7.70', '5.15', '1.39', '2.08', '3.41', '3.74', '2.18', '3.34', '3.55', '1.52', '4.52', '6.20', '2.15', '3.46', '3.46', '3.24', '3.73', '2.16', '1.53', '4.46', '6.10', '3.31', '3.26', '2.31', '1.28', '6.16', '10.14', '1.09', '11.85', '27.62', '3.27', '3.54', '2.20', '2.08', '3.45', '3.69', '3.32', '3.24', '2.31', '2.11', '3.59', '3.47', '5.26', '4.07', '1.65', '2.10', '3.45', '3.65', '6.95', '4.75', '1.45', '2.55', '3.28', '2.90', '1.39', '5.26', '7.55', '2.09', '3.42', '3.71', '1.74', '3.87', '4.74', '1.40', '5.05', '7.72', '1.14', '8.87', '18.56', '1.19', '7.51', '15.15', '3.63', '3.90', '1.96', '4.58', '4.32', '1.69', '6.02', '4.57', '1.52', '6.95', '4.90', '1.44', '2.50', '3.35', '2.91', '7.61', '4.70', '1.43', '2.49', '3.22', '3.03', '1.91', '3.51', '4.35', '2.65', '3.36', '2.73', '1.76', '4.08', '4.40', '1.45', '4.76', '7.13', '1.27', '6.11', '11.79', '2.16', '3.46', '3.44', '4.02', '3.60', '1.94', '1.56', '4.36', '5.87', '1.78', '3.61', '4.96', '4.39', '3.86', '1.81', '2.43', '3.26', '3.08', '1.08', '12.60', '31.88', '2.44', '3.54', '2.86', '2.20', '3.39', '3.41', '1.90', '3.65', '4.10', '1.78', '3.74', '4.70', '7.95', '5.24', '1.38', '4.01', '3.46', '1.99', '6.52', '4.05', '1.56', '10.45', '6.10', '1.28', '2.43', '3.34', '3.00', '1.58', '4.04', '6.19', '1.36', '5.20', '8.69', '4.58', '3.57', '1.84']

Or if you wanted the odd attribute value from only the span tags,

odds = [ 
    o.select_one('span[data-odd]').get('data-odd')
    if o.select('span[data-odd]') else None
    for o in odds_raw
]

now, the output of print(odds) would be

['1.22', None, None, '1.71', None, None, None, None, '1.74', None, '3.43', None, '3.07', None, None, '2.09', None, None, None, '3.42', None, None, None, '1.40', None, None, '11.93', '1.76', None, None, '1.60', None, None, '2.06', None, None, None, None, '3.41', None, '4.60', None, None, None, '3.33', None, None, '3.13', None, '3.55', None, '2.52', None, None, '1.23', None, None, '8.85', None, None, '2.95', None, None, '2.30', None, None, None, None, '2.79', None, '3.70', None, '1.42', None, None, '2.37', None, None, '1.67', None, None, None, '4.95', None, '2.10', None, None, None, None, '1.99', None, None, '1.63', '3.81', None, None, None, '3.65', None, None, '3.35', None, None, '3.60', None, None, '3.45', None, '1.46', None, None, '1.72', None, None, '3.22', None, None, None, '3.57', None, '2.53', None, None, '2.06', None, None, None, None, '2.07', '1.63', None, None, '3.27', None, None, None, None, '2.83', '1.50', None, None, '1.12', None, None, '1.68', None, None, '1.71', None, None, None, '3.39', None, '1.43', None, None, '1.98', None, None, None, '3.47', None, None, None, '1.77', None, None, '2.46', None, '4.76', None, None, None, '3.22', '2.04', None, None, None, None, '1.78', '2.92', None, None, None, '4.79', None, '1.51', None, None, None, None, '1.33', '2.07', None, None, None, None, '2.58', None, None, None, None, None, None, None, None, None, None, None, None, '1.92', None, None, '2.65', None, None, None, '5.77', None, '2.14', None, None, '1.67', None, None, None, '5.15', None, None, '3.41', None, None, None, '3.55', '1.52', None, None, '2.15', None, None, None, None, '2.16', '1.53', None, None, None, '3.26', None, '1.28', None, None, '1.09', None, None, None, '3.54', None, None, '3.45', None, '3.32', None, None, None, '3.59', None, '5.26', None, None, None, None, '3.65', None, None, '1.45', None, '3.28', None, '1.39', None, None, None, '3.42', None, '1.74', None, None, '1.40', None, None, '1.14', None, None, '1.19', None, None, None, None, '1.96', '4.58', None, None, '6.02', None, None, None, '4.90', None, None, None, '2.91', None, None, '1.43', '2.49', None, None, None, '3.51', None, '2.65', None, None, None, None, '4.40', '1.45', None, None, None, '6.11', None, None, '3.46', None, '4.02', None, None, '1.56', None, None, '1.78', None, None, '4.39', None, None, None, '3.26', None, '1.08', None, None, None, '3.54', None, None, '3.39', None, None, '3.65', None, None, None, '4.70', None, None, '1.38', '4.01', None, None, None, None, '1.56', None, '6.10', None, '2.43', None, None, '1.58', None, None, '1.36', None, None, None, None, '1.84']

If you want to filter out the Nones, just add

odds = [o for o in odds if o is not None]