How to append the data in DataFrame-CodePudding

I want the data in 'data frame' the code is working perfectly please solve these issue and provide data in Data Frame I try to solve it but failure to do these


            from selenium import webdriver
            from bs4 import BeautifulSoup
            import pandas as pd
            import time
                
            browser = webdriver.Chrome('F:\chromedriver.exe')
            browser.get("https://capitalonebank2.bluematrix.com/sellside/Disclosures.action")
            
            headers = {
                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.3"}
            
            for title in browser.find_elements_by_css_selector('option'):
                title.click()
                time.sleep(1)
                browser.switch_to.frame(browser.find_elements_by_css_selector("iframe")[1])
                table = browser.find_element_by_css_selector("table table")
                
                soup = BeautifulSoup(table.get_attribute("innerHTML"), "lxml")
                all_data = []
                ratings = {"BUY":[], "HOLD":[], "SELL":[]}
                lists_ = []
                for row in soup.select("tr")[-4:-1]:
                    info_list = row.select("td")
                    count = info_list[1].text
                    percent = info_list[2].text
                    
                    IBServ_count = info_list[4].text
                    IBServ_percent = info_list[5].text
                    
                    lists_.append([count, percent, IBServ_count, IBServ_percent])
                    
                ratings["BUY"] = lists_[0]
                ratings["HOLD"] = lists_[1]
                ratings["SELL"] = lists_[2]

CodePudding user response：

I tried a lot to get output from your code but didn't get because of huge bug. Actually, it's almost impossible to get desired result from that code. However, it's my best trying and working solution.

Code:

from bs4 import BeautifulSoup
import time
from selenium import webdriver
import pandas as pd

data = []
driver = webdriver.Chrome('chromedriver.exe')
#driver.maximize_window()
driver.set_window_size(1920, 1080)
time.sleep(10)

url = 'https://capitalonebank2.bluematrix.com/sellside/Disclosures.action'
driver.get(url)
time.sleep(5)

title = driver.find_elements_by_xpath('//option')
title[0].click()
time.sleep(5)

driver.switch_to.frame(driver.find_elements_by_css_selector("iframe")[1])
table = driver.find_element_by_xpath('//*[@bgcolor="#ffffff"]/table/tbody/tr/td/table')
time.sleep(5)

driver.find_element_by_xpath('//*[@bgcolor="#ffffff"]//table//tr[6]')


soup = BeautifulSoup(driver.page_source, 'lxml')

trs = soup.select('table table tr')
for tr in trs[3:8]:
  
    data.append(tr.stripped_strings)
df = pd.DataFrame(data).to_csv('table_data.csv', index = False)
#print(df)

Output:

0      1        2      3        4
0     Rating  Count  Percent  Count  Percent
1       None   None     None   None     None
2   BUY [OW]     89    67.94     41    46.07
3  HOLD [EW]     42    32.06     12    28.57
4  SELL [UW]      0     0.00      0     0.00

Output in csv:

Rating  Count   Percent Count   Percent
BUY [OW]    89  67.94   41  46.07
HOLD [EW]   42  32.06   12  28.57
SELL [UW]   0     0      0   0