Can you scrape multiple stock prices on a recurring schedule?-CodePudding

With the current code, I can scrape multiple prices, but it doesn't automatically re-scrape them every 2 minutes which is what I need.

import requests
from bs4 import BeautifulSoup
import time
import pandas as pd

mystocks = ['GOOG', 'META', 'MSFT', 'PLTR', 'TSLA', 'ZS', 'PYPL', 'SHOP', 'TTCF']
stockdata = []


def getData(symbol):
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36'}
    url = f'https://finance.yahoo.com/quote/{symbol}'
    r = requests.get(url, headers=headers)
    soup = BeautifulSoup(r.text, 'html.parser')    
    stock = {
    'symbol': symbol,
    'price': soup.find('div', {'class':'D(ib) Mend(20px)'}).find_all('fin-streamer')[0].text,
    }
    return stock
        
for item in mystocks:
        stockdata.append(getData(item))
            
def export_data(stockdata):
    df = pd.DataFrame(stockdata)
    df.to_excel("LETS GO2.xlsx")
         
if __name__ == '__main__':
    while True:
        getData(item)
        export_data(stockdata)
        time_wait = 2
        print(f'Waiting {time_wait} minutes...')
        time.sleep(time_wait * 60)

CodePudding user response：

Your for-loop is at the wrong place.

Try to put it in your while True: block to loop over every ticker every two minutes.

EDIT:

import requests
from bs4 import BeautifulSoup
import time
import pandas as pd

mystocks = ['GOOG', 'META', 'MSFT', 'PLTR', 'TSLA', 'ZS', 'PYPL', 'SHOP', 'TTCF']


def getData(symbol):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36'}
    url = f'https://finance.yahoo.com/quote/{symbol}'
    r = requests.get(url, headers=headers)
    soup = BeautifulSoup(r.text, 'html.parser')
    stock = {
        'symbol': symbol,
        'price': soup.find('div', {'class': 'D(ib) Mend(20px)'}).find_all('fin-streamer')[0].text,
    }
    return stock


def export_data(stockdata):
    df = pd.DataFrame(stockdata)
    df.to_excel("LETS GO2.xlsx")


if __name__ == "__main__":
    while True:
        stockdata = []
        for item in mystocks:
            print(item)
            stockdata.append(getData(item))
        export_data(stockdata)
        time_wait = 0.1
        print(f'Waiting {time_wait} minutes...')
        time.sleep(time_wait * 60)