For my master thesis, I am exploring the possibility to extract data from a website via web automation. The steps are as follows:

1. Sign in to the website (

   Your solution for sign in would look like

   driver = webdriver.Chrome(executable_path='C:\webdrivers\chromedriver.exe')
   driver.maximize_window()
   driver.get('https://www.metal.com/Nickel/201102250239')
   # Click on Sign In
   driver.find_element(By.XPATH, "//button[@class='button sign-in']").click()
   # Enter username
   driver.find_element(By.ID, "user_name").send_keys("your username")
   # Enter password
   driver.find_element(By.ID, "password").send_keys("your password") 
   # Click Sign In
   driver.find_element(By.XPATH, "//button[@type='submit']").click()
   

   To scrape data

   for element in driver.find_elements_by_class_name("historyBodyRow___1Bk9u"):
    elements =element.find_elements_by_tag_name("div")
    print("Date="  elements[0].text)
    print("Price Range="  elements[1].text)
    print("Avg="  elements[2].text)
    print("Change="  elements[3].text)
    print("Unit="  elements[4].text)
   

   Add To CSV

   import csv
   f = open('Path where you want to store the file', 'w')
   writer = csv.writer(f)
   for element in driver.find_elements_by_class_name("historyBodyRow___1Bk9u"):
     elements =element.find_elements_by_tag_name("div")
     entry = [elements[0].text ,elements[1].text ,elements[2].text , elements[3].text, elements[4].text]
     writer.writerow(entry)
   

   f.close