IndexError: tuple index out of range (LSTM)-CodePudding

I was following a python project to loosely predict the price of stocks when I encountered an index error with an LSTM model. This is the guide I was following and my code roughly matches: Prediction Tutorial. But for ease of access this is my code exactly:

import pandas as pd
import numpy as np
from keras.models import Sequential
from keras.layers import LSTM, Dropout, Dense
from sklearn.preprocessing import MinMaxScaler
import matplotlib.pyplot as plt
from matplotlib.pylab import rcParams
rcParams['figure.figsize'] = 20, 10

df = pd.read_csv('HistoricalData_Apple.csv')
df = df[['Date', 'Close/Last']]
df = df.replace({'\$':''}, regex=True)
df = df.astype({"Close/Last": float})
df["Date"] = pd.to_datetime(df.Date, format="%m/%d/%Y")
df.index = df['Date']

df = df.sort_index(ascending=True, axis=0)
data = pd.DataFrame(index=range(0, len(df)), columns=['Date', 'Close/Last'])

for i in range(0, len(data)):
    data["Date"][i] = df['Date'][i]
    data["Close/Last"][i] = df["Close/Last"][i]

scaler = MinMaxScaler(feature_range=(0, 1))

data.index = data.Date
data.drop("Date", axis=1, inplace=True)

final_data = data.values
train_data = final_data[0:200, :]
valid_data = final_data[200:, :]

scaler = MinMaxScaler(feature_range=(0, 1))

scaled_data = scaler.fit_transform(final_data)
x_train_data, y_train_data = [], []
for i in range(60, len(train_data)):
    x_train_data.append(scaled_data[i-60:i, 0])
    y_train_data.append(scaled_data[i, 0])

lstm_model = Sequential()
lstm_model.add(LSTM(units=50, return_sequences=True, input_shape= 
(np.shape(x_train_data)[1], 1)))
lstm_model.add(LSTM(units=50))
lstm_model.add(Dense(1))

model_data = data[len(data)-len(valid_data)-60:].values
model_data = model_data.reshape(-1, 1)
model_data = scaler.transform(model_data)

lstm_model.compile(loss='mean_squared_error', optimizer='adam')
lstm_model.fit(x_train_data, y_train_data, epochs=1, batch_size=1, verbose=2)
X_test=[]
for i in range(60, model_data.shape[0]):
    X_test.append(model_data[i-60:i, 0])
X_test = np.array(X_test)
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))

predicted_stock_price = lstm_model.predict(X_test)
predicted_stock_price = scaler.inverse_transform(predicted_stock_price)

train_data = data[:200]
valid_data = data[200:]
valid_data['Predictions'] = predicted_stock_price
plt.plot(train_data["Close"])
plt.plot(valid_data[['Close', "Predictions"]])
plt.show()

This code should be working according to the tutorial I was following, but every time I run the code I receive this error message:

2021-11-08 14:57:34.659018: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations:  AVX AVX2 To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. 

Traceback (most recent call last): File "C:/Users/ME/AppData/Roaming/JetBrains/PyCharmCE2021.2/scratches/scratch_1.py", line 42, in <module> lstm_model.add(LSTM(units=50, return_sequences=True, input_shape=(np.shape(x_train_data)[1], 1))) IndexError: tuple index out of range

I do not know what this means exactly or how to fix it.

CodePudding user response：

The expected input shape of the LSTM layer is (batch, timesteps, features). As you have only one feature you can reshape your training sequences as follows

x_train = x_train.reshape(x_train.shape[0], x_train.shape[1], 1)

as you already did for the validation sequences.

import pandas as pd
import numpy as np
import yfinance as yf
from keras.models import Sequential
from keras.layers import LSTM, Dense
from sklearn.preprocessing import MinMaxScaler
pd.options.mode.chained_assignment = None

# download the data
df = yf.download(tickers=['AAPL'], period='10y')

# split the data
train_data = df[['Close']].iloc[: - 200, :]
valid_data = df[['Close']].iloc[- 200:, :]

# scale the data
scaler = MinMaxScaler(feature_range=(0, 1))
scaler.fit(train_data)

train_data = scaler.transform(train_data)
valid_data = scaler.transform(valid_data)

# extract the training sequences
x_train, y_train = [], []

for i in range(60, train_data.shape[0]):
    x_train.append(train_data[i - 60: i, 0])
    y_train.append(train_data[i, 0])

x_train = np.array(x_train)
y_train = np.array(y_train)

# extract the validation sequences
x_valid = []

for i in range(60, valid_data.shape[0]):
    x_valid.append(valid_data[i - 60: i, 0])

x_valid = np.array(x_valid)

# reshape the sequences
x_train = x_train.reshape(x_train.shape[0], x_train.shape[1], 1)
x_valid = x_valid.reshape(x_valid.shape[0], x_valid.shape[1], 1)

# train the model
model = Sequential()
model.add(LSTM(units=50, return_sequences=True, input_shape=x_train.shape[1:]))
model.add(LSTM(units=50))
model.add(Dense(1))

model.compile(loss='mean_squared_error', optimizer='adam')
model.fit(x_train, y_train, epochs=50, batch_size=128, verbose=1)

# generate the model predictions
y_pred = model.predict(x_valid)
y_pred = scaler.inverse_transform(y_pred)
y_pred = y_pred.flatten()

# plot the model predictions
df.rename(columns={'Close': 'Actual'}, inplace=True)
df['Predicted'] = np.nan
df['Predicted'].iloc[- y_pred.shape[0]:] = y_pred
df[['Actual', 'Predicted']].plot(title='AAPL')