2 years ago
#71834
oladimeji
Incompatible shapes: [16,8,1] vs. [16,1] [[{{node loss_3/dense_7_loss/MeanSquaredError/sub}}]]
I am trying to predict the quantity of c02 using LSTM model in RNN deep learning
import numpy as np
from numpy import zeros, newaxis
import pandas as pd
import matplotlib.pyplot as plt
import scipy
from scipy import stats
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import Sequential, layers, callbacks
from tensorflow.keras.layers import LSTM, Dense, Input
from sklearn.preprocessing import StandardScaler
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from tensorflow.keras.layers import Dropout
from IPython.core.pylabtools import figsize
from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()
import seaborn as sns
df1 = pd.read_csv('CO2_O2_0_1.csv', header = 0)
1
df1.head()
Wavelength (cm-1) CO2:O2=0:1 (Absorbance)
0 1000.409 0.000104
1 1000.891 0.000085
2 1001.373 0.000053
3 1001.855 0.000111
4 1002.337 0.000251
1
df3 = pd.read_csv('CO2_O2_1_0.csv', header = 0)
2
df3.head()
Wavelength (cm-1) CO2:O2=1:0 (Absorbance)
0 1000.409 0.000148
1 1000.891 0.000137
2 1001.373 0.000052
3 1001.855 0.000004
4 1002.337 0.000108
1
df2 = pd.read_csv('CO2_O2_15_1.csv', header = 0)
2
df2.head()
Wavelength (cm-1) CO2:O2=15:1 (Absorbance)
0 1000.409 -0.000160
1 1000.891 -0.000038
2 1001.373 -0.000192
3 1001.855 -0.000356
4 1002.337 -0.000081
1
df4 = pd.read_csv('CO2_O2_1_1.csv')
2
df4.head()
Wavelength (cm-1) CO2:O2=1:1 (Absorbance)
0 1000.409 -0.000105
1 1000.891 -0.000178
2 1001.373 -0.000178
3 1001.855 -0.000148
4 1002.337 -0.000117
1
df5 = pd.read_csv('CO2_O2_1_3.csv')
2
df5.head()
Wavelength (cm-1) CO2:O2=1:3 (Absorbance)
0 1000.409 0.000109
1 1000.891 0.000107
2 1001.373 0.000184
3 1001.855 0.000163
4 1002.337 0.000012
1
df6 = pd.read_csv('CO2_O2_1_6.csv')
2
df6.head()
Wavelength (cm-1) CO2:O2=1:6 (Absorbance)
0 1000.409 0.000100
1 1000.891 0.000238
2 1001.373 0.000313
3 1001.855 0.000234
4 1002.337 0.000122
1
df7 = pd.read_csv('CO2_O2_3_1.csv')
2
df7.head()
Wavelength (cm-1) CO2:O2=3:1 (Absorbance)
0 1000.409 0.000048
1 1000.891 0.000033
2 1001.373 -0.000079
3 1001.855 -0.000263
4 1002.337 -0.000228
1
df8 = pd.read_csv('CO2_O2_5_1.csv')
2
df8.head()
Wavelength (cm-1) CO2:O2=5:1 (Absorbance)
0 1000.409 8.410000e-05
1 1000.891 -1.580000e-04
2 1001.373 -6.250000e-07
3 1001.855 1.560000e-04
4 1002.337 4.010000e-05
1
df = pd.concat([df1, df2, df3, df4, df5, df6, df7, df8], ignore_index=True, axis = 0, sort = False)
2
df.head()
Wavelength (cm-1) CO2:O2=0:1 (Absorbance) CO2:O2=15:1 (Absorbance) CO2:O2=1:0 (Absorbance) CO2:O2=1:1 (Absorbance) CO2:O2=1:3 (Absorbance) CO2:O2=1:6 (Absorbance) CO2:O2=3:1 (Absorbance) CO2:O2=5:1 (Absorbance)
0 1000.409 0.000104 NaN NaN NaN NaN NaN NaN NaN
1 1000.891 0.000085 NaN NaN NaN NaN NaN NaN NaN
2 1001.373 0.000053 NaN NaN NaN NaN NaN NaN NaN
3 1001.855 0.000111 NaN NaN NaN NaN NaN NaN NaN
4 1002.337 0.000251 NaN NaN NaN NaN NaN NaN NaN
1
df.shape
(1344, 9)
Missing Value Imputation
1
df['Wavelength (cm-1)'].fillna(value = df['Wavelength (cm-1)'].mean(), inplace = True)
2
df['CO2:O2=0:1 (Absorbance)'].fillna(value = df['CO2:O2=0:1 (Absorbance)'].mean(), inplace = True)
3
df['CO2:O2=15:1 (Absorbance)'].fillna(value = df['CO2:O2=15:1 (Absorbance)'].mean(), inplace = True)
4
df['CO2:O2=1:0 (Absorbance)'].fillna(value = df['CO2:O2=1:0 (Absorbance)'].mean(), inplace = True)
5
df['CO2:O2=1:1 (Absorbance)'].fillna(value = df['CO2:O2=1:1 (Absorbance)'].mean(), inplace = True)
6
df['CO2:O2=1:3 (Absorbance)'].fillna(value = df['CO2:O2=1:3 (Absorbance)'].mean(), inplace = True)
7
df['CO2:O2=1:6 (Absorbance)'].fillna(value = df['CO2:O2=1:6 (Absorbance)'].mean(), inplace = True)
8
df['CO2:O2=3:1 (Absorbance)'].fillna(value = df['CO2:O2=3:1 (Absorbance)'].mean(), inplace = True)
9
df['CO2:O2=5:1 (Absorbance)'].fillna(value = df['CO2:O2=5:1 (Absorbance)'].mean(), inplace = True)
1
df.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1344 entries, 0 to 1343
Data columns (total 9 columns):
Wavelength (cm-1) 1344 non-null float64
CO2:O2=0:1 (Absorbance) 1344 non-null float64
CO2:O2=15:1 (Absorbance) 1344 non-null float64
CO2:O2=1:0 (Absorbance) 1344 non-null float64
CO2:O2=1:1 (Absorbance) 1344 non-null float64
CO2:O2=1:3 (Absorbance) 1344 non-null float64
CO2:O2=1:6 (Absorbance) 1344 non-null float64
CO2:O2=3:1 (Absorbance) 1344 non-null float64
CO2:O2=5:1 (Absorbance) 1344 non-null float64
dtypes: float64(9)
memory usage: 94.6 KB
1
df.head()
Wavelength (cm-1) CO2:O2=0:1 (Absorbance) CO2:O2=15:1 (Absorbance) CO2:O2=1:0 (Absorbance) CO2:O2=1:1 (Absorbance) CO2:O2=1:3 (Absorbance) CO2:O2=1:6 (Absorbance) CO2:O2=3:1 (Absorbance) CO2:O2=5:1 (Absorbance)
0 1000.409 0.000104 0.000624 0.000282 0.001511 0.00179 0.002626 0.001083 0.000797
1 1000.891 0.000085 0.000624 0.000282 0.001511 0.00179 0.002626 0.001083 0.000797
2 1001.373 0.000053 0.000624 0.000282 0.001511 0.00179 0.002626 0.001083 0.000797
3 1001.855 0.000111 0.000624 0.000282 0.001511 0.00179 0.002626 0.001083 0.000797
4 1002.337 0.000251 0.000624 0.000282 0.001511 0.00179 0.002626 0.001083 0.000797
Outliers Treatment
1
#df[(np.abs(stats.zscore(df)) < 3).all(axis=1)]
1
df.shape
(1344, 9)
x,y split
1
x = df.loc[:,df.columns!="Wavelength (cm-1)"]
2
x.shape
(1344, 8)
1
y = df['Wavelength (cm-1)']
2
type(y)
pandas.core.series.Series
1
y.shape
(1344,)
1
x_train_full, x_test, y_train_full, y_test = train_test_split(x, y,test_size=0.2,random_state=42)
2
3
x_train, x_valid, y_train, y_valid = train_test_split(x_train_full, y_train_full, test_size = 0.2, random_state = 42)
4
x_train.head()
CO2:O2=0:1 (Absorbance) CO2:O2=15:1 (Absorbance) CO2:O2=1:0 (Absorbance) CO2:O2=1:1 (Absorbance) CO2:O2=1:3 (Absorbance) CO2:O2=1:6 (Absorbance) CO2:O2=3:1 (Absorbance) CO2:O2=5:1 (Absorbance)
1169 0.005092 0.000624 0.000282 0.001511 0.001790 0.002626 -0.000106 0.000797
1072 0.005092 0.000624 0.000282 0.001511 0.001790 0.002626 0.000616 0.000797
784 0.005092 0.000624 0.000282 0.001511 -0.000055 0.002626 0.001083 0.000797
375 0.005092 0.000624 -0.000137 0.001511 0.001790 0.002626 0.001083 0.000797
305 0.005092 -0.000090 0.000282 0.001511 0.001790 0.002626 0.001083 0.000797
1
np.random.seed(42)
2
tf.random.set_random_seed(42)
1
x_train.shape
(860, 8)
1
x_valid.shape
(215, 8)
1
x_valid_3d = x_valid.values.reshape(x_valid.shape[0],8,1).astype('float32')
2
x_valid_3d.shape
(215, 8, 1)
1
y_valid.shape
(215,)
1
#y_valid_3d = y_valid.values.reshape(y_valid.shape[0],1,1).astype('float32')
2
#y_valid_3d
1
x_test.shape
(269, 8)
1
y_train.shape
(860,)
1
y_test.shape
(269,)
1
x_train_3d = x_train.values.reshape(x_train.shape[0],8,1).astype('float32')
2
x_test_3d = x_test.values.reshape(x_test.shape[0],8,1).astype('float32')
1
x_test_3d.shape
(269, 8, 1)
1
sc = StandardScaler().fit(x_train)
2
sc_train_std = sc.transform(x_train)
3
sc_test_std = sc.transform(x_test)
4
sc_test_std
array([[ 1.78192582e-02, -5.73877445e-01, -1.17600227e-02, ...,
6.23458297e-03, 2.07732728e-02, 1.94397475e-02],
[ 1.78192582e-02, 8.53034545e+00, -1.17600227e-02, ...,
6.23458297e-03, 2.07732728e-02, 1.94397475e-02],
[ 1.78192582e-02, 5.48432794e-03, -1.17600227e-02, ...,
-1.47457945e+00, 2.07732728e-02, 1.94397475e-02],
...,
[ 1.78192582e-02, 5.48432794e-03, -9.56460901e-01, ...,
6.23458297e-03, 2.07732728e-02, 1.94397475e-02],
[ 1.78192582e-02, 5.48432794e-03, -1.17600227e-02, ...,
6.23458297e-03, 2.07732728e-02, -1.51494627e+00],
[-1.38528609e+00, 5.48432794e-03, -1.17600227e-02, ...,
6.23458297e-03, 2.07732728e-02, 1.94397475e-02]])
1
sc_test_std.shape
(269, 8)
1
sc_train_std.shape
(860, 8)
,1
1
model = Sequential()
2
model.add(LSTM(30, activation = 'relu', input_shape=[8,1], return_sequences = True))
3
model.add(Dense(30, activation='relu'))
4
model.add(Dense(1))
1
model.summary()
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
lstm_4 (LSTM) (None, 8, 30) 3840
_________________________________________________________________
dense_6 (Dense) (None, 8, 30) 930
_________________________________________________________________
dense_7 (Dense) (None, 8, 1) 31
=================================================================
Total params: 4,801
Trainable params: 4,801
Non-trainable params: 0
_________________________________________________________________
1
model.compile(loss='mse', optimizer='adam', metrics=['accuracy'])
1
checkpoint_cb = keras.callbacks.ModelCheckpoint("Best_Model.h5", save_best_only=True)
1
x_valid_3d.shape
(215, 8, 1)
1
x_train_3d.shape
(860, 8, 1)
16
1
model.fit(x_train_3d,y_train, batch_size=16, epochs =5, validation_data=(x_valid_3d,y_valid), callbacks=[checkpoint_cb], verbose=1)
Train on 860 samples, validate on 215 samples
Epoch 1/5
---------------------------------------------------------------------------
InvalidArgumentError Traceback (most recent call last)
<ipython-input-74-1e4223a8a839> in <module>()
----> 1 model.fit(x_train_3d,y_train, batch_size=16, epochs =5, validation_data=(x_valid_3d,y_valid), callbacks=[checkpoint_cb], verbose=1)
~\Anaconda3\lib\site-packages\tensorflow\python\keras\engine\training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, max_queue_size, workers, use_multiprocessing, **kwargs)
878 initial_epoch=initial_epoch,
879 steps_per_epoch=steps_per_epoch,
--> 880 validation_steps=validation_steps)
881
882 def evaluate(self,
~\Anaconda3\lib\site-packages\tensorflow\python\keras\engine\training_arrays.py in model_iteration(model, inputs, targets, sample_weights, batch_size, epochs, verbose, callbacks, val_inputs, val_targets, val_sample_weights, shuffle, initial_epoch, steps_per_epoch, validation_steps, mode, validation_in_fit, **kwargs)
327
328 # Get outputs.
--> 329 batch_outs = f(ins_batch)
330 if not isinstance(batch_outs, list):
331 batch_outs = [batch_outs]
~\Anaconda3\lib\site-packages\tensorflow\python\keras\backend.py in __call__(self, inputs)
3074
3075 fetched = self._callable_fn(*array_vals,
-> 3076 run_metadata=self.run_metadata)
3077 self._call_fetch_callbacks(fetched[-len(self._fetches):])
3078 return nest.pack_sequence_as(self._outputs_structure,
~\Anaconda3\lib\site-packages\tensorflow\python\client\session.py in __call__(self, *args, **kwargs)
1437 ret = tf_session.TF_SessionRunCallable(
1438 self._session._session, self._handle, args, status,
-> 1439 run_metadata_ptr)
1440 if run_metadata:
1441 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
~\Anaconda3\lib\site-packages\tensorflow\python\framework\errors_impl.py in __exit__(self, type_arg, value_arg, traceback_arg)
526 None, None,
527 compat.as_text(c_api.TF_Message(self.status.status)),
--> 528 c_api.TF_GetCode(self.status.status))
529 # Delete the underlying status object from memory otherwise it stays alive
530 # as there is a reference to status from this from the traceback due to
InvalidArgumentError: Incompatible shapes: [16,8,1] vs. [16,1]
[[{{node loss_3/dense_7_loss/MeanSquaredError/sub}}]]
python
deep-learning
conv-neural-network
lstm
recurrent-neural-network
0 Answers
Your Answer