I am trying to run a SARIMAX model on some mortgage prepayment data. I have a list of dataframes clustered by mortgage cohort and separate them into train and test sets based on time. Then I scale the train and test sets and run a step-wise autoarima function to come up with the best p, d, and q values for the SARIMAX I'd like to run on each cohort. I have this code here:
from pmdarima.arima import auto_arima
from statsmodels.tsa.statespace.sarimax import SARIMAX
from sklearn.preprocessing import MinMaxScaler
import re
columns = feature_df.columns.tolist()
cols = [c for c in columns if c not in ['ScheduledBalance', 'SMM', 'SRCDate', 'cluster', 'PredictionDate', 'CprTarget', 'bondsec_code', 'Coupon']]
i = 1
mapes = []
new_dfs = []
for df in dfs[0:1]:
if df.empty is False:
df.index = df['SRCDate']
#print(df.index)
train = df[df['SRCDate'] <= max(df['SRCDate']) - relativedelta(months = 3)]
test = df[df['SRCDate'] > max(df['SRCDate']) - relativedelta(months = 3)]
X_train = train[cols]
y_train = train['CprTarget']
X_test = test[cols]
y_test = test['CprTarget']
scaler = MinMaxScaler(feature_range=(0, 1))
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)
scaler_output = scaler.fit_transform(feature_df[['CprTarget']])
scaler_output =pd.DataFrame(scaler_output)
train_size=int(len(X_train))
test_size = int(len(y_test))
print(f"For {df['cluster'].unique()}")
step_wise = auto_arima(y_train,
exogenous= X_train,
start_p=1, start_q=1,
max_p=7, max_q=7,
d=1, max_d=7,
error_action='ignore',
suppress_warnings=True,
stepwise=True)
model = SARIMAX(y_train,
exog=X_train,
order=step_wise.get_params().get('order'),
enforce_invertibility=False, enforce_stationarity=False)
results = model.fit()
predictions = results.predict(start = train_size, end=train_size+test_size,exog=X_test)
actuals = pd.DataFrame(scaler_output.iloc[train_size:, 0])
predictions=pd.DataFrame(predictions)
predictions.reset_index(drop=True, inplace=True)
predictions.index=X_test.index
predictions['Actual'] = actuals['CprTarget']
predictions.rename(columns={0:'Pred'}, inplace=True)
predictions['Actual'].plot(figsize=(20,8), legend=True, color='blue')
predictions['Pred'].plot(legend=True, color='red', figsize=(20,8))
And this is the dataframe I am testing. Here is the traceback for the error I'm getting:
ValueError Traceback (most recent call last)
File ~\Anaconda3\lib\site-packages\statsmodels\tsa\statespace\mlemodel.py:1775, in MLEModel._validate_out_of_sample_exog(self, exog, out_of_sample)
1774 try:
-> 1775 exog = exog.reshape(required_exog_shape)
1776 except ValueError:
ValueError: cannot reshape array of size 620 into shape (74,20)
During handling of the above exception, another exception occurred:
ValueError Traceback (most recent call last)
Input In [128], in <cell line: 10>()
45 model = SARIMAX(y_train,
46 exog=X_train,
47 order=step_wise.get_params().get('order'),
48 enforce_invertibility=False, enforce_stationarity=False)
50 results = model.fit()
---> 52 predictions = results.predict(start = train_size, end=train_size+test_size,exog=X_test)
54 actuals = pd.DataFrame(scaler_output.iloc[train_size:, 0])
56 predictions=pd.DataFrame(predictions)
File ~\Anaconda3\lib\site-packages\statsmodels\base\wrapper.py:113, in make_wrapper.<locals>.wrapper(self, *args, **kwargs)
111 obj = data.wrap_output(func(results, *args, **kwargs), how[0], how[1:])
112 elif how:
--> 113 obj = data.wrap_output(func(results, *args, **kwargs), how)
114 return obj
File ~\Anaconda3\lib\site-packages\statsmodels\tsa\statespace\mlemodel.py:3403, in MLEResults.predict(self, start, end, dynamic, **kwargs)
3357 """
3358 In-sample prediction and out-of-sample forecasting
3359
(...)
3400 including confidence intervals.
3401 """
3402 # Perform the prediction
-> 3403 prediction_results = self.get_prediction(start, end, dynamic, **kwargs)
3404 return prediction_results.predicted_mean
File ~\Anaconda3\lib\site-packages\statsmodels\tsa\statespace\mlemodel.py:3302, in MLEResults.get_prediction(self, start, end, dynamic, index, exog, extend_model, extend_kwargs, **kwargs)
3299 extend_model = (self.model.exog is not None or
3300 not self.filter_results.time_invariant)
3301 if out_of_sample and extend_model:
-> 3302 kwargs = self.model._get_extension_time_varying_matrices(
3303 self.params, exog, out_of_sample, extend_kwargs,
3304 transformed=True, includes_fixed=True, **kwargs)
3306 # Make sure the model class has the current parameters
3307 self.model.update(self.params, transformed=True, includes_fixed=True)
File ~\Anaconda3\lib\site-packages\statsmodels\tsa\statespace\sarimax.py:1718, in SARIMAX._get_extension_time_varying_matrices(self, params, exog, out_of_sample, extend_kwargs, transformed, includes_fixed, **kwargs)
1708 """
1709 Get time-varying state space system matrices for extended model
1710
(...)
1714 special handling in the `simple_differencing=True` case.
1715 """
1717 # Get the appropriate exog for the extended sample
-> 1718 exog = self._validate_out_of_sample_exog(exog, out_of_sample)
1720 # Get the tmp endog, exog
1721 if self.simple_differencing:
File ~\Anaconda3\lib\site-packages\statsmodels\tsa\statespace\mlemodel.py:1777, in MLEModel._validate_out_of_sample_exog(self, exog, out_of_sample)
1775 exog = exog.reshape(required_exog_shape)
1776 except ValueError:
-> 1777 raise ValueError('Provided exogenous values are not of the'
1778 ' appropriate shape. Required %s, got %s.'
1779 % (str(required_exog_shape),
1780 str(exog.shape)))
1781 elif self.k_exog > 0 and exog is not None:
1782 exog = None
ValueError: Provided exogenous values are not of the appropriate shape. Required (74, 20), got (31, 20).
I'm not sure what I need to do to fix this.
