#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""

These are some examples of using the inner functions to generate individual results.

In addition, was used in practice for generating the results of the CV (cross-validation) analysis.

"""
import numpy as np
import pandas as pd
import sklearn.neural_network as skl_nn
import sklearn.cross_decomposition as cd
import sklearn.svm as skl_svm
import sklearn.linear_model as skl_lin
import matplotlib.pyplot as plt
import os
import configparser
import json
import itertools
import statsmodels.api as sm
from rich.console import Console
from rich.table import Table
import src.generalutils as gutil
import src.fcastutils as futil
import src.algfcast as algfcast
import src.figsandtables as fgstabs
import src.mlfcast as mlfcast
import importlib
importlib.reload(gutil)
importlib.reload(algfcast)
importlib.reload(fgstabs)
importlib.reload(mlfcast)
importlib.reload(futil)

config = configparser.ConfigParser()
config.optionxform = str
config.read('config.ini')

# init rich
console = Console()

# Init Bm data
bmdata = gutil.BenchmarkData(localTrafos=True)


def time_series_extent_tab():
    """
    Time series limitations - prints out first/last
    appearing index entry for monthly
    """
    dfBM = bmdata.returnAllSeries('M')
    nonmetrics = list(dict(config['proxies_u'])) + \
        list(dict(config['proxies_s']))
    dfBM = dfBM[nonmetrics]
    table = Table(title="First and last entries of proxies")
    table.add_column("Series name", justify="right",
                     style="cyan", no_wrap=True)
    table.add_column("First", style="magenta")
    table.add_column("Last", style="green")
    for col in dfBM.columns:
        table.add_row(col,
                      str(dfBM[col].dropna().head(1).index.values[0])[:10],
                      str(dfBM[col].dropna().tail(1).index.values[0])[:10])
    console.print(table)


def check_uncert_counts_at_per_art_level():
    """
    Check how many 'uncert' counts there are at the per article level
    """
    df = pd.read_csv(os.path.join(config['data']['intermed'],
                                  'DAIMmet_per_art.csv'))
    console.print(
        f"There are {100*len(df.loc[df['word_count_uncertain'] > 0])/len(df):.2f}")


time_series_extent_tab()
check_uncert_counts_at_per_art_level()


#   Settings
(target, metric, horizon, paper,
    trafo, alpha, stepSize, expanding) = ('CPIall', 'opinion', 3, 'GRDN',
                                          'none', 3, 1, False)
packagedSettings = (target, metric, horizon, paper,
                    trafo, alpha, stepSize, expanding)
metricList = ['opinion', 'tf_idf_econom']
nonmetrics = ['MGDP', 'ABJR.Q']
# ------------------------------------------------------------
# General utilities
# ------------------------------------------------------------
df = gutil.getTimeSeries(metric, target, paper, freq='M')
print(df.sample(20))
df = gutil.getTimeSeries(metric, target, paper, freq='M',
                         bmLocalTrafo=False)
print(df.sample(20))
df[target].plot()
plt.show()
(df[target].pct_change(12)*1.0e2).plot()
plt.show()
(np.log(df[target])).plot()
plt.show()
df = gutil.getTimeSeries(metricList, target, paper, freq='M')
print(df.sample(20))
df = gutil.getTimeSeries(metricList[0], list(gutil.allProxiesDict()), paper,
                         freq='M')
print(df.sample(20))
df = gutil.getTimeSeries(metricList, target, 'COMB', freq='M')
print(df.sample(20))
df = gutil.getTFTimeSeries(nonmetrics, paper,
                           norm=False, roll=False,
                           bmLocalTrafo=True, freq='M')
print(df.sample(20))
df = gutil.combinePaperTimeSeries(freqIn='M')
print(df.sample(20))
df = gutil.combineWithBMData(df, metricList, nonmetrics,
                             norm=False, roll=False,
                             bmLocalTrafo=True, freq='M')
print(gutil.paperFname(paper))
df = gutil.loadMetricsData(gutil.paperFname(paper))
print(df.sample(20))
# ------------------------------------------------------------
# Algo fcasts
# ------------------------------------------------------------
(target, metric, horizon, paper,
 trafo, alpha, stepSize, expanding) = ('CPIall',
                                       'stability',
                                       3,
                                       'GRDN',
                                       'none', 36, 1, False)
fcastSettings = (target, metric, horizon, paper,
                 trafo, alpha, stepSize, expanding)
# AR1 test
res = algfcast.predictvsAR1(*fcastSettings)
sumResHere = futil.runVSbchmarkData(res, horizon, expanding)
namesVars = json.loads(config.get("runSettings", "runSettings"))
for i, col in enumerate(namesVars[:len(fcastSettings)]):
    sumResHere[col] = fcastSettings[i]
# Factor model
res = algfcast.predictsvsFactorModel(*fcastSettings)
sumResHere = futil.runVSbchmarkData(res, horizon, expanding)
namesVars = json.loads(config.get("runSettings", "runSettings"))
for i, col in enumerate(namesVars[:len(fcastSettings)]):
    sumResHere[col] = fcastSettings[i]

# ------------------------------------------------------------
# Figures and tables
# ------------------------------------------------------------
# Example run plot
target = 'CPIall'
metric = 'tf_idf_econom'
horizon = 6
paper = 'GRDN'
trafo = 'none'
alpha = 36
stepSize = 1
expanding = False
CV = False
model = 'OLS'
specification = 'met_vs_ar1'
run_type = 'metric'
sentiment = True
saveAnalysis = False
fcastSettings = (target, metric, horizon, paper,
                 trafo, alpha, stepSize, expanding)
res = algfcast.predictvsAR1(*fcastSettings)
sumResHere = futil.runVSbchmarkData(res, horizon, expanding)
namesVars = json.loads(config.get("runSettings", "runSettings"))
for i, col in enumerate(namesVars[:len(fcastSettings)]):
    sumResHere[col] = fcastSettings[i]
fgstabs.plottingGunkTS(res,
                       target, metric, horizon, paper,
                       trafo, alpha, stepSize, expanding,
                       CV, model, specification, run_type,
                       saveAnalysis, sentiment)
fgstabs.plottingGunkTS(res,
                       target, metric, horizon, paper,
                       trafo, alpha, stepSize, expanding,
                       CV, model, specification, 'benchmark',
                       saveAnalysis, sentiment)
# ----------------------------------------------
# ML run and forecast
# ----------------------------------------------
# In-depth test of some ML models
yvar = 'CPIall'
ML_models_to_try = [
    skl_lin.Lasso(alpha=1., fit_intercept=False),
    skl_svm.SVR(C=800, gamma='auto', epsilon=0.),
    skl_lin.ElasticNet(
        alpha=1., l1_ratio=0.5, fit_intercept=False),
    skl_lin.Ridge(alpha=1, fit_intercept=False),
    skl_nn.MLPRegressor((2, 2), alpha=0.1,
                        activation='tanh',
                        solver='lbfgs',
                        max_iter=2000),
]
for ML_model in ML_models_to_try:
    print('Model name:   ' + type(ML_model).__name__)
    # Cut off point:
    cut_off = 150
    df = gutil.getTFTimeSeries(yvar, 'DAIM',
                               freq='M')
    df['lagged_yvar'] = df[yvar].shift(1)
    df[yvar] = df[yvar].shift(-3)
    df = df.dropna()
    Xvars = [x for x in df.columns if x != yvar]
    bchmrk_model = skl_lin.LinearRegression()
    bchmrk_model.fit(df['lagged_yvar'].iloc[:cut_off].values.reshape(-1, 1),
                     df[yvar].iloc[:cut_off])
    ML_model.fit(df[Xvars].iloc[:cut_off, :], df[yvar].iloc[:cut_off])
    df['prediction_ML'] = ML_model.predict(df[Xvars].iloc[:, :])
    df['prediction_ar1'] = (bchmrk_model
                            .predict(df['lagged_yvar'].values.reshape(-1, 1)))
    fig, ax = plt.subplots()
    df[['prediction_ML', yvar, 'prediction_ar1']].plot(ax=ax)
    ax.axvline(df[Xvars].index[cut_off])
    plt.show()
    print(sm.tools.eval_measures.rmse(df['prediction_ML'],
                                      df[yvar]))
    print('verus AR1 which got ' +
          str(sm.tools.eval_measures.rmse(df['prediction_ar1'],
                                          df[yvar])))
    print('----------------END--------------------------')
# AR1 - SVM/CPI/GRDN
saveAnalysis = False
sentiment = False
packageSettings = ('CPIall', 3, 'DAIM', 'none', 36,
                   1, False, False, 'SVM')
(yvar, horizon, paper,
 trafo, alpha, stepSize, expanding,
 CV, model) = packageSettings
results = mlfcast.predict_tf_vsAR1(*packageSettings)
run_type = 'metric'
metric = 'tf_matrix'
specification = 'tf_vs_AR1'
sumResHere = futil.runVSbchmarkData(results, horizon, expanding)
namesVars = json.loads(config.get("runSettings", "runSettings"))
for i, col in enumerate(namesVars[:len(packageSettings)]):
    sumResHere[col] = packageSettings[i]
fgstabs.plottingGunkTS(results,
                       yvar, metric, horizon, paper,
                       trafo, alpha, stepSize, expanding,
                       CV, model, specification, run_type,
                       saveAnalysis, sentiment)
fgstabs.plottingGunkTS(results,
                       yvar, metric, horizon, paper,
                       trafo, alpha, stepSize, expanding,
                       CV, model, specification, 'benchmark',
                       saveAnalysis, sentiment)

# FCTR
saveAnalysis = False
sentiment = False
packageSettings = ('MGDP', 9, 'DAIM', 'none', 36,
                   1, False, False, 'SVM')
(yvar, horizon, paper,
 trafo, alpha, stepSize, expanding,
 CV, model) = packageSettings
results = mlfcast.predict_tf_vsFactorModel(*packageSettings)
run_type = 'metric'
sumResHere = futil.runVSbchmarkData(results, horizon, expanding)
namesVars = json.loads(config.get("runSettings", "runSettings"))
for i, col in enumerate(namesVars[:len(packageSettings)]):
    sumResHere[col] = packageSettings[i]
fgstabs.plottingGunkTS(results,
                       yvar, metric, horizon, paper,
                       trafo, alpha, stepSize, expanding,
                       CV, model, specification, run_type,
                       saveAnalysis, sentiment)
fgstabs.plottingGunkTS(results,
                       yvar, metric, horizon, paper,
                       trafo, alpha, stepSize, expanding,
                       CV, model, specification, 'benchmark',
                       saveAnalysis, sentiment)
bmdata.returnSeries('M')['IOS']

# AR1 ML vs OLS AR1
saveAnalysis = False
sentiment = False
packageSettings = ('MGDP', 3, 'DAIM', 'none', 36,
                   1, False, False, 'SVM')
(yvar, horizon, paper,
 trafo, alpha, stepSize, expanding,
 CV, model) = packageSettings
results = mlfcast.predict_tf_vsAR1OLS(*packageSettings)
run_type = 'metric'
metric = 'tf_matrix'
specification = 'tf_vs_AR1OLS'
sumResHere = futil.runVSbchmarkData(results, horizon, expanding)
namesVars = json.loads(config.get("runSettings", "runSettings"))
for i, col in enumerate(namesVars[:len(packageSettings)]):
    sumResHere[col] = packageSettings[i]
fgstabs.plottingGunkTS(results,
                       yvar, metric, horizon, paper,
                       trafo, alpha, stepSize, expanding,
                       CV, model, specification, run_type,
                       saveAnalysis, sentiment)
fgstabs.plottingGunkTS(results,
                       yvar, metric, horizon, paper,
                       trafo, alpha, stepSize, expanding,
                       CV, model, specification, 'benchmark',
                       saveAnalysis, sentiment)
# AR1+fctrs ML vs OLS AR1+fctrs
saveAnalysis = False
sentiment = False
packageSettings = ('MGDP', 3, 'DMIR', 'none', 36,
                   1, False, False, 'NN')
(yvar, horizon, paper,
 trafo, alpha, stepSize, expanding,
 CV, model) = packageSettings
results = mlfcast.predict_tf_vsfctrOLS(*packageSettings)
run_type = 'metric'
metric = 'tf_matrix'
specification = 'tf_vs_fctrOLS'
sumResHere = futil.runVSbchmarkData(results, horizon, expanding)
namesVars = json.loads(config.get("runSettings", "runSettings"))
for i, col in enumerate(namesVars[:len(packageSettings)]):
    sumResHere[col] = packageSettings[i]
fgstabs.plottingGunkTS(results,
                       yvar, metric, horizon, paper,
                       trafo, alpha, stepSize, expanding,
                       CV, model, specification, run_type,
                       saveAnalysis, sentiment)
fgstabs.plottingGunkTS(results,
                       yvar, metric, horizon, paper,
                       trafo, alpha, stepSize, expanding,
                       CV, model, specification, 'benchmark',
                       saveAnalysis, sentiment)
# ML with CV
saveAnalysis = False
sentiment = False
packageSettings = ('MGDP', 3, 'DAIM', 'none', 36,
                   1, False, False, 'SVM')
(yvar, horizon, paper,
 trafo, alpha, stepSize, expanding,
 CV, model) = packageSettings
results = mlfcast.predict_tf_vsAR1OLS(*packageSettings)
run_type = 'metric'
metric = 'tf_matrix'
specification = 'tf_vs_AR1OLS'
sumResHere = futil.runVSbchmarkData(results, horizon, expanding)
namesVars = json.loads(config.get("runSettings", "runSettings"))
for i, col in enumerate(namesVars[:len(packageSettings)]):
    sumResHere[col] = packageSettings[i]
fgstabs.plottingGunkTS(results,
                       yvar, metric, horizon, paper,
                       trafo, alpha, stepSize, expanding,
                       CV, model, specification, run_type,
                       saveAnalysis, sentiment)
fgstabs.plottingGunkTS(results,
                       yvar, metric, horizon, paper,
                       trafo, alpha, stepSize, expanding,
                       CV, model, specification, 'benchmark',
                       saveAnalysis, sentiment)

# ----------------------------------------------
# In-depth test of some ML models with CV
modelsL = json.loads(config.get("runSettings", "MLmodels"))
alpha, stepSize, trafo, expanding, CV = 36, 1, 'none', False, False
argsCombos = [['CPIall'], [9], ['DMIR'],
              [trafo], [alpha], [stepSize], [expanding],
              [CV], modelsL]
argsCombos = list(itertools.product(*argsCombos))
saveAnalysis = False
sentiment = False
for setting in argsCombos:
    packageSettings = setting
    (yvar, horizon, paper,
     trafo, alpha, stepSize, expanding,
     CV, model) = setting
    print('Model name:   ' + model)
    results = mlfcast.predict_tf_vsAR1OLS(*setting)
    run_type = 'metric'
    metric = 'tf_matrix'
    specification = 'tf_vs_AR1OLS'
    sumResHere = futil.runVSbchmarkData(results, horizon, expanding)
    namesVars = json.loads(config.get("runSettings", "runSettings"))
    for i, col in enumerate(namesVars[:len(setting)]):
        sumResHere[col] = setting[i]
    fgstabs.plottingGunkTS(results,
                           yvar, metric, horizon, paper,
                           trafo, alpha, stepSize, expanding,
                           CV, model, specification, run_type,
                           saveAnalysis, sentiment)
    fgstabs.plottingGunkTS(results,
                           yvar, metric, horizon, paper,
                           trafo, alpha, stepSize, expanding,
                           CV, model, specification, 'benchmark',
                           saveAnalysis, sentiment)
    print('----------------END--------------------------')


# ----------------------------------------------
# Test of forecast env
xf = pd.DataFrame(np.array([['t-1', 't', 't+1', 't+2', 't+3', 't+4', 't+5'],
                            ['a', 'b', 'c', 'd', 'e', 'f', 'g'], [0, 1, 2, 3, 4, 5, 6]]).T,
                  columns=['Time', 'y', 'x'])
xf.index.name = 'time'
lag = 1
xf['ylag'] = xf['y'].shift(lag)
X_feat = xf[['x', 'ylag']]
alpha = 1
step = 1
mu_th_period = 3
futil.slice_insample_r(X_feat, mu_th_period, alpha, step)
futil.slice_oosample_r(X_feat, mu_th_period, alpha, step)
numSteps = np.int((len(xf)-step-alpha)/step)
allResultsDf = pd.DataFrame()
for mu in range(1, numSteps+1):
    print('\n -------------------')
    print('mu = '+str(mu)+' of '+str(numSteps)+', s = '+str(step))
    print('\n in-sample: \n')
    xf_IS = futil.slice_insample_r(X_feat, mu, alpha, step)
    xf_IS['mu'] = mu
    xf_IS['IS'] = True
    xf_IS['target'] = 'IS_y_hat ' + str(mu)
    print(xf_IS)
    print('\n and out-of-sample: \n')
    xf_OOS = futil.slice_oosample_r(X_feat, mu, alpha, step)
    xf_OOS['mu'] = mu
    xf_OOS['IS'] = False
    xf_OOS['target'] = 'OOS_y_hat ' + str(mu)
    print(xf_OOS)
    allResultsDf = pd.concat([allResultsDf, xf_IS, xf_OOS],
                             sort=True, axis=0)
final_results = futil.script_ISOOS_summary(allResultsDf)
print(final_results)


if __name__ == '__main__':
    # do nothing
