%% Replication code for adding daily and additionally sourced data series
%  Daily WEI series: Adjusted daily tax withholding, Rasmussen Consumer
%  Confidence Index, Unadjusted daily tax withholding

%  Additional weekly series: Bloomberg Consumer Comfort Index and Sum of
%  Initial and Continuing Claims

% LICENSE FOR CODE:
% Copyright Federal Reserve Bank of New York and Federal Reserve Bank of Dallas.
% You may reproduce, use, modify, make derivative works of, and distribute this code in whole or in part 
% so long as you keep this notice in the documentation associated with any distributed works. 
% Neither the names of the Federal Reserve Bank of New York and Federal Reserve Bank of Dallas nor the names 
% of any of the authors may be used to endorse or promote works derived from this 
% code without prior written permission. Portions of the code attributed to third 
% parties are subject to applicable third party licenses and rights. By your 
% use of this code you accept this license and any applicable third party license.
% THIS CODE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT ANY WARRANTIES OR CONDITIONS 
% OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY WARRANTIES
% OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A 
% PARTICULAR PURPOSE, EXCEPT TO THE EXTENT THAT THESE DISCLAIMERS ARE HELD TO 
% BE LEGALLY INVALID. THE FEDERAL RESERVE BANK OF NEW YORK AND THE FEDERAL 
% RESERVE BANK OF DALLAS ARE NOT, UNDER ANY CIRCUMSTANCES, LIABLE TO YOU FOR
% DAMAGES OF ANY KIND ARISING OUT OF OR IN CONNECTION WITH USE OF OR INABILITY
% TO USE THE CODE, INCLUDING, BUT NOT LIMITED TO DIRECT, INDIRECT, INCIDENTAL,
% CONSEQUENTIAL, PUNITIVE, SPECIAL OR EXEMPLARY DAMAGES, WHETHER BASED ON BREACH
% OF CONTRACT, BREACH OF WARRANTY, TORT OR OTHER LEGAL OR EQUITABLE THEORY, EVEN
% IF THE FEDERAL RESERVE BANK OF NEW YORK OR THE FEDERAL RESERVE BANK OF DALLAS
% HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES OR LOSS AND REGARDLESS
% OF WHETHER SUCH DAMAGES OR LOSS IS FORESEEABLE.
%% Preliminaries
clear; close all; clc;

%% Subset Dates
%  Note: Change if dates in Data/Data_WEI_Alt_spec.csv updated

startDate = datenum(2004, 12, 26); % Week to start series with (Note: Start of week, Sunday)
endDate   = datenum(2020, 12, 27); % Week to end series with   (Note: Start of week, Sunday)

%% Transforming Rasmussen Consumer Confidence to a Weekly Series

% Clean Rasmussen CSV file
rasRaw                 = readcell(fullfile('..', 'Data', 'rasmussen_econ_data.csv'));
indexStart             = find(strcmp(rasRaw(:, 1), 'DATE')); % Where do the column headers start
rasTable               = cell2table(rasRaw(indexStart+1:end, :), ... % Converting cell array to table
                         'VariableNames', rasRaw(indexStart, :));

rasTable.DateClean = datenum(extractAfter(rasTable.DATE, ', ')); % Cleaning dates and sorting
rasTable           = sortrows(rasTable, 'DateClean');

rasTable.CONSUMERS(isnan(rasTable.INVESTORS)) = {NaN}; % cleaning consumer index (setting NaNs based on investors index because of issue with strings)
rasTable.CONSUMERS                            = cell2mat(rasTable{:, 'CONSUMERS'});
rasTable                                      = rasTable(rasTable.DateClean >= startDate, :);

% Converting Table to Time Table
rasTT  = array2timetable(rasTable.CONSUMERS, 'RowTimes', datetime(rasTable.DateClean, 'ConvertFrom', 'datenum'));
rasTT  = retime(rasTT, 'daily', 'previous'); % Filling missing values with previous value

% Converting Daily to Weekly
rasTT  = retime(rasTT, 'weekly', 'mean');

% Subsetting Data
rasDates  = datenum(rasTT.Time);
keepIndex = rasDates >= startDate & rasDates <= endDate; 
rasSeries = rasTT.Var1(keepIndex);

%% Transforming daily adjusted tax withholding data to a weekly series

% Clean daily adjusted tax withholding data
taxTable                          = readtable(fullfile('..', 'Data', 'daily_withholding_adjusted.csv'));
taxTable.Properties.VariableNames = {'Date', 'Value'};

taxTable.Value = taxTable.Value .* 100; % Converting to percent
taxTable.Date  = datenum(taxTable.Date); % Converting to MATLAB datenum
if taxTable.Date(1) > startDate % Adjusting table if first date is greater than start date
   taxSlice       = taxTable(1, :);
   taxSlice.Date  = startDate;
   taxSlice.Value = NaN;
   taxTable       = [taxSlice; taxTable]; 
end

taxTable.Date  = taxTable.Date - 2; % Shifting dates to take mean on Tuesday - Monday week rather than Sunday-Saturday
taxTable.Date  = datetime(year(taxTable.Date) ,  month(taxTable.Date), day(taxTable.Date)); % Converting to datetime
taxTable       = table2timetable(taxTable, 'RowTimes', 'Date');

% Converting table to Time Table
taxTT = retime(taxTable,'daily', 'previous'); % Filling missing values with previous value
taxTT = retime(taxTT, 'weekly', 'mean');

% Subsetting Dates
taxDates  = datenum(taxTT.Properties.RowTimes);
keepIndex = taxDates >= startDate & taxDates <= endDate;
taxSeries = taxTT.Value(keepIndex);


%% Transforming unadjusted daily withholding to a weekly series

% Reading in unadjusted daily tax withholding
unadjTaxRaw = readtable(fullfile('..', 'Data','daily_withholding_unadjusted.csv'));

% Cleaning data
unadjTaxRaw.Date = datenum(unadjTaxRaw.Date); % Converting dates to MATLAB datenums

% Converting Table to Time Table
unadjTT  = array2timetable(unadjTaxRaw.TDW, 'RowTimes', datetime(unadjTaxRaw.Date, 'ConvertFrom', 'datenum'));
unadjTT  = retime(unadjTT, 'daily', 'previous'); % Filling missing values with previous value

% Converting Daily to Weekly
unadjTT  = retime(unadjTT, 'weekly', 'mean');

% Subsetting Data
unadjDates    = datenum(unadjTT.Time);
keepIndex     = unadjDates >= startDate & unadjDates <= endDate; 
unadjSeries   = unadjTT.Var1(keepIndex);

%% Adjusting Bloomberg Consumer Comfort Index 

% Read in raw Bloomberg Consumer Comfort Data 
bccRaw  = readcell(fullfile('..', 'Data', 'bloomberg_consumer_comfort.csv'));

% Cleaning data
bccData = bccRaw(7:end, 1:2); % Subsetting to just pull dates and index
bccData = array2table(bccData, 'VariableNames' , {'Date', 'COMF'}); % Converting to table

bccData.Date = cellfun(@(x) x, bccData.Date); % Converting dates to MATLAB datenums
bccData.Date = datenum(bccData.Date);

bccData.COMF  = cellfun(@(x) x, bccData.COMF); % Fixing data formatting

bccData.Date = bccData.Date - 7; % Shifting reference period back since Sunday in Bloomberg reflects Saturday to Sunday not start Sunday to Saturday

% Subsetting data
keepIndex = bccData.Date >= startDate & bccData.Date <= endDate;
bccSeries = bccData.COMF(keepIndex);
%% Writing Series to Data WEI Alt Spec

DataWEI = readtable(fullfile('..','Data', 'Data_WEI_Alt_Spec.csv'));

% Adding Rasmussen consumer confidence
colName = 'RAS';
DataWEI.(colName) = rasSeries;
% Adding Adjusted Tax Withholding data
colName = 'WITH';
DataWEI.(colName) = taxSeries;
% Adding Unadjusted Tax Withholding data
colName = 'TDW';
DataWEI.(colName) = unadjSeries;
% Adding Bloomberg Consumer Comfort Index
colName = 'COMF';
DataWEI.(colName) = bccSeries;
% Adding Sum of Initial and Continuing Claims
colName = 'CLAIMS';
DataWEI.(colName) = DataWEI.LICN + DataWEI.LIUN;

% Writing Data
writetable(DataWEI, fullfile('..', 'Data', 'Data_WEI_Alt_Spec.csv'));

% NOTE: Manually inputted series information into Series Info sheet in
% Data/series_info.csv
