scratch_6.py

from sysdata.sim.db_futures_sim_data import dbFuturesSimData

from systems.basesystem import System
from systems.futures.rawdata import FuturesRawData
from systems.positionsizing import PositionSizing
from systems.portfolio import Portfolios
from systems.accounts.accounts_stage import Account

import pickle
import datetime
with open("/home/rob/groupings.pick", "rb") as f:
    pd_of_groupings = pickle.load(f)

from sysquant.fitting_dates import fitDates

from systems.forecast_combine import *

class GroupedForecastCombine(ForecastCombine
):
    def get_monthly_raw_forecast_weights_estimated(self, instrument_code: str) -> pd.DataFrame:
        # will only work if in memory
        pd_of_groups_index = list(pd_of_groupings.index) + [datetime.datetime.now()]
        stacked_weights = []
        index_for_stacked_weights = []
        for idx in range(len(pd_of_groups_index))[1:-1]:
            grouping = pd_of_groupings.iloc[idx]
            print("%s: %d", (instrument_code, idx))
            ## expanding window with 20 years
            fit_start = pd_of_groups_index[max(0, idx-20)]
            fit_end = pd_of_groups_index[idx]
            start_date = fit_end
            end_date = pd_of_groups_index[idx+1]

            fit_date = fitDates(fit_start,fit_end, start_date, end_date)

            optimiser = self.calculation_of_raw_estimated_monthly_forecast_weights(
                instrument_code, fit_date, grouping)
            weights = optimiser.weights()

            stacked_weights.append(weights)
            index_for_stacked_weights.append(start_date)

        stacked_weights_pd = pd.concat(stacked_weights)

        ## index will be wrong as will have the start of fitting period
        stacked_weights_pd.index = index_for_stacked_weights

        return stacked_weights_pd

    @dont_cache
    def calculation_of_raw_estimated_monthly_forecast_weights(self,
                                                              instrument_code,
                                                              fit_date,
                                                              grouping):
        config = self.config
        # Get some useful stuff from the config
        weighting_params = copy(config.forecast_weight_estimate)

        ## we're not really in sample, it's expanding window simulated in the calling
        ##   function
        weighting_params['date_method']='in_sample'

        # which function to use for calculation
        weighting_func = resolve_function(weighting_params.pop("func"))

        returns_pre_processor = self.returns_pre_processor_for_code(instrument_code,
                                                                    fit_date,
                                                                    grouping
                                                                    )
        net_returns =returns_pre_processor.get_net_returns(instrument_code)
        is_empty = net_returns.empty
        if is_empty:
            print("No data empty weights")
            return EqualWeights(returns_pre_processor, fit_date, instrument_code)

        weight_func = weighting_func(
            returns_pre_processor,
            asset_name=instrument_code,
            log=self.log,
            **weighting_params)

        return weight_func

    @dont_cache
    def returns_pre_processor_for_code(self, instrument_code,
                                       fit_date,
                                       grouping):
        # Because we might be pooling, we get a stack of p&l data
        codes_to_use = [code for code in grouping.keys() if grouping[code]==grouping[instrument_code]]
        trading_rule_list = self.cheap_trading_rules(instrument_code)

        returns_pre_processor = self.returns_pre_processing(codes_to_use,
                                                            fit_date,
                                                            trading_rule_list=trading_rule_list)

        return returns_pre_processor

    @dont_cache
    def returns_pre_processing(self, codes_to_use: list,
                               fit_date,
                               trading_rule_list: list,
                               ):
        pandl_forecasts = self.get_pandl_forecasts(codes_to_use, trading_rule_list, fit_date)
        turnovers = self.get_turnover_for_list_of_rules(codes_to_use, trading_rule_list)
        config = self.config

        weighting_params = copy(config.forecast_weight_estimate)
        cost_params = copy(config.forecast_cost_estimates)
        weighting_params = {**weighting_params, **cost_params}

        returns_pre_processor = returnsPreProcessor(pandl_forecasts,
                                                    turnovers = turnovers,
                                                    log=self.log,
                                                    **weighting_params)

        return returns_pre_processor

    @diagnostic()
    def get_pandl_forecasts(self, codes_to_use: list,
                            trading_rule_list: list,
                            fit_date):
        # returns a dict of accountCurveGroups
        pandl_forecasts = dict()
        for code in codes_to_use:
            returns=self.get_returns_for_optimisation(code,
                                                      trading_rule_list)
            returns.costs = returns.costs[fit_date.fit_start: fit_date.fit_end]
            returns.gross = returns.gross[fit_date.fit_start: fit_date.fit_end]

            pandl_forecasts[code] = returns



        pandl_forecasts = dictOfReturnsForOptimisationWithCosts(pandl_forecasts)

        return pandl_forecasts

from sysquant.optimisation.weights import one_over_n_weights_given_asset_names

class EqualWeights(object):
    def __init__(self, returns_pre_processor, fit_date, instrument_code):
        self.columns = list(returns_pre_processor.get_net_returns(instrument_code).columns)
        self.fit_date = fit_date
    def weights(self):
        weights = one_over_n_weights_given_asset_names(self.columns)

        return pd.DataFrame(dict(weights), index = [self.fit_date.period_start])


def grouped_futures_system():

    data = dbFuturesSimData()

    config = Config(
            "private.systems.carrytrend.buildconfig.yaml")


    system = System(
        [
            Account(),
            Portfolios(),
            PositionSizing(),
            FuturesRawData(),
            GroupedForecastCombine(),
            ForecastScaleCap(),
            Rules(),
        ],
        data,
        config,
    )
    system.set_logging_level("on")
    system.set_logging_level("on")
    system.config.forecast_weight_estimate['pool_gross_returns'] = True
    system.config.forecast_weight_estimate['equalise_SR'] = False
    system.config.use_forecast_weight_estimates = True
    system.config.forecast_weight_estimate['ceiling_cost_SR'] = 999
    system.config.instruments = ['AEX', 'AUD', 'SP500', 'BUND', "SHATZ",'BOBL','US10', 'US2','US5', 'EDOLLAR', 'CRUDE_W', 'GAS_US', 'CORN', 'WHEAT']

    system.config.use_forecast_scale_estimates = False


    rules = {x: system.config.trading_rules[x] for x in\
    ['momentum4', 'momentum8', 'momentum16', 'momentum32', 'momentum64', 'carry10']}

    system.config.trading_rules = rules

    return system

system = grouped_futures_system()
wts_dict = {}
for instrument in system.get_instrument_list():
    wts_dict[instrument] = system.combForecast.get_forecast_weights(instrument)

import pickle
with open("/home/rob/wts_with_groupings.pick", "wb") as f:
    pickle.dump(wts_dict, f)

system.cache.pickle("/home/rob/system_with_groupings.pck")