Fix #1278, round 3: redux from round 2 PR

pdr_backend/sim/sim_engine.py

@@ -58,7 +58,8 @@ def __init__(
         else:
             self.multi_id = str(uuid.uuid4())
 
-        self.model: Optional[Aimodel] = None
+        self.model_UP: Optional[Aimodel] = None
+        self.model_DOWN: Optional[Aimodel] = None
 
     @property
     def predict_feed(self) -> ArgFeed:
@@ -95,71 +96,133 @@ def run(self):
     def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame):
         ppss, pdr_ss, st = self.ppss, self.ppss.predictoor_ss, self.st
         transform = pdr_ss.aimodel_data_ss.transform
-        stake_amt = pdr_ss.stake_amount.amt_eth
         others_stake = pdr_ss.others_stake.amt_eth
         revenue = pdr_ss.revenue.amt_eth
+        model_conf_thr = self.ppss.trader_ss.sim_confidence_threshold
 
         testshift = ppss.sim_ss.test_n - test_i - 1  # eg [99, 98, .., 2, 1, 0]
         data_f = AimodelDataFactory(pdr_ss)  # type: ignore[arg-type]
         predict_feed = self.predict_train_feedset.predict
         train_feeds = self.predict_train_feedset.train_on
 
         # X, ycont, and x_df are all expressed in % change wrt prev candle
-        X, ytran, yraw, x_df, _ = data_f.create_xy(
+        p = predict_feed
+        predict_feed_close = p
+        predict_feed_high = ArgFeed(p.exchange, "high", p.pair, p.timeframe)
+        predict_feed_low = ArgFeed(p.exchange, "low", p.pair, p.timeframe)
+        _, _, yraw_close, _, _ = data_f.create_xy(
             mergedohlcv_df,
             testshift,
-            predict_feed,
+            predict_feed_close,
             train_feeds,
         )
-        colnames = list(x_df.columns)
-
-        st_, fin = 0, X.shape[0] - 1
-        X_train, X_test = X[st_:fin, :], X[fin : fin + 1, :]
-        ytran_train, _ = ytran[st_:fin], ytran[fin : fin + 1]
+        X_UP, ytran_UP, yraw_high, x_df_high, _ = data_f.create_xy(
+            mergedohlcv_df,
+            testshift,
+            predict_feed_high,
+            train_feeds,
+        )
+        X_DOWN, ytran_DOWN, yraw_low, _, _ = data_f.create_xy(
+            mergedohlcv_df,
+            testshift,
+            predict_feed_low,
+            train_feeds,
+        )
+        colnames_high = list(x_df_high.columns)
 
-        cur_high, cur_low = data_f.get_highlow(mergedohlcv_df, predict_feed, testshift)
+        cur_close, next_close = yraw_close[-2], yraw_close[-1]
+        cur_high, next_high = yraw_high[-2], yraw_high[-1]
+        cur_low, next_low = yraw_low[-2], yraw_low[-1]
 
-        cur_close = yraw[-2]
-        next_close = yraw[-1]
+        st_, fin = 0, X_UP.shape[0] - 1
+        X_train_UP, X_test_UP = X_UP[st_:fin, :], X_UP[fin : fin + 1, :]
+        ytran_train_UP, _ = ytran_UP[st_:fin], ytran_UP[fin : fin + 1]
+        X_train_DOWN, X_test_DOWN = X_DOWN[st_:fin, :], X_DOWN[fin : fin + 1, :]
+        ytran_train_DOWN, _ = ytran_DOWN[st_:fin], ytran_DOWN[fin : fin + 1]
 
+        percent_change_needed = 0.002  # magic number. TODO: move to ppss.yaml
         if transform == "None":
-            y_thr = cur_close
+            y_thr_UP = cur_close * (1 + percent_change_needed)
+            y_thr_DOWN = cur_close * (1 - percent_change_needed)
         else:  # transform = "RelDiff"
-            y_thr = 0.0
-        ytrue = data_f.ycont_to_ytrue(ytran, y_thr)
+            y_thr_UP = +np.std(yraw_close) * percent_change_needed
+            y_thr_DOWN = -np.std(yraw_close) * percent_change_needed
+        ytrue_UP = data_f.ycont_to_ytrue(ytran_UP, y_thr_UP)
+        ytrue_DOWN = data_f.ycont_to_ytrue(ytran_DOWN, y_thr_DOWN)
 
-        ytrue_train, _ = ytrue[st_:fin], ytrue[fin : fin + 1]
+        ytrue_train_UP, _ = ytrue_UP[st_:fin], ytrue_UP[fin : fin + 1]
+        ytrue_train_DOWN, _ = ytrue_DOWN[st_:fin], ytrue_DOWN[fin : fin + 1]
 
         if (
-            self.model is None
+            self.model_UP is None
             or self.st.iter_number % pdr_ss.aimodel_ss.train_every_n_epochs == 0
         ):
             model_f = AimodelFactory(pdr_ss.aimodel_ss)
-            self.model = model_f.build(X_train, ytrue_train, ytran_train, y_thr)
+            self.model_UP = model_f.build(
+                X_train_UP,
+                ytrue_train_UP,
+                ytran_train_UP,
+                y_thr_UP,
+            )
+            self.model_DOWN = model_f.build(
+                X_train_DOWN,
+                ytrue_train_DOWN,
+                ytran_train_DOWN,
+                y_thr_DOWN,
+            )
 
         # current time
         recent_ut = UnixTimeMs(int(mergedohlcv_df["timestamp"].to_list()[-1]))
         timeframe: ArgTimeframe = predict_feed.timeframe  # type: ignore
         ut = UnixTimeMs(recent_ut - testshift * timeframe.ms)
 
         # predict price direction
-        prob_up: float = self.model.predict_ptrue(X_test)[0]  # in [0.0, 1.0]
-        prob_down: float = 1.0 - prob_up
-        conf_up = (prob_up - 0.5) * 2.0  # to range [0,1]
-        conf_down = (prob_down - 0.5) * 2.0  # to range [0,1]
-        conf_threshold = self.ppss.trader_ss.sim_confidence_threshold
-        pred_up: bool = prob_up > 0.5 and conf_up > conf_threshold
-        pred_down: bool = prob_up < 0.5 and conf_down > conf_threshold
-        st.probs_up.append(prob_up)
+        prob_up_UP = self.model_UP.predict_ptrue(X_test_UP)[0]
+        prob_up_DOWN = self.model_DOWN.predict_ptrue(X_test_DOWN)[0]
+        prob_down_DOWN = 1.0 - prob_up_DOWN
+
+        models_in_conflict = (prob_up_UP > 0.5 and prob_down_DOWN > 0.5) or \
+            (prob_up_UP < 0.5 and prob_down_DOWN < 0.5)
+        if models_in_conflict:
+            conf_up = conf_down = 0.0
+            pred_up = pred_down = False
+            prob_up_MERGED = 0.5
+        elif prob_up_UP >= prob_down_DOWN:
+            conf_up = (prob_up_UP - 0.5) * 2.0  # to range [0,1]
+            conf_down = 0.0
+            pred_up = conf_up > model_conf_thr
+            pred_down = False
+            prob_up_MERGED = prob_up_UP
+        else: # prob_down_DOWN > prob_up_UP
+            conf_up = 0.0
+            conf_down = (prob_down_DOWN - 0.5) * 2.0
+            pred_up = False
+            pred_down = conf_down > model_conf_thr
+            prob_up_MERGED = 1.0 - prob_down_DOWN
+
+        st.probs_up_UP.append(prob_up_UP)
+        st.ytrues_hat_UP.append(prob_up_UP > 0.5)
+        st.probs_up_DOWN.append(prob_up_DOWN)
+        st.ytrues_hat_DOWN.append(prob_up_DOWN > 0.5)
+        st.probs_up_MERGED.append(prob_up_MERGED)
 
         # predictoor: (simulate) submit predictions with stake
         acct_up_profit = acct_down_profit = 0.0
-        stake_up = stake_amt * prob_up
-        stake_down = stake_amt * (1.0 - prob_up)
+        max_stake_amt = pdr_ss.stake_amount.amt_eth
+        if models_in_conflict or not (pred_up or pred_down):
+            stake_up = stake_down = 0
+        elif prob_up_UP >= prob_down_DOWN:
+            stake_amt = max_stake_amt * conf_up
+            stake_up = stake_amt * prob_up_MERGED
+            stake_down = stake_amt * (1.0 - prob_up_MERGED)
+        else: # prob_down_DOWN > prob_up_UP
+            stake_amt = max_stake_amt * conf_down
+            stake_up = stake_amt * prob_up_MERGED
+            stake_down = stake_amt * (1.0 - prob_up_MERGED)
         acct_up_profit -= stake_up
         acct_down_profit -= stake_down
 
-        profit = self.trader.trade_iter(
+        trader_profit = self.trader.trade_iter(
             cur_close,
             pred_up,
             pred_down,
@@ -169,43 +232,46 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame):
             cur_low,
         )
 
-        st.trader_profits_USD.append(profit)
-
-        # observe true price
-        true_up = next_close > cur_close
-        st.ytrues.append(true_up)
-
-        # update classifier metrics
-        n_correct = sum(np.array(st.ytrues) == np.array(st.ytrues_hat))
-        n_trials = len(st.ytrues)
-        acc_est = n_correct / n_trials
-        acc_l, acc_u = proportion_confint(count=n_correct, nobs=n_trials)
-        (precision, recall, f1, _) = precision_recall_fscore_support(
-            st.ytrues,
-            st.ytrues_hat,
-            average="binary",
-            zero_division=0.0,
-        )
-        if min(st.ytrues) == max(st.ytrues):
-            loss = 3.0
+        st.trader_profits_USD.append(trader_profit)
+
+        # observe true price change
+        true_up_close = next_close > cur_close
+
+        if transform == "None":
+            true_up_UP = next_high > y_thr_UP
+            true_up_DOWN = next_low > y_thr_DOWN
         else:
-            loss = log_loss(st.ytrues, st.probs_up)
+            raise NotImplementedError("build me")
+        st.ytrues_UP.append(true_up_UP)
+        st.ytrues_DOWN.append(true_up_DOWN)
+
+        # update classifier performances
+        perf_UP = get_perf(st.ytrues_UP, st.ytrues_hat_UP, st.probs_up_UP)
+        perf_DOWN = get_perf(st.ytrues_DOWN, st.ytrues_hat_DOWN, st.probs_up_DOWN)
+        perf_merged = merge_tups(perf_UP, perf_DOWN)
+        (acc_est, acc_l, acc_u, precision, recall, f1, loss) = perf_merged
+
         yerr = 0.0
-        if self.model.do_regr:
-            pred_ycont = self.model.predict_ycont(X_test)[0]
+        if self.model_UP.do_regr:
+            pred_ycont_UP = self.model_UP.predict_ycont(X_test_UP)[0]
+            pred_ycont_DOWN = self.model_DOWN.predict_ycont(X_test_DOWN)[0]
             if transform == "None":
-                pred_next_close = pred_ycont
+                pred_next_high = pred_ycont_UP
+                pred_next_low = pred_ycont_DOWN
             else:  # transform = "RelDiff"
                 relchange = pred_ycont
-                pred_next_close = cur_close + relchange * cur_close
-            yerr = next_close - pred_next_close
+                pred_next_high = cur_high + relchange * cur_high
+                pred_next_low = cur_low + relchange * cur_low
+            yerr_UP = next_high - pred_next_high
+            yerr_DOWN = next_low - pred_next_low
+            yerr = np.mean([yerr_UP, yerr_DOWN])
 
         st.aim.update(acc_est, acc_l, acc_u, f1, precision, recall, loss, yerr)
 
         # track predictoor profit
-        tot_stake = others_stake + stake_amt
+        tot_stake = others_stake + stake_up + stake_down
         others_stake_correct = others_stake * pdr_ss.others_accuracy
-        if true_up:
+        if true_up_close:
             tot_stake_correct = others_stake_correct + stake_up
             percent_to_me = stake_up / tot_stake_correct
             acct_up_profit += (revenue + tot_stake) * percent_to_me
@@ -221,16 +287,16 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame):
         save_state, is_final_state = self.save_state(test_i, self.ppss.sim_ss.test_n)
 
         if save_state:
-            colnames = [shift_one_earlier(colname) for colname in colnames]
-            most_recent_x = X[-1, :]
+            cs_ = [shift_one_earlier(c_) for c_ in colnames_high]
+            most_recent_x = X_UP[-1, :]
             slicing_x = most_recent_x  # plot about the most recent x
             d = AimodelPlotdata(
-                self.model,
-                X_train,
-                ytrue_train,
-                ytran_train,
-                y_thr,
-                colnames,
+                self.model_UP,
+                X_train_UP,
+                ytrue_train_UP,
+                ytran_train_UP,
+                y_thr_UP,
+                cs_,
                 slicing_x,
             )
             self.st.iter_number = test_i
@@ -257,3 +323,27 @@ def save_state(self, i: int, N: int):
             return False, False
 
         return True, False
+
+def get_perf(ytrues, ytrues_hat, probs_up) -> tuple:
+    """Get classifier performances: accuracy, precision/recall/f1, log loss"""
+    n_correct = sum(np.array(ytrues) == np.array(ytrues_hat))
+    n_trials = len(ytrues)
+    acc_est = n_correct / n_trials
+    acc_l, acc_u = proportion_confint(count=n_correct, nobs=n_trials)
+
+    (precision, recall, f1, _) = precision_recall_fscore_support(
+        ytrues,
+        ytrues_hat,
+        average="binary",
+        zero_division=0.0,
+    )
+
+    if min(ytrues) == max(ytrues):
+        loss = 3.0 # magic number
+    else:
+        loss = log_loss(ytrues, probs_up)
+
+    return (acc_est, acc_l, acc_u, precision, recall, f1, loss)
+
+def merge_tups(tup1, tup2):
+    return (np.mean([val1, val2]) for val1, val2 in zip(tup1, tup2))

pdr_backend/sim/sim_logger.py

@@ -20,8 +20,9 @@ def __init__(self, ppss, st, test_i, ut, acct_up_profit, acct_down_profit):
         self.acct_up_profit = acct_up_profit
         self.acct_down_profit = acct_down_profit
 
-        self.n_correct = sum(np.array(st.ytrues) == np.array(st.ytrues_hat))
-        self.n_trials = len(st.ytrues)
+        # TODO: account for DOWN too
+        self.n_correct = sum(np.array(st.ytrues_UP) == np.array(st.ytrues_hat_UP))
+        self.n_trials = len(st.ytrues_UP)
 
         for key, item in st.recent_metrics(extras=["prob_up"]).items():
             setattr(self, key, item)

pdr_backend/sim/sim_plotter.py

@@ -159,7 +159,7 @@ def plot_trader_profit_vs_time(self):
 
     @enforce_types
     def plot_pdr_profit_vs_ptrue(self):
-        x = self.st.probs_up
+        x = self.st.probs_up_MERGED
         y = self.st.pdr_profits_OCEAN
         fig = go.Figure(
             go.Scatter(
@@ -179,7 +179,7 @@ def plot_pdr_profit_vs_ptrue(self):
 
     @enforce_types
     def plot_trader_profit_vs_ptrue(self):
-        x = self.st.probs_up
+        x = self.st.probs_up_MERGED
         y = self.st.trader_profits_USD
         fig = go.Figure(
             go.Scatter(

pdr_backend/sim/sim_state.py

@@ -74,9 +74,18 @@ def __init__(self):
     def init_loop_attributes(self):
         # 'i' is iteration number i
 
-        # base data
-        self.ytrues: List[bool] = []  # [i] : was-truly-up
-        self.probs_up: List[float] = []  # [i] : predicted-prob-up
+        # base data for UP classifier
+        self.ytrues_UP: List[bool] = []  # [i] : true value
+        self.ytrues_hat_UP: List[bool] = []  # [i] : model pred. value
+        self.probs_up_UP: List[float] = []  # [i] : model's pred. prob.
+
+        # base data for DOWN classifier
+        self.ytrues_DOWN: List[bool] = []  # [i] : true value
+        self.ytrues_hat_DOWN: List[bool] = []  # [i] : model pred. value
+        self.probs_up_DOWN: List[float] = []  # [i] : model's pred. prob.
+
+        # merged values
+        self.probs_up_MERGED: List[float] = []  # [i] : merged pred. prob.
 
         # aimodel metrics
         self.aim = AimodelMetrics()
@@ -105,14 +114,10 @@ def recent_metrics(
         )
 
         if extras and "prob_up" in extras:
-            rm["prob_up"] = self.probs_up[-1]
+            rm["prob_up"] = self.probs_up_UP[-1] # FIXME: account for DOWN
 
         return rm
 
-    @property
-    def ytrues_hat(self) -> List[bool]:
-        return [p > 0.5 for p in self.probs_up]
-
     @property
     def n_correct(self) -> int:
         return sum((p > 0.5) == t for p, t in zip(self.probs_up, self.ytrues))