Fix core cash-drag by topping up existing positions

Author: Rohan5commit

main.py

@@ -372,15 +372,24 @@ def run_daily_test(self, test_date=None, pipeline_stats=None, backtest_signals=N
         open_positions_df = self.core_tracker.get_open_positions()
         open_symbols = set(open_positions_df['symbol']) if not open_positions_df.empty else set()
         max_positions = self.config.get('trading', {}).get('max_positions', 10)
+        trading_cfg = self.config.get("trading", {}) if isinstance(self.config, dict) else {}
+        risk_cfg = self.config.get("risk", {}) if isinstance(self.config, dict) else {}
+        short_threshold = float(trading_cfg.get("short_threshold", -0.0))
         available_slots = max(0, max_positions - len(open_symbols))
         invested_cost = 0.0
         if not open_positions_df.empty:
             invested_cost = (open_positions_df['entry_price'] * open_positions_df['quantity']).sum()
 
         available_capital = max(0.0, current_capital - invested_cost)
         cash_pct = (available_capital / current_capital) if current_capital else 0.0
-        max_cash_pct = float(self.config.get("trading", {}).get("max_cash_pct", 1.0))
+        max_cash_pct = float(trading_cfg.get("max_cash_pct", 1.0))
         max_cash_pct = max(0.0, min(1.0, max_cash_pct))
+        min_weight = float(trading_cfg.get("min_total_weight", 0.0) or 0.0)
+        min_weight = max(0.0, min(1.0, min_weight))
+        idle_cash_caps = [float(current_capital) * max_cash_pct]
+        if min_weight > 0.0:
+            idle_cash_caps.append(float(current_capital) * max(0.0, 1.0 - min_weight))
+        allowed_idle_cash = min(idle_cash_caps) if idle_cash_caps else 0.0
         cash_drag_excess = available_capital > 0 and cash_pct > max_cash_pct
 
         if available_slots == 0 or available_capital <= 0:
@@ -401,7 +410,6 @@ def run_daily_test(self, test_date=None, pipeline_stats=None, backtest_signals=N
             portfolio_mgr = PortfolioManager(self.config_path)
             enable_shorts = bool(self.config.get("trading", {}).get("enable_shorts", False))
             max_shorts = int(self.config.get("trading", {}).get("max_shorts", 0))
-            short_threshold = float(self.config.get("trading", {}).get("short_threshold", -0.0))
 
             long_df = rank_df_filtered[rank_df_filtered["adjusted_score"] > 0].copy()
             short_df = pd.DataFrame()
@@ -453,14 +461,14 @@ def run_daily_test(self, test_date=None, pipeline_stats=None, backtest_signals=N
                     target_portfolio["weight"] = target_portfolio["weight"] / total_weight
 
                 # Guardrail: enforce minimum total weight to avoid leaving too much cash idle
-                min_weight = float(self.config.get("trading", {}).get("min_total_weight", 0.0))
                 if min_weight > 0 and 0 < total_weight < min_weight:
                     target_portfolio["weight"] = target_portfolio["weight"] * (min_weight / total_weight)
                     logger.info(f"Adjusted weights from {total_weight:.2%} to {min_weight:.2%} to deploy more capital")
 
         # Open NEW positions for selected stocks
         tp_pct = self.config.get('trading', {}).get('take_profit_pct', 0.03)
         new_positions = []
+        top_up_positions = []
         remaining_capital = float(available_capital)
         for _, row in target_portfolio.iterrows():
             symbol = row['symbol']
@@ -521,6 +529,111 @@ def run_daily_test(self, test_date=None, pipeline_stats=None, backtest_signals=N
                     })
                     remaining_capital = max(0.0, remaining_capital - allocation_dollars)
 
+        # If older under-sized positions are occupying slots, scale into the best-held names
+        # before leaving excess cash idle.
+        extra_to_deploy = max(0.0, float(remaining_capital) - float(allowed_idle_cash))
+        max_position_equity_pct = float(risk_cfg.get("max_position_equity_pct", 1.0) or 1.0)
+        max_position_equity_pct = max(0.0, min(1.0, max_position_equity_pct))
+        if extra_to_deploy > 0.0 and max_position_equity_pct > 0.0:
+            open_positions_for_top_up = self.core_tracker.get_open_positions()
+            if open_positions_for_top_up is not None and not open_positions_for_top_up.empty:
+                top_up_df = open_positions_for_top_up.copy()
+                top_up_df["symbol"] = top_up_df["symbol"].astype(str).str.strip().str.upper()
+                top_up_df["side"] = top_up_df["side"].fillna("LONG").astype(str).str.upper()
+                top_up_df["entry_price"] = pd.to_numeric(top_up_df["entry_price"], errors="coerce").fillna(0.0)
+                top_up_df["quantity"] = pd.to_numeric(top_up_df["quantity"], errors="coerce").fillna(0.0)
+                top_up_df["notional"] = top_up_df["entry_price"] * top_up_df["quantity"]
+                top_up_df["adjusted_score"] = top_up_df["symbol"].map(
+                    lambda sym: float(rank_lookup.loc[sym]["adjusted_score"]) if sym in rank_lookup.index else 0.0
+                )
+                top_up_df["rank_priority"] = top_up_df["adjusted_score"].abs()
+
+                supported_mask = (
+                    ((top_up_df["side"] == "LONG") & (top_up_df["adjusted_score"] > 0))
+                    | ((top_up_df["side"] == "SHORT") & (top_up_df["adjusted_score"] <= short_threshold))
+                )
+                top_up_candidates = top_up_df[supported_mask].copy()
+                if top_up_candidates.empty:
+                    top_up_candidates = top_up_df.copy()
+
+                top_up_candidates = top_up_candidates.sort_values(
+                    ["rank_priority", "adjusted_score"],
+                    ascending=[False, False],
+                )
+
+                max_position_notional = float(current_capital) * max_position_equity_pct
+                logger.warning(
+                    "Core cash drag remains %.2f with idle-cash cap %.2f; topping up existing positions.",
+                    float(remaining_capital),
+                    float(allowed_idle_cash),
+                )
+                for _, pos in top_up_candidates.iterrows():
+                    if extra_to_deploy <= 0.0:
+                        break
+
+                    current_notional = float(pos.get("notional", 0.0) or 0.0)
+                    room = max(0.0, max_position_notional - current_notional)
+                    if room <= 0.0:
+                        continue
+
+                    symbol = str(pos["symbol"]).strip().upper()
+                    side = str(pos.get("side") or "LONG").upper()
+                    price_data = pd.read_sql(
+                        "SELECT open FROM prices WHERE symbol=? AND date=?",
+                        conn,
+                        params=(symbol, test_date.strftime("%Y-%m-%d")),
+                    )
+                    if price_data.empty:
+                        continue
+
+                    entry_price = float(price_data.iloc[0]["open"] or 0.0)
+                    if entry_price <= 0.0:
+                        continue
+
+                    allocation_dollars = min(extra_to_deploy, room)
+                    quantity = allocation_dollars / entry_price if entry_price else 0.0
+                    if quantity <= 0.0:
+                        continue
+
+                    added = self.core_tracker.add_to_position(
+                        symbol=symbol,
+                        add_date=test_date.strftime("%Y-%m-%d"),
+                        add_price=entry_price,
+                        quantity=quantity,
+                        side=side,
+                    )
+                    if not added:
+                        continue
+
+                    reason = "Cash-drag top-up of existing position"
+                    if symbol in rank_lookup.index:
+                        info = rank_lookup.loc[symbol]
+                        reason = (
+                            f"Cash-drag top-up (pred {float(info['predicted_return']):.2%}, "
+                            f"adj {float(info['adjusted_score']):.2%}, rank {int(info['rank'])})"
+                        )
+
+                    top_up_positions.append({
+                        "symbol": symbol,
+                        "side": side,
+                        "entry_price": entry_price,
+                        "target_price": added["target_price"],
+                        "quantity": quantity,
+                        "allocation_pct": (allocation_dollars / current_capital * 100) if current_capital else 0.0,
+                        "allocation_dollars": allocation_dollars,
+                        "reason": reason,
+                    })
+                    remaining_capital = max(0.0, remaining_capital - allocation_dollars)
+                    extra_to_deploy = max(0.0, extra_to_deploy - allocation_dollars)
+
+                if extra_to_deploy > 0.0:
+                    logger.warning(
+                        "Core cash drag persists after top-ups; %.2f cash still exceeds the idle-cash cap.",
+                        extra_to_deploy,
+                    )
+
+        entries_today = list(new_positions) + list(top_up_positions)
+
         # Keep Core + AI strategies distinct by optionally preventing overlap.
         core_reserved_symbols = set(open_symbols)
         if target_portfolio is not None and hasattr(target_portfolio, "empty") and not target_portfolio.empty:
@@ -579,6 +692,7 @@ def run_daily_test(self, test_date=None, pipeline_stats=None, backtest_signals=N
         report = {
             'date': test_date.date(),
             'new_positions_opened': len(new_positions),
+            'positions_topped_up': len(top_up_positions),
             'positions_closed_at_tp': len(closed_positions),
             'open_positions': summary['open_positions'],
             'realized_pnl_today': realized_today,
@@ -835,7 +949,7 @@ def _recent_price_metrics(conn_, sym_, lookback=25):
             report_data=report,
             unrealized_df=unrealized,
             closed_positions=closed_positions,
-            new_positions=new_positions,
+            new_positions=entries_today,
             meta_insights=meta_insights,
             signal_rankings=rank_df,
             pipeline_stats=core_pipeline_stats,

positions.py

@@ -36,6 +36,12 @@ def __init__(self, config_path=None, table_name="positions"):
         self.table_name = self._validate_table_name(table_name)
         self._init_tables()
 
+    def _target_price_for_side(self, entry_price, side):
+        side = str(side or "LONG").upper()
+        if side == "SHORT":
+            return float(entry_price) * (1 - self.tp_pct)
+        return float(entry_price) * (1 + self.tp_pct)
+
     @staticmethod
     def _validate_table_name(name: str) -> str:
         # Prevent SQL injection via table_name.
@@ -86,10 +92,7 @@ def open_position(self, symbol, entry_date, entry_price, quantity, side="LONG"):
         if side not in {"LONG", "SHORT"}:
             raise ValueError(f"Unsupported side: {side}")
 
-        if side == "LONG":
-            target_price = entry_price * (1 + self.tp_pct)
-        else:
-            target_price = entry_price * (1 - self.tp_pct)
+        target_price = self._target_price_for_side(entry_price, side)
 
         conn = sqlite3.connect(self.db_path)
         cursor = conn.cursor()
@@ -120,6 +123,84 @@ def open_position(self, symbol, entry_date, entry_price, quantity, side="LONG"):
         logger.info(f"Opened position #{position_id}: {symbol} {side} @ {entry_price:.2f}, TP @ {target_price:.2f}")
         return position_id
 
+    def add_to_position(self, symbol, add_date, add_price, quantity, side=None):
+        """Add capital to an existing open position and blend the average entry price."""
+        symbol = str(symbol or "").strip().upper()
+        if not symbol:
+            return None
+
+        try:
+            add_price = float(add_price or 0.0)
+            quantity = float(quantity or 0.0)
+        except (TypeError, ValueError):
+            return None
+
+        if add_price <= 0.0 or quantity <= 0.0:
+            return None
+
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+        existing = cursor.execute(
+            f"""
+            SELECT id, side, entry_date, entry_price, quantity
+            FROM {self.table_name}
+            WHERE symbol=? AND status='OPEN'
+            """,
+            (symbol,),
+        ).fetchone()
+
+        if not existing:
+            conn.close()
+            return None
+
+        pos_id, existing_side, existing_entry_date, existing_entry_price, existing_qty = existing
+        existing_side = str(existing_side or "LONG").upper()
+        if side is not None and str(side or "LONG").upper() != existing_side:
+            conn.close()
+            raise ValueError(f"Side mismatch for add_to_position({symbol})")
+
+        existing_entry_price = float(existing_entry_price or 0.0)
+        existing_qty = float(existing_qty or 0.0)
+        new_qty = existing_qty + quantity
+        if new_qty <= 0.0:
+            conn.close()
+            return None
+
+        blended_entry = ((existing_entry_price * existing_qty) + (add_price * quantity)) / new_qty
+        target_price = self._target_price_for_side(blended_entry, existing_side)
+
+        cursor.execute(
+            f"""
+            UPDATE {self.table_name}
+            SET entry_price=?, quantity=?, target_price=?
+            WHERE id=?
+            """,
+            (blended_entry, new_qty, target_price, pos_id),
+        )
+        conn.commit()
+        conn.close()
+
+        logger.info(
+            "Added to position #%s: %s %s +%.4f @ %.2f, new avg %.2f, TP @ %.2f",
+            pos_id,
+            symbol,
+            existing_side,
+            quantity,
+            add_price,
+            blended_entry,
+            target_price,
+        )
+        return {
+            "id": pos_id,
+            "symbol": symbol,
+            "side": existing_side,
+            "entry_date": existing_entry_date or add_date,
+            "entry_price": blended_entry,
+            "quantity": new_qty,
+            "added_quantity": quantity,
+            "target_price": target_price,
+        }
+
     def check_and_close_positions(self, check_date=None):
         """
         Check all open positions against the day's price data.