Last active
February 11, 2026 14:19
-
-
Save austin-starks/b1fff758f0c59c3fc2865927ffb6310b to your computer and use it in GitHub Desktop.
Analyze Backtest Script
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python3 | |
| """ | |
| Backtest Data Analyzer - Trading Strategy Component Analysis | |
| INPUT DATA FORMAT: | |
| ------------------ | |
| Expects JSONL files in: output/raw_backtest_data/{timestamp}/ | |
| Each category file (e.g., strategySetsWithAssets.jsonl) contains: | |
| { | |
| "component": "Buy 100 percent in SPY Stock when Price > 50 Day SMA", | |
| "count": 5, | |
| "performances": [ | |
| { | |
| "backtestId": "abc123", | |
| "backtestName": "Test Portfolio", | |
| "startDate": "2020-01-01T00:00:00.000Z", | |
| "endDate": "2023-01-01T00:00:00.000Z", | |
| "statistics": { | |
| "percentChange": 45.2, | |
| "sharpeRatio": 1.8, | |
| "sortinoRatio": 2.3, | |
| "maxDrawdown": 15.4, | |
| ... | |
| } | |
| } | |
| ] | |
| } | |
| Categories analyzed: | |
| - strategySetsWithAssets / strategySetsWithoutAssets | |
| - strategiesWithAssets / strategiesWithoutAssets | |
| - conditionsWithAssets / conditionsWithoutAssets | |
| - actionsWithAssets / actionsWithoutAssets | |
| - indicatorsWithAssets / indicatorsWithoutAssets | |
| OUTPUT DATA FORMAT: | |
| ------------------- | |
| Creates: output/backtest_analysis_results/{timestamp}/ | |
| 1. summary.json - Aggregate statistics for all components | |
| 2. corpus-{category}.json - Top performing components by metric: | |
| { | |
| "category": "strategiesWithAssets", | |
| "totalComponents": 14008, | |
| "corpuses": { | |
| "high_sharpe": [{"component": "...", "meanSharpe": 12.1, ...}], | |
| "high_cagr": [...], | |
| "high_calmar": [...], | |
| "elite": [...] // Top 20% in all three metrics | |
| } | |
| } | |
| 3. corpus-worst-{category}.json - Worst performing components (if --include-worst): | |
| { | |
| "category": "strategiesWithAssets", | |
| "totalComponents": 14008, | |
| "corpuses": { | |
| "low_sharpe": [{"component": "...", "meanSharpe": -3.2, ...}], | |
| "low_cagr": [...], | |
| "low_calmar": [...], | |
| "terrible": [...] // Bottom 20% in all three metrics | |
| } | |
| } | |
| NORMALIZATION (default enabled): | |
| --------------------------------- | |
| All numeric values are normalized for better grouping: | |
| - < 1: Round to 1 decimal place (0.234 -> 0.2) | |
| - 1-10: Round to integer (8.4 -> 8) | |
| - 10-100: Round to nearest 5 (89 -> 90) | |
| - >= 100: Round to order of magnitude (1112 -> 1100) | |
| Use --raw-allocations to disable normalization. | |
| DIVERSITY SELECTION: | |
| -------------------- | |
| Uses TF-IDF + cosine similarity to select diverse high performers, | |
| preventing over-representation of similar strategy variations. | |
| """ | |
| import polars as pl | |
| import json | |
| import sys | |
| import time | |
| import argparse | |
| import re | |
| import math | |
| from pathlib import Path | |
| from datetime import datetime | |
| from typing import Dict, List, Any | |
| from sklearn.feature_extraction.text import TfidfVectorizer | |
| from sklearn.metrics.pairwise import cosine_similarity | |
| import numpy as np | |
| def normalize_component_allocations(component: str) -> str: | |
| """ | |
| Aggressively normalize ALL numbers in component strings for better grouping. | |
| Normalization rules based on magnitude: | |
| - < 1: Round to 1 decimal place (0.234 -> 0.2) | |
| - 1-10: Round to nearest integer (8.4 -> 8) | |
| - 10-100: Round to nearest 5 (47 -> 45, 89 -> 90) | |
| - >= 100: Round to order of magnitude (832 -> 800, 1112 -> 1100) | |
| Examples: | |
| "Buy 11.4 percent when Price > 89.15" -> "Buy 10 percent when Price > 90" | |
| "Sell 832.69 number of assets" -> "Sell 800 number of assets" | |
| "when Volume >= 1234567" -> "when Volume >= 1000000" | |
| """ | |
| def normalize_number(value: float) -> str: | |
| """Normalize a number based on its magnitude""" | |
| abs_val = abs(value) | |
| if abs_val == 0: | |
| return "0" | |
| elif abs_val < 1: | |
| # Round to 1 decimal place | |
| return f"{round(value, 1):.1f}".rstrip('0').rstrip('.') | |
| elif abs_val < 10: | |
| # Round to nearest integer | |
| return str(int(round(value))) | |
| elif abs_val < 100: | |
| # Round to nearest 5 | |
| rounded = int(round(value / 5) * 5) | |
| return str(rounded) | |
| else: | |
| # Round to order of magnitude | |
| magnitude = 10 ** math.floor(math.log10(abs_val)) | |
| rounded = int(round(value / magnitude) * magnitude) | |
| return str(rounded) | |
| def replace_number(match): | |
| """Replace a number with its normalized version""" | |
| value = float(match.group(0)) | |
| return normalize_number(value) | |
| # Match any number (integer or decimal, positive or negative) | |
| # This catches: 123, 45.67, -89.1234, etc. | |
| result = re.sub(r'-?\d+(?:\.\d+)?', replace_number, component) | |
| return result | |
| class BacktestDataAnalyzer: | |
| def __init__(self, timestamp: str = None, corpus_limit: int = 100, similarity_threshold: float = 0.7, raw_allocations: bool = False, include_worst: bool = True): | |
| self.timestamp = timestamp | |
| self.corpus_limit = corpus_limit | |
| self.similarity_threshold = similarity_threshold | |
| self.raw_allocations = raw_allocations | |
| self.include_worst = include_worst | |
| self.output_dir = Path(__file__).parent / "output" | |
| self.raw_data_dir = self.output_dir / "raw_backtest_data" | |
| # Create timestamped folder for this analysis run (nanoseconds for uniqueness) | |
| self.run_timestamp = str(int(time.time_ns())) | |
| self.analysis_output_dir = self.output_dir / "backtest_analysis_results" / self.run_timestamp | |
| # Create analysis results directory | |
| self.analysis_output_dir.mkdir(parents=True, exist_ok=True) | |
| def find_latest_extraction(self) -> str: | |
| """Find the most recent extraction timestamp from raw_backtest_data folders""" | |
| if not self.raw_data_dir.exists(): | |
| raise FileNotFoundError(f"Raw data directory not found: {self.raw_data_dir}") | |
| # Look for timestamped folders in raw_backtest_data directory | |
| timestamped_folders = [ | |
| d for d in self.raw_data_dir.iterdir() | |
| if d.is_dir() | |
| ] | |
| if not timestamped_folders: | |
| raise FileNotFoundError(f"No extraction folders found in {self.raw_data_dir}") | |
| # Sort by modification time, get most recent | |
| latest_folder = max(timestamped_folders, key=lambda d: d.stat().st_mtime) | |
| return latest_folder.name | |
| def find_categories(self) -> List[str]: | |
| """Auto-discover categories by scanning for JSONL files in timestamped folder""" | |
| extraction_dir = self.raw_data_dir / self.timestamp | |
| files = list(extraction_dir.glob("*.jsonl")) | |
| categories = [f.stem for f in files] | |
| return sorted(categories) | |
| def load_metadata(self) -> Dict[str, Any]: | |
| """Load extraction metadata""" | |
| metadata_path = self.raw_data_dir / self.timestamp / "metadata.json" | |
| with open(metadata_path, 'r') as f: | |
| metadata = json.load(f) | |
| print(f"Loaded metadata:") | |
| print(f" Generated at: {metadata['generatedAt']}") | |
| print(f" Total backtests: {metadata['totalBacktests']}") | |
| print(f" Total skipped: {metadata['totalSkipped']}") | |
| print() | |
| return metadata | |
| def load_category_data(self, category: str) -> pl.LazyFrame: | |
| """Load a single category JSONL file as a lazy dataframe""" | |
| filename = self.raw_data_dir / self.timestamp / f"{category}.jsonl" | |
| if not filename.exists(): | |
| print(f"Warning: {filename} not found, skipping") | |
| return None | |
| # Read JSONL and explode performances array | |
| df = ( | |
| pl.scan_ndjson(filename) | |
| .select([ | |
| pl.col("component"), | |
| pl.col("count"), | |
| pl.col("performances") | |
| ]) | |
| .explode("performances") | |
| .unnest("performances") # Flatten the performance object | |
| ) | |
| return df | |
| def calculate_aggregates_for_category(self, category: str) -> Dict[str, Any]: | |
| """Calculate aggregate statistics for a category""" | |
| print(f" Analyzing {category}...") | |
| filename = self.raw_data_dir / self.timestamp / f"{category}.jsonl" | |
| if not filename.exists(): | |
| return {} | |
| aggregates = {} | |
| # Process file line by line to calculate aggregates | |
| with open(filename, 'r') as f: | |
| for line in f: | |
| if not line.strip(): | |
| continue | |
| entry = json.loads(line) | |
| component = entry['component'] | |
| performances = entry['performances'] | |
| if len(performances) == 0: | |
| continue | |
| # Calculate normalized metrics for each performance | |
| returns = [] | |
| sharpes = [] | |
| sortinos = [] | |
| drawdowns = [] | |
| win_rates = [] | |
| profit_factors = [] | |
| for perf in performances: | |
| stats = perf['statistics'] | |
| start_date = datetime.fromisoformat(perf['startDate'].replace('Z', '+00:00')) | |
| end_date = datetime.fromisoformat(perf['endDate'].replace('Z', '+00:00')) | |
| days = (end_date - start_date).days | |
| if days <= 0: | |
| continue | |
| # Annualize metrics | |
| pct_change = (stats.get('percentChange') or 0) / 100 # Convert from percentage to decimal | |
| # Handle edge cases to avoid complex numbers and overflow | |
| # For losses >= 100%, annualization doesn't make mathematical sense | |
| # (you can't compound a negative base), so we keep the raw loss | |
| if pct_change <= -1: | |
| # Use the raw percentage change but cap extreme values | |
| annualized_return = max(pct_change, -10) | |
| else: | |
| annualized_return = ((1 + pct_change) ** (365 / days)) - 1 | |
| # Cap at 10x (1000%) to prevent overflow in variance calculations | |
| annualized_return = min(annualized_return, 10) | |
| annualized_drawdown = (stats.get('maxDrawdown') or 0) * (365 / days) ** 0.5 | |
| returns.append(annualized_return) | |
| sharpes.append(stats.get('sharpeRatio') or 0) | |
| sortinos.append(stats.get('sortinoRatio') or 0) | |
| drawdowns.append(annualized_drawdown) | |
| win_rates.append(stats.get('winRate') or 0) | |
| profit_factors.append(stats.get('profitFactor') or 0) | |
| if len(returns) == 0: | |
| continue | |
| # Calculate aggregates | |
| aggregates[component] = { | |
| "count": len(returns), | |
| "aggregates": { | |
| "count": len(returns), | |
| "meanAnnualizedReturn": sum(returns) / len(returns), | |
| "medianAnnualizedReturn": sorted(returns)[len(returns) // 2], | |
| "stdDevAnnualizedReturn": self._std_dev(returns), | |
| "minAnnualizedReturn": min(returns), | |
| "maxAnnualizedReturn": max(returns), | |
| "meanSharpe": sum(sharpes) / len(sharpes), | |
| "medianSharpe": sorted(sharpes)[len(sharpes) // 2], | |
| "meanSortino": sum(sortinos) / len(sortinos), | |
| "medianSortino": sorted(sortinos)[len(sortinos) // 2], | |
| "meanMaxDrawdown": sum(drawdowns) / len(drawdowns), | |
| "medianMaxDrawdown": sorted(drawdowns)[len(drawdowns) // 2], | |
| "meanWinRate": sum(win_rates) / len(win_rates), | |
| "medianWinRate": sorted(win_rates)[len(win_rates) // 2], | |
| "meanProfitFactor": sum(profit_factors) / len(profit_factors), | |
| "medianProfitFactor": sorted(profit_factors)[len(profit_factors) // 2], | |
| } | |
| } | |
| return aggregates | |
| def _std_dev(self, values: List[float]) -> float: | |
| """Calculate standard deviation""" | |
| if len(values) == 0: | |
| return 0.0 | |
| mean = sum(values) / len(values) | |
| variance = sum((x - mean) ** 2 for x in values) / len(values) | |
| return variance ** 0.5 | |
| def _select_diverse_components(self, components: List[Dict[str, Any]], | |
| metric_key: str, | |
| limit: int, | |
| similarity_threshold: float) -> List[Dict[str, Any]]: | |
| """ | |
| Select diverse high-performing components using TF-IDF similarity. | |
| Args: | |
| components: List of component dicts with 'component' string and performance metrics | |
| metric_key: Which metric to optimize (e.g., 'meanSharpe', 'meanAnnualizedReturn') | |
| limit: Maximum number of components to select | |
| similarity_threshold: Cosine similarity threshold (0-1). Higher = more similar. | |
| Components above this threshold to already-selected ones are penalized. | |
| Returns: | |
| List of selected diverse components | |
| """ | |
| if len(components) == 0: | |
| return [] | |
| if len(components) <= limit: | |
| return components | |
| # Extract component strings for TF-IDF | |
| component_texts = [c['component'] for c in components] | |
| # Vectorize using TF-IDF | |
| vectorizer = TfidfVectorizer( | |
| max_features=1000, | |
| ngram_range=(1, 3), # Unigrams, bigrams, trigrams | |
| stop_words=None # Keep all words since strategy descriptions are technical | |
| ) | |
| tfidf_matrix = vectorizer.fit_transform(component_texts) | |
| # Calculate similarity matrix | |
| similarity_matrix = cosine_similarity(tfidf_matrix) | |
| # Greedy diversity selection | |
| selected_indices = [] | |
| remaining_indices = list(range(len(components))) | |
| # Sort by metric (descending) | |
| sorted_indices = sorted(remaining_indices, | |
| key=lambda i: components[i][metric_key], | |
| reverse=True) | |
| # Always take the best performer first | |
| selected_indices.append(sorted_indices[0]) | |
| remaining_indices.remove(sorted_indices[0]) | |
| # Iteratively select diverse components | |
| while len(selected_indices) < limit and remaining_indices: | |
| best_candidate = None | |
| best_score = -float('inf') | |
| for idx in remaining_indices: | |
| # Calculate diversity score: performance minus similarity penalty | |
| performance = components[idx][metric_key] | |
| # Calculate max similarity to already-selected components | |
| max_similarity = max(similarity_matrix[idx][selected_idx] | |
| for selected_idx in selected_indices) | |
| # Penalize if too similar to existing selections | |
| if max_similarity > similarity_threshold: | |
| # Heavy penalty for similar components | |
| diversity_penalty = performance * (max_similarity - similarity_threshold) * 2 | |
| else: | |
| diversity_penalty = 0 | |
| score = performance - diversity_penalty | |
| if score > best_score: | |
| best_score = score | |
| best_candidate = idx | |
| if best_candidate is not None: | |
| selected_indices.append(best_candidate) | |
| remaining_indices.remove(best_candidate) | |
| else: | |
| break | |
| return [components[i] for i in selected_indices] | |
| def calculate_corpuses_for_category(self, category: str) -> Dict[str, Any]: | |
| """Calculate corpuses for a single category at component level""" | |
| # Load JSONL and explode to performance-level rows | |
| filename = self.raw_data_dir / self.timestamp / f"{category}.jsonl" | |
| df = ( | |
| pl.read_ndjson(filename) | |
| .explode("performances") | |
| .unnest("performances") | |
| .unnest("statistics") | |
| # Filter out non-trading components and malformed components | |
| .filter( | |
| ~pl.col("component").str.contains("Deposit") & | |
| ~pl.col("component").str.contains("Withdraw") & | |
| ~pl.col("component").str.contains("Alert") & | |
| # Filter out components that start with space or comma (malformed) | |
| ~pl.col("component").str.starts_with(" ") & | |
| ~pl.col("component").str.starts_with(",") & | |
| # Filter out components with "undefined" or "Sum()" (extraction errors) | |
| ~pl.col("component").str.contains("undefined") & | |
| ~pl.col("component").str.contains("Sum\\(\\)") | |
| ) | |
| ) | |
| # Apply allocation normalization if enabled | |
| if not self.raw_allocations: | |
| df = df.with_columns([ | |
| pl.col("component").map_elements( | |
| normalize_component_allocations, | |
| return_dtype=pl.Utf8 | |
| ).alias("component") | |
| ]) | |
| # Calculate time-based metrics | |
| df = df.with_columns([ | |
| # Parse dates | |
| pl.col("startDate").str.to_datetime(time_zone="UTC").alias("start"), | |
| pl.col("endDate").str.to_datetime(time_zone="UTC").alias("end"), | |
| ]).with_columns([ | |
| # Calculate days | |
| ((pl.col("end") - pl.col("start")).dt.total_days()).alias("days") | |
| ]).filter( | |
| pl.col("days") > 0 | |
| ).with_columns([ | |
| # Convert percentChange from percentage to decimal (e.g., 9.39 -> 0.0939) | |
| (pl.col("percentChange").fill_null(0) / 100).alias("returnDecimal") | |
| ]).with_columns([ | |
| # Annualized return with edge case handling | |
| pl.when(pl.col("returnDecimal") <= -1) | |
| .then(pl.col("returnDecimal").clip(-10, None)) | |
| .otherwise( | |
| (((1 + pl.col("returnDecimal")) ** (365 / pl.col("days"))) - 1).clip(None, 10) | |
| ) | |
| .alias("annualizedReturn"), | |
| # Annualized drawdown | |
| (pl.col("maxDrawdown").fill_null(0) * ((365 / pl.col("days")) ** 0.5)).alias("annualizedDrawdown"), | |
| # Fill nulls for other metrics | |
| pl.col("sharpeRatio").fill_null(0).alias("sharpe"), | |
| pl.col("sortinoRatio").fill_null(0).alias("sortino"), | |
| ]).with_columns([ | |
| # Calculate Calmar ratio | |
| pl.when(pl.col("annualizedDrawdown") != 0) | |
| .then(pl.col("annualizedReturn") / pl.col("annualizedDrawdown").abs()) | |
| .otherwise(0) | |
| .alias("calmar") | |
| ]).filter( | |
| # Filter out infinite Calmar values | |
| pl.col("calmar").abs() < 1000 | |
| ) | |
| # Aggregate by component | |
| component_stats_df = df.group_by("component").agg([ | |
| pl.col("component").count().alias("count"), | |
| pl.col("sharpe").mean().alias("meanSharpe"), | |
| pl.col("sharpe").median().alias("medianSharpe"), | |
| pl.col("sortino").mean().alias("meanSortino"), | |
| pl.col("sortino").median().alias("medianSortino"), | |
| pl.col("annualizedReturn").mean().alias("meanAnnualizedReturn"), | |
| pl.col("annualizedReturn").median().alias("medianAnnualizedReturn"), | |
| pl.col("calmar").mean().alias("meanCalmar"), | |
| pl.col("calmar").median().alias("medianCalmar"), | |
| ]).filter( | |
| # Filter out rows with no valid data | |
| pl.col("count") > 0 | |
| ) | |
| # Convert to list of dicts for diversity selection | |
| component_stats = component_stats_df.to_dicts() | |
| # Use diversity-aware selection for each corpus | |
| # For Calmar, filter out non-finite values first | |
| calmar_candidates = [c for c in component_stats if np.isfinite(c['medianCalmar'])] | |
| # Elite: filter to top 20% in each metric first, then select diverse | |
| if len(component_stats) > 0: | |
| sharpe_threshold = np.quantile([c['medianSharpe'] for c in component_stats], 0.80) | |
| cagr_threshold = np.quantile([c['medianAnnualizedReturn'] for c in component_stats], 0.80) | |
| calmar_threshold = np.quantile([c['medianCalmar'] for c in calmar_candidates], 0.80) if len(calmar_candidates) > 0 else 0 | |
| elite_candidates = [ | |
| c for c in component_stats | |
| if c['medianSharpe'] >= sharpe_threshold | |
| and c['medianAnnualizedReturn'] >= cagr_threshold | |
| and np.isfinite(c['medianCalmar']) | |
| and c['medianCalmar'] >= calmar_threshold | |
| ] | |
| else: | |
| elite_candidates = [] | |
| # Create 4 corpuses with diversity selection (using MEDIAN metrics) | |
| corpuses = { | |
| 'high_sharpe': self._select_diverse_components(component_stats, 'medianSharpe', self.corpus_limit, self.similarity_threshold), | |
| 'high_cagr': self._select_diverse_components(component_stats, 'medianAnnualizedReturn', self.corpus_limit, self.similarity_threshold), | |
| 'high_calmar': self._select_diverse_components(calmar_candidates, 'medianCalmar', self.corpus_limit, self.similarity_threshold), | |
| 'elite': self._select_diverse_components(elite_candidates, 'medianSharpe', self.corpus_limit, self.similarity_threshold) | |
| } | |
| return { | |
| 'category': category, | |
| 'totalComponents': len(component_stats), | |
| 'corpuses': corpuses | |
| } | |
| def calculate_worst_corpuses_for_category(self, category: str) -> Dict[str, Any]: | |
| """Calculate worst-performing corpuses for a single category""" | |
| # Load JSONL and explode to performance-level rows (same as best) | |
| filename = self.raw_data_dir / self.timestamp / f"{category}.jsonl" | |
| df = ( | |
| pl.read_ndjson(filename) | |
| .explode("performances") | |
| .unnest("performances") | |
| .unnest("statistics") | |
| # Filter out non-trading components and malformed components | |
| .filter( | |
| ~pl.col("component").str.contains("Deposit") & | |
| ~pl.col("component").str.contains("Withdraw") & | |
| ~pl.col("component").str.contains("Alert") | |
| ) | |
| ) | |
| # Apply allocation normalization if enabled | |
| if not self.raw_allocations: | |
| df = df.with_columns([ | |
| pl.col("component").map_elements( | |
| normalize_component_allocations, | |
| return_dtype=pl.Utf8 | |
| ).alias("component") | |
| ]) | |
| # Calculate time-based metrics (same calculation as best) | |
| df = df.with_columns([ | |
| pl.col("startDate").str.to_datetime(time_zone="UTC").alias("start"), | |
| pl.col("endDate").str.to_datetime(time_zone="UTC").alias("end"), | |
| ]).with_columns([ | |
| ((pl.col("end") - pl.col("start")).dt.total_days()).alias("days") | |
| ]).filter( | |
| pl.col("days") > 0 | |
| ).with_columns([ | |
| (pl.col("percentChange").fill_null(0) / 100).alias("returnDecimal") | |
| ]).with_columns([ | |
| pl.when(pl.col("returnDecimal") <= -1) | |
| .then(pl.col("returnDecimal").clip(-10, None)) | |
| .otherwise( | |
| (((1 + pl.col("returnDecimal")) ** (365 / pl.col("days"))) - 1).clip(None, 10) | |
| ) | |
| .alias("annualizedReturn"), | |
| (pl.col("maxDrawdown").fill_null(0) * ((365 / pl.col("days")) ** 0.5)).alias("annualizedDrawdown"), | |
| pl.col("sharpeRatio").fill_null(0).alias("sharpe"), | |
| pl.col("sortinoRatio").fill_null(0).alias("sortino"), | |
| ]).with_columns([ | |
| pl.when(pl.col("annualizedDrawdown") != 0) | |
| .then(pl.col("annualizedReturn") / pl.col("annualizedDrawdown").abs()) | |
| .otherwise(0) | |
| .alias("calmar") | |
| ]).filter( | |
| pl.col("calmar").abs() < 1000 | |
| ) | |
| # Aggregate by component | |
| component_stats_df = df.group_by("component").agg([ | |
| pl.col("component").count().alias("count"), | |
| pl.col("sharpe").mean().alias("meanSharpe"), | |
| pl.col("sharpe").median().alias("medianSharpe"), | |
| pl.col("sortino").mean().alias("meanSortino"), | |
| pl.col("sortino").median().alias("medianSortino"), | |
| pl.col("annualizedReturn").mean().alias("meanAnnualizedReturn"), | |
| pl.col("annualizedReturn").median().alias("medianAnnualizedReturn"), | |
| pl.col("calmar").mean().alias("meanCalmar"), | |
| pl.col("calmar").median().alias("medianCalmar"), | |
| ]).filter( | |
| pl.col("count") > 0 | |
| ) | |
| # Convert to list of dicts | |
| component_stats = component_stats_df.to_dicts() | |
| # For worst performers, we want LOW values (reverse logic) | |
| calmar_candidates = [c for c in component_stats if np.isfinite(c['medianCalmar'])] | |
| # Terrible: bottom 20% in each metric | |
| if len(component_stats) > 0: | |
| sharpe_threshold = np.quantile([c['medianSharpe'] for c in component_stats], 0.20) | |
| cagr_threshold = np.quantile([c['medianAnnualizedReturn'] for c in component_stats], 0.20) | |
| calmar_threshold = np.quantile([c['medianCalmar'] for c in calmar_candidates], 0.20) if len(calmar_candidates) > 0 else 0 | |
| terrible_candidates = [ | |
| c for c in component_stats | |
| if c['medianSharpe'] <= sharpe_threshold | |
| and c['medianAnnualizedReturn'] <= cagr_threshold | |
| and np.isfinite(c['medianCalmar']) | |
| and c['medianCalmar'] <= calmar_threshold | |
| ] | |
| else: | |
| terrible_candidates = [] | |
| # Create 4 worst corpuses with diversity selection (using MEDIAN metrics) | |
| corpuses = { | |
| 'low_sharpe': self._select_diverse_worst_components(component_stats, 'medianSharpe', self.corpus_limit, self.similarity_threshold), | |
| 'low_cagr': self._select_diverse_worst_components(component_stats, 'medianAnnualizedReturn', self.corpus_limit, self.similarity_threshold), | |
| 'low_calmar': self._select_diverse_worst_components(calmar_candidates, 'medianCalmar', self.corpus_limit, self.similarity_threshold), | |
| 'terrible': self._select_diverse_worst_components(terrible_candidates, 'medianSharpe', self.corpus_limit, self.similarity_threshold) | |
| } | |
| return { | |
| 'category': category, | |
| 'totalComponents': len(component_stats), | |
| 'corpuses': corpuses | |
| } | |
| def _select_diverse_worst_components(self, components: List[Dict[str, Any]], | |
| metric_key: str, | |
| limit: int, | |
| similarity_threshold: float) -> List[Dict[str, Any]]: | |
| """ | |
| Select diverse WORST-performing components using TF-IDF similarity. | |
| Similar to _select_diverse_components but optimizes for LOW values. | |
| """ | |
| if len(components) == 0: | |
| return [] | |
| if len(components) <= limit: | |
| return components | |
| # Extract component strings for TF-IDF | |
| component_texts = [c['component'] for c in components] | |
| # Vectorize using TF-IDF | |
| vectorizer = TfidfVectorizer( | |
| max_features=1000, | |
| ngram_range=(1, 3), | |
| stop_words=None | |
| ) | |
| tfidf_matrix = vectorizer.fit_transform(component_texts) | |
| # Calculate similarity matrix | |
| similarity_matrix = cosine_similarity(tfidf_matrix) | |
| # Greedy diversity selection | |
| selected_indices = [] | |
| remaining_indices = list(range(len(components))) | |
| # Sort by metric (ASCENDING for worst) | |
| sorted_indices = sorted(remaining_indices, | |
| key=lambda i: components[i][metric_key], | |
| reverse=False) # Low to high | |
| # Always take the worst performer first | |
| selected_indices.append(sorted_indices[0]) | |
| remaining_indices.remove(sorted_indices[0]) | |
| # Iteratively select diverse components | |
| while len(selected_indices) < limit and remaining_indices: | |
| best_candidate = None | |
| best_score = float('inf') # Want LOWEST score | |
| for idx in remaining_indices: | |
| # For worst performers, we want LOW metric values | |
| performance = components[idx][metric_key] | |
| # Calculate max similarity to already-selected components | |
| max_similarity = max(similarity_matrix[idx][selected_idx] | |
| for selected_idx in selected_indices) | |
| # Penalize if too similar to existing selections | |
| if max_similarity > similarity_threshold: | |
| # Penalty makes score higher (worse for minimization) | |
| diversity_penalty = abs(performance) * (max_similarity - similarity_threshold) * 2 | |
| else: | |
| diversity_penalty = 0 | |
| score = performance + diversity_penalty # Higher score = worse candidate | |
| if score < best_score: | |
| best_score = score | |
| best_candidate = idx | |
| if best_candidate is not None: | |
| selected_indices.append(best_candidate) | |
| remaining_indices.remove(best_candidate) | |
| else: | |
| break | |
| return [components[i] for i in selected_indices] | |
| def run(self): | |
| """Main execution""" | |
| print("=== Backtest Data Analysis (Python) ===\n") | |
| try: | |
| # Find or use provided timestamp | |
| if not self.timestamp: | |
| self.timestamp = self.find_latest_extraction() | |
| print(f"Using latest extraction: {self.timestamp}\n") | |
| # Load metadata | |
| metadata = self.load_metadata() | |
| # Auto-discover categories from JSONL files | |
| categories = self.find_categories() | |
| print(f"Found {len(categories)} categories: {', '.join(categories)}\n") | |
| # Analyze all categories | |
| print("Analyzing data...\n") | |
| analyzed_data = { | |
| 'metadata': { | |
| **metadata, | |
| 'sourceTimestamp': self.timestamp | |
| } | |
| } | |
| for category in categories: | |
| aggregates = self.calculate_aggregates_for_category(category) | |
| analyzed_data[category] = aggregates | |
| # Calculate corpuses for each category separately | |
| print("\nCalculating corpuses by category...\n") | |
| corpus_results = {} | |
| for category in categories: | |
| print(f" Calculating corpuses for {category}...") | |
| corpus_results[category] = self.calculate_corpuses_for_category(category) | |
| print(f" Total components: {corpus_results[category]['totalComponents']}") | |
| for corpus_name, corpus_data in corpus_results[category]['corpuses'].items(): | |
| print(f" {corpus_name}: {len(corpus_data)} entries") | |
| # Calculate worst corpuses if requested | |
| worst_corpus_results = {} | |
| if self.include_worst: | |
| print("\nCalculating worst-performer corpuses by category...\n") | |
| for category in categories: | |
| print(f" Calculating worst corpuses for {category}...") | |
| worst_corpus_results[category] = self.calculate_worst_corpuses_for_category(category) | |
| print(f" Total components: {worst_corpus_results[category]['totalComponents']}") | |
| for corpus_name, corpus_data in worst_corpus_results[category]['corpuses'].items(): | |
| print(f" {corpus_name}: {len(corpus_data)} entries") | |
| # Export results | |
| self.export_results(analyzed_data) | |
| self.export_corpuses_by_category(corpus_results) | |
| if self.include_worst: | |
| self.export_worst_corpuses_by_category(worst_corpus_results) | |
| print("\n✓ Analysis completed successfully") | |
| return 0 | |
| except Exception as e: | |
| print(f"\n✗ Analysis failed: {e}", file=sys.stderr) | |
| import traceback | |
| traceback.print_exc() | |
| return 1 | |
| def export_results(self, data: Dict[str, Any]): | |
| """Export analysis results to JSON files""" | |
| print("\nExporting aggregate statistics...") | |
| # Export summary (aggregates only - corpuses exported separately) | |
| summary_path = self.analysis_output_dir / "summary.json" | |
| with open(summary_path, 'w') as f: | |
| json.dump(data, f, indent=2, default=str) | |
| print(f" ✓ Summary exported to: {summary_path}") | |
| def export_corpuses_by_category(self, results: Dict[str, Any]): | |
| """Export separate corpus file for each category""" | |
| print("\nExporting corpuses by category...") | |
| for category, data in results.items(): | |
| filename = self.analysis_output_dir / f"corpus-{category}.json" | |
| with open(filename, 'w') as f: | |
| json.dump(data, f, indent=2) | |
| print(f" ✓ {category}: {data['totalComponents']} components") | |
| for corpus_name in data['corpuses'].keys(): | |
| print(f" - {corpus_name}: {len(data['corpuses'][corpus_name])} entries") | |
| def export_worst_corpuses_by_category(self, results: Dict[str, Any]): | |
| """Export separate worst-performer corpus file for each category""" | |
| print("\nExporting worst-performer corpuses by category...") | |
| for category, data in results.items(): | |
| filename = self.analysis_output_dir / f"corpus-worst-{category}.json" | |
| with open(filename, 'w') as f: | |
| json.dump(data, f, indent=2) | |
| print(f" ✓ {category}: {data['totalComponents']} components") | |
| for corpus_name in data['corpuses'].keys(): | |
| print(f" - {corpus_name}: {len(data['corpuses'][corpus_name])} entries") | |
| def main(): | |
| """CLI entry point""" | |
| parser = argparse.ArgumentParser(description='Analyze backtest data from JSONL files') | |
| parser.add_argument('--timestamp', type=str, help='Extraction timestamp to analyze (auto-detect if not provided)') | |
| parser.add_argument('--corpus-limit', type=int, default=100, help='Maximum number of components per corpus (default: 100)') | |
| parser.add_argument('--similarity-threshold', type=float, default=0.7, help='TF-IDF similarity threshold for diversity (0-1, default: 0.7)') | |
| parser.add_argument('--raw-allocations', action='store_true', help='Use exact numbers instead of normalizing (default: aggressively normalize ALL numbers - <1: 1 decimal, 1-10: integer, 10-100: nearest 5, 100+: order of magnitude)') | |
| parser.add_argument('--skip-worst', action='store_true', help='Skip generating worst performer corpus files (default: generate both best and worst)') | |
| args = parser.parse_args() | |
| analyzer = BacktestDataAnalyzer( | |
| timestamp=args.timestamp, | |
| corpus_limit=args.corpus_limit, | |
| similarity_threshold=args.similarity_threshold, | |
| raw_allocations=args.raw_allocations, | |
| include_worst=not args.skip_worst # Invert the flag | |
| ) | |
| sys.exit(analyzer.run()) | |
| if __name__ == '__main__': | |
| main() |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| /** | |
| * Backtest Data Extractor - Trading Strategy Component Extraction | |
| * | |
| * INPUT DATA SOURCE: | |
| * ------------------ | |
| * Queries MongoDB for completed backtests with the following criteria: | |
| * - status: "COMPLETE" | |
| * - Optional date range filter (--date-from, --date-to) | |
| * - Processes in batches using cursor-based pagination | |
| * | |
| * Each backtest document contains: | |
| * { | |
| * _id: ObjectId, | |
| * name: string, | |
| * status: "COMPLETE" | "PENDING" | "FAILED", | |
| * startDate: Date, | |
| * endDate: Date, | |
| * portfolio: { | |
| * strategies: [...], | |
| * assets: [...] | |
| * }, | |
| * statistics: { | |
| * percentChange: number, | |
| * sharpeRatio: number, | |
| * sortinoRatio: number, | |
| * maxDrawdown: number, | |
| * winRate: number, | |
| * profitFactor: number, | |
| * ... | |
| * } | |
| * } | |
| * | |
| * EXTRACTION LOGIC: | |
| * ----------------- | |
| * For each backtest, extracts components at multiple levels: | |
| * 1. Strategy Sets - Complete portfolio configurations | |
| * 2. Strategies - Individual trading strategies | |
| * 3. Conditions - Entry/exit conditions | |
| * 4. Actions - Buy/sell actions | |
| * 5. Indicators - Technical indicators used | |
| * | |
| * Each component is tracked in two variants: | |
| * - withAssets: Component string includes specific asset symbols | |
| * - withoutAssets: Component string with asset symbols normalized | |
| * | |
| * OUTPUT DATA FORMAT: | |
| * ------------------- | |
| * Creates: output/raw_backtest_data/{timestamp}/ | |
| * | |
| * 1. metadata.json - Extraction metadata: | |
| * { | |
| * "generatedAt": "2024-01-01T00:00:00.000Z", | |
| * "totalBacktests": 5000, | |
| * "totalSkipped": 10, | |
| * "dateRange": { | |
| * "earliest": "2020-01-01T00:00:00.000Z", | |
| * "latest": "2024-01-01T00:00:00.000Z" | |
| * }, | |
| * "timestamp": "1704067200000000000" | |
| * } | |
| * | |
| * 2. {category}.jsonl files (JSONL format, one component per line): | |
| * {"component":"Buy 100 percent in SPY Stock when Price > 50 Day SMA","count":5,"performances":[{"backtestId":"abc123","backtestName":"Test Portfolio","startDate":"2020-01-01T00:00:00.000Z","endDate":"2023-01-01T00:00:00.000Z","statistics":{"percentChange":45.2,"sharpeRatio":1.8,...}}]} | |
| * | |
| * Output files created: | |
| * - strategySetsWithAssets.jsonl | |
| * - strategySetsWithoutAssets.jsonl | |
| * - strategiesWithAssets.jsonl | |
| * - strategiesWithoutAssets.jsonl | |
| * - conditionsWithAssets.jsonl | |
| * - conditionsWithoutAssets.jsonl | |
| * - actionsWithAssets.jsonl | |
| * - actionsWithoutAssets.jsonl | |
| * - indicatorsWithAssets.jsonl | |
| * - indicatorsWithoutAssets.jsonl | |
| * | |
| * USAGE: | |
| * ------ | |
| * npm run analytics:extract -- [options] | |
| * | |
| * Options: | |
| * --db-env <env> Database environment (cloud|local) [default: cloud] | |
| * --batch-size <size> Number of backtests to process per batch [default: 100] | |
| * --date-from <date> Filter backtests from this date (ISO format) | |
| * --date-to <date> Filter backtests to this date (ISO format) | |
| * | |
| * Examples: | |
| * npm run analytics:extract | |
| * npm run analytics:extract -- --db-env local --batch-size 50 | |
| * npm run analytics:extract -- --date-from 2024-01-01 --date-to 2024-12-31 | |
| * | |
| * PERFORMANCE CONSIDERATIONS: | |
| * --------------------------- | |
| * - Uses streaming JSONL format to avoid memory issues with large datasets | |
| * - Cursor-based pagination prevents loading entire collection into memory | |
| * - Batch processing with configurable batch size | |
| * - Nanosecond timestamps ensure unique output directories | |
| */ | |
| import * as fs from "fs"; | |
| import * as path from "path"; | |
| import Backtester, { IBacktester } from "../models/backtester"; | |
| import { | |
| ComponentData, | |
| ExtractOptions, | |
| RawDataOutput, | |
| RawPerformance, | |
| } from "./backtestAnalytics/types"; | |
| import { | |
| extractActions, | |
| extractAllConditions, | |
| extractIndicators, | |
| extractStrategies, | |
| extractStrategySet, | |
| } from "./backtestAnalytics/utils"; | |
| import Db from "../services/database/mongo"; | |
| import moment from "moment"; | |
| import mongoose from "mongoose"; | |
| class BacktestDataExtractor { | |
| private db: Db; | |
| private options: ExtractOptions; | |
| // Aggregation maps: key -> ComponentData | |
| private strategySetsWithAssets: Map<string, ComponentData<RawPerformance>>; | |
| private strategySetsWithoutAssets: Map<string, ComponentData<RawPerformance>>; | |
| private strategiesWithAssets: Map<string, ComponentData<RawPerformance>>; | |
| private strategiesWithoutAssets: Map<string, ComponentData<RawPerformance>>; | |
| private conditionsWithAssets: Map<string, ComponentData<RawPerformance>>; | |
| private conditionsWithoutAssets: Map<string, ComponentData<RawPerformance>>; | |
| private actionsWithAssets: Map<string, ComponentData<RawPerformance>>; | |
| private actionsWithoutAssets: Map<string, ComponentData<RawPerformance>>; | |
| private indicatorsWithAssets: Map<string, ComponentData<RawPerformance>>; | |
| private indicatorsWithoutAssets: Map<string, ComponentData<RawPerformance>>; | |
| // Metadata | |
| private totalProcessed: number = 0; | |
| private totalSkipped: number = 0; | |
| private earliestDate: Date | null = null; | |
| private latestDate: Date | null = null; | |
| private timestamp: string; | |
| constructor(options: ExtractOptions) { | |
| this.options = options; | |
| this.db = new Db(options.dbEnv); | |
| // Use nanosecond timestamp for uniqueness and consistency with analysis | |
| this.timestamp = Date.now().toString() + process.hrtime()[1].toString(); | |
| // Initialize maps | |
| this.strategySetsWithAssets = new Map(); | |
| this.strategySetsWithoutAssets = new Map(); | |
| this.strategiesWithAssets = new Map(); | |
| this.strategiesWithoutAssets = new Map(); | |
| this.conditionsWithAssets = new Map(); | |
| this.conditionsWithoutAssets = new Map(); | |
| this.actionsWithAssets = new Map(); | |
| this.actionsWithoutAssets = new Map(); | |
| this.indicatorsWithAssets = new Map(); | |
| this.indicatorsWithoutAssets = new Map(); | |
| } | |
| async run(): Promise<void> { | |
| console.log("=== Backtest Data Extraction ==="); | |
| console.log(`Database: ${this.options.dbEnv}`); | |
| console.log(`Batch size: ${this.options.batchSize}`); | |
| if (this.options.dateFrom) { | |
| console.log(`Date from: ${this.options.dateFrom.toISOString()}`); | |
| } | |
| if (this.options.dateTo) { | |
| console.log(`Date to: ${this.options.dateTo.toISOString()}`); | |
| } | |
| console.log(); | |
| try { | |
| await this.connectDatabase(); | |
| await this.processBacktests(); | |
| await this.exportResults(); | |
| } catch (error) { | |
| console.error("Error during extraction:", error); | |
| throw error; | |
| } finally { | |
| await this.cleanup(); | |
| } | |
| } | |
| private async connectDatabase(): Promise<void> { | |
| await this.db.connect(); | |
| console.log("✓ Connected to database\n"); | |
| } | |
| /** | |
| * Get the Backtest model from mongoose | |
| */ | |
| private getBacktestModel() { | |
| // Ensure Backtester is loaded (triggers model registration) | |
| if (!Backtester) { | |
| throw new Error("Backtester not loaded"); | |
| } | |
| // Get the registered Backtest model | |
| const BacktestModel = mongoose.connection.models["Backtest"]; | |
| if (!BacktestModel) { | |
| throw new Error("Backtest model not found in connection.models"); | |
| } | |
| return BacktestModel; | |
| } | |
| private async processBacktests(): Promise<void> { | |
| const BacktestModel = this.getBacktestModel(); | |
| let lastId: mongoose.Types.ObjectId | null = null; | |
| let processedCount = 0; | |
| console.log("Processing backtests...\n"); | |
| while (true) { | |
| // Build filter with cursor | |
| const filter: Record<string, unknown> = { status: "COMPLETE" }; | |
| // Add date range filter | |
| if (this.options.dateFrom || this.options.dateTo) { | |
| filter.createdAt = {}; | |
| if (this.options.dateFrom) { | |
| (filter.createdAt as Record<string, unknown>).$gte = | |
| this.options.dateFrom; | |
| } | |
| if (this.options.dateTo) { | |
| (filter.createdAt as Record<string, unknown>).$lte = | |
| this.options.dateTo; | |
| } | |
| } | |
| // Add cursor pagination | |
| if (lastId) { | |
| filter._id = { $gt: lastId }; | |
| } | |
| // Fetch batch using the model | |
| const batch = (await BacktestModel.find(filter) | |
| .sort({ _id: 1 }) | |
| .limit(this.options.batchSize) | |
| .lean()) as IBacktester[]; | |
| if (batch.length === 0) { | |
| console.log("\n✓ Finished processing all backtests"); | |
| break; | |
| } | |
| // Process each backtest | |
| for (const backtest of batch) { | |
| this.processBacktest(backtest); | |
| processedCount++; | |
| if (processedCount % 100 === 0) { | |
| console.log(` Processed ${processedCount} backtests...`); | |
| } | |
| } | |
| // Update cursor | |
| lastId = batch[batch.length - 1]._id as mongoose.Types.ObjectId; | |
| } | |
| console.log(`\nTotal processed: ${this.totalProcessed}`); | |
| console.log(`Total skipped: ${this.totalSkipped}`); | |
| } | |
| private processBacktest(backtest: IBacktester): void { | |
| // Skip if no statistics | |
| if (!backtest.statistics) { | |
| this.totalSkipped++; | |
| return; | |
| } | |
| // Create raw performance object | |
| const performance: RawPerformance = { | |
| backtestId: backtest._id.toString(), | |
| backtestName: backtest.name, | |
| startDate: backtest.startDate, | |
| endDate: backtest.endDate, | |
| statistics: backtest.statistics, | |
| }; | |
| try { | |
| // Extract and store strategy sets | |
| const strategySets = extractStrategySet(backtest.portfolio); | |
| this.addToMap( | |
| this.strategySetsWithAssets, | |
| strategySets.withAssets, | |
| performance | |
| ); | |
| this.addToMap( | |
| this.strategySetsWithoutAssets, | |
| strategySets.withoutAssets, | |
| performance | |
| ); | |
| // Extract and store strategies | |
| const strategies = extractStrategies(backtest.portfolio); | |
| strategies.withAssets.forEach((s: string) => | |
| this.addToMap(this.strategiesWithAssets, s, performance) | |
| ); | |
| strategies.withoutAssets.forEach((s: string) => | |
| this.addToMap(this.strategiesWithoutAssets, s, performance) | |
| ); | |
| // Extract and store conditions | |
| const conditions = extractAllConditions(backtest.portfolio); | |
| conditions.withAssets.forEach((c: string) => | |
| this.addToMap(this.conditionsWithAssets, c, performance) | |
| ); | |
| conditions.withoutAssets.forEach((c: string) => | |
| this.addToMap(this.conditionsWithoutAssets, c, performance) | |
| ); | |
| // Extract and store actions | |
| const actions = extractActions(backtest.portfolio); | |
| actions.withAssets.forEach((a: string) => | |
| this.addToMap(this.actionsWithAssets, a, performance) | |
| ); | |
| actions.withoutAssets.forEach((a: string) => | |
| this.addToMap(this.actionsWithoutAssets, a, performance) | |
| ); | |
| // Extract and store indicators | |
| const indicators = extractIndicators(backtest.portfolio); | |
| indicators.withAssets.forEach((i) => | |
| this.addToMap(this.indicatorsWithAssets, i, performance) | |
| ); | |
| indicators.withoutAssets.forEach((i) => | |
| this.addToMap(this.indicatorsWithoutAssets, i, performance) | |
| ); | |
| this.totalProcessed++; | |
| this.updateDateRange(backtest.startDate, backtest.endDate); | |
| } catch (error: any) { | |
| console.warn( | |
| `Warning: Error processing backtest ${backtest._id}: ${error.message}` | |
| ); | |
| this.totalSkipped++; | |
| } | |
| } | |
| private addToMap( | |
| map: Map<string, ComponentData<RawPerformance>>, | |
| key: string, | |
| performance: RawPerformance | |
| ): void { | |
| if (!map.has(key)) { | |
| map.set(key, { | |
| count: 0, | |
| performances: [], | |
| }); | |
| } | |
| const component = map.get(key)!; | |
| component.count++; | |
| component.performances.push(performance); | |
| } | |
| private updateDateRange(startDate: Date, endDate: Date): void { | |
| const start = new Date(startDate); | |
| const end = new Date(endDate); | |
| if (!this.earliestDate || start < this.earliestDate) { | |
| this.earliestDate = start; | |
| } | |
| if (!this.latestDate || end > this.latestDate) { | |
| this.latestDate = end; | |
| } | |
| } | |
| /** | |
| * Write a JSONL file from a Map (streaming to avoid memory issues) | |
| */ | |
| private async writeJsonlFile( | |
| map: Map<string, ComponentData<RawPerformance>>, | |
| filename: string | |
| ): Promise<void> { | |
| // Create write stream to avoid building giant string in memory | |
| const writeStream = fs.createWriteStream(filename, { encoding: "utf-8" }); | |
| let isFirst = true; | |
| for (const [key, value] of map.entries()) { | |
| const entry = { | |
| component: key, | |
| count: value.count, | |
| performances: value.performances, | |
| }; | |
| // Add newline before all lines except the first | |
| if (!isFirst) { | |
| writeStream.write("\n"); | |
| } | |
| isFirst = false; | |
| // Write the JSON line | |
| writeStream.write(JSON.stringify(entry)); | |
| } | |
| // Close the stream and wait for it to finish | |
| return new Promise((resolve, reject) => { | |
| writeStream.end(() => resolve()); | |
| writeStream.on("error", reject); | |
| }); | |
| } | |
| private async exportResults(): Promise<void> { | |
| const baseOutputDir = path.join( | |
| __dirname, | |
| "..", | |
| "..", | |
| "..", | |
| "python", | |
| "output", | |
| "raw_backtest_data" | |
| ); | |
| // Create timestamped folder for this extraction | |
| const outputDir = path.join(baseOutputDir, this.timestamp); | |
| // Ensure output directory exists | |
| if (!fs.existsSync(outputDir)) { | |
| fs.mkdirSync(outputDir, { recursive: true }); | |
| } | |
| console.log("\nExporting results to JSONL files..."); | |
| // Write metadata file | |
| const metadata = { | |
| generatedAt: new Date(), | |
| totalBacktests: this.totalProcessed, | |
| totalSkipped: this.totalSkipped, | |
| dateRange: { | |
| earliest: this.earliestDate, | |
| latest: this.latestDate, | |
| }, | |
| timestamp: this.timestamp, | |
| }; | |
| await fs.promises.writeFile( | |
| path.join(outputDir, `metadata.json`), | |
| JSON.stringify(metadata, null, 2) | |
| ); | |
| // Write JSONL files for each component category | |
| await this.writeJsonlFile( | |
| this.strategySetsWithAssets, | |
| path.join(outputDir, `strategySetsWithAssets.jsonl`) | |
| ); | |
| await this.writeJsonlFile( | |
| this.strategySetsWithoutAssets, | |
| path.join(outputDir, `strategySetsWithoutAssets.jsonl`) | |
| ); | |
| await this.writeJsonlFile( | |
| this.strategiesWithAssets, | |
| path.join(outputDir, `strategiesWithAssets.jsonl`) | |
| ); | |
| await this.writeJsonlFile( | |
| this.strategiesWithoutAssets, | |
| path.join(outputDir, `strategiesWithoutAssets.jsonl`) | |
| ); | |
| await this.writeJsonlFile( | |
| this.conditionsWithAssets, | |
| path.join(outputDir, `conditionsWithAssets.jsonl`) | |
| ); | |
| await this.writeJsonlFile( | |
| this.conditionsWithoutAssets, | |
| path.join(outputDir, `conditionsWithoutAssets.jsonl`) | |
| ); | |
| await this.writeJsonlFile( | |
| this.actionsWithAssets, | |
| path.join(outputDir, `actionsWithAssets.jsonl`) | |
| ); | |
| await this.writeJsonlFile( | |
| this.actionsWithoutAssets, | |
| path.join(outputDir, `actionsWithoutAssets.jsonl`) | |
| ); | |
| await this.writeJsonlFile( | |
| this.indicatorsWithAssets, | |
| path.join(outputDir, `indicatorsWithAssets.jsonl`) | |
| ); | |
| await this.writeJsonlFile( | |
| this.indicatorsWithoutAssets, | |
| path.join(outputDir, `indicatorsWithoutAssets.jsonl`) | |
| ); | |
| console.log(`\n✓ Results exported to: ${outputDir}/*.jsonl`); | |
| // Print summary statistics | |
| console.log("\n=== Summary ==="); | |
| console.log( | |
| `Strategy Sets (with assets): ${this.strategySetsWithAssets.size}` | |
| ); | |
| console.log( | |
| `Strategy Sets (without assets): ${this.strategySetsWithoutAssets.size}` | |
| ); | |
| console.log(`Strategies (with assets): ${this.strategiesWithAssets.size}`); | |
| console.log( | |
| `Strategies (without assets): ${this.strategiesWithoutAssets.size}` | |
| ); | |
| console.log(`Conditions (with assets): ${this.conditionsWithAssets.size}`); | |
| console.log( | |
| `Conditions (without assets): ${this.conditionsWithoutAssets.size}` | |
| ); | |
| console.log(`Actions (with assets): ${this.actionsWithAssets.size}`); | |
| console.log(`Actions (without assets): ${this.actionsWithoutAssets.size}`); | |
| console.log(`Indicators (with assets): ${this.indicatorsWithAssets.size}`); | |
| console.log( | |
| `Indicators (without assets): ${this.indicatorsWithoutAssets.size}` | |
| ); | |
| } | |
| private async cleanup(): Promise<void> { | |
| await this.db.disconnect(); | |
| console.log("\n✓ Database connection closed"); | |
| } | |
| } | |
| // Parse CLI arguments | |
| function parseArgs(): ExtractOptions { | |
| const args = process.argv.slice(2); | |
| const options: ExtractOptions = { | |
| dbEnv: "cloud", | |
| batchSize: 100, | |
| }; | |
| for (let i = 0; i < args.length; i++) { | |
| switch (args[i]) { | |
| case "--db-env": | |
| options.dbEnv = args[++i] as "cloud" | "local"; | |
| break; | |
| case "--batch-size": | |
| options.batchSize = parseInt(args[++i], 10); | |
| break; | |
| case "--date-from": | |
| options.dateFrom = new Date(args[++i]); | |
| break; | |
| case "--date-to": | |
| options.dateTo = new Date(args[++i]); | |
| break; | |
| case "--help": | |
| console.log(` | |
| Usage: npm run analytics:extract -- [options] | |
| Options: | |
| --db-env <env> Database environment (cloud|local) [default: cloud] | |
| --batch-size <size> Number of backtests to process per batch [default: 100] | |
| --date-from <date> Filter backtests from this date (ISO format) | |
| --date-to <date> Filter backtests to this date (ISO format) | |
| --help Show this help message | |
| Example: | |
| npm run analytics:extract -- --db-env cloud --batch-size 50 --date-from 2024-01-01 | |
| `); | |
| process.exit(0); | |
| } | |
| } | |
| return options; | |
| } | |
| // Main execution | |
| if (require.main === module) { | |
| const options = parseArgs(); | |
| const extractor = new BacktestDataExtractor(options); | |
| extractor | |
| .run() | |
| .then(() => { | |
| console.log("\n✓ Extraction completed successfully"); | |
| process.exit(0); | |
| }) | |
| .catch((error) => { | |
| console.error("\n✗ Extraction failed:", error); | |
| process.exit(1); | |
| }); | |
| } | |
| export default BacktestDataExtractor; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment