Skip to content

Instantly share code, notes, and snippets.

@DonCarlosss

DonCarlosss/lorentzian_notebook.ipynb

Last active May 10, 2025 19:06
Show Gist options
  • Save DonCarlosss/7e8d417be47638cab0b36406ea3e16c1 to your computer and use it in GitHub Desktop.
Save DonCarlosss/7e8d417be47638cab0b36406ea3e16c1 to your computer and use it in GitHub Desktop.
Download ZIP
lorentzian test
Raw
gistfile1.txt
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "f77116df",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Added project root to sys.path: /Users/carletes/Documents\n",
"Imports successful.\n"
]
}
],
"source": [
"# Cell 1: Imports\n",
"import pandas as pd\n",
"import numpy as np\n",
"import os\n",
"\n",
"# To make sure we can import our custom modules from the 'core' directory\n",
"import sys\n",
"# Assuming the notebook is in 'lorentzian/notebooks/' and 'core' is in 'lorentzian/core/'\n",
"notebook_dir = os.path.dirname(os.path.abspath('__file__')) # Gets current directory of notebook\n",
"project_root = os.path.abspath(os.path.join(notebook_dir, '..'))\n",
"if project_root not in sys.path:\n",
" sys.path.insert(0, project_root)\n",
" print(f\"Added project root to sys.path: {project_root}\")\n",
"\n",
"from core.config_manager import ConfigManager\n",
"print(\"Imports successful.\")\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "8e476e2b",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Using KNOWN ABSOLUTE path for CSV: /Users/carletes/Documents/lorentzian/XAUUSD_GMT+2_US-DST-TICKS- 5-4-2010 to 4-22-2025.csv\n",
"Using tick_column_names: ['Timestamp_str', 'Bid', 'Ask']\n",
"Resample Timeframe: h\n",
"Detected project root for output files: /Users/carletes/Documents/lorentzian\n",
"\n",
"--- Final Configuration for Data Loading ---\n",
"Input CSV (Target Path): /Users/carletes/Documents/lorentzian/XAUUSD_GMT+2_US-DST-TICKS- 5-4-2010 to 4-22-2025.csv\n",
"Price column for OHLC: Bid\n",
"Output Parquet: /Users/carletes/Documents/lorentzian/processed_data/XAUUSD_OHLC_h.parquet\n"
]
}
],
"source": [
"# Cell 2: Configuration\n",
"import os # Ensure os is imported if not already from Cell 1\n",
"from core.config_manager import ConfigManager # Assuming ConfigManager is needed for other settings\n",
"\n",
"# --- Instantiate ConfigManager ---\n",
"# We still instantiate it as it holds other useful parameters like resample_timeframe, column names etc.\n",
"# We will override the data_file_path it might try to construct.\n",
"config = ConfigManager()\n",
"\n",
"# --- CRITICAL: Use the KNOWN ABSOLUTE PATH to your CSV file ---\n",
"csv_file_path = \"/Users/carletes/Documents/lorentzian/XAUUSD_GMT+2_US-DST-TICKS- 5-4-2010 to 4-22-2025.csv\"\n",
"print(f\"Using KNOWN ABSOLUTE path for CSV: {csv_file_path}\")\n",
"\n",
"# --- Define your column names here ---\n",
"# Based on your CSV preview (Timestamp, Bid, Ask)\n",
"tick_column_names = ['Timestamp_str', 'Bid', 'Ask']\n",
"print(f\"Using tick_column_names: {tick_column_names}\")\n",
"\n",
"# --- MODIFIED PART ---\n",
"# Desired OHLC timeframe for resampling\n",
"resample_timeframe = \"h\" # <--- CORRECTED to lowercase 'h' for future-proofing\n",
"print(f\"Resample Timeframe: {resample_timeframe}\")\n",
"\n",
"# Which price to use for OHLC resampling (from ConfigManager)\n",
"price_col_for_ohlc = config.tick_bid_col # This should be 'Bid' from your tick_column_names\n",
"if price_col_for_ohlc not in tick_column_names:\n",
" print(f\"Warning! Configured price_col_for_ohlc '{price_col_for_ohlc}' not in assigned tick_column_names {tick_column_names}.\")\n",
" # Fallback if 'Bid' (from config.tick_bid_col) isn't in your assigned tick_column_names\n",
" # This should not happen if tick_column_names = ['Timestamp_str', 'Bid', 'Ask'] and config.tick_bid_col = 'Bid'\n",
" price_col_for_ohlc = tick_column_names[1] \n",
" print(f\"Fallback: Using '{price_col_for_ohlc}' as price column for OHLC.\")\n",
"\n",
"\n",
"# --- Output path for the processed OHLC Parquet file ---\n",
"# Determine project root for saving processed data (assuming notebook is in 'lorentzian' or 'lorentzian/notebooks')\n",
"current_path = os.getcwd() # Gets the current working directory of the notebook kernel\n",
"project_root_for_output = current_path\n",
"# Navigate up until 'lorentzian' is the last part of the path or we hit the filesystem root\n",
"# This helps if the notebook is run from a subdirectory like 'lorentzian/notebooks'\n",
"while os.path.basename(project_root_for_output).lower() != 'lorentzian' and project_root_for_output != os.path.dirname(project_root_for_output):\n",
" project_root_for_output = os.path.dirname(project_root_for_output)\n",
"\n",
"# If 'lorentzian' was not found by going up, it means CWD might already be the project root or something else\n",
"if os.path.basename(project_root_for_output).lower() != 'lorentzian':\n",
" # If CWD itself is 'lorentzian', then project_root_for_output is already correct\n",
" if os.path.basename(current_path).lower() == 'lorentzian':\n",
" project_root_for_output = current_path\n",
" else: # Fallback if 'lorentzian' can't be clearly identified\n",
" project_root_for_output = current_path # Default to CWD for output\n",
" print(f\"Warning: Could not robustly auto-detect 'lorentzian' project root for output based on CWD. Using CWD for output: {project_root_for_output}\")\n",
"print(f\"Detected project root for output files: {project_root_for_output}\")\n",
"\n",
"\n",
"processed_data_dir = os.path.join(project_root_for_output, \"processed_data\")\n",
"os.makedirs(processed_data_dir, exist_ok=True) # Create directory if it doesn't exist\n",
"# The output filename will now use the updated resample_timeframe (e.g., \"h\")\n",
"output_parquet_path = os.path.join(processed_data_dir, \"XAUUSD_OHLC_\" + resample_timeframe + \".parquet\")\n",
"\n",
"\n",
"print(f\"\\n--- Final Configuration for Data Loading ---\")\n",
"print(f\"Input CSV (Target Path): {csv_file_path}\")\n",
"print(f\"Price column for OHLC: {price_col_for_ohlc}\")\n",
"print(f\"Output Parquet: {output_parquet_path}\") # This will now show ..._OHLC_h.parquet\n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "75bf92fd",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Preview of the first 10 rows (raw, no headers assigned yet by read_csv here):\n",
" 0 1 2\n",
"0 2010.05.04 03:01:04.777 1182.580 1183.242\n",
"1 2010.05.04 03:01:05.453 1182.598 1183.302\n",
"2 2010.05.04 03:01:10.693 1182.633 1183.337\n",
"3 2010.05.04 03:01:23.180 1182.677 1183.383\n",
"4 2010.05.04 03:01:31.364 1182.780 1183.320\n",
"5 2010.05.04 03:01:46.912 1182.704 1183.408\n",
"6 2010.05.04 03:01:51.669 1182.918 1183.278\n",
"7 2010.05.04 03:01:55.776 1182.747 1183.453\n",
"8 2010.05.04 03:02:07.965 1182.692 1183.396\n",
"9 2010.05.04 03:02:15.719 1182.658 1183.364\n",
"\n",
"Number of columns detected: 3\n",
"Column count matches 'tick_column_names' definition. Good to proceed.\n"
]
}
],
"source": [
"# Cell 3: Inspect a Small Portion of the CSV\n",
"# Load only the first few rows to understand the structure and delimiter\n",
"try:\n",
" preview_df = pd.read_csv(csv_file_path, nrows=10, header=None) # header=None because no columns in file\n",
" print(\"Preview of the first 10 rows (raw, no headers assigned yet by read_csv here):\")\n",
" print(preview_df)\n",
" print(f\"\\nNumber of columns detected: {preview_df.shape[1]}\")\n",
" if preview_df.shape[1] != len(tick_column_names):\n",
" print(f\"ERROR: Detected {preview_df.shape[1]} columns, but 'tick_column_names' in Cell 2 expects {len(tick_column_names)}.\")\n",
" print(\"Please adjust 'tick_column_names' in Cell 2 to match the actual number of columns in your CSV!\")\n",
" else:\n",
" print(\"Column count matches 'tick_column_names' definition. Good to proceed.\")\n",
"except FileNotFoundError:\n",
" print(f\"ERROR: CSV file not found at {csv_file_path}. Please check the path and filename in Cell 2.\")\n",
"except Exception as e:\n",
" print(f\"Error reading preview of CSV: {e}\")\n",
" print(\"Check file path, permissions, and CSV integrity.\")\n"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "d5908400",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Starting to process /Users/carletes/Documents/lorentzian/XAUUSD_GMT+2_US-DST-TICKS- 5-4-2010 to 4-22-2025.csv in chunks of 1000000 rows...\n",
"Processing chunk 1 (Shape: (1000000, 3))...\n",
" Chunk 1 processed. Resampled OHLC shape: (551, 5)\n",
"Processing chunk 2 (Shape: (1000000, 3))...\n",
" Chunk 2 processed. Resampled OHLC shape: (511, 5)\n",
"Processing chunk 3 (Shape: (1000000, 3))...\n",
" Chunk 3 processed. Resampled OHLC shape: (433, 5)\n",
"Processing chunk 4 (Shape: (1000000, 3))...\n",
" Chunk 4 processed. Resampled OHLC shape: (398, 5)\n",
"Processing chunk 5 (Shape: (1000000, 3))...\n",
" Chunk 5 processed. Resampled OHLC shape: (377, 5)\n",
"Processing chunk 6 (Shape: (1000000, 3))...\n",
" Chunk 6 processed. Resampled OHLC shape: (330, 5)\n",
"Processing chunk 7 (Shape: (1000000, 3))...\n",
" Chunk 7 processed. Resampled OHLC shape: (304, 5)\n",
"Processing chunk 8 (Shape: (1000000, 3))...\n",
" Chunk 8 processed. Resampled OHLC shape: (289, 5)\n",
"Processing chunk 9 (Shape: (1000000, 3))...\n",
" Chunk 9 processed. Resampled OHLC shape: (375, 5)\n",
"Processing chunk 10 (Shape: (1000000, 3))...\n",
" Chunk 10 processed. Resampled OHLC shape: (617, 5)\n",
"Processing chunk 11 (Shape: (1000000, 3))...\n",
" Chunk 11 processed. Resampled OHLC shape: (553, 5)\n",
"Processing chunk 12 (Shape: (1000000, 3))...\n",
" Chunk 12 processed. Resampled OHLC shape: (616, 5)\n",
"Processing chunk 13 (Shape: (1000000, 3))...\n",
" Chunk 13 processed. Resampled OHLC shape: (506, 5)\n",
"Processing chunk 14 (Shape: (1000000, 3))...\n",
" Chunk 14 processed. Resampled OHLC shape: (457, 5)\n",
"Processing chunk 15 (Shape: (1000000, 3))...\n",
" Chunk 15 processed. Resampled OHLC shape: (359, 5)\n",
"Processing chunk 16 (Shape: (1000000, 3))...\n",
" Chunk 16 processed. Resampled OHLC shape: (477, 5)\n",
"Processing chunk 17 (Shape: (1000000, 3))...\n",
" Chunk 17 processed. Resampled OHLC shape: (458, 5)\n",
"Processing chunk 18 (Shape: (1000000, 3))...\n",
" Chunk 18 processed. Resampled OHLC shape: (346, 5)\n",
"Processing chunk 19 (Shape: (1000000, 3))...\n",
" Chunk 19 processed. Resampled OHLC shape: (254, 5)\n",
"Processing chunk 20 (Shape: (1000000, 3))...\n",
" Chunk 20 processed. Resampled OHLC shape: (238, 5)\n",
"Processing chunk 21 (Shape: (1000000, 3))...\n",
" Chunk 21 processed. Resampled OHLC shape: (247, 5)\n",
"Processing chunk 22 (Shape: (1000000, 3))...\n",
" Chunk 22 processed. Resampled OHLC shape: (200, 5)\n",
"Processing chunk 23 (Shape: (1000000, 3))...\n",
" Chunk 23 processed. Resampled OHLC shape: (286, 5)\n",
"Processing chunk 24 (Shape: (1000000, 3))...\n",
" Chunk 24 processed. Resampled OHLC shape: (285, 5)\n",
"Processing chunk 25 (Shape: (1000000, 3))...\n",
" Chunk 25 processed. Resampled OHLC shape: (284, 5)\n",
"Processing chunk 26 (Shape: (1000000, 3))...\n",
" Chunk 26 processed. Resampled OHLC shape: (282, 5)\n",
"Processing chunk 27 (Shape: (1000000, 3))...\n",
" Chunk 27 processed. Resampled OHLC shape: (208, 5)\n",
"Processing chunk 28 (Shape: (1000000, 3))...\n",
" Chunk 28 processed. Resampled OHLC shape: (273, 5)\n",
"Processing chunk 29 (Shape: (1000000, 3))...\n",
" Chunk 29 processed. Resampled OHLC shape: (237, 5)\n",
"Processing chunk 30 (Shape: (1000000, 3))...\n",
" Chunk 30 processed. Resampled OHLC shape: (272, 5)\n",
"Processing chunk 31 (Shape: (1000000, 3))...\n",
" Chunk 31 processed. Resampled OHLC shape: (302, 5)\n",
"Processing chunk 32 (Shape: (1000000, 3))...\n",
" Chunk 32 processed. Resampled OHLC shape: (285, 5)\n",
"Processing chunk 33 (Shape: (1000000, 3))...\n",
" Chunk 33 processed. Resampled OHLC shape: (277, 5)\n",
"Processing chunk 34 (Shape: (1000000, 3))...\n",
" Chunk 34 processed. Resampled OHLC shape: (267, 5)\n",
"Processing chunk 35 (Shape: (1000000, 3))...\n",
" Chunk 35 processed. Resampled OHLC shape: (305, 5)\n",
"Processing chunk 36 (Shape: (1000000, 3))...\n",
" Chunk 36 processed. Resampled OHLC shape: (260, 5)\n",
"Processing chunk 37 (Shape: (1000000, 3))...\n",
" Chunk 37 processed. Resampled OHLC shape: (220, 5)\n",
"Processing chunk 38 (Shape: (1000000, 3))...\n",
" Chunk 38 processed. Resampled OHLC shape: (205, 5)\n",
"Processing chunk 39 (Shape: (1000000, 3))...\n",
" Chunk 39 processed. Resampled OHLC shape: (236, 5)\n",
"Processing chunk 40 (Shape: (1000000, 3))...\n",
" Chunk 40 processed. Resampled OHLC shape: (251, 5)\n",
"Processing chunk 41 (Shape: (1000000, 3))...\n",
" Chunk 41 processed. Resampled OHLC shape: (239, 5)\n",
"Processing chunk 42 (Shape: (1000000, 3))...\n",
" Chunk 42 processed. Resampled OHLC shape: (258, 5)\n",
"Processing chunk 43 (Shape: (1000000, 3))...\n",
" Chunk 43 processed. Resampled OHLC shape: (307, 5)\n",
"Processing chunk 44 (Shape: (1000000, 3))...\n",
" Chunk 44 processed. Resampled OHLC shape: (272, 5)\n",
"Processing chunk 45 (Shape: (1000000, 3))...\n",
" Chunk 45 processed. Resampled OHLC shape: (269, 5)\n",
"Processing chunk 46 (Shape: (1000000, 3))...\n",
" Chunk 46 processed. Resampled OHLC shape: (285, 5)\n",
"Processing chunk 47 (Shape: (1000000, 3))...\n",
" Chunk 47 processed. Resampled OHLC shape: (350, 5)\n",
"Processing chunk 48 (Shape: (1000000, 3))...\n",
" Chunk 48 processed. Resampled OHLC shape: (354, 5)\n",
"Processing chunk 49 (Shape: (1000000, 3))...\n",
" Chunk 49 processed. Resampled OHLC shape: (345, 5)\n",
"Processing chunk 50 (Shape: (1000000, 3))...\n",
" Chunk 50 processed. Resampled OHLC shape: (412, 5)\n",
"Processing chunk 51 (Shape: (1000000, 3))...\n",
" Chunk 51 processed. Resampled OHLC shape: (306, 5)\n",
"Processing chunk 52 (Shape: (1000000, 3))...\n",
" Chunk 52 processed. Resampled OHLC shape: (292, 5)\n",
"Processing chunk 53 (Shape: (1000000, 3))...\n",
" Chunk 53 processed. Resampled OHLC shape: (226, 5)\n",
"Processing chunk 54 (Shape: (1000000, 3))...\n",
" Chunk 54 processed. Resampled OHLC shape: (381, 5)\n",
"Processing chunk 55 (Shape: (1000000, 3))...\n",
" Chunk 55 processed. Resampled OHLC shape: (302, 5)\n",
"Processing chunk 56 (Shape: (1000000, 3))...\n",
" Chunk 56 processed. Resampled OHLC shape: (230, 5)\n",
"Processing chunk 57 (Shape: (1000000, 3))...\n",
" Chunk 57 processed. Resampled OHLC shape: (282, 5)\n",
"Processing chunk 58 (Shape: (1000000, 3))...\n",
" Chunk 58 processed. Resampled OHLC shape: (242, 5)\n",
"Processing chunk 59 (Shape: (1000000, 3))...\n",
" Chunk 59 processed. Resampled OHLC shape: (268, 5)\n",
"Processing chunk 60 (Shape: (1000000, 3))...\n",
" Chunk 60 processed. Resampled OHLC shape: (210, 5)\n",
"Processing chunk 61 (Shape: (1000000, 3))...\n",
" Chunk 61 processed. Resampled OHLC shape: (257, 5)\n",
"Processing chunk 62 (Shape: (1000000, 3))...\n",
" Chunk 62 processed. Resampled OHLC shape: (293, 5)\n",
"Processing chunk 63 (Shape: (1000000, 3))...\n",
" Chunk 63 processed. Resampled OHLC shape: (295, 5)\n",
"Processing chunk 64 (Shape: (1000000, 3))...\n",
" Chunk 64 processed. Resampled OHLC shape: (261, 5)\n",
"Processing chunk 65 (Shape: (1000000, 3))...\n",
" Chunk 65 processed. Resampled OHLC shape: (276, 5)\n",
"Processing chunk 66 (Shape: (1000000, 3))...\n",
" Chunk 66 processed. Resampled OHLC shape: (237, 5)\n",
"Processing chunk 67 (Shape: (1000000, 3))...\n",
" Chunk 67 processed. Resampled OHLC shape: (189, 5)\n",
"Processing chunk 68 (Shape: (1000000, 3))...\n",
" Chunk 68 processed. Resampled OHLC shape: (212, 5)\n",
"Processing chunk 69 (Shape: (1000000, 3))...\n",
" Chunk 69 processed. Resampled OHLC shape: (277, 5)\n",
"Processing chunk 70 (Shape: (1000000, 3))...\n",
" Chunk 70 processed. Resampled OHLC shape: (258, 5)\n",
"Processing chunk 71 (Shape: (1000000, 3))...\n",
" Chunk 71 processed. Resampled OHLC shape: (266, 5)\n",
"Processing chunk 72 (Shape: (1000000, 3))...\n",
" Chunk 72 processed. Resampled OHLC shape: (296, 5)\n",
"Processing chunk 73 (Shape: (1000000, 3))...\n",
" Chunk 73 processed. Resampled OHLC shape: (273, 5)\n",
"Processing chunk 74 (Shape: (1000000, 3))...\n",
" Chunk 74 processed. Resampled OHLC shape: (270, 5)\n",
"Processing chunk 75 (Shape: (1000000, 3))...\n",
" Chunk 75 processed. Resampled OHLC shape: (266, 5)\n",
"Processing chunk 76 (Shape: (1000000, 3))...\n",
" Chunk 76 processed. Resampled OHLC shape: (261, 5)\n",
"Processing chunk 77 (Shape: (1000000, 3))...\n",
" Chunk 77 processed. Resampled OHLC shape: (269, 5)\n",
"Processing chunk 78 (Shape: (1000000, 3))...\n",
" Chunk 78 processed. Resampled OHLC shape: (313, 5)\n",
"Processing chunk 79 (Shape: (1000000, 3))...\n",
" Chunk 79 processed. Resampled OHLC shape: (326, 5)\n",
"Processing chunk 80 (Shape: (1000000, 3))...\n",
" Chunk 80 processed. Resampled OHLC shape: (328, 5)\n",
"Processing chunk 81 (Shape: (1000000, 3))...\n",
" Chunk 81 processed. Resampled OHLC shape: (347, 5)\n",
"Processing chunk 82 (Shape: (1000000, 3))...\n",
" Chunk 82 processed. Resampled OHLC shape: (306, 5)\n",
"Processing chunk 83 (Shape: (1000000, 3))...\n",
" Chunk 83 processed. Resampled OHLC shape: (324, 5)\n",
"Processing chunk 84 (Shape: (1000000, 3))...\n",
" Chunk 84 processed. Resampled OHLC shape: (349, 5)\n",
"Processing chunk 85 (Shape: (1000000, 3))...\n",
" Chunk 85 processed. Resampled OHLC shape: (343, 5)\n",
"Processing chunk 86 (Shape: (1000000, 3))...\n",
" Chunk 86 processed. Resampled OHLC shape: (307, 5)\n",
"Processing chunk 87 (Shape: (1000000, 3))...\n",
" Chunk 87 processed. Resampled OHLC shape: (253, 5)\n",
"Processing chunk 88 (Shape: (1000000, 3))...\n",
" Chunk 88 processed. Resampled OHLC shape: (244, 5)\n",
"Processing chunk 89 (Shape: (1000000, 3))...\n",
" Chunk 89 processed. Resampled OHLC shape: (258, 5)\n",
"Processing chunk 90 (Shape: (1000000, 3))...\n",
" Chunk 90 processed. Resampled OHLC shape: (204, 5)\n",
"Processing chunk 91 (Shape: (1000000, 3))...\n",
" Chunk 91 processed. Resampled OHLC shape: (202, 5)\n",
"Processing chunk 92 (Shape: (1000000, 3))...\n",
" Chunk 92 processed. Resampled OHLC shape: (240, 5)\n",
"Processing chunk 93 (Shape: (1000000, 3))...\n",
" Chunk 93 processed. Resampled OHLC shape: (203, 5)\n",
"Processing chunk 94 (Shape: (1000000, 3))...\n",
" Chunk 94 processed. Resampled OHLC shape: (306, 5)\n",
"Processing chunk 95 (Shape: (1000000, 3))...\n",
" Chunk 95 processed. Resampled OHLC shape: (188, 5)\n",
"Processing chunk 96 (Shape: (1000000, 3))...\n",
" Chunk 96 processed. Resampled OHLC shape: (207, 5)\n",
"Processing chunk 97 (Shape: (1000000, 3))...\n",
" Chunk 97 processed. Resampled OHLC shape: (252, 5)\n",
"Processing chunk 98 (Shape: (1000000, 3))...\n",
" Chunk 98 processed. Resampled OHLC shape: (282, 5)\n",
"Processing chunk 99 (Shape: (1000000, 3))...\n",
" Chunk 99 processed. Resampled OHLC shape: (272, 5)\n",
"Processing chunk 100 (Shape: (1000000, 3))...\n",
" Chunk 100 processed. Resampled OHLC shape: (315, 5)\n",
"Processing chunk 101 (Shape: (1000000, 3))...\n",
" Chunk 101 processed. Resampled OHLC shape: (267, 5)\n",
"Processing chunk 102 (Shape: (1000000, 3))...\n",
" Chunk 102 processed. Resampled OHLC shape: (259, 5)\n",
"Processing chunk 103 (Shape: (1000000, 3))...\n",
" Chunk 103 processed. Resampled OHLC shape: (285, 5)\n",
"Processing chunk 104 (Shape: (1000000, 3))...\n",
" Chunk 104 processed. Resampled OHLC shape: (278, 5)\n",
"Processing chunk 105 (Shape: (1000000, 3))...\n",
" Chunk 105 processed. Resampled OHLC shape: (290, 5)\n",
"Processing chunk 106 (Shape: (1000000, 3))...\n",
" Chunk 106 processed. Resampled OHLC shape: (260, 5)\n",
"Processing chunk 107 (Shape: (1000000, 3))...\n",
" Chunk 107 processed. Resampled OHLC shape: (194, 5)\n",
"Processing chunk 108 (Shape: (1000000, 3))...\n",
" Chunk 108 processed. Resampled OHLC shape: (230, 5)\n",
"Processing chunk 109 (Shape: (1000000, 3))...\n",
" Chunk 109 processed. Resampled OHLC shape: (218, 5)\n",
"Processing chunk 110 (Shape: (1000000, 3))...\n",
" Chunk 110 processed. Resampled OHLC shape: (208, 5)\n",
"Processing chunk 111 (Shape: (1000000, 3))...\n",
" Chunk 111 processed. Resampled OHLC shape: (260, 5)\n",
"Processing chunk 112 (Shape: (1000000, 3))...\n",
" Chunk 112 processed. Resampled OHLC shape: (221, 5)\n",
"Processing chunk 113 (Shape: (1000000, 3))...\n",
" Chunk 113 processed. Resampled OHLC shape: (191, 5)\n",
"Processing chunk 114 (Shape: (1000000, 3))...\n",
" Chunk 114 processed. Resampled OHLC shape: (202, 5)\n",
"Processing chunk 115 (Shape: (1000000, 3))...\n",
" Chunk 115 processed. Resampled OHLC shape: (218, 5)\n",
"Processing chunk 116 (Shape: (1000000, 3))...\n",
" Chunk 116 processed. Resampled OHLC shape: (186, 5)\n",
"Processing chunk 117 (Shape: (1000000, 3))...\n",
" Chunk 117 processed. Resampled OHLC shape: (192, 5)\n",
"Processing chunk 118 (Shape: (1000000, 3))...\n",
" Chunk 118 processed. Resampled OHLC shape: (173, 5)\n",
"Processing chunk 119 (Shape: (1000000, 3))...\n",
" Chunk 119 processed. Resampled OHLC shape: (183, 5)\n",
"Processing chunk 120 (Shape: (1000000, 3))...\n",
" Chunk 120 processed. Resampled OHLC shape: (161, 5)\n",
"Processing chunk 121 (Shape: (1000000, 3))...\n",
" Chunk 121 processed. Resampled OHLC shape: (149, 5)\n",
"Processing chunk 122 (Shape: (1000000, 3))...\n",
" Chunk 122 processed. Resampled OHLC shape: (162, 5)\n",
"Processing chunk 123 (Shape: (1000000, 3))...\n",
" Chunk 123 processed. Resampled OHLC shape: (151, 5)\n",
"Processing chunk 124 (Shape: (1000000, 3))...\n",
" Chunk 124 processed. Resampled OHLC shape: (110, 5)\n",
"Processing chunk 125 (Shape: (1000000, 3))...\n",
" Chunk 125 processed. Resampled OHLC shape: (119, 5)\n",
"Processing chunk 126 (Shape: (1000000, 3))...\n",
" Chunk 126 processed. Resampled OHLC shape: (112, 5)\n",
"Processing chunk 127 (Shape: (1000000, 3))...\n",
" Chunk 127 processed. Resampled OHLC shape: (116, 5)\n",
"Processing chunk 128 (Shape: (1000000, 3))...\n",
" Chunk 128 processed. Resampled OHLC shape: (108, 5)\n",
"Processing chunk 129 (Shape: (1000000, 3))...\n",
" Chunk 129 processed. Resampled OHLC shape: (114, 5)\n",
"Processing chunk 130 (Shape: (1000000, 3))...\n",
" Chunk 130 processed. Resampled OHLC shape: (125, 5)\n",
"Processing chunk 131 (Shape: (1000000, 3))...\n",
" Chunk 131 processed. Resampled OHLC shape: (125, 5)\n",
"Processing chunk 132 (Shape: (1000000, 3))...\n",
" Chunk 132 processed. Resampled OHLC shape: (111, 5)\n",
"Processing chunk 133 (Shape: (1000000, 3))...\n",
" Chunk 133 processed. Resampled OHLC shape: (123, 5)\n",
"Processing chunk 134 (Shape: (1000000, 3))...\n",
" Chunk 134 processed. Resampled OHLC shape: (119, 5)\n",
"Processing chunk 135 (Shape: (1000000, 3))...\n",
" Chunk 135 processed. Resampled OHLC shape: (123, 5)\n",
"Processing chunk 136 (Shape: (1000000, 3))...\n",
" Chunk 136 processed. Resampled OHLC shape: (115, 5)\n",
"Processing chunk 137 (Shape: (1000000, 3))...\n",
" Chunk 137 processed. Resampled OHLC shape: (119, 5)\n",
"Processing chunk 138 (Shape: (1000000, 3))...\n",
" Chunk 138 processed. Resampled OHLC shape: (119, 5)\n",
"Processing chunk 139 (Shape: (1000000, 3))...\n",
" Chunk 139 processed. Resampled OHLC shape: (142, 5)\n",
"Processing chunk 140 (Shape: (1000000, 3))...\n",
" Chunk 140 processed. Resampled OHLC shape: (148, 5)\n",
"Processing chunk 141 (Shape: (1000000, 3))...\n",
" Chunk 141 processed. Resampled OHLC shape: (137, 5)\n",
"Processing chunk 142 (Shape: (1000000, 3))...\n",
" Chunk 142 processed. Resampled OHLC shape: (123, 5)\n",
"Processing chunk 143 (Shape: (1000000, 3))...\n",
" Chunk 143 processed. Resampled OHLC shape: (133, 5)\n",
"Processing chunk 144 (Shape: (1000000, 3))...\n",
" Chunk 144 processed. Resampled OHLC shape: (116, 5)\n",
"Processing chunk 145 (Shape: (1000000, 3))...\n",
" Chunk 145 processed. Resampled OHLC shape: (111, 5)\n",
"Processing chunk 146 (Shape: (1000000, 3))...\n",
" Chunk 146 processed. Resampled OHLC shape: (124, 5)\n",
"Processing chunk 147 (Shape: (1000000, 3))...\n",
" Chunk 147 processed. Resampled OHLC shape: (133, 5)\n",
"Processing chunk 148 (Shape: (1000000, 3))...\n",
" Chunk 148 processed. Resampled OHLC shape: (142, 5)\n",
"Processing chunk 149 (Shape: (1000000, 3))...\n",
" Chunk 149 processed. Resampled OHLC shape: (140, 5)\n",
"Processing chunk 150 (Shape: (1000000, 3))...\n",
" Chunk 150 processed. Resampled OHLC shape: (139, 5)\n",
"Processing chunk 151 (Shape: (1000000, 3))...\n",
" Chunk 151 processed. Resampled OHLC shape: (135, 5)\n",
"Processing chunk 152 (Shape: (1000000, 3))...\n",
" Chunk 152 processed. Resampled OHLC shape: (120, 5)\n",
"Processing chunk 153 (Shape: (1000000, 3))...\n",
" Chunk 153 processed. Resampled OHLC shape: (139, 5)\n",
"Processing chunk 154 (Shape: (1000000, 3))...\n",
" Chunk 154 processed. Resampled OHLC shape: (137, 5)\n",
"Processing chunk 155 (Shape: (1000000, 3))...\n",
" Chunk 155 processed. Resampled OHLC shape: (122, 5)\n",
"Processing chunk 156 (Shape: (1000000, 3))...\n",
" Chunk 156 processed. Resampled OHLC shape: (156, 5)\n",
"Processing chunk 157 (Shape: (1000000, 3))...\n",
" Chunk 157 processed. Resampled OHLC shape: (186, 5)\n",
"Processing chunk 158 (Shape: (1000000, 3))...\n",
" Chunk 158 processed. Resampled OHLC shape: (141, 5)\n",
"Processing chunk 159 (Shape: (1000000, 3))...\n",
" Chunk 159 processed. Resampled OHLC shape: (92, 5)\n",
"Processing chunk 160 (Shape: (1000000, 3))...\n",
" Chunk 160 processed. Resampled OHLC shape: (92, 5)\n",
"Processing chunk 161 (Shape: (1000000, 3))...\n",
" Chunk 161 processed. Resampled OHLC shape: (122, 5)\n",
"Processing chunk 162 (Shape: (1000000, 3))...\n",
" Chunk 162 processed. Resampled OHLC shape: (128, 5)\n",
"Processing chunk 163 (Shape: (1000000, 3))...\n",
" Chunk 163 processed. Resampled OHLC shape: (124, 5)\n",
"Processing chunk 164 (Shape: (1000000, 3))...\n",
" Chunk 164 processed. Resampled OHLC shape: (132, 5)\n",
"Processing chunk 165 (Shape: (1000000, 3))...\n",
" Chunk 165 processed. Resampled OHLC shape: (180, 5)\n",
"Processing chunk 166 (Shape: (1000000, 3))...\n",
" Chunk 166 processed. Resampled OHLC shape: (137, 5)\n",
"Processing chunk 167 (Shape: (1000000, 3))...\n",
" Chunk 167 processed. Resampled OHLC shape: (133, 5)\n",
"Processing chunk 168 (Shape: (1000000, 3))...\n",
" Chunk 168 processed. Resampled OHLC shape: (119, 5)\n",
"Processing chunk 169 (Shape: (1000000, 3))...\n",
" Chunk 169 processed. Resampled OHLC shape: (130, 5)\n",
"Processing chunk 170 (Shape: (1000000, 3))...\n",
" Chunk 170 processed. Resampled OHLC shape: (107, 5)\n",
"Processing chunk 171 (Shape: (1000000, 3))...\n",
" Chunk 171 processed. Resampled OHLC shape: (110, 5)\n",
"Processing chunk 172 (Shape: (1000000, 3))...\n",
" Chunk 172 processed. Resampled OHLC shape: (117, 5)\n",
"Processing chunk 173 (Shape: (1000000, 3))...\n",
" Chunk 173 processed. Resampled OHLC shape: (129, 5)\n",
"Processing chunk 174 (Shape: (1000000, 3))...\n",
" Chunk 174 processed. Resampled OHLC shape: (113, 5)\n",
"Processing chunk 175 (Shape: (1000000, 3))...\n",
" Chunk 175 processed. Resampled OHLC shape: (122, 5)\n",
"Processing chunk 176 (Shape: (1000000, 3))...\n",
" Chunk 176 processed. Resampled OHLC shape: (137, 5)\n",
"Processing chunk 177 (Shape: (1000000, 3))...\n",
" Chunk 177 processed. Resampled OHLC shape: (125, 5)\n",
"Processing chunk 178 (Shape: (1000000, 3))...\n",
" Chunk 178 processed. Resampled OHLC shape: (152, 5)\n",
"Processing chunk 179 (Shape: (1000000, 3))...\n",
" Chunk 179 processed. Resampled OHLC shape: (122, 5)\n",
"Processing chunk 180 (Shape: (1000000, 3))...\n",
" Chunk 180 processed. Resampled OHLC shape: (133, 5)\n",
"Processing chunk 181 (Shape: (1000000, 3))...\n",
" Chunk 181 processed. Resampled OHLC shape: (145, 5)\n",
"Processing chunk 182 (Shape: (1000000, 3))...\n",
" Chunk 182 processed. Resampled OHLC shape: (147, 5)\n",
"Processing chunk 183 (Shape: (1000000, 3))...\n",
" Chunk 183 processed. Resampled OHLC shape: (133, 5)\n",
"Processing chunk 184 (Shape: (1000000, 3))...\n",
" Chunk 184 processed. Resampled OHLC shape: (139, 5)\n",
"Processing chunk 185 (Shape: (1000000, 3))...\n",
" Chunk 185 processed. Resampled OHLC shape: (173, 5)\n",
"Processing chunk 186 (Shape: (1000000, 3))...\n",
" Chunk 186 processed. Resampled OHLC shape: (138, 5)\n",
"Processing chunk 187 (Shape: (1000000, 3))...\n",
" Chunk 187 processed. Resampled OHLC shape: (167, 5)\n",
"Processing chunk 188 (Shape: (1000000, 3))...\n",
" Chunk 188 processed. Resampled OHLC shape: (154, 5)\n",
"Processing chunk 189 (Shape: (1000000, 3))...\n",
" Chunk 189 processed. Resampled OHLC shape: (166, 5)\n",
"Processing chunk 190 (Shape: (1000000, 3))...\n",
" Chunk 190 processed. Resampled OHLC shape: (162, 5)\n",
"Processing chunk 191 (Shape: (1000000, 3))...\n",
" Chunk 191 processed. Resampled OHLC shape: (136, 5)\n",
"Processing chunk 192 (Shape: (1000000, 3))...\n",
" Chunk 192 processed. Resampled OHLC shape: (143, 5)\n",
"Processing chunk 193 (Shape: (1000000, 3))...\n",
" Chunk 193 processed. Resampled OHLC shape: (137, 5)\n",
"Processing chunk 194 (Shape: (1000000, 3))...\n",
" Chunk 194 processed. Resampled OHLC shape: (142, 5)\n",
"Processing chunk 195 (Shape: (1000000, 3))...\n",
" Chunk 195 processed. Resampled OHLC shape: (135, 5)\n",
"Processing chunk 196 (Shape: (1000000, 3))...\n",
" Chunk 196 processed. Resampled OHLC shape: (131, 5)\n",
"Processing chunk 197 (Shape: (1000000, 3))...\n",
" Chunk 197 processed. Resampled OHLC shape: (127, 5)\n",
"Processing chunk 198 (Shape: (1000000, 3))...\n",
" Chunk 198 processed. Resampled OHLC shape: (111, 5)\n",
"Processing chunk 199 (Shape: (1000000, 3))...\n",
" Chunk 199 processed. Resampled OHLC shape: (121, 5)\n",
"Processing chunk 200 (Shape: (1000000, 3))...\n",
" Chunk 200 processed. Resampled OHLC shape: (131, 5)\n",
"Processing chunk 201 (Shape: (1000000, 3))...\n",
" Chunk 201 processed. Resampled OHLC shape: (119, 5)\n",
"Processing chunk 202 (Shape: (1000000, 3))...\n",
" Chunk 202 processed. Resampled OHLC shape: (116, 5)\n",
"Processing chunk 203 (Shape: (1000000, 3))...\n",
" Chunk 203 processed. Resampled OHLC shape: (115, 5)\n",
"Processing chunk 204 (Shape: (1000000, 3))...\n",
" Chunk 204 processed. Resampled OHLC shape: (114, 5)\n",
"Processing chunk 205 (Shape: (1000000, 3))...\n",
" Chunk 205 processed. Resampled OHLC shape: (108, 5)\n",
"Processing chunk 206 (Shape: (1000000, 3))...\n",
" Chunk 206 processed. Resampled OHLC shape: (110, 5)\n",
"Processing chunk 207 (Shape: (1000000, 3))...\n",
" Chunk 207 processed. Resampled OHLC shape: (124, 5)\n",
"Processing chunk 208 (Shape: (1000000, 3))...\n",
" Chunk 208 processed. Resampled OHLC shape: (100, 5)\n",
"Processing chunk 209 (Shape: (1000000, 3))...\n",
" Chunk 209 processed. Resampled OHLC shape: (114, 5)\n",
"Processing chunk 210 (Shape: (1000000, 3))...\n",
" Chunk 210 processed. Resampled OHLC shape: (114, 5)\n",
"Processing chunk 211 (Shape: (1000000, 3))...\n",
" Chunk 211 processed. Resampled OHLC shape: (141, 5)\n",
"Processing chunk 212 (Shape: (1000000, 3))...\n",
" Chunk 212 processed. Resampled OHLC shape: (118, 5)\n",
"Processing chunk 213 (Shape: (1000000, 3))...\n",
" Chunk 213 processed. Resampled OHLC shape: (116, 5)\n",
"Processing chunk 214 (Shape: (1000000, 3))...\n",
" Chunk 214 processed. Resampled OHLC shape: (108, 5)\n",
"Processing chunk 215 (Shape: (1000000, 3))...\n",
" Chunk 215 processed. Resampled OHLC shape: (101, 5)\n",
"Processing chunk 216 (Shape: (1000000, 3))...\n",
" Chunk 216 processed. Resampled OHLC shape: (107, 5)\n",
"Processing chunk 217 (Shape: (1000000, 3))...\n",
" Chunk 217 processed. Resampled OHLC shape: (80, 5)\n",
"Processing chunk 218 (Shape: (1000000, 3))...\n",
" Chunk 218 processed. Resampled OHLC shape: (88, 5)\n",
"Processing chunk 219 (Shape: (1000000, 3))...\n",
" Chunk 219 processed. Resampled OHLC shape: (104, 5)\n",
"Processing chunk 220 (Shape: (1000000, 3))...\n",
" Chunk 220 processed. Resampled OHLC shape: (101, 5)\n",
"Processing chunk 221 (Shape: (1000000, 3))...\n",
" Chunk 221 processed. Resampled OHLC shape: (93, 5)\n",
"Processing chunk 222 (Shape: (1000000, 3))...\n",
" Chunk 222 processed. Resampled OHLC shape: (88, 5)\n",
"Processing chunk 223 (Shape: (1000000, 3))...\n",
" Chunk 223 processed. Resampled OHLC shape: (141, 5)\n",
"Processing chunk 224 (Shape: (1000000, 3))...\n",
" Chunk 224 processed. Resampled OHLC shape: (183, 5)\n",
"Processing chunk 225 (Shape: (1000000, 3))...\n",
" Chunk 225 processed. Resampled OHLC shape: (174, 5)\n",
"Processing chunk 226 (Shape: (1000000, 3))...\n",
" Chunk 226 processed. Resampled OHLC shape: (174, 5)\n",
"Processing chunk 227 (Shape: (1000000, 3))...\n",
" Chunk 227 processed. Resampled OHLC shape: (215, 5)\n",
"Processing chunk 228 (Shape: (1000000, 3))...\n",
" Chunk 228 processed. Resampled OHLC shape: (219, 5)\n",
"Processing chunk 229 (Shape: (1000000, 3))...\n",
" Chunk 229 processed. Resampled OHLC shape: (211, 5)\n",
"Processing chunk 230 (Shape: (1000000, 3))...\n",
" Chunk 230 processed. Resampled OHLC shape: (234, 5)\n",
"Processing chunk 231 (Shape: (1000000, 3))...\n",
" Chunk 231 processed. Resampled OHLC shape: (205, 5)\n",
"Processing chunk 232 (Shape: (1000000, 3))...\n",
" Chunk 232 processed. Resampled OHLC shape: (165, 5)\n",
"Processing chunk 233 (Shape: (1000000, 3))...\n",
" Chunk 233 processed. Resampled OHLC shape: (181, 5)\n",
"Processing chunk 234 (Shape: (1000000, 3))...\n",
" Chunk 234 processed. Resampled OHLC shape: (168, 5)\n",
"Processing chunk 235 (Shape: (1000000, 3))...\n",
" Chunk 235 processed. Resampled OHLC shape: (194, 5)\n",
"Processing chunk 236 (Shape: (1000000, 3))...\n",
" Chunk 236 processed. Resampled OHLC shape: (166, 5)\n",
"Processing chunk 237 (Shape: (1000000, 3))...\n",
" Chunk 237 processed. Resampled OHLC shape: (186, 5)\n",
"Processing chunk 238 (Shape: (1000000, 3))...\n",
" Chunk 238 processed. Resampled OHLC shape: (182, 5)\n",
"Processing chunk 239 (Shape: (1000000, 3))...\n",
" Chunk 239 processed. Resampled OHLC shape: (191, 5)\n",
"Processing chunk 240 (Shape: (1000000, 3))...\n",
" Chunk 240 processed. Resampled OHLC shape: (195, 5)\n",
"Processing chunk 241 (Shape: (1000000, 3))...\n",
" Chunk 241 processed. Resampled OHLC shape: (164, 5)\n",
"Processing chunk 242 (Shape: (1000000, 3))...\n",
" Chunk 242 processed. Resampled OHLC shape: (183, 5)\n",
"Processing chunk 243 (Shape: (1000000, 3))...\n",
" Chunk 243 processed. Resampled OHLC shape: (191, 5)\n",
"Processing chunk 244 (Shape: (1000000, 3))...\n",
" Chunk 244 processed. Resampled OHLC shape: (209, 5)\n",
"Processing chunk 245 (Shape: (1000000, 3))...\n",
" Chunk 245 processed. Resampled OHLC shape: (267, 5)\n",
"Processing chunk 246 (Shape: (1000000, 3))...\n",
" Chunk 246 processed. Resampled OHLC shape: (250, 5)\n",
"Processing chunk 247 (Shape: (1000000, 3))...\n",
" Chunk 247 processed. Resampled OHLC shape: (210, 5)\n",
"Processing chunk 248 (Shape: (1000000, 3))...\n",
" Chunk 248 processed. Resampled OHLC shape: (209, 5)\n",
"Processing chunk 249 (Shape: (1000000, 3))...\n",
" Chunk 249 processed. Resampled OHLC shape: (271, 5)\n",
"Processing chunk 250 (Shape: (1000000, 3))...\n",
" Chunk 250 processed. Resampled OHLC shape: (299, 5)\n",
"Processing chunk 251 (Shape: (1000000, 3))...\n",
" Chunk 251 processed. Resampled OHLC shape: (210, 5)\n",
"Processing chunk 252 (Shape: (1000000, 3))...\n",
" Chunk 252 processed. Resampled OHLC shape: (214, 5)\n",
"Processing chunk 253 (Shape: (1000000, 3))...\n",
" Chunk 253 processed. Resampled OHLC shape: (201, 5)\n",
"Processing chunk 254 (Shape: (1000000, 3))...\n",
" Chunk 254 processed. Resampled OHLC shape: (248, 5)\n",
"Processing chunk 255 (Shape: (1000000, 3))...\n",
" Chunk 255 processed. Resampled OHLC shape: (251, 5)\n",
"Processing chunk 256 (Shape: (1000000, 3))...\n",
" Chunk 256 processed. Resampled OHLC shape: (191, 5)\n",
"Processing chunk 257 (Shape: (1000000, 3))...\n",
" Chunk 257 processed. Resampled OHLC shape: (212, 5)\n",
"Processing chunk 258 (Shape: (1000000, 3))...\n",
" Chunk 258 processed. Resampled OHLC shape: (186, 5)\n",
"Processing chunk 259 (Shape: (1000000, 3))...\n",
" Chunk 259 processed. Resampled OHLC shape: (165, 5)\n",
"Processing chunk 260 (Shape: (1000000, 3))...\n",
" Chunk 260 processed. Resampled OHLC shape: (122, 5)\n",
"Processing chunk 261 (Shape: (1000000, 3))...\n",
" Chunk 261 processed. Resampled OHLC shape: (103, 5)\n",
"Processing chunk 262 (Shape: (1000000, 3))...\n",
" Chunk 262 processed. Resampled OHLC shape: (133, 5)\n",
"Processing chunk 263 (Shape: (1000000, 3))...\n",
" Chunk 263 processed. Resampled OHLC shape: (146, 5)\n",
"Processing chunk 264 (Shape: (1000000, 3))...\n",
" Chunk 264 processed. Resampled OHLC shape: (136, 5)\n",
"Processing chunk 265 (Shape: (1000000, 3))...\n",
" Chunk 265 processed. Resampled OHLC shape: (167, 5)\n",
"Processing chunk 266 (Shape: (1000000, 3))...\n",
" Chunk 266 processed. Resampled OHLC shape: (86, 5)\n",
"Processing chunk 267 (Shape: (1000000, 3))...\n",
" Chunk 267 processed. Resampled OHLC shape: (107, 5)\n",
"Processing chunk 268 (Shape: (1000000, 3))...\n",
" Chunk 268 processed. Resampled OHLC shape: (113, 5)\n",
"Processing chunk 269 (Shape: (1000000, 3))...\n",
" Chunk 269 processed. Resampled OHLC shape: (125, 5)\n",
"Processing chunk 270 (Shape: (1000000, 3))...\n",
" Chunk 270 processed. Resampled OHLC shape: (119, 5)\n",
"Processing chunk 271 (Shape: (1000000, 3))...\n",
" Chunk 271 processed. Resampled OHLC shape: (105, 5)\n",
"Processing chunk 272 (Shape: (1000000, 3))...\n",
" Chunk 272 processed. Resampled OHLC shape: (127, 5)\n",
"Processing chunk 273 (Shape: (1000000, 3))...\n",
" Chunk 273 processed. Resampled OHLC shape: (129, 5)\n",
"Processing chunk 274 (Shape: (1000000, 3))...\n",
" Chunk 274 processed. Resampled OHLC shape: (105, 5)\n",
"Processing chunk 275 (Shape: (1000000, 3))...\n",
" Chunk 275 processed. Resampled OHLC shape: (103, 5)\n",
"Processing chunk 276 (Shape: (1000000, 3))...\n",
" Chunk 276 processed. Resampled OHLC shape: (98, 5)\n",
"Processing chunk 277 (Shape: (1000000, 3))...\n",
" Chunk 277 processed. Resampled OHLC shape: (120, 5)\n",
"Processing chunk 278 (Shape: (1000000, 3))...\n",
" Chunk 278 processed. Resampled OHLC shape: (144, 5)\n",
"Processing chunk 279 (Shape: (1000000, 3))...\n",
" Chunk 279 processed. Resampled OHLC shape: (147, 5)\n",
"Processing chunk 280 (Shape: (1000000, 3))...\n",
" Chunk 280 processed. Resampled OHLC shape: (147, 5)\n",
"Processing chunk 281 (Shape: (1000000, 3))...\n",
" Chunk 281 processed. Resampled OHLC shape: (172, 5)\n",
"Processing chunk 282 (Shape: (1000000, 3))...\n",
" Chunk 282 processed. Resampled OHLC shape: (186, 5)\n",
"Processing chunk 283 (Shape: (1000000, 3))...\n",
" Chunk 283 processed. Resampled OHLC shape: (173, 5)\n",
"Processing chunk 284 (Shape: (1000000, 3))...\n",
" Chunk 284 processed. Resampled OHLC shape: (202, 5)\n",
"Processing chunk 285 (Shape: (1000000, 3))...\n",
" Chunk 285 processed. Resampled OHLC shape: (92, 5)\n",
"Processing chunk 286 (Shape: (1000000, 3))...\n",
" Chunk 286 processed. Resampled OHLC shape: (167, 5)\n",
"Processing chunk 287 (Shape: (1000000, 3))...\n",
" Chunk 287 processed. Resampled OHLC shape: (130, 5)\n",
"Processing chunk 288 (Shape: (1000000, 3))...\n",
" Chunk 288 processed. Resampled OHLC shape: (114, 5)\n",
"Processing chunk 289 (Shape: (1000000, 3))...\n",
" Chunk 289 processed. Resampled OHLC shape: (137, 5)\n",
"Processing chunk 290 (Shape: (1000000, 3))...\n",
" Chunk 290 processed. Resampled OHLC shape: (155, 5)\n",
"Processing chunk 291 (Shape: (1000000, 3))...\n",
" Chunk 291 processed. Resampled OHLC shape: (59, 5)\n",
"Processing chunk 292 (Shape: (1000000, 3))...\n",
" Chunk 292 processed. Resampled OHLC shape: (55, 5)\n",
"Processing chunk 293 (Shape: (1000000, 3))...\n",
" Chunk 293 processed. Resampled OHLC shape: (49, 5)\n",
"Processing chunk 294 (Shape: (1000000, 3))...\n",
" Chunk 294 processed. Resampled OHLC shape: (58, 5)\n",
"Processing chunk 295 (Shape: (1000000, 3))...\n",
" Chunk 295 processed. Resampled OHLC shape: (57, 5)\n",
"Processing chunk 296 (Shape: (1000000, 3))...\n",
" Chunk 296 processed. Resampled OHLC shape: (62, 5)\n",
"Processing chunk 297 (Shape: (1000000, 3))...\n",
" Chunk 297 processed. Resampled OHLC shape: (63, 5)\n",
"Processing chunk 298 (Shape: (1000000, 3))...\n",
" Chunk 298 processed. Resampled OHLC shape: (72, 5)\n",
"Processing chunk 299 (Shape: (1000000, 3))...\n",
" Chunk 299 processed. Resampled OHLC shape: (167, 5)\n",
"Processing chunk 300 (Shape: (1000000, 3))...\n",
" Chunk 300 processed. Resampled OHLC shape: (160, 5)\n",
"Processing chunk 301 (Shape: (1000000, 3))...\n",
" Chunk 301 processed. Resampled OHLC shape: (138, 5)\n",
"Processing chunk 302 (Shape: (1000000, 3))...\n",
" Chunk 302 processed. Resampled OHLC shape: (114, 5)\n",
"Processing chunk 303 (Shape: (1000000, 3))...\n",
" Chunk 303 processed. Resampled OHLC shape: (96, 5)\n",
"Processing chunk 304 (Shape: (1000000, 3))...\n",
" Chunk 304 processed. Resampled OHLC shape: (134, 5)\n",
"Processing chunk 305 (Shape: (1000000, 3))...\n",
" Chunk 305 processed. Resampled OHLC shape: (111, 5)\n",
"Processing chunk 306 (Shape: (1000000, 3))...\n",
" Chunk 306 processed. Resampled OHLC shape: (99, 5)\n",
"Processing chunk 307 (Shape: (1000000, 3))...\n",
" Chunk 307 processed. Resampled OHLC shape: (130, 5)\n",
"Processing chunk 308 (Shape: (1000000, 3))...\n",
" Chunk 308 processed. Resampled OHLC shape: (105, 5)\n",
"Processing chunk 309 (Shape: (1000000, 3))...\n",
" Chunk 309 processed. Resampled OHLC shape: (110, 5)\n",
"Processing chunk 310 (Shape: (1000000, 3))...\n",
" Chunk 310 processed. Resampled OHLC shape: (115, 5)\n",
"Processing chunk 311 (Shape: (1000000, 3))...\n",
" Chunk 311 processed. Resampled OHLC shape: (111, 5)\n",
"Processing chunk 312 (Shape: (1000000, 3))...\n",
" Chunk 312 processed. Resampled OHLC shape: (141, 5)\n",
"Processing chunk 313 (Shape: (1000000, 3))...\n",
" Chunk 313 processed. Resampled OHLC shape: (138, 5)\n",
"Processing chunk 314 (Shape: (1000000, 3))...\n",
" Chunk 314 processed. Resampled OHLC shape: (135, 5)\n",
"Processing chunk 315 (Shape: (1000000, 3))...\n",
" Chunk 315 processed. Resampled OHLC shape: (79, 5)\n",
"Processing chunk 316 (Shape: (1000000, 3))...\n",
" Chunk 316 processed. Resampled OHLC shape: (81, 5)\n",
"Processing chunk 317 (Shape: (1000000, 3))...\n",
" Chunk 317 processed. Resampled OHLC shape: (90, 5)\n",
"Processing chunk 318 (Shape: (1000000, 3))...\n",
" Chunk 318 processed. Resampled OHLC shape: (78, 5)\n",
"Processing chunk 319 (Shape: (1000000, 3))...\n",
" Chunk 319 processed. Resampled OHLC shape: (66, 5)\n",
"Processing chunk 320 (Shape: (1000000, 3))...\n",
" Chunk 320 processed. Resampled OHLC shape: (80, 5)\n",
"Processing chunk 321 (Shape: (1000000, 3))...\n",
" Chunk 321 processed. Resampled OHLC shape: (86, 5)\n",
"Processing chunk 322 (Shape: (1000000, 3))...\n",
" Chunk 322 processed. Resampled OHLC shape: (86, 5)\n",
"Processing chunk 323 (Shape: (1000000, 3))...\n",
" Chunk 323 processed. Resampled OHLC shape: (96, 5)\n",
"Processing chunk 324 (Shape: (1000000, 3))...\n",
" Chunk 324 processed. Resampled OHLC shape: (109, 5)\n",
"Processing chunk 325 (Shape: (1000000, 3))...\n",
" Chunk 325 processed. Resampled OHLC shape: (114, 5)\n",
"Processing chunk 326 (Shape: (1000000, 3))...\n",
" Chunk 326 processed. Resampled OHLC shape: (86, 5)\n",
"Processing chunk 327 (Shape: (1000000, 3))...\n",
" Chunk 327 processed. Resampled OHLC shape: (89, 5)\n",
"Processing chunk 328 (Shape: (1000000, 3))...\n",
" Chunk 328 processed. Resampled OHLC shape: (107, 5)\n",
"Processing chunk 329 (Shape: (1000000, 3))...\n",
" Chunk 329 processed. Resampled OHLC shape: (118, 5)\n",
"Processing chunk 330 (Shape: (1000000, 3))...\n",
" Chunk 330 processed. Resampled OHLC shape: (120, 5)\n",
"Processing chunk 331 (Shape: (1000000, 3))...\n",
" Chunk 331 processed. Resampled OHLC shape: (138, 5)\n",
"Processing chunk 332 (Shape: (1000000, 3))...\n",
" Chunk 332 processed. Resampled OHLC shape: (108, 5)\n",
"Processing chunk 333 (Shape: (1000000, 3))...\n",
" Chunk 333 processed. Resampled OHLC shape: (86, 5)\n",
"Processing chunk 334 (Shape: (1000000, 3))...\n",
" Chunk 334 processed. Resampled OHLC shape: (113, 5)\n",
"Processing chunk 335 (Shape: (1000000, 3))...\n",
" Chunk 335 processed. Resampled OHLC shape: (130, 5)\n",
"Processing chunk 336 (Shape: (1000000, 3))...\n",
" Chunk 336 processed. Resampled OHLC shape: (120, 5)\n",
"Processing chunk 337 (Shape: (1000000, 3))...\n",
" Chunk 337 processed. Resampled OHLC shape: (142, 5)\n",
"Processing chunk 338 (Shape: (1000000, 3))...\n",
" Chunk 338 processed. Resampled OHLC shape: (142, 5)\n",
"Processing chunk 339 (Shape: (1000000, 3))...\n",
" Chunk 339 processed. Resampled OHLC shape: (121, 5)\n",
"Processing chunk 340 (Shape: (1000000, 3))...\n",
" Chunk 340 processed. Resampled OHLC shape: (142, 5)\n",
"Processing chunk 341 (Shape: (1000000, 3))...\n",
" Chunk 341 processed. Resampled OHLC shape: (89, 5)\n",
"Processing chunk 342 (Shape: (1000000, 3))...\n",
" Chunk 342 processed. Resampled OHLC shape: (114, 5)\n",
"Processing chunk 343 (Shape: (1000000, 3))...\n",
" Chunk 343 processed. Resampled OHLC shape: (99, 5)\n",
"Processing chunk 344 (Shape: (1000000, 3))...\n",
" Chunk 344 processed. Resampled OHLC shape: (88, 5)\n",
"Processing chunk 345 (Shape: (1000000, 3))...\n",
" Chunk 345 processed. Resampled OHLC shape: (64, 5)\n",
"Processing chunk 346 (Shape: (1000000, 3))...\n",
" Chunk 346 processed. Resampled OHLC shape: (90, 5)\n",
"Processing chunk 347 (Shape: (1000000, 3))...\n",
" Chunk 347 processed. Resampled OHLC shape: (119, 5)\n",
"Processing chunk 348 (Shape: (1000000, 3))...\n",
" Chunk 348 processed. Resampled OHLC shape: (91, 5)\n",
"Processing chunk 349 (Shape: (1000000, 3))...\n",
" Chunk 349 processed. Resampled OHLC shape: (81, 5)\n",
"Processing chunk 350 (Shape: (1000000, 3))...\n",
" Chunk 350 processed. Resampled OHLC shape: (64, 5)\n",
"Processing chunk 351 (Shape: (1000000, 3))...\n",
" Chunk 351 processed. Resampled OHLC shape: (69, 5)\n",
"Processing chunk 352 (Shape: (1000000, 3))...\n",
" Chunk 352 processed. Resampled OHLC shape: (71, 5)\n",
"Processing chunk 353 (Shape: (1000000, 3))...\n",
" Chunk 353 processed. Resampled OHLC shape: (77, 5)\n",
"Processing chunk 354 (Shape: (1000000, 3))...\n",
" Chunk 354 processed. Resampled OHLC shape: (86, 5)\n",
"Processing chunk 355 (Shape: (1000000, 3))...\n",
" Chunk 355 processed. Resampled OHLC shape: (96, 5)\n",
"Processing chunk 356 (Shape: (1000000, 3))...\n",
" Chunk 356 processed. Resampled OHLC shape: (103, 5)\n",
"Processing chunk 357 (Shape: (1000000, 3))...\n",
" Chunk 357 processed. Resampled OHLC shape: (128, 5)\n",
"Processing chunk 358 (Shape: (1000000, 3))...\n",
" Chunk 358 processed. Resampled OHLC shape: (113, 5)\n",
"Processing chunk 359 (Shape: (1000000, 3))...\n",
" Chunk 359 processed. Resampled OHLC shape: (99, 5)\n",
"Processing chunk 360 (Shape: (1000000, 3))...\n",
" Chunk 360 processed. Resampled OHLC shape: (114, 5)\n",
"Processing chunk 361 (Shape: (1000000, 3))...\n",
" Chunk 361 processed. Resampled OHLC shape: (106, 5)\n",
"Processing chunk 362 (Shape: (1000000, 3))...\n",
" Chunk 362 processed. Resampled OHLC shape: (90, 5)\n",
"Processing chunk 363 (Shape: (1000000, 3))...\n",
" Chunk 363 processed. Resampled OHLC shape: (83, 5)\n",
"Processing chunk 364 (Shape: (1000000, 3))...\n",
" Chunk 364 processed. Resampled OHLC shape: (118, 5)\n",
"Processing chunk 365 (Shape: (1000000, 3))...\n",
" Chunk 365 processed. Resampled OHLC shape: (114, 5)\n",
"Processing chunk 366 (Shape: (1000000, 3))...\n",
" Chunk 366 processed. Resampled OHLC shape: (124, 5)\n",
"Processing chunk 367 (Shape: (1000000, 3))...\n",
" Chunk 367 processed. Resampled OHLC shape: (115, 5)\n",
"Processing chunk 368 (Shape: (1000000, 3))...\n",
" Chunk 368 processed. Resampled OHLC shape: (87, 5)\n",
"Processing chunk 369 (Shape: (1000000, 3))...\n",
" Chunk 369 processed. Resampled OHLC shape: (113, 5)\n",
"Processing chunk 370 (Shape: (1000000, 3))...\n",
" Chunk 370 processed. Resampled OHLC shape: (154, 5)\n",
"Processing chunk 371 (Shape: (1000000, 3))...\n",
" Chunk 371 processed. Resampled OHLC shape: (140, 5)\n",
"Processing chunk 372 (Shape: (1000000, 3))...\n",
" Chunk 372 processed. Resampled OHLC shape: (138, 5)\n",
"Processing chunk 373 (Shape: (1000000, 3))...\n",
" Chunk 373 processed. Resampled OHLC shape: (152, 5)\n",
"Processing chunk 374 (Shape: (1000000, 3))...\n",
" Chunk 374 processed. Resampled OHLC shape: (151, 5)\n",
"Processing chunk 375 (Shape: (1000000, 3))...\n",
" Chunk 375 processed. Resampled OHLC shape: (154, 5)\n",
"Processing chunk 376 (Shape: (1000000, 3))...\n",
" Chunk 376 processed. Resampled OHLC shape: (143, 5)\n",
"Processing chunk 377 (Shape: (1000000, 3))...\n",
" Chunk 377 processed. Resampled OHLC shape: (155, 5)\n",
"Processing chunk 378 (Shape: (1000000, 3))...\n",
" Chunk 378 processed. Resampled OHLC shape: (116, 5)\n",
"Processing chunk 379 (Shape: (1000000, 3))...\n",
" Chunk 379 processed. Resampled OHLC shape: (109, 5)\n",
"Processing chunk 380 (Shape: (1000000, 3))...\n",
" Chunk 380 processed. Resampled OHLC shape: (106, 5)\n",
"Processing chunk 381 (Shape: (1000000, 3))...\n",
" Chunk 381 processed. Resampled OHLC shape: (102, 5)\n",
"Processing chunk 382 (Shape: (1000000, 3))...\n",
" Chunk 382 processed. Resampled OHLC shape: (112, 5)\n",
"Processing chunk 383 (Shape: (1000000, 3))...\n",
" Chunk 383 processed. Resampled OHLC shape: (129, 5)\n",
"Processing chunk 384 (Shape: (1000000, 3))...\n",
" Chunk 384 processed. Resampled OHLC shape: (114, 5)\n",
"Processing chunk 385 (Shape: (1000000, 3))...\n",
" Chunk 385 processed. Resampled OHLC shape: (118, 5)\n",
"Processing chunk 386 (Shape: (1000000, 3))...\n",
" Chunk 386 processed. Resampled OHLC shape: (127, 5)\n",
"Processing chunk 387 (Shape: (1000000, 3))...\n",
" Chunk 387 processed. Resampled OHLC shape: (125, 5)\n",
"Processing chunk 388 (Shape: (1000000, 3))...\n",
" Chunk 388 processed. Resampled OHLC shape: (108, 5)\n",
"Processing chunk 389 (Shape: (1000000, 3))...\n",
" Chunk 389 processed. Resampled OHLC shape: (126, 5)\n",
"Processing chunk 390 (Shape: (1000000, 3))...\n",
" Chunk 390 processed. Resampled OHLC shape: (102, 5)\n",
"Processing chunk 391 (Shape: (1000000, 3))...\n",
" Chunk 391 processed. Resampled OHLC shape: (158, 5)\n",
"Processing chunk 392 (Shape: (1000000, 3))...\n",
" Chunk 392 processed. Resampled OHLC shape: (156, 5)\n",
"Processing chunk 393 (Shape: (1000000, 3))...\n",
" Chunk 393 processed. Resampled OHLC shape: (165, 5)\n",
"Processing chunk 394 (Shape: (1000000, 3))...\n",
" Chunk 394 processed. Resampled OHLC shape: (134, 5)\n",
"Processing chunk 395 (Shape: (1000000, 3))...\n",
" Chunk 395 processed. Resampled OHLC shape: (118, 5)\n",
"Processing chunk 396 (Shape: (1000000, 3))...\n",
" Chunk 396 processed. Resampled OHLC shape: (105, 5)\n",
"Processing chunk 397 (Shape: (1000000, 3))...\n",
" Chunk 397 processed. Resampled OHLC shape: (123, 5)\n",
"Processing chunk 398 (Shape: (1000000, 3))...\n",
" Chunk 398 processed. Resampled OHLC shape: (112, 5)\n",
"Processing chunk 399 (Shape: (1000000, 3))...\n",
" Chunk 399 processed. Resampled OHLC shape: (92, 5)\n",
"Processing chunk 400 (Shape: (1000000, 3))...\n",
" Chunk 400 processed. Resampled OHLC shape: (94, 5)\n",
"Processing chunk 401 (Shape: (1000000, 3))...\n",
" Chunk 401 processed. Resampled OHLC shape: (70, 5)\n",
"Processing chunk 402 (Shape: (1000000, 3))...\n",
" Chunk 402 processed. Resampled OHLC shape: (68, 5)\n",
"Processing chunk 403 (Shape: (1000000, 3))...\n",
" Chunk 403 processed. Resampled OHLC shape: (58, 5)\n",
"Processing chunk 404 (Shape: (1000000, 3))...\n",
" Chunk 404 processed. Resampled OHLC shape: (62, 5)\n",
"Processing chunk 405 (Shape: (1000000, 3))...\n",
" Chunk 405 processed. Resampled OHLC shape: (95, 5)\n",
"Processing chunk 406 (Shape: (1000000, 3))...\n",
" Chunk 406 processed. Resampled OHLC shape: (122, 5)\n",
"Processing chunk 407 (Shape: (1000000, 3))...\n",
" Chunk 407 processed. Resampled OHLC shape: (103, 5)\n",
"Processing chunk 408 (Shape: (1000000, 3))...\n",
" Chunk 408 processed. Resampled OHLC shape: (104, 5)\n",
"Processing chunk 409 (Shape: (1000000, 3))...\n",
" Chunk 409 processed. Resampled OHLC shape: (106, 5)\n",
"Processing chunk 410 (Shape: (1000000, 3))...\n",
" Chunk 410 processed. Resampled OHLC shape: (105, 5)\n",
"Processing chunk 411 (Shape: (1000000, 3))...\n",
" Chunk 411 processed. Resampled OHLC shape: (95, 5)\n",
"Processing chunk 412 (Shape: (1000000, 3))...\n",
" Chunk 412 processed. Resampled OHLC shape: (86, 5)\n",
"Processing chunk 413 (Shape: (1000000, 3))...\n",
" Chunk 413 processed. Resampled OHLC shape: (102, 5)\n",
"Processing chunk 414 (Shape: (1000000, 3))...\n",
" Chunk 414 processed. Resampled OHLC shape: (85, 5)\n",
"Processing chunk 415 (Shape: (1000000, 3))...\n",
" Chunk 415 processed. Resampled OHLC shape: (94, 5)\n",
"Processing chunk 416 (Shape: (1000000, 3))...\n",
" Chunk 416 processed. Resampled OHLC shape: (114, 5)\n",
"Processing chunk 417 (Shape: (1000000, 3))...\n",
" Chunk 417 processed. Resampled OHLC shape: (156, 5)\n",
"Processing chunk 418 (Shape: (1000000, 3))...\n",
" Chunk 418 processed. Resampled OHLC shape: (162, 5)\n",
"Processing chunk 419 (Shape: (1000000, 3))...\n",
" Chunk 419 processed. Resampled OHLC shape: (94, 5)\n",
"Processing chunk 420 (Shape: (1000000, 3))...\n",
" Chunk 420 processed. Resampled OHLC shape: (116, 5)\n",
"Processing chunk 421 (Shape: (1000000, 3))...\n",
" Chunk 421 processed. Resampled OHLC shape: (117, 5)\n",
"Processing chunk 422 (Shape: (1000000, 3))...\n",
" Chunk 422 processed. Resampled OHLC shape: (112, 5)\n",
"Processing chunk 423 (Shape: (1000000, 3))...\n",
" Chunk 423 processed. Resampled OHLC shape: (113, 5)\n",
"Processing chunk 424 (Shape: (1000000, 3))...\n",
" Chunk 424 processed. Resampled OHLC shape: (109, 5)\n",
"Processing chunk 425 (Shape: (1000000, 3))...\n",
" Chunk 425 processed. Resampled OHLC shape: (119, 5)\n",
"Processing chunk 426 (Shape: (1000000, 3))...\n",
" Chunk 426 processed. Resampled OHLC shape: (99, 5)\n",
"Processing chunk 427 (Shape: (1000000, 3))...\n",
" Chunk 427 processed. Resampled OHLC shape: (120, 5)\n",
"Processing chunk 428 (Shape: (1000000, 3))...\n",
" Chunk 428 processed. Resampled OHLC shape: (134, 5)\n",
"Processing chunk 429 (Shape: (1000000, 3))...\n",
" Chunk 429 processed. Resampled OHLC shape: (136, 5)\n",
"Processing chunk 430 (Shape: (1000000, 3))...\n",
" Chunk 430 processed. Resampled OHLC shape: (122, 5)\n",
"Processing chunk 431 (Shape: (1000000, 3))...\n",
" Chunk 431 processed. Resampled OHLC shape: (113, 5)\n",
"Processing chunk 432 (Shape: (1000000, 3))...\n",
" Chunk 432 processed. Resampled OHLC shape: (107, 5)\n",
"Processing chunk 433 (Shape: (1000000, 3))...\n",
" Chunk 433 processed. Resampled OHLC shape: (104, 5)\n",
"Processing chunk 434 (Shape: (1000000, 3))...\n",
" Chunk 434 processed. Resampled OHLC shape: (88, 5)\n",
"Processing chunk 435 (Shape: (1000000, 3))...\n",
" Chunk 435 processed. Resampled OHLC shape: (85, 5)\n",
"Processing chunk 436 (Shape: (1000000, 3))...\n",
" Chunk 436 processed. Resampled OHLC shape: (99, 5)\n",
"Processing chunk 437 (Shape: (1000000, 3))...\n",
" Chunk 437 processed. Resampled OHLC shape: (96, 5)\n",
"Processing chunk 438 (Shape: (1000000, 3))...\n",
" Chunk 438 processed. Resampled OHLC shape: (118, 5)\n",
"Processing chunk 439 (Shape: (1000000, 3))...\n",
" Chunk 439 processed. Resampled OHLC shape: (115, 5)\n",
"Processing chunk 440 (Shape: (1000000, 3))...\n",
" Chunk 440 processed. Resampled OHLC shape: (140, 5)\n",
"Processing chunk 441 (Shape: (1000000, 3))...\n",
" Chunk 441 processed. Resampled OHLC shape: (128, 5)\n",
"Processing chunk 442 (Shape: (1000000, 3))...\n",
" Chunk 442 processed. Resampled OHLC shape: (130, 5)\n",
"Processing chunk 443 (Shape: (1000000, 3))...\n",
" Chunk 443 processed. Resampled OHLC shape: (158, 5)\n",
"Processing chunk 444 (Shape: (1000000, 3))...\n",
" Chunk 444 processed. Resampled OHLC shape: (157, 5)\n",
"Processing chunk 445 (Shape: (1000000, 3))...\n",
" Chunk 445 processed. Resampled OHLC shape: (131, 5)\n",
"Processing chunk 446 (Shape: (1000000, 3))...\n",
" Chunk 446 processed. Resampled OHLC shape: (149, 5)\n",
"Processing chunk 447 (Shape: (1000000, 3))...\n",
" Chunk 447 processed. Resampled OHLC shape: (119, 5)\n",
"Processing chunk 448 (Shape: (1000000, 3))...\n",
" Chunk 448 processed. Resampled OHLC shape: (134, 5)\n",
"Processing chunk 449 (Shape: (1000000, 3))...\n",
" Chunk 449 processed. Resampled OHLC shape: (137, 5)\n",
"Processing chunk 450 (Shape: (1000000, 3))...\n",
" Chunk 450 processed. Resampled OHLC shape: (145, 5)\n",
"Processing chunk 451 (Shape: (1000000, 3))...\n",
" Chunk 451 processed. Resampled OHLC shape: (126, 5)\n",
"Processing chunk 452 (Shape: (1000000, 3))...\n",
" Chunk 452 processed. Resampled OHLC shape: (141, 5)\n",
"Processing chunk 453 (Shape: (1000000, 3))...\n",
" Chunk 453 processed. Resampled OHLC shape: (164, 5)\n",
"Processing chunk 454 (Shape: (1000000, 3))...\n",
" Chunk 454 processed. Resampled OHLC shape: (163, 5)\n",
"Processing chunk 455 (Shape: (1000000, 3))...\n",
" Chunk 455 processed. Resampled OHLC shape: (93, 5)\n",
"Processing chunk 456 (Shape: (1000000, 3))...\n",
" Chunk 456 processed. Resampled OHLC shape: (93, 5)\n",
"Processing chunk 457 (Shape: (1000000, 3))...\n",
" Chunk 457 processed. Resampled OHLC shape: (112, 5)\n",
"Processing chunk 458 (Shape: (1000000, 3))...\n",
" Chunk 458 processed. Resampled OHLC shape: (145, 5)\n",
"Processing chunk 459 (Shape: (1000000, 3))...\n",
" Chunk 459 processed. Resampled OHLC shape: (160, 5)\n",
"Processing chunk 460 (Shape: (1000000, 3))...\n",
" Chunk 460 processed. Resampled OHLC shape: (155, 5)\n",
"Processing chunk 461 (Shape: (1000000, 3))...\n",
" Chunk 461 processed. Resampled OHLC shape: (134, 5)\n",
"Processing chunk 462 (Shape: (1000000, 3))...\n",
" Chunk 462 processed. Resampled OHLC shape: (124, 5)\n",
"Processing chunk 463 (Shape: (1000000, 3))...\n",
" Chunk 463 processed. Resampled OHLC shape: (120, 5)\n",
"Processing chunk 464 (Shape: (1000000, 3))...\n",
" Chunk 464 processed. Resampled OHLC shape: (141, 5)\n",
"Processing chunk 465 (Shape: (1000000, 3))...\n",
" Chunk 465 processed. Resampled OHLC shape: (170, 5)\n",
"Processing chunk 466 (Shape: (1000000, 3))...\n",
" Chunk 466 processed. Resampled OHLC shape: (163, 5)\n",
"Processing chunk 467 (Shape: (1000000, 3))...\n",
" Chunk 467 processed. Resampled OHLC shape: (168, 5)\n",
"Processing chunk 468 (Shape: (1000000, 3))...\n",
" Chunk 468 processed. Resampled OHLC shape: (167, 5)\n",
"Processing chunk 469 (Shape: (1000000, 3))...\n",
" Chunk 469 processed. Resampled OHLC shape: (180, 5)\n",
"Processing chunk 470 (Shape: (1000000, 3))...\n",
" Chunk 470 processed. Resampled OHLC shape: (168, 5)\n",
"Processing chunk 471 (Shape: (1000000, 3))...\n",
" Chunk 471 processed. Resampled OHLC shape: (198, 5)\n",
"Processing chunk 472 (Shape: (1000000, 3))...\n",
" Chunk 472 processed. Resampled OHLC shape: (218, 5)\n",
"Processing chunk 473 (Shape: (1000000, 3))...\n",
" Chunk 473 processed. Resampled OHLC shape: (255, 5)\n",
"Processing chunk 474 (Shape: (1000000, 3))...\n",
" Chunk 474 processed. Resampled OHLC shape: (260, 5)\n",
"Processing chunk 475 (Shape: (1000000, 3))...\n",
" Chunk 475 processed. Resampled OHLC shape: (216, 5)\n",
"Processing chunk 476 (Shape: (1000000, 3))...\n",
" Chunk 476 processed. Resampled OHLC shape: (187, 5)\n",
"Processing chunk 477 (Shape: (1000000, 3))...\n",
" Chunk 477 processed. Resampled OHLC shape: (175, 5)\n",
"Processing chunk 478 (Shape: (1000000, 3))...\n",
" Chunk 478 processed. Resampled OHLC shape: (193, 5)\n",
"Processing chunk 479 (Shape: (1000000, 3))...\n",
" Chunk 479 processed. Resampled OHLC shape: (204, 5)\n",
"Processing chunk 480 (Shape: (1000000, 3))...\n",
" Chunk 480 processed. Resampled OHLC shape: (189, 5)\n",
"Processing chunk 481 (Shape: (1000000, 3))...\n",
" Chunk 481 processed. Resampled OHLC shape: (151, 5)\n",
"Processing chunk 482 (Shape: (1000000, 3))...\n",
" Chunk 482 processed. Resampled OHLC shape: (171, 5)\n",
"Processing chunk 483 (Shape: (1000000, 3))...\n",
" Chunk 483 processed. Resampled OHLC shape: (180, 5)\n",
"Processing chunk 484 (Shape: (1000000, 3))...\n",
" Chunk 484 processed. Resampled OHLC shape: (150, 5)\n",
"Processing chunk 485 (Shape: (1000000, 3))...\n",
" Chunk 485 processed. Resampled OHLC shape: (172, 5)\n",
"Processing chunk 486 (Shape: (1000000, 3))...\n",
" Chunk 486 processed. Resampled OHLC shape: (171, 5)\n",
"Processing chunk 487 (Shape: (1000000, 3))...\n",
" Chunk 487 processed. Resampled OHLC shape: (226, 5)\n",
"Processing chunk 488 (Shape: (1000000, 3))...\n",
" Chunk 488 processed. Resampled OHLC shape: (248, 5)\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/var/folders/93/_0zg38790f7c8c2ygns8b3080000gn/T/ipykernel_25668/3429916984.py:72: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.\n",
" tick_chunk_df[timestamp_col_name_in_chunk] = pd.to_datetime(tick_chunk_df[timestamp_col_name_in_chunk], errors='coerce')\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Processing chunk 489 (Shape: (1000000, 3))...\n",
" Chunk 489 processed. Resampled OHLC shape: (162, 5)\n",
"Processing chunk 490 (Shape: (1000000, 3))...\n",
" Chunk 490 processed. Resampled OHLC shape: (175, 5)\n",
"Processing chunk 491 (Shape: (1000000, 3))...\n",
" Chunk 491 processed. Resampled OHLC shape: (149, 5)\n",
"Processing chunk 492 (Shape: (1000000, 3))...\n",
" Chunk 492 processed. Resampled OHLC shape: (98, 5)\n",
"Processing chunk 493 (Shape: (1000000, 3))...\n",
" Chunk 493 processed. Resampled OHLC shape: (88, 5)\n",
"Processing chunk 494 (Shape: (1000000, 3))...\n",
" Chunk 494 processed. Resampled OHLC shape: (83, 5)\n",
"Processing chunk 495 (Shape: (1000000, 3))...\n",
" Chunk 495 processed. Resampled OHLC shape: (96, 5)\n",
"Processing chunk 496 (Shape: (1000000, 3))...\n",
" Chunk 496 processed. Resampled OHLC shape: (107, 5)\n",
"Processing chunk 497 (Shape: (1000000, 3))...\n",
" Chunk 497 processed. Resampled OHLC shape: (130, 5)\n",
"Processing chunk 498 (Shape: (1000000, 3))...\n",
" Chunk 498 processed. Resampled OHLC shape: (103, 5)\n",
"Processing chunk 499 (Shape: (1000000, 3))...\n",
" Chunk 499 processed. Resampled OHLC shape: (96, 5)\n",
"Processing chunk 500 (Shape: (1000000, 3))...\n",
" Chunk 500 processed. Resampled OHLC shape: (100, 5)\n",
"Processing chunk 501 (Shape: (1000000, 3))...\n",
" Chunk 501 processed. Resampled OHLC shape: (96, 5)\n",
"Processing chunk 502 (Shape: (1000000, 3))...\n",
" Chunk 502 processed. Resampled OHLC shape: (93, 5)\n",
"Processing chunk 503 (Shape: (1000000, 3))...\n",
" Chunk 503 processed. Resampled OHLC shape: (96, 5)\n",
"Processing chunk 504 (Shape: (1000000, 3))...\n",
" Chunk 504 processed. Resampled OHLC shape: (112, 5)\n",
"Processing chunk 505 (Shape: (1000000, 3))...\n",
" Chunk 505 processed. Resampled OHLC shape: (97, 5)\n",
"Processing chunk 506 (Shape: (1000000, 3))...\n",
" Chunk 506 processed. Resampled OHLC shape: (108, 5)\n",
"Processing chunk 507 (Shape: (1000000, 3))...\n",
" Chunk 507 processed. Resampled OHLC shape: (106, 5)\n",
"Processing chunk 508 (Shape: (1000000, 3))...\n",
" Chunk 508 processed. Resampled OHLC shape: (99, 5)\n",
"Processing chunk 509 (Shape: (1000000, 3))...\n",
" Chunk 509 processed. Resampled OHLC shape: (79, 5)\n",
"Processing chunk 510 (Shape: (1000000, 3))...\n",
" Chunk 510 processed. Resampled OHLC shape: (87, 5)\n",
"Processing chunk 511 (Shape: (1000000, 3))...\n",
" Chunk 511 processed. Resampled OHLC shape: (93, 5)\n",
"Processing chunk 512 (Shape: (1000000, 3))...\n",
" Chunk 512 processed. Resampled OHLC shape: (66, 5)\n",
"Processing chunk 513 (Shape: (1000000, 3))...\n",
" Chunk 513 processed. Resampled OHLC shape: (67, 5)\n",
"Processing chunk 514 (Shape: (1000000, 3))...\n",
" Chunk 514 processed. Resampled OHLC shape: (94, 5)\n",
"Processing chunk 515 (Shape: (1000000, 3))...\n",
" Chunk 515 processed. Resampled OHLC shape: (91, 5)\n",
"Processing chunk 516 (Shape: (1000000, 3))...\n",
" Chunk 516 processed. Resampled OHLC shape: (94, 5)\n",
"Processing chunk 517 (Shape: (1000000, 3))...\n",
" Chunk 517 processed. Resampled OHLC shape: (99, 5)\n",
"Processing chunk 518 (Shape: (1000000, 3))...\n",
" Chunk 518 processed. Resampled OHLC shape: (114, 5)\n",
"Processing chunk 519 (Shape: (1000000, 3))...\n",
" Chunk 519 processed. Resampled OHLC shape: (95, 5)\n",
"Processing chunk 520 (Shape: (1000000, 3))...\n",
" Chunk 520 processed. Resampled OHLC shape: (99, 5)\n",
"Processing chunk 521 (Shape: (1000000, 3))...\n",
" Chunk 521 processed. Resampled OHLC shape: (86, 5)\n",
"Processing chunk 522 (Shape: (1000000, 3))...\n",
" Chunk 522 processed. Resampled OHLC shape: (83, 5)\n",
"Processing chunk 523 (Shape: (1000000, 3))...\n",
" Chunk 523 processed. Resampled OHLC shape: (103, 5)\n",
"Processing chunk 524 (Shape: (1000000, 3))...\n",
" Chunk 524 processed. Resampled OHLC shape: (94, 5)\n",
"Processing chunk 525 (Shape: (1000000, 3))...\n",
" Chunk 525 processed. Resampled OHLC shape: (109, 5)\n",
"Processing chunk 526 (Shape: (1000000, 3))...\n",
" Chunk 526 processed. Resampled OHLC shape: (92, 5)\n",
"Processing chunk 527 (Shape: (1000000, 3))...\n",
" Chunk 527 processed. Resampled OHLC shape: (89, 5)\n",
"Processing chunk 528 (Shape: (1000000, 3))...\n",
" Chunk 528 processed. Resampled OHLC shape: (83, 5)\n",
"Processing chunk 529 (Shape: (1000000, 3))...\n",
" Chunk 529 processed. Resampled OHLC shape: (78, 5)\n",
"Processing chunk 530 (Shape: (1000000, 3))...\n",
" Chunk 530 processed. Resampled OHLC shape: (81, 5)\n",
"Processing chunk 531 (Shape: (1000000, 3))...\n",
" Chunk 531 processed. Resampled OHLC shape: (73, 5)\n",
"Processing chunk 532 (Shape: (1000000, 3))...\n",
" Chunk 532 processed. Resampled OHLC shape: (90, 5)\n",
"Processing chunk 533 (Shape: (1000000, 3))...\n",
" Chunk 533 processed. Resampled OHLC shape: (73, 5)\n",
"Processing chunk 534 (Shape: (1000000, 3))...\n",
" Chunk 534 processed. Resampled OHLC shape: (94, 5)\n",
"Processing chunk 535 (Shape: (1000000, 3))...\n",
" Chunk 535 processed. Resampled OHLC shape: (91, 5)\n",
"Processing chunk 536 (Shape: (1000000, 3))...\n",
" Chunk 536 processed. Resampled OHLC shape: (89, 5)\n",
"Processing chunk 537 (Shape: (1000000, 3))...\n",
" Chunk 537 processed. Resampled OHLC shape: (91, 5)\n",
"Processing chunk 538 (Shape: (1000000, 3))...\n",
" Chunk 538 processed. Resampled OHLC shape: (91, 5)\n",
"Processing chunk 539 (Shape: (1000000, 3))...\n",
" Chunk 539 processed. Resampled OHLC shape: (120, 5)\n",
"Processing chunk 540 (Shape: (1000000, 3))...\n",
" Chunk 540 processed. Resampled OHLC shape: (95, 5)\n",
"Processing chunk 541 (Shape: (1000000, 3))...\n",
" Chunk 541 processed. Resampled OHLC shape: (88, 5)\n",
"Processing chunk 542 (Shape: (1000000, 3))...\n",
" Chunk 542 processed. Resampled OHLC shape: (88, 5)\n",
"Processing chunk 543 (Shape: (1000000, 3))...\n",
" Chunk 543 processed. Resampled OHLC shape: (86, 5)\n",
"Processing chunk 544 (Shape: (1000000, 3))...\n",
" Chunk 544 processed. Resampled OHLC shape: (91, 5)\n",
"Processing chunk 545 (Shape: (1000000, 3))...\n",
" Chunk 545 processed. Resampled OHLC shape: (67, 5)\n",
"Processing chunk 546 (Shape: (1000000, 3))...\n",
" Chunk 546 processed. Resampled OHLC shape: (79, 5)\n",
"Processing chunk 547 (Shape: (1000000, 3))...\n",
" Chunk 547 processed. Resampled OHLC shape: (73, 5)\n",
"Processing chunk 548 (Shape: (1000000, 3))...\n",
" Chunk 548 processed. Resampled OHLC shape: (85, 5)\n",
"Processing chunk 549 (Shape: (1000000, 3))...\n",
" Chunk 549 processed. Resampled OHLC shape: (89, 5)\n",
"Processing chunk 550 (Shape: (1000000, 3))...\n",
" Chunk 550 processed. Resampled OHLC shape: (83, 5)\n",
"Processing chunk 551 (Shape: (1000000, 3))...\n",
" Chunk 551 processed. Resampled OHLC shape: (87, 5)\n",
"Processing chunk 552 (Shape: (1000000, 3))...\n",
" Chunk 552 processed. Resampled OHLC shape: (77, 5)\n",
"Processing chunk 553 (Shape: (1000000, 3))...\n",
" Chunk 553 processed. Resampled OHLC shape: (88, 5)\n",
"Processing chunk 554 (Shape: (1000000, 3))...\n",
" Chunk 554 processed. Resampled OHLC shape: (94, 5)\n",
"Processing chunk 555 (Shape: (1000000, 3))...\n",
" Chunk 555 processed. Resampled OHLC shape: (97, 5)\n",
"Processing chunk 556 (Shape: (1000000, 3))...\n",
" Chunk 556 processed. Resampled OHLC shape: (94, 5)\n",
"Processing chunk 557 (Shape: (1000000, 3))...\n",
" Chunk 557 processed. Resampled OHLC shape: (70, 5)\n",
"Processing chunk 558 (Shape: (1000000, 3))...\n",
" Chunk 558 processed. Resampled OHLC shape: (61, 5)\n",
"Processing chunk 559 (Shape: (1000000, 3))...\n",
" Chunk 559 processed. Resampled OHLC shape: (66, 5)\n",
"Processing chunk 560 (Shape: (1000000, 3))...\n",
" Chunk 560 processed. Resampled OHLC shape: (102, 5)\n",
"Processing chunk 561 (Shape: (475872, 3))...\n",
" Chunk 561 processed. Resampled OHLC shape: (35, 5)\n",
"\n",
"All 561 non-empty chunks processed. Final OHLC DataFrame shape: (90112, 5)\n",
"\n",
"Head of final OHLC data:\n",
" Open High Low Close Volume\n",
"Timestamp_str \n",
"2010-05-04 04:00:00+00:00 1182.580 1183.332 1180.329 1180.860 1\n",
"2010-05-04 05:00:00+00:00 1180.904 1181.470 1178.850 1179.494 1\n",
"2010-05-04 06:00:00+00:00 1179.578 1179.958 1178.502 1178.937 1\n",
"2010-05-04 07:00:00+00:00 1178.974 1180.183 1178.448 1179.969 1\n",
"2010-05-04 08:00:00+00:00 1179.958 1181.230 1179.426 1180.587 1\n",
"\n",
"Tail of final OHLC data:\n",
" Open High Low Close Volume\n",
"Timestamp_str \n",
"2025-04-22 22:00:00+00:00 3401.595 3402.195 3372.995 3381.305 1\n",
"2025-04-22 23:00:00+00:00 3381.305 3385.615 3366.525 3374.505 1\n",
"2025-04-23 00:00:00+00:00 3374.499 3382.505 3372.915 3380.075 1\n",
"2025-04-23 02:00:00+00:00 3349.515 3349.515 3312.625 3327.255 1\n",
"2025-04-23 03:00:00+00:00 3327.145 3349.795 3325.945 3336.315 1\n",
"\n",
"Processed OHLC data saved to: /Users/carletes/Documents/lorentzian/processed_data/XAUUSD_OHLC_h.parquet\n"
]
}
],
"source": [
"# Cell 4: Load and Process Data in Chunks\n",
"\n",
"# --- Define the resampling function (similar to DataLoader) ---\n",
"def resample_chunk_to_ohlc(chunk_df, price_col_name_in_chunk, timeframe_to_resample):\n",
" if chunk_df.empty or price_col_name_in_chunk not in chunk_df.columns:\n",
" print(f\" Resample warning: Chunk empty or price column '{price_col_name_in_chunk}' not found.\")\n",
" return pd.DataFrame()\n",
"\n",
" if not isinstance(chunk_df.index, pd.DatetimeIndex):\n",
" print(\" Resample warning: Chunk index is not DatetimeIndex. Skipping resampling.\")\n",
" return pd.DataFrame()\n",
" \n",
" agg_rules = {\n",
" price_col_name_in_chunk: ['first', 'max', 'min', 'last']\n",
" }\n",
" # if 'TickVolume' in chunk_df.columns: agg_rules['TickVolume'] = 'sum' # If you add tick volume\n",
"\n",
" resampled = chunk_df.resample(timeframe_to_resample, label='right', closed='right').agg(agg_rules)\n",
" \n",
" if resampled.empty:\n",
" return pd.DataFrame()\n",
"\n",
" # Flatten MultiIndex columns\n",
" if isinstance(resampled.columns, pd.MultiIndex):\n",
" resampled.columns = ['_'.join(col).strip() for col in resampled.columns.values]\n",
" rename_map = {\n",
" f'{price_col_name_in_chunk}_first': 'Open',\n",
" f'{price_col_name_in_chunk}_max': 'High',\n",
" f'{price_col_name_in_chunk}_min': 'Low',\n",
" f'{price_col_name_in_chunk}_last': 'Close',\n",
" }\n",
" # if f'TickVolume_sum' in resampled.columns: rename_map[f'TickVolume_sum'] = 'Volume'\n",
" resampled.rename(columns=rename_map, inplace=True)\n",
" else: # For pandas 2.x and later, direct renaming might be needed\n",
" new_column_names = ['Open', 'High', 'Low', 'Close']\n",
" # if 'TickVolume' in agg_rules: new_column_names.append('Volume')\n",
" if len(resampled.columns) == len(new_column_names):\n",
" resampled.columns = new_column_names\n",
" else:\n",
" print(f\" Resample warning: Column mismatch after resampling. Expected {len(new_column_names)} got {len(resampled.columns)}. Columns: {resampled.columns.tolist()}\")\n",
" return pd.DataFrame() # Return empty if structure is not as expected\n",
"\n",
" if 'Volume' not in resampled.columns:\n",
" resampled['Volume'] = 1 # Placeholder\n",
" \n",
" return resampled.dropna()\n",
"# --- End of resampling function ---\n",
"\n",
"chunk_size = 1_000_000\n",
"all_ohlc_data = []\n",
"processed_chunks_count = 0\n",
"\n",
"print(f\"\\nStarting to process {csv_file_path} in chunks of {chunk_size} rows...\")\n",
"try:\n",
" chunk_iter = pd.read_csv(\n",
" csv_file_path,\n",
" header=None,\n",
" names=tick_column_names, # Assigns ['Timestamp_str', 'Bid', 'Ask']\n",
" chunksize=chunk_size,\n",
" na_values=['null', 'NaN', ''],\n",
" low_memory=False # Can sometimes help with mixed types if not specifying dtypes\n",
" )\n",
"\n",
" for i, tick_chunk_df in enumerate(chunk_iter):\n",
" print(f\"Processing chunk {i+1} (Shape: {tick_chunk_df.shape})...\")\n",
"\n",
" # 1. Convert Timestamp\n",
" timestamp_col_name_in_chunk = tick_column_names[0] # This is 'Timestamp_str'\n",
" if config.tick_datetime_format: # From ConfigManager\n",
" tick_chunk_df[timestamp_col_name_in_chunk] = pd.to_datetime(tick_chunk_df[timestamp_col_name_in_chunk], format=config.tick_datetime_format, errors='coerce')\n",
" else:\n",
" tick_chunk_df[timestamp_col_name_in_chunk] = pd.to_datetime(tick_chunk_df[timestamp_col_name_in_chunk], errors='coerce')\n",
" \n",
" tick_chunk_df.dropna(subset=[timestamp_col_name_in_chunk], inplace=True)\n",
"\n",
" # 2. Handle Timezone\n",
" if tick_chunk_df[timestamp_col_name_in_chunk].dt.tz is None:\n",
" try:\n",
" tick_chunk_df[timestamp_col_name_in_chunk] = tick_chunk_df[timestamp_col_name_in_chunk].dt.tz_localize('UTC')\n",
" except Exception as tz_err: # Catches AmbiguousTimeError or NonExistentTimeError during DST transitions if not UTC\n",
" print(f\" Warning: Could not localize timestamp for chunk {i+1} to UTC. Error: {tz_err}. Trying to infer timezone or skipping problematic rows.\")\n",
" # Attempt to infer or skip, this part is tricky with complex local times\n",
" try:\n",
" tick_chunk_df[timestamp_col_name_in_chunk] = pd.to_datetime(tick_chunk_df[timestamp_col_name_in_chunk], errors='coerce', utc=True)\n",
" tick_chunk_df.dropna(subset=[timestamp_col_name_in_chunk], inplace=True)\n",
" except Exception as tz_infer_err:\n",
" print(f\" Could not infer UTC timezone for chunk {i+1} after error: {tz_infer_err}. Some rows might be lost.\")\n",
" continue # Skip this chunk if critical error\n",
" else:\n",
" tick_chunk_df[timestamp_col_name_in_chunk] = tick_chunk_df[timestamp_col_name_in_chunk].dt.tz_convert('UTC')\n",
" \n",
" tick_chunk_df.set_index(timestamp_col_name_in_chunk, inplace=True)\n",
"\n",
" # 3. Convert price columns to numeric\n",
" # price_col_for_ohlc is already defined in Cell 2 (e.g., 'Bid')\n",
" # We also need to ensure the other price column ('Ask') is numeric if it exists\n",
" \n",
" cols_to_convert_to_numeric = [col for col in [tick_column_names[1], tick_column_names[2]] if col in tick_chunk_df.columns]\n",
" for col_to_convert in cols_to_convert_to_numeric:\n",
" tick_chunk_df[col_to_convert] = pd.to_numeric(tick_chunk_df[col_to_convert], errors='coerce')\n",
" \n",
" tick_chunk_df.dropna(subset=cols_to_convert_to_numeric, inplace=True)\n",
" \n",
" if tick_chunk_df.empty:\n",
" print(f\" Chunk {i+1} is empty after preprocessing (timestamp/numeric conversion). Skipping.\")\n",
" continue\n",
" \n",
" # 4. Resample this chunk\n",
" # price_col_for_ohlc was defined in Cell 2 (e.g., 'Bid')\n",
" ohlc_chunk = resample_chunk_to_ohlc(tick_chunk_df, price_col_for_ohlc, resample_timeframe)\n",
" \n",
" if not ohlc_chunk.empty:\n",
" all_ohlc_data.append(ohlc_chunk)\n",
" print(f\" Chunk {i+1} processed. Resampled OHLC shape: {ohlc_chunk.shape}\")\n",
" processed_chunks_count += 1\n",
" else:\n",
" print(f\" Chunk {i+1} resulted in empty OHLC data after resampling.\")\n",
"\n",
" if all_ohlc_data:\n",
" final_ohlc_df = pd.concat(all_ohlc_data)\n",
" final_ohlc_df.sort_index(inplace=True)\n",
" print(f\"\\nAll {processed_chunks_count} non-empty chunks processed. Final OHLC DataFrame shape: {final_ohlc_df.shape}\")\n",
" \n",
" if not final_ohlc_df.empty:\n",
" print(\"\\nHead of final OHLC data:\")\n",
" print(final_ohlc_df.head())\n",
" print(\"\\nTail of final OHLC data:\")\n",
" print(final_ohlc_df.tail())\n",
" \n",
" final_ohlc_df.to_parquet(output_parquet_path)\n",
" print(f\"\\nProcessed OHLC data saved to: {output_parquet_path}\")\n",
" else:\n",
" print(\"\\nFinal OHLC DataFrame is empty after concatenating chunks.\")\n",
" \n",
" else:\n",
" print(\"\\nNo OHLC data was generated from any of the chunks.\")\n",
"\n",
"except FileNotFoundError:\n",
" print(f\"ERROR: CSV file not found at {csv_file_path}. Please check the path and filename defined in Cell 2.\")\n",
"except Exception as e:\n",
" print(f\"An error occurred during chunk processing: {e}\")\n",
" import traceback\n",
" traceback.print_exc()\n"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "4bcd64c9",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Loading Parquet file from: /Users/carletes/Documents/lorentzian/processed_data/XAUUSD_OHLC_h.parquet\n",
"\n",
"Successfully loaded Parquet file.\n",
"Shape of loaded OHLC data: (90112, 5)\n",
"\n",
"Head of loaded OHLC data:\n",
" Open High Low Close Volume\n",
"Timestamp_str \n",
"2010-05-04 04:00:00+00:00 1182.580 1183.332 1180.329 1180.860 1\n",
"2010-05-04 05:00:00+00:00 1180.904 1181.470 1178.850 1179.494 1\n",
"2010-05-04 06:00:00+00:00 1179.578 1179.958 1178.502 1178.937 1\n",
"2010-05-04 07:00:00+00:00 1178.974 1180.183 1178.448 1179.969 1\n",
"2010-05-04 08:00:00+00:00 1179.958 1181.230 1179.426 1180.587 1\n",
"\n",
"Tail of loaded OHLC data:\n",
" Open High Low Close Volume\n",
"Timestamp_str \n",
"2025-04-22 22:00:00+00:00 3401.595 3402.195 3372.995 3381.305 1\n",
"2025-04-22 23:00:00+00:00 3381.305 3385.615 3366.525 3374.505 1\n",
"2025-04-23 00:00:00+00:00 3374.499 3382.505 3372.915 3380.075 1\n",
"2025-04-23 02:00:00+00:00 3349.515 3349.515 3312.625 3327.255 1\n",
"2025-04-23 03:00:00+00:00 3327.145 3349.795 3325.945 3336.315 1\n",
"\n",
"Info about loaded OHLC data:\n",
"<class 'pandas.core.frame.DataFrame'>\n",
"DatetimeIndex: 90112 entries, 2010-05-04 04:00:00+00:00 to 2025-04-23 03:00:00+00:00\n",
"Data columns (total 5 columns):\n",
" # Column Non-Null Count Dtype \n",
"--- ------ -------------- ----- \n",
" 0 Open 90112 non-null float64\n",
" 1 High 90112 non-null float64\n",
" 2 Low 90112 non-null float64\n",
" 3 Close 90112 non-null float64\n",
" 4 Volume 90112 non-null int64 \n",
"dtypes: float64(4), int64(1)\n",
"memory usage: 4.1 MB\n",
"\n",
"Basic statistics:\n",
" Open High Low Close Volume\n",
"count 90112.000000 90112.000000 90112.000000 90112.000000 90112.0\n",
"mean 1581.561000 1583.579931 1579.501256 1581.581102 1.0\n",
"std 395.433547 396.047200 394.816852 395.480764 0.0\n",
"min 1050.008000 1050.609000 1046.231000 1050.008000 1.0\n",
"25% 1271.901750 1273.261750 1270.480250 1271.909000 1.0\n",
"50% 1490.891000 1492.657000 1488.875000 1490.903000 1.0\n",
"75% 1802.358000 1804.520500 1800.049000 1802.380250 1.0\n",
"max 3494.145000 3499.915000 3479.015000 3494.135000 1.0\n",
"\n",
"NaNs in loaded data: 0\n"
]
}
],
"source": [
"# Cell 5: Inspect Saved Parquet File\n",
"import pandas as pd\n",
"\n",
"# output_parquet_path should be defined from running Cell 2\n",
"# It should now be /Users/carletes/Documents/lorentzian/processed_data/XAUUSD_OHLC_H.parquet\n",
"\n",
"print(f\"Loading Parquet file from: {output_parquet_path}\") \n",
"try:\n",
" ohlc_final_df = pd.read_parquet(output_parquet_path)\n",
" print(\"\\nSuccessfully loaded Parquet file.\")\n",
" print(f\"Shape of loaded OHLC data: {ohlc_final_df.shape}\")\n",
" \n",
" print(\"\\nHead of loaded OHLC data:\")\n",
" print(ohlc_final_df.head())\n",
" \n",
" print(\"\\nTail of loaded OHLC data:\")\n",
" print(ohlc_final_df.tail())\n",
" \n",
" print(\"\\nInfo about loaded OHLC data:\")\n",
" ohlc_final_df.info()\n",
" \n",
" print(\"\\nBasic statistics:\")\n",
" print(ohlc_final_df.describe())\n",
" \n",
" print(f\"\\nNaNs in loaded data: {ohlc_final_df.isnull().sum().sum()}\")\n",
"\n",
"except FileNotFoundError:\n",
" print(f\"ERROR: Parquet file not found at {output_parquet_path}. Did Cell 4 complete and save the file? Check the exact filename in the 'processed_data' directory.\")\n",
"except Exception as e:\n",
" print(f\"Error loading or inspecting Parquet file: {e}\")\n",
" import traceback\n",
" traceback.print_exc() # Print full traceback for other errors\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "72069bce",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "010fe3f4",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "433f7eea",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.13.3"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Display the source blob
Display the rendered blob
Raw
lorentzian_notebook.ipynb
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment