Source code for gui.panels.extraction_panel
"""
Cerebrum Forex - Extraction Panel
OHLC data extraction with tabs: Status and Logs
"""
import logging
from datetime import datetime
from PyQt6.QtWidgets import (
QWidget, QVBoxLayout, QHBoxLayout, QLabel,
QTableWidget, QTableWidgetItem, QPushButton, QGroupBox,
QProgressBar, QHeaderView, QTabWidget, QFrame, QStackedWidget, QApplication,
QSpinBox, QMessageBox
)
from PyQt6.QtCore import Qt, QTimer, pyqtSignal
from PyQt6.QtGui import QFont, QColor
logger = logging.getLogger(__name__)
[docs]
class ExtractionPanel(QWidget):
"""Panel for OHLC extraction with tabbed layout"""
# Signal for thread-safe UI updates
_update_table_signal = pyqtSignal(dict)
def __init__(self, app_controller=None):
super().__init__()
self.app_controller = app_controller
self._init_ui()
self._timer = QTimer()
self._timer.timeout.connect(self._trigger_background_update)
# self._timer.start(5000) # DISABLE background monitoring loop to prevent UI freeze
# State
self._is_updating = False
# === STAGGERED INITIALIZATION (Non-Blocking) ===
# 1.0s: Check freshness (Fast metadata check)
QTimer.singleShot(1000, self._check_freshness_status)
# 3.0s: Full table population (Heavy CSV read)
QTimer.singleShot(3000, self._trigger_background_update)
# Clear file cache to force fresh read
self._file_cache = {}
self._cache_time = {}
self._first_load_done = False
# Connect signal for thread-safe UI updates
self._update_table_signal.connect(self._apply_background_results)
def _init_ui(self):
from config.settings import OHLC_DIR
layout = QVBoxLayout(self)
layout.setSpacing(10)
layout.setContentsMargins(10, 10, 10, 10)
# Header
header = QLabel("📊 Data Extraction")
header.setFont(QFont("Segoe UI", 16, QFont.Weight.Bold))
header.setStyleSheet("color: #0e639c;")
layout.addWidget(header)
# Controls
controls = QHBoxLayout()
from gui.widgets.styled_button import Mega3DButton, Premium3DButton
self.extract_btn = Mega3DButton("🔄 Smart Extract", color="#f59e0b", hover_color="#fbbf24", pressed_color="#d97706")
self.extract_btn.clicked.connect(self._smart_extract)
self.extract_btn.setMinimumWidth(140)
self.extract_btn.setToolTip("Smartly extract missing or outdated OHLC data from MT5 (Auto-detects gaps)")
controls.addWidget(self.extract_btn)
# === FROM YEAR PICKER ===
controls.addSpacing(15)
year_label = QLabel("📅 From:")
year_label.setFont(QFont("Segoe UI", 10))
year_label.setStyleSheet("color: #9ca3af;")
controls.addWidget(year_label)
# Year picker (enabled - user can choose start year)
self.from_year = QSpinBox()
from datetime import datetime
self.from_year.setRange(2000, datetime.now().year)
self.from_year.setValue(2015)
self.from_year.setFixedWidth(70)
self.from_year.setFixedHeight(28)
self.from_year.setFont(QFont("Cascadia Code", 10, QFont.Weight.Bold))
self.from_year.setButtonSymbols(QSpinBox.ButtonSymbols.NoButtons)
self.from_year.setAlignment(Qt.AlignmentFlag.AlignCenter)
self.from_year.setEnabled(True) # Enabled!
self.from_year.setStyleSheet("""
QSpinBox {
background: #374151;
color: #f59e0b;
border: 2px solid #4b5563;
border-radius: 4px;
padding: 2px;
}
QSpinBox:hover { border-color: #f59e0b; }
""")
self.from_year.setToolTip("Select start year for data extraction")
self.from_year.valueChanged.connect(self._on_year_changed)
controls.addWidget(self.from_year)
controls.addStretch()
self.status_label = QLabel("Ready")
self.status_label.setStyleSheet("color: #10b981; font-size: 12px;")
controls.addWidget(self.status_label)
# Server Date/Time Clock (Reference for User)
controls.addSpacing(20)
self.server_time_label = QLabel("🌐 Server: ----.--.-- --:--:--")
self.server_time_label.setFont(QFont("Cascadia Code", 10, QFont.Weight.Bold))
self.server_time_label.setStyleSheet("""
QLabel {
color: #60a5fa;
background: qlineargradient(x1:0, y1:0, x2:1, y2:1, stop:0 #1a1a1a, stop:1 #2d2d2d);
padding: 6px 15px;
border-radius: 6px;
border: 1px solid #3b82f6;
}
""")
self.server_time_label.setToolTip("Broker Server Time (Reference for data freshness)")
controls.addWidget(self.server_time_label)
# Update clock every second
self._clock_timer = QTimer()
self._clock_timer.timeout.connect(self._update_server_clock)
self._clock_timer.start(1000)
layout.addLayout(controls)
# Status label (simple status info)
self.info_label = QLabel("Checking files...")
self.info_label.setStyleSheet("color: #9ca3af; font-size: 11px; padding: 2px 5px;")
layout.addWidget(self.info_label)
# Progress
self.progress_bar = QProgressBar()
self.progress_bar.setVisible(False)
layout.addWidget(self.progress_bar)
# Tabs
self.tabs = QTabWidget()
self.tabs.setStyleSheet("""
QTabWidget::pane { border: 1px solid #3c3c3c; background: #252526; }
QTabBar::tab {
background: #2d2d2d; color: #ccc;
padding: 8px 20px; border: 1px solid #3c3c3c; border-bottom: none;
}
QTabBar::tab:selected { background: #252526; border-top: 2px solid #0e639c; }
""")
# Tab 1: Files Status
self.files_tab = self._create_files_tab()
self.tabs.addTab(self.files_tab, "📁 OHLC Files")
# Tab 2: Logs
self.logs_tab = self._create_logs_tab()
self.tabs.addTab(self.logs_tab, "📝 Logs")
layout.addWidget(self.tabs, stretch=1)
def _on_year_changed(self, year: int):
"""Save year to settings when changed"""
if self.app_controller:
settings = self.app_controller.get_settings()
settings["from_year"] = year
self.app_controller.save_settings(settings)
logger.info(f"[ExtractionPanel] Start year changed to {year}")
def _create_files_tab(self):
"""Create files status tab with loading overlay"""
tab = QWidget()
layout = QVBoxLayout(tab)
layout.setContentsMargins(5, 10, 5, 5)
# Container for table with overlay
self.table_container = QWidget()
container_layout = QVBoxLayout(self.table_container)
container_layout.setContentsMargins(0, 0, 0, 0)
# Loading overlay (transparent)
self.loading_overlay = QFrame(self.table_container)
self.loading_overlay.setStyleSheet("""
QFrame {
background-color: transparent;
}
""")
overlay_layout = QVBoxLayout(self.loading_overlay)
overlay_layout.setAlignment(Qt.AlignmentFlag.AlignCenter)
# Loading spinner label
self.loading_label = QLabel("⏳ Loading...")
self.loading_label.setFont(QFont("Segoe UI", 14, QFont.Weight.Bold))
self.loading_label.setStyleSheet("color: #60a5fa; background: transparent;")
self.loading_label.setAlignment(Qt.AlignmentFlag.AlignCenter)
overlay_layout.addWidget(self.loading_label)
# Timer label
self.loading_timer_label = QLabel("0.0s")
self.loading_timer_label.setFont(QFont("Consolas", 20, QFont.Weight.Bold))
self.loading_timer_label.setStyleSheet("color: #fbbf24; background: transparent;")
self.loading_timer_label.setAlignment(Qt.AlignmentFlag.AlignCenter)
overlay_layout.addWidget(self.loading_timer_label)
self.loading_overlay.setVisible(False)
self._loading_start_time = None
self._loading_timer = QTimer()
self._loading_timer.timeout.connect(self._update_loading_timer)
# No Data overlay (hidden by default)
self.no_data_overlay = QFrame(self.table_container)
self.no_data_overlay.setStyleSheet("""
QFrame {
background-color: rgba(30, 30, 30, 0.9);
}
""")
no_data_layout = QVBoxLayout(self.no_data_overlay)
no_data_layout.setAlignment(Qt.AlignmentFlag.AlignCenter)
no_data_label = QLabel("🚫 No Data Found")
no_data_label.setFont(QFont("Segoe UI", 16, QFont.Weight.Bold))
no_data_label.setStyleSheet("color: #ef4444; background: transparent;")
no_data_label.setAlignment(Qt.AlignmentFlag.AlignCenter)
no_data_layout.addWidget(no_data_label)
no_data_sub = QLabel("Files are missing or not accessible.\nClick 'Smart Extract' to fix.")
no_data_sub.setFont(QFont("Segoe UI", 11))
no_data_sub.setStyleSheet("color: #9ca3af; background: transparent;")
no_data_sub.setAlignment(Qt.AlignmentFlag.AlignCenter)
no_data_layout.addWidget(no_data_sub)
self.no_data_overlay.setVisible(False)
# Files table
self.files_table = QTableWidget()
self.files_table.setColumnCount(7)
self.files_table.setHorizontalHeaderLabels([
"TF", "Status", "Candles", "Train (50%)", "Val (50%)", "First Date", "Last Date"
])
# Optimize column widths
header = self.files_table.horizontalHeader()
header.setSectionResizeMode(0, QHeaderView.ResizeMode.ResizeToContents) # TF minimized
header.setSectionResizeMode(1, QHeaderView.ResizeMode.ResizeToContents) # Status
header.setSectionResizeMode(2, QHeaderView.ResizeMode.ResizeToContents) # Candles
header.setSectionResizeMode(3, QHeaderView.ResizeMode.ResizeToContents) # Train
header.setSectionResizeMode(4, QHeaderView.ResizeMode.ResizeToContents) # Val
header.setSectionResizeMode(5, QHeaderView.ResizeMode.Stretch) # First Date maximized
header.setSectionResizeMode(6, QHeaderView.ResizeMode.Stretch) # Last Date maximized
self.files_table.setAlternatingRowColors(True)
self.files_table.setStyleSheet("QTableWidget { alternate-background-color: #2d2d2d; }")
timeframes = ["1m", "5m", "15m", "30m", "1h", "4h"]
self.files_table.setRowCount(len(timeframes))
for i, tf in enumerate(timeframes):
self.files_table.setItem(i, 0, QTableWidgetItem(tf))
for j in range(1, 7):
self.files_table.setItem(i, j, QTableWidgetItem("--"))
container_layout.addWidget(self.files_table)
layout.addWidget(self.table_container)
# Auto-extraction info
self.auto_status = QLabel("Auto-extraction: Disabled (activate app to enable)")
self.auto_status.setStyleSheet("color: #6b7280; font-size: 11px; padding: 5px;")
layout.addWidget(self.auto_status)
# Bottom info note (always visible)
self.bottom_note = QLabel(
"ℹ️ Tip: Choose your start year above. Lower timeframes (1m, 5m) may have limited broker history. "
"For maximum data: MT5 → Tools → Options → Charts → Max bars = Unlimited. "
"✅ AI predictions work great with available data!"
)
self.bottom_note.setWordWrap(True)
self.bottom_note.setStyleSheet("""
QLabel {
color: #1a1a1a;
font-size: 12px;
padding: 10px 14px;
background: #f59e0b;
border-radius: 6px;
margin-top: 8px;
}
""")
layout.addWidget(self.bottom_note)
return tab
def _show_loading(self, message: str = "Loading..."):
"""Show loading overlay with timer"""
self.loading_label.setText(f"⏳ {message}")
self._loading_start_time = datetime.now()
self.loading_timer_label.setText("0.0s")
# Position overlay over table
self.loading_overlay.setGeometry(self.files_table.geometry())
self.loading_overlay.raise_()
self.loading_overlay.setVisible(True)
self._loading_timer.start(100) # Update every 100ms
def _hide_loading(self):
"""Hide loading overlay"""
self._loading_timer.stop()
self.loading_overlay.setVisible(False)
self._loading_start_time = None
def _update_loading_timer(self):
"""Update loading timer display"""
if self._loading_start_time:
elapsed = (datetime.now() - self._loading_start_time).total_seconds()
self.loading_timer_label.setText(f"{elapsed:.1f}s")
# Position overlay (in case of resize)
self.loading_overlay.setGeometry(self.files_table.geometry())
self.no_data_overlay.setGeometry(self.files_table.geometry())
[docs]
def resizeEvent(self, event):
super().resizeEvent(event)
# Ensure overlays resize with table
if hasattr(self, 'loading_overlay'):
self.loading_overlay.setGeometry(self.files_table.geometry())
if hasattr(self, 'no_data_overlay'):
self.no_data_overlay.setGeometry(self.files_table.geometry())
def _create_logs_tab(self):
"""Create logs tab"""
tab = QWidget()
layout = QVBoxLayout(tab)
layout.setContentsMargins(5, 10, 5, 5)
self.logs_table = QTableWidget()
self.logs_table.setColumnCount(4)
self.logs_table.setHorizontalHeaderLabels(["Time", "Timeframe", "Status", "Candles"])
self.logs_table.horizontalHeader().setSectionResizeMode(QHeaderView.ResizeMode.Stretch)
self.logs_table.setStyleSheet("QTableWidget { alternate-background-color: #2d2d2d; }")
layout.addWidget(self.logs_table)
return tab
def _check_freshness_status(self):
"""Check OHLC freshness on load and update button/info accordingly"""
from config.settings import OHLC_DIR
# Freshness thresholds (hours) per timeframe
# Freshness thresholds (hours) per timeframe
# Strict: trigger update if > 1.5x - 2x the timeframe duration is missing
freshness_hours = {
"1m": 0.05, # 3 minutes
"5m": 0.2, # 12 minutes
"15m": 0.5, # 30 minutes
"30m": 1.0, # 1 hour
"1h": 1.5, # 1.5 hours
"4h": 5.0, # 5 hours
"1d": 26, # 1 day + buffer
"1w": 180, # 1 week + buffer
}
# Dashboard focus: 1m to 4h
timeframes = ["1m", "5m", "15m", "30m", "1h", "4h"]
outdated_tfs = []
missing_tfs = []
fresh_count = 0
for tf in timeframes:
ohlc_file = OHLC_DIR / f"ohlc_EURUSD_{tf}.csv"
if not ohlc_file.exists():
missing_tfs.append(tf)
else:
file_age_hours = (datetime.now() - datetime.fromtimestamp(ohlc_file.stat().st_mtime)).total_seconds() / 3600
if file_age_hours > freshness_hours.get(tf, 6):
outdated_tfs.append(tf)
else:
fresh_count += 1
# Update UI based on status
if missing_tfs:
self.extract_btn.set_color(
color="#dc2626", # Red
hover="#ef4444",
pressed="#b91c1c"
)
self.status_label.setText("❌ MISSING")
self.status_label.setToolTip(
"Critical data missing.\nClick 'Smart Extract' to download."
)
self.status_label.setStyleSheet("color: #ef4444; font-size: 13px; font-weight: bold;")
self.info_label.setText(f"Missing: {', '.join(missing_tfs[:5])}")
elif outdated_tfs:
self.extract_btn.set_color(
color="#f59e0b", # Orange
hover="#fbbf24",
pressed="#d97706"
)
self.status_label.setText("⚠️ OUTDATED")
self.status_label.setToolTip(
"Local data is old.\nClick 'Smart Extract' to sync."
)
self.status_label.setStyleSheet("color: #fbbf24; font-size: 13px; font-weight: bold;")
self.info_label.setText(f"Outdated: {', '.join(outdated_tfs[:5])}")
else:
self.extract_btn.set_color(
color="#059669", # Green
hover="#10b981",
pressed="#047857"
)
self.status_label.setText("✓ VALID")
self.status_label.setToolTip("All data is up to date.")
self.status_label.setStyleSheet("color: #10b981; font-size: 13px; font-weight: bold;")
self.info_label.setText("All files are fresh.")
def _smart_extract(self):
"""Smart extraction - manual trigger only"""
logger.info("Smart Extract button clicked")
if not self.app_controller:
logger.error("Smart Extract failed: app_controller is None")
self.status_label.setText("❌ Error: No controller")
self.status_label.setStyleSheet("color: #ef4444;")
return
from config.settings import OHLC_DIR
import threading
# Show loading immediately to give instant feedback
self._show_loading("Checking Status...")
QApplication.processEvents() # Force UI update before sync check
logger.info(f"Checking OHLC files in: {OHLC_DIR}")
# Freshness thresholds (hours) - How old can data be before triggering update?
# These are generous to avoid constant updates during weekends/holidays
freshness = {
"1m": 2, # 2 hours - market moves fast
"5m": 4, # 4 hours
"15m": 8, # 8 hours
"30m": 12, # 12 hours
"1h": 24, # 1 day
"4h": 48 # 2 days
}
timeframes = ["1m", "5m", "15m", "30m", "1h", "4h"]
missing = []
outdated = []
settings = self.app_controller.get_settings()
target_year = settings.get("from_year", 2020)
for tf in timeframes:
# Check file status via controller (reads actual data dates)
info = self.app_controller.mt5.get_ohlc_status(tf)
if not info['exists']:
missing.append(tf)
else:
# Check if data starts late (e.g. starts 2020 but we want 2015)
# If start date is significantly different from target year, treat as missing/incomplete
first_date = info.get('from_date')
if first_date and first_date.year > target_year:
logger.info(f"{tf} starts in {first_date.year}, required {target_year} -> Marking as incomplete")
missing.append(tf)
continue
# Check freshness
last_date = info.get('to_date')
if last_date:
age_h = (datetime.now() - last_date).total_seconds() / 3600
if age_h > freshness.get(tf, 6):
outdated.append(tf)
logger.info(f"Missing timeframes: {missing}, Outdated: {outdated}")
# All OK - nothing to do
if not missing and not outdated:
self.extract_btn.set_color(
color="#059669", # Green
hover="#10b981",
pressed="#047857"
)
self.status_label.setText("✓ All OK")
self.status_label.setStyleSheet("color: #10b981;")
self.info_label.setText("All files are up to date. No extraction needed.")
logger.info("All OHLC files are fresh, no extraction needed")
self._hide_loading() # Important: Hide "Checking Status..." overlay
return
# Need extraction
# FIX: If we have ANY missing or incomplete files, we must run in FULL mode (is_update=False)
# to ensure those specific files are re-created from scratch.
is_update = len(missing) == 0
# Double check: If manual click, and we have outdated files that are very small (e.g. M1 empty), force full.
if not is_update:
logger.info(f"Smart Extract: Forcing FULL extraction because of missing/incomplete files: {missing}")
if missing:
self.extract_btn.set_color(
color="#dc2626", # Red
hover="#ef4444",
pressed="#b91c1c"
)
# self.info_label.setText(f"Missing: {', '.join(missing)}")
else:
self.extract_btn.set_color(
color="#f59e0b", # Orange
hover="#fbbf24",
pressed="#d97706"
)
# self.info_label.setText(f"Outdated: {', '.join(outdated)}")
self.status_label.setText("Extracting...")
self.status_label.setStyleSheet("color: #fbbf24;")
self.progress_bar.setVisible(True)
self.progress_bar.setRange(0, 0)
# Show loading overlay with timer
self._show_loading("Extracting OHLC data...")
# Run extraction in background thread
def do_extract():
try:
logger.info(f"Starting Smart Extraction (Missing: {len(missing)}, Outdated: {len(outdated)})")
settings = self.app_controller.get_settings()
from_year = settings.get("from_year", 2020)
from_date = datetime(from_year, 1, 1)
# 1. Handle MISSING/INCOMPLETE files -> Force Full Download (is_update=False)
# WARNING: This overwrites files. Only do this for files identified as broken/missing.
for tf in missing:
logger.info(f"[{tf}] Re-downloading full history (Missing/Incomplete)...")
self.app_controller.mt5.extract_ohlc(tf, is_update=False, from_date=from_date)
# 2. Handle OUTDATED files -> Update/Merge (is_update=True)
# This PRESERVES existing history (e.g. 2015 data) and appends new data.
for tf in outdated:
logger.info(f"[{tf}] Updating history (Merge)...")
self.app_controller.mt5.extract_ohlc(tf, is_update=True, from_date=from_date)
logger.info("Smart Extraction completed successfully")
# Schedule UI update back on main thread
QTimer.singleShot(0, self._after_extraction)
except Exception as e:
logger.error(f"Extraction thread error: {e}")
# Update UI to show error
QTimer.singleShot(0, lambda: self._extraction_error(str(e)))
threading.Thread(target=do_extract, daemon=True).start()
# Log with actual timeframes being extracted
tfs_to_extract = missing + outdated
tf_display = ", ".join(tfs_to_extract[:3]) + ("..." if len(tfs_to_extract) > 3 else "")
self._add_log(tf_display if tf_display else "ALL", "EXTRACT" if not is_update else "UPDATE", f"{len(missing)} missing, {len(outdated)} outdated")
def _extraction_error(self, error_msg: str):
"""Handle extraction error"""
self._hide_loading()
self.progress_bar.setVisible(False)
self.status_label.setText("❌ Error")
self.status_label.setStyleSheet("color: #ef4444;")
self.info_label.setText(f"Error: {error_msg[:50]}...")
self._add_log("--", "ERROR", error_msg[:30])
def _after_extraction(self):
"""Called after extraction completes"""
self._hide_loading()
self.progress_bar.setVisible(False)
self.extract_btn.setEnabled(True)
# 1. Clear caches immediately
self._file_cache = {}
self._cache_time = {}
logger.info("Extraction complete. Clearing caches and updating status...")
# 2. Update the main status label immediately (st_mtime is fast)
self._check_freshness_status()
# ADDED: Explicit Log Feedback
msg = "Extraction Finished."
status_type = "DONE"
if "Missing" in self.status_label.text():
msg += " (Server limit reached?)"
status_type = "WARNING"
self._add_log("ALL", "TRAINING", msg)
self._add_log("ALL", status_type, "Check results above.")
# 3. Trigger heavy background table update (pd.read_csv)
self._trigger_background_update()
# 4. Optional: Briefly show Done message if all is valid
if self.status_label.text() == "✓ VALID":
self.status_label.setText("✓ SYNCED")
QTimer.singleShot(2000, self._check_freshness_status)
def _update_server_clock(self):
"""Update server time display"""
if self.app_controller and self.app_controller.mt5:
srv_time = self.app_controller.mt5.get_server_time()
if srv_time:
# Show full date and time for better data reference
self.server_time_label.setText(f"🌐 Server: {srv_time.strftime('%Y.%m.%d %H:%M:%S')}")
# Optional: subtle color pulse or update indicator if needed
else:
pass
def _trigger_background_update(self):
"""Start background update if not running"""
# On first load, show specific loading message if not already shown
if not self._first_load_done and not self._is_updating:
if not self.loading_overlay.isVisible():
self._show_loading("Checking files...")
if self._is_updating: return
# Calculate Local - UTC offset
# This assumes CSV data is stored as UTC (pd.to_datetime unit='s' default)
local_offset = (datetime.now() - datetime.utcnow()).total_seconds()
import threading
self._is_updating = True
threading.Thread(target=self._background_update_task, args=(local_offset,), daemon=True).start()
def _background_update_task(self, offset=0):
"""Heavy lifting (CSV reading) in background"""
try:
from config.settings import OHLC_DIR
import pandas as pd
import os
from datetime import timedelta
# Gather data for all TFs
results = {} # tf -> {exists, candles, from, to, status}
timeframes = ["1m", "5m", "15m", "30m", "1h", "4h"]
for tf in timeframes:
ohlc_file = OHLC_DIR / f"ohlc_EURUSD_{tf}.csv"
if not ohlc_file.exists():
results[tf] = {"exists": False}
continue
# Check cache logic
if not hasattr(self, '_file_cache'): self._file_cache = {}
if not hasattr(self, '_cache_time'): self._cache_time = {}
try:
file_mtime = ohlc_file.stat().st_mtime
except Exception:
file_mtime = 0
cache_valid = (tf in self._file_cache and self._cache_time.get(tf) == file_mtime)
if cache_valid:
# Apply offset dynamically even for cached items?
# No, we bake it into the display string.
# If offset changes (DST change?), a restart fixes it.
results[tf] = self._file_cache[tf]
results[tf]['exists'] = True
else:
# HEAVY READ
try:
# Optimization: Get size first
size_mb = ohlc_file.stat().st_size / (1024*1024)
logger.info(f"[{tf}] Reading {ohlc_file.resolve()} ({size_mb:.2f} MB)")
df = pd.read_csv(ohlc_file, usecols=['time'])
df['time'] = pd.to_datetime(df['time'], errors='coerce')
candles = len(df)
if not df.empty:
from_d = df['time'].min()
to_d = df['time'].max()
# APPLY LOCALIZATION (UTC -> Local)
# We assume the stored helper is UTC-ish (from unit='s')
# so adding (Local - UTC) converts to Local.
if pd.notna(from_d):
from_d = from_d + timedelta(seconds=offset)
if pd.notna(to_d):
to_d = to_d + timedelta(seconds=offset)
from_s = from_d.strftime("%Y-%m-%d") if pd.notna(from_d) else "--"
to_s = to_d.strftime("%Y-%m-%d %H:%M") if pd.notna(to_d) else "--"
else:
from_s = "--"
to_s = "--"
entry = {
"exists": True,
"candles": candles,
"from_date": from_s,
"to_date": to_s
}
self._file_cache[tf] = entry
self._cache_time[tf] = file_mtime
results[tf] = entry
except Exception as e:
logger.warning(f"Error reading {tf}: {e}")
results[tf] = {"exists": True, "error": True}
# Done gathering. Emit signal for thread-safe UI update.
self._update_table_signal.emit(results)
except Exception as e:
logger.error(f"Background update failed: {e}")
finally:
self._is_updating = False
def _apply_background_results(self, results):
"""Update UI with gathered data (Main Thread)"""
try:
self._hide_loading()
self._first_load_done = True
logger.info(f"_apply_background_results CALLED with {len(results)} timeframes")
timeframes = ["1m", "5m", "15m", "30m", "1h", "4h"]
has_any_data = False
for i, tf in enumerate(timeframes):
res = results.get(tf, {})
logger.debug(f"TF {tf}: exists={res.get('exists')}, candles={res.get('candles', 'N/A')}")
if res.get("exists"):
has_any_data = True
if res.get("error"):
self.files_table.setItem(i, 1, QTableWidgetItem("⚠ Error"))
self.files_table.setItem(i, 2, QTableWidgetItem("Error"))
else:
status_item = QTableWidgetItem("✓ Ready")
status_item.setForeground(QColor("#10b981"))
self.files_table.setItem(i, 1, status_item)
count = res.get('candles', 0)
train_c = int(count * 0.5)
val_c = count - train_c
self.files_table.setItem(i, 2, QTableWidgetItem(f"{count:,}"))
self.files_table.setItem(i, 3, QTableWidgetItem(f"{train_c:,}"))
self.files_table.setItem(i, 4, QTableWidgetItem(f"{val_c:,}"))
self.files_table.setItem(i, 5, QTableWidgetItem(res.get('from_date', '--')))
self.files_table.setItem(i, 6, QTableWidgetItem(res.get('to_date', '--')))
else:
status_item = QTableWidgetItem("✗ Missing")
status_item.setForeground(QColor("#ef4444"))
self.files_table.setItem(i, 1, status_item)
for j in range(2, 7):
self.files_table.setItem(i, j, QTableWidgetItem("--"))
# Handle No Data Overlay
if not has_any_data:
self.no_data_overlay.setGeometry(self.files_table.geometry())
self.no_data_overlay.setVisible(True)
self.no_data_overlay.raise_()
else:
self.no_data_overlay.setVisible(False)
# Update from_year range based on actual data availability
earliest_year = 2030
latest_year = 2000
for tf, res in results.items():
if res.get("exists") and res.get("from_date"):
try:
from_str = res.get("from_date", "")
to_str = res.get("to_date", "")
if from_str and from_str != "--":
year = int(from_str.split("-")[0])
if year < earliest_year:
earliest_year = year
if to_str and to_str != "--":
year = int(to_str.split("-")[0])
if year > latest_year:
latest_year = year
except:
pass
# Set the from_year range based on actual data
from datetime import datetime
current_year = datetime.now().year
if earliest_year < 2030:
# Data exists: limit range to data years
self.from_year.setRange(earliest_year, min(latest_year, current_year))
self.from_year.setValue(earliest_year)
self.from_year.setToolTip(f"Data available from {earliest_year}")
else:
# No data: keep default range
self.from_year.setRange(2000, current_year)
# Update ready status
if self.app_controller and not self.app_controller.is_extracting:
self.auto_status.setText("Background Monitor: Active ⚡")
except Exception as e:
logger.error(f"UI Update failed: {e}")
def _add_log(self, timeframe, status, candles):
row = 0
self.logs_table.insertRow(row)
self.logs_table.setItem(row, 0, QTableWidgetItem(datetime.now().strftime("%H:%M:%S")))
self.logs_table.setItem(row, 1, QTableWidgetItem(timeframe))
self.logs_table.setItem(row, 2, QTableWidgetItem(status))
self.logs_table.setItem(row, 3, QTableWidgetItem(candles))
while self.logs_table.rowCount() > 20:
self.logs_table.removeRow(self.logs_table.rowCount() - 1)
[docs]
def update_profile_constraints(self, profile):
"""Update profile constraints (legacy - year picker removed, history is now automatic)"""
if profile:
logger.info(f"[ExtractionPanel] Profile set to {profile} - history is fetched automatically")