diagnostic.py

#!/usr/bin/env python3
"""
Diagnostic script to test inference pipeline components
Run this in the Space environment to identify issues
"""

import sys
import os
from datetime import datetime

print("="*60)
print("FBMC CHRONOS-2 DIAGNOSTIC SCRIPT")
print("="*60)

# Test 1: Python environment
print("\n[1] Python Environment")
print(f"  Python version: {sys.version}")
print(f"  Python path: {sys.executable}")

# Test 2: Import dependencies
print("\n[2] Importing Dependencies")
try:
    import torch
    print(f"  PyTorch: {torch.__version__}")
    print(f"  CUDA available: {torch.cuda.is_available()}")
    if torch.cuda.is_available():
        print(f"  CUDA device: {torch.cuda.get_device_name(0)}")
except Exception as e:
    print(f"  PyTorch ERROR: {e}")

try:
    import polars as pl
    print(f"  Polars: {pl.__version__}")
except Exception as e:
    print(f"  Polars ERROR: {e}")

try:
    import numpy as np
    print(f"  NumPy: {np.__version__}")
except Exception as e:
    print(f"  NumPy ERROR: {e}")

try:
    from chronos import ChronosPipeline
    print(f"  Chronos: OK")
except Exception as e:
    print(f"  Chronos ERROR: {e}")

try:
    from datasets import load_dataset
    print(f"  HF Datasets: OK")
except Exception as e:
    print(f"  HF Datasets ERROR: {e}")

# Test 3: Environment variables
print("\n[3] Environment Variables")
print(f"  HF_TOKEN: {'SET' if os.getenv('HF_TOKEN') else 'NOT SET'}")
print(f"  DEVICE: {os.getenv('DEVICE', 'cuda')}")

# Test 4: Load dataset
print("\n[4] Loading Dataset")
try:
    from datasets import load_dataset
    hf_token = os.getenv("HF_TOKEN")
    print(f"  Loading evgueni-p/fbmc-features-24month...")
    dataset = load_dataset(
        "evgueni-p/fbmc-features-24month",
        split="train",
        token=hf_token
    )
    print(f"  Dataset rows: {len(dataset)}")

    # Convert to Polars
    import polars as pl
    df = pl.from_arrow(dataset.data.table)
    print(f"  Polars shape: {df.shape}")

    # Check for target columns
    target_cols = [col for col in df.columns if col.startswith('target_border_')]
    print(f"  Target borders: {len(target_cols)}")
    if target_cols:
        print(f"  First border: {target_cols[0]}")

except Exception as e:
    print(f"  Dataset ERROR: {e}")
    import traceback
    traceback.print_exc()

# Test 5: Load Chronos model
print("\n[5] Loading Chronos Model")
try:
    from chronos import ChronosPipeline
    import torch

    device = "cuda" if torch.cuda.is_available() else "cpu"
    print(f"  Device: {device}")
    print(f"  Loading amazon/chronos-t5-large...")

    pipeline = ChronosPipeline.from_pretrained(
        "amazon/chronos-t5-large",
        device_map=device,
        torch_dtype=torch.bfloat16 if device == "cuda" else torch.float32
    )
    print(f"  Model loaded successfully!")

    # Test inference with dummy data
    print(f"\n  Testing inference with dummy data...")
    import numpy as np
    dummy_context = np.random.randn(512).astype(np.float32)

    forecast = pipeline.predict(
        context=dummy_context,
        prediction_length=24,
        num_samples=5
    )

    forecast_np = forecast.numpy()
    print(f"  Forecast shape: {forecast_np.shape}")

    # Test quantile calculation
    median = np.median(forecast_np, axis=0)
    q10 = np.quantile(forecast_np, 0.1, axis=0)
    q90 = np.quantile(forecast_np, 0.9, axis=0)

    print(f"  Quantiles calculated successfully!")
    print(f"    Median shape: {median.shape}")
    print(f"    Q10 shape: {q10.shape}")
    print(f"    Q90 shape: {q90.shape}")

except Exception as e:
    print(f"  Model ERROR: {e}")
    import traceback
    traceback.print_exc()

# Test 6: Test dynamic_forecast import
print("\n[6] Testing Module Imports")
try:
    from src.forecasting.dynamic_forecast import DynamicForecast
    print(f"  DynamicForecast: OK")
except Exception as e:
    print(f"  DynamicForecast ERROR: {e}")
    import traceback
    traceback.print_exc()

try:
    from src.forecasting.feature_availability import FeatureAvailability
    print(f"  FeatureAvailability: OK")
except Exception as e:
    print(f"  FeatureAvailability ERROR: {e}")

# Test 7: Quick inference test
print("\n[7] Full Pipeline Test (Minimal)")
try:
    print(f"  Testing run_inference function...")
    from src.forecasting.chronos_inference import run_inference

    # This will be slow but should work
    print(f"  Running smoke test for 2025-09-30...")
    print(f"  (This may take 60+ seconds...)")

    result_path = run_inference(
        run_date="2025-09-30",
        forecast_type="smoke_test",
        output_dir="/tmp"
    )

    print(f"  Result file: {result_path}")

    # Check if file has data
    import polars as pl
    df = pl.read_parquet(result_path)
    print(f"  Result shape: {df.shape}")
    print(f"  Columns: {df.columns}")

    if len(df.columns) > 1:
        print(f"  [SUCCESS] Forecast has data!")
    else:
        print(f"  [ERROR] Forecast is empty (only timestamps)")

except Exception as e:
    print(f"  Pipeline ERROR: {e}")
    import traceback
    traceback.print_exc()

print("\n" + "="*60)
print("DIAGNOSTIC COMPLETE")
print("="*60)