feat: add comprehensive diagnostic endpoint for Space debugging

app.py

@@ -67,6 +67,48 @@ def forecast_api(run_date_str, forecast_type):
         return error_path
 
 
+
+
+def run_diagnostic():
+    """Run comprehensive diagnostic tests. Returns path to report file."""
+    import subprocess
+    import sys
+    
+    output_path = "/tmp/diagnostic_report.txt"
+    
+    try:
+        result = subprocess.run(
+            [sys.executable, "diagnostic.py"],
+            capture_output=True,
+            text=True,
+            timeout=600
+        )
+        
+        with open(output_path, 'w') as f:
+            f.write("=== DIAGNOSTIC REPORT ===
+
+")
+            f.write("STDOUT:
+" + result.stdout)
+            f.write("
+
+STDERR:
+" + result.stderr)
+            f.write(f"
+
+Return code: {result.returncode}
+")
+        
+        print(f"Diagnostic completed: {output_path}")
+    except Exception as e:
+        with open(output_path, 'w') as f:
+            f.write(f"Diagnostic ERROR: {str(e)}
+")
+            import traceback
+            f.write(traceback.format_exc())
+    
+    return output_path
+
 # Build Gradio interface
 with gr.Blocks(title="FBMC Chronos-2 Forecasting") as demo:
     gr.Markdown("""
@@ -118,6 +160,15 @@ with gr.Blocks(title="FBMC Chronos-2 Forecasting") as demo:
             - Precision: bfloat16
             """)
 
+
+    gr.Markdown("---")
+    gr.Markdown("### Diagnostics")
+    with gr.Row():
+        diagnostic_btn = gr.Button("Run Diagnostics", variant="secondary")
+        diagnostic_output = gr.File(label="Diagnostic Report", type="filepath")
+    
+    diagnostic_btn.click(fn=run_diagnostic, inputs=[], outputs=diagnostic_output)
+    
     # Wire up the interface
     submit_btn.click(
         fn=forecast_api,

diagnostic.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+"""
+Diagnostic script to test inference pipeline components
+Run this in the Space environment to identify issues
+"""
+
+import sys
+import os
+from datetime import datetime
+
+print("="*60)
+print("FBMC CHRONOS-2 DIAGNOSTIC SCRIPT")
+print("="*60)
+
+# Test 1: Python environment
+print("\n[1] Python Environment")
+print(f"  Python version: {sys.version}")
+print(f"  Python path: {sys.executable}")
+
+# Test 2: Import dependencies
+print("\n[2] Importing Dependencies")
+try:
+    import torch
+    print(f"  PyTorch: {torch.__version__}")
+    print(f"  CUDA available: {torch.cuda.is_available()}")
+    if torch.cuda.is_available():
+        print(f"  CUDA device: {torch.cuda.get_device_name(0)}")
+except Exception as e:
+    print(f"  PyTorch ERROR: {e}")
+
+try:
+    import polars as pl
+    print(f"  Polars: {pl.__version__}")
+except Exception as e:
+    print(f"  Polars ERROR: {e}")
+
+try:
+    import numpy as np
+    print(f"  NumPy: {np.__version__}")
+except Exception as e:
+    print(f"  NumPy ERROR: {e}")
+
+try:
+    from chronos import ChronosPipeline
+    print(f"  Chronos: OK")
+except Exception as e:
+    print(f"  Chronos ERROR: {e}")
+
+try:
+    from datasets import load_dataset
+    print(f"  HF Datasets: OK")
+except Exception as e:
+    print(f"  HF Datasets ERROR: {e}")
+
+# Test 3: Environment variables
+print("\n[3] Environment Variables")
+print(f"  HF_TOKEN: {'SET' if os.getenv('HF_TOKEN') else 'NOT SET'}")
+print(f"  DEVICE: {os.getenv('DEVICE', 'cuda')}")
+
+# Test 4: Load dataset
+print("\n[4] Loading Dataset")
+try:
+    from datasets import load_dataset
+    hf_token = os.getenv("HF_TOKEN")
+    print(f"  Loading evgueni-p/fbmc-features-24month...")
+    dataset = load_dataset(
+        "evgueni-p/fbmc-features-24month",
+        split="train",
+        token=hf_token
+    )
+    print(f"  Dataset rows: {len(dataset)}")
+
+    # Convert to Polars
+    import polars as pl
+    df = pl.from_arrow(dataset.data.table)
+    print(f"  Polars shape: {df.shape}")
+
+    # Check for target columns
+    target_cols = [col for col in df.columns if col.startswith('target_border_')]
+    print(f"  Target borders: {len(target_cols)}")
+    if target_cols:
+        print(f"  First border: {target_cols[0]}")
+
+except Exception as e:
+    print(f"  Dataset ERROR: {e}")
+    import traceback
+    traceback.print_exc()
+
+# Test 5: Load Chronos model
+print("\n[5] Loading Chronos Model")
+try:
+    from chronos import ChronosPipeline
+    import torch
+
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    print(f"  Device: {device}")
+    print(f"  Loading amazon/chronos-t5-large...")
+
+    pipeline = ChronosPipeline.from_pretrained(
+        "amazon/chronos-t5-large",
+        device_map=device,
+        torch_dtype=torch.bfloat16 if device == "cuda" else torch.float32
+    )
+    print(f"  Model loaded successfully!")
+
+    # Test inference with dummy data
+    print(f"\n  Testing inference with dummy data...")
+    import numpy as np
+    dummy_context = np.random.randn(512).astype(np.float32)
+
+    forecast = pipeline.predict(
+        context=dummy_context,
+        prediction_length=24,
+        num_samples=5
+    )
+
+    forecast_np = forecast.numpy()
+    print(f"  Forecast shape: {forecast_np.shape}")
+
+    # Test quantile calculation
+    median = np.median(forecast_np, axis=0)
+    q10 = np.quantile(forecast_np, 0.1, axis=0)
+    q90 = np.quantile(forecast_np, 0.9, axis=0)
+
+    print(f"  Quantiles calculated successfully!")
+    print(f"    Median shape: {median.shape}")
+    print(f"    Q10 shape: {q10.shape}")
+    print(f"    Q90 shape: {q90.shape}")
+
+except Exception as e:
+    print(f"  Model ERROR: {e}")
+    import traceback
+    traceback.print_exc()
+
+# Test 6: Test dynamic_forecast import
+print("\n[6] Testing Module Imports")
+try:
+    from src.forecasting.dynamic_forecast import DynamicForecast
+    print(f"  DynamicForecast: OK")
+except Exception as e:
+    print(f"  DynamicForecast ERROR: {e}")
+    import traceback
+    traceback.print_exc()
+
+try:
+    from src.forecasting.feature_availability import FeatureAvailability
+    print(f"  FeatureAvailability: OK")
+except Exception as e:
+    print(f"  FeatureAvailability ERROR: {e}")
+
+# Test 7: Quick inference test
+print("\n[7] Full Pipeline Test (Minimal)")
+try:
+    print(f"  Testing run_inference function...")
+    from src.forecasting.chronos_inference import run_inference
+
+    # This will be slow but should work
+    print(f"  Running smoke test for 2025-09-30...")
+    print(f"  (This may take 60+ seconds...)")
+
+    result_path = run_inference(
+        run_date="2025-09-30",
+        forecast_type="smoke_test",
+        output_dir="/tmp"
+    )
+
+    print(f"  Result file: {result_path}")
+
+    # Check if file has data
+    import polars as pl
+    df = pl.read_parquet(result_path)
+    print(f"  Result shape: {df.shape}")
+    print(f"  Columns: {df.columns}")
+
+    if len(df.columns) > 1:
+        print(f"  [SUCCESS] Forecast has data!")
+    else:
+        print(f"  [ERROR] Forecast is empty (only timestamps)")
+
+except Exception as e:
+    print(f"  Pipeline ERROR: {e}")
+    import traceback
+    traceback.print_exc()
+
+print("\n" + "="*60)
+print("DIAGNOSTIC COMPLETE")
+print("="*60)