Pr 101

.devcontainer/devcontainer.json

@@ -0,0 +1,33 @@
+{
+  "name": "Python 3",
+  // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+  "image": "mcr.microsoft.com/devcontainers/python:1-3.11-bullseye",
+  "customizations": {
+    "codespaces": {
+      "openFiles": [
+        "README.md",
+        "Dashboard/streamlit_app.py"
+      ]
+    },
+    "vscode": {
+      "settings": {},
+      "extensions": [
+        "ms-python.python",
+        "ms-python.vscode-pylance"
+      ]
+    }
+  },
+  "updateContentCommand": "[ -f packages.txt ] && sudo apt update && sudo apt upgrade -y && sudo xargs apt install -y <packages.txt; [ -f requirements.txt ] && pip3 install --user -r requirements.txt; pip3 install --user streamlit; echo '✅ Packages installed and Requirements met'",
+  "postAttachCommand": {
+    "server": "streamlit run Dashboard/streamlit_app.py --server.enableCORS false --server.enableXsrfProtection false"
+  },
+  "portsAttributes": {
+    "8501": {
+      "label": "Application",
+      "onAutoForward": "openPreview"
+    }
+  },
+  "forwardPorts": [
+    8501
+  ]
+}

.env.template

@@ -1,11 +1,168 @@
-# Required
-METACULUS_TOKEN=1234567890
-
-# Optional
-OPENROUTER_API_KEY=1234567890
-PERPLEXITY_API_KEY=1234567890
-OPENAI_API_KEY=1234567890
-EXA_API_KEY=1234567890
-ASKNEWS_CLIENT_ID=1234567890
-ASKNEWS_SECRET=1234567890
-ANTHROPIC_API_KEY=1234567890
+############################################
+# SPAGBOT — ENVIRONMENT VARIABLES
+# Last updated: 2025-09-13 (Europe/Istanbul)
+############################################
+
+###############
+# METACULUS
+###############
+METACULUS_TOKEN=xxx
+METACULUS_INCLUDE_RESOLVED=0
+TOURNAMENT_ID=fall-aib-2025
+SUBMIT_PREDICTION=1
+
+###############
+# GENERAL
+###############
+GIT_SHA=
+
+###############
+# CONCURRENCY
+###############
+LLM_MAX_CONCURRENCY=4
+
+###############
+# LOGGING LIMITS
+###############
+HUMAN_LOG_MODEL_RAW_MAX_CHARS=5000
+HUMAN_LOG_RESEARCH_MAX_CHARS=15000
+RESEARCH_SNIPPET_MAX_CHARS=600
+RESEARCH_LOG_ALL_CANDIDATES=1
+
+###############
+# FORECASTING KNOBS
+###############
+FORECAST_TEMP=0.00
+FORECAST_TOP_P=0.10
+GTMC1_ACTIVATION_THRESHOLD=0.50
+
+###############
+# RESEARCH KNOBS
+###############
+RESEARCH_TEMP=0.10
+RESEARCH_TOP_P=0.90
+REQUIRE_YEAR_IF_PRESENT=0
+MIN_ANCHOR_MATCH=2
+SALVAGE_MIN_MATCH=1
+
+###############
+# CALIBRATION
+###############
+CALIBRATION_PATH=calibration/calibration_state.json
+CALIB_ADVICE_PATH=calibration/calibration_advice.json
+CALIB_WEIGHTS_PATH=calibration/calibration_weights.json
+SPAG_CALIB_K=1.0
+SPAG_CALIB_BETA=0.50
+SPAG_CALIB_HALFLIFE_DAYS=30
+
+###############
+# CSV / FILE PATHS
+###############
+FORECASTS_CSV_PATH=outputs/forecasts.csv
+TEST_POSTS_FILE=data/test_posts.jsonl
+TEST_POST_IDS=578,14333,22427,38195
+
+###############
+# MARKET SNAPSHOT
+###############
+ENABLE_MARKET_SNAPSHOT=1
+MARKET_SNAPSHOT_MAX_MATCHES=10
+
+###############
+# PROVIDER TOGGLES
+###############
+USE_OPENROUTER=1
+USE_GOOGLE=1
+ENABLE_SERPER=1
+ENABLE_SERPER_WEB=1
+ENABLE_ASKNEWS=1
+ENABLE_GROK=1
+
+###############
+# OPENROUTER (PRIMARY)
+###############
+OPENROUTER_API_KEY=xxx
+OPENROUTER_BASE_URL=https://openrouter.ai/api/v1
+OPENROUTER_FALLBACK_ID=openai/gpt-4o
+OPENROUTER_GPT5_ID=openai/gpt-4o
+OPENROUTER_GPT5_THINK_ID=openai/gpt-4o
+OPENROUTER_CLAUDE37_ID=anthropic/claude-3.7-sonnet
+
+# Alias sometimes used in code for OpenRouter:
+OR_API_KEY=xxx
+
+###############
+# OPENAI DIRECT (optional; usually unused now)
+###############
+OPENAI_API_KEY=xxx
+OPENAI_BASE_URL=https://openrouter.ai/api/v1
+
+###############
+# GEMINI (Google)
+###############
+GOOGLE_API_KEY=xxx
+GEMINI_API_KEY=xxxx
+GEMINI_MODEL=gemini-2.5-pro
+GEMINI_MODEL_ID=gemini-2.5-pro
+GEMINI_CALL_TIMEOUT_SEC=180
+GPT5_CALL_TIMEOUT_SEC=180
+
+###############
+# GROK (xAI)
+###############
+XAI_API_KEY=xxx
+XAI_BASE_URL=https://api.x.ai/v1
+XAI_GROK_ID=grok-4
+GROK_MODEL_ID=grok-4
+GROK_CALL_TIMEOUT_SEC=180
+
+
+###############
+# OTHER PROVIDERS (optional)
+###############
+PERPLEXITY_API_KEY=
+EXA_API_KEY=
+
+###############
+# CLASSIFIER / CACHES
+###############
+SPAGBOT_USE_LLM_CLASSIFIER=1
+SPAGBOT_DISABLE_CLASSIFIER_CACHE=1
+SPAGBOT_DISABLE_RESEARCH_CACHE=1
+# Optional: one markdown file per run (no subfolder) is 1; blank means one file per question
+HUMAN_LOG_FLAT=
+
+# Log GitHub Commit config 
+HUMAN_LOG_EXT=md
+DISABLE_GIT_LOGS=false
+COMMIT_LOGS=true 
+GIT_LOG_MESSAGE="chore(logs): append forecasts & run logs"
+GIT_REMOTE_NAME=origin
+GIT_BRANCH_NAME=main 
+
+###############
+# COST TRACKING (local use) USD per 1000 tokens
+###############
+MODEL_COSTS_JSON={
+  "openai/gpt-4o":{"prompt":2.5,"completion":10.0},
+  "anthropic/claude-3.7-sonnet":{"prompt":3.0,"completion":15.0},
+  "gemini-2.5-pro":{"prompt":1.25,"completion":10.0},
+  "grok-4":{"prompt":3.0,"completion":15.0}
+}
+
+###############
+# RESOLVER CONNECTORS
+###############
+# ACLED credentials (prefer refresh token; helper exchanges it for a new access token)
+ACLED_REFRESH_TOKEN=
+# Optional fallbacks for manual authentication (avoid storing secrets when possible)
+# ACLED_USERNAME=
+# ACLED_PASSWORD=
+# Legacy override; if set, treated as an access token
+# ACLED_ACCESS_TOKEN=
+
+# IOM DTM API keys
+DTM_API_PRIMARY_KEY=
+DTM_API_SECONDARY_KEY=
+# Optional custom header name (defaults to x-api-key)
+# DTM_API_HEADER_NAME=

.gitattributes

@@ -0,0 +1,11 @@
+# Normalize line endings
+* text=auto
+
+# Always use LF endings for shell scripts
+*.sh text eol=lf
+
+# Force LF for requirements.txt
+requirements.txt text eol=lf
+
+# Ensure hooks (in tracked .githooks folder) also use LF
+.githooks/* text eol=lf

.githooks/pre-commit

@@ -0,0 +1,24 @@
+#!/bin/sh
+# Re-exec this script in bash (prefer WSL bash if available), because sh doesn't support 'pipefail'.
+if [ -z "$BASH" ]; then
+  if command -v wsl.exe >/dev/null 2>&1; then
+    exec wsl bash "$0" "$@"
+  elif command -v bash >/dev/null 2>&1; then
+    exec bash "$0" "$@"
+  else
+    echo "pre-commit: bash not found; cannot run hook." >&2
+    exit 1
+  fi
+fi
+
+# Now we're in bash
+set -euo pipefail
+
+# Block committing these files unless OVERRIDE=1 is set
+if git diff --cached --name-only | grep -E -q '^(forecasts\.csv|[Dd]ashboard/data/forecasts\.parquet)$'; then
+  echo "❌ Blocked: committing forecasts.csv or Dashboard/data/forecasts.parquet"
+  echo "   (override with: OVERRIDE=1 git commit ...)"
+  if [ "${OVERRIDE:-}" != "1" ]; then
+    exit 1
+  fi
+fi

.github/workflows/build_dashboard_data.yml

@@ -0,0 +1,54 @@
+name: Build dashboard data
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ main ]
+    paths:
+      - "forecasts.csv"   # ← only rebuild when the CSV changes
+
+permissions:
+  contents: write  # required so we can push the parquet back to the repo
+
+jobs:
+  build-parquet:
+    runs-on: ubuntu-24.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install parquet deps
+        run: |
+          python -m pip install --upgrade pip
+          pip install pandas pyarrow
+
+      - name: Build Dashboard/data/forecasts.parquet
+        run: |
+          python - <<'PY'
+          import os, pandas as pd, pathlib
+          # Prefer the root CSV; fall back to forecast_logs if needed
+          candidates = ["forecasts.csv", "forecast_logs/forecasts.csv"]
+          path = next((p for p in candidates if os.path.exists(p)), None)
+          if not path:
+              raise SystemExit("Could not find forecasts.csv; adjust candidates.")
+
+          out_dir = pathlib.Path("Dashboard/data")  # Capital D path
+          out_dir.mkdir(parents=True, exist_ok=True)
+
+          df = pd.read_csv(path, low_memory=False)
+          out_path = out_dir / "forecasts.parquet"
+          df.to_parquet(out_path, index=False)
+          print("✅ wrote", out_path, "rows:", len(df))
+          PY
+
+      - name: Commit updated parquet
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add -f Dashboard/data/forecasts.parquet
+          git commit -m "chore(dashboard): refresh parquet snapshot" || echo "No changes"
+          git push