Add GitHub workflow to auto-build SWE-Bench images and push to GHCR

.github/workflows/build-swe-bench-images.yml

@@ -0,0 +1,216 @@
+name: Build SWE-Bench Images
+
+on:
+  pull_request:  # for debugging
+  workflow_dispatch:
+    inputs:
+      dataset:
+        description: 'Dataset name (e.g., princeton-nlp/SWE-bench_Verified)'
+        required: true
+        default: 'princeton-nlp/SWE-bench_Verified'
+        type: string
+      split:
+        description: 'Dataset split (e.g., test, dev)'
+        required: true
+        default: 'test'
+        type: string
+      max-workers:
+        description: 'Number of concurrent builds'
+        required: false
+        default: '2'
+        type: string
+      n-limit:
+        description: 'Limit number of images to build (for testing). Leave blank for no limit.'
+        required: false
+        default: '10'
+        type: string
+
+# Reasonable defaults for automatic (push) runs; workflow_dispatch can override these.
+env:
+  DATASET: princeton-nlp/SWE-bench_Verified
+  SPLIT: test
+  MAX_WORKERS: '32'
+  N_LIMIT: '500'
+
+concurrency:
+  group: build-swe-bench-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  build-and-push:
+    runs-on:
+      labels: blacksmith-32vcpu-ubuntu-2204
+
+    # Allow pushing to GHCR and commenting on issues
+    permissions:
+      contents: read
+      packages: write
+      issues: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      # If this was a manual dispatch, override defaults with provided inputs.
+      - name: Apply workflow_dispatch overrides (if any)
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        run: |
+          if [ -n "${{ inputs.dataset }}" ]; then echo "DATASET=${{ inputs.dataset }}" >> "$GITHUB_ENV"; fi
+          if [ -n "${{ inputs.split }}" ]; then echo "SPLIT=${{ inputs.split }}" >> "$GITHUB_ENV"; fi
+          if [ -n "${{ inputs.max-workers }}" ]; then echo "MAX_WORKERS=${{ inputs.max-workers }}" >> "$GITHUB_ENV"; fi
+          # Empty string means "no limit"
+          if [ -n "${{ inputs.n-limit }}" ]; then echo "N_LIMIT=${{ inputs.n-limit }}" >> "$GITHUB_ENV"; else echo "N_LIMIT=" >> "$GITHUB_ENV"; fi
+
+      - name: Set up Docker Buildx with Blacksmith
+        uses: useblacksmith/setup-docker-builder@v1
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
+      - name: Install dependencies
+        run: |
+          make build
+
+      - name: Build and push SWE-Bench images
+        run: |
+          set -euo pipefail
+
+          CMD="uv run benchmarks/swe_bench/build_images.py \
+            --dataset '${DATASET}' \
+            --split '${SPLIT}' \
+            --image ghcr.io/openhands/eval-agent-server \
+            --push \
+            --max-workers '${MAX_WORKERS}'"
+
+          # Only include --n-limit if provided (non-empty)
+          if [ -n "${N_LIMIT}" ]; then
+            CMD="$CMD --n-limit '${N_LIMIT}'"
+          fi
+
+          echo "Running: $CMD"
+          eval "$CMD"
+        env:
+          DOCKER_BUILDKIT: 1
+          BUILDKIT_PROGRESS: plain
+
+      - name: Upload build manifest
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: build-manifest-${{ github.run_id }}
+          path: |
+            builds/**/manifest.jsonl
+            builds/**/summary.json
+          retention-days: 30
+
+      - name: Archive build logs
+        if: always()
+        run: |
+          if [ -d builds ]; then
+            # Create tar archive to avoid filename restrictions (colons, etc.)
+            tar -czf build-logs.tar.gz builds/
+            echo "Build logs archived successfully"
+          else
+            echo "No builds directory found"
+          fi
+
+      - name: Upload build logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: build-logs-${{ github.run_id }}
+          path: build-logs.tar.gz
+          retention-days: 7
+          if-no-files-found: warn
+
+      - name: Display build summary
+        if: always()
+        run: |
+          if ls builds/*/summary.json >/dev/null 2>&1; then
+            echo "## Build Summary" >> "$GITHUB_STEP_SUMMARY"
+            cat builds/*/summary.json | python -m json.tool >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Comment on tracker issue
+        if: success()
+        run: |
+          # Get SDK version from submodule
+          SDK_SHA=$(git submodule status vendor/software-agent-sdk | awk '{print $1}' | sed 's/^[+-]//')
+
+          # Find all manifest.jsonl files
+          MANIFEST_FILES=$(find builds -name "manifest.jsonl" -type f 2>/dev/null)
+
+          if [ -z "$MANIFEST_FILES" ]; then
+            echo "No manifest.jsonl files found in builds directory"
+            echo "Build may have completed but produced no images"
+            exit 0
+          fi
+
+          # Count total images built
+          TOTAL_IMAGES=$(cat $MANIFEST_FILES 2>/dev/null | wc -l)
+
+          if [ "$TOTAL_IMAGES" -eq 0 ]; then
+            echo "No images found in manifest files"
+            echo "Skipping comment as there are no built images to report"
+            exit 0
+          fi
+
+          # Extract all tags and format them as a markdown list (one tag per image)
+          TAGS=$(cat $MANIFEST_FILES | python -c "
+          import sys
+          import json
+          for line in sys.stdin:
+              data = json.loads(line.strip())
+              if data.get('tags') and len(data['tags']) > 0:
+                  # Only show the first tag per image to reduce clutter
+                  print(f'- \`{data[\"tags\"][0]}\`')
+          ")
+
+          # Determine how the workflow was triggered
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            TRIGGER="Manual trigger (workflow_dispatch)"
+          elif [ "${{ github.event_name }}" = "pull_request" ]; then
+            TRIGGER="Pull request [#${{ github.event.pull_request.number }}](${{ github.event.pull_request.html_url }})"
+          else
+            TRIGGER="${{ github.event_name }}"
+          fi
+
+          # Create the comment body
+          COMMENT_BODY=$(cat <<EOF
+          ## Build Complete ✅
+
+          **Dataset:** \`${DATASET}\`
+          **Split:** \`${SPLIT}\`
+          **SDK Version:** [\`${SDK_SHA:0:7}\`](https://github.com/OpenHands/software-agent-sdk/commit/${SDK_SHA})
+          **Workflow Run:** [#${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
+          **Triggered by:** ${TRIGGER}
+
+          <details>
+          <summary>Built Tags (${TOTAL_IMAGES} images)</summary>
+
+          ${TAGS}
+
+          </details>
+          EOF
+          )
+
+          # Post comment to issue #81
+          curl -L -X POST \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            "${{ github.api_url }}/repos/${{ github.repository }}/issues/81/comments" \
+            -d "$(jq -n --arg body "$COMMENT_BODY" '{body: $body}')"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -205,7 +205,7 @@ cython_debug/
 workspace/
 
 # IDE and editor directories
-.openhands/
+.openhands/*
 !.openhands/setup.sh
 !.openhands/microagents/
 .vscode/
@@ -215,5 +215,5 @@ workspace/
 !.llm_config/example.json
 
 # Evaluation outputs
-./eval_outputs
-./builds
+eval_outputs/
+builds/

.openhands/microagents/repo.md

@@ -84,7 +84,7 @@ make build  # Rebuild environment
 5. Update README.md with usage instructions
 
 # LLM Configuration
-LLM configs use JSON matching the [LLM class schema](https://github.com/All-Hands-AI/agent-sdk/blob/main/openhands/sdk/llm/llm.py#L93):
+LLM configs use JSON matching the [LLM class schema](https://github.com/OpenHands/software-agent-sdk/blob/main/openhands/sdk/llm/llm.py#L93):
 ```json
 {
   "model": "litellm_proxy/anthropic/claude-sonnet-4-20250514",