From 093ca96e0ca3cdafceccb8be03719f5cfe608a16 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 10 Sep 2025 14:23:34 -0700 Subject: [PATCH 01/11] [bfops/unreal-ci]: comment out git ref due to issues --- .github/workflows/ci.yml | 34 ++++++++++++++++++++++++++++++++++ restart-test.sh | 15 +++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 restart-test.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84576981d69..46d52d5b4f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -243,6 +243,40 @@ jobs: cargo run --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update -- self-install --root-dir="${ROOT_DIR}" --yes "${ROOT_DIR}"/spacetime --root-dir="${ROOT_DIR}" help + unreal_engine_tests: + name: Unreal Engine Tests + runs-on: spacetimedb-runner + container: + image: ghcr.io/epicgames/unreal-engine:dev-5.6 + options: --user 0:0 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GHCR_TOKEN }} + steps: +# - name: Find Git ref +# env: +# GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# shell: bash +# run: | +# PR_NUMBER="${{ github.event.inputs.pr_number || null }}" +# if test -n "${PR_NUMBER}"; then +# GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" +# else +# GIT_REF="${{ github.ref }}" +# fi +# echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" + - name: Checkout sources + uses: actions/checkout@v4 + with: + ref: ${{ env.GIT_REF }} + - uses: dsherret/rust-toolchain-file@v1 + - name: Run Unreal Engine tests + working-directory: sdks/unreal + env: + UE_ROOT_PATH: /home/ue4/UnrealEngine/Engine/Binaries/Linux/ + run: | + cargo test + cli_docs: name: Check CLI docs permissions: read-all diff --git a/restart-test.sh b/restart-test.sh new file mode 100644 index 00000000000..3aa3dca78e6 --- /dev/null +++ b/restart-test.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -euo pipefail + +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null debug@bitcraft-test-spacetimedb-2 sudo docker restart spacetimedb & +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null debug@bitcraft-test-spacetimedb-3 sudo docker restart spacetimedb & +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null debug@bitcraft-test-spacetimedb-4 sudo docker restart spacetimedb & +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null debug@bitcraft-test-spacetimedb-5 sudo docker restart spacetimedb & +# ssh debug@bitcraft-test-spacetimedb-6 docker restart spacetimedb & +# ssh debug@bitcraft-test-spacetimedb-7 docker restart spacetimedb & +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null debug@bitcraft-test-spacetimedb-controller sudo docker restart spacetimedb & + +wait + +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null debug@bitcraft-test-tools-1 sudo docker restart bitcraft-relay-server From e2381f777fae4f16af6a08fb4467a578243a6f3d Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 10 Sep 2025 14:45:15 -0700 Subject: [PATCH 02/11] [bfops/unreal-ci]: debug step --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46d52d5b4f4..fad816da6ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -265,6 +265,9 @@ jobs: # GIT_REF="${{ github.ref }}" # fi # echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" + - name: Find unreal binary + run: | + find / -name '*Unreal*' - name: Checkout sources uses: actions/checkout@v4 with: From ef4c610126945fc6f928ef94b88d3c0ed4729092 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 10 Sep 2025 15:07:51 -0700 Subject: [PATCH 03/11] [bfops/unreal-ci]: debug changes --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fad816da6ee..626120badd3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -267,7 +267,10 @@ jobs: # echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - name: Find unreal binary run: | - find / -name '*Unreal*' + find /home/ue4 -name 'Engine' + find /home/ue4 -name 'Build' + find /home/ue4 -name 'BatchFiles' + find /home/ue4 -name 'Build.sh' - name: Checkout sources uses: actions/checkout@v4 with: From 016c9d8c9652aa530f8fdd223c6e61fcd7df56d2 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 10 Sep 2025 15:08:13 -0700 Subject: [PATCH 04/11] [bfops/unreal-ci]: debug changes --- .github/workflows/ci.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 626120badd3..c274de0180d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -271,17 +271,17 @@ jobs: find /home/ue4 -name 'Build' find /home/ue4 -name 'BatchFiles' find /home/ue4 -name 'Build.sh' - - name: Checkout sources - uses: actions/checkout@v4 - with: - ref: ${{ env.GIT_REF }} - - uses: dsherret/rust-toolchain-file@v1 - - name: Run Unreal Engine tests - working-directory: sdks/unreal - env: - UE_ROOT_PATH: /home/ue4/UnrealEngine/Engine/Binaries/Linux/ - run: | - cargo test +# - name: Checkout sources +# uses: actions/checkout@v4 +# with: +# ref: ${{ env.GIT_REF }} +# - uses: dsherret/rust-toolchain-file@v1 +# - name: Run Unreal Engine tests +# working-directory: sdks/unreal +# env: +# UE_ROOT_PATH: /home/ue4/UnrealEngine/Engine/Binaries/Linux/ +# run: | +# cargo test cli_docs: name: Check CLI docs From 5399cee44b6212640b35f7dfdc6d069acce5d436 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 10 Sep 2025 15:18:27 -0700 Subject: [PATCH 05/11] [bfops/unreal-ci]: fix CI --- .github/workflows/ci.yml | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c274de0180d..c196243a334 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -265,23 +265,17 @@ jobs: # GIT_REF="${{ github.ref }}" # fi # echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - - name: Find unreal binary + - name: Checkout sources + uses: actions/checkout@v4 + with: + ref: ${{ env.GIT_REF }} + - uses: dsherret/rust-toolchain-file@v1 + - name: Run Unreal Engine tests + working-directory: sdks/unreal + env: + UE_ROOT_PATH: /home/ue4/UnrealEngine run: | - find /home/ue4 -name 'Engine' - find /home/ue4 -name 'Build' - find /home/ue4 -name 'BatchFiles' - find /home/ue4 -name 'Build.sh' -# - name: Checkout sources -# uses: actions/checkout@v4 -# with: -# ref: ${{ env.GIT_REF }} -# - uses: dsherret/rust-toolchain-file@v1 -# - name: Run Unreal Engine tests -# working-directory: sdks/unreal -# env: -# UE_ROOT_PATH: /home/ue4/UnrealEngine/Engine/Binaries/Linux/ -# run: | -# cargo test + cargo test cli_docs: name: Check CLI docs From cae5a17fb21b10a8771d9e4120649c3ab093e278 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 10 Sep 2025 18:47:57 -0700 Subject: [PATCH 06/11] [bfops/unreal-ci]: try a non-root approach --- .github/workflows/ci.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c196243a334..d1a5f3aba7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -248,11 +248,20 @@ jobs: runs-on: spacetimedb-runner container: image: ghcr.io/epicgames/unreal-engine:dev-5.6 - options: --user 0:0 credentials: username: ${{ github.actor }} password: ${{ secrets.GHCR_TOKEN }} + env: + # We override this because the default directories are not writeable by the default user inside the container. + # An alternative would be to run as the root user (by settings `options: --user 0:0`) but then Unreal itself gets + # worried. + RUNNER_TEMP: /tmp + ACTIONS_RUNNER_TEMP: /tmp steps: + - name: Prepare runner temp + run: | + mkdir -p /tmp/_runner_file_commands + chmod -R a+rwx /tmp # - name: Find Git ref # env: # GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 7cf9a130bba267c38b0678b5b6bbf130c658dc4a Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 10 Sep 2025 19:09:00 -0700 Subject: [PATCH 07/11] [bfops/unreal-ci]: try a non-root approach --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1a5f3aba7a..0fc79e6bef9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -255,13 +255,13 @@ jobs: # We override this because the default directories are not writeable by the default user inside the container. # An alternative would be to run as the root user (by settings `options: --user 0:0`) but then Unreal itself gets # worried. - RUNNER_TEMP: /tmp - ACTIONS_RUNNER_TEMP: /tmp + RUNNER_TEMP: /tmp/actions_tmp/${{ github.run_id }} + ACTIONS_RUNNER_TEMP: /tmp/actions_tmp/${{ github.run_id }} steps: - name: Prepare runner temp run: | - mkdir -p /tmp/_runner_file_commands - chmod -R a+rwx /tmp + set -eux + mkdir -p "$RUNNER_TEMP/_runner_file_commands" # - name: Find Git ref # env: # GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From a7c5058b00166394c06d128604a488ed79b67332 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 10 Sep 2025 19:52:37 -0700 Subject: [PATCH 08/11] [bfops/unreal-ci]: try a non-root approach --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0fc79e6bef9..4ec9039478d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -258,10 +258,15 @@ jobs: RUNNER_TEMP: /tmp/actions_tmp/${{ github.run_id }} ACTIONS_RUNNER_TEMP: /tmp/actions_tmp/${{ github.run_id }} steps: - - name: Prepare runner temp + - name: Prepare runner temp in a writeable dir run: | set -eux mkdir -p "$RUNNER_TEMP/_runner_file_commands" + # Point the core file-command files away from /__w/_temp to our writable dir + echo "GITHUB_STATE=$RUNNER_TEMP/_runner_file_commands/save_state" >> "$GITHUB_ENV" + echo "GITHUB_OUTPUT=$RUNNER_TEMP/_runner_file_commands/output" >> "$GITHUB_ENV" + echo "GITHUB_ENV=$RUNNER_TEMP/_runner_file_commands/env" >> "$GITHUB_ENV" + echo "GITHUB_STEP_SUMMARY=$RUNNER_TEMP/_runner_file_commands/summary" >> "$GITHUB_ENV" # - name: Find Git ref # env: # GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 7f18d18f7a64242f2e16eb21d41d510865829748 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 10 Sep 2025 20:05:41 -0700 Subject: [PATCH 09/11] [bfops/unreal-ci]: run as root --- .github/workflows/ci.yml | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ec9039478d..90a8dc14779 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -251,22 +251,12 @@ jobs: credentials: username: ${{ github.actor }} password: ${{ secrets.GHCR_TOKEN }} - env: - # We override this because the default directories are not writeable by the default user inside the container. - # An alternative would be to run as the root user (by settings `options: --user 0:0`) but then Unreal itself gets - # worried. - RUNNER_TEMP: /tmp/actions_tmp/${{ github.run_id }} - ACTIONS_RUNNER_TEMP: /tmp/actions_tmp/${{ github.run_id }} + # Run as root because otherwise we get permission denied for various directories inside the container. I tried doing dances to allow it to run + # without this (reassigning env vars and stuff), but was unable to get it to work and it felt like an uphill battle. + options: --user 0:0 steps: - - name: Prepare runner temp in a writeable dir - run: | - set -eux - mkdir -p "$RUNNER_TEMP/_runner_file_commands" - # Point the core file-command files away from /__w/_temp to our writable dir - echo "GITHUB_STATE=$RUNNER_TEMP/_runner_file_commands/save_state" >> "$GITHUB_ENV" - echo "GITHUB_OUTPUT=$RUNNER_TEMP/_runner_file_commands/output" >> "$GITHUB_ENV" - echo "GITHUB_ENV=$RUNNER_TEMP/_runner_file_commands/env" >> "$GITHUB_ENV" - echo "GITHUB_STEP_SUMMARY=$RUNNER_TEMP/_runner_file_commands/summary" >> "$GITHUB_ENV" +# Uncomment this before merging so that it will run properly if run manually through the GH actions flow. It was playing weird with rolled back +# commits though. # - name: Find Git ref # env: # GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 4fbd6d8e270ca8a79202a97e720e25fbe6dd7708 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 10 Sep 2025 21:32:35 -0700 Subject: [PATCH 10/11] [bfops/unreal-ci]: comments --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90a8dc14779..48e489a4972 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -245,10 +245,16 @@ jobs: unreal_engine_tests: name: Unreal Engine Tests + # This can't go on e.g. ubuntu-latest because that runner runs out of disk space. ChatGPT suggested that the general solution tends to be to use + # a custom runner. runs-on: spacetimedb-runner container: image: ghcr.io/epicgames/unreal-engine:dev-5.6 credentials: + # Note(bfops): I don't think that `github.actor` needs to match the user that the token is for, because I'm using a token for my account and + # it seems to be totally happy. + # However, the token needs to be for a user that has access to the EpicGames org (see + # https://dev.epicgames.com/documentation/en-us/unreal-engine/downloading-source-code-in-unreal-engine?application_version=5.6) username: ${{ github.actor }} password: ${{ secrets.GHCR_TOKEN }} # Run as root because otherwise we get permission denied for various directories inside the container. I tried doing dances to allow it to run From e785d71f8c69adaf138c07213c82d7df244c68cf Mon Sep 17 00:00:00 2001 From: John Detter <4099508+jdetter@users.noreply.github.com> Date: Wed, 17 Sep 2025 17:57:17 -0500 Subject: [PATCH 11/11] Workflow should be working --- .github/workflows/ci.yml | 42 +++++++++++++++++++++++++++++++++++++++- restart-test.sh | 15 -------------- 2 files changed, 41 insertions(+), 16 deletions(-) delete mode 100644 restart-test.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 48e489a4972..825d27e2981 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -285,7 +285,47 @@ jobs: env: UE_ROOT_PATH: /home/ue4/UnrealEngine run: | - cargo test + + apt-get update + apt-get install -y acl curl ca-certificates + + REPO="$GITHUB_WORKSPACE" + # Let ue4 read/write the workspace & tool caches without changing ownership + for p in "$REPO" "${RUNNER_TEMP:-/__t}" "${RUNNER_TOOL_CACHE:-/__t}"; do + [ -d "$p" ] && setfacl -R -m u:ue4:rwX -m d:u:ue4:rwX "$p" || true + done + + # Rust tool caches live under the runner tool cache so they persist + export CARGO_HOME="${RUNNER_TOOL_CACHE:-/__t}/cargo" + export RUSTUP_HOME="${RUNNER_TOOL_CACHE:-/__t}/rustup" + mkdir -p "$CARGO_HOME" "$RUSTUP_HOME" + chown -R ue4:ue4 "$CARGO_HOME" "$RUSTUP_HOME" + + # Make sure the UE build script is executable (and parents traversable) + UE_DIR="${UE_ROOT_PATH:-/home/ue4/UnrealEngine}" + chmod a+rx "$UE_DIR" "$UE_DIR/Engine" "$UE_DIR/Engine/Build" "$UE_DIR/Engine/Build/BatchFiles/Linux" || true + chmod a+rx "$UE_DIR/Engine/Build/BatchFiles/Linux/Build.sh" || true + + # Run the build & tests as ue4 (who owns the UE tree) + sudo -E -H -u ue4 env \ + HOME=/home/ue4 \ + XDG_CONFIG_HOME=/home/ue4/.config \ + CARGO_HOME="$CARGO_HOME" \ + RUSTUP_HOME="$RUSTUP_HOME" \ + PATH="$CARGO_HOME/bin:$PATH" \ + bash -lc ' + set -euxo pipefail + # Install rustup for ue4 if needed (uses the shared caches) + if ! command -v cargo >/dev/null 2>&1; then + curl -sSf https://sh.rustup.rs | sh -s -- -y + fi + rustup show >/dev/null + git config --global --add safe.directory "$GITHUB_WORKSPACE" || true + + cd "$GITHUB_WORKSPACE/sdks/unreal" + cargo --version + cargo test + ' cli_docs: name: Check CLI docs diff --git a/restart-test.sh b/restart-test.sh deleted file mode 100644 index 3aa3dca78e6..00000000000 --- a/restart-test.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null debug@bitcraft-test-spacetimedb-2 sudo docker restart spacetimedb & -ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null debug@bitcraft-test-spacetimedb-3 sudo docker restart spacetimedb & -ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null debug@bitcraft-test-spacetimedb-4 sudo docker restart spacetimedb & -ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null debug@bitcraft-test-spacetimedb-5 sudo docker restart spacetimedb & -# ssh debug@bitcraft-test-spacetimedb-6 docker restart spacetimedb & -# ssh debug@bitcraft-test-spacetimedb-7 docker restart spacetimedb & -ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null debug@bitcraft-test-spacetimedb-controller sudo docker restart spacetimedb & - -wait - -ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null debug@bitcraft-test-tools-1 sudo docker restart bitcraft-relay-server