Added the Unreal SDK work for codegen, testing, and the plugin (#3223)

# Description of Changes

Closes #3219 
This adds the Unreal SDK, the new Unreal test cases, updates the test
runner to handle Unreal, codegen updates for Unreal, and a QuickStart
Chat.

# API and ABI breaking changes

No breaking changes.

# Expected complexity level and risk

2 - This impacts the subcommand generate.rs to include unrealcpp and
crates/testing to expand for Unreal

# Testing

- [x] Run the new Unreal tests 
- [x] Run any previous automation testing - with all the changes to
generate/testing I'm uncertain if there is an impact
- [x] Review the new CLI generate documentation changes

---------

Co-authored-by: Phoebe Goldman <[email protected]>
Co-authored-by: Zeke Foppa <[email protected]>
Co-authored-by: Zeke Foppa <[email protected]>
Co-authored-by: John Detter <[email protected]>

Author: 4 people

.github/workflows/ci.yml

@@ -103,9 +103,10 @@ jobs:
         run: |
           sudo mkdir /stdb
           sudo chmod 777 /stdb
-
+      
       - name: Run cargo test
-        run: cargo test --all
+        #Note: Unreal tests will be run separately
+        run: cargo test --all -- --skip unreal
 
       - name: Check that the test outputs are up-to-date
         run: bash tools/check-diff.sh
@@ -243,6 +244,90 @@ jobs:
           cargo run --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update -- self-install --root-dir="${ROOT_DIR}" --yes
           "${ROOT_DIR}"/spacetime --root-dir="${ROOT_DIR}" help
 
+  unreal_engine_tests:
+    name: Unreal Engine Tests
+    # This can't go on e.g. ubuntu-latest because that runner runs out of disk space. ChatGPT suggested that the general solution tends to be to use
+    # a custom runner.
+    runs-on: spacetimedb-runner
+    container:
+      image: ghcr.io/epicgames/unreal-engine:dev-5.6
+      credentials:
+        # Note(bfops): I don't think that `github.actor` needs to match the user that the token is for, because I'm using a token for my account and
+        # it seems to be totally happy.
+        # However, the token needs to be for a user that has access to the EpicGames org (see
+        # https://dev.epicgames.com/documentation/en-us/unreal-engine/downloading-source-code-in-unreal-engine?application_version=5.6)
+        username: ${{ github.actor }}
+        password: ${{ secrets.GHCR_TOKEN }}
+      # Run as root because otherwise we get permission denied for various directories inside the container. I tried doing dances to allow it to run
+      # without this (reassigning env vars and stuff), but was unable to get it to work and it felt like an uphill battle.
+      options: --user 0:0
+    steps:
+# Uncomment this before merging so that it will run properly if run manually through the GH actions flow. It was playing weird with rolled back
+# commits though.
+#      - name: Find Git ref
+#        env:
+#          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+#        shell: bash
+#        run: |
+#          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
+#          if test -n "${PR_NUMBER}"; then
+#            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
+#          else
+#            GIT_REF="${{ github.ref }}"
+#          fi
+#          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.GIT_REF }}
+      - uses: dsherret/rust-toolchain-file@v1
+      - name: Run Unreal Engine tests
+        working-directory: sdks/unreal
+        env:
+          UE_ROOT_PATH: /home/ue4/UnrealEngine
+        run: |
+
+          apt-get update
+          apt-get install -y acl curl ca-certificates
+
+          REPO="$GITHUB_WORKSPACE"
+          # Let ue4 read/write the workspace & tool caches without changing ownership
+          for p in "$REPO" "${RUNNER_TEMP:-/__t}" "${RUNNER_TOOL_CACHE:-/__t}"; do
+            [ -d "$p" ] && setfacl -R -m u:ue4:rwX -m d:u:ue4:rwX "$p" || true
+          done
+
+          # Rust tool caches live under the runner tool cache so they persist
+          export CARGO_HOME="${RUNNER_TOOL_CACHE:-/__t}/cargo"
+          export RUSTUP_HOME="${RUNNER_TOOL_CACHE:-/__t}/rustup"
+          mkdir -p "$CARGO_HOME" "$RUSTUP_HOME"
+          chown -R ue4:ue4 "$CARGO_HOME" "$RUSTUP_HOME"
+
+          # Make sure the UE build script is executable (and parents traversable)
+          UE_DIR="${UE_ROOT_PATH:-/home/ue4/UnrealEngine}"
+          chmod a+rx "$UE_DIR" "$UE_DIR/Engine" "$UE_DIR/Engine/Build" "$UE_DIR/Engine/Build/BatchFiles/Linux" || true
+          chmod a+rx "$UE_DIR/Engine/Build/BatchFiles/Linux/Build.sh" || true
+
+          # Run the build & tests as ue4 (who owns the UE tree)
+          sudo -E -H -u ue4 env \
+            HOME=/home/ue4 \
+            XDG_CONFIG_HOME=/home/ue4/.config \
+            CARGO_HOME="$CARGO_HOME" \
+            RUSTUP_HOME="$RUSTUP_HOME" \
+            PATH="$CARGO_HOME/bin:$PATH" \
+            bash -lc '
+              set -euxo pipefail
+              # Install rustup for ue4 if needed (uses the shared caches)
+              if ! command -v cargo >/dev/null 2>&1; then
+                curl -sSf https://sh.rustup.rs | sh -s -- -y
+              fi
+              rustup show >/dev/null
+              git config --global --add safe.directory "$GITHUB_WORKSPACE" || true
+
+              cd "$GITHUB_WORKSPACE/sdks/unreal"
+              cargo --version
+              cargo test
+            '
+
   cli_docs:
     name: Check CLI docs
     permissions: read-all

Cargo.lock

@@ -26,6 +26,7 @@ members = [
   "crates/sats",
   "crates/schema",
   "sdks/rust",
+  "sdks/unreal",
   "crates/snapshot",
   "crates/sqltest",
   "crates/sql-parser",

Cargo.toml

@@ -5,7 +5,7 @@ use clap::parser::ValueSource;
 use clap::Arg;
 use clap::ArgAction::Set;
 use fs_err as fs;
-use spacetimedb_codegen::{generate, Csharp, Lang, OutputFile, Rust, TypeScript, AUTO_GENERATED_PREFIX};
+use spacetimedb_codegen::{generate, Csharp, Lang, OutputFile, Rust, TypeScript, UnrealCpp, AUTO_GENERATED_PREFIX};
 use spacetimedb_lib::de::serde::DeserializeWrapper;
 use spacetimedb_lib::{sats, RawModuleDef};
 use spacetimedb_schema;
@@ -25,7 +25,7 @@ use std::io::Read;
 pub fn cli() -> clap::Command {
     clap::Command::new("generate")
         .about("Generate client files for a spacetime module.")
-        .override_usage("spacetime generate --lang <LANG> --out-dir <DIR> [--project-path <DIR> | --bin-path <PATH>]")
+        .override_usage("spacetime generate --lang <LANG> --out-dir <DIR> [--project-path <DIR> | --bin-path <PATH> | --module-name <MODULE_NAME> | --uproject-dir <DIR>]")
         .arg(
             Arg::new("wasm_file")
                 .value_parser(clap::value_parser!(PathBuf))
@@ -57,17 +57,32 @@ pub fn cli() -> clap::Command {
         .arg(
             Arg::new("out_dir")
                 .value_parser(clap::value_parser!(PathBuf))
-                .required(true)
                 .long("out-dir")
                 .short('o')
-                .help("The system path (absolute or relative) to the generate output directory"),
+                .help("The system path (absolute or relative) to the generate output directory")
+                .required_if_eq("lang", "rust")
+                .required_if_eq("lang", "csharp")
+                .required_if_eq("lang", "typescript"),
+        )
+        .arg(
+            Arg::new("uproject_dir")
+                .value_parser(clap::value_parser!(PathBuf))
+                .long("uproject-dir")
+                .help("Path to the Unreal project directory, replaces --out-dir for Unreal generation (only used with --lang unrealcpp)")
+                .required_if_eq("lang", "unrealcpp")
         )
         .arg(
             Arg::new("namespace")
                 .default_value("SpacetimeDB.Types")
                 .long("namespace")
                 .help("The namespace that should be used"),
         )
+        .arg(
+            Arg::new("module_name")
+                .long("module-name")
+                .help("The module name that should be used for DLL export macros (required for lang unrealcpp)")
+                .required_if_eq("lang", "unrealcpp")
+        )
         .arg(
             Arg::new("lang")
                 .required(true)
@@ -86,6 +101,11 @@ pub fn cli() -> clap::Command {
         )
         .arg(common_args::yes())
         .after_help("Run `spacetime help publish` for more detailed information.")
+        .group(
+            clap::ArgGroup::new("output_dir")
+                .args(["out_dir", "uproject_dir"])
+                .required(true)
+        )
 }
 
 pub async fn exec(config: Config, args: &clap::ArgMatches) -> anyhow::Result<()> {
@@ -101,16 +121,21 @@ pub async fn exec_ex(
     let project_path = args.get_one::<PathBuf>("project_path").unwrap();
     let wasm_file = args.get_one::<PathBuf>("wasm_file").cloned();
     let json_module = args.get_many::<PathBuf>("json_module");
-    let out_dir = args.get_one::<PathBuf>("out_dir").unwrap();
     let lang = *args.get_one::<Language>("lang").unwrap();
     let namespace = args.get_one::<String>("namespace").unwrap();
+    let module_name = args.get_one::<String>("module_name");
     let force = args.get_flag("force");
     let build_options = args.get_one::<String>("build_options").unwrap();
 
     if args.value_source("namespace") == Some(ValueSource::CommandLine) && lang != Language::Csharp {
         return Err(anyhow::anyhow!("--namespace is only supported with --lang csharp"));
     }
 
+    let out_dir = args
+        .get_one::<PathBuf>("out_dir")
+        .or_else(|| args.get_one::<PathBuf>("uproject_dir"))
+        .unwrap();
+
     let module: ModuleDef = if let Some(mut json_module) = json_module {
         let DeserializeWrapper::<RawModuleDef>(module) = if let Some(path) = json_module.next() {
             serde_json::from_slice(&fs::read(path)?)?
@@ -136,11 +161,19 @@ pub async fn exec_ex(
     let mut paths = BTreeSet::new();
 
     let csharp_lang;
+    let unreal_cpp_lang;
     let gen_lang = match lang {
         Language::Csharp => {
             csharp_lang = Csharp { namespace };
             &csharp_lang as &dyn Lang
         }
+        Language::UnrealCpp => {
+            unreal_cpp_lang = UnrealCpp {
+                module_name: module_name.as_ref().unwrap(),
+                uproject_dir: out_dir,
+            };
+            &unreal_cpp_lang as &dyn Lang
+        }
         Language::Rust => &Rust,
         Language::TypeScript => &TypeScript,
     };
@@ -156,9 +189,15 @@ pub async fn exec_ex(
         paths.insert(path);
     }
 
+    // For Unreal, we want to clean up just the module directory, not the entire uproject directory tree.
+    let cleanup_root = match lang {
+        Language::UnrealCpp => out_dir.join("Source").join(module_name.as_ref().unwrap()),
+        _ => out_dir.clone(),
+    };
+
     // TODO: We should probably just delete all generated files before we generate any, rather than selectively deleting some afterward.
     let mut auto_generated_buf: [u8; AUTO_GENERATED_PREFIX.len()] = [0; AUTO_GENERATED_PREFIX.len()];
-    let files_to_delete = walkdir::WalkDir::new(out_dir)
+    let files_to_delete = walkdir::WalkDir::new(&cleanup_root)
         .into_iter()
         .map(|entry_result| {
             let entry = entry_result?;
@@ -213,17 +252,19 @@ pub enum Language {
     Csharp,
     TypeScript,
     Rust,
+    UnrealCpp,
 }
 
 impl clap::ValueEnum for Language {
     fn value_variants<'a>() -> &'a [Self] {
-        &[Self::Csharp, Self::TypeScript, Self::Rust]
+        &[Self::Csharp, Self::TypeScript, Self::Rust, Self::UnrealCpp]
     }
     fn to_possible_value(&self) -> Option<PossibleValue> {
         Some(match self {
             Self::Csharp => clap::builder::PossibleValue::new("csharp").aliases(["c#", "cs"]),
             Self::TypeScript => clap::builder::PossibleValue::new("typescript").aliases(["ts", "TS"]),
             Self::Rust => clap::builder::PossibleValue::new("rust").aliases(["rs", "RS"]),
+            Self::UnrealCpp => PossibleValue::new("unrealcpp").aliases(["uecpp", "ue5cpp", "unreal"]),
         })
     }
 }
@@ -236,6 +277,9 @@ impl Language {
             Language::TypeScript => {
                 // TODO: implement formatting.
             }
+            Language::UnrealCpp => {
+                // TODO: implement formatting.
+            }
         }
 
         Ok(())