Skip to content

Commit 0fdfc67

Browse files
authored
testscripts: better caching with Docker (#2245)
Mount a Docker volume to /nix when running `docker-testscripts` to speed up tests. Also set `XDG_CACHE_HOME=/nix/cache` so that the tarball and eval cache persist to the volume. Update `setupTestEnv` to share the pip cache between tests (the same way we do with Nix's cache). This cuts the duration of python linker tests in half (down from about 1 minute to about 30s).
1 parent 8c30551 commit 0fdfc67

File tree

6 files changed

+60
-57
lines changed

6 files changed

+60
-57
lines changed

devbox.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,8 @@
5757
"trap 'rm -f testscripts-linux-amd64 testscripts-linux-arm64' EXIT",
5858
"GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go test -c -o testscripts-linux-amd64",
5959
"GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go test -c -o testscripts-linux-arm64",
60-
"image=$(docker build --quiet --tag devbox-ubuntu:noble --platform linux/amd64 .)",
61-
"docker run --rm --platform linux/amd64 -e DEVBOX_RUN_FAILING_TESTS -e DEVBOX_RUN_PROJECT_TESTS -e DEVBOX_DEBUG $image \"$@\"",
60+
"image=$(docker build --quiet --tag devbox-testscripts-ubuntu:noble --platform linux/amd64 .)",
61+
"docker run --rm --mount type=volume,src=devbox-testscripts-amd64,dst=/nix --platform linux/amd64 -e DEVBOX_RUN_FAILING_TESTS -e DEVBOX_RUN_PROJECT_TESTS -e DEVBOX_DEBUG $image \"$@\"",
6262
],
6363
},
6464
},

testscripts/Dockerfile

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,19 @@ FROM $BASEIMAGE:$BASETAG
1919
ARG TARGETOS
2020
ARG TARGETARCH
2121

22+
ENV XDG_CACHE_HOME=/nix/cache
23+
2224
COPY --from=installer --link /nix-installer-$TARGETOS-$TARGETARCH nix-installer
2325
RUN <<EOF
2426
set -e
2527

28+
mkdir -p "$XDG_CACHE_HOME"
29+
2630
# Setting this env var is the same as passing --extra-conf to nix-installer.
2731
export NIX_INSTALLER_EXTRA_CONF="filter-syscalls = false
28-
sandbox = false"
32+
sandbox = false
33+
experimental-features = nix-command flakes fetch-closure ca-derivations
34+
use-xdg-base-directories = true"
2935

3036
./nix-installer install linux --no-confirm --logger full --init none
3137
rm nix-installer
@@ -35,6 +41,6 @@ ENV SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
3541
COPY --link */*.test.txt /devbox/testscripts/
3642
COPY --link testscripts-$TARGETOS-$TARGETARCH /devbox/testscripts/test
3743

38-
VOLUME /nix/store
44+
VOLUME "/nix"
3945
WORKDIR /devbox/testscripts
4046
ENTRYPOINT [ "/devbox/testscripts/test" ]

testscripts/languages/python_old_glibc.test.txt

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ except psycopg2.OperationalError:
3131
},
3232
"env": {
3333
"PIP_DISABLE_PIP_VERSION_CHECK": "1",
34-
"PIP_NO_CACHE_DIR": "1",
3534
"PIP_NO_INPUT": "1",
3635
"PIP_NO_PYTHON_VERSION_WARNING": "1",
3736
"PIP_PROGRESS_BAR": "off",

testscripts/testrunner/examplesrunner.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ func runSingleDevboxTestscript(t *testing.T, dir, projectDir string) {
9090
t.Error(err)
9191
}
9292

93-
params := getTestscriptParams(t, testscriptDir)
93+
params := getTestscriptParams(testscriptDir)
9494

9595
// save a reference to the original params.Setup so that we can wrap it below
9696
setup := params.Setup

testscripts/testrunner/setupenv.go

Lines changed: 46 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -13,68 +13,66 @@ import (
1313
"go.jetpack.io/devbox/internal/xdg"
1414
)
1515

16-
func setupTestEnv(t *testing.T, envs *testscript.Env) error {
17-
setupPATH(envs)
18-
19-
setupHome(t, envs)
20-
21-
err := setupCacheHome(envs)
22-
if err != nil {
23-
return err
24-
}
25-
26-
propagateEnvVars(envs,
16+
// setupTestEnv configures env for devbox tests.
17+
func setupTestEnv(env *testscript.Env) error {
18+
setupPATH(env)
19+
setupHome(env)
20+
setupCacheHome(env)
21+
propagateEnvVars(env,
2722
debug.DevboxDebug, // to enable extra logging
2823
"SSL_CERT_FILE", // so HTTPS works with Nix-installed certs
2924
)
3025
return nil
3126
}
3227

33-
func setupHome(t *testing.T, envs *testscript.Env) {
34-
// We set a HOME env-var because:
35-
// 1. testscripts overrides it to /no-home, presumably to improve isolation
36-
// 2. but many language tools rely on a $HOME being set, and break due to 1.
37-
// examples include ~/.dotnet folder and GOCACHE=$HOME/Library/Caches/go-build
38-
envs.Setenv(envir.Home, t.TempDir())
28+
// setupHome sets the test's HOME to a unique temp directory. The testscript
29+
// package sets it to /no-home by default (presumably to improve isolation), but
30+
// this breaks most programs.
31+
func setupHome(env *testscript.Env) {
32+
env.Setenv(envir.Home, env.T().(testing.TB).TempDir())
3933
}
4034

41-
func setupPATH(envs *testscript.Env) {
42-
// Ensure path is empty so that we rely only on the PATH set by devbox
43-
// itself.
44-
// The one entry we need to keep is the /bin directory in the testing directory.
45-
// That directory is setup by the testing framework itself, and it's what allows
46-
// us to call our own custom "devbox" command.
47-
oldPath := envs.Getenv(envir.Path)
48-
newPath := strings.Split(oldPath, ":")[0]
49-
envs.Setenv(envir.Path, newPath)
35+
// setupPATH removes all directories from the test's PATH to ensure that it only
36+
// uses the PATH set by devbox. The one exception is the testscript's bin
37+
// directory, which contains the commands given to testscript.RunMain
38+
// (such as devbox itself).
39+
func setupPATH(env *testscript.Env) {
40+
s, _, _ := strings.Cut(env.Getenv(envir.Path), string(filepath.ListSeparator))
41+
env.Setenv(envir.Path, s)
5042
}
5143

52-
func setupCacheHome(envs *testscript.Env) error {
53-
// Both devbox itself and nix occasionally create some files in
54-
// XDG_CACHE_HOME (which defaults to ~/.cache). For purposes of this
55-
// test set it to a location within the test's working directory:
56-
cacheHome := filepath.Join(envs.WorkDir, ".cache")
57-
envs.Setenv(envir.XDGCacheHome, cacheHome)
58-
err := os.MkdirAll(cacheHome, 0o755) // Ensure dir exists.
59-
if err != nil {
60-
return err
61-
}
44+
// setupCacheHome sets the test's XDG_CACHE_HOME to a unique temp directory so
45+
// that it doesn't share caches with other tests or the user's system. For
46+
// programs where this would make tests too slow, it symlinks specific cache
47+
// subdirectories to a shared location that persists between test runs. For
48+
// example, $WORK/.cache/nix would symlink to $XDG_CACHE_HOME/devbox-tests/nix
49+
// so that Nix doesn't re-download tarballs for every test.
50+
func setupCacheHome(env *testscript.Env) {
51+
t := env.T().(testing.TB) //nolint:varnamelen
6252

63-
// There is one directory we do want to share across tests: nix's cache.
64-
// Without it tests are very slow, and nix would end up re-downloading
65-
// nixpkgs every time.
66-
// Here we create a shared location for nix's cache, and symlink from
67-
// the test's working directory.
68-
err = os.MkdirAll(xdg.CacheSubpath("devbox-tests/nix"), 0o755) // Ensure dir exists.
53+
cacheHome := filepath.Join(env.WorkDir, ".cache")
54+
env.Setenv(envir.XDGCacheHome, cacheHome)
55+
err := os.MkdirAll(cacheHome, 0o755)
6956
if err != nil {
70-
return err
71-
}
72-
err = os.Symlink(xdg.CacheSubpath("devbox-tests/nix"), filepath.Join(cacheHome, "nix"))
73-
if err != nil {
74-
return err
57+
t.Fatal("create XDG_CACHE_HOME for test:", err)
7558
}
7659

77-
return nil
60+
// Symlink cache subdirectories that we want to share and persist
61+
// between tests.
62+
sharedCacheDir := xdg.CacheSubpath("devbox-tests")
63+
for _, subdir := range []string{"nix", "pip"} {
64+
sharedSubdir := filepath.Join(sharedCacheDir, subdir)
65+
err := os.MkdirAll(sharedSubdir, 0o755)
66+
if err != nil {
67+
t.Fatal("create shared XDG_CACHE_HOME subdir:", err)
68+
}
69+
70+
testSubdir := filepath.Join(cacheHome, subdir)
71+
err = os.Symlink(sharedSubdir, testSubdir)
72+
if err != nil {
73+
t.Fatal("symlink test's XDG_CACHE_HOME subdir to shared XDG_CACHE_HOME subdir:", err)
74+
}
75+
}
7876
}
7977

8078
// propagateEnvVars propagates the values of environment variables to the test

testscripts/testrunner/testrunner.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ func RunTestscripts(t *testing.T, testscriptsDir string) {
4343
continue
4444
}
4545

46-
testscript.Run(t, getTestscriptParams(t, dir))
46+
testscript.Run(t, getTestscriptParams(dir))
4747
}
4848
}
4949

@@ -80,12 +80,12 @@ func copyFileCmd(script *testscript.TestScript, neg bool, args []string) {
8080
script.Check(err)
8181
}
8282

83-
func getTestscriptParams(t *testing.T, dir string) testscript.Params {
83+
func getTestscriptParams(dir string) testscript.Params {
8484
return testscript.Params{
8585
Dir: dir,
8686
RequireExplicitExec: true,
8787
TestWork: false, // Set to true if you're trying to debug a test.
88-
Setup: func(env *testscript.Env) error { return setupTestEnv(t, env) },
88+
Setup: func(env *testscript.Env) error { return setupTestEnv(env) },
8989
Cmds: map[string]func(ts *testscript.TestScript, neg bool, args []string){
9090
"cp": copyFileCmd,
9191
"devboxjson.packages.contains": assertDevboxJSONPackagesContains,

0 commit comments

Comments
 (0)