[zeroconfig] Implement --autodetect flag (#2325)

## Summary

Closes DEV-151

(re) implements something similar to our old "planner" framework but
only for python/poetry. It's only opt-in.

Using the `--autodetect` with `init` flag generates a devbox.json with
any detection results.

Currently the framework only only uses the most relevant detector, but
could be modified to use multiple detectors.

## How was it tested?

`devbox init --autodetect --dry-run`

Author: mikeland73

devbox.json

@@ -1,43 +1,43 @@
 {
-  "name": "devbox",
+  "name":        "devbox",
   "description": "Instant, easy, and predictable development environments",
   "packages": {
-    "go": "latest",
+    "go":                          "latest",
     "runx:golangci/golangci-lint": "latest",
-    "runx:mvdan/gofumpt": "latest"
+    "runx:mvdan/gofumpt":          "latest",
   },
   "env": {
     "GOENV": "off",
-    "PATH": "$PATH:$PWD/dist"
+    "PATH":  "$PATH:$PWD/dist",
   },
   "shell": {
     "init_hook": [
       // Remove Go environment variables that might've been inherited from the
       // user's environment and could affect the build.
       "test -z $FISH_VERSION && \\",
       "unset       CGO_ENABLED GO111MODULE GOARCH GOFLAGS GOMOD GOOS GOROOT GOTOOLCHAIN GOWORK || \\",
-      "set --erase CGO_ENABLED GO111MODULE GOARCH GOFLAGS GOMOD GOOS GOROOT GOTOOLCHAIN GOWORK"
+      "set --erase CGO_ENABLED GO111MODULE GOARCH GOFLAGS GOMOD GOOS GOROOT GOTOOLCHAIN GOWORK",
     ],
     "scripts": {
       // Build devbox for the current platform
-      "build": "go build -o dist/devbox ./cmd/devbox",
+      "build":              "go build -o dist/devbox ./cmd/devbox",
       "build-darwin-amd64": "GOOS=darwin GOARCH=amd64 go build -o dist/devbox-darwin-amd64 ./cmd/devbox",
       "build-darwin-arm64": "GOOS=darwin GOARCH=arm64 go build -o dist/devbox-darwin-arm64 ./cmd/devbox",
-      "build-linux-amd64": "GOOS=linux GOARCH=amd64 go build -o dist/devbox-linux-amd64 ./cmd/devbox",
-      "build-linux-arm64": "GOOS=linux GOARCH=arm64 go build -o dist/devbox-linux-arm64 ./cmd/devbox",
+      "build-linux-amd64":  "GOOS=linux GOARCH=amd64 go build -o dist/devbox-linux-amd64 ./cmd/devbox",
+      "build-linux-arm64":  "GOOS=linux GOARCH=arm64 go build -o dist/devbox-linux-arm64 ./cmd/devbox",
       "build-all": [
         "devbox run build-darwin-amd64",
         "devbox run build-darwin-arm64",
         "devbox run build-linux-amd64",
-        "devbox run build-linux-arm64"
+        "devbox run build-linux-arm64",
       ],
       // Open VSCode
-      "code": "code .",
-      "lint": "golangci-lint run --timeout 5m && scripts/gofumpt.sh",
-      "fmt": "scripts/gofumpt.sh",
-      "test": "go test -race -cover ./...",
+      "code":               "code .",
+      "lint":               "golangci-lint run --timeout 5m && scripts/gofumpt.sh",
+      "fmt":                "scripts/gofumpt.sh",
+      "test":               "go test -race -cover ./...",
       "test-projects-only": "DEVBOX_RUN_PROJECT_TESTS=1 go test -v -timeout ${DEVBOX_GOLANG_TEST_TIMEOUT:-30m} ./... -run \"TestExamples|TestScriptsWithProjects\"",
-      "update-examples": "devbox run build && go run testscripts/testrunner/updater/main.go",
+      "update-examples":    "devbox run build && go run testscripts/testrunner/updater/main.go",
       // Updates the Flake's vendorHash: First run `go mod vendor` to vendor
       // the dependencies, then hash the vendor directory with Nix.
       // The hash is saved to the `vendor-hash` file, which is then
@@ -68,8 +68,8 @@
         "GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go test -c -o testscripts-linux-amd64",
         "GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go test -c -o testscripts-linux-arm64",
         "image=$(docker build --quiet --tag devbox-testscripts-ubuntu:noble --platform linux/amd64 .)",
-        "docker run --rm --mount type=volume,src=devbox-testscripts-amd64,dst=/nix --platform linux/amd64 -e DEVBOX_RUN_FAILING_TESTS -e DEVBOX_RUN_PROJECT_TESTS -e DEVBOX_DEBUG $image \"$@\""
-      ]
-    }
-  }
+        "docker run --rm --mount type=volume,src=devbox-testscripts-amd64,dst=/nix --platform linux/amd64 -e DEVBOX_RUN_FAILING_TESTS -e DEVBOX_RUN_PROJECT_TESTS -e DEVBOX_DEBUG $image \"$@\"",
+      ],
+    },
+  },
 }

internal/autodetect/autodetect.go

@@ -0,0 +1,75 @@
+package autodetect
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"go.jetpack.io/devbox/internal/autodetect/detector"
+	"go.jetpack.io/devbox/internal/devbox"
+	"go.jetpack.io/devbox/internal/devbox/devopt"
+)
+
+func PopulateConfig(ctx context.Context, path string, stderr io.Writer) error {
+	pkgs, err := packages(ctx, path)
+	if err != nil {
+		return err
+	}
+	devbox, err := devbox.Open(&devopt.Opts{
+		Dir:    path,
+		Stderr: stderr,
+	})
+	if err != nil {
+		return err
+	}
+	return devbox.Add(ctx, pkgs, devopt.AddOpts{})
+}
+
+func DryRun(ctx context.Context, path string, stderr io.Writer) error {
+	pkgs, err := packages(ctx, path)
+	if err != nil {
+		return err
+	} else if len(pkgs) == 0 {
+		fmt.Fprintln(stderr, "No packages to add")
+		return nil
+	}
+	fmt.Fprintln(stderr, "Packages to add:")
+	for _, pkg := range pkgs {
+		fmt.Fprintln(stderr, pkg)
+	}
+	return nil
+}
+
+func detectors(path string) []detector.Detector {
+	return []detector.Detector{
+		&detector.PythonDetector{Root: path},
+		&detector.PoetryDetector{Root: path},
+	}
+}
+
+func packages(ctx context.Context, path string) ([]string, error) {
+	mostRelevantDetector, err := relevantDetector(path)
+	if err != nil || mostRelevantDetector == nil {
+		return nil, err
+	}
+	return mostRelevantDetector.Packages(ctx)
+}
+
+// relevantDetector returns the most relevant detector for the given path.
+// We could modify this to return a list of detectors and their scores or
+// possibly grouped detectors by category (e.g. python, server, etc.)
+func relevantDetector(path string) (detector.Detector, error) {
+	relevantScore := 0.0
+	var mostRelevantDetector detector.Detector
+	for _, detector := range detectors(path) {
+		score, err := detector.Relevance(path)
+		if err != nil {
+			return nil, err
+		}
+		if score > relevantScore {
+			relevantScore = score
+			mostRelevantDetector = detector
+		}
+	}
+	return mostRelevantDetector, nil
+}

internal/autodetect/detector/detector.go

@@ -0,0 +1,8 @@
+package detector
+
+import "context"
+
+type Detector interface {
+	Relevance(path string) (float64, error)
+	Packages(ctx context.Context) ([]string, error)
+}

internal/autodetect/detector/poetry.go

@@ -0,0 +1,85 @@
+package detector
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/pelletier/go-toml/v2"
+	"go.jetpack.io/devbox/internal/searcher"
+)
+
+type PoetryDetector struct {
+	PythonDetector
+	Root string
+}
+
+var _ Detector = &PoetryDetector{}
+
+func (d *PoetryDetector) Relevance(path string) (float64, error) {
+	pyprojectPath := filepath.Join(d.Root, "pyproject.toml")
+	_, err := os.Stat(pyprojectPath)
+	if err == nil {
+		return d.maxRelevance(), nil
+	}
+	if os.IsNotExist(err) {
+		return 0, nil
+	}
+	return 0, err
+}
+
+func (d *PoetryDetector) Packages(ctx context.Context) ([]string, error) {
+	pyprojectPath := filepath.Join(d.Root, "pyproject.toml")
+	pyproject, err := os.ReadFile(pyprojectPath)
+	if err != nil {
+		return nil, err
+	}
+
+	var pyprojectToml struct {
+		Tool struct {
+			Poetry struct {
+				Version      string `toml:"version"`
+				Dependencies struct {
+					Python string `toml:"python"`
+				} `toml:"dependencies"`
+			} `toml:"poetry"`
+		} `toml:"tool"`
+	}
+	err = toml.Unmarshal(pyproject, &pyprojectToml)
+	if err != nil {
+		return nil, err
+	}
+
+	poetryVersion := determineBestVersion(ctx, "poetry", pyprojectToml.Tool.Poetry.Version)
+	pythonVersion := determineBestVersion(ctx, "python", pyprojectToml.Tool.Poetry.Dependencies.Python)
+
+	return []string{"python@" + pythonVersion, "poetry@" + poetryVersion}, nil
+}
+
+func determineBestVersion(ctx context.Context, pkg, version string) string {
+	if version == "" {
+		return "latest"
+	}
+
+	version = sanitizeVersion(version)
+
+	_, err := searcher.Client().ResolveV2(ctx, pkg, version)
+	if err != nil {
+		return "latest"
+	}
+
+	return version
+}
+
+func sanitizeVersion(version string) string {
+	// Remove non-numeric characters and 'v' prefix
+	sanitized := strings.TrimPrefix(version, "v")
+	return regexp.MustCompile(`[^\d.]`).ReplaceAllString(sanitized, "")
+}
+
+func (d *PoetryDetector) maxRelevance() float64 {
+	// this is arbitrary, but we want to prioritize poetry over python
+	return d.PythonDetector.maxRelevance() + 1
+}

internal/autodetect/detector/python.go

@@ -0,0 +1,33 @@
+package detector
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+)
+
+type PythonDetector struct {
+	Root string
+}
+
+var _ Detector = &PythonDetector{}
+
+func (d *PythonDetector) Relevance(path string) (float64, error) {
+	requirementsPath := filepath.Join(d.Root, "requirements.txt")
+	_, err := os.Stat(requirementsPath)
+	if err == nil {
+		return d.maxRelevance(), nil
+	}
+	if os.IsNotExist(err) {
+		return 0, nil
+	}
+	return 0, err
+}
+
+func (d *PythonDetector) Packages(ctx context.Context) ([]string, error) {
+	return []string{"python@latest"}, nil
+}
+
+func (d *PythonDetector) maxRelevance() float64 {
+	return 1.0
+}

internal/boxcli/global.go

@@ -63,7 +63,7 @@ func ensureGlobalConfig() (string, error) {
 	if err != nil {
 		return "", err
 	}
-	_, err = devbox.InitConfig(globalConfigPath)
+	err = devbox.InitConfig(globalConfigPath)
 	if err != nil {
 		return "", err
 	}

internal/boxcli/init.go

@@ -7,10 +7,17 @@ import (
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 
+	"go.jetpack.io/devbox/internal/autodetect"
 	"go.jetpack.io/devbox/internal/devbox"
 )
 
+type initFlags struct {
+	auto   bool
+	dryRun bool
+}
+
 func initCmd() *cobra.Command {
+	flags := &initFlags{}
 	command := &cobra.Command{
 		Use:   "init [<dir>]",
 		Short: "Initialize a directory as a devbox project",
@@ -19,16 +26,33 @@ func initCmd() *cobra.Command {
 			"You can then add packages using `devbox add`",
 		Args: cobra.MaximumNArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return runInitCmd(args)
+			return runInitCmd(cmd, args, flags)
 		},
 	}
 
+	command.Flags().BoolVar(&flags.auto, "auto", false, "Automatically detect packages to add")
+	command.Flags().BoolVar(&flags.dryRun, "dry-run", false, "Dry run")
+	_ = command.Flags().MarkHidden("auto")
+	_ = command.Flags().MarkHidden("dry-run")
+
 	return command
 }
 
-func runInitCmd(args []string) error {
+func runInitCmd(cmd *cobra.Command, args []string, flags *initFlags) error {
 	path := pathArg(args)
 
-	_, err := devbox.InitConfig(path)
+	if flags.auto && flags.dryRun {
+		return autodetect.DryRun(cmd.Context(), path, cmd.ErrOrStderr())
+	}
+
+	err := devbox.InitConfig(path)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	if flags.auto {
+		err = autodetect.PopulateConfig(cmd.Context(), path, cmd.ErrOrStderr())
+	}
+
 	return errors.WithStack(err)
 }

internal/devbox/devbox.go

@@ -71,7 +71,7 @@ type Devbox struct {
 
 var legacyPackagesWarningHasBeenShown = false
 
-func InitConfig(dir string) (bool, error) {
+func InitConfig(dir string) error {
 	return devconfig.Init(dir)
 }
 

internal/devbox/devbox_test.go

@@ -122,7 +122,7 @@ func TestComputeDevboxPathWhenRemoving(t *testing.T) {
 
 func devboxForTesting(t *testing.T) *Devbox {
 	path := t.TempDir()
-	_, err := devconfig.Init(path)
+	err := devconfig.Init(path)
 	require.NoError(t, err, "InitConfig should not fail")
 	d, err := Open(&devopt.Opts{
 		Dir:    path,

internal/devbox/util.go

@@ -55,7 +55,7 @@ func ensureDevboxUtilityConfig() (string, error) {
 		return "", err
 	}
 
-	_, err = InitConfig(path)
+	err = InitConfig(path)
 	if err != nil {
 		return "", err
 	}