Add pip to dbotconf (#307)

CharlieTLe · pellared · MrAlias · web-flow · commit 6fd2bd763e91 · 2023-05-22T07:23:48.000-07:00
* Add pip to dbotconf requirements.txt files will now be tracked with dependabot. This is related to open-telemetry/opentelemetry-go#3996. * Create struct for updates * Update dbotconf/internal/verify_test.go Co-authored-by: Robert Pająk <pellared@hotmail.com> * fix shadow declaration of "updates" * Generate change log * Update .chloggen/pip.yaml * Update dbotconf/internal/conf.go Co-authored-by: Tyler Yahn <MrAlias@users.noreply.github.com> * Update dbotconf/internal/testdata/dependabot.yml --------- Co-authored-by: Robert Pająk <pellared@hotmail.com> Co-authored-by: Tyler Yahn <MrAlias@users.noreply.github.com>
diff --git a/.chloggen/pip.yaml b/.chloggen/pip.yaml
@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. crosslink)
+component: dbotconf
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Add support for pip package ecosystem."
+
+# One or more tracking issues related to the change
+issues: [307]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
diff --git a/dbotconf/internal/conf.go b/dbotconf/internal/conf.go
@@ -19,13 +19,15 @@ const (
 	ghPkgEco     = "github-actions"
 	dockerPkgEco = "docker"
 	gomodPkgEco  = "gomod"
+	pipPkgEco    = "pip"
 )
 
 var (
 	weeklySchedule = schedule{Interval: "weekly", Day: "sunday"}
 	actionLabels   = []string{"dependencies", "actions", "Skip Changelog"}
 	dockerLabels   = []string{"dependencies", "docker", "Skip Changelog"}
 	goLabels       = []string{"dependencies", "go", "Skip Changelog"}
+	pipLabels      = []string{"dependencies", "python", "Skip Changelog"}
 )
 
 type dependabotConfig struct {
diff --git a/dbotconf/internal/generate.go b/dbotconf/internal/generate.go
@@ -30,7 +30,7 @@ const header = "# File generated by dbotconf; DO NOT EDIT."
 var buildConfigFunc = buildConfig
 
 // buildConfig constructs a dependabotConfig for all modules in the repo.
-func buildConfig(root string, mods []*modfile.File, dockerFiles []string) (*dependabotConfig, error) {
+func buildConfig(root string, mods []*modfile.File, dockerFiles []string, pipFiles []string) (*dependabotConfig, error) {
 	c := &dependabotConfig{
 		Version: version2,
 		Updates: []update{
@@ -70,6 +70,21 @@ func buildConfig(root string, mods []*modfile.File, dockerFiles []string) (*depe
 			Schedule:         weeklySchedule,
 		})
 	}
+
+	for _, p := range pipFiles {
+		local, err := localPath(root, p)
+		if err != nil {
+			return nil, err
+		}
+
+		c.Updates = append(c.Updates, update{
+			PackageEcosystem: pipPkgEco,
+			Directory:        local,
+			Labels:           pipLabels,
+			Schedule:         weeklySchedule,
+		})
+	}
+
 	return c, nil
 }
 
@@ -88,7 +103,12 @@ func generate() error {
 		return err
 	}
 
-	c, err := buildConfigFunc(root, mods, dockerFiles)
+	pipFiles, err := allPipFunc(root)
+	if err != nil {
+		return err
+	}
+
+	c, err := buildConfigFunc(root, mods, dockerFiles, pipFiles)
 	if err != nil {
 		return err
 	}
diff --git a/dbotconf/internal/generate_test.go b/dbotconf/internal/generate_test.go
@@ -69,8 +69,11 @@ func TestBuildConfig(t *testing.T) {
 		"/home/user/repo/a/",
 		"/home/user/repo/b/",
 	}
+	pipFiles := []string{
+		"/home/user/repo/requirements.txt",
+	}
 
-	got, err := buildConfig(root, mods, dockerFiles)
+	got, err := buildConfig(root, mods, dockerFiles, pipFiles)
 	require.NoError(t, err)
 	assert.Equal(t, &dependabotConfig{
 		Version: version2,
@@ -82,6 +85,7 @@ func TestBuildConfig(t *testing.T) {
 			newUpdate(gomodPkgEco, "/", goLabels),
 			newUpdate(gomodPkgEco, "/a", goLabels),
 			newUpdate(gomodPkgEco, "/b", goLabels),
+			newUpdate(pipPkgEco, "/", pipLabels),
 		},
 	}, got)
 }
@@ -119,11 +123,17 @@ func TestRunGenerateReturnBuildConfigError(t *testing.T) {
 	allDockerFunc = func(string) ([]string, error) {
 		return nil, nil
 	}
+	t.Cleanup(func(f func(string) ([]string, error)) func() {
+		return func() { allPipFunc = f }
+	}(allPipFunc))
+	allPipFunc = func(string) ([]string, error) {
+		return nil, nil
+	}
 
-	t.Cleanup(func(f func(string, []*modfile.File, []string) (*dependabotConfig, error)) func() {
+	t.Cleanup(func(f func(string, []*modfile.File, []string, []string) (*dependabotConfig, error)) func() {
 		return func() { buildConfigFunc = f }
 	}(buildConfigFunc))
-	buildConfigFunc = func(string, []*modfile.File, []string) (*dependabotConfig, error) {
+	buildConfigFunc = func(string, []*modfile.File, []string, []string) (*dependabotConfig, error) {
 		return nil, assert.AnError
 	}
 	assert.ErrorIs(t, generate(), assert.AnError)
diff --git a/dbotconf/internal/mods.go b/dbotconf/internal/mods.go
@@ -27,6 +27,7 @@ import (
 var (
 	allModsFunc           = allMods
 	allDockerFunc         = allDocker
+	allPipFunc            = allPip
 	configuredUpdatesFunc = configuredUpdates
 )
 
@@ -51,6 +52,10 @@ func allDocker(root string) ([]string, error) {
 	return repo.FindFilePatternDirs(root, "*Dockerfile*")
 }
 
+func allPip(root string) ([]string, error) {
+	return repo.FindFilePatternDirs(root, "*requirements.txt")
+}
+
 // localModPath returns the dependabot appropriate directory name for module
 // mod that resides in a repo with root.
 func localModPath(root string, mod *modfile.File) (string, error) {
diff --git a/dbotconf/internal/testdata/dependabot.yml b/dbotconf/internal/testdata/dependabot.yml
@@ -1,57 +1,66 @@
 # File generated by dbotconf; DO NOT EDIT.
 version: 2
 updates:
-    - package-ecosystem: github-actions
-      directory: /
-      labels:
-        - dependencies
-        - actions
-        - Skip Changelog
-      schedule:
-        interval: weekly
-        day: sunday
-    - package-ecosystem: docker
-      directory: /
-      labels:
-        - dependencies
-        - docker
-        - Skip Changelog
-      schedule:
-        interval: weekly
-        day: sunday
-    - package-ecosystem: docker
-      directory: /a/b/example
-      labels:
-        - dependencies
-        - docker
-        - Skip Changelog
-      schedule:
-        interval: weekly
-        day: sunday
-    - package-ecosystem: gomod
-      directory: /
-      labels:
-        - dependencies
-        - actions
-        - Skip Changelog
-      schedule:
-        interval: weekly
-        day: sunday
-    - package-ecosystem: gomod
-      directory: /a
-      labels:
-        - dependencies
-        - actions
-        - Skip Changelog
-      schedule:
-        interval: weekly
-        day: sunday
-    - package-ecosystem: gomod
-      directory: /a/b
-      labels:
-        - dependencies
-        - actions
-        - Skip Changelog
-      schedule:
-        interval: weekly
-        day: sunday
+  - package-ecosystem: github-actions
+    directory: /
+    labels:
+      - dependencies
+      - actions
+      - Skip Changelog
+    schedule:
+      interval: weekly
+      day: sunday
+  - package-ecosystem: docker
+    directory: /
+    labels:
+      - dependencies
+      - docker
+      - Skip Changelog
+    schedule:
+      interval: weekly
+      day: sunday
+  - package-ecosystem: docker
+    directory: /a/b/example
+    labels:
+      - dependencies
+      - docker
+      - Skip Changelog
+    schedule:
+      interval: weekly
+      day: sunday
+  - package-ecosystem: gomod
+    directory: /
+    labels:
+      - dependencies
+      - actions
+      - Skip Changelog
+    schedule:
+      interval: weekly
+      day: sunday
+  - package-ecosystem: gomod
+    directory: /a
+    labels:
+      - dependencies
+      - actions
+      - Skip Changelog
+    schedule:
+      interval: weekly
+      day: sunday
+  - package-ecosystem: gomod
+    directory: /a/b
+    labels:
+      - dependencies
+      - actions
+      - Skip Changelog
+    schedule:
+      interval: weekly
+      day: sunday
+  - package-ecosystem: pip
+    directory: /
+    labels:
+      - dependencies
+      - python
+      - Skip Changelog
+    schedule:
+      interval: weekly
+      day: sunday
diff --git a/dbotconf/internal/verify.go b/dbotconf/internal/verify.go
@@ -29,37 +29,46 @@ var (
 	errNotEnoughArg = errors.New("path argument required")
 )
 
-// configuredUpdates returns the set of Go modules and Dockerfiles dependabot
-// is configured to check updates for.
-func configuredUpdates(path string) (mods map[string]struct{}, docker map[string]struct{}, err error) {
+type updates struct {
+	mods   map[string]struct{}
+	docker map[string]struct{}
+	pip    map[string]struct{}
+}
+
+// configuredUpdates returns updates configured in the dependabot configuration
+func configuredUpdates(path string) (u updates, err error) {
 	f, err := os.Open(filepath.Clean(path))
 	if errors.Is(err, os.ErrNotExist) {
-		return nil, nil, fmt.Errorf("dependabot configuration file does not exist: %s", path)
+		return updates{}, fmt.Errorf("dependabot configuration file does not exist: %s", path)
 	}
 	if err != nil {
-		return nil, nil, fmt.Errorf("failed to read dependabot configuration file: %s", path)
+		return updates{}, fmt.Errorf("failed to read dependabot configuration file: %s", path)
 	}
 
 	var c dependabotConfig
 	if err := yaml.NewDecoder(f).Decode(&c); err != nil {
-		return nil, nil, fmt.Errorf("invalid dependabot configuration: %w", err)
+		return updates{}, fmt.Errorf("invalid dependabot configuration: %w", err)
 	}
 
-	mods = make(map[string]struct{})
-	docker = make(map[string]struct{})
+	mods := make(map[string]struct{})
+	docker := make(map[string]struct{})
+	pip := make(map[string]struct{})
 	for _, u := range c.Updates {
 		if u.PackageEcosystem == dockerPkgEco {
 			docker[u.Directory] = struct{}{}
 		}
 		if u.PackageEcosystem == gomodPkgEco {
 			mods[u.Directory] = struct{}{}
 		}
+		if u.PackageEcosystem == pipPkgEco {
+			pip[u.Directory] = struct{}{}
+		}
 	}
-	return mods, docker, nil
+	return updates{mods, docker, pip}, nil
 }
 
-// verify ensures dependabot configuration contains a check for all modules and
-// Dockerfiles.
+// verify ensures dependabot configuration contains a check for all modules,
+// Dockerfiles, and requirements.txt files.
 func verify(args []string) error {
 	switch len(args) {
 	case 0:
@@ -80,7 +89,13 @@ func verify(args []string) error {
 		return err
 	}
 
-	modUp, dockerUp, err := configuredUpdatesFunc(args[0])
+	pipFiles, err := allPipFunc(root)
+	if err != nil {
+		return err
+	}
+
+	u, err := configuredUpdatesFunc(args[0])
+
 	if err != nil {
 		return err
 	}
@@ -92,7 +107,7 @@ func verify(args []string) error {
 			return err
 		}
 
-		if _, ok := modUp[local]; !ok {
+		if _, ok := u.mods[local]; !ok {
 			missingMod = append(missingMod, local)
 		}
 	}
@@ -103,19 +118,33 @@ func verify(args []string) error {
 			return err
 		}
 
-		if _, ok := dockerUp[local]; !ok {
+		if _, ok := u.docker[local]; !ok {
 			missingDocker = append(missingDocker, local)
 		}
 	}
+	var missingPip []string
+	for _, p := range pipFiles {
+		local, err := localPath(root, p)
+		if err != nil {
+			return err
+		}
 
-	if len(missingMod) > 0 || len(missingDocker) > 0 {
+		if _, ok := u.pip[local]; !ok {
+			missingPip = append(missingPip, local)
+		}
+	}
+
+	if len(missingMod) > 0 || len(missingDocker) > 0 || len(missingPip) > 0 {
 		msg := "missing update check(s):"
 		if len(missingMod) > 0 {
 			msg = fmt.Sprintf("%s\n- Go mod files: %s", msg, strings.Join(missingMod, ", "))
 		}
 		if len(missingDocker) > 0 {
 			msg = fmt.Sprintf("%s\n- Dockerfiles: %s", msg, strings.Join(missingDocker, ", "))
 		}
+		if len(missingPip) > 0 {
+			msg = fmt.Sprintf("%s\n- Pip files: %s", msg, strings.Join(missingPip, ", "))
+		}
 		msg += "\n"
 		return errors.New(msg)
 	}
diff --git a/dbotconf/internal/verify_test.go b/dbotconf/internal/verify_test.go

Original file line number	Diff line number	Diff line change
`@@ -19,13 +19,15 @@ const (`
`19`	`19`	`ghPkgEco = "github-actions"`
`20`	`20`	`dockerPkgEco = "docker"`
`21`	`21`	`gomodPkgEco = "gomod"`
	`22`	`+ pipPkgEco = "pip"`
`22`	`23`	`)`
`23`	`24`
`24`	`25`	`var (`
`25`	`26`	`weeklySchedule = schedule{Interval: "weekly", Day: "sunday"}`
`26`	`27`	`actionLabels = []string{"dependencies", "actions", "Skip Changelog"}`
`27`	`28`	`dockerLabels = []string{"dependencies", "docker", "Skip Changelog"}`
`28`	`29`	`goLabels = []string{"dependencies", "go", "Skip Changelog"}`
	`30`	`+ pipLabels = []string{"dependencies", "python", "Skip Changelog"}`
`29`	`31`	`)`
`30`	`32`
`31`	`33`	`type dependabotConfig struct {`