Add support for incremental stamping (#175)

* Add support for incremental stamping
* Update docs with stamp parameter

Author: randrei-adobe

README.md

@@ -464,6 +464,16 @@ As a result of the setup above the `create_gitops_prs` tool will open up to 2 po
 
 The GitOps pull request is only created (or new commits added) if the `gitops` target changes the state for the target deployment branch. The source pull request will remain open (and keep accumulation GitOps results) until the pull request is merged and source branch is deleted.
 
+The `--stamp` parameter allows for the replacement of certain placeholders, but only when the `gitops` target changes the output's digest compared to the one already saved. The new digest of the unstamped data is also saved with the manifest. The digest is kept in a file in the same location as the YAML file, with a `.digest` extension added to its name. This is helpful when the manifests have volatile information that shouldn't be the only factor causing changes in the target deployment branch.
+
+Here are the placeholders that can be replaced:
+
+| Placeholder      | Replacement                                    |
+|------------------|-------------------------------------------------|
+| `{{GIT_REVISION}}` | Result of `git rev-parse HEAD`                  |
+| `{{UTC_DATE}}`     | Result of `date -u`                             |
+| `{{GIT_BRANCH}}`   | The `branch_name` argument given to `create_gitops_prs` |
+
 `--dry_run` parameter can be used to test the tool without creating any pull requests. The tool will print the list of the potential pull requests. It is recommended to run the tool in the dry run mode as a part of the CI test suite to verify that the tool is configured correctly.
 
 <a name="multiple-release-branches-gitops-workflow"></a>

gitops/digester/BUILD

@@ -0,0 +1,20 @@
+# Copyright 2024 Adobe. All rights reserved.
+# This file is licensed to you under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License. You may obtain a copy
+# of the License at http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+# OF ANY KIND, either express or implied. See the License for the specific language
+# governing permissions and limitations under the License.
+
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+licenses(["notice"])  # Apache 2.0
+
+go_library(
+    name = "go_default_library",
+    srcs = ["digester.go"],
+    importpath = "github.com/adobe/rules_gitops/gitops/digester",
+    visibility = ["//visibility:public"],
+)

gitops/digester/digester.go

@@ -0,0 +1,74 @@
+/*
+Copyright 2024 Adobe. All rights reserved.
+This file is licensed to you under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License. You may obtain a copy
+of the License at http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under
+the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+OF ANY KIND, either express or implied. See the License for the specific language
+governing permissions and limitations under the License.
+*/
+package digester
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"io"
+	"log"
+	"os"
+)
+
+// CalculateDigest calculates the SHA256 digest of a file specified by the given path
+func CalculateDigest(path string) string {
+	if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
+		return ""
+	}
+
+	fi, err := os.Open(path)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer fi.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, fi); err != nil {
+		log.Fatal(err)
+	}
+
+	return hex.EncodeToString(h.Sum(nil))
+}
+
+// GetDigest retrieves the digest of a file from a file with the same name but with a ".digest" extension
+func GetDigest(path string) string {
+	digestPath := path + ".digest"
+
+	if _, err := os.Stat(digestPath); errors.Is(err, os.ErrNotExist) {
+		return ""
+	}
+
+	digest, err := os.ReadFile(digestPath)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return string(digest)
+}
+
+// VerifyDigest verifies the integrity of a file by comparing its calculated digest with the stored digest
+func VerifyDigest(path string) bool {
+	return CalculateDigest(path) == GetDigest(path)
+}
+
+// SaveDigest calculates the digest of a file at the given path and saves it to a file with the same name but with a ".digest" extension.
+func SaveDigest(path string) {
+	digest := CalculateDigest(path)
+
+	digestPath := path + ".digest"
+
+	err := os.WriteFile(digestPath, []byte(digest), 0666)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

gitops/git/git.go

@@ -12,12 +12,14 @@ governing permissions and limitations under the License.
 package git
 
 import (
+	"bufio"
 	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
 	oe "os/exec"
 	"path/filepath"
+	"strings"
 
 	"github.com/adobe/rules_gitops/gitops/exec"
 )
@@ -114,6 +116,28 @@ func (r *Repo) Commit(message, gitopsPath string) bool {
 	return true
 }
 
+// RestoreFile restores the specified file in the repository to its original state
+func (r *Repo) RestoreFile(fileName string) {
+	exec.Mustex(r.Dir, "git", "checkout", "--", fileName)
+}
+
+// GetChangedFiles returns a list of files that have been changed in the repository
+func (r *Repo) GetChangedFiles() []string {
+	s, err := exec.Ex(r.Dir, "git", "diff", "--name-only")
+	if err != nil {
+		log.Fatalf("ERROR: %s", err)
+	}
+	var files []string
+	sc := bufio.NewScanner(strings.NewReader(s))
+	for sc.Scan() {
+		files = append(files, sc.Text())
+	}
+	if err := sc.Err(); err != nil {
+		log.Fatalf("ERROR: %s", err)
+	}
+	return files
+}
+
 // IsClean returns true if there is no local changes (nothing to commit)
 func (r *Repo) IsClean() bool {
 	cmd := oe.Command("git", "status", "--porcelain")

gitops/prer/BUILD

@@ -21,11 +21,13 @@ go_library(
         "//gitops/analysis:go_default_library",
         "//gitops/bazel:go_default_library",
         "//gitops/commitmsg:go_default_library",
+        "//gitops/digester:go_default_library",
         "//gitops/exec:go_default_library",
         "//gitops/git:go_default_library",
         "//gitops/git/bitbucket:go_default_library",
         "//gitops/git/github:go_default_library",
         "//gitops/git/gitlab:go_default_library",
+        "//templating/fasttemplate:go_default_library",
         "//vendor/github.com/golang/protobuf/proto:go_default_library",
     ],
 )

gitops/prer/create_gitops_prs.go

@@ -25,11 +25,13 @@ import (
 	"github.com/adobe/rules_gitops/gitops/analysis"
 	"github.com/adobe/rules_gitops/gitops/bazel"
 	"github.com/adobe/rules_gitops/gitops/commitmsg"
+	"github.com/adobe/rules_gitops/gitops/digester"
 	"github.com/adobe/rules_gitops/gitops/exec"
 	"github.com/adobe/rules_gitops/gitops/git"
 	"github.com/adobe/rules_gitops/gitops/git/bitbucket"
 	"github.com/adobe/rules_gitops/gitops/git/github"
 	"github.com/adobe/rules_gitops/gitops/git/gitlab"
+	"github.com/adobe/rules_gitops/templating/fasttemplate"
 
 	proto "github.com/golang/protobuf/proto"
 )
@@ -71,6 +73,7 @@ var (
 	gitopsKind             SliceFlags
 	gitopsRuleName         SliceFlags
 	gitopsRuleAttr         SliceFlags
+	stamp                  = flag.Bool("stamp", false, "Stamp results of gitops targets with volatile information")
 	dryRun                 = flag.Bool("dry_run", false, "Do not create PRs, just print what would be done")
 )
 
@@ -101,6 +104,40 @@ func bazelQuery(query string) *analysis.CqueryResult {
 	return qr
 }
 
+func getGitStatusDict(workdir *git.Repo, gitCommit, branchName string) map[string]interface{} {
+	utcDate, err := exec.Ex("", "date", "-u")
+	if err != nil {
+		log.Fatal(err)
+	}
+	utcDate = strings.TrimSpace(utcDate)
+
+	ctx := map[string]interface{}{
+		"GIT_REVISION": gitCommit,
+		"UTC_DATE":     utcDate,
+		"GIT_BRANCH":   branchName,
+	}
+
+	return ctx
+}
+
+func stampFile(fullPath string, ctx map[string]interface{}) {
+	template, err := os.ReadFile(fullPath)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	outf, err := os.OpenFile(fullPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer outf.Close()
+
+	_, err = fasttemplate.Execute(string(template), "{{", "}}", outf, ctx)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
 func main() {
 	flag.Parse()
 	if *workspace != "" {
@@ -187,6 +224,21 @@ func main() {
 			bin := bazel.TargetToExecutable(target)
 			exec.Mustex("", bin, "--nopush", "--nobazel", "--deployment_root", gitopsdir)
 		}
+		if *stamp {
+			changedFiles := workdir.GetChangedFiles()
+			if len(changedFiles) > 0 {
+				ctx := getGitStatusDict(workdir, *gitCommit, *branchName)
+				for _, filePath := range changedFiles {
+					fullPath := gitopsdir + "/" + filePath
+					if digester.VerifyDigest(fullPath) {
+						workdir.RestoreFile(fullPath)
+					} else {
+						digester.SaveDigest(fullPath)
+						stampFile(fullPath, ctx)
+					}
+				}
+			}
+		}
 		if workdir.Commit(fmt.Sprintf("GitOps for release branch %s from %s commit %s\n%s", *releaseBranch, *branchName, *gitCommit, commitmsg.Generate(targets)), *gitopsPath) {
 			log.Println("branch", branch, "has changes, push is required")
 			updatedGitopsTargets = append(updatedGitopsTargets, targets...)