Skip to content

Improve go mod checks with hack/verify-go-modules.sh #3373

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Improve go mod checks with hack/verify-go-modules.sh #3373

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2fea1a2
hack/verify-go-modules.sh: check for dirty gomod and version mismatch
gman0 Apr 3, 2025
6cc8cd2
cli-doc: tidy'd and sync'd go mod
gman0 Apr 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docs/generators/cli-doc/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,8 @@ require (
k8s.io/component-base v0.31.6 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/kustomize/api v0.17.2 // indirect
sigs.k8s.io/kustomize/kyaml v0.17.1 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
Expand Down
8 changes: 4 additions & 4 deletions docs/generators/cli-doc/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -296,10 +296,10 @@ k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag=
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98=
k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0=
k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE=
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A=
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
sigs.k8s.io/kustomize/api v0.17.2 h1:E7/Fjk7V5fboiuijoZHgs4aHuexi5Y2loXlVOAVAG5g=
sigs.k8s.io/kustomize/api v0.17.2/go.mod h1:UWTz9Ct+MvoeQsHcJ5e+vziRRkwimm3HytpZgIYqye0=
sigs.k8s.io/kustomize/kyaml v0.17.1 h1:TnxYQxFXzbmNG6gOINgGWQt09GghzgTP6mIurOgrLCQ=
Expand Down
84 changes: 60 additions & 24 deletions hack/verify-go-modules.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,38 @@ set -o pipefail

REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)

mapfile -t DIRS < <(find "${REPO_ROOT}" -name go.mod -print0 | xargs -0 dirname)
if [[ ! -f "${REPO_ROOT}/go.work" ]]; then
(
cd "${REPO_ROOT}"
go work init
go work use -r .
)
fi

# Array of all modules inside the repository, sorted in
# descending topological order (dependant comes before its dependee).
mapfile -t MODULE_DIRS < <(
for mod in $(tsort <(
# Lines in format "<Dependant> <Dependee>" are fed into `tsort` to perform the topological sort.
for dir in $(go list -m -json | jq -r '.Dir'); do
(
cd "${dir}"
# Prints the dependency graph where we extract only the direct
# github.com/kcp-dev/kcp/* dependencies of the currently examined module,
# and we discard the dependency version (the line is in format
# "<This module> <Its dependency>@<Version>", so we split by '@' and get the
# first part). We skip when no such deps are found.
go mod graph \
| grep -E "^$(go mod edit -json | jq -r '.Module.Path') github.com/kcp-dev/kcp/" \
| cut -d'@' -f1 \
|| true
)
done
) | tac); do # We need to reverse the lines (with `tac`) to have a descending order.
# We have sorted module paths. We need to convert them into their respective directory locations.
go list -m -json "$mod" | jq -r '.Dir'
done
)

# list_deps lists dependencies of the supplied go.mod file (dependencies
# with version "v0.0.0" are skipped). The output is a json dictionary in the
Expand Down Expand Up @@ -55,42 +86,40 @@ function diff_version_deps {
has_deps="${1}"
wants_deps="${2}"
jq -s '
# Extracts v<Maj>.<Min> from a semantic version string.
def major_minor_semver: capture("v(?<major>\\d+)\\.(?<minor>\\d+)")
| "v" + .major + "." + .minor;

map(to_entries) # Convert both input dicts into two separate arrays.
| add # Concatenate those two arrays into a single one.
| group_by(.key) # Group items by `.key`.
| map( # Map each selected item into { "<Dep>": {"has": "<Version0>", "wants": "<Version1>"} }.
select(
# If grouping resulted in two items, it means both arrays have this dependency.
length == 2
# Compare the v<Maj>.<Min> of both items.
and (.[0].value | major_minor_semver) != (.[1].value | major_minor_semver)
# And the dependency has a version mismatch.
and (.[0].value != .[1].value)
)
| { (.[0].key): { "has": .[0].value, "wants": .[1].value } }
)
| add // empty
' "${has_deps}" "${wants_deps}"
}

# print_diff_version_deps prints the output of diff_version_deps as a
# human-readable multi-line text.
# print_diff_version_deps prints the output of diff_version_deps (expected in stdin)
# as a human-readable multi-line text.
# Returns 1 if any lines were printed (i.e. errors were found).
function print_diff_version_deps {
jq -r "to_entries | map(\"Warning: version mismatch: has \(.key)@\(.value.has), but \(.value.wants) expected\") | join(\"\\n\")"
jq -r '
to_entries
| map("Version mismatch: \(.key) is \(.value.has), but \(.value.wants) expected")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
| map("Version mismatch: \(.key) is \(.value.has), but \(.value.wants) expected")
| map(" Version mismatch: \(.key) is \(.value.has), but \(.value.wants) expected")

Just as a visual separator in the output - I felt like the mismatches were going missing a bit in the output

| join("\n")
'
}

# Compares versions of dependencies in the supplied go.mod to
# makes sure they are in line with the ones declared in
# k8s.io/kubernetes module and prints the result.
function compare_mod_versions {
# k8s.io/kubernetes module.
function compare_mod_versions_with_k8s_deps {
gomod_file="${1}"
echo "Verifying dependency versions in ${gomod_file} against ${k8s_gomod}"
deps="$(list_deps ${gomod_file})"

diff_version_deps <(echo "${deps}") <(echo "${k8s_deps}") \
| print_diff_version_deps "${gomod_file}"
diff_version_deps <(echo "${deps}") <(echo "${k8s_deps}")
}

function gomod_filepath_for {
Expand All @@ -101,20 +130,27 @@ function gomod_filepath_for {
k8s_gomod="$(gomod_filepath_for k8s.io/kubernetes)"
k8s_deps="$(list_deps ${k8s_gomod})"

for dir in "${DIRS[@]}"; do
for dir in "${MODULE_DIRS[@]}"; do
(
cd "$dir"
echo "Verifying ${dir}"

if ! git diff --quiet HEAD -- go.mod go.sum; then
git --no-pager diff HEAD -- go.mod go.sum
echo "Error: go.mod and/or go.sum in ${dir} files have been modified, inspect and commit them before continuing" 1>&2
if ! go mod tidy -diff ; then
echo "Error: go.mod and/or go.sum is not clean, run 'go mod tidy' before continuing" 1>&2
exit 1
fi

compare_mod_versions "${dir}/go.mod"
gomod_file="${dir}/go.mod"
echo "Verifying dependency versions in ${gomod_file} against ${k8s_gomod}"

diff="$(compare_mod_versions_with_k8s_deps ${dir}/go.mod)"
if [[ -n "${diff}" ]]; then
echo "${diff}" | print_diff_version_deps 1>&2
echo "${diff}" | grep -q "k8s.io/" && {
echo "Error: dependencies from '*k8s.io/*' must be in line with ${k8s_gomod}" 1>&2
exit 1
} || true
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
} || true
}

Personally I'd prefer the script to not error but to continue going through the go.mods

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
} || true
echo "${diff}" | grep -q "k8s.io/" && {
echo "Error: dependencies from '*k8s.io/*' must be in line with ${k8s_gomod}" 1>&2
}

Sorry like this - the exit will still kill the script as {} is only a scope, not a subshell.
Instead the errors could be added up and then add a exit $errors at the end

echo "Continuing because no fatal errors found"
fi
)
done

compare_mod_versions "$(gomod_filepath_for github.com/kcp-dev/client-go)"
compare_mod_versions "$(gomod_filepath_for github.com/kcp-dev/apimachinery/v2)"