feat: Add --skip-archived-repos flag (#33)

Author: jphuynh

README.md

@@ -332,6 +332,7 @@ echo "gruntwork-io/terragrunt gruntwork-io/terratest" | git-xargs \
 | `--github-org`         | If you want to target every repo in a Github org that your GITHUB_OAUTH_TOKEN has access to, pass the name of the Organization with this flag, to page through every repo via the Github API and target it                                                                                                                                                                                                                     | String  | No       |
 | `--commit-message`     | The commit message to use when creating commits. If you supply this flag, but neither the optional `--pull-request-title` or `--pull-request-description` flags, then the commit message value will be used for all three.                                                                                                                                                                                                     | String  | No       |
 | `--skip-pull-requests` | If you don't want any pull requests opened, but would rather have your changes committed directly to your specified branch, pass this flag. Note that it won't work if your Github repo is configured with branch protections on the branch you're trying to commit directly to!                                                                                                                                               | Boolean | No       |
+| `--skip-archived-repos`| If you want to exclude archived (read-only) repositories from the list of targeted repos, pass this flag.                                                                                                                                                                                                                                                                                                                      | Boolean | No       |
 | `--dry-run`            | If you are in the process of testing out `git-xargs` or your intial set of targeted repos, but you don't want to make any changes via the Github API (pushing your local changes or opening pull requests) you can pass the dry-run branch. This is useful because the output report will still tell you which repos would have been affected, without actually making changes via the Github API to your remote repositories. | Boolean | No       |
 
 ## Best practices, tips and tricks

cmd/git-xargs.go

@@ -22,6 +22,7 @@ func parseGitXargsConfig(c *cli.Context) (*config.GitXargsConfig, error) {
 	config := config.NewGitXargsConfig()
 	config.DryRun = c.Bool("dry-run")
 	config.SkipPullRequests = c.Bool("skip-pull-requests")
+	config.SkipArchivedRepos = c.Bool("skip-archived-repos")
 	config.BranchName = c.String("branch-name")
 	config.CommitMessage = c.String("commit-message")
 	config.PullRequestTitle = c.String("pull-request-title")

common/common.go

@@ -6,6 +6,7 @@ const (
 	GithubOrgFlagName              = "github-org"
 	DryRunFlagName                 = "dry-run"
 	SkipPullRequestsFlagName       = "skip-pull-requests"
+	SkipArchivedReposFlagName      = "skip-archived-repos"
 	RepoFlagName                   = "repo"
 	ReposFileFlagName              = "repos"
 	CommitMessageFlagName          = "commit-message"
@@ -30,6 +31,10 @@ var (
 		Name:  SkipPullRequestsFlagName,
 		Usage: "When skip-pull-requests is set to true, no pull requests will be opened. All changes will be committed and pushed to the specified branch directly.",
 	}
+	GenericSkipArchivedReposFlag = cli.BoolFlag{
+		Name:  SkipArchivedReposFlagName,
+		Usage: "Used in conjunction with github-org, will exclude archived repositories.",
+	}
 	GenericRepoFlag = cli.StringSliceFlag{
 		Name:  RepoFlagName,
 		Usage: "A single repo name to run the command on in the format of <github-organization/repo-name>. Can be invoked multiple times with different repo names",

config/config.go

@@ -14,6 +14,7 @@ import (
 type GitXargsConfig struct {
 	DryRun                 bool
 	SkipPullRequests       bool
+	SkipArchivedRepos      bool
 	BranchName             string
 	CommitMessage          string
 	PullRequestTitle       string
@@ -33,6 +34,7 @@ func NewGitXargsConfig() *GitXargsConfig {
 	return &GitXargsConfig{
 		DryRun:                 false,
 		SkipPullRequests:       false,
+		SkipArchivedRepos:      false,
 		BranchName:             "",
 		CommitMessage:          common.DefaultCommitMessage,
 		PullRequestTitle:       common.DefaultPullRequestTitle,

main.go

@@ -64,6 +64,7 @@ func setupApp() *cli.App {
 		common.GenericGithubOrgFlag,
 		common.GenericDryRunFlag,
 		common.GenericSkipPullRequestFlag,
+		common.GenericSkipArchivedReposFlag,
 		common.GenericRepoFlag,
 		common.GenericRepoFileFlag,
 		common.GenericBranchFlag,

mocks/mocks.go

@@ -14,10 +14,14 @@ var ownerName = "gruntwork-io"
 var repoName1 = "terragrunt"
 var repoName2 = "terratest"
 var repoName3 = "fetch"
+var repoName4 = "terraform-kubernetes-helm"
 
 var repoURL1 = "https://github.com/gruntwork-io/terragrunt"
 var repoURL2 = "https://github.com/gruntwork-io/terratest"
 var repoURL3 = "https://github.com/gruntwork-io/fetch"
+var repoURL4 = "https://github.com/gruntwork-io/terraform-kubernetes-helm"
+
+var archivedFlag = true
 
 var MockGithubRepositories = []*github.Repository{
 	&github.Repository{
@@ -41,6 +45,14 @@ var MockGithubRepositories = []*github.Repository{
 		Name:    &repoName3,
 		HTMLURL: &repoURL3,
 	},
+	&github.Repository{
+		Owner: &github.User{
+		Login: &ownerName,
+		},
+		Name:    &repoName4,
+		HTMLURL: &repoURL4,
+		Archived: &archivedFlag,
+	},
 }
 
 // This mocks the PullRequest service in go-github that is used in production to call the associated Github endpoint

repository/fetch-repos.go

@@ -84,11 +84,31 @@ func getReposByOrg(config *config.GitXargsConfig) ([]*github.Repository, error)
 	}
 
 	for {
+		var reposToAdd []*github.Repository
 		repos, resp, err := config.GithubClient.Repositories.ListByOrg(context.Background(), config.GithubOrg, opt)
 		if err != nil {
 			return allRepos, errors.WithStackTrace(err)
 		}
-		allRepos = append(allRepos, repos...)
+
+		// github.RepositoryListByOrgOptions doesn't seem to be able to filter out archived repos
+		// So re-slice the repos list if --skip-archived-repos is passed and the repository is in archived/read-only state
+		for i, repo := range repos {
+			if config.SkipArchivedRepos && repo.GetArchived() {
+				logger.WithFields(logrus.Fields{
+					"Name": repo.GetFullName(),
+				}).Debug("Skipping archived repository")
+
+				// Track repos to skip because of archived status for our final run report
+				config.Stats.TrackSingle(stats.ReposArchivedSkipped, repo)
+
+				reposToAdd = append(repos[:i], repos[i+1:]...)
+			} else {
+				reposToAdd = repos
+			}
+		}
+
+		allRepos = append(allRepos, reposToAdd...)
+
 		if resp.NextPage == 0 {
 			break
 		}

repository/fetch-repos_test.go

@@ -34,7 +34,7 @@ func TestGetFileDefinedRepos(t *testing.T) {
 
 	githubRepos, reposLookupErr := getFileDefinedRepos(config.GithubClient, allowedRepos, config.Stats)
 
-	assert.Equal(t, len(githubRepos), len(mocks.MockGithubRepositories))
+	assert.Equal(t, len(githubRepos), len(allowedRepos))
 	assert.NoError(t, reposLookupErr)
 }
 
@@ -51,3 +51,18 @@ func TestGetReposByOrg(t *testing.T) {
 	assert.Equal(t, len(githubRepos), len(mocks.MockGithubRepositories))
 	assert.NoError(t, reposByOrgLookupErr)
 }
+
+// TestSkipArchivedRepos ensures that you can filter out archived repositories
+func TestSkipArchivedRepos(t *testing.T) {
+	t.Parallel()
+
+	config := config.NewGitXargsTestConfig()
+	config.GithubOrg = "gruntwork-io"
+	config.SkipArchivedRepos = true
+	config.GithubClient = mocks.ConfigureMockGithubClient()
+
+	githubRepos, reposByOrgLookupErr := getReposByOrg(config)
+
+	assert.Equal(t, len(githubRepos), len(mocks.MockGithubRepositories) -1)
+	assert.NoError(t, reposByOrgLookupErr)
+}

stats/stats.go

@@ -14,6 +14,8 @@ const (
 	DryRunSet types.Event = "dry-run-set-no-changes-made"
 	// ReposSelected denotes all the repositories that were targeted for processing by this tool AFTER filtering was applied to determine valid repos
 	ReposSelected types.Event = "repos-selected-pre-processing"
+	// ReposArchivedSkipped denotes all the repositories that were skipped from the list of repos to clone because the skip-archived-repos was set to true
+	ReposArchivedSkipped types.Event = "repos-archived-skipped"
 	// TargetBranchNotFound denotes the special branch used by this tool to make changes on was not found on lookup, suggesting it should be created
 	TargetBranchNotFound types.Event = "target-branch-not-found"
 	// TargetBranchAlreadyExists denotes the special branch used by this tool was already found (so it was likely already created by a previous run)
@@ -70,6 +72,7 @@ var allEvents = []types.AnnotatedEvent{
 	{Event: FetchedViaGithubAPI, Description: "Repos successfully fetched via Github API"},
 	{Event: DryRunSet, Description: "Repos that were not modified in any way because this was a dry-run"},
 	{Event: ReposSelected, Description: "All repos that were targeted for processing AFTER filtering missing / malformed repos"},
+	{Event: ReposArchivedSkipped, Description: "All repos that were filtered out with the --skip-archived-repos flag"},
 	{Event: TargetBranchNotFound, Description: "Repos whose target branch was not found"},
 	{Event: TargetBranchAlreadyExists, Description: "Repos whose target branch already existed"},
 	{Event: TargetBranchLookupErr, Description: "Repos whose target branches could not be looked up due to an API error"},
@@ -96,6 +99,7 @@ var allEvents = []types.AnnotatedEvent{
 // RunStats will be a stats-tracker class that keeps score of which repos were touched, which were considered for update, which had branches made, PRs made, which were missing workflows or contexts, or had out of date workflows syntax values, etc
 type RunStats struct {
 	repos                 map[types.Event][]*github.Repository
+	skippedArchivedRepos  map[types.Event][]*github.Repository
 	pulls                 map[string]string
 	command               []string
 	fileProvidedRepos     []*types.AllowedRepo
@@ -112,6 +116,7 @@ func NewStatsTracker() *RunStats {
 
 	t := &RunStats{
 		repos:                 make(map[types.Event][]*github.Repository),
+		skippedArchivedRepos:  make(map[types.Event][]*github.Repository),
 		pulls:                 make(map[string]string),
 		command:               []string{},
 		fileProvidedRepos:     fileProvidedRepos,
@@ -134,6 +139,11 @@ func (r *RunStats) GetRepos() map[types.Event][]*github.Repository {
 	return r.repos
 }
 
+// GetSkippedArchivedRepos returns the inner map of events to *github.Repositories that are excluded from the targeted repos list
+func (r *RunStats) GetSkippedArchivedRepos() map[types.Event][]*github.Repository {
+	return r.skippedArchivedRepos
+}
+
 // GetPullRequests returns the inner representation of the pull requests that were opened during the lifecycle of a given run
 func (r *RunStats) GetPullRequests() map[string]string {
 	return r.pulls
@@ -211,6 +221,7 @@ func (r *RunStats) TrackMultiple(event types.Event, repos []*github.Repository)
 func (r *RunStats) GenerateRunReport() *types.RunReport {
 	return &types.RunReport{
 		Repos:          r.GetRepos(),
+		SkippedRepos:   r.GetSkippedArchivedRepos(),
 		Command:        r.command,
 		RuntimeSeconds: r.GetTotalRunSeconds(), FileProvidedRepos: r.GetFileProvidedRepos(),
 		PullRequests: r.GetPullRequests(),

types/types.go

@@ -17,6 +17,7 @@ type ReducedRepo struct {
 
 type RunReport struct {
 	Repos             map[Event][]*github.Repository
+	SkippedRepos      map[Event][]*github.Repository
 	Command           []string
 	RuntimeSeconds    int
 	FileProvidedRepos []*AllowedRepo