Introduce new statuses: SKIP, MANU, ERRO

Author: hypnoglow

check/check.go

@@ -38,9 +38,12 @@ const (
 	WARN State = "WARN"
 	// INFO informational message
 	INFO State = "INFO"
-
 	// SKIP for when a check should be skipped.
-	SKIP = "skip"
+	SKIP State = "SKIP"
+	// MANU for when a check is manual.
+	MANU State = "MANU"
+	// ERRO for errors in tests.
+	ERRO State = "ERRO"
 
 	// MASTER a master node
 	MASTER NodeType = "master"
@@ -58,8 +61,11 @@ const (
 	// MANAGEDSERVICES a node to run managedservices from
 	MANAGEDSERVICES = "managedservices"
 
-	// MANUAL Check Type
-	MANUAL string = "manual"
+	// TypeSkip is skip check type.
+	TypeSkip = "skip"
+
+	// TypeManual is manual check type.
+	TypeManual = "manual"
 )
 
 // Check contains information about a recommendation in the
@@ -118,18 +124,18 @@ func (c *Check) run() State {
 		return c.State
 	}
 
-	// If check type is skip, force result to INFO
-	if c.Type == SKIP {
+	// If check type is skip, force result to SKIP
+	if c.Type == TypeSkip {
 		c.Reason = "Test marked as skip"
-		c.State = INFO
+		c.State = SKIP
 		glog.V(3).Info(c.Reason)
 		return c.State
 	}
 
-	// If check type is manual force result to WARN
-	if c.Type == MANUAL {
+	// If check type is manual, force result to MANU
+	if c.Type == TypeManual {
 		c.Reason = "Test marked as a manual test"
-		c.State = WARN
+		c.State = MANU
 		glog.V(3).Info(c.Reason)
 		return c.State
 	}
@@ -172,11 +178,7 @@ func (c *Check) run() State {
 
 	if err != nil {
 		c.Reason = err.Error()
-		if c.Scored {
-			c.State = FAIL
-		} else {
-			c.State = WARN
-		}
+		c.State = ERRO
 		glog.V(3).Info(c.Reason)
 	}
 

check/check_test.go

@@ -27,9 +27,27 @@ func TestCheck_Run(t *testing.T) {
 	}
 
 	testCases := []TestCase{
-		{name: "Manual check should WARN", check: Check{Type: MANUAL}, Expected: WARN},
-		{name: "Skip check should INFO", check: Check{Type: "skip"}, Expected: INFO},
-		{name: "Unscored check (with no type) should WARN on failure", check: Check{Scored: false}, Expected: WARN},
+		{
+			name: "Manual check should MANU",
+			check: Check{
+				Type: TypeManual,
+			},
+			Expected: MANU,
+		},
+		{
+			name: "Skip check should SKIP",
+			check: Check{
+				Type: TypeSkip,
+			},
+			Expected: SKIP,
+		},
+		{
+			name: "Unscored check (with no type) should WARN on failure",
+			check: Check{
+				Scored: false,
+			},
+			Expected: WARN,
+		},
 		{
 			name: "Unscored check that pass should PASS",
 			check: Check{
@@ -42,9 +60,21 @@ func TestCheck_Run(t *testing.T) {
 			},
 			Expected: PASS,
 		},
-
-		{name: "Check with no tests should WARN", check: Check{Scored: true}, Expected: WARN},
-		{name: "Scored check with empty tests should FAIL", check: Check{Scored: true, Tests: &tests{}}, Expected: FAIL},
+		{
+			name: "Check with no tests should WARN",
+			check: Check{
+				Scored: true,
+			},
+			Expected: WARN,
+		},
+		{
+			name: "Scored check with empty tests should FAIL",
+			check: Check{
+				Scored: true,
+				Tests:  &tests{},
+			},
+			Expected: FAIL,
+		},
 		{
 			name: "Scored check that doesn't pass should FAIL",
 			check: Check{

check/controls.go

@@ -64,6 +64,9 @@ type Group struct {
 	Fail   int      `json:"fail"`
 	Warn   int      `json:"warn"`
 	Info   int      `json:"info"`
+	Skip   int      `json:"skip"`
+	Manu   int      `json:"manu"`
+	Erro   int      `json:"erro"`
 	Text   string   `json:"desc"`
 	Checks []*Check `json:"results"`
 }
@@ -74,6 +77,9 @@ type Summary struct {
 	Fail int `json:"total_fail"`
 	Warn int `json:"total_warn"`
 	Info int `json:"total_info"`
+	Skip int `json:"total_skip"`
+	Manu int `json:"total_manu"`
+	Erro int `json:"total_erro"`
 }
 
 // Predicate a predicate on the given Group and Check arguments.
@@ -99,7 +105,7 @@ func NewControls(t NodeType, in []byte, detectedVersion string) (*Controls, erro
 func (controls *Controls) RunChecks(runner Runner, filter Predicate, skipIDMap map[string]bool) Summary {
 	var g []*Group
 	m := make(map[string]*Group)
-	controls.Summary.Pass, controls.Summary.Fail, controls.Summary.Warn, controls.Info = 0, 0, 0, 0
+	controls.Summary = Summary{}
 
 	for _, group := range controls.Groups {
 		for _, check := range group.Checks {
@@ -111,8 +117,8 @@ func (controls *Controls) RunChecks(runner Runner, filter Predicate, skipIDMap m
 			_, groupSkippedViaCmd := skipIDMap[group.ID]
 			_, checkSkippedViaCmd := skipIDMap[check.ID]
 
-			if group.Type == SKIP || groupSkippedViaCmd || checkSkippedViaCmd {
-				check.Type = SKIP
+			if group.Type == TypeSkip || groupSkippedViaCmd || checkSkippedViaCmd {
+				check.Type = TypeSkip
 			}
 
 			state := runner.Run(check)
@@ -158,8 +164,14 @@ func (controls *Controls) JUnit() ([]byte, error) {
 	suite := reporters.JUnitTestSuite{
 		Name:      controls.Text,
 		TestCases: []reporters.JUnitTestCase{},
-		Tests:     controls.Summary.Pass + controls.Summary.Fail + controls.Summary.Info + controls.Summary.Warn,
-		Failures:  controls.Summary.Fail,
+		Tests: controls.Summary.Pass +
+			controls.Summary.Fail +
+			controls.Summary.Info +
+			controls.Summary.Warn +
+			controls.Summary.Skip +
+			controls.Summary.Manu +
+			controls.Summary.Erro,
+		Failures: controls.Summary.Fail,
 	}
 	for _, g := range controls.Groups {
 		for _, check := range g.Checks {
@@ -179,11 +191,10 @@ func (controls *Controls) JUnit() ([]byte, error) {
 			}
 
 			switch check.State {
-			case FAIL:
+			case FAIL, ERRO:
 				tc.FailureMessage = &reporters.JUnitFailureMessage{Message: check.Remediation}
-			case WARN, INFO:
-				// WARN and INFO are two different versions of skipped tests. Either way it would be a false positive/negative to report
-				// it any other way.
+			case WARN, INFO, SKIP, MANU:
+				// Different versions of skipped tests. It would be a false positive/negative to report it any other way.
 				tc.Skipped = &reporters.JUnitSkipped{}
 			case PASS:
 			default:
@@ -226,7 +237,7 @@ func (controls *Controls) ASFF() ([]*securityhub.AwsSecurityFinding, error) {
 	tf := ti.Format(time.RFC3339)
 	for _, g := range controls.Groups {
 		for _, check := range g.Checks {
-			if check.State == FAIL || check.State == WARN {
+			if check.State == FAIL || check.State == WARN || check.State == MANU || check.State == ERRO {
 				// ASFF ProductFields['Actual result'] can't be longer than 1024 characters
 				actualValue := check.ActualValue
 				remediation := check.Remediation
@@ -292,6 +303,7 @@ func getConfig(name string) (string, error) {
 	}
 	return r, nil
 }
+
 func summarize(controls *Controls, state State) {
 	switch state {
 	case PASS:
@@ -302,6 +314,12 @@ func summarize(controls *Controls, state State) {
 		controls.Summary.Warn++
 	case INFO:
 		controls.Summary.Info++
+	case SKIP:
+		controls.Summary.Skip++
+	case MANU:
+		controls.Summary.Manu++
+	case ERRO:
+		controls.Summary.Erro++
 	default:
 		glog.Warningf("Unrecognized state %s", state)
 	}
@@ -317,6 +335,12 @@ func summarizeGroup(group *Group, state State) {
 		group.Warn++
 	case INFO:
 		group.Info++
+	case SKIP:
+		group.Skip++
+	case MANU:
+		group.Manu++
+	case ERRO:
+		group.Erro++
 	default:
 		glog.Warningf("Unrecognized state %s", state)
 	}

check/controls_test.go

@@ -132,10 +132,10 @@ groups:
 		controls.RunChecks(normalRunner, allChecks, skipMap)
 
 		G1 := controls.Groups[0]
-		assertEqualGroupSummary(t, 0, 0, 3, 0, G1)
+		assertEqualGroupSummary(t, 0, 0, 0, 0, 3, 0, 0, G1)
 
 		G2 := controls.Groups[1]
-		assertEqualGroupSummary(t, 0, 0, 2, 0, G2)
+		assertEqualGroupSummary(t, 0, 0, 0, 0, 2, 0, 0, G2)
 	})
 }
 
@@ -163,7 +163,7 @@ groups:
 		controls.RunChecks(normalRunner, allChecks, emptySkipList)
 
 		G1 := controls.Groups[0]
-		assertEqualGroupSummary(t, 0, 0, 1, 0, G1)
+		assertEqualGroupSummary(t, 0, 0, 0, 0, 1, 0, 0, G1)
 	})
 }
 
@@ -214,7 +214,7 @@ groups:
 		G1 := controls.Groups[0]
 		assert.Equal(t, "G1", G1.ID)
 		assert.Equal(t, "G1/C1", G1.Checks[0].ID)
-		assertEqualGroupSummary(t, 1, 0, 0, 0, G1)
+		assertEqualGroupSummary(t, 1, 0, 0, 0, 0, 0, 0, G1)
 		// and
 		G2 := controls.Groups[1]
 		assert.Equal(t, "G2", G2.ID)
@@ -225,12 +225,15 @@ groups:
 		assert.Equal(t, "SomeSampleFlag=true", G2.Checks[0].Tests.TestItems[0].Flag)
 		assert.Equal(t, "Edit the config file /this/is/a/file/path and set SomeSampleFlag to true.\n", G2.Checks[0].Remediation)
 		assert.Equal(t, true, G2.Checks[0].Scored)
-		assertEqualGroupSummary(t, 0, 1, 0, 0, G2)
+		assertEqualGroupSummary(t, 0, 1, 0, 0, 0, 0, 0, G2)
 		// and
 		assert.Equal(t, 1, controls.Summary.Pass)
 		assert.Equal(t, 1, controls.Summary.Fail)
 		assert.Equal(t, 0, controls.Summary.Info)
 		assert.Equal(t, 0, controls.Summary.Warn)
+		assert.Equal(t, 0, controls.Summary.Skip)
+		assert.Equal(t, 0, controls.Summary.Manu)
+		assert.Equal(t, 0, controls.Summary.Erro)
 		// and
 		runner.AssertExpectations(t)
 	})
@@ -267,6 +270,9 @@ func TestControls_JUnitIncludesJSON(t *testing.T) {
 					Pass: 100,
 					Warn: 101,
 					Info: 102,
+					Skip: 0,
+					Manu: 0,
+					Erro: 0,
 				},
 				Groups: []*Group{
 					{
@@ -283,7 +289,7 @@ func TestControls_JUnitIncludesJSON(t *testing.T) {
     </testcase>
 </testsuite>`),
 		}, {
-			desc: "Warn and Info are considered skips and failed tests properly reported",
+			desc: "WARN, INFO, SKIP, MANU are considered skips and failed tests properly reported",
 			input: &Controls{
 				Groups: []*Group{
 					{
@@ -293,6 +299,9 @@ func TestControls_JUnitIncludesJSON(t *testing.T) {
 							{ID: "check2id", Text: "check2text", State: INFO},
 							{ID: "check3id", Text: "check3text", State: WARN},
 							{ID: "check4id", Text: "check4text", State: FAIL},
+							{ID: "check5id", Text: "check5text", State: SKIP},
+							{ID: "check6id", Text: "check6text", State: MANU},
+							{ID: "check7id", Text: "check7text", State: ERRO},
 						},
 					},
 				},
@@ -313,6 +322,18 @@ func TestControls_JUnitIncludesJSON(t *testing.T) {
         <failure type=""></failure>
         <system-out>{&#34;test_number&#34;:&#34;check4id&#34;,&#34;test_desc&#34;:&#34;check4text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditEnv&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;FAIL&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;IsMultiple&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
     </testcase>
+    <testcase name="check5id check5text" classname="" time="0">
+        <skipped></skipped>
+        <system-out>{&#34;test_number&#34;:&#34;check5id&#34;,&#34;test_desc&#34;:&#34;check5text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditEnv&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;SKIP&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;IsMultiple&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
+    </testcase>
+    <testcase name="check6id check6text" classname="" time="0">
+        <skipped></skipped>
+        <system-out>{&#34;test_number&#34;:&#34;check6id&#34;,&#34;test_desc&#34;:&#34;check6text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditEnv&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;MANU&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;IsMultiple&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
+    </testcase>
+    <testcase name="check7id check7text" classname="" time="0">
+        <failure type=""></failure>
+        <system-out>{&#34;test_number&#34;:&#34;check7id&#34;,&#34;test_desc&#34;:&#34;check7text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditEnv&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;ERRO&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;IsMultiple&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
+    </testcase>
 </testsuite>`),
 		},
 	}
@@ -355,12 +376,15 @@ func TestControls_JUnitIncludesJSON(t *testing.T) {
 	}
 }
 
-func assertEqualGroupSummary(t *testing.T, pass, fail, info, warn int, actual *Group) {
+func assertEqualGroupSummary(t *testing.T, pass, fail, info, warn, skip, manu, erro int, actual *Group) {
 	t.Helper()
 	assert.Equal(t, pass, actual.Pass)
 	assert.Equal(t, fail, actual.Fail)
 	assert.Equal(t, info, actual.Info)
 	assert.Equal(t, warn, actual.Warn)
+	assert.Equal(t, skip, actual.Skip)
+	assert.Equal(t, manu, actual.Manu)
+	assert.Equal(t, erro, actual.Erro)
 }
 
 func TestControls_ASFF(t *testing.T) {
@@ -388,6 +412,9 @@ func TestControls_ASFF(t *testing.T) {
 					Pass: 100,
 					Warn: 101,
 					Info: 102,
+					Skip: 0,
+					Manu: 0,
+					Erro: 0,
 				},
 				Groups: []*Group{
 					{