feat: multi-source Layer4 implementation (for discussion)

layer4/assessment_log.go

@@ -80,6 +80,16 @@ func (a *AssessmentLog) Run(targetData interface{}) Result {
 	return a.Result
 }
 
+// SetConfidenceLevel sets the evaluator's confidence level in this specific assessment result.
+// Returns an error if the assessment hasn't been run yet.
+func (a *AssessmentLog) SetConfidenceLevel(confidenceLevel ConfidenceLevel) error {
+	if a.Result == NotRun {
+		return errors.New("cannot set confidence level before assessment has been run")
+	}
+	a.ConfidenceLevel = confidenceLevel
+	return nil
+}
+
 // precheck verifies that the assessment has all the required fields.
 // It returns an error if the assessment is not valid.
 func (a *AssessmentLog) precheck() error {

layer4/assessment_log_test.go

@@ -1,6 +1,7 @@
 package layer4
 
 import (
+	"strings"
 	"testing"
 )
 
@@ -185,8 +186,84 @@ func TestNewAssessment(t *testing.T) {
 			if !data.expectedError && err != nil {
 				t.Errorf("expected no error, got %v", err)
 			}
-			if assessment == nil && !data.expectedError {
-				t.Error("expected assessment object, got nil")
+		if assessment == nil && !data.expectedError {
+			t.Error("expected assessment object, got nil")
+		}
+	})
+	}
+}
+
+func TestSetConfidenceLevel(t *testing.T) {
+	tests := []struct {
+		name           string
+		setupResult    Result // Result to set before calling SetConfidenceLevel
+		confidenceLevel ConfidenceLevel
+		wantErr        bool
+		expectedErr   string
+	}{
+		{
+			name:           "Cannot set confidence level before Run",
+			setupResult:    NotRun,
+			confidenceLevel: High,
+			wantErr:        true,
+			expectedErr:    "cannot set confidence level before assessment has been run",
+		},
+		{
+			name:           "Can set confidence level to High after Run",
+			setupResult:    Passed,
+			confidenceLevel: High,
+			wantErr:        false,
+		},
+		{
+			name:           "Can set confidence level to Medium after Run",
+			setupResult:    Passed,
+			confidenceLevel: Medium,
+			wantErr:        false,
+		},
+		{
+			name:           "Can set confidence level to Low after Run",
+			setupResult:    Passed,
+			confidenceLevel: Low,
+			wantErr:        false,
+		},
+		{
+			name:           "Can set confidence level after Failed result",
+			setupResult:    Failed,
+			confidenceLevel: High,
+			wantErr:        false,
+		},
+		{
+			name:           "Can set confidence level after NeedsReview result",
+			setupResult:    NeedsReview,
+			confidenceLevel: Low,
+			wantErr:        false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assessment, err := NewAssessment("test", "test description", []string{"test"}, []AssessmentStep{passingAssessmentStep})
+			if err != nil {
+				t.Fatalf("Failed to create assessment: %v", err)
+			}
+
+			// Set the result to simulate assessment completion (or NotRun state)
+			assessment.Result = tt.setupResult
+
+			err = assessment.SetConfidenceLevel(tt.confidenceLevel)
+			if tt.wantErr {
+				if err == nil {
+					t.Error("expected error, got nil")
+				} else if tt.expectedErr != "" && !strings.Contains(err.Error(), tt.expectedErr) {
+					t.Errorf("expected error containing %q, got %q", tt.expectedErr, err.Error())
+				}
+			} else {
+				if err != nil {
+					t.Errorf("unexpected error: %v", err)
+				}
+				if assessment.ConfidenceLevel != tt.confidenceLevel {
+					t.Errorf("expected confidence level %v, got %v", tt.confidenceLevel, assessment.ConfidenceLevel)
+				}
 			}
 		})
 	}

layer4/confidence_level.go

@@ -0,0 +1,77 @@
+package layer4
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/ossf/gemara/internal/loaders"
+)
+
+// ConfidenceLevel indicates the evaluator's confidence level in an assessment result.
+// This is designed to restrict the possible confidence level values to a set of known levels.
+type ConfidenceLevel int
+
+const (
+	// Undetermined indicates the confidence level could not be determined (default).
+	Undetermined ConfidenceLevel = iota
+	// Low indicates the evaluator has low confidence in this result.
+	Low
+	// Medium indicates the evaluator has moderate confidence in this result.
+	Medium
+	// High indicates the evaluator has high confidence in this result.
+	High
+)
+
+var confidenceLevelToString = map[ConfidenceLevel]string{
+	Undetermined: "Undetermined",
+	Low:          "Low",
+	Medium:       "Medium",
+	High:         "High",
+}
+
+var stringToConfidenceLevel = map[string]ConfidenceLevel{
+	"Undetermined": Undetermined,
+	"Low":          Low,
+	"Medium":       Medium,
+	"High":         High,
+}
+
+func (c *ConfidenceLevel) String() string {
+	return confidenceLevelToString[*c]
+}
+
+// MarshalYAML ensures that ConfidenceLevel is serialized as a string in YAML
+func (c *ConfidenceLevel) MarshalYAML() (interface{}, error) {
+	return c.String(), nil
+}
+
+// UnmarshalYAML ensures that ConfidenceLevel can be deserialized from a YAML string
+func (c *ConfidenceLevel) UnmarshalYAML(data []byte) error {
+	var s string
+	if err := loaders.UnmarshalYAML(data, &s); err != nil {
+		return err
+	}
+	if val, ok := stringToConfidenceLevel[s]; ok {
+		*c = val
+		return nil
+	}
+	return fmt.Errorf("invalid ConfidenceLevel: %s", s)
+}
+
+// MarshalJSON ensures that ConfidenceLevel is serialized as a string in JSON
+func (c *ConfidenceLevel) MarshalJSON() ([]byte, error) {
+	return json.Marshal(c.String())
+}
+
+// UnmarshalJSON ensures that ConfidenceLevel can be deserialized from a JSON string
+func (c *ConfidenceLevel) UnmarshalJSON(data []byte) error {
+	var s string
+	if err := json.Unmarshal(data, &s); err != nil {
+		return err
+	}
+	if val, ok := stringToConfidenceLevel[s]; ok {
+		*c = val
+		return nil
+	}
+	return fmt.Errorf("invalid ConfidenceLevel: %s", s)
+}

layer4/confidence_level_test.go

@@ -0,0 +1,149 @@
+package layer4
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfidenceLevel_String(t *testing.T) {
+	tests := []struct {
+		name  string
+		level ConfidenceLevel
+		want  string
+	}{
+		{
+			name:  "Undetermined level",
+			level: Undetermined,
+			want:  "Undetermined",
+		},
+		{
+			name:  "Low level",
+			level: Low,
+			want:  "Low",
+		},
+		{
+			name:  "Medium level",
+			level: Medium,
+			want:  "Medium",
+		},
+		{
+			name:  "High level",
+			level: High,
+			want:  "High",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tt.level.String()
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestConfidenceLevel_UnmarshalYAML(t *testing.T) {
+	tests := []struct {
+		name    string
+		data    string
+		want    ConfidenceLevel
+		wantErr bool
+	}{
+		{
+			name:    "Undetermined level",
+			data:    "Undetermined",
+			want:    Undetermined,
+			wantErr: false,
+		},
+		{
+			name:    "Low level",
+			data:    "Low",
+			want:    Low,
+			wantErr: false,
+		},
+		{
+			name:    "Medium level",
+			data:    "Medium",
+			want:    Medium,
+			wantErr: false,
+		},
+		{
+			name:    "High level",
+			data:    "High",
+			want:    High,
+			wantErr: false,
+		},
+		{
+			name:    "Invalid level",
+			data:    "Invalid",
+			want:    Undetermined,
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var level ConfidenceLevel
+			err := level.UnmarshalYAML([]byte(tt.data))
+			if tt.wantErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tt.want, level)
+			}
+		})
+	}
+}
+
+func TestConfidenceLevel_UnmarshalJSON(t *testing.T) {
+	tests := []struct {
+		name    string
+		data    string
+		want    ConfidenceLevel
+		wantErr bool
+	}{
+		{
+			name:    "Undetermined level",
+			data:    `"Undetermined"`,
+			want:    Undetermined,
+			wantErr: false,
+		},
+		{
+			name:    "Low level",
+			data:    `"Low"`,
+			want:    Low,
+			wantErr: false,
+		},
+		{
+			name:    "Medium level",
+			data:    `"Medium"`,
+			want:    Medium,
+			wantErr: false,
+		},
+		{
+			name:    "High level",
+			data:    `"High"`,
+			want:    High,
+			wantErr: false,
+		},
+		{
+			name:    "Invalid level",
+			data:    `"Invalid"`,
+			want:    Undetermined,
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var level ConfidenceLevel
+			err := level.UnmarshalJSON([]byte(tt.data))
+			if tt.wantErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tt.want, level)
+			}
+		})
+	}
+}

layer4/evaluation_plan.go

@@ -43,7 +43,7 @@ type Checklist struct {
 }
 
 // ToChecklist converts an EvaluationPlan into a structured Checklist.
-func (e EvaluationPlan) ToChecklist() (Checklist, error) {
+func (e *EvaluationPlan) ToChecklist() (Checklist, error) {
 	checklist := Checklist{}
 
 	if e.Metadata.Id != "" {
@@ -92,7 +92,7 @@ func (e EvaluationPlan) ToChecklist() (Checklist, error) {
 
 // ToMarkdownChecklist converts an evaluation plan into a markdown checklist.
 // Generates a pre-execution checklist showing what needs to be checked.
-func (e EvaluationPlan) ToMarkdownChecklist() (string, error) {
+func (e *EvaluationPlan) ToMarkdownChecklist() (string, error) {
 	checklist, err := e.ToChecklist()
 	if err != nil {
 		return "", fmt.Errorf("failed to build checklist: %w", err)

layer4/evaluation_plan_test.go

@@ -82,7 +82,7 @@ func Test_ToMarkdownChecklist(t *testing.T) {
 				Metadata: Metadata{
 					Id:      "plan-2024-01",
 					Version: "1.0.0",
-					Author: Author{
+					Author: Actor{
 						Name:    "gemara",
 						Uri:     "https://github.com/ossf/gemara",
 						Version: "1.0.0",
@@ -150,7 +150,7 @@ func Test_ToMarkdownChecklist(t *testing.T) {
 					},
 				},
 				Metadata: Metadata{
-					Author: Author{Name: "test"},
+					Author: Actor{Name: "test"},
 					MappingReferences: []MappingReference{
 						{
 							Id:      "OSPS-B",
@@ -174,7 +174,7 @@ func Test_ToMarkdownChecklist(t *testing.T) {
 				Plans: []AssessmentPlan{},
 				Metadata: Metadata{
 					Id: "empty-plan",
-					Author: Author{
+					Author: Actor{
 						Name: "test",
 					},
 				},
@@ -235,7 +235,7 @@ func Test_ToChecklist(t *testing.T) {
 		},
 		Metadata: Metadata{
 			Id: "test-plan",
-			Author: Author{
+			Author: Actor{
 				Name:    "test-author",
 				Version: "1.0.0",
 			},