diff --git a/layer4/assessment_log.go b/layer4/assessment_log.go index a764d77..c471af2 100644 --- a/layer4/assessment_log.go +++ b/layer4/assessment_log.go @@ -80,6 +80,16 @@ func (a *AssessmentLog) Run(targetData interface{}) Result { return a.Result } +// SetConfidenceLevel sets the evaluator's confidence level in this specific assessment result. +// Returns an error if the assessment hasn't been run yet. +func (a *AssessmentLog) SetConfidenceLevel(confidenceLevel ConfidenceLevel) error { + if a.Result == NotRun { + return errors.New("cannot set confidence level before assessment has been run") + } + a.ConfidenceLevel = confidenceLevel + return nil +} + // precheck verifies that the assessment has all the required fields. // It returns an error if the assessment is not valid. func (a *AssessmentLog) precheck() error { diff --git a/layer4/assessment_log_test.go b/layer4/assessment_log_test.go index f61a57a..57d2cd4 100644 --- a/layer4/assessment_log_test.go +++ b/layer4/assessment_log_test.go @@ -1,6 +1,7 @@ package layer4 import ( + "strings" "testing" ) @@ -185,8 +186,84 @@ func TestNewAssessment(t *testing.T) { if !data.expectedError && err != nil { t.Errorf("expected no error, got %v", err) } - if assessment == nil && !data.expectedError { - t.Error("expected assessment object, got nil") + if assessment == nil && !data.expectedError { + t.Error("expected assessment object, got nil") + } + }) + } +} + +func TestSetConfidenceLevel(t *testing.T) { + tests := []struct { + name string + setupResult Result // Result to set before calling SetConfidenceLevel + confidenceLevel ConfidenceLevel + wantErr bool + expectedErr string + }{ + { + name: "Cannot set confidence level before Run", + setupResult: NotRun, + confidenceLevel: High, + wantErr: true, + expectedErr: "cannot set confidence level before assessment has been run", + }, + { + name: "Can set confidence level to High after Run", + setupResult: Passed, + confidenceLevel: High, + wantErr: false, + }, + { + name: "Can set confidence level to Medium after Run", + setupResult: Passed, + confidenceLevel: Medium, + wantErr: false, + }, + { + name: "Can set confidence level to Low after Run", + setupResult: Passed, + confidenceLevel: Low, + wantErr: false, + }, + { + name: "Can set confidence level after Failed result", + setupResult: Failed, + confidenceLevel: High, + wantErr: false, + }, + { + name: "Can set confidence level after NeedsReview result", + setupResult: NeedsReview, + confidenceLevel: Low, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assessment, err := NewAssessment("test", "test description", []string{"test"}, []AssessmentStep{passingAssessmentStep}) + if err != nil { + t.Fatalf("Failed to create assessment: %v", err) + } + + // Set the result to simulate assessment completion (or NotRun state) + assessment.Result = tt.setupResult + + err = assessment.SetConfidenceLevel(tt.confidenceLevel) + if tt.wantErr { + if err == nil { + t.Error("expected error, got nil") + } else if tt.expectedErr != "" && !strings.Contains(err.Error(), tt.expectedErr) { + t.Errorf("expected error containing %q, got %q", tt.expectedErr, err.Error()) + } + } else { + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if assessment.ConfidenceLevel != tt.confidenceLevel { + t.Errorf("expected confidence level %v, got %v", tt.confidenceLevel, assessment.ConfidenceLevel) + } } }) } diff --git a/layer4/confidence_level.go b/layer4/confidence_level.go new file mode 100644 index 0000000..c5fdb1d --- /dev/null +++ b/layer4/confidence_level.go @@ -0,0 +1,77 @@ +package layer4 + +import ( + "encoding/json" + "fmt" + + "github.com/ossf/gemara/internal/loaders" +) + +// ConfidenceLevel indicates the evaluator's confidence level in an assessment result. +// This is designed to restrict the possible confidence level values to a set of known levels. +type ConfidenceLevel int + +const ( + // Undetermined indicates the confidence level could not be determined (default). + Undetermined ConfidenceLevel = iota + // Low indicates the evaluator has low confidence in this result. + Low + // Medium indicates the evaluator has moderate confidence in this result. + Medium + // High indicates the evaluator has high confidence in this result. + High +) + +var confidenceLevelToString = map[ConfidenceLevel]string{ + Undetermined: "Undetermined", + Low: "Low", + Medium: "Medium", + High: "High", +} + +var stringToConfidenceLevel = map[string]ConfidenceLevel{ + "Undetermined": Undetermined, + "Low": Low, + "Medium": Medium, + "High": High, +} + +func (c *ConfidenceLevel) String() string { + return confidenceLevelToString[*c] +} + +// MarshalYAML ensures that ConfidenceLevel is serialized as a string in YAML +func (c *ConfidenceLevel) MarshalYAML() (interface{}, error) { + return c.String(), nil +} + +// UnmarshalYAML ensures that ConfidenceLevel can be deserialized from a YAML string +func (c *ConfidenceLevel) UnmarshalYAML(data []byte) error { + var s string + if err := loaders.UnmarshalYAML(data, &s); err != nil { + return err + } + if val, ok := stringToConfidenceLevel[s]; ok { + *c = val + return nil + } + return fmt.Errorf("invalid ConfidenceLevel: %s", s) +} + +// MarshalJSON ensures that ConfidenceLevel is serialized as a string in JSON +func (c *ConfidenceLevel) MarshalJSON() ([]byte, error) { + return json.Marshal(c.String()) +} + +// UnmarshalJSON ensures that ConfidenceLevel can be deserialized from a JSON string +func (c *ConfidenceLevel) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + if val, ok := stringToConfidenceLevel[s]; ok { + *c = val + return nil + } + return fmt.Errorf("invalid ConfidenceLevel: %s", s) +} diff --git a/layer4/confidence_level_test.go b/layer4/confidence_level_test.go new file mode 100644 index 0000000..61045e8 --- /dev/null +++ b/layer4/confidence_level_test.go @@ -0,0 +1,149 @@ +package layer4 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConfidenceLevel_String(t *testing.T) { + tests := []struct { + name string + level ConfidenceLevel + want string + }{ + { + name: "Undetermined level", + level: Undetermined, + want: "Undetermined", + }, + { + name: "Low level", + level: Low, + want: "Low", + }, + { + name: "Medium level", + level: Medium, + want: "Medium", + }, + { + name: "High level", + level: High, + want: "High", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.level.String() + assert.Equal(t, tt.want, got) + }) + } +} + +func TestConfidenceLevel_UnmarshalYAML(t *testing.T) { + tests := []struct { + name string + data string + want ConfidenceLevel + wantErr bool + }{ + { + name: "Undetermined level", + data: "Undetermined", + want: Undetermined, + wantErr: false, + }, + { + name: "Low level", + data: "Low", + want: Low, + wantErr: false, + }, + { + name: "Medium level", + data: "Medium", + want: Medium, + wantErr: false, + }, + { + name: "High level", + data: "High", + want: High, + wantErr: false, + }, + { + name: "Invalid level", + data: "Invalid", + want: Undetermined, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var level ConfidenceLevel + err := level.UnmarshalYAML([]byte(tt.data)) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, level) + } + }) + } +} + +func TestConfidenceLevel_UnmarshalJSON(t *testing.T) { + tests := []struct { + name string + data string + want ConfidenceLevel + wantErr bool + }{ + { + name: "Undetermined level", + data: `"Undetermined"`, + want: Undetermined, + wantErr: false, + }, + { + name: "Low level", + data: `"Low"`, + want: Low, + wantErr: false, + }, + { + name: "Medium level", + data: `"Medium"`, + want: Medium, + wantErr: false, + }, + { + name: "High level", + data: `"High"`, + want: High, + wantErr: false, + }, + { + name: "Invalid level", + data: `"Invalid"`, + want: Undetermined, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var level ConfidenceLevel + err := level.UnmarshalJSON([]byte(tt.data)) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, level) + } + }) + } +} diff --git a/layer4/evaluation_plan.go b/layer4/evaluation_plan.go index 1c634de..007ad00 100644 --- a/layer4/evaluation_plan.go +++ b/layer4/evaluation_plan.go @@ -43,7 +43,7 @@ type Checklist struct { } // ToChecklist converts an EvaluationPlan into a structured Checklist. -func (e EvaluationPlan) ToChecklist() (Checklist, error) { +func (e *EvaluationPlan) ToChecklist() (Checklist, error) { checklist := Checklist{} if e.Metadata.Id != "" { @@ -92,7 +92,7 @@ func (e EvaluationPlan) ToChecklist() (Checklist, error) { // ToMarkdownChecklist converts an evaluation plan into a markdown checklist. // Generates a pre-execution checklist showing what needs to be checked. -func (e EvaluationPlan) ToMarkdownChecklist() (string, error) { +func (e *EvaluationPlan) ToMarkdownChecklist() (string, error) { checklist, err := e.ToChecklist() if err != nil { return "", fmt.Errorf("failed to build checklist: %w", err) diff --git a/layer4/evaluation_plan_test.go b/layer4/evaluation_plan_test.go index 0ddbe0c..f95fd1d 100644 --- a/layer4/evaluation_plan_test.go +++ b/layer4/evaluation_plan_test.go @@ -82,7 +82,7 @@ func Test_ToMarkdownChecklist(t *testing.T) { Metadata: Metadata{ Id: "plan-2024-01", Version: "1.0.0", - Author: Author{ + Author: Actor{ Name: "gemara", Uri: "https://github.com/ossf/gemara", Version: "1.0.0", @@ -150,7 +150,7 @@ func Test_ToMarkdownChecklist(t *testing.T) { }, }, Metadata: Metadata{ - Author: Author{Name: "test"}, + Author: Actor{Name: "test"}, MappingReferences: []MappingReference{ { Id: "OSPS-B", @@ -174,7 +174,7 @@ func Test_ToMarkdownChecklist(t *testing.T) { Plans: []AssessmentPlan{}, Metadata: Metadata{ Id: "empty-plan", - Author: Author{ + Author: Actor{ Name: "test", }, }, @@ -235,7 +235,7 @@ func Test_ToChecklist(t *testing.T) { }, Metadata: Metadata{ Id: "test-plan", - Author: Author{ + Author: Actor{ Name: "test-author", Version: "1.0.0", }, diff --git a/layer4/execution_type.go b/layer4/execution_type.go new file mode 100644 index 0000000..8c31511 --- /dev/null +++ b/layer4/execution_type.go @@ -0,0 +1,68 @@ +package layer4 + +import ( + "encoding/json" + "fmt" + + "github.com/ossf/gemara/internal/loaders" +) + +// ExecutionType specifies whether an actor's execution mode is automated or manual. +type ExecutionType int + +const ( + // Automated indicates the evaluator is a tool or script that runs without human intervention. + Automated ExecutionType = iota + // Manual indicates the evaluator requires human review or judgment. + Manual +) + +var evaluatorTypeToString = map[ExecutionType]string{ + Automated: "Automated", + Manual: "Manual", +} + +var stringToEvaluatorType = map[string]ExecutionType{ + "Automated": Automated, + "Manual": Manual, +} + +func (e *ExecutionType) String() string { + return evaluatorTypeToString[*e] +} + +// MarshalYAML ensures that ExecutionType is serialized as a string in YAML +func (e *ExecutionType) MarshalYAML() (interface{}, error) { + return e.String(), nil +} + +// UnmarshalYAML ensures that ExecutionType can be deserialized from a YAML string +func (e *ExecutionType) UnmarshalYAML(data []byte) error { + var s string + if err := loaders.UnmarshalYAML(data, &s); err != nil { + return err + } + if val, ok := stringToEvaluatorType[s]; ok { + *e = val + return nil + } + return fmt.Errorf("invalid ExecutionType: %s", s) +} + +// MarshalJSON ensures that ExecutionType is serialized as a string in JSON +func (e *ExecutionType) MarshalJSON() ([]byte, error) { + return json.Marshal(e.String()) +} + +// UnmarshalJSON ensures that ExecutionType can be deserialized from a JSON string +func (e *ExecutionType) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + if val, ok := stringToEvaluatorType[s]; ok { + *e = val + return nil + } + return fmt.Errorf("invalid ExecutionType: %s", s) +} diff --git a/layer4/execution_type_test.go b/layer4/execution_type_test.go new file mode 100644 index 0000000..30f46b2 --- /dev/null +++ b/layer4/execution_type_test.go @@ -0,0 +1,115 @@ +package layer4 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestExecutionType_String(t *testing.T) { + tests := []struct { + name string + typ ExecutionType + want string + }{ + { + name: "Automated type", + typ: Automated, + want: "Automated", + }, + { + name: "Manual type", + typ: Manual, + want: "Manual", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.typ.String() + assert.Equal(t, tt.want, got) + }) + } +} + +func TestExecutionType_UnmarshalYAML(t *testing.T) { + tests := []struct { + name string + data string + want ExecutionType + wantErr bool + }{ + { + name: "Automated type", + data: "Automated", + want: Automated, + wantErr: false, + }, + { + name: "Manual type", + data: "Manual", + want: Manual, + wantErr: false, + }, + { + name: "Invalid type", + data: "Invalid", + want: Automated, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var typ ExecutionType + err := typ.UnmarshalYAML([]byte(tt.data)) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, typ) + } + }) + } +} + +func TestExecutionType_UnmarshalJSON(t *testing.T) { + tests := []struct { + name string + data string + want ExecutionType + wantErr bool + }{ + { + name: "Automated type", + data: `"Automated"`, + want: Automated, + wantErr: false, + }, + { + name: "Manual type", + data: `"Manual"`, + want: Manual, + wantErr: false, + }, + { + name: "Invalid type", + data: `"Invalid"`, + want: Automated, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var typ ExecutionType + err := typ.UnmarshalJSON([]byte(tt.data)) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, typ) + } + }) + } +} diff --git a/layer4/generated_types.go b/layer4/generated_types.go index 10e2f21..8dd7492 100644 --- a/layer4/generated_types.go +++ b/layer4/generated_types.go @@ -2,32 +2,49 @@ package layer4 -// EvaluationPlan defines how a set of Layer 2 controls are to be evaluated. -type EvaluationPlan struct { +// EvaluationDocument defines how a set of Layer 2 controls are to be evaluated and the associated outcomes of the evaluation. +type EvaluationDocument struct { Metadata Metadata `json:"metadata" yaml:"metadata"` - Plans []AssessmentPlan `json:"plans" yaml:"plans"` + EvaluationPlan *EvaluationPlan `json:"evaluation-plan,omitempty" yaml:"evaluation-plan,omitempty"` + + EvaluationLogs []EvaluationLog `json:"evaluation-logs,omitempty" yaml:"evaluation-logs,omitempty"` } -// Metadata contains metadata about the Layer 4 evaluation plan and log. +// Metadata contains common fields shared across all metadata types. type Metadata struct { Id string `json:"id" yaml:"id"` Version string `json:"version,omitempty" yaml:"version,omitempty"` - Author Author `json:"author" yaml:"author"` + Author Actor `json:"author" yaml:"author"` + + Description string `json:"description,omitempty" yaml:"description,omitempty"` MappingReferences []MappingReference `json:"mapping-references,omitempty" yaml:"mapping-references,omitempty"` } -// Author contains the information about the entity that produced the evaluation plan or log. -type Author struct { +// Actor represents an entity (human or tool) that can perform actions in evaluations. +type Actor struct { + // Id uniquely identifies the actor. + Id string `json:"id" yaml:"id"` + + // Name provides the name of the actor. Name string `json:"name" yaml:"name"` - Uri string `json:"uri,omitempty" yaml:"uri,omitempty"` + // Type specifies how the evaluation is executed (automated tool or manual/human evaluator). + Type ExecutionType `json:"type" yaml:"type"` + // Version specifies the version of the actor (if applicable, e.g., for tools). Version string `json:"version,omitempty" yaml:"version,omitempty"` + // Description provides additional context about the actor. + Description string `json:"description,omitempty" yaml:"description,omitempty"` + + // Uri provides a general URI for the actor information. + Uri string `json:"uri,omitempty" yaml:"uri,omitempty"` + + // Contact provides contact information for the actor. Contact Contact `json:"contact,omitempty" yaml:"contact,omitempty"` } @@ -60,6 +77,16 @@ type MappingReference struct { Url string `json:"url,omitempty" yaml:"url,omitempty"` } +// EvaluationPlan defines how a set of Layer 2 controls are to be evaluated. +type EvaluationPlan struct { + Metadata Metadata `json:"metadata" yaml:"metadata"` + + // Evaluators defines the assessment evaluators that can be used to execute assessment procedures. + Evaluators []Actor `json:"evaluators" yaml:"evaluators"` + + Plans []AssessmentPlan `json:"plans" yaml:"plans"` +} + // AssessmentPlan defines all testing procedures for a control id. type AssessmentPlan struct { // Control points to the Layer 2 control being evaluated. @@ -88,9 +115,9 @@ type Mapping struct { // EvaluationLog contains the results of evaluating a set of Layer 2 controls. type EvaluationLog struct { - Evaluations []*ControlEvaluation `json:"evaluations" yaml:"evaluations"` - Metadata Metadata `json:"metadata,omitempty" yaml:"metadata,omitempty"` + + Evaluations []*ControlEvaluation `json:"evaluations" yaml:"evaluations"` } // ControlEvaluation contains the results of evaluating a single Layer 4 control. @@ -142,6 +169,9 @@ type AssessmentLog struct { // Recommendation provides guidance on how to address a failed assessment. Recommendation string `json:"recommendation,omitempty" yaml:"recommendation,omitempty"` + + // ConfidenceLevel indicates the evaluator's confidence level in this specific assessment result. + ConfidenceLevel ConfidenceLevel `json:"confidence-level,omitempty" yaml:"confidence-level,omitempty"` } type Datetime string @@ -168,6 +198,34 @@ type AssessmentProcedure struct { // Documentation provides a URL to documentation that describes how the assessment procedure evaluates the control requirement Documentation string `json:"documentation,omitempty" yaml:"documentation,omitempty"` + + // Evaluators lists which assessment evaluators can execute this procedure. + Evaluators []EvaluatorMapping `json:"evaluators" yaml:"evaluators"` + + // Strategy defines the rules for aggregating results from multiple evaluators running the same procedure. + Strategy *Strategy `json:"strategy,omitempty" yaml:"strategy,omitempty"` +} + +// EvaluatorMapping maps an assessment evaluator to a procedure. +type EvaluatorMapping struct { + Id string `json:"id" yaml:"id"` + + // Authoritative determines how this evaluator participates in conflict resolution when using AuthoritativeConfirmation strategy. + // If true, the evaluator can trigger findings independently. If false (default), the evaluator requires confirmation from authoritative evaluators before triggering findings. + // Note: This field is only used with AuthoritativeConfirmation strategy. With Strict (default) or ManualOverride strategies, this field is ignored. + Authoritative bool `json:"authoritative,omitempty" yaml:"authoritative,omitempty"` + + // Remarks provides context about why this evaluator-procedure combination was chosen. + Remarks string `json:"remarks,omitempty" yaml:"remarks,omitempty"` +} + +// Strategy defines the rules for resolving conflicts between multiple evaluator results. +type Strategy struct { + // ConflictRuleType specifies the aggregation logic used to resolve conflicts when multiple evaluators provide results for the same assessment procedure. + ConflictRuleType ConflictRuleType `json:"conflict-rule-type" yaml:"conflict-rule-type"` + + // Remarks provides context for why this specific conflict resolution strategy was chosen. + Remarks string `json:"remarks,omitempty" yaml:"remarks,omitempty"` } type Email string diff --git a/layer4/loaders.go b/layer4/loaders.go new file mode 100644 index 0000000..80d90ca --- /dev/null +++ b/layer4/loaders.go @@ -0,0 +1,29 @@ +package layer4 + +import ( + "fmt" + "path" + + "github.com/ossf/gemara/internal/loaders" +) + +// LoadFile loads data from a YAML or JSON file at the provided path. +// If run multiple times for the same data type, this method will override previous data. +func (e *EvaluationPlan) LoadFile(sourcePath string) error { + ext := path.Ext(sourcePath) + switch ext { + case ".yaml", ".yml": + err := loaders.LoadYAML(sourcePath, e) + if err != nil { + return err + } + case ".json": + err := loaders.LoadJSON(sourcePath, e) + if err != nil { + return fmt.Errorf("error loading json: %w", err) + } + default: + return fmt.Errorf("unsupported file extension: %s", ext) + } + return nil +} diff --git a/layer4/loaders_test.go b/layer4/loaders_test.go new file mode 100644 index 0000000..092eb0e --- /dev/null +++ b/layer4/loaders_test.go @@ -0,0 +1,48 @@ +package layer4 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +var tests = []struct { + name string + sourcePath string + wantErr bool +}{ + { + name: "Bad path", + sourcePath: "file://bad-path.yaml", + wantErr: true, + }, + { + name: "Bad YAML", + sourcePath: "file://test-data/bad.yaml", + wantErr: true, + }, + { + name: "Good YAML — Multi Tool Plan", + sourcePath: "file://test-data/multi-tool-plan.yaml", + wantErr: false, + }, +} + +func Test_LoadFile(t *testing.T) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := &EvaluationPlan{} + err := e.LoadFile(tt.sourcePath) + if (err == nil) == tt.wantErr { + t.Errorf("EvaluationPlan.LoadFile() error = %v, wantErr %v", err, tt.wantErr) + } + if !tt.wantErr { + // Validate that the evaluation plan was loaded successfully + if len(e.Plans) > 0 { + assert.NotEmpty(t, e.Plans[0].Control.ReferenceId, "Control reference ID should not be empty") + assert.NotEmpty(t, e.Plans[0].Control.EntryId, "Control entry ID should not be empty") + } + } + }) + } +} diff --git a/layer4/strategy.go b/layer4/strategy.go new file mode 100644 index 0000000..27017e3 --- /dev/null +++ b/layer4/strategy.go @@ -0,0 +1,77 @@ +package layer4 + +import ( + "encoding/json" + "fmt" + + "github.com/ossf/gemara/internal/loaders" +) + +// ConflictRuleType specifies the type of aggregation logic used to resolve conflicts +// when multiple evaluators provide results for the same assessment procedure. +// This is designed to restrict the possible conflict rule values to a set of known types. +type ConflictRuleType int + +const ( + // Strict indicates that if any evaluator reports a failure, the overall + // procedure result is failed, regardless of other evaluator results. + Strict ConflictRuleType = iota + // ManualOverride gives precedence to manual review evaluators over automated + // evaluators when results conflict. + ManualOverride + // AuthoritativeConfirmation treats non-authoritative evaluators + // as requiring confirmation from authoritative evaluators before triggering findings. + AuthoritativeConfirmation +) + +var conflictRuleTypeToString = map[ConflictRuleType]string{ + Strict: "Strict", + ManualOverride: "ManualOverride", + AuthoritativeConfirmation: "AuthoritativeConfirmation", +} + +var stringToConflictRuleType = map[string]ConflictRuleType{ + "Strict": Strict, + "ManualOverride": ManualOverride, + "AuthoritativeConfirmation": AuthoritativeConfirmation, +} + +func (c *ConflictRuleType) String() string { + return conflictRuleTypeToString[*c] +} + +// MarshalYAML ensures that ConflictRuleType is serialized as a string in YAML +func (c *ConflictRuleType) MarshalYAML() (interface{}, error) { + return c.String(), nil +} + +// UnmarshalYAML ensures that ConflictRuleType can be deserialized from a YAML string +func (c *ConflictRuleType) UnmarshalYAML(data []byte) error { + var s string + if err := loaders.UnmarshalYAML(data, &s); err != nil { + return err + } + if val, ok := stringToConflictRuleType[s]; ok { + *c = val + return nil + } + return fmt.Errorf("invalid ConflictRuleType: %s", s) +} + +// MarshalJSON ensures that ConflictRuleType is serialized as a string in JSON +func (c *ConflictRuleType) MarshalJSON() ([]byte, error) { + return json.Marshal(c.String()) +} + +// UnmarshalJSON ensures that ConflictRuleType can be deserialized from a JSON string +func (c *ConflictRuleType) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + if val, ok := stringToConflictRuleType[s]; ok { + *c = val + return nil + } + return fmt.Errorf("invalid ConflictRuleType: %s", s) +} diff --git a/layer4/strategy_test.go b/layer4/strategy_test.go new file mode 100644 index 0000000..5b710ba --- /dev/null +++ b/layer4/strategy_test.go @@ -0,0 +1,132 @@ +package layer4 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConflictRuleType_String(t *testing.T) { + tests := []struct { + name string + rule ConflictRuleType + want string + }{ + { + name: "Strict rule", + rule: Strict, + want: "Strict", + }, + { + name: "ManualOverride rule", + rule: ManualOverride, + want: "ManualOverride", + }, + { + name: "AuthoritativeConfirmation rule", + rule: AuthoritativeConfirmation, + want: "AuthoritativeConfirmation", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.rule.String() + assert.Equal(t, tt.want, got) + }) + } +} + +func TestConflictRuleType_UnmarshalYAML(t *testing.T) { + tests := []struct { + name string + data string + want ConflictRuleType + wantErr bool + }{ + { + name: "Strict rule", + data: "Strict", + want: Strict, + wantErr: false, + }, + { + name: "ManualOverride rule", + data: "ManualOverride", + want: ManualOverride, + wantErr: false, + }, + { + name: "AuthoritativeConfirmation rule", + data: "AuthoritativeConfirmation", + want: AuthoritativeConfirmation, + wantErr: false, + }, + { + name: "Invalid rule", + data: "Invalid", + want: Strict, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var rule ConflictRuleType + err := rule.UnmarshalYAML([]byte(tt.data)) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, rule) + } + }) + } +} + +func TestConflictRuleType_UnmarshalJSON(t *testing.T) { + tests := []struct { + name string + data string + want ConflictRuleType + wantErr bool + }{ + { + name: "Strict rule", + data: `"Strict"`, + want: Strict, + wantErr: false, + }, + { + name: "ManualOverride rule", + data: `"ManualOverride"`, + want: ManualOverride, + wantErr: false, + }, + { + name: "AuthoritativeConfirmation rule", + data: `"AuthoritativeConfirmation"`, + want: AuthoritativeConfirmation, + wantErr: false, + }, + { + name: "Invalid rule", + data: `"Invalid"`, + want: Strict, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var rule ConflictRuleType + err := rule.UnmarshalJSON([]byte(tt.data)) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, rule) + } + }) + } +} diff --git a/layer4/test-data/bad.yaml b/layer4/test-data/bad.yaml new file mode 100644 index 0000000..ed6062d --- /dev/null +++ b/layer4/test-data/bad.yaml @@ -0,0 +1,6 @@ +metadata: + id: test-plan + invalid-field: this should cause an error +plans: + - invalid: structure + invalid: yaml structure diff --git a/layer4/test-data/multi-tool-plan.yaml b/layer4/test-data/multi-tool-plan.yaml new file mode 100644 index 0000000..5d22cde --- /dev/null +++ b/layer4/test-data/multi-tool-plan.yaml @@ -0,0 +1,205 @@ +metadata: + id: test-multi-tool-osps-plan + version: "1.0.0" + author: + id: test-plan-generator + name: Test Evaluation Plan Generator + type: Automated + uri: https://example.com/test-plans + version: "1.0.0" + mapping-references: + - id: OSPS-B + title: Open Source Project Security Baseline + version: "2025" + description: The Open Source Project Security (OSPS) Baseline +evaluators: + - id: pvtr-baseline-scanner + name: PVTR Baseline Scanner + type: Automated + description: Automated security baseline scanner for GitHub repositories + uri: https://github.com/revanite-io/pvtr-github-repo + + - id: openssf-scorecard + name: OpenSSF Scorecard + type: Automated + description: Security health scorecard for open source projects + uri: https://github.com/ossf/scorecard + + - id: manual-review + name: Manual Security Review + type: Manual + description: Human expert review of security controls and documentation + uri: https://example.com/manual-review-guide + + - id: ci-test-detector + name: CI Test Suite Detector + type: Automated + description: Automated tool that detects test suites in CI/CD pipeline configurations + uri: https://example.com/ci-test-detector + + - id: test-coverage-analyzer + name: Test Coverage Analyzer + type: Automated + description: Analyzes test coverage metrics and identifies gaps in test suites + uri: https://example.com/test-coverage-analyzer +plans: + - control: + reference-id: OSPS-B + entry-id: OSPS-AC-03 + assessments: + - requirement: + reference-id: OSPS-B + entry-id: OSPS-AC-03.01 + strategy: + conflict-rule-type: Strict + procedures: + - id: check-branch-protection-automated + name: Automated Branch Protection Check + description: | + Verify that branch protection rules prevent direct commits to the primary branch + by querying the version control system API. Multiple automated tools can perform this check. + evaluators: + - id: pvtr-baseline-scanner + authoritative: true + - id: openssf-scorecard + authoritative: true + strategy: + conflict-rule-type: Strict + remarks: | + Any automated tool reporting a failure should cause the procedure to fail. + + - id: check-branch-protection-manual + name: Manual Branch Protection Review + description: | + Review repository settings in the GitHub UI to confirm branch protection is configured + and requires pull request reviews before merging. + evaluators: + - id: manual-review + authoritative: true + + - control: + reference-id: OSPS-B + entry-id: OSPS-QA-06 + assessments: + - requirement: + reference-id: OSPS-B + entry-id: OSPS-QA-06.01 + strategy: + conflict-rule-type: Strict + remarks: | + Both automated detection and manual verification are required. Automated tools + check CI/CD configuration, but manual review ensures tests are actually meaningful + and properly configured. Any failure from either procedure should cause the requirement to fail. + procedures: + - id: check-ci-test-automated + name: Automated CI Test Suite Detection + description: | + Automatically detect test suites configured in CI/CD pipelines by analyzing + workflow files, pipeline configurations, and test execution patterns. This checks + that tests are configured to run automatically before commits are accepted. + evaluators: + - id: ci-test-detector + authoritative: true + - id: pvtr-baseline-scanner + authoritative: true + - id: test-coverage-analyzer + authoritative: false + strategy: + conflict-rule-type: AuthoritativeConfirmation + remarks: | + Authoritative evaluators (ci-test-detector, pvtr-baseline-scanner) can independently + confirm test suite presence. Test-coverage-analyzer is non-authoritative and provides + additional insights but requires authoritative confirmation before triggering findings. + + - id: check-ci-test-manual + name: Manual CI Test Suite Verification + description: | + Manually review CI/CD pipeline configurations and test execution logs to verify + that automated test suites are properly configured, actually run, and produce + meaningful results. This ensures tests aren't just present but are functional. + evaluators: + - id: manual-review + authoritative: true + strategy: + conflict-rule-type: ManualOverride + remarks: | + Manual review can override automated detection results when human judgment + determines that automated tools have false positives or missed configurations. + + - control: + reference-id: OSPS-B + entry-id: OSPS-QA-07 + assessments: + - requirement: + reference-id: OSPS-B + entry-id: OSPS-QA-07.01 + strategy: + conflict-rule-type: Strict + remarks: | + Both configuration verification and behavior verification are required. Configuration + checks ensure the rules are set up correctly, while behavior checks verify the system + actually enforces them. Any failure from either procedure should cause the requirement to fail. + procedures: + - id: check-approval-configuration + name: Approval Requirement Configuration Verification + description: | + Verify that branch protection rules are configured to require at least one + non-author approval before merging to the primary branch. This checks the actual + configuration settings in the version control system (e.g., GitHub branch protection + rules, GitLab merge request settings) to ensure the requirement is properly configured. + evaluators: + - id: pvtr-baseline-scanner + authoritative: true + - id: openssf-scorecard + authoritative: true + strategy: + conflict-rule-type: Strict + + - id: check-approval-enforcement-behavior + name: Approval Requirement Enforcement Behavior Verification + description: | + Verify that the version control system actually enforces the approval requirement + by checking recent merge history, pull request patterns, and attempting to verify + that merges without non-author approval were blocked or prevented. This behavior + check ensures the configuration is not just present but actively enforced. + evaluators: + - id: pvtr-baseline-scanner + authoritative: true + + - control: + reference-id: OSPS-B + entry-id: OSPS-QA-05 + assessments: + - requirement: + reference-id: OSPS-B + entry-id: OSPS-QA-05.01 + procedures: + - id: check-binary-files + name: Binary File Detection + description: | + Scan the repository to detect binary or executable files that should + not be stored in version control. + evaluators: + - id: pvtr-baseline-scanner + authoritative: true + + - control: + reference-id: OSPS-B + entry-id: OSPS-GV-03 + assessments: + - requirement: + reference-id: OSPS-B + entry-id: OSPS-GV-03.01 + procedures: + - id: check-contribution-guide + name: Contribution Guide Detection + description: | + Scan the repository for contribution guide files (CONTRIBUTING.md, + CONTRIBUTING.rst, etc.) and verify their presence. + evaluators: + - id: pvtr-baseline-scanner + authoritative: true + - id: manual-review + authoritative: true + strategy: + conflict-rule-type: ManualOverride diff --git a/layer4/test-data/pvtr-baseline-scan.yaml b/layer4/test-data/pvtr-baseline-scan.yaml index c45662e..3991515 100644 --- a/layer4/test-data/pvtr-baseline-scan.yaml +++ b/layer4/test-data/pvtr-baseline-scan.yaml @@ -1,14 +1,28 @@ -service_name: pvtr -plugin_name: github-repo -payload: omitted -evaluation-set: - - name: "" - control-id: OSPS-AC-01 +metadata: + id: pvtr-baseline-scan + mapping-references: + - id: OSPS-B + title: Open Source Project Security Baseline + version: '2024' + author: + id: pvtr-github-repo # References evaluator from the plan + name: PVTR GitHub Repo Plugin + type: Automated + uri: https://github.com/revanite-io/pvtr-github-repo +evaluations: + - name: OSPS-AC-01 result: Passed message: Two-factor authentication is configured as required by the parent organization - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-AC-01 assessment-logs: - - requirement-id: OSPS-AC-01.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-AC-01.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-AC-01.01 applicability: - Maturity Level 1 - Maturity Level 2 @@ -19,17 +33,21 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/access_control.orgRequiresMFA steps-executed: 1 - start: 2025-08-22T16:02:00.000000000Z - end: 2025-08-22T16:02:00.000003708Z - value: null - changes: {} - - name: "" - control-id: OSPS-AC-02 + start: 2025-08-22T16:02:00+00:00 + end: 2025-08-22T16:02:00+00:00 + - name: OSPS-AC-02 result: Passed message: This control is enforced by GitHub for all projects - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-AC-02 assessment-logs: - - requirement-id: OSPS-AC-02.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-AC-02.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-AC-02.01 applicability: - Maturity Level 1 - Maturity Level 2 @@ -40,17 +58,21 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.GithubBuiltIn steps-executed: 1 - start: 2025-08-22T16:02:00.000000000Z - end: 2025-08-22T16:02:00.000001208Z - value: null - changes: {} - - name: "" - control-id: OSPS-AC-03 + start: 2025-08-22T16:02:00+00:00 + end: 2025-08-22T16:02:00+00:00 + - name: OSPS-AC-03 result: Passed message: Branch protection rule prevents deletions - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-AC-03 assessment-logs: - - requirement-id: OSPS-AC-03.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-AC-03.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-AC-03.01 applicability: - Maturity Level 1 - Maturity Level 2 @@ -62,11 +84,14 @@ evaluation-set: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.IsCodeRepo - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/access_control.branchProtectionRestrictsPushes steps-executed: 2 - start: 2025-08-22T16:02:00.000000000Z - end: 2025-08-22T16:02:00.000002750Z - value: null - changes: {} - - requirement-id: OSPS-AC-03.02 + start: 2025-08-22T16:02:00+00:00 + end: 2025-08-22T16:02:00+00:00 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-AC-03.02 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-AC-03.02 applicability: - Maturity Level 1 - Maturity Level 2 @@ -77,17 +102,21 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/access_control.branchProtectionPreventsDeletion steps-executed: 1 - start: 2025-08-22T16:02:00.000000000Z - end: 2025-08-22T16:02:00.000001167Z - value: null - changes: {} - - name: "" - control-id: OSPS-AC-04 + start: 2025-08-22T16:02:00+00:00 + end: 2025-08-22T16:02:00+00:00 + - name: OSPS-AC-04 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-AC-04 assessment-logs: - - requirement-id: OSPS-AC-04.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-AC-04.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-AC-04.01 applicability: - Maturity Level 2 - Maturity Level 3 @@ -97,16 +126,20 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/access_control.workflowDefaultReadPermissions steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-BR-01 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-BR-01 result: Needs Review message: Not implemented - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-BR-01 assessment-logs: - - requirement-id: OSPS-BR-01.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-BR-01.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-BR-01.01 applicability: - Maturity Level 1 - Maturity Level 2 @@ -118,11 +151,14 @@ evaluation-set: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.IsCodeRepo - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/build_release.cicdSanitizedInputParameters steps-executed: 2 - start: 2025-08-22T16:02:00.000000000Z - end: 2025-08-22T16:02:01.711621250Z - value: null - changes: {} - - requirement-id: OSPS-BR-01.02 + start: 2025-08-22T16:02:00+00:00 + end: 2025-08-22T16:02:01+00:00 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-BR-01.02 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-BR-01.02 applicability: - Maturity Level 1 - Maturity Level 2 @@ -133,17 +169,21 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.NotImplemented steps-executed: 1 - start: 2025-08-22T16:02:00.000000000Z - end: 2025-08-22T16:02:00.000000708Z - value: null - changes: {} - - name: "" - control-id: OSPS-BR-02 + start: 2025-08-22T16:02:00+00:00 + end: 2025-08-22T16:02:00+00:00 + - name: OSPS-BR-02 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-BR-02 assessment-logs: - - requirement-id: OSPS-BR-02.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-BR-02.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-BR-02.01 applicability: - Maturity Level 2 - Maturity Level 3 @@ -154,16 +194,20 @@ evaluation-set: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.HasMadeReleases - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/build_release.releaseHasUniqueIdentifier steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-BR-03 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-BR-03 result: Needs Review message: No official distribution points found in Security Insights data - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-BR-03 assessment-logs: - - requirement-id: OSPS-BR-03.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-BR-03.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-BR-03.01 applicability: - Maturity Level 1 - Maturity Level 2 @@ -175,11 +219,14 @@ evaluation-set: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.HasSecurityInsightsFile - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/build_release.ensureInsightsLinksUseHTTPS steps-executed: 2 - start: 2025-08-22T16:02:00.000000000Z - end: 2025-08-22T16:02:00.000003417Z - value: null - changes: {} - - requirement-id: OSPS-BR-03.02 + start: 2025-08-22T16:02:00+00:00 + end: 2025-08-22T16:02:00+00:00 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-BR-03.02 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-BR-03.02 applicability: - Maturity Level 1 - Maturity Level 2 @@ -190,17 +237,21 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/build_release.distributionPointsUseHTTPS steps-executed: 1 - start: 2025-08-22T16:02:00.000000000Z - end: 2025-08-22T16:02:00.000000584Z - value: null - changes: {} - - name: "" - control-id: OSPS-BR-04 + start: 2025-08-22T16:02:00+00:00 + end: 2025-08-22T16:02:00+00:00 + - name: OSPS-BR-04 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-BR-04 assessment-logs: - - requirement-id: OSPS-BR-04.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-BR-04.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-BR-04.01 applicability: - Maturity Level 2 - Maturity Level 3 @@ -211,16 +262,20 @@ evaluation-set: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.HasMadeReleases - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/build_release.ensureLatestReleaseHasChangelog steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-BR-05 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-BR-05 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-BR-05 assessment-logs: - - requirement-id: OSPS-BR-05.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-BR-05.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-BR-05.01 applicability: - Maturity Level 2 - Maturity Level 3 @@ -230,16 +285,20 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.NotImplemented steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-BR-06 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-BR-06 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-BR-06 assessment-logs: - - requirement-id: OSPS-BR-06.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-BR-06.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-BR-06.01 applicability: - Maturity Level 2 - Maturity Level 3 @@ -251,16 +310,20 @@ evaluation-set: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.HasSecurityInsightsFile - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/build_release.insightsHasSlsaAttestation steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-DO-01 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-DO-01 result: Failed message: User guide was NOT specified in Security Insights data - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-DO-01 assessment-logs: - - requirement-id: OSPS-DO-01.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-DO-01.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-DO-01.01 applicability: - Maturity Level 1 - Maturity Level 2 @@ -273,16 +336,20 @@ evaluation-set: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.HasSecurityInsightsFile - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/docs.hasUserGuides steps-executed: 3 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-DO-02 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-DO-02 result: Failed message: Repository does not accept vulnerability reports - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-DO-02 assessment-logs: - - requirement-id: OSPS-DO-02.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-DO-02.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-DO-02.01 applicability: - Maturity Level 1 - Maturity Level 2 @@ -295,16 +362,20 @@ evaluation-set: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.HasIssuesOrDiscussionsEnabled - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/docs.acceptsVulnReports steps-executed: 3 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-DO-03 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-DO-03 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-DO-03 assessment-logs: - - requirement-id: OSPS-DO-03.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-DO-03.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-DO-03.01 applicability: - Maturity Level 3 description: When the project has made a release, the project documentation MUST contain instructions to verify the integrity and authenticity of the release assets. @@ -315,16 +386,20 @@ evaluation-set: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.HasSecurityInsightsFile - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/docs.hasSignatureVerificationGuide steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-DO-04 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-DO-04 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-DO-04 assessment-logs: - - requirement-id: OSPS-DO-04.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-DO-04.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-DO-04.01 applicability: - Maturity Level 3 description: When the project has made a release, the project documentation MUST include a descriptive statement about the scope and duration of support for each release. @@ -333,16 +408,20 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/docs.hasSupportDocs steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-DO-05 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-DO-05 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-DO-05 assessment-logs: - - requirement-id: OSPS-DO-05.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-DO-05.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-DO-05.01 applicability: - Maturity Level 3 description: When the project has made a release, the project documentation MUST provide a descriptive statement when releases or versions will no longer receive security updates. @@ -351,16 +430,20 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/docs.hasSupportDocs steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-DO-06 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-DO-06 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-DO-06 assessment-logs: - - requirement-id: OSPS-DO-06.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-DO-06.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-DO-06.01 applicability: - Maturity Level 2 - Maturity Level 3 @@ -373,16 +456,20 @@ evaluation-set: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.HasSecurityInsightsFile - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/docs.hasDependencyManagementPolicy steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-GV-01 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-GV-01 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-GV-01 assessment-logs: - - requirement-id: OSPS-GV-01.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-GV-01.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-GV-01.01 applicability: - Maturity Level 2 - Maturity Level 3 @@ -395,10 +482,13 @@ evaluation-set: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/governance.coreTeamIsListed - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/governance.projectAdminsListed steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - requirement-id: OSPS-GV-01.02 + start: 2025-08-22T16:02:00+00:00 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-GV-01.02 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-GV-01.02 applicability: - Maturity Level 2 - Maturity Level 3 @@ -408,16 +498,20 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/governance.hasRolesAndResponsibilities steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-GV-02 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-GV-02 result: Passed message: Issues are enabled for the repository - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-GV-02 assessment-logs: - - requirement-id: OSPS-GV-02.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-GV-02.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-GV-02.01 applicability: - Maturity Level 1 - Maturity Level 2 @@ -428,32 +522,39 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.HasIssuesOrDiscussionsEnabled steps-executed: 1 - start: 2025-08-22T16:02:00.000000000Z - end: 2025-08-22T16:02:00.000000292Z - value: null - changes: {} - - name: "" - control-id: OSPS-GV-03 + start: 2025-08-22T16:02:00+00:00 + end: 2025-08-22T16:02:00+00:00 + - name: OSPS-GV-03 result: Needs Review - message: "Contributing guide was found via GitHub API (Recommendation: Add code of conduct location to Security Insights data)" - corrupted-state: false + message: 'Contributing guide was found via GitHub API (Recommendation: Add code of conduct location to Security Insights data)' + control: + reference-id: OSPS-B + entry-id: OSPS-GV-03 assessment-logs: - - requirement-id: OSPS-GV-03.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-GV-03.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-GV-03.01 applicability: - Maturity Level 1 - Maturity Level 2 - Maturity Level 3 description: While active, the project documentation MUST include an explanation of the contribution process. result: Needs Review - message: "Contributing guide was found via GitHub API (Recommendation: Add code of conduct location to Security Insights data)" + message: 'Contributing guide was found via GitHub API (Recommendation: Add code of conduct location to Security Insights data)' steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/governance.hasContributionGuide steps-executed: 1 - start: 2025-08-22T16:02:00.000000000Z - end: 2025-08-22T16:02:00.000000792Z - value: null - changes: {} - - requirement-id: OSPS-GV-03.02 + start: 2025-08-22T16:02:00+00:00 + end: 2025-08-22T16:02:00+00:00 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-GV-03.02 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-GV-03.02 applicability: - Maturity Level 2 - Maturity Level 3 @@ -466,16 +567,20 @@ evaluation-set: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.IsActive - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/governance.hasContributionReviewPolicy steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-GV-04 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-GV-04 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-GV-04 assessment-logs: - - requirement-id: OSPS-GV-04.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-GV-04.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-GV-04.01 applicability: - Maturity Level 3 description: While active, the project documentation MUST have a policy that code contributors are reviewed prior to granting escalated permissions to sensitive resources. @@ -484,16 +589,20 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.NotImplemented steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-LE-01 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-LE-01 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-LE-01 assessment-logs: - - requirement-id: OSPS-LE-01.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-LE-01.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-LE-01.01 applicability: - Maturity Level 2 - Maturity Level 3 @@ -503,16 +612,20 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.GithubTermsOfService steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-LE-02 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-LE-02 result: Needs Review message: All license found are OSI or FSF approved - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-LE-02 assessment-logs: - - requirement-id: OSPS-LE-02.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-LE-02.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-LE-02.01 applicability: - Maturity Level 1 - Maturity Level 2 @@ -524,17 +637,21 @@ evaluation-set: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/legal.foundLicense - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/legal.goodLicense steps-executed: 2 - start: 2025-08-22T16:02:00.000000000Z - end: 2025-08-22T16:02:00.269504834Z - value: null - changes: {} - - name: "" - control-id: OSPS-LE-03 + start: 2025-08-22T16:02:00+00:00 + end: 2025-08-22T16:02:00+00:00 + - name: OSPS-LE-03 result: Passed message: GitHub releases include the license(s) in the released source code. - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-LE-03 assessment-logs: - - requirement-id: OSPS-LE-03.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-LE-03.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-LE-03.01 applicability: - Maturity Level 1 - Maturity Level 2 @@ -545,11 +662,14 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/legal.foundLicense steps-executed: 1 - start: 2025-08-22T16:02:00.000000000Z - end: 2025-08-22T16:02:00.000000875Z - value: null - changes: {} - - requirement-id: OSPS-LE-03.02 + start: 2025-08-22T16:02:00+00:00 + end: 2025-08-22T16:02:00+00:00 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-LE-03.02 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-LE-03.02 applicability: - Maturity Level 1 - Maturity Level 2 @@ -560,17 +680,21 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/legal.releasesLicensed steps-executed: 1 - start: 2025-08-22T16:02:00.000000000Z - end: 2025-08-22T16:02:00.000000375Z - value: null - changes: {} - - name: "" - control-id: OSPS-QA-01 + start: 2025-08-22T16:02:00+00:00 + end: 2025-08-22T16:02:00+00:00 + - name: OSPS-QA-01 result: Passed message: This control is enforced by GitHub for all projects - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-QA-01 assessment-logs: - - requirement-id: OSPS-QA-01.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-QA-01.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-QA-01.01 applicability: - Maturity Level 1 - Maturity Level 2 @@ -581,11 +705,14 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/quality.repoIsPublic steps-executed: 1 - start: 2025-08-22T16:02:00.000000000Z - end: 2025-08-22T16:02:00.000000958Z - value: null - changes: {} - - requirement-id: OSPS-QA-01.02 + start: 2025-08-22T16:02:00+00:00 + end: 2025-08-22T16:02:00+00:00 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-QA-01.02 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-QA-01.02 applicability: - Maturity Level 1 - Maturity Level 2 @@ -596,17 +723,21 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.GithubBuiltIn steps-executed: 1 - start: 2025-08-22T16:02:00.000000000Z - end: 2025-08-22T16:02:00.000000375Z - value: null - changes: {} - - name: "" - control-id: OSPS-QA-02 + start: 2025-08-22T16:02:00+00:00 + end: 2025-08-22T16:02:00+00:00 + - name: OSPS-QA-02 result: Passed message: Found 8 dependency manifests from GitHub API - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-QA-02 assessment-logs: - - requirement-id: OSPS-QA-02.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-QA-02.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-QA-02.01 applicability: - Maturity Level 1 - Maturity Level 2 @@ -617,11 +748,14 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/quality.verifyDependencyManagement steps-executed: 1 - start: 2025-08-22T16:02:00.000000000Z - end: 2025-08-22T16:02:00.000002667Z - value: null - changes: {} - - requirement-id: OSPS-QA-02.02 + start: 2025-08-22T16:02:00+00:00 + end: 2025-08-22T16:02:00+00:00 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-QA-02.02 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-QA-02.02 applicability: - Maturity Level 3 description: When the project has made a release, all compiled released software assets MUST be delivered with a software bill of materials. @@ -630,16 +764,20 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.NotImplemented steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-QA-03 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-QA-03 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-QA-03 assessment-logs: - - requirement-id: OSPS-QA-03.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-QA-03.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-QA-03.01 applicability: - Maturity Level 2 - Maturity Level 3 @@ -650,16 +788,20 @@ evaluation-set: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/quality.statusChecksAreRequiredByRulesets - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/quality.statusChecksAreRequiredByBranchProtection steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-QA-04 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-QA-04 result: Failed message: Insights does NOT contains a list of repositories - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-QA-04 assessment-logs: - - requirement-id: OSPS-QA-04.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-QA-04.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-QA-04.01 applicability: - Maturity Level 1 - Maturity Level 2 @@ -673,16 +815,20 @@ evaluation-set: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.IsActive - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/quality.insightsListsRepositories steps-executed: 4 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-QA-05 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-QA-05 result: Passed message: No common binary file extensions were found in the repository - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-QA-05 assessment-logs: - - requirement-id: OSPS-QA-05.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-QA-05.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-QA-05.01 applicability: - Maturity Level 1 - Maturity Level 2 @@ -693,17 +839,21 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/quality.noBinariesInRepo steps-executed: 1 - start: 2025-08-22T16:02:00.000000000Z - end: 2025-08-22T16:02:00.729000709Z - value: null - changes: {} - - name: "" - control-id: OSPS-QA-06 + start: 2025-08-22T16:02:00+00:00 + end: 2025-08-22T16:02:00+00:00 + - name: OSPS-QA-06 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-QA-06 assessment-logs: - - requirement-id: OSPS-QA-06.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-QA-06.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-QA-06.01 applicability: - Maturity Level 2 - Maturity Level 3 @@ -714,10 +864,13 @@ evaluation-set: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.IsCodeRepo - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/quality.hasOneOrMoreStatusChecks steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - requirement-id: OSPS-QA-06.02 + start: 2025-08-22T16:02:00+00:00 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-QA-06.02 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-QA-06.02 applicability: - Maturity Level 3 description: While active, project's documentation MUST clearly document when and how tests are run. @@ -726,10 +879,13 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/quality.documentsTestExecution steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - requirement-id: OSPS-QA-06.03 + start: 2025-08-22T16:02:00+00:00 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-QA-06.03 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-QA-06.03 applicability: - Maturity Level 3 description: While active, the project's documentation MUST include a policy that all major changes to the software produced by the project should add or update tests of the functionality in an automated test suite. @@ -739,16 +895,20 @@ evaluation-set: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.IsCodeRepo - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/quality.documentsTestMaintenancePolicy steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-QA-07 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-QA-07 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-QA-07 assessment-logs: - - requirement-id: OSPS-QA-07.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-QA-07.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-QA-07.01 applicability: - Maturity Level 3 description: When a commit is made to the primary branch, the project's version control system MUST require at least one non-author approval of the changes before merging. @@ -757,16 +917,20 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/quality.requiresNonAuthorApproval steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-SA-01 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-SA-01 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-SA-01 assessment-logs: - - requirement-id: OSPS-SA-01.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-SA-01.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-SA-01.01 applicability: - Maturity Level 2 - Maturity Level 3 @@ -776,16 +940,20 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.NotImplemented steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-SA-02 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-SA-02 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-SA-02 assessment-logs: - - requirement-id: OSPS-SA-02.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-SA-02.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-SA-02.01 applicability: - Maturity Level 2 - Maturity Level 3 @@ -793,18 +961,22 @@ evaluation-set: result: Not Run message: "" steps: - - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.NotImplemented + - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.NotImplemented steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-SA-03 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-SA-03 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-SA-03 assessment-logs: - - requirement-id: OSPS-SA-03.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-SA-03.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-SA-03.01 applicability: - Maturity Level 2 - Maturity Level 3 @@ -814,10 +986,13 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.NotImplemented steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - requirement-id: OSPS-SA-03.02 + start: 2025-08-22T16:02:00+00:00 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-SA-03.02 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-SA-03.02 applicability: - Maturity Level 3 description: When the project has made a release, the project MUST perform a threat modeling and attack surface analysis to understand and protect against attacks on critical code paths, functions, and interactions within the system. @@ -826,16 +1001,20 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.NotImplemented steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-VM-01 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-VM-01 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-VM-01 assessment-logs: - - requirement-id: OSPS-VM-01.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-VM-01.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-VM-01.01 applicability: - Maturity Level 2 - Maturity Level 3 @@ -843,18 +1022,22 @@ evaluation-set: result: Not Run message: "" steps: - - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.NotImplemented + - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.NotImplemented steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-VM-02 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-VM-02 result: Failed message: Security contacts were not specified in Security Insights data - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-VM-02 assessment-logs: - - requirement-id: OSPS-VM-02.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-VM-02.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-VM-02.01 applicability: - Maturity Level 1 description: While active, the project documentation MUST contain security contacts. @@ -864,16 +1047,20 @@ evaluation-set: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.IsCodeRepo - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/vuln_management.hasSecContact steps-executed: 2 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-VM-03 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-VM-03 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-VM-03 assessment-logs: - - requirement-id: OSPS-VM-03.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-VM-03.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-VM-03.01 applicability: - Maturity Level 2 - Maturity Level 3 @@ -883,16 +1070,20 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.NotImplemented steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-VM-04 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-VM-04 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-VM-04 assessment-logs: - - requirement-id: OSPS-VM-04.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-VM-04.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-VM-04.01 applicability: - Maturity Level 2 - Maturity Level 3 @@ -902,10 +1093,13 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.NotImplemented steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - requirement-id: OSPS-VM-04.02 + start: 2025-08-22T16:02:00+00:00 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-VM-04.02 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-VM-04.02 applicability: - Maturity Level 3 description: While active, any vulnerabilities in the software components not affecting the project MUST be accounted for in a VEX document, augmenting the vulnerability report with non-exploitability details. @@ -914,16 +1108,20 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.NotImplemented steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-VM-05 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-VM-05 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-VM-05 assessment-logs: - - requirement-id: OSPS-VM-05.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-VM-05.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-VM-05.01 applicability: - Maturity Level 3 description: While active, the project documentation MUST include a policy that defines a threshold for remediation of SCA findings related to vulnerabilities and licenses. @@ -932,10 +1130,13 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.NotImplemented steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - requirement-id: OSPS-VM-05.02 + start: 2025-08-22T16:02:00+00:00 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-VM-05.02 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-VM-05.02 applicability: - Maturity Level 3 description: While active, the project documentation MUST include a policy to address SCA violations prior to any release. @@ -944,10 +1145,13 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.NotImplemented steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - requirement-id: OSPS-VM-05.03 + start: 2025-08-22T16:02:00+00:00 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-VM-05.03 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-VM-05.03 applicability: - Maturity Level 3 description: While active, all changes to the project's codebase MUST be automatically evaluated against a documented policy for malicious dependencies and known vulnerabilities in dependencies, then blocked in the event of violations, except when declared and suppressed as non-exploitable. @@ -956,16 +1160,20 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.NotImplemented steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - name: "" - control-id: OSPS-VM-06 + start: 2025-08-22T16:02:00+00:00 + - name: OSPS-VM-06 result: Not Run message: "" - corrupted-state: false + control: + reference-id: OSPS-B + entry-id: OSPS-VM-06 assessment-logs: - - requirement-id: OSPS-VM-06.01 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-VM-06.01 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-VM-06.01 applicability: - Maturity Level 3 description: While active, the project documentation MUST include a policy that defines a threshold for remediation of SAST findings. @@ -974,10 +1182,13 @@ evaluation-set: steps: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.HasDependencyManagementPolicy steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} - - requirement-id: OSPS-VM-06.02 + start: 2025-08-22T16:02:00+00:00 + - requirement: + reference-id: OSPS-B + entry-id: OSPS-VM-06.02 + procedure: + reference-id: OSPS-B + entry-id: PROC-OSPS-VM-06.02 applicability: - Maturity Level 3 description: While active, all changes to the project's codebase MUST be automatically evaluated against a documented policy for security weaknesses and blocked in the event of violations except when declared and suppressed as non-exploitable. @@ -988,6 +1199,4 @@ evaluation-set: - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.HasSecurityInsightsFile - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/vuln_management.sastToolDefined steps-executed: 0 - start: 2025-08-22T16:02:00.000000000Z - value: null - changes: {} \ No newline at end of file + start: 2025-08-22T16:02:00+00:00 diff --git a/layer4/to_sarif_test.go b/layer4/to_sarif_test.go index fa6fdf9..9576a5e 100644 --- a/layer4/to_sarif_test.go +++ b/layer4/to_sarif_test.go @@ -29,7 +29,7 @@ func TestToSARIF(t *testing.T) { name: "basic conversion with multiple results", artifactURI: "", catalog: nil, - evaluationLog: makeEvaluationLog(Author{ + evaluationLog: makeEvaluationLog(Actor{ Name: "gemara", Uri: "https://github.com/ossf/gemara", Version: "1.0.0", @@ -58,7 +58,7 @@ func TestToSARIF(t *testing.T) { name: "with artifactURI parameter", artifactURI: "README.md", catalog: nil, - evaluationLog: makeEvaluationLog(Author{ + evaluationLog: makeEvaluationLog(Actor{ Name: "gemara", Uri: "https://github.com/test/repo", Version: "1.0.0", @@ -83,7 +83,7 @@ func TestToSARIF(t *testing.T) { name: "empty author URI", artifactURI: "", catalog: nil, - evaluationLog: makeEvaluationLog(Author{ + evaluationLog: makeEvaluationLog(Actor{ Name: "gemara", Uri: "", Version: "1.0.0", @@ -108,7 +108,7 @@ func TestToSARIF(t *testing.T) { name: "with catalog enrichment", artifactURI: "README.md", catalog: testCatalog, - evaluationLog: makeEvaluationLog(Author{ + evaluationLog: makeEvaluationLog(Actor{ Name: "test-tool", Uri: "https://github.com/test/tool", Version: "1.0.0", @@ -147,7 +147,7 @@ func TestToSARIF(t *testing.T) { name: "without catalog", artifactURI: "README.md", catalog: nil, - evaluationLog: makeEvaluationLog(Author{ + evaluationLog: makeEvaluationLog(Actor{ Name: "test-tool", Uri: "https://github.com/test/tool", Version: "1.0.0", @@ -182,7 +182,7 @@ func TestToSARIF(t *testing.T) { name: "catalog recommendation when assessment log has none", artifactURI: "README.md", catalog: testCatalog, - evaluationLog: makeEvaluationLog(Author{ + evaluationLog: makeEvaluationLog(Actor{ Name: "test-tool", Uri: "https://github.com/test/tool", Version: "1.0.0", @@ -267,7 +267,7 @@ func TestToSARIF_ResultLevels(t *testing.T) { for _, tt := range tests { t.Run(tt.result.String(), func(t *testing.T) { - evaluationLog := makeEvaluationLog(Author{ + evaluationLog := makeEvaluationLog(Actor{ Name: "test", Uri: "https://test", Version: "1.0.0", @@ -290,8 +290,26 @@ func TestToSARIF_ResultLevels(t *testing.T) { // Helper functions -func makeEvaluationLog(author Author, logs []*AssessmentLog) EvaluationLog { +func makeEvaluationLog(evaluator Actor, logs []*AssessmentLog) EvaluationLog { + // Convert evaluator to author (evaluator is the author of the log) + // The author.id references the evaluator from the plan return EvaluationLog{ + Metadata: Metadata{ + Id: "test-evaluation-log", + Author: Actor{ + Id: evaluator.Id, + Name: evaluator.Name, + Uri: evaluator.Uri, + Version: evaluator.Version, + }, + MappingReferences: []MappingReference{ + { + Id: "TEST-REF", + Title: "Test Reference", + Version: "1.0", + }, + }, + }, Evaluations: []*ControlEvaluation{ { Name: "Example Control", @@ -300,7 +318,6 @@ func makeEvaluationLog(author Author, logs []*AssessmentLog) EvaluationLog { AssessmentLogs: logs, }, }, - Metadata: Metadata{Author: author}, } } diff --git a/schemas/layer-4.cue b/schemas/layer-4.cue index 7dd0469..a138de8 100644 --- a/schemas/layer-4.cue +++ b/schemas/layer-4.cue @@ -4,23 +4,33 @@ import "time" @go(layer4) +// EvaluationDocument defines how a set of Layer 2 controls are to be evaluated and the associated outcomes of the evaluation. +#EvaluationDocument: { + metadata: #Metadata + "evaluation-plan"?: #EvaluationPlan @go(EvaluationPlan,optional=nillable) + "evaluation-logs"?: [...#EvaluationLog] @go(EvaluationLogs) +} + // EvaluationPlan defines how a set of Layer 2 controls are to be evaluated. #EvaluationPlan: { metadata: #Metadata + // Evaluators defines the assessment evaluators that can be used to execute assessment procedures. + evaluators: [...#Actor] @go(Evaluators) plans: [...#AssessmentPlan] } // EvaluationLog contains the results of evaluating a set of Layer 2 controls. #EvaluationLog: { + "metadata"?: #Metadata "evaluations": [#ControlEvaluation, ...#ControlEvaluation] @go(Evaluations,type=[]*ControlEvaluation) - "metadata"?: #Metadata @go(Metadata) } -// Metadata contains metadata about the Layer 4 evaluation plan and log. +// Metadata contains common fields shared across all metadata types. #Metadata: { - id: string - version?: string - author: #Author + id: string + version?: string + author: #Actor + description?: string "mapping-references"?: [...#MappingReference] @go(MappingReferences) @yaml("mapping-references,omitempty") } @@ -43,14 +53,27 @@ import "time" remarks?: string } -// Author contains the information about the entity that produced the evaluation plan or log. -#Author: { - "name": string - "uri"?: string - "version"?: string - "contact"?: #Contact @go(Contact) +// Actor represents an entity (human or tool) that can perform actions in evaluations. +#Actor: { + // Id uniquely identifies the actor. + id: string + // Name provides the name of the actor. + name: string + // Type specifies how the evaluation is executed (automated tool or manual/human evaluator). + type: #ExecutionType @go(Type) + // Version specifies the version of the actor (if applicable, e.g., for tools). + version?: string + // Description provides additional context about the actor. + description?: string + // Uri provides a general URI for the actor information. + uri?: =~"^https?://[^\\s]+$" + // Contact provides contact information for the actor. + contact?: #Contact @go(Contact) } +// ExecutionType specifies how the evaluation is executed (automated or manual). +#ExecutionType: "Automated" | "Manual" @go(-) + // ControlEvaluation contains the results of evaluating a single Layer 4 control. #ControlEvaluation: { name: string @@ -89,6 +112,8 @@ import "time" end?: #Datetime // Recommendation provides guidance on how to address a failed assessment. recommendation?: string + // ConfidenceLevel indicates the evaluator's confidence level in this specific assessment result. + "confidence-level"?: #ConfidenceLevel @go(ConfidenceLevel) } #AssessmentStep: string @go(-) @@ -97,6 +122,9 @@ import "time" #Datetime: time.Format("2006-01-02T15:04:05Z07:00") @go(Datetime) +// ConfidenceLevel indicates the evaluator's confidence level in an assessment result. +#ConfidenceLevel: "Undetermined" | "Low" | "Medium" | "High" @go(-) + // AssessmentPlan defines all testing procedures for a control id. #AssessmentPlan: { // Control points to the Layer 2 control being evaluated. @@ -128,8 +156,34 @@ import "time" description: string // Documentation provides a URL to documentation that describes how the assessment procedure evaluates the control requirement documentation?: =~"^https?://[^\\s]+$" + // Evaluators lists which assessment evaluators can execute this procedure. + evaluators: [...#EvaluatorMapping] @go(Evaluators) + // Strategy defines the rules for aggregating results from multiple evaluators running the same procedure. + strategy?: #Strategy @go(Strategy,optional=nillable) +} + +// EvaluatorMapping maps an assessment evaluator to a procedure. +#EvaluatorMapping: { + id: string + // Authoritative determines how this evaluator participates in conflict resolution when using AuthoritativeConfirmation strategy. + // If true, the evaluator can trigger findings independently. If false (default), the evaluator requires confirmation from authoritative evaluators before triggering findings. + // Note: This field is only used with AuthoritativeConfirmation strategy. With Strict (default) or ManualOverride strategies, this field is ignored. + authoritative?: bool @go(Authoritative) + // Remarks provides context about why this evaluator-procedure combination was chosen. + remarks?: string } +// Strategy defines the rules for resolving conflicts between multiple evaluator results. +#Strategy: { + // ConflictRuleType specifies the aggregation logic used to resolve conflicts when multiple evaluators provide results for the same assessment procedure. + "conflict-rule-type": #ConflictRuleType @go(ConflictRuleType) + // Remarks provides context for why this specific conflict resolution strategy was chosen. + remarks?: string +} + +// ConflictRuleType defines how to resolve conflicts when multiple evaluators produce different results. +#ConflictRuleType: "Strict" | "ManualOverride" | "AuthoritativeConfirmation" @go(-) + #Contact: { // The contact person's name. name: string diff --git a/spec/README.md b/spec/README.md new file mode 100644 index 0000000..9e64982 --- /dev/null +++ b/spec/README.md @@ -0,0 +1,11 @@ +# Specifications + +This directory contains specifications for Gemara. + +## Specifications + +- **[Finding Determination Specification](finding-determination.md)** - Defines how assessment results are aggregated and how conflicts between multiple evaluators are resolved. Covers result types, conflict resolution strategies, and use cases for GRC Engineering workflows. + +## Overview + +The specifications in this directory describe the machine-readable formats and behavioral rules for the Gemara model layers. \ No newline at end of file diff --git a/spec/finding-determination.md b/spec/finding-determination.md new file mode 100644 index 0000000..6aac38a --- /dev/null +++ b/spec/finding-determination.md @@ -0,0 +1,241 @@ +# Finding Determination Specification + + +* [Finding Determination Specification](#finding-determination-specification) + * [Abstract](#abstract) + * [Overview](#overview) + * [Notations and Terminology](#notations-and-terminology) + * [Notational Conventions](#notational-conventions) + * [Terminology](#terminology) + * [Finding](#finding) + * [Evaluator](#evaluator) + * [Conflict Resolution Strategy](#conflict-resolution-strategy) + * [Authoritative Evaluator](#authoritative-evaluator) + * [Result Types](#result-types) + * [Conflict Resolution Strategies](#conflict-resolution-strategies) + * [Strict Strategy](#strict-strategy) + * [ManualOverride Strategy](#manualoverride-strategy) + * [AuthoritativeConfirmation Strategy](#authoritativeconfirmation-strategy) + * [Use Cases](#use-cases) + * [Strategy Selection Anti-Patterns](#strategy-selection-anti-patterns) + + +## Abstract + +This specification defines how findings are determined when multiple assessment evaluators and procedures provide results for the same assessment requirement in Layer 4 Evaluation Plans. +It specifies conflict resolution strategies and result type semantics to ensure consistent finding determination across implementations. + +## Overview + +Layer 4 Evaluation Plans support multiple assessment evaluators running assessment procedures to evaluate control requirements. +When multiple evaluators run the same procedure, or multiple procedures evaluate the same requirement, conflict resolution strategies determine how their results are combined to determine if there is a finding. +This specification provides a formal definition of these strategies to ensure consistent and predictable behavior across implementations. + +## Notations and Terminology + +### Notational Conventions + +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC 2119](https://tools.ietf.org/html/rfc2119). + +### Terminology + +This specification defines the following terms: + +#### Finding + +A finding is a documented observation that a Layer 2 control requirement, as referenced by a Layer 3 policy, is not being met or is not implemented correctly. + +A finding is determined when one or more evaluator results indicate non-compliance with the assessed control requirement. Only results of `Failed`, `Unknown`, or `NeedsReview` constitute findings; +`Passed` and `NotApplicable` results indicate compliance or inapplicability and do not produce findings. +Findings **MUST** be supported by evidence from assessment logs that document the evaluation results, and serve as the basis for Layer 5 enforcement actions and remediation activities. + +#### Evaluator + +An evaluator is a tool, process, or person that executes an assessment procedure and produces a result. + +#### Conflict Resolution Strategy + +A conflict resolution strategy is an algorithm that determines how multiple evaluator results are combined to produce a single finding determination. + +#### Authoritative Evaluator + +Evaluator authoritativeness determines how an evaluator participates in conflict resolution when using the `AuthoritativeConfirmation` strategy. +Evaluators can be marked as `authoritative` (can trigger findings independently) or `non-authoritative` (requires confirmation from authoritative evaluators to trigger findings). + +The distinction between authoritative and non-authoritative evaluators maps directly to enforcement readiness: + +- **Authoritative evaluators** represent procedures that are ready for enforcement. When authoritative evaluators report failures, findings are determined and can trigger enforcement actions (blocking, remediation, alerts). +- **Non-authoritative evaluators** represent procedures that collect risk data but are not ready to trigger enforcement. Non-authoritative evaluators provide risk visibility and inform decision-making, but their failures do not independently trigger enforcement actions. + +**Default Behavior**: If the `authoritative` field is not explicitly set, evaluators default to `authoritative: false` (non-authoritative). +However, since the default conflict resolution strategy is `Strict`, this default has no effect on finding determination - `Strict` ignores the `authoritative` field and treats all evaluators equally. + +## Result Types + +The following result types are used in Layer 4: + +- **NotRun**: The assessment was not executed +- **Passed**: The assessment passed successfully +- **Failed**: The assessment failed +- **NeedsReview**: The assessment requires manual review +- **NotApplicable**: The assessment is not applicable to the current context +- **Unknown**: The assessment result is unknown or indeterminate + +## Conflict Resolution Strategies + +For aggregating results within a single log (e.g., multiple steps within an assessment), implementations MUST use severity-based determination, where the most severe result takes precedence according to the hierarchy: `Failed > Unknown > NeedsReview > Passed > NotApplicable`. +If all results are `NotRun`, no finding **MUST** be determined. This severity-based aggregation strategy **MUST** be used if a finding determination is inconclusive using other strategies. + +Three multi-source conflict resolution strategies are defined in this specification. Implementations **MUST** support all strategies. + +### Strict Strategy + +The Strict strategy determines that a finding exists if ANY evaluator reports a failure, regardless of other evaluator results. This strategy provides zero tolerance for failures and is the simplest conflict resolution approach. + +**Security-First Design**: `Strict` applies uniform zero-tolerance logic to all non-passing results (`Failed`, `Unknown`, `NeedsReview`). This makes `Strict` ideal for organizations that want predictable, consistent behavior and absolute zero-tolerance for security violations, ensuring that any evaluator reporting a problem triggers a finding. + +**Process**: When using the Strict strategy, a finding **MUST** be determined according to the following priority order: + +1. If ANY evaluator reports `Failed`, then **Finding exists** (Failed) +2. Else if ANY evaluator reports `Unknown`, then **Finding exists** (Unknown) +3. Else if ANY evaluator reports `NeedsReview`, then **Finding exists** (NeedsReview) +4. Else if ALL evaluator results are `Passed`, then **No finding** (Passed) +5. Else if ALL evaluator results are `NotApplicable`, then **No finding** (NotApplicable) + +Evaluators with `NotRun` results MUST be excluded from the determination process. All evaluators are treated equally when using the Strict strategy. + +### ManualOverride Strategy + +The ManualOverride strategy gives precedence to manual review evaluators over automated evaluators when determining findings from conflicting results. + +**Process**: When using the ManualOverride strategy: + +1. Separate results into manual and automated evaluator results based on the evaluator's ExecutionType (`Manual` vs `Automated`). +2. If manual evaluators exist: + - If any manual evaluator reports `Failed`: **Finding exists** (Failed) + - Else if any manual evaluator reports `Unknown`: **Finding exists** (Unknown) + - Else if any manual evaluator reports `NeedsReview`: **Finding exists** (NeedsReview) + - Else if all manual evaluators report `Passed`: **No finding** (Passed) + - Else: **No finding** (NotApplicable) + +Evaluators with `NotRun` results **MUST** be excluded from the determination process. + +### AuthoritativeConfirmation Strategy + +The AuthoritativeConfirmation strategy treats non-authoritative evaluators as requiring confirmation from authoritative evaluators before triggering findings. + +**Process**: When using the AuthoritativeConfirmation strategy: + +1. Separate evaluators into authoritative and non-authoritative groups based on their `authoritative` field. Evaluators without an explicit `authoritative` field default to `authoritative: false` (non-authoritative). +2. Authoritative evaluators trigger findings using Strict logic: + - If any authoritative evaluator reports `Failed`: **Finding exists** (Failed) + - Else if any authoritative evaluator reports `Unknown`: **Finding exists** (Unknown) + - Else if any authoritative evaluator reports `NeedsReview`: **Finding exists** (NeedsReview) + - Else if all authoritative evaluators report `Passed`: Continue to step 3 +3. Non-authoritative evaluators require confirmation: + - If only non-authoritative evaluators report `Failed`: **No finding** (non-authoritative cannot trigger alone) + - If non-authoritative evaluator reports `Failed` AND authoritative evaluator reports `Passed`: **No finding** (contradicted) + - If non-authoritative evaluator reports `Failed` AND authoritative evaluator reports `Failed`: **Finding exists** (confirmed) + - If non-authoritative evaluator reports `Failed` AND authoritative evaluator reports `Unknown` or `NeedsReview`: **Finding exists** (escalated for investigation) + - If all evaluators (authoritative and non-authoritative) report `Passed`: **No finding** (Passed) + - If all evaluators report `NotApplicable`: **No finding** (NotApplicable) + +Evaluators with `NotRun` results MUST be excluded from the determination process. + +**Key Behaviors**: + +- **Authoritative evaluators** can trigger findings independently using zero-tolerance logic. If any authoritative evaluator reports a non-passing result, a finding is immediately determined, regardless of non-authoritative evaluator results. +- **Non-authoritative evaluators** can only trigger findings when: + - They confirm an authoritative evaluator's failure (both report `Failed`) + - They escalate an unclear authoritative result (authoritative reports `Unknown`/`NeedsReview` and non-authoritative reports `Failed`) +- **Non-authoritative evaluators cannot**: + - Trigger findings independently (if only non-authoritative evaluators report failures, no finding is determined) + - Override authoritative evaluators (if authoritative evaluators report `Passed`, non-authoritative failures are ignored) + +## Use Cases + +Layer 4 Evaluation Plans serve as the foundation for Layer 5 enforcement decisions. The evaluation results inform what enforcement actions should be taken, such as blocking deployments, triggering remediation, or generating alerts. +However, not all procedures in an Evaluation Plan need to trigger enforcement actions. + +Use the following decision matrix to select the appropriate conflict resolution strategy: + +| Scenario | Recommended Strategy | Rationale | +|----------------------------------------------------------------------|-----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------| +| **All evaluators are equally trusted and validated** | `Strict` | Zero tolerance for failures. Any evaluator reporting a problem triggers a finding. Simplest and most predictable behavior. | +| **Multiple evaluators, need human judgment for final determination** | `ManualOverride` | Manual review takes precedence. Automated tools provide initial screening, but human reviewers make final decisions. | +| **Gradual rollout of new enforcement** | `AuthoritativeConfirmation` | Start with evaluators as non-authoritative to collect baseline data. Promote to authoritative once validated. | +| **Experimental or unvalidated evaluators** | `AuthoritativeConfirmation` | Mark experimental evaluators as non-authoritative. They provide visibility but don't trigger enforcement until confirmed by authoritative evaluators. | +| **Simple, predictable behavior needed** | `Strict` | No complex logic. Any failure = finding. Easiest to understand and maintain. | +| **Automated tools need human verification** | `ManualOverride` | Automated tools flag issues, but require manual confirmation before triggering findings. | +| **Collecting metrics before enforcing** | `AuthoritativeConfirmation` | Run evaluators non-authoritatively to understand violation patterns, then promote to authoritative for enforcement. | + +**Quick Decision Guide:** + +``` +Do you need human judgment for final decisions? +├─ YES → Use ManualOverride +└─ NO → Do you need to distinguish enforcement-ready from experimental evaluators? + ├─ YES → Use AuthoritativeConfirmation + └─ NO → Use Strict (default) +``` + +A typical workflow for introducing promoting procedures from non-authoritative to authoritative: + +1. **Add to Evaluation Plan (Non-Authoritative)** + - Add new procedure with evaluators (default is `authoritative: false`, so no explicit setting needed) + - Use `AuthoritativeConfirmation` strategy to enable authoritative/non-authoritative behavior + - Run evaluations to collect baseline data + - Understand violation patterns and impact + - Validate that the evaluator produces accurate results + +2. **Assess and Remediate** + - Analyze non-authoritative evaluator results + - Fix critical violations discovered during baseline collection + - Validate procedure accuracy and false positive rates + - Ensure evaluator is ready for enforcement + +3. **Promote to Authoritative** + - Change evaluator `authoritative` field to `true` (explicit opt-in required) + - Now triggers findings and enforcement actions independently + - Monitor enforcement impact + - Verify that enforcement actions are appropriate + +4. **Maintain Some Procedures as Non-Authoritative** + - Keep informational/audit-only procedures as non-authoritative (default `authoritative: false`) + - Maintain experimental procedures as non-authoritative until validated + - Use non-authoritative for risk visibility without enforcement + - With `AuthoritativeConfirmation` strategy, non-authoritative evaluators require confirmation + +**Example: Understanding Defaults** + +```yaml +# Example 1: Default behavior (security-first) +procedures: + - id: check-branch-protection + evaluators: + - id: security-scanner + # No authoritative field = defaults to false (but ignored by Strict) + # No strategy = defaults to Strict + # Result: Any failure from security-scanner triggers a finding (Strict ignores authoritative) +``` + +```yaml +# Example 2: Explicit non-authoritative (opt-out) +procedures: + - id: experimental-check + evaluators: + - id: new-tool + authoritative: false # Explicitly opt-out + strategy: + conflict-rule-type: AuthoritativeConfirmation + # Result: new-tool provides visibility but doesn't trigger findings alone +``` + +### Strategy Selection Anti-Patterns + +**Avoid these patterns:** + +- ❌ **Using `AuthoritativeConfirmation` with all evaluators as non-authoritative** - No findings will ever be triggered. At least one authoritative evaluator is required. +- ❌ **Using `ManualOverride` when all evaluators have ExecutionType `Automated`** - Falls back to `Strict`, defeating the purpose of manual override. At least one evaluator with ExecutionType `Manual` is required for ManualOverride to function as intended. +- ❌ **Mixing strategies inconsistently** - Use the same strategy at both Assessment and Procedure levels unless there's a clear reason for different behavior. +- ❌ **Setting `authoritative: false` without using `AuthoritativeConfirmation`** - The field is ignored by other strategies, which can be confusing.