feat: ttl-delete audit log entries (#7450)

<!-- Describe what has changed in this PR -->
**What changed?**
Adds domain-id based TTL for domain-audit log entries, since automated
testing domains are fairly noisy and probably of fairly limited value
beyond a short period of time.


<!-- Tell your future self why have you made these changes -->
**Why?**


<!-- How have you verified this change? Tested locally? Added a unit
test? Checked in staging env? -->
**How did you test it?**


<!-- Assuming the worst case, what can be broken when deploying this
change to production? -->
**Potential risks**

<!-- Is it notable for release? e.g. schema updates, configuration or
data migration required? If so, please mention it, and also update
CHANGELOG.md -->
**Release notes**

<!-- Is there any documentation updates should be made for config,
https://cadenceworkflow.io/docs/operation-guide/setup/ ? If so, please
open an PR in https://github.com/cadence-workflow/cadence-docs -->
**Documentation Changes**

---------

Signed-off-by: David Porter <[email protected]>

Author: davidporter-id-au

common/dynamicconfig/dynamicproperties/constants.go

@@ -2961,6 +2961,13 @@ const (
 	// Allowed filters: domainName, taskListName, taskListType
 	TaskIsolationPollerWindow
 
+	// DomainAuditLogTTL is the TTL for domain audit log entries
+	// KeyName: system.domainAuditLogTTL
+	// Value type: Duration
+	// Default value: 365 days (1 year)
+	// Allowed filters: DomainID
+	DomainAuditLogTTL
+
 	// LastDurationKey must be the last one in this const group
 	LastDurationKey
 )
@@ -5491,6 +5498,12 @@ var DurationKeys = map[DurationKey]DynamicDuration{
 		Description:  "TaskIsolationDuration is the time period for which we attempt to respect tasklist isolation before allowing any poller to process the task",
 		DefaultValue: time.Second * 2,
 	},
+	DomainAuditLogTTL: {
+		KeyName:      "system.domainAuditLogTTL",
+		Filters:      []Filter{DomainID},
+		Description:  "DomainAuditLogTTL is the TTL for domain audit log entries",
+		DefaultValue: time.Hour * 24 * 365, // 1 year
+	},
 }
 
 var MapKeys = map[MapKey]DynamicMap{

common/persistence/config.go

@@ -36,6 +36,7 @@ type (
 		ReadNoSQLHistoryTaskFromDataBlob         dynamicproperties.BoolPropertyFn
 		ReadNoSQLShardFromDataBlob               dynamicproperties.BoolPropertyFn
 		SerializationEncoding                    dynamicproperties.StringPropertyFn
+		DomainAuditLogTTL                        dynamicproperties.DurationPropertyFnWithDomainIDFilter
 	}
 )
 
@@ -50,5 +51,6 @@ func NewDynamicConfiguration(dc *dynamicconfig.Collection) *DynamicConfiguration
 		ReadNoSQLHistoryTaskFromDataBlob:         dc.GetBoolProperty(dynamicproperties.ReadNoSQLHistoryTaskFromDataBlob),
 		ReadNoSQLShardFromDataBlob:               dc.GetBoolProperty(dynamicproperties.ReadNoSQLShardFromDataBlob),
 		SerializationEncoding:                    dc.GetStringProperty(dynamicproperties.SerializationEncoding),
+		DomainAuditLogTTL:                        dc.GetDurationPropertyFilteredByDomainID(dynamicproperties.DomainAuditLogTTL),
 	}
 }

common/persistence/data_store_interfaces.go

@@ -887,6 +887,7 @@ type (
 		Identity        string
 		IdentityType    string
 		Comment         string
+		TTLSeconds      int64 // TTL for the audit log entry in seconds
 	}
 
 	// InternalGetDomainAuditLogsResponse is the response for GetDomainAuditLogs

common/persistence/domain_audit_manager.go

@@ -112,6 +112,9 @@ func (m *domainAuditManagerImpl) CreateDomainAuditLog(
 		stateAfterBlob = blob
 	}
 
+	// Get TTL from dynamic config using domain ID
+	ttlDuration := m.dc.DomainAuditLogTTL(request.DomainID)
+
 	return m.persistence.CreateDomainAuditLog(ctx, &InternalCreateDomainAuditLogRequest{
 		DomainID:        request.DomainID,
 		EventID:         request.EventID,
@@ -123,6 +126,7 @@ func (m *domainAuditManagerImpl) CreateDomainAuditLog(
 		Identity:        request.Identity,
 		IdentityType:    request.IdentityType,
 		Comment:         request.Comment,
+		TTLSeconds:      int64(ttlDuration.Seconds()),
 	})
 }
 

common/persistence/nosql/nosql_domain_audit_store.go

@@ -69,6 +69,7 @@ func (m *nosqlDomainAuditStore) CreateDomainAuditLog(
 		Identity:            request.Identity,
 		IdentityType:        request.IdentityType,
 		Comment:             request.Comment,
+		TTLSeconds:          request.TTLSeconds,
 	}
 
 	err := m.db.InsertDomainAuditLog(ctx, row)

common/persistence/nosql/nosqlplugin/cassandra/domain_audit_log.go

@@ -32,7 +32,7 @@ const (
 	templateInsertDomainAuditLogQuery = `INSERT INTO domain_audit_log (` +
 		`domain_id, event_id, state_before, state_before_encoding, state_after, state_after_encoding, ` +
 		`operation_type, created_time, last_updated_time, identity, identity_type, comment) ` +
-		`VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+		`VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) USING TTL ?`
 
 	templateSelectDomainAuditLogsQuery = `SELECT ` +
 		`event_id, domain_id, state_before, state_before_encoding, state_after, state_after_encoding, ` +
@@ -57,6 +57,7 @@ func (db *CDB) InsertDomainAuditLog(ctx context.Context, row *nosqlplugin.Domain
 		row.Identity,
 		row.IdentityType,
 		row.Comment,
+		row.TTLSeconds,
 	).WithContext(ctx)
 
 	err := query.Exec()

common/persistence/nosql/nosqlplugin/types.go

@@ -265,6 +265,7 @@ type (
 		Identity            string
 		IdentityType        string
 		Comment             string
+		TTLSeconds          int64 // TTL for the audit log entry in seconds
 	}
 
 	// DomainAuditLogFilter contains the filter criteria for querying domain audit logs

common/persistence/persistence-tests/persistenceTestBase.go

@@ -160,6 +160,7 @@ func NewTestBaseWithNoSQL(t *testing.T, options *TestBaseOptions) *TestBase {
 		ReadNoSQLHistoryTaskFromDataBlob:         dynamicproperties.GetBoolPropertyFn(false),
 		SerializationEncoding:                    dynamicproperties.GetStringPropertyFn(string(constants.EncodingTypeThriftRW)),
 		ReadNoSQLShardFromDataBlob:               dynamicproperties.GetBoolPropertyFn(true),
+		DomainAuditLogTTL:                        func(domainID string) time.Duration { return time.Hour * 24 * 365 }, // 1 year default
 	}
 	params := TestBaseParams{
 		DefaultTestCluster:    testCluster,
@@ -192,6 +193,7 @@ func NewTestBaseWithSQL(t *testing.T, options *TestBaseOptions) *TestBase {
 		ReadNoSQLHistoryTaskFromDataBlob:         dynamicproperties.GetBoolPropertyFn(false),
 		SerializationEncoding:                    dynamicproperties.GetStringPropertyFn(string(constants.EncodingTypeThriftRW)),
 		ReadNoSQLShardFromDataBlob:               dynamicproperties.GetBoolPropertyFn(true),
+		DomainAuditLogTTL:                        func(domainID string) time.Duration { return time.Hour * 24 * 365 }, // 1 year default
 	}
 	params := TestBaseParams{
 		DefaultTestCluster:    testCluster,

config/dynamicconfig/development.yaml

@@ -71,6 +71,8 @@ system.writeVisibilityStoreName:
   - value: "db"
 system.readVisibilityStoreName:
   - value: "db"
+system.domainAuditLogTTL:
+  - value: "15m"
 matching.enableClientAutoConfig:
 - value: true
 shardDistributor.MigrationMode: