Skip to content

TRT-2814: Eliminate prow_job_runs_report_matview #3828

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mstaeble wants to merge 1 commit into
base: main
Choose a base branch
from
Open

TRT-2814: Eliminate prow_job_runs_report_matview #3828

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
416 changes: 332 additions & 84 deletions pkg/api/job_runs.go
View file
Open in desktop

Large diffs are not rendered by default.

13 changes: 3 additions & 10 deletions pkg/db/query/cumulative_query.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,11 @@ func nameMatchConditions(matches TestNameMatches) (conditions []string, args []a
}
for _, prefix := range matches.Prefixes {
conditions = append(conditions, "tests.name LIKE ?")
args = append(args, escapeLikeMetachars(prefix)+"%")
args = append(args, filter.EscapeLikeMetachars(prefix)+"%")
}
for _, sub := range matches.Substrings {
conditions = append(conditions, "tests.name ILIKE ?")
args = append(args, "%"+escapeLikeMetachars(sub)+"%")
args = append(args, "%"+filter.EscapeLikeMetachars(sub)+"%")
}
return conditions, args
}
Expand All @@ -95,13 +95,6 @@ func buildTestsJoinCondition(matches TestNameMatches) (string, []any) {
return testsJoinClause + " AND (" + strings.Join(conditions, " OR ") + ")", args
}

// escapeLikeMetachars escapes LIKE/ILIKE metacharacters (%, _, \) so they
// match literally.
func escapeLikeMetachars(s string) string {
r := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`)
return r.Replace(s)
}

// DateRange defines a half-open date interval [Start, End) used to compute
// period counts from prefix sums in test_cumulative_summaries.
//
Expand Down Expand Up @@ -193,7 +186,7 @@ func variantFilterConditions(variantFilter *filter.Filter) (conditions []string,
}
args = append(args, item.Value)
case filter.OperatorHasEntryContaining, filter.OperatorContains:
pattern := "%" + escapeLikeMetachars(strings.ToLower(item.Value)) + "%"
pattern := "%" + filter.EscapeLikeMetachars(strings.ToLower(item.Value)) + "%"
if item.Not {
conditions = append(conditions, "NOT EXISTS (SELECT 1 FROM variant_combinations vc, LATERAL unnest(vc.variants) AS v(item) WHERE vc.id = variant_combination_id AND LOWER(v.item) LIKE ?)")
} else {
Expand Down
4 changes: 2 additions & 2 deletions pkg/db/query/cumulative_query_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,9 @@ func TestEscapeLikeMetachars(t *testing.T) {
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := escapeLikeMetachars(tc.input)
got := filter.EscapeLikeMetachars(tc.input)
if got != tc.expected {
t.Errorf("escapeLikeMetachars(%q) = %q, want %q", tc.input, got, tc.expected)
t.Errorf("filter.EscapeLikeMetachars(%q) = %q, want %q", tc.input, got, tc.expected)
}
})
}
Expand Down
77 changes: 1 addition & 76 deletions pkg/db/views.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,7 @@ const replaceTimeNow = "|||TIMENOW|||"
const timestampFormat = "2006-01-02 15:04:05"

// TODO: for historical sippy we need to specify the pinnedDate and not use NOW
var PostgresMatViews = []PostgresView{
{
Name: "prow_job_runs_report_matview",
Definition: jobRunsReportMatView,
IndexColumns: []string{"id"},
AdditionalIndexes: []string{"release, timestamp DESC"},
},
}
var PostgresMatViews = []PostgresView{}

// PostgresViews are regular, non-materialized views:
var PostgresViews = []PostgresView{}
Expand Down Expand Up @@ -147,71 +140,3 @@ func syncPostgresViews(db *gorm.DB, reportEnd *time.Time) error {

return nil
}

// jobRunsReportMatView limits all data to a 90-day window. This is intentional:
// prow_job_run_tests is heavily partitioned and scanning beyond 90 days is expensive
// with no consumer needing older per-test failure/flake details in this view.
const jobRunsReportMatView = `
WITH test_results AS (
SELECT prow_job_run_tests.prow_job_run_id,
prow_job_run_tests.prow_job_run_release,
count(tests.id) FILTER (WHERE prow_job_run_tests.status = 12) AS failed_test_count,
array_agg(tests.name) FILTER (WHERE prow_job_run_tests.status = 12) AS failed_test_names,
count(tests.id) FILTER (WHERE prow_job_run_tests.status = 13) AS flaked_test_count,
array_agg(tests.name) FILTER (WHERE prow_job_run_tests.status = 13) AS flaked_test_names
FROM prow_job_run_tests
JOIN tests ON tests.id = prow_job_run_tests.test_id
WHERE prow_job_run_tests.status IN (12, 13)
AND prow_job_run_tests.prow_job_run_timestamp >= |||TIMENOW||| - interval '90 days'
GROUP BY prow_job_run_tests.prow_job_run_id, prow_job_run_tests.prow_job_run_release
),
pull_requests AS (
SELECT
DISTINCT ON(prow_job_runs.id)
prow_job_runs.id as id,
prow_pull_requests.link,
prow_pull_requests.sha,
prow_pull_requests.org,
prow_pull_requests.author,
prow_pull_requests.repo
FROM
prow_pull_requests
INNER JOIN
prow_job_run_prow_pull_requests ON prow_job_run_prow_pull_requests.prow_pull_request_id = prow_pull_requests.id
INNER JOIN
prow_job_runs ON prow_job_run_prow_pull_requests.prow_job_run_id = prow_job_runs.id
WHERE prow_job_runs."timestamp" >= |||TIMENOW||| - interval '90 days'
GROUP BY prow_job_runs.id, prow_pull_requests.link, prow_pull_requests.sha, prow_pull_requests.org, prow_pull_requests.repo, prow_pull_requests.author
)
SELECT prow_job_runs.id,
prow_jobs.release,
prow_jobs.name,
prow_jobs.name AS job,
prow_jobs.variants,
regexp_replace(prow_jobs.name, 'periodic-ci-openshift-(multiarch|release)-master-(ci|nightly)-[0-9]+.[0-9]+-'::text, ''::text) AS brief_name,
prow_job_runs.overall_result,
prow_job_runs.url AS test_grid_url,
prow_job_runs.url,
prow_job_runs.succeeded,
prow_job_runs.infrastructure_failure,
prow_job_runs.known_failure,
(EXTRACT(epoch FROM (prow_job_runs."timestamp" AT TIME ZONE 'utc'::text)) * 1000::numeric)::bigint AS "timestamp",
prow_job_runs.id AS prow_id,
prow_job_runs.cluster AS cluster,
prow_job_runs.labels as labels,
test_results.flaked_test_names AS flaked_test_names,
test_results.flaked_test_count AS test_flakes,
test_results.failed_test_names AS failed_test_names,
test_results.failed_test_count AS test_failures,
pull_requests.link as pull_request_link,
pull_requests.sha as pull_request_sha,
pull_requests.org as pull_request_org,
pull_requests.repo as pull_request_repo,
pull_requests.author as pull_request_author
FROM prow_job_runs
LEFT JOIN test_results ON test_results.prow_job_run_id = prow_job_runs.id
AND test_results.prow_job_run_release = prow_job_runs.prow_job_release
LEFT JOIN pull_requests ON pull_requests.id = prow_job_runs.id
JOIN prow_jobs ON prow_job_runs.prow_job_id = prow_jobs.id
WHERE prow_job_runs."timestamp" >= |||TIMENOW||| - interval '90 days'
`
177 changes: 68 additions & 109 deletions pkg/filter/filterable.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,19 +73,31 @@ func optNot(not bool) string {
return ""
}

// WrapNot wraps a SQL expression in NOT(...) when negated.
func WrapNot(sql string, not bool) string {
if not {
return fmt.Sprintf("NOT(%s)", sql)
}
return sql
}

// EscapeLikeMetachars escapes LIKE/ILIKE metacharacters (%, _, \) so they
// match literally in PostgreSQL pattern expressions.
func EscapeLikeMetachars(s string) string {
r := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`)
return r.Replace(s)
}

// ilikeFilter returns the SQL filter and parameters for ILIKE pattern matching,
// handling both string fields (using ILIKE directly) and array fields (using unnest with EXISTS).
func ilikeFilter(field, pattern string, not bool, filterable Filterable, fieldName string) (string, interface{}) {
func ilikeFilter(field, pattern string, not bool, filterable Filterable, fieldName string) (string, any) {
var sql string
if filterable != nil && filterable.GetFieldType(fieldName) == apitype.ColumnTypeArray {
return fmt.Sprintf("%s EXISTS (SELECT 1 FROM unnest(%s) AS elem WHERE elem ILIKE ?)", optNot(not), field), pattern
sql = fmt.Sprintf("EXISTS (SELECT 1 FROM unnest(%s) AS elem WHERE elem ILIKE ?)", field)
} else {
sql = fmt.Sprintf("%s ILIKE ?", field)
}
return fmt.Sprintf("%s %s ILIKE ?", field, optNot(not)), pattern
}

// applyIlikeFilter applies an ILIKE filter to a GORM DB handle, handling both string and array fields.
func applyIlikeFilter(db *gorm.DB, field, pattern string, not bool, filterable Filterable, fieldName string) *gorm.DB {
filterSQL, params := ilikeFilter(field, pattern, not, filterable, fieldName)
return db.Where(filterSQL, params)
return WrapNot(sql, not), pattern
}

func (f FilterItem) isEmptyFilter(field string, filterable Filterable, forBQ bool) string {
Expand All @@ -97,131 +109,75 @@ func (f FilterItem) isEmptyFilter(field string, filterable Filterable, forBQ boo
sql = fmt.Sprintf("(%s IS NULL or ARRAY_LENGTH(%s) = 0)", field, field)
}
}
if f.Not {
return fmt.Sprintf("NOT(%s)", sql)
}
return sql
return WrapNot(sql, f.Not)
}

func (f FilterItem) orFilterToSQL(db *gorm.DB, filterable Filterable) (orFilter string, orParams interface{}) { //nolint
field := fmt.Sprintf("%q", f.Field)
if filterable != nil && filterable.GetFieldType(f.Field) == apitype.ColumnTypeTimestamp {
field = fmt.Sprintf("extract(epoch from %s at time zone 'utc') * 1000", f.Field)
}

// FilterItemToSQL returns a SQL fragment and parameter for a filter item
// applied to the given column expression. The column is used as-is with no
// type-aware transformations: ILIKE operators match the column value directly
// rather than unnesting array elements, and timestamps are not converted to
// epoch milliseconds. Use FilterFieldToSQL when the column may be an array
// or timestamp type.
func (f FilterItem) FilterItemToSQL(column string) (string, any, error) {
var sql string
var param any
switch f.Operator {
case OperatorHasEntry:
if f.Not {
return fmt.Sprintf("%s IS NULL OR ? != ALL(%s)", field, field), f.Value
}
return fmt.Sprintf("? = ANY(%s)", field), f.Value
sql, param = fmt.Sprintf("? = ANY(COALESCE(%s, '{}'))", column), f.Value
case OperatorHasEntryContaining, OperatorContains:
return ilikeFilter(field, fmt.Sprintf("%%%s%%", f.Value), f.Not, filterable, f.Field)
sql, param = fmt.Sprintf("%s ILIKE ?", column), fmt.Sprintf("%%%s%%", EscapeLikeMetachars(f.Value))
case OperatorEquals, OperatorArithmeticEquals:
if f.Not {
return fmt.Sprintf("%s != ?", field), f.Value
}
return fmt.Sprintf("%s = ?", field), f.Value
sql, param = fmt.Sprintf("%s = ?", column), f.Value
case OperatorArithmeticGreaterThan:
if f.Not {
return fmt.Sprintf("%s <= ?", field), f.Value
}
return fmt.Sprintf("%s > ?", field), f.Value
sql, param = fmt.Sprintf("%s > ?", column), f.Value
case OperatorArithmeticGreaterThanOrEquals:
if f.Not {
return fmt.Sprintf("%s < ?", field), f.Value
}
return fmt.Sprintf("%s >= ?", field), f.Value
sql, param = fmt.Sprintf("%s >= ?", column), f.Value
case OperatorArithmeticLessThan:
if f.Not {
return fmt.Sprintf("%s >= ?", field), f.Value
}
return fmt.Sprintf("%s < ?", field), f.Value
sql, param = fmt.Sprintf("%s < ?", column), f.Value
case OperatorArithmeticLessThanOrEquals:
if f.Not {
return fmt.Sprintf("%s > ?", field), f.Value
}
return fmt.Sprintf("%s <= ?", field), f.Value
sql, param = fmt.Sprintf("%s <= ?", column), f.Value
case OperatorArithmeticNotEquals:
if f.Not {
return fmt.Sprintf("%s = ?", field), f.Value
}
return fmt.Sprintf("%s <> ?", field), f.Value
sql, param = fmt.Sprintf("%s <> ?", column), f.Value
case OperatorStartsWith:
return ilikeFilter(field, fmt.Sprintf("%s%%", f.Value), f.Not, filterable, f.Field)
sql, param = fmt.Sprintf("%s ILIKE ?", column), fmt.Sprintf("%s%%", EscapeLikeMetachars(f.Value))
case OperatorEndsWith:
return ilikeFilter(field, fmt.Sprintf("%%%s", f.Value), f.Not, filterable, f.Field)
sql, param = fmt.Sprintf("%s ILIKE ?", column), fmt.Sprintf("%%%s", EscapeLikeMetachars(f.Value))
case OperatorIsEmpty:
return f.isEmptyFilter(field, filterable, false), nil
sql = fmt.Sprintf("%s IS NULL", column)
case OperatorIsNotEmpty:
return fmt.Sprintf("%s IS %s NULL", field, optNot(!f.Not)), nil
sql = fmt.Sprintf("%s IS NOT NULL", column)
default:
return "", nil, fmt.Errorf("unsupported operator %q for field %q", f.Operator, f.Field)
}

return "UnknownFilterOperator()", nil // cause SQL to fail in obvious way
return WrapNot(sql, f.Not), param, nil
}

func (f FilterItem) andFilterToSQL(db *gorm.DB, filterable Filterable) *gorm.DB { //nolint
// FilterFieldToSQL returns a SQL fragment and parameter for a filter item,
// with array and timestamp type awareness from the filterable.
func (f FilterItem) FilterFieldToSQL(filterable Filterable) (string, any) {
field := fmt.Sprintf("%q", f.Field)
if filterable != nil && filterable.GetFieldType(f.Field) == apitype.ColumnTypeTimestamp {
field = fmt.Sprintf("extract(epoch from %s at time zone 'utc') * 1000", f.Field)
}

// Operators that need array-aware handling delegate to specialized helpers;
// all other operators use the common scalar implementation.
switch f.Operator {
case OperatorHasEntry:
if f.Not {
db = db.Where(fmt.Sprintf("%s IS NULL OR ? != ALL(%s)", field, field), f.Value)
} else {
db = db.Where(fmt.Sprintf("? = ANY(%s)", field), f.Value)
}
case OperatorHasEntryContaining, OperatorContains:
db = applyIlikeFilter(db, field, fmt.Sprintf("%%%s%%", f.Value), f.Not, filterable, f.Field)
case OperatorEquals, OperatorArithmeticEquals:
if f.Not {
db = db.Not(fmt.Sprintf("%s = ?", field), f.Value)
} else {
db = db.Where(fmt.Sprintf("%s = ?", field), f.Value)
}
case OperatorArithmeticGreaterThan:
if f.Not {
db = db.Not(fmt.Sprintf("%s > ?", field), f.Value)
} else {
db = db.Where(fmt.Sprintf("%s > ?", field), f.Value)
}
case OperatorArithmeticGreaterThanOrEquals:
if f.Not {
db = db.Not(fmt.Sprintf("%s >= ?", field), f.Value)
} else {
db = db.Where(fmt.Sprintf("%s >= ?", field), f.Value)
}
case OperatorArithmeticLessThan:
if f.Not {
db = db.Not(fmt.Sprintf("%s < ?", field), f.Value)
} else {
db = db.Where(fmt.Sprintf("%s < ?", field), f.Value)
}
case OperatorArithmeticLessThanOrEquals:
if f.Not {
db = db.Not(fmt.Sprintf("%s <= ?", field), f.Value)
} else {
db = db.Where(fmt.Sprintf("%s <= ?", field), f.Value)
}
case OperatorArithmeticNotEquals:
if f.Not {
db = db.Not(fmt.Sprintf("%s <> ?", field), f.Value)
} else {
db = db.Where(fmt.Sprintf("%s <> ?", field), f.Value)
}
return ilikeFilter(field, fmt.Sprintf("%%%s%%", EscapeLikeMetachars(f.Value)), f.Not, filterable, f.Field)
case OperatorStartsWith:
db = applyIlikeFilter(db, field, fmt.Sprintf("%s%%", f.Value), f.Not, filterable, f.Field)
return ilikeFilter(field, fmt.Sprintf("%s%%", EscapeLikeMetachars(f.Value)), f.Not, filterable, f.Field)
case OperatorEndsWith:
db = applyIlikeFilter(db, field, fmt.Sprintf("%%%s", f.Value), f.Not, filterable, f.Field)
return ilikeFilter(field, fmt.Sprintf("%%%s", EscapeLikeMetachars(f.Value)), f.Not, filterable, f.Field)
case OperatorIsEmpty:
db = db.Where(f.isEmptyFilter(field, filterable, false))
case OperatorIsNotEmpty:
db = db.Where(fmt.Sprintf("%s IS %s NULL", field, optNot(!f.Not)))
return f.isEmptyFilter(field, filterable, false), nil
}

return db
sql, param, err := f.FilterItemToSQL(field)
if err != nil {
return "UnknownFilterOperator()", nil
}
return sql, param
}

func (f FilterItem) toBQStr(filterable Filterable, paramIndex int) (sql string, params []bigquery.QueryParameter) { //nolint
Expand Down Expand Up @@ -441,16 +397,19 @@ filterOuterLoop:
}

func (filters Filter) ToSQL(db *gorm.DB, filterable Filterable) *gorm.DB {

orFilters := []string{}
orFilterParams := []interface{}{}
var orFilters []string
var orFilterParams []interface{}

for _, f := range filters.Items {
q, p := f.FilterFieldToSQL(filterable)
switch filters.LinkOperator {
case LinkOperatorAnd, "":
db = f.andFilterToSQL(db, filterable)
if p != nil {
db = db.Where(q, p)
} else {
db = db.Where(q)
}
case LinkOperatorOr:
q, p := f.orFilterToSQL(db, filterable)
orFilters = append(orFilters, q)
if p != nil {
orFilterParams = append(orFilterParams, p)
Expand Down
Loading