diff --git a/cmd/sippy/automatejira.go b/cmd/sippy/automatejira.go index 1b40ba8538..64b3e084da 100644 --- a/cmd/sippy/automatejira.go +++ b/cmd/sippy/automatejira.go @@ -14,9 +14,7 @@ import ( "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/sets" - "github.com/openshift/sippy/pkg/api" "github.com/openshift/sippy/pkg/api/componentreadiness" - bqprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/bigquery" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/cache" jiratype "github.com/openshift/sippy/pkg/apis/jira/v1" @@ -34,6 +32,7 @@ type AutomateJiraFlags struct { ConfigFlags *configflags.ConfigFlags PostgresFlags *flags.PostgresFlags JiraFlags flags.JiraFlags + DataProvider string SippyURL string IncludeComponentsStr string // IncludeComponents is a set of string in the format of jiraProject:jiraComponent @@ -70,6 +69,7 @@ func (f *AutomateJiraFlags) BindFlags(fs *pflag.FlagSet) { fs.StringArrayVar(&f.ColumnThresholdStrs, "column-threshold", f.ColumnThresholdStrs, "A threshold of red cell counts over which a jira issue will be created against a component corresponding to an interesting variant of a column (e.g. Bare Metal Hardware Provisioning for metal platform). The format of the threshold string is [variant]:[value]:[threshold] (e.g. Platform:metal:3).") fs.StringVar(&f.JiraAccount, "jira-account", f.JiraAccount, "The jira account used to automate jira") fs.BoolVar(&f.DryRun, "dry-run", f.DryRun, "Print the tasks of automating jiras without real interaction with jira.") + fs.StringVar(&f.DataProvider, "data-provider", "default", "Data provider: default, bigquery, or postgres") } func (f *AutomateJiraFlags) Validate(allVariants crtest.JobVariants) error { @@ -152,11 +152,6 @@ func NewAutomateJiraCommand() *cobra.Command { if err != nil { log.WithError(err).Fatal("unable to load views") } - releases, err := api.GetReleases(context.Background(), bigQueryClient, false) - if err != nil { - log.WithError(err).Fatal("error querying releases") - } - jiraClient, err := f.JiraFlags.GetJiraClient() if err != nil { return errors.WithMessage(err, "couldn't get jira client") @@ -165,7 +160,14 @@ func NewAutomateJiraCommand() *cobra.Command { return fmt.Errorf("couldn't get jira client: jira auth is not configured") } - provider := bqprovider.NewBigQueryProvider(bigQueryClient) + dbc, err := f.PostgresFlags.GetDBClient() + if err != nil { + log.WithError(err).Warn("unable to connect to postgres, will use BigQuery for release metadata") + } + provider, err := newDataProvider(f.DataProvider, bigQueryClient, dbc, cacheClient) + if err != nil { + return err + } allVariants, errs := componentreadiness.GetJobVariants(ctx, provider) if len(errs) > 0 { return fmt.Errorf("failed to get job variants: %v", errs) @@ -178,10 +180,11 @@ func NewAutomateJiraCommand() *cobra.Command { return errors.WithMessage(err, "error validating options") } - dbc, err := f.PostgresFlags.GetDBClient() + releases, err := provider.QueryReleases(ctx) if err != nil { - log.WithError(err).Fatal("unable to connect to postgres") + log.WithError(err).Fatal("error querying releases") } + j, err := jiraautomator.NewJiraAutomator( jiraClient, bigQueryClient, provider, dbc, cacheOpts, views.ComponentReadiness, releases, f.SippyURL, f.JiraAccount, diff --git a/cmd/sippy/component_readiness.go b/cmd/sippy/component_readiness.go index 6801cc0d93..0ffcd2dfbf 100644 --- a/cmd/sippy/component_readiness.go +++ b/cmd/sippy/component_readiness.go @@ -17,7 +17,6 @@ import ( "gopkg.in/yaml.v3" resources "github.com/openshift/sippy" - bqprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/bigquery" "github.com/openshift/sippy/pkg/apis/cache" v1 "github.com/openshift/sippy/pkg/apis/config/v1" "github.com/openshift/sippy/pkg/bigquery" @@ -38,8 +37,9 @@ type ComponentReadinessFlags struct { APIFlags *flags.APIFlags JiraFlags *flags.JiraFlags - Config string - LogLevel string + Config string + DataProvider string + LogLevel string } func NewComponentReadinessCommand() *cobra.Command { @@ -88,6 +88,7 @@ func (f *ComponentReadinessFlags) BindFlags(flagSet *pflag.FlagSet) { f.APIFlags.BindFlags(flagSet) f.JiraFlags.BindFlags(flagSet) flagSet.StringVar(&f.LogLevel, "log-level", f.LogLevel, "Log level (trace,debug,info,warn,error) (default info)") + flagSet.StringVar(&f.DataProvider, "data-provider", "default", "Data provider: default, bigquery, or postgres") } func (f *ComponentReadinessFlags) Validate() error { @@ -187,7 +188,10 @@ func (f *ComponentReadinessFlags) runServerMode() error { log.WithError(err).Warn("unable to initialize Jira client, bug filing will be disabled") } - crDataProvider := bqprovider.NewBigQueryProvider(bigQueryClient) + crDataProvider, err := newDataProvider(f.DataProvider, bigQueryClient, dbc, cacheClient) + if err != nil { + return err + } server := sippyserver.NewServer( sippyserver.ModeOpenShift, diff --git a/cmd/sippy/serve.go b/cmd/sippy/serve.go index a456812f00..a1d6e665f5 100644 --- a/cmd/sippy/serve.go +++ b/cmd/sippy/serve.go @@ -19,11 +19,13 @@ import ( resources "github.com/openshift/sippy" "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" bqprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/bigquery" + mixedprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/mixed" pgprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/postgres" "github.com/openshift/sippy/pkg/apis/cache" "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/bigquery/bqlabel" "github.com/openshift/sippy/pkg/dataloader/prowloader/gcs" + "github.com/openshift/sippy/pkg/db" "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/flags" "github.com/openshift/sippy/pkg/flags/configflags" @@ -70,7 +72,7 @@ func (f *ServerFlags) BindFlags(flagSet *pflag.FlagSet) { f.ConfigFlags.BindFlags(flagSet) f.APIFlags.BindFlags(flagSet) f.JiraFlags.BindFlags(flagSet) - flagSet.StringVar(&f.DataProvider, "data-provider", "bigquery", "Data provider for component readiness: bigquery, postgres") + flagSet.StringVar(&f.DataProvider, "data-provider", "default", "Data provider: default, bigquery, or postgres") } func (f *ServerFlags) Validate() error { @@ -110,7 +112,7 @@ func NewServeCommand() *cobra.Command { var gcsClient *storage.Client var crDataProvider dataprovider.DataProvider switch f.DataProvider { - case "bigquery": + case "default", "bigquery": if f.GoogleCloudFlags.ServiceAccountCredentialFile != "" { opCtx := bqlabel.OperationalContext{ App: bqlabel.AppSippy, @@ -132,16 +134,12 @@ func NewServeCommand() *cobra.Command { if bigQueryClient != nil && f.CacheFlags.EnablePersistentCaching { bigQueryClient = f.CacheFlags.DecorateBiqQueryClientWithPersistentCache(bigQueryClient) } - - crDataProvider = bqprovider.NewBigQueryProvider(bigQueryClient) } + } - case "postgres": - crDataProvider = pgprovider.NewPostgresProvider(dbc, cacheClient) - log.Info("Using Postgres data provider for component readiness") - - default: - return fmt.Errorf("unknown --data-provider %q, must be bigquery or postgres", f.DataProvider) + crDataProvider, err = newDataProvider(f.DataProvider, bigQueryClient, dbc, cacheClient) + if err != nil { + return err } gcsClient, err = gcs.NewGCSClient(context.TODO(), @@ -262,3 +260,33 @@ func NewServeCommand() *cobra.Command { f.BindFlags(cmd.Flags()) return cmd } + +func newDataProvider(name string, bigQueryClient *bigquery.Client, dbc *db.DB, cacheClient cache.Cache) (dataprovider.DataProvider, error) { + switch name { + case "default": + if bigQueryClient != nil && dbc != nil { + return mixedprovider.NewMixedProvider(bigQueryClient, dbc, cacheClient), nil + } else if bigQueryClient != nil { + return bqprovider.NewBigQueryProvider(bigQueryClient), nil + } else if dbc != nil { + return pgprovider.NewPostgresProvider(dbc, cacheClient), nil + } + return nil, fmt.Errorf("default data provider requires at least one of BigQuery or PostgreSQL to be configured") + + case "bigquery": + if bigQueryClient != nil { + return bqprovider.NewBigQueryProvider(bigQueryClient), nil + } + return nil, fmt.Errorf("bigquery data provider requires google-service-account-credential-file to be configured") + + case "postgres": + if dbc == nil { + return nil, fmt.Errorf("postgres data provider requires a database connection") + } + log.Info("Using Postgres data provider for component readiness") + return pgprovider.NewPostgresProvider(dbc, cacheClient), nil + + default: + return nil, fmt.Errorf("unknown --data-provider %q, must be default, bigquery, or postgres", name) + } +} diff --git a/pkg/api/componentreadiness/dataprovider/mixed/provider.go b/pkg/api/componentreadiness/dataprovider/mixed/provider.go new file mode 100644 index 0000000000..64377cc485 --- /dev/null +++ b/pkg/api/componentreadiness/dataprovider/mixed/provider.go @@ -0,0 +1,81 @@ +package mixed + +import ( + "context" + "time" + + "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" + "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/bigquery" + "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/postgres" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" + "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" + "github.com/openshift/sippy/pkg/apis/cache" + v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + bqcachedclient "github.com/openshift/sippy/pkg/bigquery" + "github.com/openshift/sippy/pkg/db" +) + +var _ dataprovider.DataProvider = &MixedProvider{} + +// MixedProvider wraps both a BigQuery and PostgreSQL provider, routing +// release metadata queries to PostgreSQL and everything else to BigQuery. +type MixedProvider struct { + bq *bigquery.BigQueryProvider + pg *postgres.PostgresProvider +} + +func NewMixedProvider(bqClient *bqcachedclient.Client, dbc *db.DB, cacheClient cache.Cache) *MixedProvider { + return &MixedProvider{ + bq: bigquery.NewBigQueryProvider(bqClient), + pg: postgres.NewPostgresProvider(dbc, cacheClient), + } +} + +func (p *MixedProvider) Cache() cache.Cache { + return p.bq.Cache() +} + +func (p *MixedProvider) QueryReleases(ctx context.Context) ([]v1.Release, error) { + return p.pg.QueryReleases(ctx) +} + +func (p *MixedProvider) QueryReleaseDates(ctx context.Context, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) { + return p.pg.QueryReleaseDates(ctx, reqOptions) +} + +func (p *MixedProvider) QueryJobVariants(ctx context.Context) (crtest.JobVariants, []error) { + return p.bq.QueryJobVariants(ctx) +} + +func (p *MixedProvider) QueryUniqueVariantValues(ctx context.Context, field string, nested bool) ([]string, error) { + return p.bq.QueryUniqueVariantValues(ctx, field, nested) +} + +func (p *MixedProvider) QueryBaseTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions) (map[string]crstatus.TestStatus, []error) { + return p.bq.QueryBaseTestStatus(ctx, reqOptions) +} + +func (p *MixedProvider) QuerySampleTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, includeVariants map[string][]string, start, end time.Time) (map[string]crstatus.TestStatus, []error) { + return p.bq.QuerySampleTestStatus(ctx, reqOptions, includeVariants, start, end) +} + +func (p *MixedProvider) QueryBaseJobRunTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions) (map[string][]crstatus.TestJobRunRows, []error) { + return p.bq.QueryBaseJobRunTestStatus(ctx, reqOptions) +} + +func (p *MixedProvider) QuerySampleJobRunTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, includeVariants map[string][]string, start, end time.Time) (map[string][]crstatus.TestJobRunRows, []error) { + return p.bq.QuerySampleJobRunTestStatus(ctx, reqOptions, includeVariants, start, end) +} + +func (p *MixedProvider) QueryJobRuns(ctx context.Context, reqOptions reqopts.RequestOptions, release string, start, end time.Time) (map[string]dataprovider.JobRunStats, error) { + return p.bq.QueryJobRuns(ctx, reqOptions, release, start, end) +} + +func (p *MixedProvider) QueryJobVariantValues(ctx context.Context, jobNames, variantKeys []string) (map[string]map[string]string, error) { + return p.bq.QueryJobVariantValues(ctx, jobNames, variantKeys) +} + +func (p *MixedProvider) LookupJobVariants(ctx context.Context, jobName string) (map[string]string, error) { + return p.bq.LookupJobVariants(ctx, jobName) +} diff --git a/pkg/api/componentreadiness/dataprovider/postgres/provider.go b/pkg/api/componentreadiness/dataprovider/postgres/provider.go index 73be13acdb..edba44f127 100644 --- a/pkg/api/componentreadiness/dataprovider/postgres/provider.go +++ b/pkg/api/componentreadiness/dataprovider/postgres/provider.go @@ -12,6 +12,7 @@ import ( "github.com/lib/pq" "k8s.io/apimachinery/pkg/util/sets" + "github.com/openshift/sippy/pkg/api" "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" "github.com/openshift/sippy/pkg/api/componentreadiness/utils" "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" @@ -133,96 +134,16 @@ func (p *PostgresProvider) QueryJobVariants(ctx context.Context) (crtest.JobVari return variants, nil } -// releaseMetadata holds hardcoded release info for known releases. -// This avoids needing a releases table — we derive release names from prow_jobs -// and fill in metadata from this map. -var releaseMetadata = map[string]struct { - previousRelease string - gaOffsetDays int // 0 = no GA date (in development) - product string // empty = defaults to "OCP" -}{ - "4.17": {previousRelease: "4.16", gaOffsetDays: -540}, - "4.18": {previousRelease: "4.17", gaOffsetDays: -395}, - "4.19": {previousRelease: "4.18", gaOffsetDays: -289}, - "4.20": {previousRelease: "4.19", gaOffsetDays: -163}, - "4.21": {previousRelease: "4.20", gaOffsetDays: -58}, - "4.22": {previousRelease: "4.21"}, - "5.0": {previousRelease: "4.22"}, -} - func (p *PostgresProvider) QueryReleases(ctx context.Context) ([]v1.Release, error) { - var releaseNames []string - err := p.dbc.DB.WithContext(ctx).Raw(`SELECT DISTINCT release FROM prow_jobs WHERE deleted_at IS NULL ORDER BY release DESC`). - Pluck("release", &releaseNames).Error - if err != nil { - return nil, fmt.Errorf("querying releases: %w", err) - } - - caps := map[v1.ReleaseCapability]bool{ - v1.ComponentReadinessCap: true, - v1.FeatureGatesCap: true, - v1.MetricsCap: true, - v1.PayloadTagsCap: true, - v1.SippyClassicCap: true, - } - - now := time.Now().UTC() - var releases []v1.Release - for _, name := range releaseNames { - rel := v1.Release{ - Release: name, - Capabilities: caps, - Product: "OCP", - } - if meta, ok := releaseMetadata[name]; ok { - rel.PreviousRelease = meta.previousRelease - if meta.gaOffsetDays != 0 { - ga := now.AddDate(0, 0, meta.gaOffsetDays) - rel.GADate = &ga - } - if meta.product != "" { - rel.Product = meta.product - } - } - releases = append(releases, rel) - } - return releases, nil + return api.GetReleasesFromDB(ctx, p.dbc) } -func (p *PostgresProvider) QueryReleaseDates(ctx context.Context, _ reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) { - // Derive time ranges from actual data in the DB rather than hardcoded GA dates. - // This ensures fallback queries find data where it actually exists. - type releaseRange struct { - Release string - Start time.Time - End time.Time - } - var ranges []releaseRange - err := p.dbc.DB.WithContext(ctx).Raw(` - SELECT pj.release, - MIN(pjr.timestamp) AS start, - MAX(pjr.timestamp) AS end - FROM prow_job_runs pjr - JOIN prow_jobs pj ON pj.id = pjr.prow_job_id - WHERE pj.deleted_at IS NULL AND pjr.deleted_at IS NULL - GROUP BY pj.release - ORDER BY pj.release DESC - `).Scan(&ranges).Error +func (p *PostgresProvider) QueryReleaseDates(ctx context.Context, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) { + timeRanges, err := api.GetReleaseDatesFromDB(ctx, p.dbc, reqOptions) if err != nil { - return nil, []error{fmt.Errorf("querying release dates: %w", err)} - } - - var dates []crtest.ReleaseTimeRange - for _, r := range ranges { - start := r.Start - end := r.End - dates = append(dates, crtest.ReleaseTimeRange{ - Release: r.Release, - Start: &start, - End: &end, - }) + return nil, []error{err} } - return dates, nil + return timeRanges, nil } func (p *PostgresProvider) QueryUniqueVariantValues(ctx context.Context, field string, nested bool) ([]string, error) { diff --git a/pkg/api/job_runs.go b/pkg/api/job_runs.go index 7fa69ec96a..2364b97b8e 100644 --- a/pkg/api/job_runs.go +++ b/pkg/api/job_runs.go @@ -372,8 +372,7 @@ func JobRunRiskAnalysis( compareRelease := jobRun.ProwJob.Release neverStableJob := false if compareRelease == "Presubmits" { - // Get latest release from the DB: - ar, err := GetReleases(ctx, bqc, false) + ar, err := GetReleasesFromDB(ctx, dbc) if err != nil { return apitype.ProwJobRunRiskAnalysis{}, err } @@ -422,8 +421,7 @@ func JobRunRiskAnalysis( } if totalJobRuns < 20 { - // go back to the prior release and get more jobIds to compare against - releases, err := GetReleases(ctx, bqc, false) + releases, err := GetReleasesFromDB(ctx, dbc) if err != nil { logger.WithError(err).Error("Failed to get releases for prior release lookup") } else { diff --git a/pkg/api/releases.go b/pkg/api/releases.go index 33a744a274..541f4b720a 100644 --- a/pkg/api/releases.go +++ b/pkg/api/releases.go @@ -16,6 +16,8 @@ import ( "gorm.io/gorm" apitype "github.com/openshift/sippy/pkg/apis/api" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" + "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" sippyv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" bqcachedclient "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/bigquery/bqlabel" @@ -24,6 +26,7 @@ import ( "github.com/openshift/sippy/pkg/db/query" "github.com/openshift/sippy/pkg/filter" "github.com/openshift/sippy/pkg/testidentification" + "github.com/openshift/sippy/pkg/util" ) func PrintPullRequestsReport(w http.ResponseWriter, req *http.Request, dbClient *db.DB) { @@ -540,9 +543,33 @@ func GetReleaseRowsFromBigQuery(ctx context.Context, client *bqcachedclient.Clie return rows, nil } -// GetReleasesFromDB queries release metadata from the release_definitions table -// and converts to []sippyv1.Release for use by existing callers. +// GetReleaseDatesFromDB derives CR time ranges from release_definitions GA dates. +func GetReleaseDatesFromDB(ctx context.Context, dbc *db.DB, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, error) { + if dbc == nil || dbc.DB == nil { + return nil, fmt.Errorf("no database connection available for release dates") + } + releases, err := GetReleasesFromDB(ctx, dbc) + if err != nil { + return nil, err + } + var timeRanges []crtest.ReleaseTimeRange + for _, release := range releases { + tr := crtest.ReleaseTimeRange{Release: release.Release} + if release.GADate != nil { + prior := util.AdjustReleaseTime(*release.GADate, true, "30", reqOptions.CacheOption.CRTimeRoundingFactor, reqOptions.CacheOption.CRTimeRoundingOffset) + tr.Start = &prior + tr.End = release.GADate + } + timeRanges = append(timeRanges, tr) + } + return timeRanges, nil +} + +// GetReleasesFromDB queries release metadata from the release_definitions table. func GetReleasesFromDB(ctx context.Context, dbc *db.DB) ([]sippyv1.Release, error) { + if dbc == nil || dbc.DB == nil { + return nil, fmt.Errorf("no database connection available for releases") + } var defs []models.ReleaseDefinition err := dbc.DB.WithContext(ctx).Order("development_start_date DESC").Find(&defs).Error if err != nil { diff --git a/pkg/api/releases_test.go b/pkg/api/releases_test.go index 62967569cb..9c6a778de2 100644 --- a/pkg/api/releases_test.go +++ b/pkg/api/releases_test.go @@ -4,70 +4,114 @@ import ( "testing" "time" - "cloud.google.com/go/bigquery" - "cloud.google.com/go/civil" - - sippyv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" - + "github.com/lib/pq" "github.com/stretchr/testify/assert" apitype "github.com/openshift/sippy/pkg/apis/api" + "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" + sippyv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/openshift/sippy/pkg/db/models" ) -func TestTransformRelease(t *testing.T) { - - devStart420, _ := time.Parse(time.RFC3339, "2025-04-18T00:00:00.00Z") - devStart419, _ := time.Parse(time.RFC3339, "2024-11-25T00:00:00.00Z") - gaDate419, _ := time.Parse(time.RFC3339, "2025-05-09T00:00:00.00Z") +func TestDefinitionToRelease(t *testing.T) { + ga := time.Date(2026, 6, 9, 0, 0, 0, 0, time.UTC) + devStart := time.Date(2025, 12, 1, 0, 0, 0, 0, time.UTC) tests := []struct { - name string - releaseRow sippyv1.ReleaseRow - expectedRelease sippyv1.Release + name string + def models.ReleaseDefinition + expected sippyv1.Release }{ { - name: "release without devel start", - releaseRow: sippyv1.ReleaseRow{Release: "4.20", ReleaseStatus: bigquery.NullString{Valid: true, StringVal: "Development"}}, - expectedRelease: sippyv1.Release{Release: "4.20", Status: "Development"}, + name: "all fields populated", + def: models.ReleaseDefinition{ + Release: "4.22", + PreviousRelease: "4.21", + GADate: &ga, + DevelopmentStartDate: &devStart, + Product: "OCP", + Status: "Full Support", + Capabilities: pq.StringArray{"componentReadiness", "metrics", "payloadTags"}, + }, + expected: sippyv1.Release{ + Release: "4.22", + PreviousRelease: "4.21", + GADate: &ga, + DevelopmentStartDate: &devStart, + Product: "OCP", + Status: "Full Support", + Capabilities: map[sippyv1.ReleaseCapability]bool{ + "componentReadiness": true, + "metrics": true, + "payloadTags": true, + }, + }, }, { - name: "release with devel start", - releaseRow: sippyv1.ReleaseRow{Release: "4.20", ReleaseStatus: bigquery.NullString{Valid: true, StringVal: "Development"}, DevelStartDate: civil.Date{ - Year: 2025, - Month: 4, - Day: 18, - }}, - expectedRelease: sippyv1.Release{Release: "4.20", Status: "Development", DevelopmentStartDate: &devStart420}, + name: "nil GA date (in development)", + def: models.ReleaseDefinition{ + Release: "5.0", + PreviousRelease: "4.22", + Product: "OCP", + Status: "Development", + Capabilities: pq.StringArray{"componentReadiness"}, + }, + expected: sippyv1.Release{ + Release: "5.0", + PreviousRelease: "4.22", + Product: "OCP", + Status: "Development", + Capabilities: map[sippyv1.ReleaseCapability]bool{"componentReadiness": true}, + }, }, { - name: "release with ga date", - releaseRow: sippyv1.ReleaseRow{Release: "4.19", ReleaseStatus: bigquery.NullString{Valid: true, StringVal: "Development"}, DevelStartDate: civil.Date{ - Year: 2024, - Month: 11, - Day: 25, - }, GADate: bigquery.NullDate{ - Date: civil.Date{ - Year: 2025, - Month: 5, - Day: 9}, - Valid: true, - }}, - expectedRelease: sippyv1.Release{Release: "4.19", Status: "Development", DevelopmentStartDate: &devStart419, GADate: &gaDate419}, + name: "empty capabilities", + def: models.ReleaseDefinition{ + Release: "automation", + Product: "OCP", + Capabilities: pq.StringArray{}, + }, + expected: sippyv1.Release{ + Release: "automation", + Product: "OCP", + Capabilities: map[sippyv1.ReleaseCapability]bool{}, + }, + }, + { + name: "nil capabilities", + def: models.ReleaseDefinition{ + Release: "3.11", + Product: "OCP", + Capabilities: nil, + }, + expected: sippyv1.Release{ + Release: "3.11", + Product: "OCP", + Capabilities: map[sippyv1.ReleaseCapability]bool{}, + }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - release := transformRelease(tc.releaseRow) - assert.Equal(t, tc.expectedRelease.Release, release.Release, "unexpected release") - assert.Equal(t, tc.expectedRelease.Status, release.Status, "unexpected status") - assert.Equal(t, tc.expectedRelease.GADate, release.GADate, "unexpected status") - assert.Equal(t, tc.expectedRelease.DevelopmentStartDate, release.DevelopmentStartDate, "unexpected devel start") + result := DefinitionToRelease(tc.def) + assert.Equal(t, tc.expected, result) }) } } +func TestGetReleasesFromDB_NilDB(t *testing.T) { + _, err := GetReleasesFromDB(t.Context(), nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "no database connection") +} + +func TestGetReleaseDatesFromDB_NilDB(t *testing.T) { + _, err := GetReleaseDatesFromDB(t.Context(), nil, reqopts.RequestOptions{}) + assert.Error(t, err) + assert.Contains(t, err.Error(), "no database connection") +} + func buildFakeReleaseHealthReport(osVersion string) apitype.ReleaseHealthReport { return apitype.ReleaseHealthReport{ ReleaseTag: models.ReleaseTag{ diff --git a/pkg/api/utils.go b/pkg/api/utils.go index 3d859906f9..75367eb51a 100644 --- a/pkg/api/utils.go +++ b/pkg/api/utils.go @@ -1,52 +1,15 @@ package api import ( - "context" "fmt" "net/http" "net/url" "strings" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" - log "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/sets" - - "github.com/openshift/sippy/pkg/apis/cache" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" - bqclient "github.com/openshift/sippy/pkg/bigquery" ) -type releaseGenerator struct { - client *bqclient.Client -} - -func (r *releaseGenerator) ListReleases(ctx context.Context) ([]v1.Release, []error) { - releases, err := GetReleasesFromBigQuery(ctx, r.client) - if err != nil { - log.WithError(err).Error("error getting releases from bigquery") - return releases, []error{err} - } - return releases, nil -} - -// GetReleases gets all the releases defined in the BQ Releases table. -func GetReleases(ctx context.Context, bqc *bqclient.Client, forceRefresh bool) ([]v1.Release, error) { - releaseGen := releaseGenerator{bqc} - - var err error - rels, errs := GetDataFromCacheOrGenerate[[]v1.Release]( - ctx, - bqc.Cache, - cache.RequestOptions{ForceRefresh: forceRefresh}, - NewCacheSpec(v1.Release{}, "Releases~", nil), // no cache options needed here, global list - releaseGen.ListReleases, - []v1.Release{}) - if len(errs) > 0 { - err = errs[0] - } - return rels, err -} - // VariantsStringToSet converts comma separated variant string into a set; also validates that the variants are known func VariantsStringToSet(allJobVariants crtest.JobVariants, variantsString string) (sets.Set[string], error) { variantSet := sets.New[string]() diff --git a/pkg/dataloader/releasedefloader/releasedefloader_test.go b/pkg/dataloader/releasedefloader/releasedefloader_test.go new file mode 100644 index 0000000000..92ca2811b8 --- /dev/null +++ b/pkg/dataloader/releasedefloader/releasedefloader_test.go @@ -0,0 +1,137 @@ +package releasedefloader + +import ( + "testing" + "time" + + "cloud.google.com/go/bigquery" + "cloud.google.com/go/civil" + "github.com/lib/pq" + "github.com/stretchr/testify/assert" + + sippyv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" +) + +func TestReleaseRowToDefinition(t *testing.T) { + tests := []struct { + name string + row sippyv1.ReleaseRow + expectedRelease string + expectedMajor int + expectedMinor int + expectedPatch *int + expectedPrevious string + expectedProduct string + expectedStatus string + expectedHasGA bool + expectedHasDevel bool + expectedCapabilities pq.StringArray + }{ + { + name: "fully populated release", + row: sippyv1.ReleaseRow{ + Release: "4.22", + Major: 4, + Minor: 22, + PreviousRelease: bigquery.NullString{Valid: true, StringVal: "4.21"}, + Product: bigquery.NullString{Valid: true, StringVal: "OCP"}, + ReleaseStatus: bigquery.NullString{Valid: true, StringVal: "Full Support"}, + GADate: bigquery.NullDate{Valid: true, Date: civil.Date{Year: 2026, Month: 6, Day: 9}}, + DevelStartDate: civil.Date{Year: 2025, Month: 12, Day: 1}, + Capabilities: []sippyv1.ReleaseCapability{"metrics", "componentReadiness"}, + }, + expectedRelease: "4.22", + expectedMajor: 4, + expectedMinor: 22, + expectedPrevious: "4.21", + expectedProduct: "OCP", + expectedStatus: "Full Support", + expectedHasGA: true, + expectedHasDevel: true, + expectedCapabilities: pq.StringArray{"componentReadiness", "metrics"}, + }, + { + name: "in-development release with no GA date", + row: sippyv1.ReleaseRow{ + Release: "5.0", + Major: 5, + Minor: 0, + Product: bigquery.NullString{Valid: true, StringVal: "OCP"}, + ReleaseStatus: bigquery.NullString{Valid: true, StringVal: "Development"}, + DevelStartDate: civil.Date{Year: 2026, Month: 1, Day: 15}, + Capabilities: []sippyv1.ReleaseCapability{"componentReadiness"}, + }, + expectedRelease: "5.0", + expectedMajor: 5, + expectedMinor: 0, + expectedProduct: "OCP", + expectedStatus: "Development", + expectedHasGA: false, + expectedHasDevel: true, + expectedCapabilities: pq.StringArray{"componentReadiness"}, + }, + { + name: "null optional fields", + row: sippyv1.ReleaseRow{ + Release: "automation", + PreviousRelease: bigquery.NullString{Valid: false}, + Product: bigquery.NullString{Valid: false}, + ReleaseStatus: bigquery.NullString{Valid: false}, + GADate: bigquery.NullDate{Valid: false}, + }, + expectedRelease: "automation", + expectedHasGA: false, + expectedHasDevel: false, + expectedCapabilities: pq.StringArray{}, + }, + { + name: "release with patch version", + row: sippyv1.ReleaseRow{ + Release: "4.22.1", + Major: 4, + Minor: 22, + Patch: bigquery.NullInt64{Valid: true, Int64: 1}, + }, + expectedRelease: "4.22.1", + expectedMajor: 4, + expectedMinor: 22, + expectedPatch: intPtr(1), + expectedHasDevel: false, + expectedCapabilities: pq.StringArray{}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + def := ReleaseRowToDefinition(tc.row) + + assert.Equal(t, tc.expectedRelease, def.Release) + assert.Equal(t, tc.expectedMajor, def.Major) + assert.Equal(t, tc.expectedMinor, def.Minor) + assert.Equal(t, tc.expectedPatch, def.Patch) + assert.Equal(t, tc.expectedPrevious, def.PreviousRelease) + assert.Equal(t, tc.expectedProduct, def.Product) + assert.Equal(t, tc.expectedStatus, def.Status) + + if tc.expectedHasGA { + assert.NotNil(t, def.GADate) + assert.Equal(t, time.UTC, def.GADate.Location()) + } else { + assert.Nil(t, def.GADate) + } + + if tc.expectedHasDevel { + assert.NotNil(t, def.DevelopmentStartDate) + assert.Equal(t, time.UTC, def.DevelopmentStartDate.Location()) + } else { + assert.Nil(t, def.DevelopmentStartDate) + } + + assert.Equal(t, tc.expectedCapabilities, def.Capabilities) + }) + } +} + +func intPtr(i int) *int { + return &i +} diff --git a/pkg/mcp/tools/releases.go b/pkg/mcp/tools/releases.go index 7ef6e6834d..6df402fcb4 100644 --- a/pkg/mcp/tools/releases.go +++ b/pkg/mcp/tools/releases.go @@ -40,8 +40,10 @@ func (rt *ReleasesTool) GetHandler() func(ctx context.Context, request mcp.CallT return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { log.Debug("Handling get_releases tool call") - // Get releases from BigQuery (never force refresh for MCP) - releases, err := api.GetReleases(ctx, rt.deps.BigQueryClient, false) + releases, err := api.GetReleasesFromDB(ctx, rt.deps.DBClient) + if err != nil && rt.deps.BigQueryClient != nil { + releases, err = api.GetReleasesFromBigQuery(ctx, rt.deps.BigQueryClient) + } if err != nil { log.WithError(err).Error("error querying releases") return rt.CreateErrorResponse(fmt.Errorf("error querying releases: %w", err)) diff --git a/pkg/sippyserver/server.go b/pkg/sippyserver/server.go index 31a659383f..a25e7b2dac 100644 --- a/pkg/sippyserver/server.go +++ b/pkg/sippyserver/server.go @@ -197,13 +197,8 @@ type Server struct { rateLimiters map[string]*rateLimiter } -// getReleases returns release data, preferring the BigQuery client with caching -// when available, falling back to the data provider for mock mode. -func (s *Server) getReleases(ctx context.Context, forceRefresh ...bool) ([]sippyv1.Release, error) { - if s.bigQueryClient != nil { - refresh := len(forceRefresh) > 0 && forceRefresh[0] - return api.GetReleases(ctx, s.bigQueryClient, refresh) - } +// getReleases returns release data via the configured data provider. +func (s *Server) getReleases(ctx context.Context) ([]sippyv1.Release, error) { if s.crDataProvider != nil { return s.crDataProvider.QueryReleases(ctx) } @@ -999,8 +994,8 @@ func (s *Server) jsonTestRunsAndOutputsFromBigQuery(w http.ResponseWriter, req * outputs, err := api.GetTestRunsAndOutputsFromBigQuery(req.Context(), s.bigQueryClient, testID, prowJobRunIDList, prowJobNames, includeSuccess, startDate, endDate) if err != nil { - log.WithError(err).Error("error querying test runs from bigquery") - failureResponse(w, http.StatusInternalServerError, "error querying test runs from bigquery") + log.WithError(err).Error("error querying test runs") + failureResponse(w, http.StatusInternalServerError, "error querying test runs") return } @@ -1014,11 +1009,11 @@ func (s *Server) jsonComponentTestVariantsFromBigQuery(w http.ResponseWriter, re } outputs, errs := componentreadiness.GetComponentTestVariants(req.Context(), s.crDataProvider) if len(errs) > 0 { - log.Warningf("%d errors were encountered while querying test variants from big query:", len(errs)) + log.Warningf("%d errors were encountered while querying test variants:", len(errs)) for _, err := range errs { log.Error(err.Error()) } - failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying test variants from big query: %v", errs)) + failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying test variants: %v", errs)) return } api.RespondWithJSON(http.StatusOK, w, outputs) @@ -1031,11 +1026,11 @@ func (s *Server) jsonJobVariantsFromBigQuery(w http.ResponseWriter, req *http.Re } outputs, errs := componentreadiness.GetJobVariants(req.Context(), s.crDataProvider) if len(errs) > 0 { - log.Warningf("%d errors were encountered while querying job variants from big query:", len(errs)) + log.Warningf("%d errors were encountered while querying job variants:", len(errs)) for _, err := range errs { log.Error(err.Error()) } - failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying job variants from big query: %v", errs)) + failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying job variants: %v", errs)) return } api.RespondWithJSON(http.StatusOK, w, outputs) @@ -1126,7 +1121,7 @@ func (s *Server) getComponentReportFromRequest(req *http.Request) (componentrepo baseURL, ) if len(errs) > 0 { - return componentreport.ComponentReport{}, fmt.Errorf("error querying component from big query: %v", errs) + return componentreport.ComponentReport{}, fmt.Errorf("error querying component: %v", errs) } // Add any warnings from parsing to the report @@ -1172,11 +1167,11 @@ func (s *Server) jsonComponentReportTestDetailsFromBigQuery(w http.ResponseWrite baseURL := api.GetBaseFrontendURL(req) outputs, errs := componentreadiness.GetTestDetails(req.Context(), s.crDataProvider, s.db, reqOptions, allReleases, baseURL) if len(errs) > 0 { - log.Warningf("%d errors were encountered while querying component test details from big query:", len(errs)) + log.Warningf("%d errors were encountered while querying component test details:", len(errs)) for _, err := range errs { log.Error(err.Error()) } - failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying component test details from big query: %v", errs)) + failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying component test details: %v", errs)) return } api.RespondWithJSON(http.StatusOK, w, outputs) @@ -1244,15 +1239,14 @@ func (s *Server) jsonTestDetailsReportFromDB(w http.ResponseWriter, req *http.Re } func (s *Server) jsonReleasesReportFromDB(w http.ResponseWriter, req *http.Request) { - forceRefresh := req.URL.Query().Get("forceRefresh") != "" - releases, err := s.getReleases(req.Context(), forceRefresh) + releases, err := s.getReleases(req.Context()) if err != nil { log.WithError(err).Error("error querying releases") failureResponse(w, http.StatusInternalServerError, "error querying releases") return } - // Get last updated time from database if available + // Get last updated time if available var lastUpdated time.Time if s.db != nil { type LastUpdatedQuery struct { @@ -2407,7 +2401,7 @@ func (s *Server) Serve() { }, { EndpointPath: "/api/autocomplete/{field}", - Description: "Autocompletes queries from database", + Description: "Autocompletes queries", Capabilities: []string{LocalDBCapability}, HandlerFunc: s.jsonAutocompleteFromDB, }, diff --git a/test/e2e/componentreadiness/componentreadiness_test.go b/test/e2e/componentreadiness/componentreadiness_test.go index 770744a1cf..f7d56e0573 100644 --- a/test/e2e/componentreadiness/componentreadiness_test.go +++ b/test/e2e/componentreadiness/componentreadiness_test.go @@ -62,9 +62,9 @@ func TestRegressionCacheLoader(t *testing.T) { require.NoError(t, err, "error parsing seed views") require.Greater(t, len(sippyViews.ComponentReadiness), 0, "no views found in seed-views.yaml") - // Get release configs from BigQuery - releaseConfigs, err := api.GetReleasesFromBigQuery(ctx, bqClient) - require.NoError(t, err, "error getting releases from bigquery") + // Get release configs from PostgreSQL + releaseConfigs, err := api.GetReleasesFromDB(ctx, dbc) + require.NoError(t, err, "error getting releases from postgres") // Build a regression store regressionStore := componentreadiness.NewPostgresRegressionStore(dbc, nil)