Switch release consumers from BigQuery to PostgreSQL

Switch all release data consumers to read from the release_definitions
table in PostgreSQL, populated by the release-definitions loader from
the prior commit.

Key changes:
- getReleases() in server.go reads exclusively from PG
- Remove QueryReleases and QueryReleaseDates from the DataProvider
  interface and both implementations (BQ and PG providers)
- GetComponentReport, RefreshMetricsDB, and syncRegressions call
  GetReleasesFromDB directly instead of going through the provider
- Add GetReleaseDatesFromDB to derive CR time ranges from PG
- Update releasefallback and test_details to use GetReleaseDatesFromDB
- Delete BQ releasedates.go
- Switch automatejira, MCP releases tool, and job_runs to use PG
- Remove hardcoded releaseMetadata map from PG provider
- Fix stale "from big query" error messages in server.go

Ref: TRT-2734

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: mstaeble

cmd/sippy/automatejira.go

@@ -152,7 +152,11 @@ func NewAutomateJiraCommand() *cobra.Command {
 			if err != nil {
 				log.WithError(err).Fatal("unable to load views")
 			}
-			releases, err := api.GetReleases(context.Background(), bigQueryClient, false)
+			dbc, err := f.PostgresFlags.GetDBClient()
+			if err != nil {
+				log.WithError(err).Fatal("unable to connect to postgres")
+			}
+			releases, err := api.GetReleasesFromDB(ctx, dbc)
 			if err != nil {
 				log.WithError(err).Fatal("error querying releases")
 			}
@@ -178,10 +182,6 @@ func NewAutomateJiraCommand() *cobra.Command {
 				return errors.WithMessage(err, "error validating options")
 			}
 
-			dbc, err := f.PostgresFlags.GetDBClient()
-			if err != nil {
-				log.WithError(err).Fatal("unable to connect to postgres")
-			}
 			j, err := jiraautomator.NewJiraAutomator(
 				jiraClient, bigQueryClient, provider, dbc, cacheOpts,
 				views.ComponentReadiness, releases, f.SippyURL, f.JiraAccount,

cmd/sippy/load.go

@@ -37,7 +37,7 @@ import (
 	"github.com/openshift/sippy/pkg/dataloader/prowloader"
 	"github.com/openshift/sippy/pkg/dataloader/prowloader/gcs"
 	"github.com/openshift/sippy/pkg/dataloader/prowloader/github"
-	releasedefloader "github.com/openshift/sippy/pkg/dataloader/releasedefloader"
+	"github.com/openshift/sippy/pkg/dataloader/releasedefloader"
 	"github.com/openshift/sippy/pkg/dataloader/releaseloader"
 	"github.com/openshift/sippy/pkg/dataloader/testownershiploader"
 	"github.com/openshift/sippy/pkg/db"
@@ -176,9 +176,12 @@ func NewLoadCommand() *cobra.Command {
 				}
 			}
 			if len(releaseConfigs) == 0 && bigqueryErr == nil {
-				releaseConfigs, err = api.GetReleasesFromBigQuery(context.Background(), bqc)
-				if err != nil {
-					return errors.Wrapf(err, "error querying releases from bigquery")
+				rows, bqErr := api.GetReleaseRowsFromBigQuery(context.Background(), bqc)
+				if bqErr != nil {
+					return errors.Wrapf(bqErr, "error querying releases from bigquery")
+				}
+				for _, row := range rows {
+					releaseConfigs = append(releaseConfigs, api.DefinitionToRelease(releasedefloader.ReleaseRowToDefinition(row)))
 				}
 			}
 

cmd/sippy/seed_data.go

@@ -16,6 +16,7 @@ import (
 	"github.com/spf13/pflag"
 	"gopkg.in/yaml.v3"
 
+	"github.com/openshift/sippy/pkg/api"
 	componentreadiness "github.com/openshift/sippy/pkg/api/componentreadiness"
 	pgprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/postgres"
 	"github.com/openshift/sippy/pkg/api/componentreadiness/utils"
@@ -746,7 +747,7 @@ func syncRegressions(dbc *db.DB) error {
 	provider := pgprovider.NewPostgresProvider(dbc, nil)
 	ctx := context.Background()
 
-	releases, err := provider.QueryReleases(ctx)
+	releases, err := api.GetReleasesFromDB(ctx, dbc)
 	if err != nil {
 		return fmt.Errorf("querying releases: %w", err)
 	}

pkg/api/componentreadiness/component_report.go

@@ -82,7 +82,7 @@ func GetComponentReport(
 	reqOptions reqopts.RequestOptions,
 	baseURL string,
 ) (report crtype.ComponentReport, errs []error) {
-	releaseConfigs, err := provider.QueryReleases(ctx)
+	releaseConfigs, err := api.GetReleasesFromDB(ctx, dbc)
 	if err != nil {
 		return report, []error{err}
 	}
@@ -278,7 +278,7 @@ func (c *ComponentReportGenerator) initializeMiddleware() {
 	c.middlewares = middleware.List{}
 	// Initialize all our middleware applicable to this request.
 	if c.ReqOptions.AdvancedOption.IncludeMultiReleaseAnalysis && c.ReqOptions.SampleRelease.PullRequestOptions == nil {
-		c.middlewares = append(c.middlewares, releasefallback.NewReleaseFallbackMiddleware(c.dataProvider, c.ReqOptions, c.releaseConfigs))
+		c.middlewares = append(c.middlewares, releasefallback.NewReleaseFallbackMiddleware(c.dataProvider, c.ReqOptions, c.releaseConfigs, c.dbc))
 	}
 	if c.dbc != nil {
 		c.middlewares = append(c.middlewares, regressiontracker.NewRegressionTrackerMiddleware(c.dbc, c.ReqOptions))

pkg/api/componentreadiness/dataprovider/bigquery/provider.go

@@ -19,7 +19,6 @@ import (
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/crtest"
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts"
 	apiCache "github.com/openshift/sippy/pkg/apis/cache"
-	v1 "github.com/openshift/sippy/pkg/apis/sippy/v1"
 	bqcachedclient "github.com/openshift/sippy/pkg/bigquery"
 	"github.com/openshift/sippy/pkg/bigquery/bqlabel"
 	"github.com/openshift/sippy/pkg/util/param"
@@ -170,14 +169,6 @@ func (p *BigQueryProvider) QueryJobVariants(ctx context.Context) (crtest.JobVari
 	return variants, nil
 }
 
-func (p *BigQueryProvider) QueryReleaseDates(ctx context.Context, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) {
-	return GetReleaseDatesFromBigQuery(ctx, p.client, reqOptions)
-}
-
-func (p *BigQueryProvider) QueryReleases(ctx context.Context) ([]v1.Release, error) {
-	return apiPkg.GetReleasesFromBigQuery(ctx, p.client)
-}
-
 func (p *BigQueryProvider) QueryUniqueVariantValues(ctx context.Context, field string, nested bool) ([]string, error) {
 	unnest := ""
 	if nested {

pkg/api/componentreadiness/dataprovider/bigquery/releasedates.go

@@ -8,7 +8,6 @@ import (
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/crtest"
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts"
 	"github.com/openshift/sippy/pkg/apis/cache"
-	v1 "github.com/openshift/sippy/pkg/apis/sippy/v1"
 )
 
 // TestStatusQuerier fetches aggregated test pass/fail counts.
@@ -40,12 +39,6 @@ type MetadataQuerier interface {
 	// QueryJobVariants returns all variant names and their possible values.
 	QueryJobVariants(ctx context.Context) (crtest.JobVariants, []error)
 
-	// QueryReleaseDates returns the time ranges for each known release.
-	QueryReleaseDates(ctx context.Context, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error)
-
-	// QueryReleases returns known release configurations.
-	QueryReleases(ctx context.Context) ([]v1.Release, error)
-
 	// QueryUniqueVariantValues returns distinct values for a variant column
 	// from the past 60 days.
 	QueryUniqueVariantValues(ctx context.Context, field string, nested bool) ([]string, error)

pkg/api/componentreadiness/dataprovider/interface.go

@@ -18,7 +18,6 @@ import (
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/crtest"
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts"
 	"github.com/openshift/sippy/pkg/apis/cache"
-	v1 "github.com/openshift/sippy/pkg/apis/sippy/v1"
 	"github.com/openshift/sippy/pkg/db"
 )
 
@@ -133,98 +132,6 @@ func (p *PostgresProvider) QueryJobVariants(ctx context.Context) (crtest.JobVari
 	return variants, nil
 }
 
-// releaseMetadata holds hardcoded release info for known releases.
-// This avoids needing a releases table — we derive release names from prow_jobs
-// and fill in metadata from this map.
-var releaseMetadata = map[string]struct {
-	previousRelease string
-	gaOffsetDays    int    // 0 = no GA date (in development)
-	product         string // empty = defaults to "OCP"
-}{
-	"4.17": {previousRelease: "4.16", gaOffsetDays: -540},
-	"4.18": {previousRelease: "4.17", gaOffsetDays: -395},
-	"4.19": {previousRelease: "4.18", gaOffsetDays: -289},
-	"4.20": {previousRelease: "4.19", gaOffsetDays: -163},
-	"4.21": {previousRelease: "4.20", gaOffsetDays: -58},
-	"4.22": {previousRelease: "4.21"},
-	"5.0":  {previousRelease: "4.22"},
-}
-
-func (p *PostgresProvider) QueryReleases(ctx context.Context) ([]v1.Release, error) {
-	var releaseNames []string
-	err := p.dbc.DB.WithContext(ctx).Raw(`SELECT DISTINCT release FROM prow_jobs WHERE deleted_at IS NULL ORDER BY release DESC`).
-		Pluck("release", &releaseNames).Error
-	if err != nil {
-		return nil, fmt.Errorf("querying releases: %w", err)
-	}
-
-	caps := map[v1.ReleaseCapability]bool{
-		v1.ComponentReadinessCap: true,
-		v1.FeatureGatesCap:       true,
-		v1.MetricsCap:            true,
-		v1.PayloadTagsCap:        true,
-		v1.SippyClassicCap:       true,
-	}
-
-	now := time.Now().UTC()
-	var releases []v1.Release
-	for _, name := range releaseNames {
-		rel := v1.Release{
-			Release:      name,
-			Capabilities: caps,
-			Product:      "OCP",
-		}
-		if meta, ok := releaseMetadata[name]; ok {
-			rel.PreviousRelease = meta.previousRelease
-			if meta.gaOffsetDays != 0 {
-				ga := now.AddDate(0, 0, meta.gaOffsetDays)
-				rel.GADate = &ga
-			}
-			if meta.product != "" {
-				rel.Product = meta.product
-			}
-		}
-		releases = append(releases, rel)
-	}
-	return releases, nil
-}
-
-func (p *PostgresProvider) QueryReleaseDates(ctx context.Context, _ reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) {
-	// Derive time ranges from actual data in the DB rather than hardcoded GA dates.
-	// This ensures fallback queries find data where it actually exists.
-	type releaseRange struct {
-		Release string
-		Start   time.Time
-		End     time.Time
-	}
-	var ranges []releaseRange
-	err := p.dbc.DB.WithContext(ctx).Raw(`
-		SELECT pj.release,
-		       MIN(pjr.timestamp) AS start,
-		       MAX(pjr.timestamp) AS end
-		FROM prow_job_runs pjr
-		JOIN prow_jobs pj ON pj.id = pjr.prow_job_id
-		WHERE pj.deleted_at IS NULL AND pjr.deleted_at IS NULL
-		GROUP BY pj.release
-		ORDER BY pj.release DESC
-	`).Scan(&ranges).Error
-	if err != nil {
-		return nil, []error{fmt.Errorf("querying release dates: %w", err)}
-	}
-
-	var dates []crtest.ReleaseTimeRange
-	for _, r := range ranges {
-		start := r.Start
-		end := r.End
-		dates = append(dates, crtest.ReleaseTimeRange{
-			Release: r.Release,
-			Start:   &start,
-			End:     &end,
-		})
-	}
-	return dates, nil
-}
-
 func (p *PostgresProvider) QueryUniqueVariantValues(ctx context.Context, field string, nested bool) ([]string, error) {
 	if nested {
 		// Return all variant key names

pkg/api/componentreadiness/dataprovider/postgres/provider.go

@@ -16,6 +16,7 @@ import (
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails"
 	apiCache "github.com/openshift/sippy/pkg/apis/cache"
 	v1 "github.com/openshift/sippy/pkg/apis/sippy/v1"
+	"github.com/openshift/sippy/pkg/db"
 	log "github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/sets"
 
@@ -35,9 +36,11 @@ func NewReleaseFallbackMiddleware(
 	provider dataprovider.DataProvider,
 	reqOptions reqopts.RequestOptions,
 	releaseConfigs []v1.Release,
+	dbc *db.DB,
 ) *ReleaseFallback {
 	return &ReleaseFallback{
 		dataProvider:   provider,
+		dbc:            dbc,
 		log:            log.WithField("middleware", "ReleaseFallback"),
 		reqOptions:     reqOptions,
 		releaseConfigs: releaseConfigs,
@@ -55,6 +58,7 @@ func NewReleaseFallbackMiddleware(
 // This is done when we have sufficient test coverage, and a better pass rate.
 type ReleaseFallback struct {
 	dataProvider               dataprovider.DataProvider
+	dbc                        *db.DB
 	cachedFallbackTestStatuses *FallbackReleases
 	log                        log.FieldLogger
 	reqOptions                 reqopts.RequestOptions
@@ -173,7 +177,7 @@ func (r *ReleaseFallback) PostAnalysis(testKey crtest.Identification, testStats
 func (r *ReleaseFallback) getFallbackBaseQueryStatus(ctx context.Context,
 	allJobVariants crtest.JobVariants,
 	release string, start, end time.Time) []error {
-	generator := newFallbackTestQueryReleasesGenerator(r.dataProvider, r.reqOptions, allJobVariants, release, start, end, r.releaseConfigs)
+	generator := newFallbackTestQueryReleasesGenerator(r.dataProvider, r.dbc, r.reqOptions, allJobVariants, release, start, end, r.releaseConfigs)
 
 	cachedFallbackTestStatuses, errs := api.GetDataFromCacheOrGenerate[*FallbackReleases](
 		ctx, r.dataProvider.Cache(), r.reqOptions.CacheOption,
@@ -193,9 +197,9 @@ func (r *ReleaseFallback) QueryTestDetails(ctx context.Context, wg *sync.WaitGro
 	r.log.Infof("Querying fallback override test statuses for %d test ID options", len(r.reqOptions.TestIDOptions))
 
 	// Lookup all release dates, we're going to need them
-	timeRanges, errs := r.dataProvider.QueryReleaseDates(ctx, r.reqOptions)
-	if errs != nil {
-		utils.EnqueueAsync(wg, errCh, errs...)
+	timeRanges, err := api.GetReleaseDatesFromDB(ctx, r.dbc, r.reqOptions)
+	if err != nil {
+		utils.EnqueueAsync(wg, errCh, err)
 		return
 	}
 
@@ -296,6 +300,7 @@ func (r *ReleaseFallback) TestDetailsAnalyze(report *testdetails.Report) error {
 // each, which can then be used to return the best basis data from those past releases for comparison.
 type fallbackTestQueryReleasesGenerator struct {
 	dataProvider               dataprovider.DataProvider
+	dbc                        *db.DB
 	cacheOption                apiCache.RequestOptions
 	allJobVariants             crtest.JobVariants
 	BaseRelease                string
@@ -309,6 +314,7 @@ type fallbackTestQueryReleasesGenerator struct {
 
 func newFallbackTestQueryReleasesGenerator(
 	provider dataprovider.DataProvider,
+	dbc *db.DB,
 	reqOptions reqopts.RequestOptions,
 	allJobVariants crtest.JobVariants,
 	release string, start, end time.Time,
@@ -317,6 +323,7 @@ func newFallbackTestQueryReleasesGenerator(
 
 	generator := fallbackTestQueryReleasesGenerator{
 		dataProvider:   provider,
+		dbc:            dbc,
 		cacheOption:    reqOptions.CacheOption,
 		allJobVariants: allJobVariants,
 		BaseRelease:    release,
@@ -335,7 +342,7 @@ type fallbackTestQueryReleasesGeneratorCacheKey struct {
 	BaseEnd     time.Time
 	// VariantDBGroupBy is the only field within VariantOption that is used here
 	VariantDBGroupBy sets.Set[string]
-	// CRTimeRoundingFactor is used by GetReleaseDatesFromBigQuery
+	// CRTimeRoundingFactor is used by GetReleaseDatesFromDB
 	CRTimeRoundingFactor time.Duration
 	CRTimeRoundingOffset time.Duration
 	// KeyTestNames affects the BuildComponentReportQuery results via filtering logic
@@ -360,10 +367,10 @@ func (f *fallbackTestQueryReleasesGenerator) getCacheKey() fallbackTestQueryRele
 func (f *fallbackTestQueryReleasesGenerator) getTestFallbackReleases(ctx context.Context) (*FallbackReleases, []error) {
 	wg := sync.WaitGroup{}
 	f.CachedFallbackTestStatuses = newFallbackReleases()
-	timeRanges, errs := f.dataProvider.QueryReleaseDates(ctx, f.ReqOptions)
+	timeRanges, err := api.GetReleaseDatesFromDB(ctx, f.dbc, f.ReqOptions)
 
-	if errs != nil {
-		return nil, errs
+	if err != nil {
+		return nil, []error{err}
 	}
 
 	selectedTimeRanges := calculateDefaultFallbackReleases(f.BaseRelease, timeRanges, f.releaseConfigs)

pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go

@@ -164,7 +164,7 @@ func Test_PreAnalysis(t *testing.T) {
 	}
 	for i, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			rfb := NewReleaseFallbackMiddleware(nil, test.reqOpts, releaseConfigs)
+			rfb := NewReleaseFallbackMiddleware(nil, test.reqOpts, releaseConfigs, nil)
 			rfb.cachedFallbackTestStatuses = &tests[i].fallbackReleases
 			err := rfb.PreAnalysis(test.testKey, test.testStats)
 			assert.NoError(t, err)