TRT-2734: switch release consumers from BigQuery to PostgreSQL

cmd/sippy/automatejira.go

@@ -14,9 +14,7 @@ import (
 	"github.com/spf13/pflag"
 	"k8s.io/apimachinery/pkg/util/sets"
 
-	"github.com/openshift/sippy/pkg/api"
 	"github.com/openshift/sippy/pkg/api/componentreadiness"
-	bqprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/bigquery"
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/crtest"
 	"github.com/openshift/sippy/pkg/apis/cache"
 	jiratype "github.com/openshift/sippy/pkg/apis/jira/v1"
@@ -34,6 +32,7 @@ type AutomateJiraFlags struct {
 	ConfigFlags             *configflags.ConfigFlags
 	PostgresFlags           *flags.PostgresFlags
 	JiraFlags               flags.JiraFlags
+	DataProvider            string
 	SippyURL                string
 	IncludeComponentsStr    string
 	// IncludeComponents is a set of string in the format of jiraProject:jiraComponent
@@ -70,6 +69,7 @@ func (f *AutomateJiraFlags) BindFlags(fs *pflag.FlagSet) {
 	fs.StringArrayVar(&f.ColumnThresholdStrs, "column-threshold", f.ColumnThresholdStrs, "A threshold of red cell counts over which a jira issue will be created against a component corresponding to an interesting variant of a column (e.g. Bare Metal Hardware Provisioning for metal platform). The format of the threshold string is [variant]:[value]:[threshold] (e.g. Platform:metal:3).")
 	fs.StringVar(&f.JiraAccount, "jira-account", f.JiraAccount, "The jira account used to automate jira")
 	fs.BoolVar(&f.DryRun, "dry-run", f.DryRun, "Print the tasks of automating jiras without real interaction with jira.")
+	fs.StringVar(&f.DataProvider, "data-provider", "default", "Data provider: default, bigquery, or postgres")
 }
 
 func (f *AutomateJiraFlags) Validate(allVariants crtest.JobVariants) error {
@@ -152,11 +152,6 @@ func NewAutomateJiraCommand() *cobra.Command {
 			if err != nil {
 				log.WithError(err).Fatal("unable to load views")
 			}
-			releases, err := api.GetReleases(context.Background(), bigQueryClient, false)
-			if err != nil {
-				log.WithError(err).Fatal("error querying releases")
-			}
-
 			jiraClient, err := f.JiraFlags.GetJiraClient()
 			if err != nil {
 				return errors.WithMessage(err, "couldn't get jira client")
@@ -165,7 +160,14 @@ func NewAutomateJiraCommand() *cobra.Command {
 				return fmt.Errorf("couldn't get jira client: jira auth is not configured")
 			}
 
-			provider := bqprovider.NewBigQueryProvider(bigQueryClient)
+			dbc, err := f.PostgresFlags.GetDBClient()
+			if err != nil {
+				log.WithError(err).Warn("unable to connect to postgres, will use BigQuery for release metadata")
+			}
+			provider, err := newDataProvider(f.DataProvider, bigQueryClient, dbc, cacheClient)
+			if err != nil {
+				return err
+			}
 			allVariants, errs := componentreadiness.GetJobVariants(ctx, provider)
 			if len(errs) > 0 {
 				return fmt.Errorf("failed to get job variants: %v", errs)
@@ -178,10 +180,11 @@ func NewAutomateJiraCommand() *cobra.Command {
 				return errors.WithMessage(err, "error validating options")
 			}
 
-			dbc, err := f.PostgresFlags.GetDBClient()
+			releases, err := provider.QueryReleases(ctx)
 			if err != nil {
-				log.WithError(err).Fatal("unable to connect to postgres")
+				log.WithError(err).Fatal("error querying releases")
 			}
+
 			j, err := jiraautomator.NewJiraAutomator(
 				jiraClient, bigQueryClient, provider, dbc, cacheOpts,
 				views.ComponentReadiness, releases, f.SippyURL, f.JiraAccount,

cmd/sippy/component_readiness.go

@@ -17,7 +17,6 @@ import (
 	"gopkg.in/yaml.v3"
 
 	resources "github.com/openshift/sippy"
-	bqprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/bigquery"
 	"github.com/openshift/sippy/pkg/apis/cache"
 	v1 "github.com/openshift/sippy/pkg/apis/config/v1"
 	"github.com/openshift/sippy/pkg/bigquery"
@@ -38,8 +37,9 @@ type ComponentReadinessFlags struct {
 	APIFlags                *flags.APIFlags
 	JiraFlags               *flags.JiraFlags
 
-	Config   string
-	LogLevel string
+	Config       string
+	DataProvider string
+	LogLevel     string
 }
 
 func NewComponentReadinessCommand() *cobra.Command {
@@ -88,6 +88,7 @@ func (f *ComponentReadinessFlags) BindFlags(flagSet *pflag.FlagSet) {
 	f.APIFlags.BindFlags(flagSet)
 	f.JiraFlags.BindFlags(flagSet)
 	flagSet.StringVar(&f.LogLevel, "log-level", f.LogLevel, "Log level (trace,debug,info,warn,error) (default info)")
+	flagSet.StringVar(&f.DataProvider, "data-provider", "default", "Data provider: default, bigquery, or postgres")
 }
 
 func (f *ComponentReadinessFlags) Validate() error {
@@ -187,7 +188,10 @@ func (f *ComponentReadinessFlags) runServerMode() error {
 		log.WithError(err).Warn("unable to initialize Jira client, bug filing will be disabled")
 	}
 
-	crDataProvider := bqprovider.NewBigQueryProvider(bigQueryClient)
+	crDataProvider, err := newDataProvider(f.DataProvider, bigQueryClient, dbc, cacheClient)
+	if err != nil {
+		return err
+	}
 
 	server := sippyserver.NewServer(
 		sippyserver.ModeOpenShift,

cmd/sippy/serve.go

@@ -19,11 +19,13 @@ import (
 	resources "github.com/openshift/sippy"
 	"github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider"
 	bqprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/bigquery"
+	mixedprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/mixed"
 	pgprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/postgres"
 	"github.com/openshift/sippy/pkg/apis/cache"
 	"github.com/openshift/sippy/pkg/bigquery"
 	"github.com/openshift/sippy/pkg/bigquery/bqlabel"
 	"github.com/openshift/sippy/pkg/dataloader/prowloader/gcs"
+	"github.com/openshift/sippy/pkg/db"
 	"github.com/openshift/sippy/pkg/db/models"
 	"github.com/openshift/sippy/pkg/flags"
 	"github.com/openshift/sippy/pkg/flags/configflags"
@@ -70,7 +72,7 @@ func (f *ServerFlags) BindFlags(flagSet *pflag.FlagSet) {
 	f.ConfigFlags.BindFlags(flagSet)
 	f.APIFlags.BindFlags(flagSet)
 	f.JiraFlags.BindFlags(flagSet)
-	flagSet.StringVar(&f.DataProvider, "data-provider", "bigquery", "Data provider for component readiness: bigquery, postgres")
+	flagSet.StringVar(&f.DataProvider, "data-provider", "default", "Data provider: default, bigquery, or postgres")
 }
 
 func (f *ServerFlags) Validate() error {
@@ -110,7 +112,7 @@ func NewServeCommand() *cobra.Command {
 			var gcsClient *storage.Client
 			var crDataProvider dataprovider.DataProvider
 			switch f.DataProvider {
-			case "bigquery":
+			case "default", "bigquery":
 				if f.GoogleCloudFlags.ServiceAccountCredentialFile != "" {
 					opCtx := bqlabel.OperationalContext{
 						App:     bqlabel.AppSippy,
@@ -132,16 +134,12 @@ func NewServeCommand() *cobra.Command {
 					if bigQueryClient != nil && f.CacheFlags.EnablePersistentCaching {
 						bigQueryClient = f.CacheFlags.DecorateBiqQueryClientWithPersistentCache(bigQueryClient)
 					}
-
-					crDataProvider = bqprovider.NewBigQueryProvider(bigQueryClient)
 				}
+			}
 
-			case "postgres":
-				crDataProvider = pgprovider.NewPostgresProvider(dbc, cacheClient)
-				log.Info("Using Postgres data provider for component readiness")
-
-			default:
-				return fmt.Errorf("unknown --data-provider %q, must be bigquery or postgres", f.DataProvider)
+			crDataProvider, err = newDataProvider(f.DataProvider, bigQueryClient, dbc, cacheClient)
+			if err != nil {
+				return err
 			}
 
 			gcsClient, err = gcs.NewGCSClient(context.TODO(),
@@ -262,3 +260,33 @@ func NewServeCommand() *cobra.Command {
 	f.BindFlags(cmd.Flags())
 	return cmd
 }
+
+func newDataProvider(name string, bigQueryClient *bigquery.Client, dbc *db.DB, cacheClient cache.Cache) (dataprovider.DataProvider, error) {
+	switch name {
+	case "default":
+		if bigQueryClient != nil && dbc != nil {
+			return mixedprovider.NewMixedProvider(bigQueryClient, dbc, cacheClient), nil
+		} else if bigQueryClient != nil {
+			return bqprovider.NewBigQueryProvider(bigQueryClient), nil
+		} else if dbc != nil {
+			return pgprovider.NewPostgresProvider(dbc, cacheClient), nil
+		}
+		return nil, fmt.Errorf("default data provider requires at least one of BigQuery or PostgreSQL to be configured")
+
+	case "bigquery":
+		if bigQueryClient != nil {
+			return bqprovider.NewBigQueryProvider(bigQueryClient), nil
+		}
+		return nil, fmt.Errorf("bigquery data provider requires google-service-account-credential-file to be configured")
+
+	case "postgres":
+		if dbc == nil {
+			return nil, fmt.Errorf("postgres data provider requires a database connection")
+		}
+		log.Info("Using Postgres data provider for component readiness")
+		return pgprovider.NewPostgresProvider(dbc, cacheClient), nil
+
+	default:
+		return nil, fmt.Errorf("unknown --data-provider %q, must be default, bigquery, or postgres", name)
+	}
+}

pkg/api/componentreadiness/dataprovider/mixed/provider.go

@@ -0,0 +1,81 @@
+package mixed
+
+import (
+	"context"
+	"time"
+
+	"github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider"
+	"github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/bigquery"
+	"github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/postgres"
+	"github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus"
+	"github.com/openshift/sippy/pkg/apis/api/componentreport/crtest"
+	"github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts"
+	"github.com/openshift/sippy/pkg/apis/cache"
+	v1 "github.com/openshift/sippy/pkg/apis/sippy/v1"
+	bqcachedclient "github.com/openshift/sippy/pkg/bigquery"
+	"github.com/openshift/sippy/pkg/db"
+)
+
+var _ dataprovider.DataProvider = &MixedProvider{}
+
+// MixedProvider wraps both a BigQuery and PostgreSQL provider, routing
+// release metadata queries to PostgreSQL and everything else to BigQuery.
+type MixedProvider struct {
+	bq *bigquery.BigQueryProvider
+	pg *postgres.PostgresProvider
+}
+
+func NewMixedProvider(bqClient *bqcachedclient.Client, dbc *db.DB, cacheClient cache.Cache) *MixedProvider {
+	return &MixedProvider{
+		bq: bigquery.NewBigQueryProvider(bqClient),
+		pg: postgres.NewPostgresProvider(dbc, cacheClient),
+	}
+}
+
+func (p *MixedProvider) Cache() cache.Cache {
+	return p.bq.Cache()
+}
+
+func (p *MixedProvider) QueryReleases(ctx context.Context) ([]v1.Release, error) {
+	return p.pg.QueryReleases(ctx)
+}
+
+func (p *MixedProvider) QueryReleaseDates(ctx context.Context, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) {
+	return p.pg.QueryReleaseDates(ctx, reqOptions)
+}
+
+func (p *MixedProvider) QueryJobVariants(ctx context.Context) (crtest.JobVariants, []error) {
+	return p.bq.QueryJobVariants(ctx)
+}
+
+func (p *MixedProvider) QueryUniqueVariantValues(ctx context.Context, field string, nested bool) ([]string, error) {
+	return p.bq.QueryUniqueVariantValues(ctx, field, nested)
+}
+
+func (p *MixedProvider) QueryBaseTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions) (map[string]crstatus.TestStatus, []error) {
+	return p.bq.QueryBaseTestStatus(ctx, reqOptions)
+}
+
+func (p *MixedProvider) QuerySampleTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, includeVariants map[string][]string, start, end time.Time) (map[string]crstatus.TestStatus, []error) {
+	return p.bq.QuerySampleTestStatus(ctx, reqOptions, includeVariants, start, end)
+}
+
+func (p *MixedProvider) QueryBaseJobRunTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions) (map[string][]crstatus.TestJobRunRows, []error) {
+	return p.bq.QueryBaseJobRunTestStatus(ctx, reqOptions)
+}
+
+func (p *MixedProvider) QuerySampleJobRunTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, includeVariants map[string][]string, start, end time.Time) (map[string][]crstatus.TestJobRunRows, []error) {
+	return p.bq.QuerySampleJobRunTestStatus(ctx, reqOptions, includeVariants, start, end)
+}
+
+func (p *MixedProvider) QueryJobRuns(ctx context.Context, reqOptions reqopts.RequestOptions, release string, start, end time.Time) (map[string]dataprovider.JobRunStats, error) {
+	return p.bq.QueryJobRuns(ctx, reqOptions, release, start, end)
+}
+
+func (p *MixedProvider) QueryJobVariantValues(ctx context.Context, jobNames, variantKeys []string) (map[string]map[string]string, error) {
+	return p.bq.QueryJobVariantValues(ctx, jobNames, variantKeys)
+}
+
+func (p *MixedProvider) LookupJobVariants(ctx context.Context, jobName string) (map[string]string, error) {
+	return p.bq.LookupJobVariants(ctx, jobName)
+}

pkg/api/componentreadiness/dataprovider/postgres/provider.go

@@ -12,6 +12,7 @@ import (
 	"github.com/lib/pq"
 	"k8s.io/apimachinery/pkg/util/sets"
 
+	"github.com/openshift/sippy/pkg/api"
 	"github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider"
 	"github.com/openshift/sippy/pkg/api/componentreadiness/utils"
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus"
@@ -133,96 +134,16 @@ func (p *PostgresProvider) QueryJobVariants(ctx context.Context) (crtest.JobVari
 	return variants, nil
 }
 
-// releaseMetadata holds hardcoded release info for known releases.
-// This avoids needing a releases table — we derive release names from prow_jobs
-// and fill in metadata from this map.
-var releaseMetadata = map[string]struct {
-	previousRelease string
-	gaOffsetDays    int    // 0 = no GA date (in development)
-	product         string // empty = defaults to "OCP"
-}{
-	"4.17": {previousRelease: "4.16", gaOffsetDays: -540},
-	"4.18": {previousRelease: "4.17", gaOffsetDays: -395},
-	"4.19": {previousRelease: "4.18", gaOffsetDays: -289},
-	"4.20": {previousRelease: "4.19", gaOffsetDays: -163},
-	"4.21": {previousRelease: "4.20", gaOffsetDays: -58},
-	"4.22": {previousRelease: "4.21"},
-	"5.0":  {previousRelease: "4.22"},
-}
-
 func (p *PostgresProvider) QueryReleases(ctx context.Context) ([]v1.Release, error) {
-	var releaseNames []string
-	err := p.dbc.DB.WithContext(ctx).Raw(`SELECT DISTINCT release FROM prow_jobs WHERE deleted_at IS NULL ORDER BY release DESC`).
-		Pluck("release", &releaseNames).Error
-	if err != nil {
-		return nil, fmt.Errorf("querying releases: %w", err)
-	}
-
-	caps := map[v1.ReleaseCapability]bool{
-		v1.ComponentReadinessCap: true,
-		v1.FeatureGatesCap:       true,
-		v1.MetricsCap:            true,
-		v1.PayloadTagsCap:        true,
-		v1.SippyClassicCap:       true,
-	}
-
-	now := time.Now().UTC()
-	var releases []v1.Release
-	for _, name := range releaseNames {
-		rel := v1.Release{
-			Release:      name,
-			Capabilities: caps,
-			Product:      "OCP",
-		}
-		if meta, ok := releaseMetadata[name]; ok {
-			rel.PreviousRelease = meta.previousRelease
-			if meta.gaOffsetDays != 0 {
-				ga := now.AddDate(0, 0, meta.gaOffsetDays)
-				rel.GADate = &ga
-			}
-			if meta.product != "" {
-				rel.Product = meta.product
-			}
-		}
-		releases = append(releases, rel)
-	}
-	return releases, nil
+	return api.GetReleasesFromDB(ctx, p.dbc)
 }
 
-func (p *PostgresProvider) QueryReleaseDates(ctx context.Context, _ reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) {
-	// Derive time ranges from actual data in the DB rather than hardcoded GA dates.
-	// This ensures fallback queries find data where it actually exists.
-	type releaseRange struct {
-		Release string
-		Start   time.Time
-		End     time.Time
-	}
-	var ranges []releaseRange
-	err := p.dbc.DB.WithContext(ctx).Raw(`
-		SELECT pj.release,
-		       MIN(pjr.timestamp) AS start,
-		       MAX(pjr.timestamp) AS end
-		FROM prow_job_runs pjr
-		JOIN prow_jobs pj ON pj.id = pjr.prow_job_id
-		WHERE pj.deleted_at IS NULL AND pjr.deleted_at IS NULL
-		GROUP BY pj.release
-		ORDER BY pj.release DESC
-	`).Scan(&ranges).Error
+func (p *PostgresProvider) QueryReleaseDates(ctx context.Context, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) {
+	timeRanges, err := api.GetReleaseDatesFromDB(ctx, p.dbc, reqOptions)
 	if err != nil {
-		return nil, []error{fmt.Errorf("querying release dates: %w", err)}
-	}
-
-	var dates []crtest.ReleaseTimeRange
-	for _, r := range ranges {
-		start := r.Start
-		end := r.End
-		dates = append(dates, crtest.ReleaseTimeRange{
-			Release: r.Release,
-			Start:   &start,
-			End:     &end,
-		})
+		return nil, []error{err}
 	}
-	return dates, nil
+	return timeRanges, nil
 }
 
 func (p *PostgresProvider) QueryUniqueVariantValues(ctx context.Context, field string, nested bool) ([]string, error) {