Skip to content

TRT-2736: Fix sippy_prow_jobs_loaded metric to report total queried count #3782

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-bot merged 1 commit into from
Jul 20, 2026
Merged

TRT-2736: Fix sippy_prow_jobs_loaded metric to report total queried count #3782

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 3 additions & 9 deletions pkg/dataloader/prowloader/prow.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (
"regexp"
"strconv"
"sync"
"sync/atomic"
"time"

"cloud.google.com/go/bigquery"
Expand Down Expand Up @@ -72,8 +71,6 @@ type ProwLoader struct {
releaseRegexps map[string][]*regexp.Regexp
config *v1config.SippyConfig
ghCommenter *commenter.GitHubCommenter
jobsImportedCount atomic.Int32
jobsProcessedCount atomic.Int32
gcsClient *storage.Client
promPusher *push.Pusher
loadSince *time.Time
Expand Down Expand Up @@ -259,6 +256,8 @@ func (pl *ProwLoader) Load() {
pl.errors = append(pl.errors, errors.Wrap(err, "error pre-fetching labels from BigQuery"))
}

prowLoaderQueriedMetricGauge.Set(float64(len(prowJobs)))

// Match jobs to releases and bulk-upsert ProwJob definitions before
// the concurrent processing loop. The prowJobCache is read-only after
// this point.
Expand All @@ -274,7 +273,6 @@ func (pl *ProwLoader) Load() {
queue := make(chan *prow.ProwJob)
results := make(chan *jobRunResult, len(entries))
fetchErrsCh := make(chan error, len(entries))
total := len(entries)

go func() {
defer close(queue)
Expand Down Expand Up @@ -306,8 +304,6 @@ func (pl *ProwLoader) Load() {
if result != nil {
results <- result
}
pl.jobsImportedCount.Add(1)
log.Infof("%d of %d job runs processed", pl.jobsImportedCount.Load(), total)
}
}(fetchCtx)
}
Expand Down Expand Up @@ -339,9 +335,7 @@ func (pl *ProwLoader) Load() {
log.Infof("finished importing new job runs in %+v", time.Since(start))

if pl.promPusher != nil {
prowLoaderQueriedMetricGauge.Set(float64(pl.jobsImportedCount.Load()))
pl.promPusher.Collector(prowLoaderQueriedMetricGauge)
prowLoaderProcessedMetricGauge.Set(float64(pl.jobsProcessedCount.Load()))
pl.promPusher.Collector(prowLoaderProcessedMetricGauge)
}
}
Expand Down Expand Up @@ -1037,7 +1031,6 @@ func (pl *ProwLoader) fetchJobRunResult(ctx context.Context, pj *prow.ProwJob) (
return nil, err
}

pl.jobsProcessedCount.Add(1)
return result, nil
}

Expand Down Expand Up @@ -1240,6 +1233,7 @@ func (pl *ProwLoader) accumulateAndWriteJobRuns(ctx context.Context, results <-c
if total > 0 {
log.WithField("runs", total).Info("all job run batches committed")
}
prowLoaderProcessedMetricGauge.Set(float64(total))
return nil
}

Expand Down