Skip to content

Commit

Permalink
fix(HMS-2002): cleanup stats logging
Browse files Browse the repository at this point in the history
  • Loading branch information
lzap authored and adiabramovitch committed Jun 29, 2023
1 parent 69024fb commit f494205
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 9 deletions.
12 changes: 11 additions & 1 deletion internal/background/db_stats.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (

func dbStatsLoop(ctx context.Context, sleep time.Duration) {
logger := zerolog.Ctx(ctx)
logger.Debug().Msgf("Started database statistics routine")
logger.Debug().Msgf("Started database statistics routine with tick interval %.2f seconds", sleep.Seconds())
defer func() {
logger.Debug().Msgf("Database statistics routine exited")
}()
Expand All @@ -36,15 +36,25 @@ func dbStatsLoop(ctx context.Context, sleep time.Duration) {
}

func dbStatsTick(ctx context.Context) error {
logger := zerolog.Ctx(ctx)
sdao := dao.GetStatDao(ctx)
stats, err := sdao.Get(ctx)
if err != nil {
return fmt.Errorf("stats error: %w", err)
}

success, failure := 0, 0
for _, s := range stats.Usage24h {
if s.Result == "success" {
success += 1
}
if s.Result == "failure" {
failure += 1
}
metrics.SetReservations24hCount(s.Result, s.Provider, s.Count)
}
logger.Debug().Msgf("Reservation statistics for last 24 hours: success=%d, failure=%d", success, failure)

for _, s := range stats.Usage28d {
metrics.SetReservations28dCount(s.Result, s.Provider, s.Count)
}
Expand Down
13 changes: 5 additions & 8 deletions internal/background/queue_metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package background

import (
"context"
"math/rand"
"time"

"github.com/RHEnVision/provisioning-backend/internal/metrics"
Expand All @@ -12,23 +11,21 @@ import (

// jobQueueMetricLoop is a background function that runs for all workers.
// It polls job queue statistics from Redis as well as in-flight counters.
// It waits a random delay between 0 and sleep for better polling spread
// since job queue size is a global metric (redis queue length).
//
//nolint:gosec
func jobQueueMetricLoop(ctx context.Context, sleep time.Duration, name string) {
logger := zerolog.Ctx(ctx)

// spread polling intervals
randSleep := rand.Int63() % sleep.Milliseconds()
logger.Debug().Msgf("Job queue metric delay %dms", randSleep)
time.Sleep(time.Duration(randSleep) * time.Millisecond)
logger.Debug().Msgf("Started Redis statistics routine with tick interval %.2f seconds", sleep.Seconds())
defer func() {
logger.Debug().Msgf("Redis statistics routine exited")
}()
ticker := time.NewTicker(sleep)

for {
select {
case <-ticker.C:
stats := jq.Stats(ctx)
logger.Debug().Msgf("Job queue statistics: enqueued=%d, in-flight=%d", stats.EnqueuedJobs, stats.InFlight)
metrics.SetJobQueueSize(stats.EnqueuedJobs)
metrics.SetJobQueueInFlight(name, stats.InFlight)

Expand Down

0 comments on commit f494205

Please sign in to comment.