Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
a316393
RHINENG-33842: Add a Kafka batch read loop that commits after the han…
MichaelMraka Nov 13, 2026
2565260
RHINENG-33842: Read one Kafka message as a batch of size 1.
MichaelMraka Nov 13, 2026
b5aedde
RHINENG-33842: Retry fatal Kafka batch handler errors on the same slice.
MichaelMraka Nov 13, 2026
c7922ca
RHINENG-33842: Update every Kafka reader to batched HandleMessages.
MichaelMraka Nov 13, 2026
5b26169
RHINENG-33842: Aggregate advisory updates from a Kafka batch instead …
MichaelMraka Nov 13, 2026
4a4eb11
RHINENG-33842: flag to bypass messages
MichaelMraka Nov 14, 2026
86a13ae
RHINENG-33842: Add account_advisory.updated so each row records when …
MichaelMraka Nov 14, 2026
d28c4cc
RHINENG-33842: Map account_advisory.updated on AccountAdvisory.
MichaelMraka Nov 14, 2026
1815f07
RHINENG-33842: Recount account_advisory per message advisory instead …
MichaelMraka Nov 14, 2026
5cd91a5
RHINENG-33842: Skip an advisory recount when account_advisory.updated…
MichaelMraka Nov 14, 2026
348fa71
RHINENG-33842: Key advisory update messages by account so one reader …
MichaelMraka Nov 14, 2026
c2248d5
RHINENG-33842: Serialize an advisory recount when its aggregate row i…
MichaelMraka Nov 14, 2026
8d74b90
RHINENG-33842: Keep subsecond precision on advisory update produced_a…
MichaelMraka Nov 14, 2026
d9c9a13
RHINENG-33842: Pass the Kafka message trace context to handlers of mu…
MichaelMraka Nov 15, 2026
7415d13
RHINENG-33842: Require a recount to keep the aggregate row and write …
MichaelMraka Nov 15, 2026
f5074f8
RHINENG-33842: Index system_advisories by account and advisory so rec…
MichaelMraka Nov 15, 2026
84bf188
RHINENG-33842: Publish advisory updates for both workspaces when a sy…
MichaelMraka Nov 15, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions aggregator/aggregator.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,22 @@ import (

var (
advisoryUpdateTopic string
enableBypass bool
enableNotifications bool
consumerCount int
)

func readPodConfig() {
consumerCount = utils.PodConfig.GetInt("consumer_count", 1)
// Toggle bypassing of message processing (in case something goes wrong and we need to clear the queue)
enableBypass = utils.PodConfig.GetBool("bypass", false)
enableNotifications = utils.PodConfig.GetBool("enable_notifications", false)
batchSize = utils.PodConfig.GetInt("advisory_batch_size", 4000)
flushTimeout = time.Duration(utils.PodConfig.GetInt("advisory_flush_timeout_ms", 500)) * time.Millisecond
flushTimeout = time.Duration(utils.PodConfig.GetInt("advisory_flush_timeout_ms", 5000)) * time.Millisecond
}

func configure() {
advisoryUpdateTopic = utils.FailIfEmpty(utils.CoreCfg.AdvisoryUpdateTopic, "ADVISORY_UPDATE_TOPIC")
initBuffer()
configureNotifications()
}

Expand All @@ -45,7 +47,7 @@ func runServer() {
func subscribeToAdvisoryUpdates(wg *sync.WaitGroup, readerBuilder mqueue.CreateReader) {
handler := mqueue.MakeRetryingHandler(advisoryUpdateHandler)
for i := 0; i < consumerCount; i++ {
mqueue.SpawnReader(base.Context, wg, advisoryUpdateTopic, readerBuilder, handler)
mqueue.SpawnReader(base.Context, wg, advisoryUpdateTopic, readerBuilder, batchSize, flushTimeout, handler)
utils.LogDebug("spawned advisory update reader", i)
}
utils.LogInfo("connected to kafka topics")
Expand Down
222 changes: 157 additions & 65 deletions aggregator/events.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,103 +3,195 @@ package aggregator
import (
"app/base"
"app/base/database"
"app/base/models"
"app/base/mqueue"
"app/base/utils"
"bytes"
"context"
"sync"
"errors"
"fmt"
"sort"
"time"

"github.com/bytedance/sonic"
"github.com/lib/pq"
"github.com/google/uuid"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)

var (
batchSize int
flushTimeout time.Duration
advisoryBuffer []mqueue.AdvisoryUpdateEvent
bufferLock sync.Mutex
flushTimer *time.Timer
batchSize int
flushTimeout time.Duration
)

func initBuffer() {
advisoryBuffer = make([]mqueue.AdvisoryUpdateEvent, 0, batchSize+1)
flushTimer = time.AfterFunc(87600*time.Hour, func() {
utils.LogInfo("flushing advisory buffer after timeout")
flushAdvisoryBuffer()
})
}
const advisoryCountSQL = `
SELECT count(sa.*) FILTER (WHERE sa.status_id = 0) AS systems_installable,
count(sa.*) AS systems_applicable
FROM system_advisories sa
JOIN system_inventory si
ON sa.rh_account_id = si.rh_account_id AND sa.system_id = si.id
Comment thread
sourcery-ai[bot] marked this conversation as resolved.
Resolved
WHERE si.stale = FALSE
AND sa.rh_account_id = ?
AND sa.advisory_id = ?
AND si.rh_account_id = ?
Comment thread
sourcery-ai[bot] marked this conversation as resolved.
Resolved
AND si.workspace_id = ?
GROUP BY sa.advisory_id, sa.rh_account_id, si.workspace_id`
Comment thread
sourcery-ai[bot] marked this conversation as resolved.
Resolved

func flushAdvisoryBuffer() {
bufferLock.Lock()
if len(advisoryBuffer) == 0 {
bufferLock.Unlock()
return
}
// Copy and unlock before processing so new events can accumulate while DB work runs
batch := make([]mqueue.AdvisoryUpdateEvent, len(advisoryBuffer))
copy(batch, advisoryBuffer)
advisoryBuffer = advisoryBuffer[:0]
bufferLock.Unlock()

grouped := groupAdvisoryUpdates(batch)
processAdvisoryBatch(base.Context, grouped)
type advisoryRecountCount struct {
SystemsInstallable int
SystemsApplicable int
}

func advisoryUpdateHandler(_ context.Context, m mqueue.KafkaMessage) error {
var event mqueue.AdvisoryUpdateEvent
if err := sonic.Unmarshal(m.Value, &event); err != nil {
utils.LogError("err", err, "could not deserialize advisory update event")
func advisoryUpdateHandler(ctx context.Context, msgs []mqueue.KafkaMessage) error {
if enableBypass {
utils.LogWarn("Advisory update processing bypassed")
return nil
}
events := make([]mqueue.AdvisoryUpdateEvent, 0, len(msgs))
for _, m := range msgs {
var event mqueue.AdvisoryUpdateEvent
if err := sonic.Unmarshal(m.Value, &event); err != nil {
utils.LogError("err", err, "could not deserialize advisory update event")
continue
}
events = append(events, event)
}
if len(events) == 0 {
return nil
}
return processAdvisoryBatch(ctx, groupAdvisoryUpdates(events))
}

bufferLock.Lock()
advisoryBuffer = append(advisoryBuffer, event)
flushTimer.Reset(flushTimeout)
shouldFlush := len(advisoryBuffer) >= batchSize
bufferLock.Unlock()
type advisoryRecountKey struct {
RhAccountID int
WorkspaceID uuid.UUID
AdvisoryID int64
}

if shouldFlush {
utils.LogInfo("flushing full advisory buffer")
flushAdvisoryBuffer()
}
return nil
type advisoryRecount struct {
advisoryRecountKey
ProducedAt time.Time
}

func groupAdvisoryUpdates(events []mqueue.AdvisoryUpdateEvent) map[int][]int64 {
sets := make(map[int]map[int64]struct{})
for _, e := range events {
if _, ok := sets[e.RhAccountID]; !ok {
sets[e.RhAccountID] = make(map[int64]struct{})
func groupAdvisoryUpdates(events []mqueue.AdvisoryUpdateEvent) []advisoryRecount {
newest := make(map[advisoryRecountKey]time.Time)
for i := range events {
event := &events[i]
producedAt := time.Time{}
if ts := event.ProducedAt.Time(); ts != nil {
producedAt = *ts
}
for _, id := range e.AdvisoryIDs {
sets[e.RhAccountID][id] = struct{}{}
for _, advisoryID := range event.AdvisoryIDs {
key := advisoryRecountKey{event.RhAccountID, event.WorkspaceID, advisoryID}
Comment thread
sourcery-ai[bot] marked this conversation as resolved.
Resolved
Comment thread
sourcery-ai[bot] marked this conversation as resolved.
Resolved
if existing, ok := newest[key]; !ok || producedAt.After(existing) {
newest[key] = producedAt
}
}
}

grouped := make(map[int][]int64, len(sets))
for accID, idSet := range sets {
ids := make([]int64, 0, len(idSet))
for id := range idSet {
ids = append(ids, id)
}
grouped[accID] = ids
requests := make([]advisoryRecount, 0, len(newest))
for key, producedAt := range newest {
requests = append(requests, advisoryRecount{
RhAccountID: key.RhAccountID,
WorkspaceID: key.WorkspaceID,
AdvisoryID: key.AdvisoryID,
ProducedAt: producedAt,
})
}
return grouped
sort.Slice(requests, func(i, j int) bool {
if requests[i].RhAccountID != requests[j].RhAccountID {
return requests[i].RhAccountID < requests[j].RhAccountID
}
if c := bytes.Compare(requests[i].WorkspaceID[:], requests[j].WorkspaceID[:]); c != 0 {
return c < 0
}
return requests[i].AdvisoryID < requests[j].AdvisoryID
})
return requests
}

func processAdvisoryBatch(ctx context.Context, grouped map[int][]int64) {
for rhAccountID, advisoryIDs := range grouped {
utils.LogInfo("rh_account_id", rhAccountID, "advisory_count", len(advisoryIDs), "refreshing account advisory caches")
err := database.DB(ctx).Exec("SELECT refresh_account_advisory_caches_multi(?, ?)", pq.Array(advisoryIDs), rhAccountID).Error //nolint:lll
func processAdvisoryBatch(ctx context.Context, requests []advisoryRecount) error {
utils.LogInfo("advisory_count", len(requests), "refreshing account advisory caches")

accountAdvisories := make(map[int][]int64)
for _, req := range requests {
err := recountAdvisory(ctx, req.RhAccountID, req.WorkspaceID, req.AdvisoryID, req.ProducedAt)
if err != nil {
utils.LogError("err", err, "rh_account_id", rhAccountID, "failed to refresh account advisory caches")
continue
utils.LogError("err", err, "rh_account_id", req.RhAccountID,
"failed to refresh account advisory caches")
return base.WrapFatalDBError(err, "refresh account advisory caches")
}
accountAdvisories[req.RhAccountID] = append(accountAdvisories[req.RhAccountID], req.AdvisoryID)
}

CheckAdvisoryDrift(rhAccountID, advisoryIDs)

for rhAccountID, advisoryIDs := range accountAdvisories {
if err := publishNewAdvisoryNotification(rhAccountID, advisoryIDs); err != nil {
utils.LogError("err", err, "rh_account_id", rhAccountID, "failed to publish new advisory notification")
return base.WrapFatalKafkaError(err, "publish new advisory notification")
Comment thread
sourcery-ai[bot] marked this conversation as resolved.
Resolved
}
Comment thread
sourcery-ai[bot] marked this conversation as resolved.
Resolved
}
return nil
}

// Use two-integer lock as namespace + row key.
const accountAdvisoryLockNamespace int32 = 1634890860 // "artl" in ASCII as Advisory Recount Lock Namespace

func advisoryRecountLockKey(rhAccountID int, workspaceID uuid.UUID, advisoryID int64) string {
return fmt.Sprintf("%d/%s/%d", rhAccountID, workspaceID, advisoryID)
}

func recountAdvisory(ctx context.Context, rhAccountID int, workspaceID uuid.UUID, advisoryID int64,
producedAt time.Time) error {
return database.DB(ctx).Transaction(func(tx *gorm.DB) error {
// Serialize this key even when account_advisory SELECT ... FOR UPDATE returns nothing
// so an older upsert can overwrite newer counts or a delayed delete can remove a fresh row.
err := tx.Exec("SELECT pg_advisory_xact_lock(?::integer, hashtext(?))",
accountAdvisoryLockNamespace, advisoryRecountLockKey(rhAccountID, workspaceID, advisoryID)).Error
if err != nil {
return err
}

var row models.AccountAdvisory
err = tx.Clauses(clause.Locking{Strength: "UPDATE"}).
Where("rh_account_id = ? AND workspace_id = ? AND advisory_id = ?", rhAccountID, workspaceID, advisoryID).
Take(&row).Error
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return err
}
if err == nil && row.Updated.After(producedAt) {
Comment thread
MichaelMraka marked this conversation as resolved.
Resolved
Comment thread
sourcery-ai[bot] marked this conversation as resolved.
Resolved
Comment thread
sourcery-ai[bot] marked this conversation as resolved.
Resolved
Comment thread
sourcery-ai[bot] marked this conversation as resolved.
Resolved
utils.LogWarn(
"rh_account_id", rhAccountID,
"workspace_id", workspaceID,
"advisory_id", advisoryID,
"updated", row.Updated,
"produced_at", producedAt,
"skipping advisory recount, updated is younger than produced_at",
)
return nil
}

var counts []advisoryRecountCount
err = tx.Raw(advisoryCountSQL, rhAccountID, advisoryID, rhAccountID, workspaceID).Scan(&counts).Error
if err != nil {
return err
}
if len(counts) == 0 {
return tx.Where("rh_account_id = ? AND workspace_id = ? AND advisory_id = ?",
rhAccountID, workspaceID, advisoryID).
Delete(&models.AccountAdvisory{}).Error
}

rec := models.AccountAdvisory{
AdvisoryID: advisoryID,
RhAccountID: rhAccountID,
WorkspaceID: workspaceID,
SystemsApplicable: counts[0].SystemsApplicable,
SystemsInstallable: counts[0].SystemsInstallable,
Updated: time.Now(),
Comment thread
sourcery-ai[bot] marked this conversation as resolved.
Resolved
}
return database.OnConflictUpdateMulti(tx,
[]string{"rh_account_id", "workspace_id", "advisory_id"},
"systems_installable", "systems_applicable", "updated").
Create(&rec).Error
})
}
Loading
Loading