From a316393f9ad610109d6c044469a86cc5176fad34 Mon Sep 17 00:00:00 2001 From: Michael Mraka Date: Fri, 13 Nov 2026 22:21:36 +0100 Subject: [PATCH 01/17] RHINENG-33842: Add a Kafka batch read loop that commits after the handler succeeds. --- base/mqueue/batch_test.go | 177 +++++++++++++++++++++++++++++ base/mqueue/mqueue.go | 5 +- base/mqueue/mqueue_impl_gokafka.go | 114 +++++++++++++++---- base/mqueue/testing.go | 5 +- 4 files changed, 279 insertions(+), 22 deletions(-) create mode 100644 base/mqueue/batch_test.go diff --git a/base/mqueue/batch_test.go b/base/mqueue/batch_test.go new file mode 100644 index 000000000..1f8261742 --- /dev/null +++ b/base/mqueue/batch_test.go @@ -0,0 +1,177 @@ +package mqueue + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/segmentio/kafka-go" + "github.com/stretchr/testify/assert" +) + +func TestHandleMessageBatchesBlocksForFirstMessage(t *testing.T) { + ch := make(chan kafka.Message) + fetch := func(ctx context.Context) (kafka.Message, error) { + select { + case <-ctx.Done(): + return kafka.Message{}, ctx.Err() + case m := <-ch: + return m, nil + } + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan struct{}) + var got []KafkaMessage + go func() { + handleMessageBatches(ctx, "topic", fetch, func(context.Context, ...kafka.Message) error { + return nil + }, 1, time.Hour, func(_ context.Context, msgs []KafkaMessage) error { + got = msgs + cancel() + return nil + }) + close(done) + }() + + select { + case <-done: + t.Fatal("returned before the first message") + case <-time.After(30 * time.Millisecond): + } + + ch <- kafka.Message{Offset: 1, Value: []byte("one")} + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("did not return after the first message") + } + assert.Equal(t, []KafkaMessage{{Value: []byte("one")}}, got) +} + +func TestHandleMessageBatchesStopsAtN(t *testing.T) { + queued := []kafka.Message{ + {Offset: 1, Value: []byte("a")}, + {Offset: 2, Value: []byte("b")}, + {Offset: 3, Value: []byte("c")}, + } + calls := 0 + fetch := func(ctx context.Context) (kafka.Message, error) { + calls++ + if calls > len(queued) { + <-ctx.Done() + return kafka.Message{}, ctx.Err() + } + return queued[calls-1], nil + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var got []KafkaMessage + handleMessageBatches(ctx, "topic", fetch, func(_ context.Context, _ ...kafka.Message) error { + cancel() + return nil + }, 3, time.Hour, func(_ context.Context, msgs []KafkaMessage) error { + assert.Equal(t, 3, calls) + got = append([]KafkaMessage(nil), msgs...) + return nil + }) + assert.Equal(t, []KafkaMessage{ + {Value: []byte("a")}, + {Value: []byte("b")}, + {Value: []byte("c")}, + }, got) +} + +func TestHandleMessageBatchesReturnsPartialBatchOnTimeout(t *testing.T) { + var calls uint8 = 0 + fetch := func(ctx context.Context) (kafka.Message, error) { + calls++ + if calls <= 2 { + return kafka.Message{Offset: int64(calls), Value: []byte{calls}}, nil + } + <-ctx.Done() + return kafka.Message{}, ctx.Err() + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var got []KafkaMessage + var committed []kafka.Message + handleMessageBatches(ctx, "topic", fetch, func(_ context.Context, msgs ...kafka.Message) error { + committed = append(committed, msgs...) + cancel() + return nil + }, 10, 40*time.Millisecond, func(_ context.Context, msgs []KafkaMessage) error { + got = append([]KafkaMessage(nil), msgs...) + return nil + }) + assert.Equal(t, []KafkaMessage{{Value: []byte{1}}, {Value: []byte{2}}}, got) + assert.Equal(t, []kafka.Message{ + {Offset: 1, Value: []byte{1}}, + {Offset: 2, Value: []byte{2}}, + }, committed) +} + +func TestHandleMessageBatchesCancelBeforeFirstMessage(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + fetch := func(ctx context.Context) (kafka.Message, error) { + <-ctx.Done() + return kafka.Message{}, ctx.Err() + } + handled := false + committed := false + handleMessageBatches(ctx, "topic", fetch, func(context.Context, ...kafka.Message) error { + committed = true + return nil + }, 5, time.Second, func(context.Context, []KafkaMessage) error { + handled = true + return nil + }) + assert.False(t, handled) + assert.False(t, committed) +} + +func TestHandleMessageBatchesCancelWhileWaitingDropsBatch(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + calls := 0 + fetch := func(callCtx context.Context) (kafka.Message, error) { + calls++ + if calls == 1 { + return kafka.Message{Offset: 7, Value: []byte("x")}, nil + } + cancel() + <-callCtx.Done() + return kafka.Message{}, callCtx.Err() + } + handled := false + committed := false + handleMessageBatches(ctx, "topic", fetch, func(context.Context, ...kafka.Message) error { + committed = true + return nil + }, 5, time.Hour, func(context.Context, []KafkaMessage) error { + handled = true + return nil + }) + assert.False(t, handled) + assert.False(t, committed) +} + +func TestHandleMessageBatchesHandlerErrorSkipsCommit(t *testing.T) { + fetch := func(context.Context) (kafka.Message, error) { + return kafka.Message{Value: []byte("a")}, nil + } + committed := false + assert.Panics(t, func() { + handleMessageBatches(context.Background(), "topic", fetch, func(context.Context, ...kafka.Message) error { + committed = true + return nil + }, 1, time.Second, func(context.Context, []KafkaMessage) error { + return errors.New("boom") + }) + }) + assert.False(t, committed) +} diff --git a/base/mqueue/mqueue.go b/base/mqueue/mqueue.go index 06d2be990..9564a3044 100644 --- a/base/mqueue/mqueue.go +++ b/base/mqueue/mqueue.go @@ -10,6 +10,7 @@ import ( "io" "strings" "sync" + "time" "github.com/lestrrat-go/backoff/v2" "github.com/segmentio/kafka-go" @@ -19,7 +20,7 @@ const errContextCanceled = "context canceled" // By wrapping raw value we can add new methods & ensure methods of wrapped type are callable type Reader interface { - HandleMessages(ctx context.Context, handler MessageHandler) + HandleMessages(ctx context.Context, n int, wait time.Duration, handler BatchHandler) io.Closer } @@ -56,6 +57,8 @@ type KafkaMessage struct { type MessageHandler func(ctx context.Context, message KafkaMessage) error +type BatchHandler func(ctx context.Context, messages []KafkaMessage) error + func MakeRetryingHandler(handler MessageHandler) MessageHandler { return func(ctx context.Context, message KafkaMessage) error { var err error diff --git a/base/mqueue/mqueue_impl_gokafka.go b/base/mqueue/mqueue_impl_gokafka.go index 68304f810..1e3e3ecf2 100644 --- a/base/mqueue/mqueue_impl_gokafka.go +++ b/base/mqueue/mqueue_impl_gokafka.go @@ -6,6 +6,7 @@ import ( "context" "crypto/tls" "crypto/x509" + "errors" "fmt" "io" "os" @@ -17,48 +18,123 @@ import ( kafkaPlain "github.com/segmentio/kafka-go/sasl/plain" kafkaScram "github.com/segmentio/kafka-go/sasl/scram" log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/trace" ) type kafkaGoReaderImpl struct { kafka.Reader } -func (t *kafkaGoReaderImpl) HandleMessages(ctx context.Context, handler MessageHandler) { +func (t *kafkaGoReaderImpl) HandleMessages(ctx context.Context, n int, wait time.Duration, handler BatchHandler) { + handleMessageBatches(ctx, t.Config().Topic, t.FetchMessage, t.CommitMessages, n, wait, handler) +} + +func handleMessageBatches(ctx context.Context, topic string, fetch func(context.Context) (kafka.Message, error), + commit func(context.Context, ...kafka.Message) error, n int, wait time.Duration, handler BatchHandler) { + if n < 1 { + n = 1 + } for { - m, err := t.FetchMessage(ctx) + batch, err := readBatch(ctx, topic, fetch, n, wait) if err != nil { - // Both context cancellation or reader.Close() racing are expected during shutdown - if err.Error() == errContextCanceled || err == io.EOF { + if readerStopped(err) { break } - utils.LogError("err", err, "unable to read message from Kafka reader") + utils.LogError("err", err, "unable to read message batch from Kafka reader") panic(err) } - if log.IsLevelEnabled(log.TraceLevel) { - utils.LogTrace("count", len(m.Headers), "kafka message headers") - for _, h := range m.Headers { - utils.LogTrace("key", h.Key, "value", string(h.Value), "kafka message header") - } + + handlerCtx := ctx + if len(batch.app) == 1 { + handlerCtx = batch.singleCtx } - // At this level, all errors are fatal - kafkaMessage := KafkaMessage{Key: m.Key, Value: m.Value, Headers: m.Headers} - msgCtx, span := telemetry.ConsumerContext(ctx, t.Config().Topic, m.Headers) - err = handler(msgCtx, kafkaMessage) - telemetry.End(span, err) + err = handler(handlerCtx, batch.app) + batch.endSpans(err) if err != nil { utils.LogPanic("err", err, "Handler failed") } - err = t.CommitMessages(ctx, m) - if err != nil { - if err.Error() == errContextCanceled { + if err = commit(ctx, batch.raw...); err != nil { + if readerStopped(err) { break } - utils.LogError("err", err, "unable to commit kafka message") + utils.LogError("err", err, "unable to commit kafka message batch") panic(err) } } } +type fetchedBatch struct { + app []KafkaMessage + raw []kafka.Message + spans []trace.Span + singleCtx context.Context +} + +func (b *fetchedBatch) add(ctx context.Context, topic string, m kafka.Message) { + if log.IsLevelEnabled(log.TraceLevel) { + utils.LogTrace("count", len(m.Headers), "kafka message headers") + } + msgCtx, span := telemetry.ConsumerContext(ctx, topic, m.Headers) + if len(b.app) == 0 { + b.singleCtx = msgCtx + } + b.app = append(b.app, KafkaMessage{Key: m.Key, Value: m.Value, Headers: m.Headers}) + b.raw = append(b.raw, m) + b.spans = append(b.spans, span) +} + +func (b *fetchedBatch) endSpans(err error) { + for _, span := range b.spans { + telemetry.End(span, err) + } +} + +func readBatch(ctx context.Context, topic string, fetch func(context.Context) (kafka.Message, error), + n int, wait time.Duration) (fetchedBatch, error) { + batch := fetchedBatch{ + app: make([]KafkaMessage, 0, n), + raw: make([]kafka.Message, 0, n), + spans: make([]trace.Span, 0, n), + } + first, err := fetch(ctx) + if err != nil { + return fetchedBatch{}, err + } + batch.add(ctx, topic, first) + if len(batch.app) >= n { + return batch, nil + } + + err = batch.fill(ctx, topic, fetch, n, wait) + if err != nil { + batch.endSpans(err) + return fetchedBatch{}, err + } + return batch, nil +} + +func (b *fetchedBatch) fill(ctx context.Context, topic string, fetch func(context.Context) (kafka.Message, error), + n int, wait time.Duration) error { + fetchCtx, cancel := context.WithTimeout(ctx, wait) + defer cancel() + for len(b.app) < n { + m, err := fetch(fetchCtx) + if err == nil { + b.add(ctx, topic, m) + continue + } + if errors.Is(err, context.DeadlineExceeded) { + return nil + } + return err + } + return nil +} + +func readerStopped(err error) bool { + return err.Error() == errContextCanceled || errors.Is(err, context.Canceled) || err == io.EOF +} + type kafkaGoWriterImpl struct { *kafka.Writer } diff --git a/base/mqueue/testing.go b/base/mqueue/testing.go index ea3feff35..a25e67608 100644 --- a/base/mqueue/testing.go +++ b/base/mqueue/testing.go @@ -3,12 +3,13 @@ package mqueue import ( "context" "sync/atomic" + "time" ) type mockReader struct{} -func (t *mockReader) HandleMessages(_ context.Context, _ MessageHandler) {} -func (t *mockReader) Close() error { return nil } +func (t *mockReader) HandleMessages(_ context.Context, _ int, _ time.Duration, _ BatchHandler) {} +func (t *mockReader) Close() error { return nil } // Count how many times reader is created. func CreateCountedMockReader(cnt *int32) CreateReader { From 25652601b0fbe0cdbffbb8d2db306ebf8cdd23b4 Mon Sep 17 00:00:00 2001 From: Michael Mraka Date: Fri, 13 Nov 2026 22:21:53 +0100 Subject: [PATCH 02/17] RHINENG-33842: Read one Kafka message as a batch of size 1. --- base/mqueue/batch_test.go | 26 ++++++++++++++++++++++++++ base/mqueue/mqueue.go | 12 ++++++++++++ 2 files changed, 38 insertions(+) diff --git a/base/mqueue/batch_test.go b/base/mqueue/batch_test.go index 1f8261742..996c6983a 100644 --- a/base/mqueue/batch_test.go +++ b/base/mqueue/batch_test.go @@ -10,6 +10,32 @@ import ( "github.com/stretchr/testify/assert" ) +func TestAsBatchHandler(t *testing.T) { + var got []KafkaMessage + err := AsBatchHandler(func(_ context.Context, m KafkaMessage) error { + got = append(got, m) + return nil + })(context.Background(), []KafkaMessage{{Value: []byte("a")}}) + assert.NoError(t, err) + assert.Equal(t, []KafkaMessage{{Value: []byte("a")}}, got) + + got = nil + err = AsBatchHandler(func(_ context.Context, m KafkaMessage) error { + got = append(got, m) + return nil + })(context.Background(), []KafkaMessage{{Value: []byte("a")}, {Value: []byte("b")}}) + assert.NoError(t, err) + assert.Equal(t, []KafkaMessage{{Value: []byte("a")}, {Value: []byte("b")}}, got) + + calls := 0 + err = AsBatchHandler(func(context.Context, KafkaMessage) error { + calls++ + return errors.New("boom") + })(context.Background(), []KafkaMessage{{Value: []byte("a")}, {Value: []byte("b")}}) + assert.EqualError(t, err, "boom") + assert.Equal(t, 1, calls) +} + func TestHandleMessageBatchesBlocksForFirstMessage(t *testing.T) { ch := make(chan kafka.Message) fetch := func(ctx context.Context) (kafka.Message, error) { diff --git a/base/mqueue/mqueue.go b/base/mqueue/mqueue.go index 9564a3044..fddd9faa8 100644 --- a/base/mqueue/mqueue.go +++ b/base/mqueue/mqueue.go @@ -59,6 +59,18 @@ type MessageHandler func(ctx context.Context, message KafkaMessage) error type BatchHandler func(ctx context.Context, messages []KafkaMessage) error +// AsBatchHandler calls handler for every message and returns the first error. +func AsBatchHandler(handler MessageHandler) BatchHandler { + return func(ctx context.Context, msgs []KafkaMessage) error { + for _, m := range msgs { + if err := handler(ctx, m); err != nil { + return err + } + } + return nil + } +} + func MakeRetryingHandler(handler MessageHandler) MessageHandler { return func(ctx context.Context, message KafkaMessage) error { var err error From b5aeddeafea99ce5249c4c2a82e1df5b1742afbd Mon Sep 17 00:00:00 2001 From: Michael Mraka Date: Fri, 13 Nov 2026 22:22:27 +0100 Subject: [PATCH 03/17] RHINENG-33842: Retry fatal Kafka batch handler errors on the same slice. --- base/mqueue/mqueue.go | 6 +++--- base/mqueue/mqueue_test.go | 27 ++++++++++++++++----------- evaluator/advisory_update_test.go | 4 ++-- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/base/mqueue/mqueue.go b/base/mqueue/mqueue.go index fddd9faa8..eeeeaf6f8 100644 --- a/base/mqueue/mqueue.go +++ b/base/mqueue/mqueue.go @@ -71,8 +71,8 @@ func AsBatchHandler(handler MessageHandler) BatchHandler { } } -func MakeRetryingHandler(handler MessageHandler) MessageHandler { - return func(ctx context.Context, message KafkaMessage) error { +func MakeRetryingHandler(handler BatchHandler) BatchHandler { + return func(ctx context.Context, messages []KafkaMessage) error { var err error var attempt int @@ -80,7 +80,7 @@ func MakeRetryingHandler(handler MessageHandler) MessageHandler { backoffState := policy.Start(backoffCtx) defer cancel() for backoff.Continue(backoffState) { - if err = handler(ctx, message); err == nil || !errors.Is(err, base.ErrFatal) { + if err = handler(ctx, messages); err == nil || !errors.Is(err, base.ErrFatal) { return nil } utils.LogError("err", err, "attempt", attempt, "Try failed") diff --git a/base/mqueue/mqueue_test.go b/base/mqueue/mqueue_test.go index ddeb50333..0ba063959 100644 --- a/base/mqueue/mqueue_test.go +++ b/base/mqueue/mqueue_test.go @@ -4,7 +4,6 @@ import ( "app/base" "app/base/utils" "context" - "errors" "sync" "testing" @@ -24,8 +23,8 @@ func TestRoundTripKafkaGo(t *testing.T) { defer reader.Close() var eventOut PlatformEvent - go reader.HandleMessages(t.Context(), func(_ context.Context, m KafkaMessage) error { - return sonic.Unmarshal(m.Value, &eventOut) + go reader.HandleMessages(t.Context(), 1, 0, func(_ context.Context, m []KafkaMessage) error { + return sonic.Unmarshal(m[0].Value, &eventOut) }) writer := NewKafkaWriterFromEnv("test") @@ -45,21 +44,27 @@ func TestSpawnReader(t *testing.T) { assert.Equal(t, 1, int(nReaders)) } -func TestRetry(t *testing.T) { +func TestRetryBatchFatalThenSuccess(t *testing.T) { i := 0 - handler := func(_ context.Context, _ KafkaMessage) error { + handler := func(_ context.Context, _ []KafkaMessage) error { i++ - if i < 2 { - return errors.New("Failed") + if i < 3 { + return base.ErrFatal } return nil } - // Without retry handler should fail - assert.Error(t, handler(base.Context, msg)) + assert.NoError(t, MakeRetryingHandler(handler)(base.Context, []KafkaMessage{msg})) + assert.Equal(t, 3, i) +} + +func TestRetryBatchAlwaysFatal(t *testing.T) { + handler := func(_ context.Context, _ []KafkaMessage) error { + return base.ErrFatal + } - // With retry we handler should eventually succeed - assert.NoError(t, MakeRetryingHandler(handler)(base.Context, msg)) + err := MakeRetryingHandler(handler)(base.Context, []KafkaMessage{msg}) + assert.ErrorIs(t, err, base.ErrFatal) } func TestWriteMessagesInjectsTraceparent(t *testing.T) { diff --git a/evaluator/advisory_update_test.go b/evaluator/advisory_update_test.go index efc4cd2c8..54b595a9b 100644 --- a/evaluator/advisory_update_test.go +++ b/evaluator/advisory_update_test.go @@ -144,8 +144,8 @@ func TestAdvisoryUpdateKafkaRoundTrip(t *testing.T) { }() var received mqueue.KafkaMessage - go reader.HandleMessages(t.Context(), func(_ context.Context, m mqueue.KafkaMessage) error { - received = m + go reader.HandleMessages(t.Context(), 1, 0, func(_ context.Context, m []mqueue.KafkaMessage) error { + received = m[0] return nil }) From c7922ca6ee703070f745c5a40123488264bb27ce Mon Sep 17 00:00:00 2001 From: Michael Mraka Date: Fri, 13 Nov 2026 22:23:13 +0100 Subject: [PATCH 04/17] RHINENG-33842: Update every Kafka reader to batched HandleMessages. --- aggregator/aggregator.go | 4 ++-- base/mqueue/mqueue.go | 11 +++++++---- base/mqueue/mqueue_test.go | 20 ++++++++++++++++++-- evaluator/evaluate.go | 4 ++-- listener/listener.go | 6 ++++-- 5 files changed, 33 insertions(+), 12 deletions(-) diff --git a/aggregator/aggregator.go b/aggregator/aggregator.go index b151c6233..2591239af 100644 --- a/aggregator/aggregator.go +++ b/aggregator/aggregator.go @@ -43,9 +43,9 @@ func runServer() { } func subscribeToAdvisoryUpdates(wg *sync.WaitGroup, readerBuilder mqueue.CreateReader) { - handler := mqueue.MakeRetryingHandler(advisoryUpdateHandler) + handler := mqueue.MakeRetryingHandler(mqueue.AsBatchHandler(advisoryUpdateHandler)) for i := 0; i < consumerCount; i++ { - mqueue.SpawnReader(base.Context, wg, advisoryUpdateTopic, readerBuilder, handler) + mqueue.SpawnReader(base.Context, wg, advisoryUpdateTopic, readerBuilder, 1, 0, handler) utils.LogDebug("spawned advisory update reader", i) } utils.LogInfo("connected to kafka topics") diff --git a/base/mqueue/mqueue.go b/base/mqueue/mqueue.go index eeeeaf6f8..34d3fe838 100644 --- a/base/mqueue/mqueue.go +++ b/base/mqueue/mqueue.go @@ -97,16 +97,19 @@ type CreateReader func(topic string) Reader type CreateWriter func(topic string) Writer func runReader(ctx context.Context, wg *sync.WaitGroup, topic string, createReader CreateReader, - msgHandler MessageHandler) { + batchSize int, batchWait time.Duration, handler BatchHandler) { defer wg.Done() defer utils.LogPanics(true) reader := createReader(topic) defer reader.Close() - reader.HandleMessages(ctx, msgHandler) + reader.HandleMessages(ctx, batchSize, batchWait, handler) } func SpawnReader(ctx context.Context, wg *sync.WaitGroup, topic string, createReader CreateReader, - msgHandler MessageHandler) { + batchSize int, batchWait time.Duration, handler BatchHandler) { + if handler == nil { + panic("reader loop has no handler") + } wg.Add(1) - go runReader(ctx, wg, topic, createReader, msgHandler) + go runReader(ctx, wg, topic, createReader, batchSize, batchWait, handler) } diff --git a/base/mqueue/mqueue_test.go b/base/mqueue/mqueue_test.go index 0ba063959..65b5a3b82 100644 --- a/base/mqueue/mqueue_test.go +++ b/base/mqueue/mqueue_test.go @@ -6,6 +6,7 @@ import ( "context" "sync" "testing" + "time" "github.com/bytedance/sonic" "github.com/google/uuid" @@ -38,12 +39,27 @@ func TestRoundTripKafkaGo(t *testing.T) { func TestSpawnReader(t *testing.T) { var nReaders int32 wg := sync.WaitGroup{} - SpawnReader(base.Context, &wg, "", CreateCountedMockReader(&nReaders), - func(_ context.Context, _ KafkaMessage) error { return nil }) + SpawnReader(base.Context, &wg, "", CreateCountedMockReader(&nReaders), 1, 0, + func(_ context.Context, _ []KafkaMessage) error { return nil }) wg.Wait() assert.Equal(t, 1, int(nReaders)) } +func TestSpawnReaderBatch(t *testing.T) { + var nReaders int32 + wg := sync.WaitGroup{} + SpawnReader(base.Context, &wg, "", CreateCountedMockReader(&nReaders), 2, time.Millisecond, + func(context.Context, []KafkaMessage) error { return nil }) + wg.Wait() + assert.Equal(t, 1, int(nReaders)) +} + +func TestSpawnReaderPanicsWithoutHandler(t *testing.T) { + assert.PanicsWithValue(t, "reader loop has no handler", func() { + SpawnReader(base.Context, &sync.WaitGroup{}, "", CreateCountedMockReader(new(int32)), 1, 0, nil) + }) +} + func TestRetryBatchFatalThenSuccess(t *testing.T) { i := 0 handler := func(_ context.Context, _ []KafkaMessage) error { diff --git a/evaluator/evaluate.go b/evaluator/evaluate.go index c06558e64..4965aba37 100644 --- a/evaluator/evaluate.go +++ b/evaluator/evaluate.go @@ -823,11 +823,11 @@ func run(wg *sync.WaitGroup, readerBuilder mqueue.CreateReader) { loadCache() - var handler = mqueue.MakeRetryingHandler(evaluateHandler) + var handler = mqueue.MakeRetryingHandler(mqueue.AsBatchHandler(evaluateHandler)) // We create multiple consumers, and hope that the partition rebalancing // algorithm assigns each consumer a single partition for i := 0; i < consumerCount; i++ { - mqueue.SpawnReader(base.Context, wg, evalTopic, readerBuilder, handler) + mqueue.SpawnReader(base.Context, wg, evalTopic, readerBuilder, 1, 0, handler) } } diff --git a/listener/listener.go b/listener/listener.go index 4eb7be171..e7bfb23d2 100644 --- a/listener/listener.go +++ b/listener/listener.go @@ -125,12 +125,14 @@ func runReaders(wg *sync.WaitGroup, readerBuilder mqueue.CreateReader) { // We create multiple consumers, and hope that the partition rebalancing // algorithm assigns each consumer a single partition for i := 0; i < eventsConsumers; i++ { - mqueue.SpawnReader(base.Context, wg, eventsTopic, readerBuilder, mqueue.MakeRetryingHandler(EventsMessageHandler)) + mqueue.SpawnReader(base.Context, wg, eventsTopic, readerBuilder, 1, 0, + mqueue.MakeRetryingHandler(mqueue.AsBatchHandler(EventsMessageHandler))) utils.LogDebug("spawned eventsTopic reader", i) } if enableTemplates { for i := 0; i < templatesConsumers; i++ { - mqueue.SpawnReader(base.Context, wg, templatesTopic, readerBuilder, mqueue.MakeRetryingHandler(TemplatesMessageHandler)) //nolint:lll + mqueue.SpawnReader(base.Context, wg, templatesTopic, readerBuilder, 1, 0, + mqueue.MakeRetryingHandler(mqueue.AsBatchHandler(TemplatesMessageHandler))) utils.LogDebug("spawned templatesTopic reader", i) } } From 5b26169660196214f0eb4426640df1c825e90d3e Mon Sep 17 00:00:00 2001 From: Michael Mraka Date: Fri, 13 Nov 2026 22:23:56 +0100 Subject: [PATCH 05/17] RHINENG-33842: Aggregate advisory updates from a Kafka batch instead of an in-memory buffer. --- aggregator/aggregator.go | 7 ++--- aggregator/events.go | 64 +++++++++++---------------------------- aggregator/events_test.go | 16 +--------- 3 files changed, 21 insertions(+), 66 deletions(-) diff --git a/aggregator/aggregator.go b/aggregator/aggregator.go index 2591239af..59f806571 100644 --- a/aggregator/aggregator.go +++ b/aggregator/aggregator.go @@ -21,12 +21,11 @@ func readPodConfig() { consumerCount = utils.PodConfig.GetInt("consumer_count", 1) enableNotifications = utils.PodConfig.GetBool("enable_notifications", false) batchSize = utils.PodConfig.GetInt("advisory_batch_size", 4000) - flushTimeout = time.Duration(utils.PodConfig.GetInt("advisory_flush_timeout_ms", 500)) * time.Millisecond + flushTimeout = time.Duration(utils.PodConfig.GetInt("advisory_flush_timeout_ms", 5000)) * time.Millisecond } func configure() { advisoryUpdateTopic = utils.FailIfEmpty(utils.CoreCfg.AdvisoryUpdateTopic, "ADVISORY_UPDATE_TOPIC") - initBuffer() configureNotifications() } @@ -43,9 +42,9 @@ func runServer() { } func subscribeToAdvisoryUpdates(wg *sync.WaitGroup, readerBuilder mqueue.CreateReader) { - handler := mqueue.MakeRetryingHandler(mqueue.AsBatchHandler(advisoryUpdateHandler)) + handler := mqueue.MakeRetryingHandler(advisoryUpdateHandler) for i := 0; i < consumerCount; i++ { - mqueue.SpawnReader(base.Context, wg, advisoryUpdateTopic, readerBuilder, 1, 0, handler) + mqueue.SpawnReader(base.Context, wg, advisoryUpdateTopic, readerBuilder, batchSize, flushTimeout, handler) utils.LogDebug("spawned advisory update reader", i) } utils.LogInfo("connected to kafka topics") diff --git a/aggregator/events.go b/aggregator/events.go index 196829f0d..aa6d00fd6 100644 --- a/aggregator/events.go +++ b/aggregator/events.go @@ -6,7 +6,6 @@ import ( "app/base/mqueue" "app/base/utils" "context" - "sync" "time" "github.com/bytedance/sonic" @@ -14,55 +13,24 @@ import ( ) var ( - batchSize int - flushTimeout time.Duration - advisoryBuffer []mqueue.AdvisoryUpdateEvent - bufferLock sync.Mutex - flushTimer *time.Timer + batchSize int + flushTimeout time.Duration ) -func initBuffer() { - advisoryBuffer = make([]mqueue.AdvisoryUpdateEvent, 0, batchSize+1) - flushTimer = time.AfterFunc(87600*time.Hour, func() { - utils.LogInfo("flushing advisory buffer after timeout") - flushAdvisoryBuffer() - }) -} - -func flushAdvisoryBuffer() { - bufferLock.Lock() - if len(advisoryBuffer) == 0 { - bufferLock.Unlock() - return +func advisoryUpdateHandler(ctx context.Context, msgs []mqueue.KafkaMessage) error { + events := make([]mqueue.AdvisoryUpdateEvent, 0, len(msgs)) + for _, m := range msgs { + var event mqueue.AdvisoryUpdateEvent + if err := sonic.Unmarshal(m.Value, &event); err != nil { + utils.LogError("err", err, "could not deserialize advisory update event") + continue + } + events = append(events, event) } - // Copy and unlock before processing so new events can accumulate while DB work runs - batch := make([]mqueue.AdvisoryUpdateEvent, len(advisoryBuffer)) - copy(batch, advisoryBuffer) - advisoryBuffer = advisoryBuffer[:0] - bufferLock.Unlock() - - grouped := groupAdvisoryUpdates(batch) - processAdvisoryBatch(base.Context, grouped) -} - -func advisoryUpdateHandler(_ context.Context, m mqueue.KafkaMessage) error { - var event mqueue.AdvisoryUpdateEvent - if err := sonic.Unmarshal(m.Value, &event); err != nil { - utils.LogError("err", err, "could not deserialize advisory update event") + if len(events) == 0 { return nil } - - bufferLock.Lock() - advisoryBuffer = append(advisoryBuffer, event) - flushTimer.Reset(flushTimeout) - shouldFlush := len(advisoryBuffer) >= batchSize - bufferLock.Unlock() - - if shouldFlush { - utils.LogInfo("flushing full advisory buffer") - flushAdvisoryBuffer() - } - return nil + return processAdvisoryBatch(ctx, groupAdvisoryUpdates(events)) } func groupAdvisoryUpdates(events []mqueue.AdvisoryUpdateEvent) map[int][]int64 { @@ -87,19 +55,21 @@ func groupAdvisoryUpdates(events []mqueue.AdvisoryUpdateEvent) map[int][]int64 { return grouped } -func processAdvisoryBatch(ctx context.Context, grouped map[int][]int64) { +func processAdvisoryBatch(ctx context.Context, grouped map[int][]int64) error { for rhAccountID, advisoryIDs := range grouped { utils.LogInfo("rh_account_id", rhAccountID, "advisory_count", len(advisoryIDs), "refreshing account advisory caches") err := database.DB(ctx).Exec("SELECT refresh_account_advisory_caches_multi(?, ?)", pq.Array(advisoryIDs), rhAccountID).Error //nolint:lll if err != nil { utils.LogError("err", err, "rh_account_id", rhAccountID, "failed to refresh account advisory caches") - continue + return base.WrapFatalDBError(err, "refresh account advisory caches") } CheckAdvisoryDrift(rhAccountID, advisoryIDs) if err := publishNewAdvisoryNotification(rhAccountID, advisoryIDs); err != nil { utils.LogError("err", err, "rh_account_id", rhAccountID, "failed to publish new advisory notification") + return base.WrapFatalKafkaError(err, "publish new advisory notification") } } + return nil } diff --git a/aggregator/events_test.go b/aggregator/events_test.go index 671fecfb3..aeb7e5fe1 100644 --- a/aggregator/events_test.go +++ b/aggregator/events_test.go @@ -8,7 +8,6 @@ import ( "app/base/utils" "sort" "testing" - "time" "github.com/bytedance/sonic" "github.com/stretchr/testify/assert" @@ -69,21 +68,8 @@ func TestBufferedEventsProcessedOnBatchThreshold(t *testing.T) { assert.Nil(t, database.TestDB().Exec("SELECT refresh_advisory_caches(NULL, 1)").Error) defer database.DeleteAccountAdvisoryByAccount(t, 1) - batchSize = 3 - flushTimeout = time.Hour - initBuffer() - msg := toKafkaMessage(t, mqueue.AdvisoryUpdateEvent{RhAccountID: 1, AdvisoryIDs: []int64{1, 2}}) - - // First two events accumulate in the buffer - assert.Nil(t, advisoryUpdateHandler(base.Context, msg)) - assert.Equal(t, 1, len(advisoryBuffer)) - assert.Nil(t, advisoryUpdateHandler(base.Context, msg)) - assert.Equal(t, 2, len(advisoryBuffer)) - - // Third event triggers flush and processAdvisoryBatch runs - assert.Nil(t, advisoryUpdateHandler(base.Context, msg)) - assert.Equal(t, 0, len(advisoryBuffer)) + assert.Nil(t, advisoryUpdateHandler(base.Context, []mqueue.KafkaMessage{msg, msg, msg})) // Verify account_advisory was populated var count int64 From 4a4eb11ab9da1bf644aae53ecf8af868327e8c6e Mon Sep 17 00:00:00 2001 From: Michael Mraka Date: Sat, 14 Nov 2026 09:39:52 +0100 Subject: [PATCH 06/17] RHINENG-33842: flag to bypass messages in case of emergency we can turn it on and just clear the queue without actually processing messages --- aggregator/aggregator.go | 3 +++ aggregator/events.go | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/aggregator/aggregator.go b/aggregator/aggregator.go index 59f806571..42684351d 100644 --- a/aggregator/aggregator.go +++ b/aggregator/aggregator.go @@ -13,12 +13,15 @@ import ( var ( advisoryUpdateTopic string + enableBypass bool enableNotifications bool consumerCount int ) func readPodConfig() { consumerCount = utils.PodConfig.GetInt("consumer_count", 1) + // Toggle bypassing of message processing (in case something goes wrong and we need to clear the queue) + enableBypass = utils.PodConfig.GetBool("bypass", false) enableNotifications = utils.PodConfig.GetBool("enable_notifications", false) batchSize = utils.PodConfig.GetInt("advisory_batch_size", 4000) flushTimeout = time.Duration(utils.PodConfig.GetInt("advisory_flush_timeout_ms", 5000)) * time.Millisecond diff --git a/aggregator/events.go b/aggregator/events.go index aa6d00fd6..7f8d8ad18 100644 --- a/aggregator/events.go +++ b/aggregator/events.go @@ -18,6 +18,10 @@ var ( ) func advisoryUpdateHandler(ctx context.Context, msgs []mqueue.KafkaMessage) error { + if enableBypass { + utils.LogWarn("Advisory update processing bypassed") + return nil + } events := make([]mqueue.AdvisoryUpdateEvent, 0, len(msgs)) for _, m := range msgs { var event mqueue.AdvisoryUpdateEvent From 86a13aed92e414c2cf31eb98de56d6217096f4a3 Mon Sep 17 00:00:00 2001 From: Michael Mraka Date: Sat, 14 Nov 2026 16:35:57 +0000 Subject: [PATCH 07/17] RHINENG-33842: Add account_advisory.updated so each row records when it was written. --- .../migrations/171_account_advisory_updated.down.sql | 2 ++ database_admin/migrations/171_account_advisory_updated.up.sql | 2 ++ database_admin/schema/create_schema.sql | 3 ++- docs/md/database.md | 2 +- 4 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 database_admin/migrations/171_account_advisory_updated.down.sql create mode 100644 database_admin/migrations/171_account_advisory_updated.up.sql diff --git a/database_admin/migrations/171_account_advisory_updated.down.sql b/database_admin/migrations/171_account_advisory_updated.down.sql new file mode 100644 index 000000000..7d6e9f076 --- /dev/null +++ b/database_admin/migrations/171_account_advisory_updated.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE account_advisory + DROP COLUMN IF EXISTS updated; diff --git a/database_admin/migrations/171_account_advisory_updated.up.sql b/database_admin/migrations/171_account_advisory_updated.up.sql new file mode 100644 index 000000000..045faf8b6 --- /dev/null +++ b/database_admin/migrations/171_account_advisory_updated.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE account_advisory + ADD COLUMN IF NOT EXISTS updated TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(); diff --git a/database_admin/schema/create_schema.sql b/database_admin/schema/create_schema.sql index a7aa74856..5a684de26 100644 --- a/database_admin/schema/create_schema.sql +++ b/database_admin/schema/create_schema.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS schema_migrations INSERT INTO schema_migrations -VALUES (170, false); +VALUES (171, false); -- --------------------------------------------------------------------------- -- Functions @@ -911,6 +911,7 @@ CREATE TABLE IF NOT EXISTS account_advisory systems_applicable INT NOT NULL DEFAULT 0, systems_installable INT NOT NULL DEFAULT 0, notified TIMESTAMP WITH TIME ZONE NULL, + updated TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), CONSTRAINT account_advisory_advisory_id FOREIGN KEY (advisory_id) REFERENCES advisory_metadata (id), diff --git a/docs/md/database.md b/docs/md/database.md index c1d7f1ed6..4f80cdef7 100644 --- a/docs/md/database.md +++ b/docs/md/database.md @@ -7,7 +7,7 @@ Main database tables description: - **advisory_metadata** - stores info about advisories (`description`, `summary`, `solution` etc.). It's synced and stored on trigger by `vmaas_sync` component. It allows to display detail information about the advisory. - **system_advisories** - stores info about advisories evaluated for particular systems (system - advisory M-N mapping table). `system_id` references **system_inventory.id**. Contains info when system advisory was firstly reported and patched (if so). Records are created and updated by `evaluator` component. It allows to display list of advisories related to a system. - **advisory_account_data** - stores info about all advisories detected within at least one system that belongs to a given account. So it provides overall statistics about system advisories displayed by the application. -- **account_advisory** - workspace-scoped version of `advisory_account_data`. Stores per-advisory aggregate counts (`systems_applicable`, `systems_installable`) and notification state for each workspace within an account. Keyed by `(rh_account_id, workspace_id, advisory_id)`, partitioned by `rh_account_id` (32 partitions). +- **account_advisory** - workspace-scoped version of `advisory_account_data`. Stores per-advisory aggregate counts (`systems_applicable`, `systems_installable`) and notification state for each workspace within an account. Keyed by `(rh_account_id, workspace_id, advisory_id)`, partitioned by `rh_account_id` (32 partitions). `updated` is `TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()`. The aggregator sets it when it inserts or updates a recounted row. - **package_name** - names of the packages installed on systems - **package** - list of all packages versions, precisely all EVRAs (epoch-version-release-arch) - **system_package2** - list of packages installed on a system From d28c4ccf00ddae3fd439fd1e42278f07612310ba Mon Sep 17 00:00:00 2001 From: Michael Mraka Date: Sat, 14 Nov 2026 16:37:27 +0000 Subject: [PATCH 08/17] RHINENG-33842: Map account_advisory.updated on AccountAdvisory. --- base/models/models.go | 1 + 1 file changed, 1 insertion(+) diff --git a/base/models/models.go b/base/models/models.go index 0481f2820..aac43dd99 100644 --- a/base/models/models.go +++ b/base/models/models.go @@ -309,6 +309,7 @@ type AccountAdvisory struct { SystemsApplicable int SystemsInstallable int Notified *time.Time + Updated time.Time `gorm:"autoUpdateTime"` } func (AccountAdvisory) TableName() string { From 1815f078c40ce12502c4e537ca85dcab20f6d1be Mon Sep 17 00:00:00 2001 From: Michael Mraka Date: Sat, 14 Nov 2026 16:39:46 +0000 Subject: [PATCH 09/17] RHINENG-33842: Recount account_advisory per message advisory instead of the account-wide SQL refresh. --- aggregator/events.go | 136 ++++++++++++++++++++++++++++++------ aggregator/events_test.go | 142 ++++++++++++++++++++++++++------------ 2 files changed, 211 insertions(+), 67 deletions(-) diff --git a/aggregator/events.go b/aggregator/events.go index 7f8d8ad18..4f6408f4c 100644 --- a/aggregator/events.go +++ b/aggregator/events.go @@ -3,13 +3,19 @@ package aggregator import ( "app/base" "app/base/database" + "app/base/models" "app/base/mqueue" "app/base/utils" + "bytes" "context" + "errors" + "sort" "time" "github.com/bytedance/sonic" - "github.com/lib/pq" + "github.com/google/uuid" + "gorm.io/gorm" + "gorm.io/gorm/clause" ) var ( @@ -17,6 +23,24 @@ var ( flushTimeout time.Duration ) +const advisoryCountSQL = ` +SELECT count(sa.*) FILTER (WHERE sa.status_id = 0) AS systems_installable, + count(sa.*) AS systems_applicable + FROM system_advisories sa + JOIN system_inventory si + ON sa.rh_account_id = si.rh_account_id AND sa.system_id = si.id + WHERE si.stale = FALSE + AND sa.rh_account_id = ? + AND sa.advisory_id = ? + AND si.rh_account_id = ? + AND si.workspace_id = ? + GROUP BY sa.advisory_id, sa.rh_account_id, si.workspace_id` + +type advisoryRecountCount struct { + SystemsInstallable int + SystemsApplicable int +} + func advisoryUpdateHandler(ctx context.Context, msgs []mqueue.KafkaMessage) error { if enableBypass { utils.LogWarn("Advisory update processing bypassed") @@ -37,39 +61,69 @@ func advisoryUpdateHandler(ctx context.Context, msgs []mqueue.KafkaMessage) erro return processAdvisoryBatch(ctx, groupAdvisoryUpdates(events)) } -func groupAdvisoryUpdates(events []mqueue.AdvisoryUpdateEvent) map[int][]int64 { - sets := make(map[int]map[int64]struct{}) - for _, e := range events { - if _, ok := sets[e.RhAccountID]; !ok { - sets[e.RhAccountID] = make(map[int64]struct{}) +type advisoryRecountKey struct { + RhAccountID int + WorkspaceID uuid.UUID + AdvisoryID int64 +} + +type advisoryRecount struct { + advisoryRecountKey + ProducedAt time.Time +} + +func groupAdvisoryUpdates(events []mqueue.AdvisoryUpdateEvent) []advisoryRecount { + newest := make(map[advisoryRecountKey]time.Time) + for i := range events { + event := &events[i] + producedAt := time.Time{} + if ts := event.ProducedAt.Time(); ts != nil { + producedAt = *ts } - for _, id := range e.AdvisoryIDs { - sets[e.RhAccountID][id] = struct{}{} + for _, advisoryID := range event.AdvisoryIDs { + key := advisoryRecountKey{event.RhAccountID, event.WorkspaceID, advisoryID} + if existing, ok := newest[key]; !ok || producedAt.After(existing) { + newest[key] = producedAt + } } } - grouped := make(map[int][]int64, len(sets)) - for accID, idSet := range sets { - ids := make([]int64, 0, len(idSet)) - for id := range idSet { - ids = append(ids, id) - } - grouped[accID] = ids + requests := make([]advisoryRecount, 0, len(newest)) + for key, producedAt := range newest { + requests = append(requests, advisoryRecount{ + RhAccountID: key.RhAccountID, + WorkspaceID: key.WorkspaceID, + AdvisoryID: key.AdvisoryID, + ProducedAt: producedAt, + }) } - return grouped + sort.Slice(requests, func(i, j int) bool { + if requests[i].RhAccountID != requests[j].RhAccountID { + return requests[i].RhAccountID < requests[j].RhAccountID + } + if c := bytes.Compare(requests[i].WorkspaceID[:], requests[j].WorkspaceID[:]); c != 0 { + return c < 0 + } + return requests[i].AdvisoryID < requests[j].AdvisoryID + }) + return requests } -func processAdvisoryBatch(ctx context.Context, grouped map[int][]int64) error { - for rhAccountID, advisoryIDs := range grouped { - utils.LogInfo("rh_account_id", rhAccountID, "advisory_count", len(advisoryIDs), "refreshing account advisory caches") - err := database.DB(ctx).Exec("SELECT refresh_account_advisory_caches_multi(?, ?)", pq.Array(advisoryIDs), rhAccountID).Error //nolint:lll +func processAdvisoryBatch(ctx context.Context, requests []advisoryRecount) error { + utils.LogInfo("advisory_count", len(requests), "refreshing account advisory caches") + + accountAdvisories := make(map[int][]int64) + for _, req := range requests { + err := recountAdvisory(ctx, req.RhAccountID, req.WorkspaceID, req.AdvisoryID) if err != nil { - utils.LogError("err", err, "rh_account_id", rhAccountID, "failed to refresh account advisory caches") + utils.LogError("err", err, "rh_account_id", req.RhAccountID, + "failed to refresh account advisory caches") return base.WrapFatalDBError(err, "refresh account advisory caches") } + accountAdvisories[req.RhAccountID] = append(accountAdvisories[req.RhAccountID], req.AdvisoryID) + } - CheckAdvisoryDrift(rhAccountID, advisoryIDs) - + for rhAccountID, advisoryIDs := range accountAdvisories { if err := publishNewAdvisoryNotification(rhAccountID, advisoryIDs); err != nil { utils.LogError("err", err, "rh_account_id", rhAccountID, "failed to publish new advisory notification") return base.WrapFatalKafkaError(err, "publish new advisory notification") @@ -77,3 +131,39 @@ func processAdvisoryBatch(ctx context.Context, grouped map[int][]int64) error { } return nil } + +func recountAdvisory(ctx context.Context, rhAccountID int, workspaceID uuid.UUID, advisoryID int64) error { + return database.DB(ctx).Transaction(func(tx *gorm.DB) error { + var row models.AccountAdvisory + err := tx.Clauses(clause.Locking{Strength: "UPDATE"}). + Where("rh_account_id = ? AND workspace_id = ? AND advisory_id = ?", rhAccountID, workspaceID, advisoryID). + Take(&row).Error + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return err + } + + var counts []advisoryRecountCount + err = tx.Raw(advisoryCountSQL, rhAccountID, advisoryID, rhAccountID, workspaceID).Scan(&counts).Error + if err != nil { + return err + } + if len(counts) == 0 { + return tx.Where("rh_account_id = ? AND workspace_id = ? AND advisory_id = ?", + rhAccountID, workspaceID, advisoryID). + Delete(&models.AccountAdvisory{}).Error + } + + rec := models.AccountAdvisory{ + AdvisoryID: advisoryID, + RhAccountID: rhAccountID, + WorkspaceID: workspaceID, + SystemsApplicable: counts[0].SystemsApplicable, + SystemsInstallable: counts[0].SystemsInstallable, + Updated: time.Now(), + } + return database.OnConflictUpdateMulti(tx, + []string{"rh_account_id", "workspace_id", "advisory_id"}, + "systems_installable", "systems_applicable", "updated"). + Create(&rec).Error + }) +} diff --git a/aggregator/events_test.go b/aggregator/events_test.go index aeb7e5fe1..4acd306e5 100644 --- a/aggregator/events_test.go +++ b/aggregator/events_test.go @@ -4,61 +4,51 @@ import ( "app/base" "app/base/core" "app/base/database" + "app/base/models" "app/base/mqueue" "app/base/utils" - "sort" "testing" "github.com/bytedance/sonic" + "github.com/google/uuid" "github.com/stretchr/testify/assert" ) -func toKafkaMessage(t *testing.T, event mqueue.AdvisoryUpdateEvent) mqueue.KafkaMessage { - t.Helper() - data, err := sonic.Marshal(event) - assert.Nil(t, err) - return mqueue.KafkaMessage{Value: data} -} - -func TestGroupAdvisoryUpdatesSingleAccount(t *testing.T) { - events := []mqueue.AdvisoryUpdateEvent{ - {RhAccountID: 1, AdvisoryIDs: []int64{1, 2}}, - {RhAccountID: 1, AdvisoryIDs: []int64{2, 3}}, - } - grouped := groupAdvisoryUpdates(events) - - assert.Len(t, grouped, 1) - ids := grouped[1] - sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) - assert.Equal(t, []int64{1, 2, 3}, ids) -} +var ( + recountWorkspace = uuid.MustParse("00000000-0000-0000-0000-000000000001") + otherWorkspace = uuid.MustParse("00000000-0000-0000-0000-000000000002") +) -func TestGroupAdvisoryUpdatesMultipleAccounts(t *testing.T) { +func TestDedupeAdvisoryUpdatesKeepsYoungestProducedAt(t *testing.T) { + older := time.Date(2026, 11, 14, 10, 0, 0, 0, time.UTC) + newer := older.Add(time.Minute) events := []mqueue.AdvisoryUpdateEvent{ - {RhAccountID: 1, AdvisoryIDs: []int64{1, 2}}, - {RhAccountID: 2, AdvisoryIDs: []int64{5}}, - {RhAccountID: 1, AdvisoryIDs: []int64{3}}, + {RhAccountID: 2, WorkspaceID: recountWorkspace, AdvisoryIDs: []int64{5}, + ProducedAt: types.Rfc3339Timestamp(older)}, + {RhAccountID: 1, WorkspaceID: otherWorkspace, AdvisoryIDs: []int64{2, 1}, + ProducedAt: types.Rfc3339Timestamp(older)}, + {RhAccountID: 1, WorkspaceID: recountWorkspace, AdvisoryIDs: []int64{2, 1}, + ProducedAt: types.Rfc3339Timestamp(older)}, + {RhAccountID: 1, WorkspaceID: recountWorkspace, AdvisoryIDs: []int64{1}, + ProducedAt: types.Rfc3339Timestamp(newer)}, } - grouped := groupAdvisoryUpdates(events) - assert.Len(t, grouped, 2) + got := groupAdvisoryUpdates(events) - ids1 := grouped[1] - sort.Slice(ids1, func(i, j int) bool { return ids1[i] < ids1[j] }) - assert.Equal(t, []int64{1, 2, 3}, ids1) - assert.Equal(t, []int64{int64(5)}, grouped[2]) + assert.Equal(t, []advisoryRecount{ + {RhAccountID: 1, WorkspaceID: recountWorkspace, AdvisoryID: 1, ProducedAt: newer}, + {RhAccountID: 1, WorkspaceID: recountWorkspace, AdvisoryID: 2, ProducedAt: older}, + {RhAccountID: 1, WorkspaceID: otherWorkspace, AdvisoryID: 1, ProducedAt: older}, + {RhAccountID: 1, WorkspaceID: otherWorkspace, AdvisoryID: 2, ProducedAt: older}, + {RhAccountID: 2, WorkspaceID: recountWorkspace, AdvisoryID: 5, ProducedAt: older}, + }, got) } -func TestGroupAdvisoryUpdatesDeduplicates(t *testing.T) { - events := []mqueue.AdvisoryUpdateEvent{ - {RhAccountID: 1, AdvisoryIDs: []int64{1, 1, 2}}, - {RhAccountID: 1, AdvisoryIDs: []int64{2, 2, 1}}, - } - grouped := groupAdvisoryUpdates(events) - - ids := grouped[1] - sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) - assert.Equal(t, []int64{1, 2}, ids) +func toKafkaMessage(t *testing.T, event mqueue.AdvisoryUpdateEvent) mqueue.KafkaMessage { + t.Helper() + data, err := sonic.Marshal(event) + assert.Nil(t, err) + return mqueue.KafkaMessage{Value: data} } func TestBufferedEventsProcessedOnBatchThreshold(t *testing.T) { @@ -68,13 +58,77 @@ func TestBufferedEventsProcessedOnBatchThreshold(t *testing.T) { assert.Nil(t, database.TestDB().Exec("SELECT refresh_advisory_caches(NULL, 1)").Error) defer database.DeleteAccountAdvisoryByAccount(t, 1) - msg := toKafkaMessage(t, mqueue.AdvisoryUpdateEvent{RhAccountID: 1, AdvisoryIDs: []int64{1, 2}}) + assert.Nil(t, database.TestDB(). + Where("rh_account_id = 1 AND workspace_id = ? AND advisory_id = 1", otherWorkspace). + Delete(&models.AccountAdvisory{}).Error) + assert.Nil(t, database.TestDB().Create(&models.AccountAdvisory{ + AdvisoryID: 1, RhAccountID: 1, WorkspaceID: otherWorkspace, + SystemsInstallable: 77, SystemsApplicable: 77, + }).Error) + + msg := toKafkaMessage(t, mqueue.AdvisoryUpdateEvent{ + RhAccountID: 1, WorkspaceID: recountWorkspace, AdvisoryIDs: []int64{2, 1}, + }) assert.Nil(t, advisoryUpdateHandler(base.Context, []mqueue.KafkaMessage{msg, msg, msg})) - // Verify account_advisory was populated var count int64 assert.Nil(t, database.TestDB().Table("account_advisory"). - Where("rh_account_id = 1 AND advisory_id IN (?)", []int64{1, 2}). + Where("rh_account_id = 1 AND workspace_id = ? AND advisory_id IN (?)", recountWorkspace, []int64{1, 2}). Count(&count).Error) - assert.True(t, count > 0, "account_advisory rows should be created after flush") + assert.Equal(t, int64(2), count) + + installable := database.PluckInt(database.TestDB().Table("account_advisory"). + Where("rh_account_id = 1 AND workspace_id = ? AND advisory_id = 1", otherWorkspace), + "systems_installable") + assert.Equal(t, 77, installable) +} + +func TestRecountUpdatesCountsAndDeletesEmpty(t *testing.T) { + utils.SkipWithoutDB(t) + core.SetupTestEnvironment() + + database.DeleteAccountAdvisory(t, 1, recountWorkspace.String(), []int64{1, 9}) + t.Cleanup(func() { + database.DeleteAccountAdvisory(t, 1, recountWorkspace.String(), []int64{1, 9}) + }) + + assert.Nil(t, database.TestDB().Create(&models.AccountAdvisory{ + AdvisoryID: 1, RhAccountID: 1, WorkspaceID: recountWorkspace, + SystemsInstallable: 99, SystemsApplicable: 99, + }).Error) + + assert.Nil(t, recountAdvisory(base.Context, 1, recountWorkspace, 1)) + + var expected int + assert.Nil(t, database.TestDB().Raw(` + SELECT count(sa.*) FILTER (WHERE sa.status_id = 0) + FROM system_advisories sa + JOIN system_inventory si + ON sa.rh_account_id = si.rh_account_id AND sa.system_id = si.id + JOIN system_patch sp + ON si.id = sp.system_id AND sp.rh_account_id = si.rh_account_id + WHERE sp.last_evaluation IS NOT NULL + AND si.stale = FALSE + AND sa.rh_account_id = ? + AND si.workspace_id = ? + AND sa.advisory_id = ?`, + 1, recountWorkspace, 1).Scan(&expected).Error) + + got := database.PluckInt(database.TestDB().Table("account_advisory"). + Where("rh_account_id = 1 AND workspace_id = ? AND advisory_id = 1", recountWorkspace), + "systems_installable") + assert.Equal(t, expected, got) + assert.NotEqual(t, 99, got) + + assert.Nil(t, database.TestDB().Create(&models.AccountAdvisory{ + AdvisoryID: 9, RhAccountID: 1, WorkspaceID: recountWorkspace, + SystemsInstallable: 1, SystemsApplicable: 1, + }).Error) + assert.Nil(t, recountAdvisory(base.Context, 1, recountWorkspace, 9)) + + var left int64 + assert.Nil(t, database.TestDB().Table("account_advisory"). + Where("rh_account_id = 1 AND workspace_id = ? AND advisory_id = 9", recountWorkspace). + Count(&left).Error) + assert.Equal(t, int64(0), left) } From 5cd91a5daa4ed5f54832ddf7bf1399f632504e43 Mon Sep 17 00:00:00 2001 From: Michael Mraka Date: Sat, 14 Nov 2026 16:42:16 +0000 Subject: [PATCH 10/17] RHINENG-33842: Skip an advisory recount when account_advisory.updated is younger than the message. --- aggregator/events.go | 16 +++++++++-- aggregator/events_test.go | 60 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/aggregator/events.go b/aggregator/events.go index 4f6408f4c..6661fa99d 100644 --- a/aggregator/events.go +++ b/aggregator/events.go @@ -114,7 +114,7 @@ func processAdvisoryBatch(ctx context.Context, requests []advisoryRecount) error accountAdvisories := make(map[int][]int64) for _, req := range requests { - err := recountAdvisory(ctx, req.RhAccountID, req.WorkspaceID, req.AdvisoryID) + err := recountAdvisory(ctx, req.RhAccountID, req.WorkspaceID, req.AdvisoryID, req.ProducedAt) if err != nil { utils.LogError("err", err, "rh_account_id", req.RhAccountID, "failed to refresh account advisory caches") @@ -132,7 +132,8 @@ func processAdvisoryBatch(ctx context.Context, requests []advisoryRecount) error return nil } -func recountAdvisory(ctx context.Context, rhAccountID int, workspaceID uuid.UUID, advisoryID int64) error { +func recountAdvisory(ctx context.Context, rhAccountID int, workspaceID uuid.UUID, advisoryID int64, + producedAt time.Time) error { return database.DB(ctx).Transaction(func(tx *gorm.DB) error { var row models.AccountAdvisory err := tx.Clauses(clause.Locking{Strength: "UPDATE"}). @@ -141,6 +142,17 @@ func recountAdvisory(ctx context.Context, rhAccountID int, workspaceID uuid.UUID if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { return err } + if err == nil && row.Updated.After(producedAt) { + utils.LogWarn( + "rh_account_id", rhAccountID, + "workspace_id", workspaceID, + "advisory_id", advisoryID, + "updated", row.Updated, + "produced_at", producedAt, + "skipping advisory recount, updated is younger than produced_at", + ) + return nil + } var counts []advisoryRecountCount err = tx.Raw(advisoryCountSQL, rhAccountID, advisoryID, rhAccountID, workspaceID).Scan(&counts).Error diff --git a/aggregator/events_test.go b/aggregator/events_test.go index 4acd306e5..d32ae8173 100644 --- a/aggregator/events_test.go +++ b/aggregator/events_test.go @@ -6,8 +6,10 @@ import ( "app/base/database" "app/base/models" "app/base/mqueue" + "app/base/types" "app/base/utils" "testing" + "time" "github.com/bytedance/sonic" "github.com/google/uuid" @@ -68,6 +70,7 @@ func TestBufferedEventsProcessedOnBatchThreshold(t *testing.T) { msg := toKafkaMessage(t, mqueue.AdvisoryUpdateEvent{ RhAccountID: 1, WorkspaceID: recountWorkspace, AdvisoryIDs: []int64{2, 1}, + ProducedAt: types.Rfc3339Timestamp(time.Now().Add(time.Hour)), }) assert.Nil(t, advisoryUpdateHandler(base.Context, []mqueue.KafkaMessage{msg, msg, msg})) @@ -97,7 +100,7 @@ func TestRecountUpdatesCountsAndDeletesEmpty(t *testing.T) { SystemsInstallable: 99, SystemsApplicable: 99, }).Error) - assert.Nil(t, recountAdvisory(base.Context, 1, recountWorkspace, 1)) + assert.Nil(t, recountAdvisory(base.Context, 1, recountWorkspace, 1, time.Now().Add(time.Hour))) var expected int assert.Nil(t, database.TestDB().Raw(` @@ -124,7 +127,7 @@ func TestRecountUpdatesCountsAndDeletesEmpty(t *testing.T) { AdvisoryID: 9, RhAccountID: 1, WorkspaceID: recountWorkspace, SystemsInstallable: 1, SystemsApplicable: 1, }).Error) - assert.Nil(t, recountAdvisory(base.Context, 1, recountWorkspace, 9)) + assert.Nil(t, recountAdvisory(base.Context, 1, recountWorkspace, 9, time.Now().Add(time.Hour))) var left int64 assert.Nil(t, database.TestDB().Table("account_advisory"). @@ -132,3 +135,56 @@ func TestRecountUpdatesCountsAndDeletesEmpty(t *testing.T) { Count(&left).Error) assert.Equal(t, int64(0), left) } + +func setAdvisoryUpdated(t *testing.T, advisoryID int64, installable int, updated time.Time) { + t.Helper() + database.DeleteAccountAdvisory(t, 1, recountWorkspace.String(), []int64{advisoryID}) + assert.Nil(t, database.TestDB().Create(&models.AccountAdvisory{ + AdvisoryID: advisoryID, RhAccountID: 1, WorkspaceID: recountWorkspace, + SystemsInstallable: installable, SystemsApplicable: installable, + }).Error) + assert.Nil(t, database.TestDB().Model(&models.AccountAdvisory{}). + Where("rh_account_id = 1 AND workspace_id = ? AND advisory_id = ?", recountWorkspace, advisoryID). + UpdateColumns(map[string]interface{}{ + "systems_installable": installable, + "systems_applicable": installable, + "updated": updated, + }).Error) +} + +func TestRecountSkipsWhenUpdatedIsYoungerThanProducedAt(t *testing.T) { + utils.SkipWithoutDB(t) + core.SetupTestEnvironment() + t.Cleanup(func() { database.DeleteAccountAdvisory(t, 1, recountWorkspace.String(), []int64{1}) }) + + producedAt := time.Now().UTC().Truncate(time.Microsecond) + setAdvisoryUpdated(t, 1, 99, producedAt.Add(time.Hour)) + + assert.Nil(t, recountAdvisory(base.Context, 1, recountWorkspace, 1, producedAt)) + + got := database.PluckInt(database.TestDB().Table("account_advisory"). + Where("rh_account_id = 1 AND workspace_id = ? AND advisory_id = 1", recountWorkspace), + "systems_installable") + assert.Equal(t, 99, got) +} + +func TestRecountRunsWhenUpdatedIsOlderOrEqual(t *testing.T) { + utils.SkipWithoutDB(t) + core.SetupTestEnvironment() + t.Cleanup(func() { database.DeleteAccountAdvisory(t, 1, recountWorkspace.String(), []int64{1}) }) + + producedAt := time.Now().UTC().Truncate(time.Microsecond) + setAdvisoryUpdated(t, 1, 99, producedAt.Add(-time.Hour)) + assert.Nil(t, recountAdvisory(base.Context, 1, recountWorkspace, 1, producedAt)) + got := database.PluckInt(database.TestDB().Table("account_advisory"). + Where("rh_account_id = 1 AND workspace_id = ? AND advisory_id = 1", recountWorkspace), + "systems_installable") + assert.NotEqual(t, 99, got) + + setAdvisoryUpdated(t, 1, 99, producedAt) + assert.Nil(t, recountAdvisory(base.Context, 1, recountWorkspace, 1, producedAt)) + got = database.PluckInt(database.TestDB().Table("account_advisory"). + Where("rh_account_id = 1 AND workspace_id = ? AND advisory_id = 1", recountWorkspace), + "systems_installable") + assert.NotEqual(t, 99, got) +} From 348fa71abbfe2664366509d6dd5e620987cf0e8c Mon Sep 17 00:00:00 2001 From: Michael Mraka Date: Sat, 14 Nov 2026 22:28:29 +0000 Subject: [PATCH 11/17] RHINENG-33842: Key advisory update messages by account so one reader handles each account. --- base/mqueue/advisory_update_event.go | 3 ++- base/mqueue/advisory_update_event_test.go | 2 ++ base/mqueue/mqueue_impl_gokafka.go | 7 +++++++ base/mqueue/mqueue_test.go | 12 ++++++++++++ evaluator/advisory_update.go | 2 +- evaluator/advisory_update_test.go | 4 +++- 6 files changed, 27 insertions(+), 3 deletions(-) diff --git a/base/mqueue/advisory_update_event.go b/base/mqueue/advisory_update_event.go index 1fafd7419..b57dec6c5 100644 --- a/base/mqueue/advisory_update_event.go +++ b/base/mqueue/advisory_update_event.go @@ -3,6 +3,7 @@ package mqueue import ( "app/base/types" "context" + "strconv" "github.com/bytedance/sonic" "github.com/google/uuid" @@ -23,7 +24,7 @@ func (event *AdvisoryUpdateEvent) createKafkaMessage() (KafkaMessage, error) { if err != nil { return KafkaMessage{}, errors.Wrap(err, "Serializing advisory update event") } - return KafkaMessage{Value: data}, nil + return KafkaMessage{Key: []byte(strconv.Itoa(event.RhAccountID)), Value: data}, nil } func (events AdvisoryUpdateEvents) WriteEvents(ctx context.Context, w Writer) error { diff --git a/base/mqueue/advisory_update_event_test.go b/base/mqueue/advisory_update_event_test.go index 5aca66546..c56b125f3 100644 --- a/base/mqueue/advisory_update_event_test.go +++ b/base/mqueue/advisory_update_event_test.go @@ -59,6 +59,8 @@ func TestAdvisoryUpdateEventsWriteEvents(t *testing.T) { mockWriter := writer.(*MockKafkaWriter) assert.Equal(t, 2, len(mockWriter.Messages)) + assert.Equal(t, []byte("1"), mockWriter.Messages[0].Key) + assert.Equal(t, []byte("2"), mockWriter.Messages[1].Key) var firstEvent AdvisoryUpdateEvent err = sonic.Unmarshal(mockWriter.Messages[0].Value, &firstEvent) diff --git a/base/mqueue/mqueue_impl_gokafka.go b/base/mqueue/mqueue_impl_gokafka.go index 1e3e3ecf2..bf8d9cf35 100644 --- a/base/mqueue/mqueue_impl_gokafka.go +++ b/base/mqueue/mqueue_impl_gokafka.go @@ -174,6 +174,13 @@ func NewKafkaWriterFromEnv(topic string) Writer { return &kafkaGoWriterImpl{newKafkaWriterFromEnv(topic)} } +// NewKeyedKafkaWriterFromEnv routes each message by hashing its key, so the same key stays on one partition. +func NewKeyedKafkaWriterFromEnv(topic string) Writer { + writer := newKafkaWriterFromEnv(topic) + writer.Balancer = &kafka.Hash{} + return &kafkaGoWriterImpl{writer} +} + func NewGzipKafkaWriterFromEnv(topic string, batchBytes int64) Writer { writer := newKafkaWriterFromEnv(topic) writer.BatchBytes = batchBytes diff --git a/base/mqueue/mqueue_test.go b/base/mqueue/mqueue_test.go index 65b5a3b82..678cda4c6 100644 --- a/base/mqueue/mqueue_test.go +++ b/base/mqueue/mqueue_test.go @@ -10,6 +10,7 @@ import ( "github.com/bytedance/sonic" "github.com/google/uuid" + "github.com/segmentio/kafka-go" "github.com/stretchr/testify/assert" ) @@ -36,6 +37,17 @@ func TestRoundTripKafkaGo(t *testing.T) { }) } +func TestKeyedKafkaWriterRoutesSameAccountToSamePartition(t *testing.T) { + w := NewKeyedKafkaWriterFromEnv("patchman.advisory.update").(*kafkaGoWriterImpl) + assert.IsType(t, &kafka.Hash{}, w.Balancer) + + partitions := []int{0, 1, 2, 3, 4, 5} + msg := kafka.Message{Key: []byte("42")} + first := w.Balancer.Balance(msg, partitions...) + second := w.Balancer.Balance(msg, partitions...) + assert.Equal(t, first, second) +} + func TestSpawnReader(t *testing.T) { var nReaders int32 wg := sync.WaitGroup{} diff --git a/evaluator/advisory_update.go b/evaluator/advisory_update.go index 4b2b82e05..951429a27 100644 --- a/evaluator/advisory_update.go +++ b/evaluator/advisory_update.go @@ -15,7 +15,7 @@ var advisoryUpdatePublisher mqueue.Writer func configureAdvisoryUpdates() { if topic := utils.CoreCfg.AdvisoryUpdateTopic; topic != "" { - advisoryUpdatePublisher = mqueue.NewKafkaWriterFromEnv(topic) + advisoryUpdatePublisher = mqueue.NewKeyedKafkaWriterFromEnv(topic) } } diff --git a/evaluator/advisory_update_test.go b/evaluator/advisory_update_test.go index 54b595a9b..28ea725f1 100644 --- a/evaluator/advisory_update_test.go +++ b/evaluator/advisory_update_test.go @@ -82,6 +82,7 @@ func TestPublishAdvisoryUpdates(t *testing.T) { err := publishAdvisoryUpdates(system, advisories) assert.NoError(t, err) assert.Len(t, mockWriter.Messages, 1) + assert.Equal(t, []byte("3"), mockWriter.Messages[0].Key) var event mqueue.AdvisoryUpdateEvent assert.NoError(t, sonic.Unmarshal(mockWriter.Messages[0].Value, &event)) @@ -137,7 +138,7 @@ func TestAdvisoryUpdateKafkaRoundTrip(t *testing.T) { t.Skip("ADVISORY_UPDATE_TOPIC not configured") } - advisoryUpdatePublisher = mqueue.NewKafkaWriterFromEnv(topic) + advisoryUpdatePublisher = mqueue.NewKeyedKafkaWriterFromEnv(topic) reader := mqueue.NewKafkaReaderFromEnv(topic) defer func() { assert.NoError(t, reader.Close(), "failed to close Kafka reader") @@ -176,6 +177,7 @@ func TestAdvisoryUpdateKafkaRoundTrip(t *testing.T) { }) var event mqueue.AdvisoryUpdateEvent assert.NoError(t, sonic.Unmarshal(received.Value, &event), "message should be valid AdvisoryUpdateEvent JSON") + assert.Equal(t, []byte("3"), received.Key) assert.Equal(t, rhAccountID, event.RhAccountID) assert.NotEqual(t, uuid.Nil, event.WorkspaceID) assert.NotEmpty(t, event.AdvisoryIDs) From c2248d56edaca5c6bad59f96838fd48cb5e7fa6f Mon Sep 17 00:00:00 2001 From: Michael Mraka Date: Sat, 14 Nov 2026 23:24:50 +0000 Subject: [PATCH 12/17] RHINENG-33842: Serialize an advisory recount when its aggregate row is missing. --- aggregator/events.go | 18 +++++++++++++++++- aggregator/events_test.go | 40 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/aggregator/events.go b/aggregator/events.go index 6661fa99d..2ebeb6306 100644 --- a/aggregator/events.go +++ b/aggregator/events.go @@ -9,6 +9,7 @@ import ( "bytes" "context" "errors" + "fmt" "sort" "time" @@ -132,11 +133,26 @@ func processAdvisoryBatch(ctx context.Context, requests []advisoryRecount) error return nil } +// Use two-integer lock as namespace + row key. +const accountAdvisoryLockNamespace int32 = 1634890860 // "artl" in ASCII as Advisory Recount Lock Namespace + +func advisoryRecountLockKey(rhAccountID int, workspaceID uuid.UUID, advisoryID int64) string { + return fmt.Sprintf("%d/%s/%d", rhAccountID, workspaceID, advisoryID) +} + func recountAdvisory(ctx context.Context, rhAccountID int, workspaceID uuid.UUID, advisoryID int64, producedAt time.Time) error { return database.DB(ctx).Transaction(func(tx *gorm.DB) error { + // Serialize this key even when account_advisory SELECT ... FOR UPDATE returns nothing + // so an older upsert can overwrite newer counts or a delayed delete can remove a fresh row. + err := tx.Exec("SELECT pg_advisory_xact_lock(?::integer, hashtext(?))", + accountAdvisoryLockNamespace, advisoryRecountLockKey(rhAccountID, workspaceID, advisoryID)).Error + if err != nil { + return err + } + var row models.AccountAdvisory - err := tx.Clauses(clause.Locking{Strength: "UPDATE"}). + err = tx.Clauses(clause.Locking{Strength: "UPDATE"}). Where("rh_account_id = ? AND workspace_id = ? AND advisory_id = ?", rhAccountID, workspaceID, advisoryID). Take(&row).Error if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { diff --git a/aggregator/events_test.go b/aggregator/events_test.go index d32ae8173..43a25866e 100644 --- a/aggregator/events_test.go +++ b/aggregator/events_test.go @@ -188,3 +188,43 @@ func TestRecountRunsWhenUpdatedIsOlderOrEqual(t *testing.T) { "systems_installable") assert.NotEqual(t, 99, got) } + +func TestRecountAdvisoryWaitsForKeyLock(t *testing.T) { + utils.SkipWithoutDB(t) + core.SetupTestEnvironment() + t.Cleanup(func() { database.DeleteAccountAdvisory(t, 1, recountWorkspace.String(), []int64{1}) }) + + hold := database.TestDB().Begin() + t.Cleanup(func() { _ = hold.Rollback().Error }) + lockKey := advisoryRecountLockKey(1, recountWorkspace, 1) + assert.Nil(t, hold.Exec("SELECT pg_advisory_xact_lock(?::integer, hashtext(?))", + accountAdvisoryLockNamespace, lockKey).Error) + + done := make(chan error, 1) + go func() { + done <- recountAdvisory(base.Context, 1, recountWorkspace, 1, time.Now().Add(time.Hour)) + }() + + deadline := time.Now().Add(2 * time.Second) + waiting := 0 + for time.Now().Before(deadline) { + assert.Nil(t, database.TestDB().Raw(` + SELECT count(*) FROM pg_stat_activity + WHERE wait_event_type = 'Lock' + AND wait_event = 'advisory' + AND query LIKE 'SELECT pg_advisory_xact_lock%'`).Scan(&waiting).Error) + if waiting > 0 { + break + } + time.Sleep(20 * time.Millisecond) + } + assert.Greater(t, waiting, 0, "recount should wait on the key lock") + + assert.Nil(t, hold.Rollback().Error) + select { + case err := <-done: + assert.Nil(t, err) + case <-time.After(5 * time.Second): + t.Fatal("recount did not finish after the key lock was released") + } +} From 8d74b900465446aad5d5ed5493fb5c4246a8d9ba Mon Sep 17 00:00:00 2001 From: Michael Mraka Date: Sat, 14 Nov 2026 23:44:36 +0000 Subject: [PATCH 13/17] RHINENG-33842: Keep subsecond precision on advisory update produced_at so a later same-second event is not skipped. --- aggregator/events_test.go | 29 +++++++++++++++++++---- base/mqueue/advisory_update_event.go | 8 +++---- base/mqueue/advisory_update_event_test.go | 2 +- evaluator/advisory_update.go | 2 +- evaluator/advisory_update_test.go | 2 +- 5 files changed, 31 insertions(+), 12 deletions(-) diff --git a/aggregator/events_test.go b/aggregator/events_test.go index 43a25866e..b260008ad 100644 --- a/aggregator/events_test.go +++ b/aggregator/events_test.go @@ -21,18 +21,37 @@ var ( otherWorkspace = uuid.MustParse("00000000-0000-0000-0000-000000000002") ) +func TestSameSecondProducedAtIsNotOlderAfterRoundTrip(t *testing.T) { + second := time.Date(2026, 11, 15, 12, 0, 0, 0, time.UTC) + updated := second.Add(400 * time.Millisecond) + producedAt := second.Add(800 * time.Millisecond) + msg := toKafkaMessage(t, mqueue.AdvisoryUpdateEvent{ + RhAccountID: 1, + WorkspaceID: recountWorkspace, + AdvisoryIDs: []int64{1}, + ProducedAt: types.Rfc3339TimestampWithZ(producedAt), + }) + + var event mqueue.AdvisoryUpdateEvent + assert.NoError(t, sonic.Unmarshal(msg.Value, &event)) + got := event.ProducedAt.Time() + assert.NotNil(t, got) + assert.True(t, got.Equal(producedAt)) + assert.False(t, updated.After(*got)) +} + func TestDedupeAdvisoryUpdatesKeepsYoungestProducedAt(t *testing.T) { older := time.Date(2026, 11, 14, 10, 0, 0, 0, time.UTC) newer := older.Add(time.Minute) events := []mqueue.AdvisoryUpdateEvent{ {RhAccountID: 2, WorkspaceID: recountWorkspace, AdvisoryIDs: []int64{5}, - ProducedAt: types.Rfc3339Timestamp(older)}, + ProducedAt: types.Rfc3339TimestampWithZ(older)}, {RhAccountID: 1, WorkspaceID: otherWorkspace, AdvisoryIDs: []int64{2, 1}, - ProducedAt: types.Rfc3339Timestamp(older)}, + ProducedAt: types.Rfc3339TimestampWithZ(older)}, {RhAccountID: 1, WorkspaceID: recountWorkspace, AdvisoryIDs: []int64{2, 1}, - ProducedAt: types.Rfc3339Timestamp(older)}, + ProducedAt: types.Rfc3339TimestampWithZ(older)}, {RhAccountID: 1, WorkspaceID: recountWorkspace, AdvisoryIDs: []int64{1}, - ProducedAt: types.Rfc3339Timestamp(newer)}, + ProducedAt: types.Rfc3339TimestampWithZ(newer)}, } got := groupAdvisoryUpdates(events) @@ -70,7 +89,7 @@ func TestBufferedEventsProcessedOnBatchThreshold(t *testing.T) { msg := toKafkaMessage(t, mqueue.AdvisoryUpdateEvent{ RhAccountID: 1, WorkspaceID: recountWorkspace, AdvisoryIDs: []int64{2, 1}, - ProducedAt: types.Rfc3339Timestamp(time.Now().Add(time.Hour)), + ProducedAt: types.Rfc3339TimestampWithZ(time.Now().Add(time.Hour)), }) assert.Nil(t, advisoryUpdateHandler(base.Context, []mqueue.KafkaMessage{msg, msg, msg})) diff --git a/base/mqueue/advisory_update_event.go b/base/mqueue/advisory_update_event.go index b57dec6c5..69d933619 100644 --- a/base/mqueue/advisory_update_event.go +++ b/base/mqueue/advisory_update_event.go @@ -11,10 +11,10 @@ import ( ) type AdvisoryUpdateEvent struct { - RhAccountID int `json:"rh_account_id"` - WorkspaceID uuid.UUID `json:"workspace_id"` - AdvisoryIDs []int64 `json:"advisory_ids"` - ProducedAt types.Rfc3339Timestamp `json:"produced_at"` + RhAccountID int `json:"rh_account_id"` + WorkspaceID uuid.UUID `json:"workspace_id"` + AdvisoryIDs []int64 `json:"advisory_ids"` + ProducedAt types.Rfc3339TimestampWithZ `json:"produced_at"` } type AdvisoryUpdateEvents []AdvisoryUpdateEvent diff --git a/base/mqueue/advisory_update_event_test.go b/base/mqueue/advisory_update_event_test.go index c56b125f3..4ad7d6ed1 100644 --- a/base/mqueue/advisory_update_event_test.go +++ b/base/mqueue/advisory_update_event_test.go @@ -13,7 +13,7 @@ import ( var ( testWorkspaceID = uuid.New() - testNow = types.Rfc3339Timestamp(time.Now()) + testNow = types.Rfc3339TimestampWithZ(time.Now()) ) func TestAdvisoryUpdateEventMarshal(t *testing.T) { diff --git a/evaluator/advisory_update.go b/evaluator/advisory_update.go index 951429a27..912999704 100644 --- a/evaluator/advisory_update.go +++ b/evaluator/advisory_update.go @@ -50,7 +50,7 @@ func publishAdvisoryUpdates(system *models.SystemPlatformV2, advisoriesByName ex RhAccountID: system.Inventory.RhAccountID, WorkspaceID: system.Inventory.WorkspaceID, AdvisoryIDs: advisoryIDs, - ProducedAt: types.Rfc3339Timestamp(time.Now()), + ProducedAt: types.Rfc3339TimestampWithZ(time.Now()), } if err := mqueue.SendMessages(base.Context, advisoryUpdatePublisher, &mqueue.AdvisoryUpdateEvents{event}); err != nil { return errors.Wrap(err, "writing advisory update events") diff --git a/evaluator/advisory_update_test.go b/evaluator/advisory_update_test.go index 28ea725f1..b5ac0ac59 100644 --- a/evaluator/advisory_update_test.go +++ b/evaluator/advisory_update_test.go @@ -48,7 +48,7 @@ func TestCreateAdvisoryUpdateEvent(t *testing.T) { RhAccountID: system.Inventory.RhAccountID, WorkspaceID: system.Inventory.WorkspaceID, AdvisoryIDs: changedAdvisoryIDs, - ProducedAt: types.Rfc3339Timestamp(time.Now()), + ProducedAt: types.Rfc3339TimestampWithZ(time.Now()), } assert.Equal(t, rhAccountID, event.RhAccountID) assert.Equal(t, wsID, event.WorkspaceID) From d9c9a134c80f0938258d294b16d5020c00a76f8b Mon Sep 17 00:00:00 2001 From: Michael Mraka Date: Sun, 15 Nov 2026 11:35:00 +0100 Subject: [PATCH 14/17] RHINENG-33842: Pass the Kafka message trace context to handlers of multi-message batches. --- base/mqueue/batch_test.go | 53 ++++++++++++++++++++++++++++++ base/mqueue/mqueue_impl_gokafka.go | 18 ++++------ 2 files changed, 60 insertions(+), 11 deletions(-) diff --git a/base/mqueue/batch_test.go b/base/mqueue/batch_test.go index 996c6983a..3bd2a7ebc 100644 --- a/base/mqueue/batch_test.go +++ b/base/mqueue/batch_test.go @@ -1,6 +1,7 @@ package mqueue import ( + "app/base/telemetry" "context" "errors" "testing" @@ -8,6 +9,8 @@ import ( "github.com/segmentio/kafka-go" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace" ) func TestAsBatchHandler(t *testing.T) { @@ -77,6 +80,56 @@ func TestHandleMessageBatchesBlocksForFirstMessage(t *testing.T) { assert.Equal(t, []KafkaMessage{{Value: []byte("one")}}, got) } +func TestHandleMessageBatchesMultiMessageUsesMessageTrace(t *testing.T) { + t.Setenv("OTEL_ENABLED", "true") + t.Setenv("OTEL_SAMPLING_RATE", "1.0") + t.Setenv("OTEL_BSP_EXPORT_TIMEOUT", "1") + t.Setenv("OTEL_BSP_SCHEDULE_DELAY", "1") + require.NoError(t, telemetry.Init()) + t.Cleanup(func() { _ = telemetry.Shutdown(context.Background()) }) + + traceID, err := trace.TraceIDFromHex("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + require.NoError(t, err) + spanID, err := trace.SpanIDFromHex("bbbbbbbbbbbbbbbb") + require.NoError(t, err) + remote := trace.NewSpanContext(trace.SpanContextConfig{ + TraceID: traceID, + SpanID: spanID, + TraceFlags: trace.FlagsSampled, + Remote: true, + }) + headers := telemetry.Inject(trace.ContextWithRemoteSpanContext(context.Background(), remote), nil) + queued := []kafka.Message{ + {Offset: 1, Value: []byte("a"), Headers: headers}, + {Offset: 2, Value: []byte("b"), Headers: headers}, + } + calls := 0 + fetch := func(ctx context.Context) (kafka.Message, error) { + calls++ + if calls > len(queued) { + <-ctx.Done() + return kafka.Message{}, ctx.Err() + } + return queued[calls-1], nil + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var got trace.TraceID + var valid bool + handleMessageBatches(ctx, "topic", fetch, func(context.Context, ...kafka.Message) error { + cancel() + return nil + }, 2, time.Hour, func(handlerCtx context.Context, _ []KafkaMessage) error { + sc := trace.SpanFromContext(handlerCtx).SpanContext() + valid = sc.IsValid() + got = sc.TraceID() + return nil + }) + assert.True(t, valid) + assert.Equal(t, traceID, got) +} + func TestHandleMessageBatchesStopsAtN(t *testing.T) { queued := []kafka.Message{ {Offset: 1, Value: []byte("a")}, diff --git a/base/mqueue/mqueue_impl_gokafka.go b/base/mqueue/mqueue_impl_gokafka.go index bf8d9cf35..1c253fc58 100644 --- a/base/mqueue/mqueue_impl_gokafka.go +++ b/base/mqueue/mqueue_impl_gokafka.go @@ -44,11 +44,7 @@ func handleMessageBatches(ctx context.Context, topic string, fetch func(context. panic(err) } - handlerCtx := ctx - if len(batch.app) == 1 { - handlerCtx = batch.singleCtx - } - err = handler(handlerCtx, batch.app) + err = handler(batch.messageCtx, batch.app) batch.endSpans(err) if err != nil { utils.LogPanic("err", err, "Handler failed") @@ -64,10 +60,10 @@ func handleMessageBatches(ctx context.Context, topic string, fetch func(context. } type fetchedBatch struct { - app []KafkaMessage - raw []kafka.Message - spans []trace.Span - singleCtx context.Context + app []KafkaMessage + raw []kafka.Message + spans []trace.Span + messageCtx context.Context } func (b *fetchedBatch) add(ctx context.Context, topic string, m kafka.Message) { @@ -75,8 +71,8 @@ func (b *fetchedBatch) add(ctx context.Context, topic string, m kafka.Message) { utils.LogTrace("count", len(m.Headers), "kafka message headers") } msgCtx, span := telemetry.ConsumerContext(ctx, topic, m.Headers) - if len(b.app) == 0 { - b.singleCtx = msgCtx + if b.messageCtx == nil { + b.messageCtx = msgCtx } b.app = append(b.app, KafkaMessage{Key: m.Key, Value: m.Value, Headers: m.Headers}) b.raw = append(b.raw, m) From 7415d136e260b6c250a825b1bdfaa1809340d37b Mon Sep 17 00:00:00 2001 From: Michael Mraka Date: Sun, 15 Nov 2026 13:01:59 +0000 Subject: [PATCH 15/17] RHINENG-33842: Require a recount to keep the aggregate row and write the source count. --- aggregator/events_test.go | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/aggregator/events_test.go b/aggregator/events_test.go index b260008ad..700d5babf 100644 --- a/aggregator/events_test.go +++ b/aggregator/events_test.go @@ -193,19 +193,33 @@ func TestRecountRunsWhenUpdatedIsOlderOrEqual(t *testing.T) { t.Cleanup(func() { database.DeleteAccountAdvisory(t, 1, recountWorkspace.String(), []int64{1}) }) producedAt := time.Now().UTC().Truncate(time.Microsecond) + var counts []advisoryRecountCount + assert.Nil(t, database.TestDB().Raw(advisoryCountSQL, 1, int64(1), 1, recountWorkspace).Scan(&counts).Error) + if !assert.NotEmpty(t, counts) { + return + } + expected := counts[0].SystemsInstallable + + assertRecounted := func() { + t.Helper() + var rows int64 + assert.Nil(t, database.TestDB().Table("account_advisory"). + Where("rh_account_id = 1 AND workspace_id = ? AND advisory_id = 1", recountWorkspace). + Count(&rows).Error) + assert.Equal(t, int64(1), rows) + got := database.PluckInt(database.TestDB().Table("account_advisory"). + Where("rh_account_id = 1 AND workspace_id = ? AND advisory_id = 1", recountWorkspace), + "systems_installable") + assert.Equal(t, expected, got) + } + setAdvisoryUpdated(t, 1, 99, producedAt.Add(-time.Hour)) assert.Nil(t, recountAdvisory(base.Context, 1, recountWorkspace, 1, producedAt)) - got := database.PluckInt(database.TestDB().Table("account_advisory"). - Where("rh_account_id = 1 AND workspace_id = ? AND advisory_id = 1", recountWorkspace), - "systems_installable") - assert.NotEqual(t, 99, got) + assertRecounted() setAdvisoryUpdated(t, 1, 99, producedAt) assert.Nil(t, recountAdvisory(base.Context, 1, recountWorkspace, 1, producedAt)) - got = database.PluckInt(database.TestDB().Table("account_advisory"). - Where("rh_account_id = 1 AND workspace_id = ? AND advisory_id = 1", recountWorkspace), - "systems_installable") - assert.NotEqual(t, 99, got) + assertRecounted() } func TestRecountAdvisoryWaitsForKeyLock(t *testing.T) { From f5074f84be5af36280aa9fd08faf9ba959fa05a5 Mon Sep 17 00:00:00 2001 From: Michael Mraka Date: Sun, 15 Nov 2026 13:07:44 +0000 Subject: [PATCH 16/17] RHINENG-33842: Index system_advisories by account and advisory so recounts avoid an account scan. --- .../172_system_advisories_account_advisory_idx.down.sql | 1 + .../172_system_advisories_account_advisory_idx.up.sql | 1 + database_admin/schema/create_schema.sql | 4 +++- 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 database_admin/migrations/172_system_advisories_account_advisory_idx.down.sql create mode 100644 database_admin/migrations/172_system_advisories_account_advisory_idx.up.sql diff --git a/database_admin/migrations/172_system_advisories_account_advisory_idx.down.sql b/database_admin/migrations/172_system_advisories_account_advisory_idx.down.sql new file mode 100644 index 000000000..ef56211b9 --- /dev/null +++ b/database_admin/migrations/172_system_advisories_account_advisory_idx.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS system_advisories_rh_account_id_advisory_id_idx; diff --git a/database_admin/migrations/172_system_advisories_account_advisory_idx.up.sql b/database_admin/migrations/172_system_advisories_account_advisory_idx.up.sql new file mode 100644 index 000000000..ba76b2294 --- /dev/null +++ b/database_admin/migrations/172_system_advisories_account_advisory_idx.up.sql @@ -0,0 +1 @@ +CREATE INDEX ON system_advisories (rh_account_id, advisory_id); diff --git a/database_admin/schema/create_schema.sql b/database_admin/schema/create_schema.sql index 5a684de26..24dcc08f7 100644 --- a/database_admin/schema/create_schema.sql +++ b/database_admin/schema/create_schema.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS schema_migrations INSERT INTO schema_migrations -VALUES (171, false); +VALUES (172, false); -- --------------------------------------------------------------------------- -- Functions @@ -870,6 +870,8 @@ GRANT SELECT, INSERT, UPDATE, DELETE ON system_advisories TO listener; -- vmaas_sync needs to delete culled systems, which cascades to system_advisories GRANT SELECT, DELETE ON system_advisories TO vmaas_sync; +CREATE INDEX ON system_advisories (rh_account_id, advisory_id); + -- advisory_account_data CREATE TABLE IF NOT EXISTS advisory_account_data ( From 84bf188508a03a3317e3787edf5086aac030b039 Mon Sep 17 00:00:00 2001 From: Michael Mraka Date: Sun, 15 Nov 2026 14:12:17 +0000 Subject: [PATCH 17/17] RHINENG-33842: Publish advisory updates for both workspaces when a system moves so both aggregate counts are recounted. --- deploy/clowdapp.yaml | 1 + listener/listener.go | 4 ++ listener/upload.go | 91 ++++++++++++++++++++++++++++++++-------- listener/upload_test.go | 92 +++++++++++++++++++++++++++++++++++------ 4 files changed, 158 insertions(+), 30 deletions(-) diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml index 698dc53a3..2b97f385a 100644 --- a/deploy/clowdapp.yaml +++ b/deploy/clowdapp.yaml @@ -192,6 +192,7 @@ objects: - {name: KAFKA_WRITER_MAX_ATTEMPTS, value: '${KAFKA_WRITER_MAX_ATTEMPTS}'} - {name: EVENTS_TOPIC, value: platform.inventory.events} - {name: EVAL_TOPIC, value: patchman.evaluator.upload} + - {name: ADVISORY_UPDATE_TOPIC, value: 'patchman.advisory.update'} - {name: CREATED_SYSTEMS_TOPIC, value: patchman.evaluator.user-evaluation} - {name: PAYLOAD_TRACKER_TOPIC, value: platform.payload-status} - {name: TEMPLATE_TOPIC, value: platform.content-sources.template} diff --git a/listener/listener.go b/listener/listener.go index e7bfb23d2..48d366a3b 100644 --- a/listener/listener.go +++ b/listener/listener.go @@ -25,6 +25,7 @@ var ( templatesTopic string templatesConsumers int evalWriter mqueue.Writer + advisoryUpdateWriter mqueue.Writer createdSystemsWriter mqueue.Writer ptWriter mqueue.Writer validReporters map[string]int @@ -62,6 +63,9 @@ func configure() { evalWriter = mqueue.NewKafkaWriterFromEnv(evalTopic) ptWriter = mqueue.NewKafkaWriterFromEnv(ptTopic) createdSystemsWriter = mqueue.NewKafkaWriterFromEnv(createdTopic) + if topic := utils.CoreCfg.AdvisoryUpdateTopic; topic != "" { + advisoryUpdateWriter = mqueue.NewKeyedKafkaWriterFromEnv(topic) + } updatedEventsBuffer.initEventBuffer(&evalWriter, &ptWriter, evalTopic) createdEventsBuffer.initEventBuffer(&createdSystemsWriter, &ptWriter, createdTopic) diff --git a/listener/upload.go b/listener/upload.go index b9e6d211a..707d6d2a3 100644 --- a/listener/upload.go +++ b/listener/upload.go @@ -317,13 +317,14 @@ func hostTemplate(tx *gorm.DB, accountID int, host *Host) *int64 { // Stores or updates base system profile, returning inventory + patch aggregate. // templateID must be resolved before SELECT ... FOR UPDATE on system_inventory. func updateSystemPlatform(tx *gorm.DB, accountID int, host *Host, - yumUpdates *YumUpdates, updatesReq *vmaas.UpdatesV3Request, templateID *int64) (*models.SystemPlatformV2, error) { + yumUpdates *YumUpdates, updatesReq *vmaas.UpdatesV3Request, templateID *int64) (*models.SystemPlatformV2, + mqueue.AdvisoryUpdateEvents, error) { defer utils.ObserveSecondsSince(time.Now(), messagePartDuration.WithLabelValues("update-system-platform")) // NOTE: if we add a map to vmaas.UpdatesV3Request in the future, we need to use // `encoder.Encode(updatesReq, encoder.SortMapKeys)` to compute the hash correctly updatesReqJSON, err := sonic.Marshal(updatesReq) if err != nil { - return nil, errors.Wrap(err, "Serializing vmaas request") + return nil, nil, errors.Wrap(err, "Serializing vmaas request") } inventoryID := host.ID hash := sha256.Sum256(updatesReqJSON) @@ -374,7 +375,7 @@ func updateSystemPlatform(tx *gorm.DB, accountID int, host *Host, updatesReqJSONString := string(updatesReqJSON) workspaceID, workspaceName, err := hostWorkspace(host) if err != nil { - return nil, err + return nil, nil, err } systemPlatform := &models.SystemPlatformV2{ Inventory: models.SystemInventory{ @@ -423,34 +424,40 @@ func updateSystemPlatform(tx *gorm.DB, accountID int, host *Host, }, } - type OldChecksums struct { - JSONChecksum string `gorm:"column:json_checksum"` - YumChecksum string `gorm:"column:yum_checksum"` + type priorInventory struct { + ID int64 + JSONChecksum string + YumChecksum string + WorkspaceID uuid.UUID } - var oldChecksums OldChecksums - // Lock the system_inventory row and read prior json_checksum / yum_checksum for incremental updates. - tx.Clauses(clause.Locking{Strength: "UPDATE"}). + var prior priorInventory + // Lock the system_inventory row and read prior checksums and workspace. + err = tx.Clauses(clause.Locking{Strength: "UPDATE"}). Model(&models.SystemInventory{}). Where("rh_account_id = ? AND inventory_id = ?", accountID, inventoryID). - Select("json_checksum, yum_checksum"). - First(&oldChecksums) + Select("id, json_checksum, yum_checksum, workspace_id"). + First(&prior).Error + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil, base.WrapFatalDBError(err, "locking system_inventory") + } + rowExists := err == nil shouldUpdateRepos := false var addedRepos, addedSysRepos, deletedSysRepos int64 // Skip updating vmaas_json if the checksum haven't changed. Should reduce TOAST trashing - if oldChecksums.JSONChecksum != jsonChecksum { + if prior.JSONChecksum != jsonChecksum { colsToUpdate = append(colsToUpdate, "vmaas_json", "json_checksum", "reporter_id") shouldUpdateRepos = true } // Skip updating yum_updates if the checksum haven't changed. - if oldChecksums.YumChecksum != yumChecksum { + if prior.YumChecksum != yumChecksum { colsToUpdate = append(colsToUpdate, "yum_updates", "yum_checksum") } if err := storeOrUpdateSysPlatform(tx, systemPlatform, colsToUpdate); err != nil { - return nil, errors.Wrap(err, "Unable to save or update system in database") + return nil, nil, errors.Wrap(err, "Unable to save or update system in database") } if shouldUpdateRepos { @@ -460,7 +467,7 @@ func updateSystemPlatform(tx *gorm.DB, accountID int, host *Host, if err != nil { utils.LogError("repository_list", updatesReq.RepositoryList, "inventoryID", inventoryID, "repos failed to insert") - return nil, errors.Wrap(err, "unable to update system repos") + return nil, nil, errors.Wrap(err, "unable to update system repos") } } @@ -468,7 +475,44 @@ func updateSystemPlatform(tx *gorm.DB, accountID int, host *Host, len(updatesReq.RepositoryList), "modules", len(updatesReq.GetModulesList()), "addedRepos", addedRepos, "addedSysRepos", addedSysRepos, "deletedSysRepos", deletedSysRepos, "System created or updated successfully") - return systemPlatform, nil + events, err := workspaceMoveEvents(tx, accountID, prior.ID, prior.WorkspaceID, workspaceID, rowExists) + if err != nil { + return nil, nil, err + } + return systemPlatform, events, nil +} + +func workspaceMoveEvents(tx *gorm.DB, accountID int, systemID int64, oldWorkspace, newWorkspace uuid.UUID, + rowExists bool) (mqueue.AdvisoryUpdateEvents, error) { + if !rowExists || oldWorkspace == newWorkspace { + return nil, nil + } + var advisoryIDs []int64 + err := tx.Model(&models.SystemAdvisories{}). + Where("rh_account_id = ? AND system_id = ?", accountID, systemID). + Order("advisory_id"). + Pluck("advisory_id", &advisoryIDs).Error + if err != nil { + return nil, base.WrapFatalDBError(err, "loading system advisories for workspace move") + } + if len(advisoryIDs) == 0 { + return nil, nil + } + now := types.Rfc3339TimestampWithZ(time.Now()) + return mqueue.AdvisoryUpdateEvents{ + { + RhAccountID: accountID, + WorkspaceID: oldWorkspace, + AdvisoryIDs: advisoryIDs, + ProducedAt: now, + }, + { + RhAccountID: accountID, + WorkspaceID: newWorkspace, + AdvisoryIDs: advisoryIDs, + ProducedAt: now, + }, + }, nil } // nolint: funlen @@ -818,7 +862,7 @@ func processUpload(ctx context.Context, host *Host, yumUpdates *YumUpdates) (*mo if skip, err := skipRecentlyDeleted(tx, host.ID); err != nil || skip { return nil, err } - sys, err := updateSystemPlatform(tx, accountID, host, yumUpdates, &updatesReq, templateID) + sys, events, err := updateSystemPlatform(tx, accountID, host, yumUpdates, &updatesReq, templateID) if err != nil { return nil, errors.Wrap(err, "saving system into the database") } @@ -826,9 +870,22 @@ func processUpload(ctx context.Context, host *Host, yumUpdates *YumUpdates) (*mo if err != nil { return nil, base.WrapFatalDBError(err, "committing changes") } + if err := publishWorkspaceAdvisoryUpdates(events); err != nil { + utils.LogError("inventoryID", host.ID, "err", err, "failed to publish workspace advisory updates") + } return sys, nil } +func publishWorkspaceAdvisoryUpdates(events mqueue.AdvisoryUpdateEvents) error { + if len(events) == 0 || advisoryUpdateWriter == nil { + return nil + } + if err := mqueue.SendMessages(base.Context, advisoryUpdateWriter, &events); err != nil { + return errors.Wrap(err, "writing workspace advisory updates") + } + return nil +} + func hostWorkspace(host *Host) (uuid.UUID, *string, error) { l := len(host.Groups) if l == 0 { diff --git a/listener/upload_test.go b/listener/upload_test.go index 3b15e5d1b..74c33d7db 100644 --- a/listener/upload_test.go +++ b/listener/upload_test.go @@ -99,7 +99,7 @@ func TestUpdateSystemPlatform(t *testing.T) { Basearch: utils.PtrString("x86_64"), } - sys1, err := updateSystemPlatform(database.TestDB(), accountID1, createTestInvHost(t), nil, &req, nil) + sys1, _, err := updateSystemPlatform(database.TestDB(), accountID1, createTestInvHost(t), nil, &req, nil) assert.Nil(t, err) reporterID1 := 1 @@ -109,7 +109,7 @@ func TestUpdateSystemPlatform(t *testing.T) { host2 := createTestInvHost(t) host2.Reporter = "yupana" req.PackageList = []string{"package0", "package1"} - sys2, err := updateSystemPlatform(database.TestDB(), accountID2, host2, nil, &req, nil) + sys2, _, err := updateSystemPlatform(database.TestDB(), accountID2, host2, nil, &req, nil) assert.Nil(t, err) reporterID2 := 3 @@ -152,31 +152,31 @@ func TestUpdateSystemPlatformIndependentIdentityIDs(t *testing.T) { ev.Host.SystemProfile.OwnerID = &ownerID1 ev.Host.SubscriptionManagerID = &ownerID1 - _, err = updateSystemPlatform(database.TestDB(), acc, &ev.Host, nil, &reqProfile, nil) + _, _, err = updateSystemPlatform(database.TestDB(), acc, &ev.Host, nil, &reqProfile, nil) require.NoError(t, err) assertSystemInventoryProfileMatchesHost(t, testInventoryID, &ev.Host) ownerID2 := uuid.MustParse("22222222-2222-2222-2222-222222222222") ev.Host.SystemProfile.OwnerID = &ownerID2 - _, err = updateSystemPlatform(database.TestDB(), acc, &ev.Host, nil, &reqProfile, nil) + _, _, err = updateSystemPlatform(database.TestDB(), acc, &ev.Host, nil, &reqProfile, nil) require.NoError(t, err) assertSystemInventoryProfileMatchesHost(t, testInventoryID, &ev.Host) subscriptionManagerID := uuid.MustParse("33333333-3333-3333-3333-333333333333") ev.Host.SubscriptionManagerID = &subscriptionManagerID - _, err = updateSystemPlatform(database.TestDB(), acc, &ev.Host, nil, &reqProfile, nil) + _, _, err = updateSystemPlatform(database.TestDB(), acc, &ev.Host, nil, &reqProfile, nil) require.NoError(t, err) assertSystemInventoryProfileMatchesHost(t, testInventoryID, &ev.Host) ev.Host.SystemProfile.OwnerID = nil - _, err = updateSystemPlatform(database.TestDB(), acc, &ev.Host, nil, &reqProfile, nil) + _, _, err = updateSystemPlatform(database.TestDB(), acc, &ev.Host, nil, &reqProfile, nil) require.NoError(t, err) assertSystemInventoryProfileMatchesHost(t, testInventoryID, &ev.Host) ev.Host.SystemProfile.OwnerID = &ownerID2 ev.Host.SubscriptionManagerID = nil - _, err = updateSystemPlatform(database.TestDB(), acc, &ev.Host, nil, &reqProfile, nil) + _, _, err = updateSystemPlatform(database.TestDB(), acc, &ev.Host, nil, &reqProfile, nil) require.NoError(t, err) assertSystemInventoryProfileMatchesHost(t, testInventoryID, &ev.Host) @@ -222,7 +222,7 @@ func TestUpdateSystemPlatformRefreshesInventoryProfileOnConflict(t *testing.T) { } ev := createTestUploadEvent(testOrgID, testInventoryID, "puptoo", true, false, "created") ev.Host.StaleTimestamp = &correctTime - _, err = updateSystemPlatform(database.TestDB(), acc, &ev.Host, nil, &reqProfile, nil) + _, _, err = updateSystemPlatform(database.TestDB(), acc, &ev.Host, nil, &reqProfile, nil) require.NoError(t, err) assertSystemInventoryProfileMatchesHost(t, testInventoryID, &ev.Host) @@ -233,7 +233,7 @@ func TestUpdateSystemPlatformRefreshesInventoryProfileOnConflict(t *testing.T) { ev.Host.SystemProfile.Workloads.Ansible.ControllerVersion = "2.13.0" ev.Host.SystemProfile.Workloads.Mssql.Version = "16.0" - _, err = updateSystemPlatform(database.TestDB(), acc, &ev.Host, nil, &reqProfile, nil) + _, _, err = updateSystemPlatform(database.TestDB(), acc, &ev.Host, nil, &reqProfile, nil) require.NoError(t, err) assertSystemInventoryProfileMatchesHost(t, testInventoryID, &ev.Host) @@ -459,7 +459,7 @@ func TestUpdateSystemPlatformYumUpdates(t *testing.T) { req := vmaas.UpdatesV3Request{} - _, err = updateSystemPlatform(database.TestDB(), accountID1, &hostEvent.Host, yumUpdates, &req, nil) + _, _, err = updateSystemPlatform(database.TestDB(), accountID1, &hostEvent.Host, yumUpdates, &req, nil) assert.Nil(t, err) reporterID1 := 1 @@ -470,7 +470,7 @@ func TestUpdateSystemPlatformYumUpdates(t *testing.T) { // check that yumUpdates has been updated (keep the same Host so profile columns are not wiped) yumUpdates.RawParsed = []byte("{}") - _, err = updateSystemPlatform(database.TestDB(), accountID1, &hostEvent.Host, yumUpdates, &req, nil) + _, _, err = updateSystemPlatform(database.TestDB(), accountID1, &hostEvent.Host, yumUpdates, &req, nil) assert.Nil(t, err) assertYumUpdatesInDB(t, testInventoryID, yumUpdates) assertSystemInventoryProfileMatchesHost(t, testInventoryID, &hostEvent.Host) @@ -481,7 +481,7 @@ func TestUpdateSystemPlatformYumUpdates(t *testing.T) { hostEvent.Host.SystemProfile.Workloads.Sap.Sids = []string{"sid-yum-test"} hostEvent.Host.SystemProfile.Workloads.Ansible.ControllerVersion = "2.14.0" hostEvent.Host.SystemProfile.Workloads.Mssql.Version = "17.0" - _, err = updateSystemPlatform(database.TestDB(), accountID1, &hostEvent.Host, yumUpdates, &req, nil) + _, _, err = updateSystemPlatform(database.TestDB(), accountID1, &hostEvent.Host, yumUpdates, &req, nil) assert.Nil(t, err) assertYumUpdatesInDB(t, testInventoryID, yumUpdates) assertSystemInventoryProfileMatchesHost(t, testInventoryID, &hostEvent.Host) @@ -492,7 +492,7 @@ func TestUpdateSystemPlatformYumUpdates(t *testing.T) { hostEvent.Host.SystemProfile.Workloads.Sap.Sids = nil hostEvent.Host.SystemProfile.Workloads.Ansible.ControllerVersion = "" hostEvent.Host.SystemProfile.Workloads.Mssql.Version = "" - _, err = updateSystemPlatform(database.TestDB(), accountID1, &hostEvent.Host, yumUpdates, &req, nil) + _, _, err = updateSystemPlatform(database.TestDB(), accountID1, &hostEvent.Host, yumUpdates, &req, nil) assert.Nil(t, err) assertYumUpdatesInDB(t, testInventoryID, yumUpdates) assertSystemInventoryProfileMatchesHost(t, testInventoryID, &hostEvent.Host) @@ -796,3 +796,69 @@ func TestHostTemplateCandlepinFailure(t *testing.T) { templateID := hostTemplate(database.TestDB(), accountID, host) assert.Nil(t, templateID) } + +func TestUpdateSystemPlatformEmitsBothWorkspaces(t *testing.T) { + utils.SkipWithoutDB(t) + core.SetupTestEnvironment() + + invID := uuid.MustParse("99c0ffee-0000-0000-0000-0000000000aa") + createTestSystemInDB(t, invID, 1, "workspace-move") + t.Cleanup(func() { deleteTestSystemInDB(t, invID) }) + + var inv models.SystemInventory + require.NoError(t, database.TestDB().Where("inventory_id = ?", invID).First(&inv).Error) + oldWS := uuid.MustParse("00000000-0000-0000-0000-000000000001") + newWS := uuid.MustParse("00000000-0000-0000-0000-000000000002") + require.NoError(t, database.TestDB().Model(&inv).Updates(map[string]any{ + "workspace_id": oldWS, + "workspace_name": "old", + }).Error) + require.NoError(t, database.TestDB().Create(&[]models.SystemAdvisories{ + {RhAccountID: 1, SystemID: inv.ID, AdvisoryID: 2, StatusID: 1}, + {RhAccountID: 1, SystemID: inv.ID, AdvisoryID: 1, StatusID: 0}, + }).Error) + + host := createTestInvHost(t) + host.ID = invID + host.Groups = []inventory.Group{{ID: newWS.String(), Name: "new"}} + req := vmaas.UpdatesV3Request{} + + _, events, err := updateSystemPlatform(database.TestDB(), 1, host, nil, &req, nil) + require.NoError(t, err) + require.Len(t, events, 2) + assert.Equal(t, 1, events[0].RhAccountID) + assert.Equal(t, oldWS, events[0].WorkspaceID) + assert.Equal(t, []int64{1, 2}, events[0].AdvisoryIDs) + assert.Equal(t, newWS, events[1].WorkspaceID) + assert.Equal(t, []int64{1, 2}, events[1].AdvisoryIDs) + assert.False(t, events[0].ProducedAt.Time().IsZero()) + + _, events, err = updateSystemPlatform(database.TestDB(), 1, host, nil, &req, nil) + require.NoError(t, err) + assert.Empty(t, events) +} + +func TestPublishWorkspaceAdvisoryUpdates(t *testing.T) { + writer := &mqueue.MockKafkaWriter{} + advisoryUpdateWriter = writer + t.Cleanup(func() { advisoryUpdateWriter = nil }) + + oldWS := uuid.MustParse("00000000-0000-0000-0000-000000000001") + newWS := uuid.MustParse("00000000-0000-0000-0000-000000000002") + events := mqueue.AdvisoryUpdateEvents{ + {RhAccountID: 1, WorkspaceID: oldWS, AdvisoryIDs: []int64{1, 2}}, + {RhAccountID: 1, WorkspaceID: newWS, AdvisoryIDs: []int64{1, 2}}, + } + require.NoError(t, publishWorkspaceAdvisoryUpdates(events)) + require.Len(t, writer.Messages, 2) + assert.Equal(t, []byte("1"), writer.Messages[0].Key) + assert.Equal(t, []byte("1"), writer.Messages[1].Key) + + var first, second mqueue.AdvisoryUpdateEvent + require.NoError(t, sonic.Unmarshal(writer.Messages[0].Value, &first)) + require.NoError(t, sonic.Unmarshal(writer.Messages[1].Value, &second)) + assert.Equal(t, oldWS, first.WorkspaceID) + assert.Equal(t, newWS, second.WorkspaceID) + assert.Equal(t, []int64{1, 2}, first.AdvisoryIDs) + assert.Equal(t, []int64{1, 2}, second.AdvisoryIDs) +}