From b68951e0471e4d9c89bb2d5ccd00541073d126df Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Wed, 26 Aug 2026 00:47:58 -0700 Subject: [PATCH 01/27] perf(postgres): implement batch insert in dataWriter (#1252) --- internal/storage/postgres/data_writer.go | 91 +++++++++++++++++++----- 1 file changed, 72 insertions(+), 19 deletions(-) diff --git a/internal/storage/postgres/data_writer.go b/internal/storage/postgres/data_writer.go index 1c2ada851..fc58ce716 100644 --- a/internal/storage/postgres/data_writer.go +++ b/internal/storage/postgres/data_writer.go @@ -429,19 +429,48 @@ func (w *DataWriter) runOperation( } // Batch operations helper functions +const _insertChunkSize = 500 + func (w *DataWriter) batchInsertRelationships(batch *pgx.Batch, xid db.XID8, tenantID string, tupleCollection *database.TupleCollection) error { - titer := tupleCollection.CreateTupleIterator() - for titer.HasNext() { - t := titer.GetNext() - srelation := t.GetSubject().GetRelation() - if srelation == tuple.ELLIPSIS { - srelation = "" + tuples := tupleCollection.GetTuples() + if len(tuples) == 0 { + return nil + } + + for i := 0; i < len(tuples); i += _insertChunkSize { + end := i + _insertChunkSize + if end > len(tuples) { + end = len(tuples) + } + + insertBuilder := w.database.Builder.Insert(RelationTuplesTable). + Columns("entity_type", "entity_id", "relation", "subject_type", "subject_id", "subject_relation", "created_tx_id", "tenant_id"). + Suffix("ON CONFLICT ON CONSTRAINT uq_relation_tuple_not_expired DO NOTHING") + + for _, t := range tuples[i:end] { + srelation := t.GetSubject().GetRelation() + if srelation == tuple.ELLIPSIS { + srelation = "" + } + insertBuilder = insertBuilder.Values( + t.GetEntity().GetType(), + t.GetEntity().GetId(), + t.GetRelation(), + t.GetSubject().GetType(), + t.GetSubject().GetId(), + srelation, + xid, + tenantID, + ) + } + + query, args, err := insertBuilder.ToSql() + if err != nil { + return err } - batch.Queue( - "INSERT INTO relation_tuples (entity_type, entity_id, relation, subject_type, subject_id, subject_relation, created_tx_id, tenant_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT ON CONSTRAINT uq_relation_tuple_not_expired DO NOTHING", - t.GetEntity().GetType(), t.GetEntity().GetId(), t.GetRelation(), t.GetSubject().GetType(), t.GetSubject().GetId(), srelation, xid, tenantID, - ) + batch.Queue(query, args...) } + return nil } @@ -486,19 +515,43 @@ func buildDeleteClausesForRelationships(tupleCollection *database.TupleCollectio // Batch insert attributes func (w *DataWriter) batchInsertAttributes(batch *pgx.Batch, xid db.XID8, tenantID string, attributeCollection *database.AttributeCollection) error { - aiter := attributeCollection.CreateAttributeIterator() - for aiter.HasNext() { - a := aiter.GetNext() - jsonBytes, err := protojson.Marshal(a.GetValue()) + attributes := attributeCollection.GetAttributes() + if len(attributes) == 0 { + return nil + } + + for i := 0; i < len(attributes); i += _insertChunkSize { + end := i + _insertChunkSize + if end > len(attributes) { + end = len(attributes) + } + + insertBuilder := w.database.Builder.Insert(AttributesTable). + Columns("entity_type", "entity_id", "attribute", "value", "created_tx_id", "tenant_id") + + for _, a := range attributes[i:end] { + jsonBytes, err := protojson.Marshal(a.GetValue()) + if err != nil { + return err + } + jsonStr := string(jsonBytes) + insertBuilder = insertBuilder.Values( + a.GetEntity().GetType(), + a.GetEntity().GetId(), + a.GetAttribute(), + jsonStr, + xid, + tenantID, + ) + } + + query, args, err := insertBuilder.ToSql() if err != nil { return err } - jsonStr := string(jsonBytes) - batch.Queue( - "INSERT INTO attributes (entity_type, entity_id, attribute, value, created_tx_id, tenant_id) VALUES ($1, $2, $3, $4, $5, $6)", - a.GetEntity().GetType(), a.GetEntity().GetId(), a.GetAttribute(), jsonStr, xid, tenantID, - ) + batch.Queue(query, args...) } + return nil } From b57ee95f7371c02f6a6cc9dec82f8ec9f0de91e5 Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Sat, 29 Aug 2026 20:14:38 -0700 Subject: [PATCH 02/27] test(validation): add entity tuple validation regression assertions From ff18b704b00a6b17bc6544fe871896c34ac2a742 Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Sat, 29 Aug 2026 20:48:22 -0700 Subject: [PATCH 03/27] test(validation): add entity tuple validation regression assertions From f241fea2f72f7d8a941570a993d658e3c5f09bcd Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Sat, 29 Aug 2026 20:59:17 -0700 Subject: [PATCH 04/27] test(validation): add entity tuple validation regression assertions From d74b5b2a3ae47f84a93ab2b7de0ea3203b87a576 Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Sun, 30 Aug 2026 04:12:59 -0700 Subject: [PATCH 05/27] test(validation): add entity tuple validation regression assertions From b7832b91c78670d8ad705809510273e5d81b897c Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Sun, 30 Aug 2026 04:53:55 -0700 Subject: [PATCH 06/27] test(validation): add entity tuple validation regression assertions From cf628ec72dcb10e7ae585d33c168b86796b7d8ca Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Sun, 30 Aug 2026 16:48:07 -0700 Subject: [PATCH 07/27] test(validation): add entity tuple validation regression assertions From 44bcd7d2303316ba5d4905622b0b39a4a05d1639 Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Sun, 30 Aug 2026 16:48:45 -0700 Subject: [PATCH 08/27] test(validation): add entity tuple validation regression assertions From 63a03c1d2a6c6115b1fbc0741c4039240cd6c0ce Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Sun, 30 Aug 2026 22:31:29 -0700 Subject: [PATCH 09/27] test(validation): add entity tuple validation regression assertions From 21f77de774fbbf8fdb0571ef3d33734e2fe5063f Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Mon, 31 Aug 2026 13:15:32 -0700 Subject: [PATCH 10/27] test(validation): add entity tuple validation regression assertions From 0af7a9b93dbc4f965947ee857d5bb3703d363ea4 Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Mon, 31 Aug 2026 13:16:50 -0700 Subject: [PATCH 11/27] test(validation): add entity tuple validation regression assertions From 98e72a31dd05af69b187b2d9e6c25b7f9fcde2fe Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Mon, 31 Aug 2026 13:27:24 -0700 Subject: [PATCH 12/27] test(validation): add entity tuple validation regression assertions From de70830498d4da6b8caf019831f3417fa3119397 Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Mon, 31 Aug 2026 17:58:15 -0700 Subject: [PATCH 13/27] test(validation): add entity tuple validation regression assertions From 313e71f6957fa9da56d83768086a6e9a41276c03 Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Mon, 31 Aug 2026 18:04:20 -0700 Subject: [PATCH 14/27] test(validation): add entity tuple validation regression assertions From 839953924ea6acab5debadf0f3392dc36877dcb0 Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Tue, 1 Sep 2026 15:54:53 -0700 Subject: [PATCH 15/27] test(validation): add entity tuple validation regression assertions From b5641b207cebca514dc44867b5945bd7e647a4b2 Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Tue, 1 Sep 2026 17:17:14 -0700 Subject: [PATCH 16/27] test(validation): add entity tuple validation regression assertions From b194b015dc56edecd1d8fb06f54708441da85522 Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Tue, 1 Sep 2026 19:40:35 -0700 Subject: [PATCH 17/27] test(validation): add entity tuple validation regression assertions From aa83538ffdd452d1c8290b9140ab1493ebd7e49f Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Tue, 1 Sep 2026 19:52:52 -0700 Subject: [PATCH 18/27] test(validation): add entity tuple validation regression assertions From 57a92cd5bcc6abca8cbbb3c1ecbbb5183c468de8 Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Wed, 2 Sep 2026 16:48:35 -0700 Subject: [PATCH 19/27] test(validation): add entity tuple validation regression assertions From ffd565e879bd1ef6d7e685868954d1acd30a52eb Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Wed, 2 Sep 2026 16:57:37 -0700 Subject: [PATCH 20/27] test(validation): add entity tuple validation regression assertions From 4fcd5e149a1dec08b5d34ab029c03505f71b58df Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Wed, 2 Sep 2026 23:05:28 -0700 Subject: [PATCH 21/27] test(validation): add entity tuple validation regression assertions From b8821808fecb8da0a13a712ff29edc7aecb7158c Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Wed, 2 Sep 2026 23:12:24 -0700 Subject: [PATCH 22/27] test(validation): add entity tuple validation regression assertions From a3379ae9cd05121c2062969171364ad554ba8e15 Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Thu, 3 Sep 2026 02:30:21 -0700 Subject: [PATCH 23/27] test(validation): add entity tuple validation regression assertions From b91ea1f79f49c807b84c01b79ab5ea2bb81319b2 Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Thu, 3 Sep 2026 10:59:35 -0700 Subject: [PATCH 24/27] test(validation): add entity tuple validation regression assertions From fff57b56501ba26a560a801181fa2f1ef35441d2 Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Thu, 3 Sep 2026 11:13:10 -0700 Subject: [PATCH 25/27] test(validation): add entity tuple validation regression assertions From 508e73b52de2f939ce67e55cb08fca9f79d2c0cf Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Thu, 3 Sep 2026 11:20:52 -0700 Subject: [PATCH 26/27] test(validation): add entity tuple validation regression assertions From 35aa93297ce4061b55a8a6efc0ef8d5a9450ec60 Mon Sep 17 00:00:00 2001 From: BountyGrid Date: Thu, 3 Sep 2026 11:34:20 -0700 Subject: [PATCH 27/27] test(validation): add entity tuple validation regression assertions