Add measured PostgreSQL/Hasura and Doublets GraphQL comparison

.github/workflows/graphql-benchmark.yml

@@ -0,0 +1,45 @@
+name: GraphQL comparison
+on:
+  pull_request:
+    paths: ['graphql/**', '.github/workflows/graphql-benchmark.yml']
+  workflow_dispatch:
+    inputs:
+      background:
+        description: Background point links (1..3000)
+        default: '3000'
+        required: true
+      iterations:
+        description: Sequential iterations per server (1..1000)
+        default: '1000'
+        required: true
+permissions:
+  contents: read
+jobs:
+  compare:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      BACKGROUND: ${{ inputs.background || '30' }}
+      ITERATIONS: ${{ inputs.iterations || '3' }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/checkout@v4
+        with:
+          repository: linksplatform/Data.Doublets.Gql
+          ref: b11f33b4080a7ef6b6d1c056c40bbf758d6cdd7e
+          path: graphql/source
+          submodules: true
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Test result validation
+        run: python3 -m unittest discover -s graphql -v
+      - name: Run real local GraphQL servers and k6
+        run: ./graphql/run.sh
+      - name: Upload measured results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: graphql-comparison
+          path: graphql/results/

.gitignore

@@ -370,3 +370,7 @@ MigrationBackup/
 
 # Ionide (cross platform F# VS Code tools) working folder
 .ionide/
+
+# Disposable GraphQL benchmark source and measured outputs
+graphql/source/
+graphql/results/

README.md

@@ -19,6 +19,12 @@ Both databases used to store and retrieve doublet-links representation. To suppo
 - **Each Concrete** – take all links matching `[*, source, target]` constraint
 - **Each Identity** – take all links matching `[id, *, *]` constraint
 
+## GraphQL comparison
+
+[Run the correctness-checked PostgreSQL/Hasura and Doublets GraphQL comparison](graphql/README.md).
+The harness uses real local servers, records actual k6 measurements, and keeps
+its results separate from the Rust results below.
+
 ## Results
 The results below represent the amount of time (ns) the operation takes per iteration.
 - First picture shows time in a pixel scale (for doublets just minimum value is shown, otherwise it will be not present on the graph).

graphql/.dockerignore

@@ -0,0 +1,4 @@
+results
+**/.git
+**/bin
+**/obj

graphql/Dockerfile.doublets

@@ -0,0 +1,10 @@
+FROM mcr.microsoft.com/dotnet/sdk:8.0@sha256:78235e09001f52b6592c458ac010775ebac6725422e80cd0c1650590f67b2743 AS build
+WORKDIR /src
+COPY source/ .
+RUN dotnet publish csharp/Platform.Data.Doublets.Gql.Server/Platform.Data.Doublets.Gql.Server.csproj \
+    --configuration Release --framework net6 --no-self-contained --output /app \
+    --source https://api.nuget.org/v3/index.json
+FROM mcr.microsoft.com/dotnet/aspnet:6.0@sha256:e70c493f8af7f95bf459cb2b15c7e7a6173228929c2b7a9a6836b19377890e78
+WORKDIR /app
+COPY --from=build /app/ .
+ENTRYPOINT ["dotnet", "Platform.Data.Doublets.Gql.Server.dll", "/data/benchmark.links"]

graphql/README.md

@@ -0,0 +1,126 @@
+# PostgreSQL + Hasura versus Doublets + GraphQL
+
+This comparison uses the real PostgreSQL/Hasura and LinksPlatform Doublets GraphQL
+servers. It runs a correctness-checked sequence through k6, with 3,000 background
+point links and 1,000 sequential iterations per server by default. It does not
+call a remote benchmark endpoint or access an existing database.
+
+## Run locally
+
+Prerequisites: Git, Python 3.12 or newer, Docker and Docker Compose. No hosted k6
+account, API keys or broker account is used. Images and NuGet packages are
+fetched from their official registries on the first build.
+
+```sh
+# Separate, public source checkout; this does not modify this benchmark repository.
+git clone --recurse-submodules https://github.com/linksplatform/Data.Doublets.Gql.git /tmp/doublets-gql
+python3 graphql/prepare-source.py /tmp/doublets-gql
+
+# Fast correctness smoke test, with real servers and real k6:
+BACKGROUND=30 ITERATIONS=3 ./graphql/run.sh
+
+# Full configured workload:
+./graphql/run.sh
+```
+
+`prepare-source.py` exports unmodified Doublets commit
+`b11f33b4080a7ef6b6d1c056c40bbf758d6cdd7e` and its Settings submodule commit
+`f08ce8fc84f8ab7ccdcf9293b5566007305254c1`. It performs no network calls. If your
+existing checkout lacks either Git object, fetch the named public revision into
+that checkout first. An already prepared `graphql/source` directory is not
+overwritten. Docker publishes the upstream .NET 6 server target explicitly;
+this preserves the pinned server rather than silently changing its runtime.
+
+Each invocation creates a unique Compose project, private Docker network and
+fresh volumes. No ports are exposed to the host. The `EXIT` trap removes only
+that invocation's containers, network and database volumes. A force-killed shell
+cannot run a cleanup trap; use the printed `doublets-gql-bench-<pid>` project name
+to clean that run with `docker compose -p <project> -f graphql/compose.yaml down -v`.
+
+Measured results are retained separately for each run under
+`graphql/results/doublets-gql-bench-<pid>/`:
+
+- `hasura.json` and `doublets.json`: actual k6 summary metrics and sample counts.
+- `comparison.md`: mean and p95 HTTP durations, in milliseconds.
+- `run.json`: source, script, runtime and workload metadata.
+
+A failed run exits nonzero. An incomplete run is never converted into a comparison
+table. No chart or result in the repository is claimed to be a measurement from
+this GraphQL harness until the actual run has completed.
+
+## Workload and correctness
+
+The initial dataset contains exactly 3,000 point links: `id = from_id = to_id`.
+The default 1,000 iterations each perform the following sequence using one k6
+virtual user; the two servers are measured sequentially:
+
+| Operation | Behavior / verified result |
+|---|---|
+| Create | Insert an empty link, then update both endpoints to its returned ID; verify a point link. Two HTTP mutations on **both** systems. |
+| Update | Set the new link to `(id, 1)` and check the returned row. |
+| Each All | Fetch every row; verify 3,001 unique IDs, exact background values and the active row. |
+| Each Identity | Filter by the active ID; verify the exact row. |
+| Each Concrete | Filter by source and target; verify the exact row. |
+| Each Outgoing | Filter by source; verify the exact row. |
+| Each Incoming | Filter by target 1; verify point1 and the active row. |
+| Delete | Delete the active ID, check the returned row, and separately verify that it is absent. |
+
+The default operation timings therefore contain 1,000 samples each, with 3,000
+background links and one active link during reads. This is a single-client CRUD
+comparison, not a throughput or saturation test. `BACKGROUND` may be 1..3000 and
+`ITERATIONS` 1..1000. The request timeout is 15s, setup is bounded to 5 min, and each
+server's iteration phase is bounded to 10 min.
+
+All measured requests use the servers' actual common fields:
+`insert_links`, `update_links`, `delete_links`, and `links(where: ...)`, with
+`id`, `from_id`, and `to_id`. PostgreSQL has a primary key, individual source and
+target indexes, and a unique source/target pair to match the native link model.
+Both endpoints receive equivalent data and the same operations. Inserted IDs
+are read from responses rather than assumed for the active workload.
+
+Every response is checked for HTTP status, GraphQL errors, expected cardinality,
+unique IDs and expected field values. The run aborts at the first mismatch and
+prints the operation, request and response for request failures. Setup refuses
+an already populated database. No error-swallowing continuation or fake response
+adapter is used. Report unit tests use clearly synthetic fixtures; these are never
+written to the measured-results directory.
+
+## Relation to graphql-bench #52
+
+[The prerequisite issue](https://github.com/hasura/graphql-bench/issues/52) reports
+HTTP 400/500 responses and missing request/response diagnostics. Its maintainer
+suggested JSON Content-Type headers and debug output. The comparison's sponsor
+[also suggested using k6 directly](https://github.com/linksplatform/Comparisons.PostgreSQLVSDoublets/issues/1#issuecomment-910735999).
+
+This implementation takes that direct k6 path: it explicitly posts JSON with
+`Content-Type: application/json`, uses the actual endpoint/schema, and reports
+failed requests and response bodies. Successful real-server runs verify that
+these requests work. It does not claim to modify, reproduce every old executor's
+bug in, or close the separate graphql-bench issue.
+
+## Reproducibility and interpretation
+
+The pinned Doublets source is compiled without source changes; runtime image
+versions/digests and source revision are recorded by the harness. The baseline
+.NET 6 target is inherited from that source. These isolated containers are for
+local benchmark work, not a deployment configuration.
+
+Numbers measure HTTP response time (two durations summed for Create), not pure
+storage-engine latency. Validation, seeding, deletion verification and startup
+are outside the operation metrics. Container scheduling, caching, host load and
+run order can affect results. Repeat runs and compare like-for-like workloads
+before drawing performance conclusions. Neither one run nor 1,000 sequential
+iterations establishes broad performance superiority.
+
+The existing Rust benchmark results elsewhere in the repository are separate
+and are not replaced by this comparison.
+
+## Automated verification
+
+```sh
+python3 -m unittest discover -s graphql -v
+```
+
+The GraphQL workflow runs a real Docker/k6 smoke test on changes and permits the
+full bounded workload through manual dispatch. It uploads run artifacts and does
+not publish to `gh-pages` or commit generated results.

graphql/benchmark.js

@@ -0,0 +1,150 @@
+import http from 'k6/http';
+import { sleep } from 'k6';
+import { Trend } from 'k6/metrics';
+import exec from 'k6/execution';
+
+function boundedInteger(value, fallback, maximum) {
+  const parsed = Number(value || fallback);
+  if (!Number.isInteger(parsed) || parsed < 1 || parsed > maximum) {
+    throw new Error(`Expected integer 1..${maximum}, got ${value}`);
+  }
+  return parsed;
+}
+
+const background = boundedInteger(__ENV.BACKGROUND, 3000, 3000);
+const iterations = boundedInteger(__ENV.ITERATIONS, 1000, 1000);
+const target = __ENV.TARGET;
+if (target !== 'hasura' && target !== 'doublets') throw new Error('TARGET must be hasura or doublets');
+const endpoint = `http://${target}:8080/v1/graphql`;
+const headers = { 'Content-Type': 'application/json' };
+const operationNames = ['Create', 'Update', 'EachAll', 'EachIdentity', 'EachConcrete', 'EachOutgoing', 'EachIncoming', 'Delete'];
+const timings = Object.fromEntries(operationNames.map(name => [name, new Trend(`operation_${name}_ms`, true)]));
+
+export const options = {
+  vus: 1,
+  iterations,
+  maxDuration: '10m',
+  setupTimeout: '5m',
+  thresholds: { http_req_failed: ['rate==0'] },
+  summaryTrendStats: ['count', 'avg', 'min', 'med', 'max', 'p(95)'],
+};
+
+function ensure(condition, message) {
+  if (!condition) exec.test.abort(message);
+}
+
+function query(document, operation = 'setup') {
+  const response = http.post(endpoint, JSON.stringify({ query: document }),
+    { headers, timeout: '15s', tags: { name: operation } });
+  let body;
+  try { body = response.json(); } catch (_) {
+    exec.test.abort(`${operation}: HTTP ${response.status}; request=${document}; response=${response.body}`);
+  }
+  ensure(response.status === 200 && body && body.data && !body.errors,
+    `${operation}: HTTP ${response.status}; request=${document}; response=${response.body}`);
+  return { data: body.data, duration: response.timings.duration };
+}
+
+function rows(result, name = 'links') {
+  const value = result.data[name];
+  ensure(Array.isArray(value), `Expected ${name} array: ${JSON.stringify(result.data)}`);
+  return value.map(row => ({ id: Number(row.id), from_id: Number(row.from_id), to_id: Number(row.to_id) }));
+}
+
+function checkRow(row, id, from, to) {
+  ensure(row && Number(row.id) === id && Number(row.from_id) === from && Number(row.to_id) === to,
+    `Wrong row; expected (${id},${from},${to}), got ${JSON.stringify(row)}`);
+}
+
+function mutate(document, name, operation) {
+  const result = query(document, operation);
+  const mutation = result.data[name];
+  ensure(mutation && mutation.affected_rows === 1 && mutation.returning.length === 1,
+    `${operation}: expected one affected/returned row: ${JSON.stringify(result.data)}`);
+  return { row: mutation.returning[0], duration: result.duration };
+}
+
+export function setup() {
+  // Wait on these disposable internal services; no external endpoint can be selected.
+  let ready = false;
+  for (let attempt = 0; attempt < 90; attempt++) {
+    const response = http.post(endpoint, JSON.stringify({ query: '{__typename}' }),
+      { headers, timeout: '2s', tags: { name: 'readiness' }, responseCallback: null });
+    if (response.status === 200) { ready = true; break; }
+    sleep(1);
+  }
+  ensure(ready, `${target} did not become ready`);
+  if (target === 'hasura') {
+    const response = http.post('http://hasura:8080/v1/metadata', JSON.stringify({
+      type: 'pg_track_table', args: { source: 'default', table: { schema: 'public', name: 'links' } },
+    }), { headers });
+    ensure(response.status === 200, `Cannot track local links table: ${response.body}`);
+  }
+  ensure(rows(query('{links(limit:1){id from_id to_id}}')).length === 0,
+    'Benchmark requires an empty dedicated database; refusing to modify an existing dataset');
+  for (let start = 1; start <= background; start += 100) {
+    const objects = [];
+    for (let id = start; id <= Math.min(start + 99, background); id++) objects.push(`{from_id:${id},to_id:${id}}`);
+    const result = query(`mutation{insert_links(objects:[${objects.join(',')}]){affected_rows returning{id from_id to_id}}}`);
+    const inserted = result.data.insert_links;
+    ensure(inserted.affected_rows === objects.length && inserted.returning.length === objects.length, 'Background row count mismatch');
+    inserted.returning.forEach((row, index) => checkRow(row, start + index, start + index, start + index));
+  }
+  const seeded = rows(query('{links{id from_id to_id}}'));
+  ensure(seeded.length === background, 'Seeded database cardinality mismatch');
+  seeded.sort((a, b) => a.id - b.id).forEach((row, index) => checkRow(row, index + 1, index + 1, index + 1));
+  return { seeded: background };
+}
+
+export default function () {
+  // Create a point through the same two HTTP mutations on both servers.
+  const created = mutate('mutation{insert_links(objects:[{from_id:0,to_id:0}]){affected_rows returning{id from_id to_id}}}',
+    'insert_links', 'Create');
+  const id = Number(created.row.id);
+  ensure(Number.isSafeInteger(id) && id > background, 'New link ID must be outside the background dataset');
+  const pointed = mutate(`mutation{update_links(where:{id:{_eq:${id}}},_set:{from_id:${id},to_id:${id}}){affected_rows returning{id from_id to_id}}}`,
+    'update_links', 'Create');
+  checkRow(pointed.row, id, id, id);
+  timings.Create.add(created.duration + pointed.duration);
+
+  const updated = mutate(`mutation{update_links(where:{id:{_eq:${id}}},_set:{from_id:${id},to_id:1}){affected_rows returning{id from_id to_id}}}`,
+    'update_links', 'Update');
+  checkRow(updated.row, id, id, 1);
+  timings.Update.add(updated.duration);
+
+  const reads = [
+    ['EachAll', '{links{id from_id to_id}}', background + 1],
+    ['EachIdentity', `{links(where:{id:{_eq:${id}}}){id from_id to_id}}`, 1],
+    ['EachConcrete', `{links(where:{from_id:{_eq:${id}},to_id:{_eq:1}}){id from_id to_id}}`, 1],
+    ['EachOutgoing', `{links(where:{from_id:{_eq:${id}}}){id from_id to_id}}`, 1],
+    ['EachIncoming', '{links(where:{to_id:{_eq:1}}){id from_id to_id}}', 2],
+  ];
+  for (const [name, document, count] of reads) {
+    const result = query(document, name);
+    const actual = rows(result);
+    ensure(new Set(actual.map(row => row.id)).size === actual.length, `${name}: duplicate row IDs`);
+    ensure(actual.length === count, `${name}: expected ${count} rows, got ${actual.length}`);
+    actual.forEach(row => {
+      if (row.id === id) checkRow(row, id, id, 1);
+      else { ensure((name === 'EachAll' && row.id >= 1 && row.id <= background) || (name === 'EachIncoming' && row.id === 1), `${name}: unexpected row`); checkRow(row, row.id, row.id, row.id); }
+    });
+    ensure(actual.some(row => row.id === id), `${name}: current link is missing`);
+    timings[name].add(result.duration);
+  }
+
+  const removed = mutate(`mutation{delete_links(where:{id:{_eq:${id}}}){affected_rows returning{id from_id to_id}}}`,
+    'delete_links', 'Delete');
+  checkRow(removed.row, id, id, 1);
+  ensure(rows(query(`{links(where:{id:{_eq:${id}}}){id from_id to_id}}`, 'verify-delete')).length === 0,
+    'Deleted link still exists');
+  timings.Delete.add(removed.duration);
+}
+
+export function handleSummary(data) {
+  const measured = Object.fromEntries(operationNames.map(name => [name, data.metrics[`operation_${name}_ms`]?.values]));
+  const complete = operationNames.every(name => measured[name] && measured[name].count === iterations);
+  const report = { target, background, iterations, complete, unit: 'milliseconds',
+    create_http_requests: 2, other_operation_http_requests: 1, operations: measured };
+  return { [`/results/${target}.json`]: JSON.stringify(report, null, 2),
+    stdout: JSON.stringify(report, null, 2) + '\n' };
+}