diff --git a/Makefile b/Makefile index 9a0a0cb65a..a42d702e92 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ TAG?=test # TAG?=v0.0.1 binary: - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o $(BINARY_NAME) ./cmd/main.go + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o $(BINARY_NAME) ./cmd .PHONY: check-legacy-packages check-legacy-packages: diff --git a/build/Dockerfile b/build/Dockerfile index 757bbba378..ce2187a596 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -7,7 +7,7 @@ ARG TARGETOS TARGETARCH RUN --mount=target=. \ --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg \ - GOOS=$TARGETOS GOARCH=$TARGETARCH GOEXPERIMENT=greenteagc go build -o /out/node-agent -ldflags="-s -w" ./cmd/main.go + GOOS=$TARGETOS GOARCH=$TARGETARCH GOEXPERIMENT=greenteagc go build -o /out/node-agent -ldflags="-s -w" ./cmd RUN --mount=target=. \ --mount=type=cache,target=/root/.cache/go-build \ diff --git a/build/Dockerfile.debug b/build/Dockerfile.debug index b21938e5af..576648e3bb 100644 --- a/build/Dockerfile.debug +++ b/build/Dockerfile.debug @@ -8,7 +8,7 @@ RUN go install github.com/go-delve/delve/cmd/dlv@latest RUN --mount=target=. \ --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg \ - GOOS=$TARGETOS GOARCH=$TARGETARCH GOEXPERIMENT=greenteagc go build -o /out/node-agent ./cmd/main.go + GOOS=$TARGETOS GOARCH=$TARGETARCH GOEXPERIMENT=greenteagc go build -o /out/node-agent ./cmd FROM gcr.io/distroless/static-debian13:debug diff --git a/cmd/gvisor_start_linux.go b/cmd/gvisor_start_linux.go new file mode 100644 index 0000000000..36c0196f3c --- /dev/null +++ b/cmd/gvisor_start_linux.go @@ -0,0 +1,96 @@ +//go:build linux + +package main + +import ( + "context" + "errors" + "time" + + "github.com/kubescape/go-logger" + "github.com/kubescape/node-agent/pkg/config" + "github.com/kubescape/node-agent/pkg/gvisor" + gvcri "github.com/kubescape/node-agent/pkg/gvisor/cri" + "github.com/kubescape/node-agent/pkg/otelsetup" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + coreclient "k8s.io/client-go/kubernetes/typed/core/v1" +) + +// startGVisorCollection is independent of the host PID/mount-namespace inventory. +// In particular it does not inject a sandbox start into an exec/profile path. +func startGVisorCollection(ctx context.Context, cfg config.Config, runtimeEndpoint string, core coreclient.CoreV1Interface, logsReady bool) (func(), error) { + if !cfg.GVisorStart.Enabled { + return func() {}, nil + } + if err := cfg.GVisorStart.Validate(); err != nil { + return nil, err + } + if !logsReady { + return nil, errors.New("gVisor observations require an initialized OTLP log exporter") + } + if cfg.GVisorStart.RuntimeEndpoint != "" { + runtimeEndpoint = cfg.GVisorStart.RuntimeEndpoint + } + inventory, err := gvcri.New(runtimeEndpoint, cfg.GVisorStart.RuntimeHandler) + if err != nil { + return nil, err + } + checkCtx, cancelCheck := context.WithTimeout(ctx, 2*time.Second) + err = inventory.Check(checkCtx) + cancelCheck() + if err != nil { + inventory.Close() + return nil, err + } + sourceCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + lookup := func(ctx context.Context, id string) (gvisor.Identity, error) { + identity, err := inventory.Resolve(ctx, id) + if err != nil { + return gvisor.Identity{}, err + } + pod, err := core.Pods(identity.Namespace).Get(ctx, identity.PodName, metav1.GetOptions{}) + if err != nil || !gVisorPodMatches(identity, pod, cfg) { + return gvisor.Identity{}, errors.New("gVisor Pod identity is unresolved or excluded") + } + return identity, nil + } + go func() { + defer close(done) + if err := gvisor.RunIntegration(sourceCtx, cfg.GVisorStart, cfg.NodeName, lookup, otelsetup.EmitRuntimeStart); err != nil { + // Never log CRI bodies or Sentry bytes. A source failure is degraded + // telemetry, not evidence that no sandbox started. + logger.L().Warning("gVisor start source stopped; runtime visibility is degraded") + } + }() + return func() { + cancel() + <-done + _ = inventory.Close() + }, nil +} + +func gVisorPodMatches(identity gvisor.Identity, pod *corev1.Pod, cfg config.Config) bool { + if pod == nil || pod.Namespace != identity.Namespace || pod.Name != identity.PodName || + string(pod.UID) != identity.PodUID || pod.Spec.NodeName != cfg.NodeName || pod.DeletionTimestamp != nil || + cfg.IgnoreContainer(pod.Namespace, pod.Name, pod.Labels) { + return false + } + for _, c := range pod.Spec.Containers { + if c.Name == identity.ContainerName { + return true + } + } + for _, c := range pod.Spec.InitContainers { + if c.Name == identity.ContainerName { + return true + } + } + for _, c := range pod.Spec.EphemeralContainers { + if c.Name == identity.ContainerName { + return true + } + } + return false +} diff --git a/cmd/gvisor_start_linux_test.go b/cmd/gvisor_start_linux_test.go new file mode 100644 index 0000000000..d69a35d0a5 --- /dev/null +++ b/cmd/gvisor_start_linux_test.go @@ -0,0 +1,55 @@ +//go:build linux + +package main + +import ( + "context" + "testing" + + "github.com/kubescape/node-agent/pkg/config" + "github.com/kubescape/node-agent/pkg/gvisor" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestGVisorPodIdentityAndNamespacePolicy(t *testing.T) { + identity := gvisor.Identity{Namespace: "apps", PodName: "sandbox", PodUID: "uid", ContainerName: "app"} + for _, tc := range []string{"verified", "wrong_uid", "wrong_node", "wrong_namespace", "missing_container", "excluded_namespace", "excluded_label"} { + t.Run(tc, func(t *testing.T) { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "apps", Name: "sandbox", UID: "uid", Labels: map[string]string{"ignore": "yes"}}, Spec: corev1.PodSpec{NodeName: "node", Containers: []corev1.Container{{Name: "app"}}}} + cfg := config.Config{NodeName: "node"} + switch tc { + case "wrong_uid": + pod.UID = "new-uid" + case "wrong_node": + pod.Spec.NodeName = "other-node" + case "wrong_namespace": + pod.Namespace = "other" + case "missing_container": + pod.Spec.Containers = nil + case "excluded_namespace": + cfg.ExcludeNamespaces = []string{"apps"} + case "excluded_label": + cfg.ExcludeLabels = map[string][]string{"ignore": {"yes"}} + } + if got := gVisorPodMatches(identity, pod, cfg); got != (tc == "verified") { + t.Fatalf("identity/policy accepted=%v", got) + } + }) + } +} + +func TestGVisorDisabledStartupDoesNoIO(t *testing.T) { + stop, err := startGVisorCollection(context.Background(), config.Config{}, "", nil, false) + if err != nil { + t.Fatal(err) + } + stop() +} + +func TestGVisorRequiresLogExport(t *testing.T) { + cfg := config.Config{GVisorStart: gvisor.IntegrationConfig{Enabled: true, SocketPath: "/run/kubescape/starts.sock"}} + if _, err := startGVisorCollection(context.Background(), cfg, "", nil, false); err == nil { + t.Fatal("enabled source silently used a no-op exporter") + } +} diff --git a/cmd/main.go b/cmd/main.go index 670565a3e0..48f0d8ca22 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -142,6 +142,7 @@ func main() { if err != nil { logger.L().Warning("OTEL init failed, running without telemetry", helpers.Error(err)) } + otelLogsReady := err == nil && (os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") != "" || os.Getenv("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT") != "") defer func() { shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -570,6 +571,13 @@ func main() { } // start watching + stopGVisor, gvisorErr := startGVisorCollection(ctx, cfg, igK8sClient.RuntimeConfig.SocketPath, k8sClient.GetKubernetesClient().CoreV1(), otelLogsReady) + if gvisorErr != nil { + logger.L().Warning("gVisor start source unavailable; runtime visibility is degraded", helpers.Error(gvisorErr)) + } else { + defer stopGVisor() + } + dWatcher.Start(ctx) defer dWatcher.Stop(ctx) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index d40819627b..b32b9bc739 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -734,3 +734,10 @@ For more information, see: - [Main README](../README.md) - [Alert Bulking](ALERT_BULKING.md) - [Kubescape Documentation](https://kubescape.io/docs/) + +## Experimental gVisor start source + +`gVisorStart.enabled` is disabled by default. See the [source configuration and +identity/export contract](features/gvisor-start-source.md) before enabling it on +a self-managed Kubernetes test node. It requires a local CRI socket, a configured +runsc trace sink and an initialized OTLP log exporter. diff --git a/docs/experiments/gvisor-start-integration-20261102/README.md b/docs/experiments/gvisor-start-integration-20261102/README.md new file mode 100644 index 0000000000..0097b29eb8 --- /dev/null +++ b/docs/experiments/gvisor-start-integration-20261102/README.md @@ -0,0 +1,141 @@ +# Node-agent gVisor start integration, 2 November 2026 + +This is a live trial of the rebuilt Kubernetes node-agent, following the +[profile comparison in #1053](https://github.com/kubescape/node-agent/pull/1053). +The receiver, CRI/Kubernetes identity join and existing OTLP log exporter run +inside the actual node-agent process. The standalone start probe is not the +producer for this evidence. + +## Observed results + +| Case | Expected observations | Received | +|---|---:|---:| +| Repeated application-container starts | 20 | 20 | +| Two overlapping gVisor workloads | 2 | 2 | +| gVisor workload in an excluded namespace | 0 | 0 | +| Valid synthetic frame claiming an unknown ID | 0 | 0 | +| Ordinary-runtime control | 0 | 0 | + +All 24 workloads (22 accepted gVisor, one excluded gVisor and one ordinary) +completed with exit code 0. Each accepted container ID, sandbox ID, Pod UID, +Pod name/namespace, node and container name matched the independently queried +CRI/Kubernetes inventory. The concurrent workloads' execution intervals overlap. +There was exactly one retained observation per accepted container, and each +reported zero sender drops. These are bounded results, not delivery guarantees. + +See the [verification](evidence/verification.json), +[identity ledger](evidence/runtime-ledger.json), +[received observations](evidence/observations.jsonl) and +[environment](evidence/environment.json). All records are marked +`source=gvisor_trace`, `kind=container_started`, `identity_status=verified`, +with `identity_origin=local_cri_and_kubernetes`. + +The ordinary control produced no trace input naturally; runtime-handler +rejection is additionally covered by the CRI unit tests. The unknown-ID control +sends a valid remote-sink frame but does not establish a real runtime start. +The trial does not authenticate the sender or prove application execution from +the start record. Workload completion is separate runtime evidence. + +## Privacy validation + +The fixture puts the harmless marker `GVS_INTEGRATION_CANARY_2026` in both +argv and the OCI working directory. A mounted empty directory supplies that +working directory. The marker in fixture source is intentional. + +The collector scans the complete incoming OTLP request for the literal marker +**before** filtering retained fields. It records a violation flag without +copying the offending payload. Every retained observation has +`wire_canary_found=false` and `unexpected_attribute_count=0`. A collector +regression test injects an extra sensitive attribute and proves that filtering +cannot falsely clear the wire check. The node-agent's ordinary logs were also +scanned locally and contained no literal marker; those raw logs are not included. +This checks the literal synthetic marker and the fixed export contract, not +every possible encoding of arbitrary credentials. + +The published run repeats an initial successful 20-plus-two trial with the +stronger pre-filter wire check. The earlier run remains in the VM for audit; +only the final run is used in the table above. + +## Environment + +- Dedicated Ubuntu ARM64 Multipass VM, four vCPUs, 6 GiB RAM, kernel + 6.8.0-147-generic. +- K3s v1.37.1+k3s1, local CRI endpoint + `/run/k3s/containerd/containerd.sock`, runsc release-20261019.0 with systrap. +- Node-agent built from this PR's working tree on base `0ad6adb2` using the + repository's Go 1.27.0 toolchain and pinned dependencies, with CGO disabled. + The actual binary SHA-256 is retained in the environment summary. +- The rebuilt binary was mounted into the existing official v0.3.293 test + image and selected as its command. This is not a published image release. +- BusyBox is pinned by image-index digest in the fixture. Runtime image IDs + are recorded per workload. +- Runtime detection, application-profile learning and host sensing were + disabled for this source/export trial. The earlier eBPF comparison remains + separate evidence; no guest-syscall comparison is inferred here. + +The test retains the earlier experimental storage/cluster setup, including +its documented root storage workaround. It is not a production chart recipe. +No managed GKE or x86_64 claim is made. + +## Reproduction + +Use only the dedicated self-managed test VM and preserve the original +DaemonSet, ConfigMap and runsc runtime settings privately before changing them. +Do not export kubeconfig, service-account tokens, cloud credentials or raw OCI +specifications into a PR. + +1. Build `./cmd` and `./docs/experiments/gvisor-start-integration-20261102/otlp-capture` + for Linux ARM64 with Go 1.27.0. Install the test binary into the experimental + node-agent Pod using a read-only hostPath mount and command override. Keep + the existing image's gadget data and host mounts. Record the binary hash. +2. Run the capture utility on a trusted VM-local address with a new output + file. Set `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` on node-agent to that collector. + The plaintext transport used for this isolated local experiment is not a + public collector deployment. +3. Enable the [source configuration](../../features/gvisor-start-source.md), + with the K3s CRI endpoint, `runtimeHandler=runsc` and + `/run/kubescape/gvisor-events.sock`. Keep namespace selection limited to + `gvisor-comparison`. Create the root-owned socket directory with mode 0700. +4. For newly created gVisor Pods only, configure runsc's pod-init configuration + with the existing `container/start` remote sink pointing to that same + socket. Do not overwrite another monitor's Default session. This experiment + owns its runtime settings and newly created sandboxes. +5. Put the fixtures and binaries in + `/home/ubuntu/gvisor-trial/integration-20261102`. Start the collector with + `--output` pointing to `observations.jsonl` there. Wait for agent startup + and a successful version-1 seqpacket handshake, then run `trial.py --output` + with the `evidence` subdirectory as its new output path. It performs 20 + repeated starts and the unknown-ID/ordinary controls. +6. Run `parallel-trial.py`, then `verify.py` before deleting Pods. Verification + checks every export against retained Pod and CRI identities, exact counts, + concurrent execution overlap, the pre-filter privacy flags and local agent + log scan. Checks remain active under `python -O` and `PYTHONOPTIMIZE`; + overlap and privacy verdicts are derived from the checked inputs. An error + is a failed experiment, not evidence of no starts. +7. Export only the sanitized observation/verification files. Restore the + original ConfigMap, DaemonSet and runsc settings, remove only the experiment + workloads, allow the OTEL shutdown flush, then stop the collector and VM. + Do not delete the broader `kubescape` namespace. + +The fixtures use fixed trial names and refuse to overwrite result directories. +Use fresh names/directories or explicitly archive and clean up a previous +experiment before repeating. Kubernetes/CRI errors and OCI specifications can +contain the marker; the fixtures do not retain those raw responses as evidence. + +## Validation and remaining limits + +The Linux ARM64 node-agent built successfully. Focused Linux tests passed for +receiver/integration behavior, CRI verification, Kubernetes UID/node/filter +checks, opt-in config and the distinct OTEL record contract. Linux-targeted +vet passed for the changed packages. Portable CRI and OTEL tests also passed +with race detection on macOS. The capture privacy regression and the existing +22 Python gVisor-tool tests passed. This was not the full repository suite or +a Linux race run. + +The previously unexplained 19/20 start run is not retrospectively explained. +This source remains experimental and disabled by default. It can miss very +short-lived containers during identity rechecks, discard queued starts on +cancellation, or lose records through the bounded OTEL batch exporter. Detailed +health metrics, fair buffering, reconnect/recovery policy, overhead benchmarks, +stop semantics and network signals remain follow-up work. There is no actor or +session attribution and no claim of in-sandbox behavioral enforcement. diff --git a/docs/experiments/gvisor-start-integration-20261102/evidence-sha256.json b/docs/experiments/gvisor-start-integration-20261102/evidence-sha256.json new file mode 100644 index 0000000000..4e751b2054 --- /dev/null +++ b/docs/experiments/gvisor-start-integration-20261102/evidence-sha256.json @@ -0,0 +1,6 @@ +{ + "environment.json": "c6e016c7c689f0b3e8098817324524a416e627e9231968cb1bddc7a136f3263e", + "observations.jsonl": "c74d21a207fefc2275b08fbe4752ead692545922b93f6596efdad1c595d7ed36", + "runtime-ledger.json": "68e86ff45e6715404eeff75e871fd49438444046acea04791642c12f7382848c", + "verification.json": "cd45cf843f4dc04c4b76ac7725ee9881c867690418c1e3e38c44f3b67c08ec41" +} diff --git a/docs/experiments/gvisor-start-integration-20261102/evidence/environment.json b/docs/experiments/gvisor-start-integration-20261102/evidence/environment.json new file mode 100644 index 0000000000..37c5bf2078 --- /dev/null +++ b/docs/experiments/gvisor-start-integration-20261102/evidence/environment.json @@ -0,0 +1,23 @@ +{ + "base_revision": "0ad6adb2", + "source": "working tree of this PR; test binary mounted into released base image", + "compiler": "go1.27.0", + "target": "linux/arm64, CGO_ENABLED=0", + "node_agent_binary_sha256": "8b3125d5acaca3ef3e45339590ab7e45d9ccdd91c0bdcb521dccef2f6cc610cb", + "base_image": "quay.io/kubescape/node-agent:v0.3.293", + "base_image_id": "sha256:0f31e6cfb343bfa2e486d59fe064c25a02c70bbc3fef4321dc31923fbe090577", + "node_agent_ready": true, + "kernel": "6.8.0-147-generic", + "k3s": "k3s version v1.37.1+k3s1 (356c0254)", + "runsc": "runsc version release-20261019.0", + "platform": "systrap", + "source_config": { + "enabled": true, + "socketPath": "/run/kubescape/gvisor-events.sock", + "runtimeEndpoint": "/run/k3s/containerd/containerd.sock", + "runtimeHandler": "runsc" + }, + "application_profiles_enabled_during_trial": false, + "namespace_allowlist": "gvisor-comparison", + "otlp_destination": "local disposable VM collector" +} diff --git a/docs/experiments/gvisor-start-integration-20261102/evidence/observations.jsonl b/docs/experiments/gvisor-start-integration-20261102/evidence/observations.jsonl new file mode 100644 index 0000000000..09b84bb336 --- /dev/null +++ b/docs/experiments/gvisor-start-integration-20261102/evidence/observations.jsonl @@ -0,0 +1,22 @@ +{"body":"RuntimeObservation","container.id":"2ae17c2e3fcb9444daf05322f242a39a881455f977603be282db7e9e27e5a42f","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793640206237419177,"pod_name":"integration-20261102-01","pod_uid":"fba05711-9919-466b-8d09-ebfe041ec319","runtime_handler":"runsc","sandbox_id":"167e20a3b3748e158637dfe3d27ea044646c76232dab6d49a111a538738bb3fd","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"caf1b1300c87d3d3e7f5e46c6107fc2f378c4ee34963dba2c8e7e452e369b292","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793640211426747254,"pod_name":"integration-20261102-02","pod_uid":"8b8d62c3-c3d6-4225-a55d-14247809460c","runtime_handler":"runsc","sandbox_id":"5f82b4a510c86b7b9ba214c3353eb72cd32670dc2808ab536f651b87de36cffe","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"568a853e091d24b68daad6265de448e22454f629e0c9139488620fe6ff3ced25","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793640217688521099,"pod_name":"integration-20261102-03","pod_uid":"9a6ad8fb-79f4-45d1-95fa-487e2d11c0c0","runtime_handler":"runsc","sandbox_id":"7a480690ac895bd9b4a7ccb313927ecce057c98b5c91c21828f8bc20e8ede82a","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"e01ab172e550220f313ba14bc1a478e6f88b2fc20a421931d9433663c644f950","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793640223320638830,"pod_name":"integration-20261102-04","pod_uid":"34b2e58d-a226-4920-a87b-786c3a883999","runtime_handler":"runsc","sandbox_id":"e61334d9e2c84280d1e9f8c592b31bb1efc2e086e4a4ba012f0655793a946a12","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"ba339c194948e20b308a4c9486cf004b9e0f3c0dc0383e9158867dffd5ff7744","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793640228433458390,"pod_name":"integration-20261102-05","pod_uid":"bb9a9df9-2d59-4296-9a78-48018d4a3f25","runtime_handler":"runsc","sandbox_id":"f81f534d72cc43f266b79adf4adee4ae302e7c0b24a580a44e1da958c80c9d50","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"80f8448ba17491bbdfd47bfb92521b6042a27db69f8aedfc0f01dafcb3512dea","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793640234382696515,"pod_name":"integration-20261102-06","pod_uid":"683870cb-2b11-436c-b881-53d9f2f266d9","runtime_handler":"runsc","sandbox_id":"8e61de9922ce9046013a5d4444f72af8e2c0bb671223e5bbd33a0d4644cb8874","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"83e543882caedc5748c1e84caa4100f1a6296e91ab3d816ffc03fb80139dfe80","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793640239613191257,"pod_name":"integration-20261102-07","pod_uid":"a7009f33-1898-441f-b740-a60c376ca75e","runtime_handler":"runsc","sandbox_id":"10e50738e754873cea01aaf6dcdd55d3e6ca86cd1931141d08bdf64bd1b96674","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"0d5bbeaef2f9ce425563c2136313b783f4ecc102d0582e341f78f78020a6209e","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793640245480774386,"pod_name":"integration-20261102-08","pod_uid":"33bebcf9-fd36-40f6-94f9-39c647b76d8d","runtime_handler":"runsc","sandbox_id":"a99a1a9c51e378dd4da2bfcfdc7006f8e11fb1e96686ec0da02d06209baf6be2","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"8a0c7c19e8443f90d6a15ed53a3a5b1b6e577a10f1df9966ea384a86282fcd45","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793640251371540250,"pod_name":"integration-20261102-09","pod_uid":"73134249-d399-4e0d-8803-a477a24ef4ff","runtime_handler":"runsc","sandbox_id":"47724cab53de02d4d4a4e254a83fab1cee03b8489912537bd1aab46c13aed9d9","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"27c02447a5debbf74fa5aad0092eecb8031964ae683c72cd560cc629b00714ba","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793640256537290422,"pod_name":"integration-20261102-10","pod_uid":"743dcff6-fd09-4d0b-af7e-05cc3b6c347c","runtime_handler":"runsc","sandbox_id":"7f129390f042036ea8f53c43145e311b39ab187ec10c805e2929122a40ea7687","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"059c82fa710832180c14f4bbb0b89d3fc04ec59752068734a48be1606d149958","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793640262561126237,"pod_name":"integration-20261102-11","pod_uid":"08664fd3-da20-4923-b5b2-ee0e118b7fee","runtime_handler":"runsc","sandbox_id":"ae98dc1521fba83e82b60e27d5065003794c7396a4f6590bfcc7108bf8d607c3","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"e3f6d4ecfdd90f5e83c7b89ee7767442cad9218e1464641a84cc8643dcdc2e0a","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793640270696327065,"pod_name":"integration-20261102-12","pod_uid":"c11c628a-4267-415c-aac4-9feeca8c9873","runtime_handler":"runsc","sandbox_id":"4a54afc590581357b82feef6708ee1d773ac2b497cfcba08f24a0268315505e7","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"c5cd39964ba04c85b31c3087d8b7fed2ebb7abdd425069655915565a70834f14","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793640275841682323,"pod_name":"integration-20261102-13","pod_uid":"8cce4814-e697-4244-a8de-369e11991256","runtime_handler":"runsc","sandbox_id":"e5a7c78a76e69b75358fa0bddc032b9d493f9b4f836f6598a481d01d46ca9fcd","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"0c72d57beb9dc99b95d5bcb94f0a851308ce358c01522cfc7dd3303274269dbc","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793640282045689470,"pod_name":"integration-20261102-14","pod_uid":"c33f922f-28c6-4db3-8766-99f258122116","runtime_handler":"runsc","sandbox_id":"b346d0af0dada0235443356ef24ceba5429067c93b041846c1535149ca70305d","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"3c58338375a2f39dc897b033025ac8a9254439f723a5bf7adc9f0d289ce3868a","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793640287978282273,"pod_name":"integration-20261102-15","pod_uid":"36d15e13-e07e-49c0-95ad-cacf14277f3b","runtime_handler":"runsc","sandbox_id":"3d6a581bb004b60a5092df1ff47491999886b09a85c4d9cbd7ab334db697e957","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"32af4a56581d3fe679ea828ea5f76cae42ef665f0b2a9b0bb067180f155c7943","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793640293816133445,"pod_name":"integration-20261102-16","pod_uid":"aa1cd447-9d57-465a-bcef-6825d0bef00c","runtime_handler":"runsc","sandbox_id":"a38de3dfb2c0dfb425b99a8b830ed198ca018252bedf7586211a2339392422ef","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"cab95800925135c5142ec3de9ae93e9739479fe705ab1f39ee201cb9086ca3b9","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793640299015776157,"pod_name":"integration-20261102-17","pod_uid":"45e6b57e-59ad-4955-ac47-28829f943328","runtime_handler":"runsc","sandbox_id":"a5864d5bdb07ca3a2bdbcbff3a2b110fcafe1a6b6306a67c06a6d05ad6f1738b","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"ce2d0ad2c8165e48516bc3f14c3adbc334770c6caffee1f909044953f5598967","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793640304956851582,"pod_name":"integration-20261102-18","pod_uid":"f514b541-60e7-45fb-a387-12a3c0ddc736","runtime_handler":"runsc","sandbox_id":"f255b4deeaeb771477a641614cdcc8992d2681e922d3e3d1b6905b1d87c8553f","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"eadf1521998507035a239a7a3568084083d1ed1cabcba55be1a1c8173bf3e594","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793640310050196968,"pod_name":"integration-20261102-19","pod_uid":"054d790a-578f-40d1-8d47-01b3c9a43f1a","runtime_handler":"runsc","sandbox_id":"aac7807837829bf40faabc1c3a04966dc101cf006915ee74826cba43072064b3","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"178d829e12eda938526ea76c89125ab0a18b912893d57f1dd6aac98f86fa70ed","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793640316309314282,"pod_name":"integration-20261102-20","pod_uid":"3ce7b777-e2d7-4bf7-927e-cfa41e3510cf","runtime_handler":"runsc","sandbox_id":"9de089c069910c8bbf2219b363864f711ad9411fbcb1d7fab446955489c10648","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"d1218107dba456e968fe1efb979d258f4869738cab8d9da1801f74874c65cef6","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793641247566904483,"pod_name":"integration-parallel-a-20261102","pod_uid":"216cfb65-13e8-4758-aa8d-c86059b7c1a1","runtime_handler":"runsc","sandbox_id":"9c639e70f1efc8d87314b0fe141fef59b3063472fe0b83ea7693790bceddd1d4","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} +{"body":"RuntimeObservation","container.id":"42b1a0aec71b119afabb046cec444922c05845f23a4cbaaac7ff9fe901a2832a","container_name":"workload","identity_origin":"local_cri_and_kubernetes","identity_status":"verified","kind":"container_started","namespace":"gvisor-comparison","node":"gvisor-trial","observed_time_unix_nano":1793641247577331703,"pod_name":"integration-parallel-b-20261102","pod_uid":"65d55738-6f7a-4c29-a370-1ae35388dc8b","runtime_handler":"runsc","sandbox_id":"20a10bec71ed551ecc390dd9ea6c0c2d81b4bcfc809a24b317dfd1d20010dc81","sender_reported_drops":0,"source":"gvisor_trace","unexpected_attribute_count":0,"wire_canary_found":false} diff --git a/docs/experiments/gvisor-start-integration-20261102/evidence/runtime-ledger.json b/docs/experiments/gvisor-start-integration-20261102/evidence/runtime-ledger.json new file mode 100644 index 0000000000..5a361a9f72 --- /dev/null +++ b/docs/experiments/gvisor-start-integration-20261102/evidence/runtime-ledger.json @@ -0,0 +1,290 @@ +[ + { + "name": "integration-20261102-01", + "namespace": "gvisor-comparison", + "pod_uid": "fba05711-9919-466b-8d09-ebfe041ec319", + "container_id": "2ae17c2e3fcb9444daf05322f242a39a881455f977603be282db7e9e27e5a42f", + "sandbox_id": "167e20a3b3748e158637dfe3d27ea044646c76232dab6d49a111a538738bb3fd", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:23:26Z", + "finished_at": "2026-11-02T17:23:30Z" + }, + { + "name": "integration-20261102-02", + "namespace": "gvisor-comparison", + "pod_uid": "8b8d62c3-c3d6-4225-a55d-14247809460c", + "container_id": "caf1b1300c87d3d3e7f5e46c6107fc2f378c4ee34963dba2c8e7e452e369b292", + "sandbox_id": "5f82b4a510c86b7b9ba214c3353eb72cd32670dc2808ab536f651b87de36cffe", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:23:31Z", + "finished_at": "2026-11-02T17:23:35Z" + }, + { + "name": "integration-20261102-03", + "namespace": "gvisor-comparison", + "pod_uid": "9a6ad8fb-79f4-45d1-95fa-487e2d11c0c0", + "container_id": "568a853e091d24b68daad6265de448e22454f629e0c9139488620fe6ff3ced25", + "sandbox_id": "7a480690ac895bd9b4a7ccb313927ecce057c98b5c91c21828f8bc20e8ede82a", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:23:37Z", + "finished_at": "2026-11-02T17:23:41Z" + }, + { + "name": "integration-20261102-04", + "namespace": "gvisor-comparison", + "pod_uid": "34b2e58d-a226-4920-a87b-786c3a883999", + "container_id": "e01ab172e550220f313ba14bc1a478e6f88b2fc20a421931d9433663c644f950", + "sandbox_id": "e61334d9e2c84280d1e9f8c592b31bb1efc2e086e4a4ba012f0655793a946a12", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:23:43Z", + "finished_at": "2026-11-02T17:23:47Z" + }, + { + "name": "integration-20261102-05", + "namespace": "gvisor-comparison", + "pod_uid": "bb9a9df9-2d59-4296-9a78-48018d4a3f25", + "container_id": "ba339c194948e20b308a4c9486cf004b9e0f3c0dc0383e9158867dffd5ff7744", + "sandbox_id": "f81f534d72cc43f266b79adf4adee4ae302e7c0b24a580a44e1da958c80c9d50", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:23:48Z", + "finished_at": "2026-11-02T17:23:52Z" + }, + { + "name": "integration-20261102-06", + "namespace": "gvisor-comparison", + "pod_uid": "683870cb-2b11-436c-b881-53d9f2f266d9", + "container_id": "80f8448ba17491bbdfd47bfb92521b6042a27db69f8aedfc0f01dafcb3512dea", + "sandbox_id": "8e61de9922ce9046013a5d4444f72af8e2c0bb671223e5bbd33a0d4644cb8874", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:23:54Z", + "finished_at": "2026-11-02T17:23:58Z" + }, + { + "name": "integration-20261102-07", + "namespace": "gvisor-comparison", + "pod_uid": "a7009f33-1898-441f-b740-a60c376ca75e", + "container_id": "83e543882caedc5748c1e84caa4100f1a6296e91ab3d816ffc03fb80139dfe80", + "sandbox_id": "10e50738e754873cea01aaf6dcdd55d3e6ca86cd1931141d08bdf64bd1b96674", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:23:59Z", + "finished_at": "2026-11-02T17:24:03Z" + }, + { + "name": "integration-20261102-08", + "namespace": "gvisor-comparison", + "pod_uid": "33bebcf9-fd36-40f6-94f9-39c647b76d8d", + "container_id": "0d5bbeaef2f9ce425563c2136313b783f4ecc102d0582e341f78f78020a6209e", + "sandbox_id": "a99a1a9c51e378dd4da2bfcfdc7006f8e11fb1e96686ec0da02d06209baf6be2", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:24:05Z", + "finished_at": "2026-11-02T17:24:09Z" + }, + { + "name": "integration-20261102-09", + "namespace": "gvisor-comparison", + "pod_uid": "73134249-d399-4e0d-8803-a477a24ef4ff", + "container_id": "8a0c7c19e8443f90d6a15ed53a3a5b1b6e577a10f1df9966ea384a86282fcd45", + "sandbox_id": "47724cab53de02d4d4a4e254a83fab1cee03b8489912537bd1aab46c13aed9d9", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:24:11Z", + "finished_at": "2026-11-02T17:24:15Z" + }, + { + "name": "integration-20261102-10", + "namespace": "gvisor-comparison", + "pod_uid": "743dcff6-fd09-4d0b-af7e-05cc3b6c347c", + "container_id": "27c02447a5debbf74fa5aad0092eecb8031964ae683c72cd560cc629b00714ba", + "sandbox_id": "7f129390f042036ea8f53c43145e311b39ab187ec10c805e2929122a40ea7687", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:24:16Z", + "finished_at": "2026-11-02T17:24:20Z" + }, + { + "name": "integration-20261102-11", + "namespace": "gvisor-comparison", + "pod_uid": "08664fd3-da20-4923-b5b2-ee0e118b7fee", + "container_id": "059c82fa710832180c14f4bbb0b89d3fc04ec59752068734a48be1606d149958", + "sandbox_id": "ae98dc1521fba83e82b60e27d5065003794c7396a4f6590bfcc7108bf8d607c3", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:24:22Z", + "finished_at": "2026-11-02T17:24:26Z" + }, + { + "name": "integration-20261102-12", + "namespace": "gvisor-comparison", + "pod_uid": "c11c628a-4267-415c-aac4-9feeca8c9873", + "container_id": "e3f6d4ecfdd90f5e83c7b89ee7767442cad9218e1464641a84cc8643dcdc2e0a", + "sandbox_id": "4a54afc590581357b82feef6708ee1d773ac2b497cfcba08f24a0268315505e7", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:24:30Z", + "finished_at": "2026-11-02T17:24:34Z" + }, + { + "name": "integration-20261102-13", + "namespace": "gvisor-comparison", + "pod_uid": "8cce4814-e697-4244-a8de-369e11991256", + "container_id": "c5cd39964ba04c85b31c3087d8b7fed2ebb7abdd425069655915565a70834f14", + "sandbox_id": "e5a7c78a76e69b75358fa0bddc032b9d493f9b4f836f6598a481d01d46ca9fcd", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:24:35Z", + "finished_at": "2026-11-02T17:24:39Z" + }, + { + "name": "integration-20261102-14", + "namespace": "gvisor-comparison", + "pod_uid": "c33f922f-28c6-4db3-8766-99f258122116", + "container_id": "0c72d57beb9dc99b95d5bcb94f0a851308ce358c01522cfc7dd3303274269dbc", + "sandbox_id": "b346d0af0dada0235443356ef24ceba5429067c93b041846c1535149ca70305d", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:24:42Z", + "finished_at": "2026-11-02T17:24:46Z" + }, + { + "name": "integration-20261102-15", + "namespace": "gvisor-comparison", + "pod_uid": "36d15e13-e07e-49c0-95ad-cacf14277f3b", + "container_id": "3c58338375a2f39dc897b033025ac8a9254439f723a5bf7adc9f0d289ce3868a", + "sandbox_id": "3d6a581bb004b60a5092df1ff47491999886b09a85c4d9cbd7ab334db697e957", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:24:47Z", + "finished_at": "2026-11-02T17:24:51Z" + }, + { + "name": "integration-20261102-16", + "namespace": "gvisor-comparison", + "pod_uid": "aa1cd447-9d57-465a-bcef-6825d0bef00c", + "container_id": "32af4a56581d3fe679ea828ea5f76cae42ef665f0b2a9b0bb067180f155c7943", + "sandbox_id": "a38de3dfb2c0dfb425b99a8b830ed198ca018252bedf7586211a2339392422ef", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:24:53Z", + "finished_at": "2026-11-02T17:24:57Z" + }, + { + "name": "integration-20261102-17", + "namespace": "gvisor-comparison", + "pod_uid": "45e6b57e-59ad-4955-ac47-28829f943328", + "container_id": "cab95800925135c5142ec3de9ae93e9739479fe705ab1f39ee201cb9086ca3b9", + "sandbox_id": "a5864d5bdb07ca3a2bdbcbff3a2b110fcafe1a6b6306a67c06a6d05ad6f1738b", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:24:59Z", + "finished_at": "2026-11-02T17:25:03Z" + }, + { + "name": "integration-20261102-18", + "namespace": "gvisor-comparison", + "pod_uid": "f514b541-60e7-45fb-a387-12a3c0ddc736", + "container_id": "ce2d0ad2c8165e48516bc3f14c3adbc334770c6caffee1f909044953f5598967", + "sandbox_id": "f255b4deeaeb771477a641614cdcc8992d2681e922d3e3d1b6905b1d87c8553f", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:25:04Z", + "finished_at": "2026-11-02T17:25:08Z" + }, + { + "name": "integration-20261102-19", + "namespace": "gvisor-comparison", + "pod_uid": "054d790a-578f-40d1-8d47-01b3c9a43f1a", + "container_id": "eadf1521998507035a239a7a3568084083d1ed1cabcba55be1a1c8173bf3e594", + "sandbox_id": "aac7807837829bf40faabc1c3a04966dc101cf006915ee74826cba43072064b3", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:25:10Z", + "finished_at": "2026-11-02T17:25:14Z" + }, + { + "name": "integration-20261102-20", + "namespace": "gvisor-comparison", + "pod_uid": "3ce7b777-e2d7-4bf7-927e-cfa41e3510cf", + "container_id": "178d829e12eda938526ea76c89125ab0a18b912893d57f1dd6aac98f86fa70ed", + "sandbox_id": "9de089c069910c8bbf2219b363864f711ad9411fbcb1d7fab446955489c10648", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:25:16Z", + "finished_at": "2026-11-02T17:25:20Z" + }, + { + "name": "integration-ordinary-20261102", + "namespace": "gvisor-comparison", + "pod_uid": "fc0f1071-ac57-41ef-8411-31b45318f689", + "container_id": "aed920bb94017dd3aebf96f34861ca8d77d6ed6b770468087cf82007409291b3", + "sandbox_id": "c908ca47c4be71d1e1bdbc2bbf4dbcea77bc55929a97f29db25acc94dbca09e6", + "runtime": "io.containerd.runc.v2", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:25:21Z", + "finished_at": "2026-11-02T17:25:25Z" + }, + { + "name": "integration-parallel-a-20261102", + "namespace": "gvisor-comparison", + "pod_uid": "216cfb65-13e8-4758-aa8d-c86059b7c1a1", + "container_id": "d1218107dba456e968fe1efb979d258f4869738cab8d9da1801f74874c65cef6", + "sandbox_id": "9c639e70f1efc8d87314b0fe141fef59b3063472fe0b83ea7693790bceddd1d4", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:40:47Z", + "finished_at": "2026-11-02T17:40:51Z" + }, + { + "name": "integration-parallel-b-20261102", + "namespace": "gvisor-comparison", + "pod_uid": "65d55738-6f7a-4c29-a370-1ae35388dc8b", + "container_id": "42b1a0aec71b119afabb046cec444922c05845f23a4cbaaac7ff9fe901a2832a", + "sandbox_id": "20a10bec71ed551ecc390dd9ea6c0c2d81b4bcfc809a24b317dfd1d20010dc81", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:40:47Z", + "finished_at": "2026-11-02T17:40:51Z" + }, + { + "name": "integration-excluded-20261102", + "namespace": "gvisor-excluded-20261102", + "pod_uid": "0bc23ccd-3f10-4d22-b10f-ce98404e7b41", + "container_id": "f692169893e95ba7da9d4c29080c1ee6525400f6c0f8830f434480a445428539", + "sandbox_id": "8236b465e2f2557515b3bd974798fe43c9c2a9afb4fa2e6355a73c6f6736b6e7", + "runtime": "io.containerd.runsc.v1", + "exit_code": 0, + "image_id": "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662", + "started_at": "2026-11-02T17:40:54Z", + "finished_at": "2026-11-02T17:40:58Z" + } +] diff --git a/docs/experiments/gvisor-start-integration-20261102/evidence/verification.json b/docs/experiments/gvisor-start-integration-20261102/evidence/verification.json new file mode 100644 index 0000000000..4e11b2a6fb --- /dev/null +++ b/docs/experiments/gvisor-start-integration-20261102/evidence/verification.json @@ -0,0 +1,14 @@ +{ + "repeated_expected": 20, + "repeated_exported": 20, + "concurrent_expected": 2, + "concurrent_exported": 2, + "concurrent_execution_overlap": true, + "excluded_namespace_exports": 0, + "unknown_id_exports": 0, + "ordinary_runtime_exports": 0, + "literal_canary_in_exports_or_agent_logs": false, + "wire_privacy_checks_before_filtering": true, + "unexpected_observation_attributes": 0, + "identity_cross_checks": "exact container ID, sandbox ID, Pod UID/name/namespace, node, container name" +} diff --git a/docs/experiments/gvisor-start-integration-20261102/otlp-capture/main.go b/docs/experiments/gvisor-start-integration-20261102/otlp-capture/main.go new file mode 100644 index 0000000000..cff692ee41 --- /dev/null +++ b/docs/experiments/gvisor-start-integration-20261102/otlp-capture/main.go @@ -0,0 +1,94 @@ +// A disposable experiment collector. It keeps only RuntimeObservation records +// and a fixed attribute allowlist, not general node-agent logs or arguments. +package main + +import ( + "bytes" + "context" + "encoding/json" + "flag" + "net" + "os" + "os/signal" + "sync" + "syscall" + + logs "go.opentelemetry.io/proto/otlp/collector/logs/v1" + "google.golang.org/grpc" + "google.golang.org/protobuf/proto" +) + +type capture struct { + logs.UnimplementedLogsServiceServer + mu sync.Mutex + file *os.File +} + +func (c *capture) Export(_ context.Context, request *logs.ExportLogsServiceRequest) (*logs.ExportLogsServiceResponse, error) { + c.mu.Lock() + defer c.mu.Unlock() + allowed := map[string]bool{"source": true, "kind": true, "identity_status": true, "identity_origin": true, "node": true, "container.id": true, "sandbox_id": true, "runtime_handler": true, "namespace": true, "pod_name": true, "pod_uid": true, "container_name": true, "sender_reported_drops": true} + wire, err := proto.Marshal(request) + if err != nil { + return nil, err + } + canaryFound := bytes.Contains(wire, []byte("GVS_INTEGRATION_CANARY_2026")) + clear(wire) + if canaryFound { + if err := json.NewEncoder(c.file).Encode(map[string]any{"body": "CapturePrivacyFailure", "canary_received": true}); err != nil { + return nil, err + } + } + for _, resource := range request.ResourceLogs { + for _, scope := range resource.ScopeLogs { + for _, record := range scope.LogRecords { + if record.Body.GetStringValue() != "RuntimeObservation" { + continue + } + output := map[string]any{"body": "RuntimeObservation", "observed_time_unix_nano": record.ObservedTimeUnixNano, "wire_canary_found": canaryFound} + unexpected := 0 + for _, kv := range record.Attributes { + if !allowed[kv.Key] { + unexpected++ + continue + } + if kv.Key == "sender_reported_drops" { + output[kv.Key] = kv.Value.GetIntValue() + } else { + output[kv.Key] = kv.Value.GetStringValue() + } + } + output["unexpected_attribute_count"] = unexpected + if err := json.NewEncoder(c.file).Encode(output); err != nil { + return nil, err + } + } + } + } + if err := c.file.Sync(); err != nil { + return nil, err + } + return &logs.ExportLogsServiceResponse{}, nil +} +func main() { + listen := flag.String("listen", "127.0.0.1:4317", "bind only a trusted local experiment address") + output := flag.String("output", "observations.jsonl", "new sanitized output file") + flag.Parse() + file, err := os.OpenFile(*output, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + panic("cannot create experiment output") + } + defer file.Close() + listener, err := net.Listen("tcp", *listen) + if err != nil { + panic("cannot listen for experiment logs") + } + server := grpc.NewServer(grpc.MaxRecvMsgSize(1 << 20)) + logs.RegisterLogsServiceServer(server, &capture{file: file}) + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + go func() { <-ctx.Done(); server.Stop() }() + if err := server.Serve(listener); err != nil && ctx.Err() == nil { + panic("experiment log collector failed") + } +} diff --git a/docs/experiments/gvisor-start-integration-20261102/otlp-capture/main_test.go b/docs/experiments/gvisor-start-integration-20261102/otlp-capture/main_test.go new file mode 100644 index 0000000000..2d39d56b0f --- /dev/null +++ b/docs/experiments/gvisor-start-integration-20261102/otlp-capture/main_test.go @@ -0,0 +1,52 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + + logs "go.opentelemetry.io/proto/otlp/collector/logs/v1" + common "go.opentelemetry.io/proto/otlp/common/v1" + logpb "go.opentelemetry.io/proto/otlp/logs/v1" +) + +func TestCaptureChecksWireBeforeFiltering(t *testing.T) { + for _, canary := range []bool{false, true} { + t.Run(map[bool]string{false: "clean", true: "sensitive_extra_attribute"}[canary], func(t *testing.T) { + file, err := os.CreateTemp(t.TempDir(), "capture") + if err != nil { + t.Fatal(err) + } + defer file.Close() + record := &logpb.LogRecord{Body: &common.AnyValue{Value: &common.AnyValue_StringValue{StringValue: "RuntimeObservation"}}} + if canary { + record.Attributes = []*common.KeyValue{{Key: "args", Value: &common.AnyValue{Value: &common.AnyValue_StringValue{StringValue: "GVS_INTEGRATION_CANARY_2026"}}}} + } + request := &logs.ExportLogsServiceRequest{ResourceLogs: []*logpb.ResourceLogs{{ScopeLogs: []*logpb.ScopeLogs{{LogRecords: []*logpb.LogRecord{record}}}}}} + collector := &capture{file: file} + if _, err := collector.Export(context.Background(), request); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(file.Name()) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), "GVS_INTEGRATION_CANARY_2026") { + t.Fatal("sensitive attribute retained") + } + lines := strings.Split(strings.TrimSpace(string(raw)), "\n") + var last map[string]any + if err := json.Unmarshal([]byte(lines[len(lines)-1]), &last); err != nil { + t.Fatal(err) + } + if last["wire_canary_found"] != canary { + t.Fatal("wire canary detection defeated by filtering") + } + if canary && (len(lines) != 2 || last["unexpected_attribute_count"] != float64(1)) { + t.Fatal("privacy violation was hidden") + } + }) + } +} diff --git a/docs/experiments/gvisor-start-integration-20261102/parallel-trial.py b/docs/experiments/gvisor-start-integration-20261102/parallel-trial.py new file mode 100644 index 0000000000..1a562f12fe --- /dev/null +++ b/docs/experiments/gvisor-start-integration-20261102/parallel-trial.py @@ -0,0 +1,15 @@ +"""Follow repeated trial with two overlapping workloads and an excluded Pod.""" +import concurrent.futures,json,pathlib,subprocess,time +from trial import CANARY,K,query,run_pod + +base=pathlib.Path('/home/ubuntu/gvisor-trial/integration-20261102') +with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + rows=list(pool.map(run_pod,['integration-parallel-a-20261102','integration-parallel-b-20261102'])) +(base/'evidence/concurrent-ledger.json').write_text(json.dumps(rows,indent=2)+'\n') +namespace='gvisor-excluded-20261102' +result=subprocess.run(K+['create','namespace',namespace],capture_output=True,timeout=20) +if result.returncode:raise RuntimeError('cannot create fresh excluded namespace') +excluded=run_pod('integration-excluded-20261102',namespace=namespace) +(base/'evidence/excluded-ledger.json').write_text(json.dumps([excluded],indent=2)+'\n') +time.sleep(7) +print('Concurrent and excluded workloads completed') diff --git a/docs/experiments/gvisor-start-integration-20261102/trial.py b/docs/experiments/gvisor-start-integration-20261102/trial.py new file mode 100644 index 0000000000..6c92bcefa1 --- /dev/null +++ b/docs/experiments/gvisor-start-integration-20261102/trial.py @@ -0,0 +1,75 @@ +"""Run only on the dedicated disposable K3s VM, after source/collector setup. + +Creates harmless, digest-pinned workloads. Never retain raw Pod/OCI specs or +errors as evidence; they can contain the synthetic argv/cwd privacy marker. +""" +import argparse,json,pathlib,socket,struct,subprocess,time + +CANARY='GVS_INTEGRATION_CANARY_2026' +IMAGE='docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662' +NS='gvisor-comparison' +K=['k3s','kubectl','--request-timeout=15s'] + + +def query(args): + result=subprocess.run(args,capture_output=True,text=True,timeout=20) + if result.returncode:raise RuntimeError('experiment query failed') + return json.loads(result.stdout) + + +def readiness(): + client=socket.socket(socket.AF_UNIX,socket.SOCK_SEQPACKET) + client.settimeout(3) + client.connect('/run/kubescape/gvisor-events.sock') + client.send(b'\x08\x01') + assert client.recv(32)==b'\x08\x01' + return client + + +def workload(name,namespace=NS,ordinary=False): + spec={'restartPolicy':'Never','containers':[{'name':'workload','image':IMAGE,'imagePullPolicy':'IfNotPresent','command':['/bin/sh','-c','sleep 4',CANARY],'workingDir':'/'+CANARY,'volumeMounts':[{'name':'private-cwd','mountPath':'/'+CANARY}],'resources':{'limits':{'cpu':'500m','memory':'256Mi'}}}],'volumes':[{'name':'private-cwd','emptyDir':{}}]} + if not ordinary:spec['runtimeClassName']='gvisor' + return {'apiVersion':'v1','kind':'Pod','metadata':{'name':name,'namespace':namespace},'spec':spec} + + +def run_pod(name,namespace=NS,ordinary=False): + result=subprocess.run(K+['create','-f','-'],input=json.dumps(workload(name,namespace,ordinary)),capture_output=True,text=True,timeout=20) + if result.returncode:raise RuntimeError('cannot create fresh experiment Pod') + deadline=time.monotonic()+90 + while True: + pod=query(K+['get','pod',name,'-n',namespace,'-o','json']) + statuses=pod.get('status',{}).get('containerStatuses',[]) + if statuses and 'terminated' in statuses[0].get('state',{}): + status=statuses[0];end=status['state']['terminated'];id=status['containerID'].split('://',1)[1] + info=query(['k3s','ctr','-n','k8s.io','containers','info',id]) + containers=query(['k3s','crictl','ps','-a','--id',id,'-o','json'])['containers'] + assert len(containers)==1 and containers[0]['id']==id + return {'name':name,'namespace':namespace,'pod_uid':pod['metadata']['uid'],'container_id':id,'sandbox_id':containers[0]['podSandboxId'],'runtime':info['Runtime']['Name'],'exit_code':end['exitCode'],'image_id':status['imageID']} + if time.monotonic()>deadline:raise RuntimeError('experiment Pod did not complete') + time.sleep(.25) + + +def main(): + parser=argparse.ArgumentParser();parser.add_argument('--output',required=True);parser.add_argument('--count',type=int,default=20);args=parser.parse_args() + out=pathlib.Path(args.output);out.mkdir(mode=0o700,exist_ok=False) + # The trace sink and listener must complete a real handshake before start. + readiness().close() + results=[] + try: + for index in range(1,args.count+1): + results.append(run_pod('integration-20261102-%02d'%index)) + (out/'runtime-ledger.json').write_text(json.dumps(results,indent=2)+'\n') + assert results[-1]['exit_code']==0 + # A valid frame claiming an ID absent from the runtime must not export. + client=readiness();unknown=b'd'*64 + assert not query(['k3s','crictl','ps','-a','--id',unknown.decode(),'-o','json'])['containers'] + client.send(struct.pack(' time.Second { + t.Fatal("CRI RPC ignored deadline") + } + valid := tc == "exact_created" || tc == "exact_running" + if (err == nil) != valid { + t.Fatalf("identity verification: got %v, valid=%v", err, valid) + } + if valid && (identity.ContainerID != id || identity.SandboxID != sandboxID || identity.PodUID != "pod-uid") { + t.Fatal("inventory fields changed") + } + }) + } +} + +func TestGVisorCRIRejectsRemoteEndpoint(t *testing.T) { + if _, err := New("tcp://127.0.0.1:1234", "runsc"); err == nil { + t.Fatal("remote endpoint accepted") + } +} diff --git a/pkg/gvisor/integration.go b/pkg/gvisor/integration.go new file mode 100644 index 0000000000..cffaadfc9d --- /dev/null +++ b/pkg/gvisor/integration.go @@ -0,0 +1,52 @@ +package gvisor + +import ( + "context" + "errors" + "path/filepath" +) + +// IntegrationConfig enables a separate experimental observation source. Empty +// configuration performs no I/O. RuntimeEndpoint defaults to the node's CRI +// socket at the call site; RuntimeHandler defaults to runsc. +type IntegrationConfig struct { + Enabled bool `mapstructure:"enabled"` + SocketPath string `mapstructure:"socketPath"` + RuntimeEndpoint string `mapstructure:"runtimeEndpoint"` + RuntimeHandler string `mapstructure:"runtimeHandler"` +} + +func (c IntegrationConfig) Validate() error { + if !c.Enabled { + return nil + } + if !filepath.IsAbs(c.SocketPath) { + return errors.New("gVisor start collection requires an absolute socketPath") + } + if c.RuntimeEndpoint != "" && !filepath.IsAbs(c.RuntimeEndpoint) { + return errors.New("gVisor runtimeEndpoint must be an absolute local CRI socket path") + } + return nil +} + +// Identity contains only allowlisted local-runtime inventory fields. It does +// not authenticate the trace sender or identify an actor/session. +type Identity struct { + ContainerID string + SandboxID string + RuntimeHandler string + Namespace string + PodName string + PodUID string + ContainerName string +} + +// Observation is distinct from an exec event, profile entry or security alert. +// Container/start precedes application execution. ObservedAt is receiver time. +type Observation struct { + Start + Identity Identity + Node string +} + +type InventoryLookup func(context.Context, string) (Identity, error) diff --git a/pkg/gvisor/integration_linux.go b/pkg/gvisor/integration_linux.go new file mode 100644 index 0000000000..476113184e --- /dev/null +++ b/pkg/gvisor/integration_linux.go @@ -0,0 +1,47 @@ +//go:build linux + +package gvisor + +import ( + "context" + "errors" + "time" +) + +// RunIntegration joins identity both before queuing and immediately before +// emission. A short-lived container that disappears between those joins is +// deliberately not attributed using a stale cache. Lookup must honor ctx. +// Emit must not block on network or file I/O; the OTEL batch processor is the +// intended consumer. Callers own the runtime client's lifetime. +func RunIntegration(ctx context.Context, cfg IntegrationConfig, node string, lookup InventoryLookup, emit func(context.Context, Observation)) error { + if !cfg.Enabled { + return nil + } + if err := cfg.Validate(); err != nil { + return err + } + if node == "" || lookup == nil || emit == nil { + return errors.New("gVisor integration requires node, inventory and event export") + } + resolve := func(ctx context.Context, id string) (Identity, bool) { + callCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + identity, err := lookup(callCtx, id) + return identity, err == nil && identity.ContainerID == id && + identity.SandboxID != "" && identity.Namespace != "" && identity.PodName != "" && + identity.PodUID != "" && identity.ContainerName != "" && identity.RuntimeHandler != "" + } + receiver := Receiver{ + SocketPath: cfg.SocketPath, + Resolve: func(id string) bool { + _, ok := resolve(ctx, id) + return ok + }, + OnStart: func(ctx context.Context, start Start) { + if identity, ok := resolve(ctx, start.ContainerID); ok && ctx.Err() == nil { + emit(ctx, Observation{Start: start, Identity: identity, Node: node}) + } + }, + } + return receiver.Run(ctx) +} diff --git a/pkg/gvisor/integration_linux_test.go b/pkg/gvisor/integration_linux_test.go new file mode 100644 index 0000000000..72d280dc53 --- /dev/null +++ b/pkg/gvisor/integration_linux_test.go @@ -0,0 +1,104 @@ +//go:build linux + +package gvisor + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "golang.org/x/sys/unix" +) + +func TestGVisorIntegrationVerifiesBeforeQueueAndExport(t *testing.T) { + for _, mode := range []string{"verified", "unknown", "disappeared", "canceled_lookup"} { + t.Run(mode, func(t *testing.T) { + dir := t.TempDir() + if err := os.Chmod(dir, 0700); err != nil { + t.Fatal(err) + } + cfg := IntegrationConfig{Enabled: true, SocketPath: filepath.Join(dir, "starts.sock")} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var calls atomic.Int32 + lookup := func(ctx context.Context, id string) (Identity, error) { + n := calls.Add(1) + if mode == "canceled_lookup" { + <-ctx.Done() + return Identity{}, ctx.Err() + } + if mode == "unknown" || (mode == "disappeared" && n > 1) { + return Identity{}, errors.New("unknown") + } + return Identity{ContainerID: id, SandboxID: "sandbox", Namespace: "apps", PodName: "pod", PodUID: "uid", ContainerName: "app", RuntimeHandler: "runsc"}, nil + } + emitted := make(chan Observation, 1) + done := make(chan error, 1) + go func() { + done <- RunIntegration(ctx, cfg, "node", lookup, func(_ context.Context, o Observation) { emitted <- o }) + }() + fd := connectReceiver(t, cfg.SocketPath) + defer unix.Close(fd) + sendPacket(t, fd, []byte{8, 1}) + if _, err := unix.Read(fd, make([]byte, 32)); err != nil { + t.Fatal(err) + } + sendPacket(t, fd, startFrame("known", "known", "SECRET_ARG_AND_CWD")) + deadline := time.Now().Add(time.Second) + for calls.Load() < 1 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if mode == "verified" { + select { + case got := <-emitted: + data, err := json.Marshal(got) + if err != nil || strings.Contains(string(data), "SECRET_ARG_AND_CWD") { + t.Fatal("sensitive fields retained in observation") + } + if got.Source != "gvisor_trace" || got.Identity.ContainerID != "known" || got.Node != "node" || calls.Load() != 2 { + t.Fatalf("incorrect verified observation: %+v", got) + } + case <-time.After(time.Second): + t.Fatal("verified start not emitted") + } + } else { + if mode == "disappeared" { + deadline := time.Now().Add(time.Second) + for calls.Load() < 2 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if calls.Load() != 2 { + t.Fatal("dispatch did not recheck identity") + } + } + time.Sleep(30 * time.Millisecond) + select { + case <-emitted: + t.Fatal("unresolved start exported") + default: + } + } + cancel() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("lookup/shutdown ignored cancellation") + } + }) + } +} + +func TestGVisorIntegrationDisabledDoesNoIO(t *testing.T) { + if err := RunIntegration(context.Background(), IntegrationConfig{}, "", nil, nil); err != nil { + t.Fatal(err) + } +} diff --git a/pkg/otelsetup/runtime_start.go b/pkg/otelsetup/runtime_start.go new file mode 100644 index 0000000000..6104461e0f --- /dev/null +++ b/pkg/otelsetup/runtime_start.go @@ -0,0 +1,36 @@ +package otelsetup + +import ( + "context" + + "github.com/kubescape/node-agent/pkg/gvisor" + "go.opentelemetry.io/otel/attribute" + otellog "go.opentelemetry.io/otel/log" +) + +// EmitRuntimeStart exports an observation through the existing OTEL log batch +// processor. It is intentionally not a SecurityAlert or a host eBPF event. +// No delivery acknowledgement is implied; OTEL's bounded queue can drop logs. +func EmitRuntimeStart(ctx context.Context, event gvisor.Observation) { + var record otellog.Record + record.SetTimestamp(event.ObservedAt) + record.SetObservedTimestamp(event.ObservedAt) + record.SetSeverity(otellog.SeverityInfo) + record.SetBody(attribute.StringValue("RuntimeObservation")) + record.AddAttributes( + attribute.String("source", "gvisor_trace"), + attribute.String("kind", "container_started"), + attribute.String("identity_status", "verified"), + attribute.String("identity_origin", "local_cri_and_kubernetes"), + attribute.String("node", event.Node), + attribute.String("container.id", event.Identity.ContainerID), + attribute.String("sandbox_id", event.Identity.SandboxID), + attribute.String("runtime_handler", event.Identity.RuntimeHandler), + attribute.String("namespace", event.Identity.Namespace), + attribute.String("pod_name", event.Identity.PodName), + attribute.String("pod_uid", event.Identity.PodUID), + attribute.String("container_name", event.Identity.ContainerName), + attribute.Int64("sender_reported_drops", int64(event.Dropped)), + ) + Logger().Emit(ctx, record) +} diff --git a/pkg/otelsetup/runtime_start_test.go b/pkg/otelsetup/runtime_start_test.go new file mode 100644 index 0000000000..73b6de2c48 --- /dev/null +++ b/pkg/otelsetup/runtime_start_test.go @@ -0,0 +1,44 @@ +package otelsetup + +import ( + "context" + "reflect" + "testing" + "time" + + "github.com/kubescape/node-agent/pkg/gvisor" + otellog "go.opentelemetry.io/otel/log" + "go.opentelemetry.io/otel/log/global" + "go.opentelemetry.io/otel/log/logtest" +) + +func TestGVisorObservationUsesDistinctLogContract(t *testing.T) { + recorder := logtest.NewRecorder() + previous := global.GetLoggerProvider() + global.SetLoggerProvider(recorder) + t.Cleanup(func() { global.SetLoggerProvider(previous) }) + when := time.Now() + EmitRuntimeStart(context.Background(), gvisor.Observation{ + Start: gvisor.Start{Source: "gvisor_trace", ContainerID: "cid", ObservedAt: when, Dropped: 3}, + Identity: gvisor.Identity{ContainerID: "cid", SandboxID: "sid", RuntimeHandler: "runsc", Namespace: "apps", PodName: "pod", PodUID: "uid", ContainerName: "app"}, Node: "node", + }) + var records []logtest.Record + for _, group := range recorder.Result() { + records = append(records, group...) + } + if len(records) != 1 { + t.Fatalf("got %d records", len(records)) + } + r := records[0] + if r.Body.AsString() != "RuntimeObservation" || r.Severity != otellog.SeverityInfo || !r.Timestamp.Equal(when) || !r.ObservedTimestamp.Equal(when) { + t.Fatal("wrong observation semantics") + } + got := map[string]any{} + for _, a := range r.Attributes { + got[string(a.Key)] = a.Value.AsInterface() + } + want := map[string]any{"source": "gvisor_trace", "kind": "container_started", "identity_status": "verified", "identity_origin": "local_cri_and_kubernetes", "node": "node", "container.id": "cid", "sandbox_id": "sid", "runtime_handler": "runsc", "namespace": "apps", "pod_name": "pod", "pod_uid": "uid", "container_name": "app", "sender_reported_drops": int64(3)} + if !reflect.DeepEqual(got, want) { + t.Fatalf("unexpected attributes: %#v", got) + } +} diff --git a/scripts/test_gvisor_integration_verifier.py b/scripts/test_gvisor_integration_verifier.py new file mode 100644 index 0000000000..52af5c7897 --- /dev/null +++ b/scripts/test_gvisor_integration_verifier.py @@ -0,0 +1,88 @@ +import copy +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +EXPERIMENT = ROOT / 'docs/experiments/gvisor-start-integration-20261102' +sys.path.insert(0, str(EXPERIMENT)) +spec = importlib.util.spec_from_file_location('integration_verifier', EXPERIMENT / 'verify.py') +verifier = importlib.util.module_from_spec(spec) +spec.loader.exec_module(verifier) +sys.path.pop(0) + + +class VerificationTests(unittest.TestCase): + def setUp(self): + self.rows = json.loads((EXPERIMENT / 'evidence/runtime-ledger.json').read_text()) + self.events = [json.loads(line) for line in (EXPERIMENT / 'evidence/observations.jsonl').read_text().splitlines()] + + def raw(self): + return b'\n'.join(json.dumps(event).encode() for event in self.events) + + def test_retained_evidence(self): + result = verifier.verify(self.rows, self.raw(), b'') + self.assertEqual(result, json.loads((EXPERIMENT / 'evidence/verification.json').read_text())) + + def test_missing_event(self): + self.events.pop() + with self.assertRaises(ValueError): + verifier.verify(self.rows, self.raw(), b'') + + def test_uid_mismatch(self): + self.events[0]['pod_uid'] = 'wrong' + with self.assertRaises(ValueError): + verifier.verify(self.rows, self.raw(), b'') + + def test_privacy_failures(self): + for field, value in [('wire_canary_found', True), ('unexpected_attribute_count', 1)]: + with self.subTest(field=field): + events = copy.deepcopy(self.events) + events[0][field] = value + raw = b'\n'.join(json.dumps(event).encode() for event in events) + with self.assertRaises(ValueError): + verifier.verify(self.rows, raw, b'') + with self.assertRaises(ValueError): + verifier.verify(self.rows, self.raw(), verifier.CANARY.encode()) + with self.assertRaises(ValueError): + verifier.verify(self.rows, self.raw() + verifier.CANARY.encode(), b'') + + def test_no_overlap(self): + parallel = [row for row in self.rows if row['name'].startswith('integration-parallel-')] + parallel[1]['started_at'] = parallel[0]['finished_at'] + with self.assertRaises(ValueError): + verifier.verify(self.rows, self.raw(), b'') + + def test_optimized_python_cannot_publish_failed_verification(self): + self.events.pop() + with tempfile.TemporaryDirectory() as folder: + base = Path(folder) + (base / 'rows.json').write_text(json.dumps(self.rows)) + (base / 'events.jsonl').write_bytes(self.raw()) + program = """ +import json,pathlib,sys +sys.path.insert(0,sys.argv[1]) +import verify +base=pathlib.Path(sys.argv[2]) +result=verify.verify(json.loads((base/'rows.json').read_text()),(base/'events.jsonl').read_bytes(),b'') +(base/'verification.json').write_text(json.dumps(result)) +""" + for optimized in (False, True): + env = os.environ.copy() + env.pop('PYTHONOPTIMIZE', None) + if optimized: + env['PYTHONOPTIMIZE'] = '1' + command = [sys.executable] + (['-O'] if optimized else []) + result = subprocess.run(command + ['-c', program, str(EXPERIMENT), str(base)], + env=env, capture_output=True, timeout=10) + self.assertNotEqual(result.returncode, 0) + self.assertFalse((base / 'verification.json').exists()) + + +if __name__ == '__main__': + unittest.main()