feat(solid): work on Linked Web Storage servers

.github/workflows/ci.yml

@@ -578,7 +578,7 @@ jobs:
       # the run; on main, each runs to its end, saving its images.
       fail-fast: ${{ github.ref != 'refs/heads/main' }}
       matrix:
-        server: [css-8, pivot, nextcloud, jss]
+        server: [css-8, pivot, nextcloud, jss, css-lws, css-lws-only]
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:

.github/workflows/interop.yml

@@ -6,11 +6,13 @@ name: Interop
 
 # The app's use cases and Solid adapters against the advisory Solid
 # servers (docs/testing.md), whose results are reported, not a gate: one
-# job per server, as CI runs the blocking ones. A job fails only on a test
-# failing that its servers/<id>/expected-failures.json does not list, or
-# when no test ran; the job's summary says what passed, what failed as
-# expected and why (by its cause on docs/interop.md), and what passes
-# though listed. A release commit,
+# job per server, as CI runs the blocking ones; and the user journeys in
+# Chromium against the servers with Linked Web Storage. A job fails only on a test
+# or journey failing that its server's expected-failures.json does not
+# list (e2e/pod/servers/<id>/ for the tests, e2e/journeys/servers/<id>/
+# for the journeys), or when none ran; the job's summary says what
+# passed, what failed as expected and why (by its cause on
+# docs/interop.md), and what passes though listed. A release commit,
 # whose push its paths match (package.json, package-lock.json), runs
 # nothing: it changes only the version's files, so its code is the
 # commit's before it, which Interop already ran on
@@ -21,6 +23,8 @@ on:
     branches: [main]
     paths:
       - e2e/pod/**
+      - e2e/journeys/**
+      - apps/**
       - packages/**
       - ns/**
       - decks/**
@@ -56,7 +60,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        server: [css-8, pivot, nextcloud, jss]
+        server: [css-8, pivot, nextcloud, jss, css-lws, css-lws-only]
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -97,3 +101,75 @@ jobs:
           name: pod-logs-${{ matrix.server }}
           path: e2e/pod/logs/
           retention-days: 7
+
+  # The user journeys in Chromium (docs/testing.md#user-journeys) against
+  # each server with Linked Web Storage, behind TLS as its signed-in
+  # instance, which the app signs in to with solid-memo-client: Inrupt's
+  # client gives no ID token to exchange for a storage's (docs/lws.md).
+  # The images come from the pod job's cache entries, which this job
+  # only restores. compare.ts fails the job on a journey failing that the
+  # server's e2e/journeys/servers/<id>/expected-failures.json does not
+  # list. Timing-dependent: end to end, on a shared runner.
+  journeys:
+    name: journeys (${{ matrix.server }})
+    if: >-
+      !startsWith(github.event.head_commit.message, 'chore(release): ')
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        server: [css-lws, css-lws-only]
+    env:
+      JOURNEY_SERVER: ${{ matrix.server }}
+      SOLID_CLIENT: solid-memo-client
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
+        with:
+          node-version-file: .nvmrc
+          cache: npm
+      - run: npm ci --ignore-scripts
+      - id: key
+        run: echo "key=$(node e2e/pod/servers.ts key ${{ matrix.server }})" >> "$GITHUB_OUTPUT"
+      - uses: ./.github/actions/e2e-images
+        with:
+          name: pod-${{ matrix.server }}
+          key: ${{ steps.key.outputs.key }}
+      - run: npm run css:prepare -w @solid-memo/e2e-journeys
+      # The app alone, without the docs' diagrams, which need Chromium
+      # and have nothing of the client.
+      - run: npx turbo run build --filter=@solid-memo/web
+        env:
+          DOCS_DIAGRAMS: skip
+      - id: playwright
+        run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
+      - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        id: browsers
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-${{ runner.os }}-${{ steps.playwright.outputs.version }}-chromium
+      - if: steps.browsers.outputs.cache-hit != 'true'
+        run: npx playwright install chromium
+      - uses: ./.github/actions/playwright-deps
+        with:
+          browser: chromium
+      # compare.ts says whether the run went as expected, not Playwright.
+      - id: journeys
+        run: npm run journeys -- --project=chromium
+        continue-on-error: true
+      - if: always()
+        run: node e2e/journeys/harness/summary.ts journeys-${{ matrix.server }}-${{ github.run_attempt }}
+      - run: node e2e/journeys/harness/compare.ts ${{ matrix.server }}
+      # Every failed journey's trace, expected or not.
+      - if: failure() || steps.journeys.outcome == 'failure'
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: journeys-${{ matrix.server }}-${{ github.run_attempt }}
+          path: |
+            e2e/journeys/playwright-report/
+            e2e/journeys/test-results/
+            e2e/journeys/logs/
+          retention-days: 7

cspell.config.yaml

@@ -161,6 +161,7 @@ words:
   - Gitea
   - igrant
   - Inrupt
+  - jeswr
   - ixuz
   - katex
   - linkify

docs/authentication.md

@@ -407,6 +407,13 @@ sent to such an address would name another server. The test is on the
 parsed address, so every spelling of a `//` at the start of the path is
 caught, and a path with one further along (`/a//b`) isn't refused here.
 
+On `solid-memo-client`, a storage that speaks Linked Web Storage gets a
+Bearer token the client exchanges the ID token for, sent within that
+storage only. The client answers its challenge, and sends the request
+again, before authFetch sees the response
+([solid-memo-client.md](solid-memo-client.md#linked-web-storage)).
+Inrupt's sign-in client gives the app no ID token short of ending the session, so it can't.
+
 When a request fails and wasn't aborted, authFetch asks the same
 address with a HEAD as no one, the redirect not followed:
 
@@ -439,6 +446,12 @@ session is gone. authFetch announces it
   `WWW-Authenticate`). The own storages are those the user's profile
   names, read as no one, and those entered on this page.
 
+On `solid-memo-client`, a 401 from a Linked Web Storage storage reaches
+authFetch only after the client tried to answer its challenge: a token
+exchange, and the request sent again. A storage's token the storage
+refuses is exchanged anew once first, so an expired one ends no
+session.
+
 A 401 from any other server, such as an instance the user registered
 in someone else's pod, says nothing of the session. The caller reads
 it as refused access, and the user stays logged in. A logout isn't an

docs/boundaries.md

@@ -88,8 +88,8 @@ boundaries."
 | `composition` | `application`, `domain`, `vocab`, `solid`, `browser` | |
 | `web` | `application`, `domain`, `vocab`, `ui`, `composition`, `studio`, `site` | `composition`, and `studio` with a dynamic `import()`: `src/main.tsx` only; `site`: `vite.config.ts` (the site's build, which renders the docs) and tests only |
 | `studio` | `application`, `domain`, `vocab`, `ui` | |
-| `e2e-pod` | `application`, `domain`, `vocab`, `solid` | |
-| `solid-memo-client` | `solid` | No package imports it: the build puts its modules in place of `solid`'s client modules ([below](#the-solid-client-libraries)) |
+| `e2e-pod` | `application`, `domain`, `vocab`, `solid`, `solid-memo-client` | `solid-memo-client`: `signedInClient.ts` only, which signs in to a Linked Web Storage server |
+| `solid-memo-client` | `solid` | No package the app is made of imports it: the build puts its modules in place of `solid`'s client modules ([below](#the-solid-client-libraries)) |
 | `e2e-journeys` | `ui`, `studio` | `ui` and `studio`: `harness/strings.ts` only, for their messages |
 
 Browser code is the `src/` of `markdown`, `vocab`, `domain`,

docs/data-model.md

@@ -221,7 +221,7 @@ flowchart LR
 
   Every blocking server the end-to-end tests start
   ([testing.md](testing.md#commands)), and Pivot and Community Solid
-  Server 8, keep another app's file and its folder, and delete a folder
+  Server 8 (with LWS too), keep another app's file and its folder, and delete a folder
   holding only Solid Memo's documents with its access rules
   ([foreignData.integration.test.ts](../e2e/pod/src/foreignData.integration.test.ts)).
   Solid-Nextcloud lists a folder's ACL among its contents, so there such

docs/docs.ttl

@@ -77,6 +77,13 @@
     schema:learningResourceType "reference" ;
     dcterms:abstract "the second Solid client, on no Inrupt library: what it replaces, switching to it with SOLID_CLIENT=solid-memo-client, how it signs in and keeps the session, the public-read grant, how it differs from Inrupt's, and its tests."@en .
 
+<lws.md>
+    a schema:TechArticle ;
+    schema:isPartOf <#the-code> ;
+    schema:position 46 ;
+    schema:learningResourceType "explanation" ;
+    dcterms:abstract "how the app works on a Linked Web Storage server: finding the storage, why it needs the Solid Protocol's requests too, signing in with a token exchange, and how it's tested."@en .
+
 <testing.md>
     a schema:TechArticle ;
     schema:isPartOf <#the-code> ;

docs/interop.md

@@ -8,25 +8,31 @@ it, with no Solid Memo involved.
 
 The tests and how they run are in [testing.md](testing.md).
 The Interop workflow runs them weekly and on changes to the app's Solid
-code. Each server's `e2e/pod/servers/<id>/expected-failures.json` names
-the tests a cause fails, by the cause's id on this page.
+code, and the user journeys on the servers with Linked Web Storage.
+Each server's `e2e/pod/servers/<id>/expected-failures.json` names the
+tests a cause fails, and its `e2e/journeys/servers/<id>/expected-failures.json`
+the journeys, by the cause's id on this page.
 
 A cause here is one that the tests hit and a live run reproduced. The
 servers are set up for testing, and each entry says when the setup
 plays a part. A server not on this page, or with no entries, passes
-every test.
+every test and journey.
 
 ## How to read an entry
 
 - **Requirement** is the strongest clause the behavior goes against,
   with its level: a MUST of the [Solid Protocol 0.11](https://solidproject.org/TR/protocol)
   or of [Web Access Control 1.0.0](https://solidproject.org/TR/wac), a
-  definition in an RDF, SPARQL or HTTP specification, or none. An
-  optional feature done wrong says so.
+  definition in an RDF, SPARQL or HTTP specification, or none. On a
+  server with Linked Web Storage, it may be a clause of
+  [LWS Protocol 1.0](https://www.w3.org/TR/lws10-core/). An optional
+  feature done wrong says so. So does something the app needs that LWS
+  doesn't define.
 - **Reproduction** uses `$POD` for the pod's root URL, ending in `/`.
   The test pods are open to anyone, so the requests carry no
   credentials, unless an entry says it's from the
-  [signed-in tests](testing.md#the-signed-in-tests). Headers and bodies are trimmed to what matters.
+  [signed-in tests](testing.md#the-signed-in-tests) or the
+  [user journeys](testing.md#user-journeys), which sign in. Headers and bodies are trimmed to what matters.
 - **Upstream** links the issue or pull request that tracks the cause,
   or says that none does yet.
 
@@ -543,3 +549,28 @@ The same PUT to `card.ttl` is served as sent.
 
 **Reported to the maintainers in private.** It gets an entry like the
 others once a fixed release is out.
+
+## Community Solid Server with Linked Web Storage
+
+`css-lws`: the Linked Web Storage branch of a fork of the Community
+Solid Server, not yet merged
+([jeswr/CommunitySolidServer#126](https://github.com/jeswr/CommunitySolidServer/pull/126)),
+at commit
+[`ef6d664f`](https://github.com/jeswr/CommunitySolidServer/tree/ef6d664f1253662cbfcb0f1786fe45d32849e067)
+(version 8.0.0-alpha.3). Its configuration is the fork's
+`config/solid-lws.json`: the Solid Protocol and LWS side by side, with
+Web Access Control. The pod tests run it in memory, its root open to
+anyone. The signed-in tests and the user journeys run its signed-in
+instance, behind TLS.
+
+No causes: every pod test and every journey passes.
+
+## Community Solid Server with Linked Web Storage alone
+
+`css-lws-only`: the same commit, with the fork's `config/lws.json`, LWS
+alone with Web Access Control. This configuration takes the Solid
+Protocol's requests too, so the app's requests are answered as on
+`css-lws`. The pod tests and the signed-in instance are run as for
+`css-lws`.
+
+No causes: every pod test and every journey passes.

docs/lws.md

@@ -0,0 +1,103 @@
+# Linked Web Storage
+
+Solid Memo is written for the Solid Protocol. The W3C's Linked Web
+Storage (LWS) Working Group is writing its successor,
+[LWS Protocol 1.0](https://www.w3.org/TR/lws10-core/), still a Working
+Draft. This page explains how the app works on a storage that speaks
+LWS, and why each part is made the way it is. The servers it's tested
+on are in [testing.md](testing.md), and what the tests find on them in
+[interop.md](interop.md).
+
+## What differs
+
+An app that works on a Solid pod asks a few things of it that LWS
+doesn't ask a server to give:
+
+| The app needs | Solid Protocol | LWS |
+|---|---|---|
+| The user's storage | `pim:storage` in the profile, or the folder typed `pim:Storage` | A `Link` with `rel="https://www.w3.org/ns/lws#storage"` on every answer, and the storage's description (`application/lws+cid`), which names its root container |
+| A folder's contents | Turtle, `ldp:contains` | `application/lws+json`, page by page |
+| A new document at a name the app chooses | PUT with `If-None-Match: *` | POST to its container, which names it; the name asked for is a hint |
+| An edit of a document | SPARQL Update or N3 Patch | JSON Patch, or the whole document again with PUT |
+| Sign-in | Solid-OIDC, a DPoP-bound access token | The OpenID Connect ID token exchanged for a Bearer token for each storage |
+| A release anyone can read | Web Access Control or Access Control Policies | Nothing yet |
+
+A server can speak both. The Community Solid Server with LWS
+(`css-lws`) does, and answers every request the app makes as a Solid
+pod does. So does its LWS-alone configuration (`css-lws-only`). The app
+doesn't work on a server that refuses the Solid Protocol's requests
+([why](#why-the-app-needs-solid-requests)).
+
+## Finding the storage
+
+The app reads `pim:storage` from the user's profile first. Without one,
+it walks from the WebID's document up its folders and asks each with a
+HEAD. A Solid folder typed `pim:Storage` is the storage. An LWS answer
+links its storage instead; the app reads that storage's description
+and takes the root container it names, if the WebID's document is in
+it ([lwsStorage.ts](../packages/solid/src/lwsStorage.ts),
+`solidStorageGateway.ts`).
+
+A deck's folder that's a storage's root holds the user's other data
+too, so the app claims none of its documents
+([deckPlace.ts](../packages/solid/src/deckPlace.ts)). On LWS, a folder
+is a root when the storage description names it as one: the storage
+link alone doesn't say so, since every resource in the storage carries
+it.
+
+**Why the app asks for Turtle first.** On the Community Solid Server
+with LWS, the storage's URI is its root container's too. Asked for
+anything, it answers with the storage's description, which has none of
+the Solid links: no `pim:Storage` type, no link to the ACL. Each HEAD
+that reads a resource's links asks for `text/turtle, */*;q=0.1`
+(`TURTLE_FIRST`, `client/linkHeader.ts`). A server that has Turtle
+answers with it, its Solid links kept. A server that lists a folder
+only as LWS has it answers too, where Turtle alone would get
+a 406.
+
+## Why the app needs Solid requests
+
+The app makes each request as the Solid Protocol has it, and doesn't
+translate it for a server that refuses them. Every server it's tested
+on takes them. Translating them would help only part of the way: an
+LWS server names a new resource itself, and the name a client asks for
+is only a hint, but every address in an instance is the app's to choose
+([data-model.md](data-model.md)). A catalog names its decks' documents
+by address, and a type index names the instance. LWS has an open issue
+for a creation at a name the client chooses
+([w3c/lws-protocol#173](https://github.com/w3c/lws-protocol/issues/173)).
+
+## Sign-in
+
+An LWS storage takes a Bearer token that its authorization server
+issues for the OpenID Connect ID token. The app's own Solid client
+([solid-memo-client.md](solid-memo-client.md#linked-web-storage)) does
+that exchange when a storage answers 401 with an LWS challenge, and
+keeps one token for each storage. Inrupt's sign-in client gives the
+app the ID token only in the address that ends the session, so a build that signs in to an LWS storage is made with
+`SOLID_CLIENT=solid-memo-client`.
+
+## How it's tested
+
+[The signed-in tests](testing.md#the-signed-in-tests) sign in to an LWS
+storage without a browser. [The user journeys](testing.md#user-journeys)
+sign in with one, in Chromium, on both servers. `JOURNEY_SERVER=css-lws`
+or `css-lws-only` runs them against that server's signed-in instance:
+the fork's own configuration behind TLS, with a storage for each pod.
+The app is built with `SOLID_CLIENT=solid-memo-client`. The Interop
+workflow runs them.
+
+Some journeys read or write the pod themselves, as another app or to
+check what the app left. There the harness does what the app does: it
+logs the account in on the authorization-code flow and exchanges its ID
+token for the storage's token (`harness/podToken.ts`). The Solid-OIDC
+token that client credentials give is no LWS storage's.
+
+**Why the harness reads the server's keys first.** The fork makes the
+key it signs tokens with the first time it needs one. On a newly started
+server, several token exchanges at once can each make a key, and the
+server keeps only one of them. A token signed with any other key is then
+refused ("signature verification failed"). So the harness reads the
+server's keys (`/.well-known/lws/jwks`) once before the journeys start.
+That read makes the key, and every exchange after it uses the same one.
+The fix belongs in the fork.