diff --git a/.gitignore b/.gitignore index f130b04..6bd8076 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,24 @@ +# Build outputs bin/ -/logs/ -__pycache__/ -*.py[cod] +dist/ +web-interface/dist/ +docs/public/ +docs/resources/ +*.hugo_build.lock + +# Dependencies +node_modules/ + +# Runtime data & local environment data/ +/logs/ .direnv/ .fake-backend-config.json +*.local + +# Python caches +__pycache__/ +*.py[cod] # OS files -.DS_Store \ No newline at end of file +.DS_Store diff --git a/README.md b/README.md index ad6502a..b34c67a 100644 --- a/README.md +++ b/README.md @@ -23,17 +23,17 @@ administrator SSH access. Kubernetes/containerd run workload containers. Start with the [documentation index](docs/README.md). -- [Using the portal, teams and jobs](docs/department-rollout.md) -- [k3s setup, accelerator allocation and diagnostics](docs/k3s-setup.md) -- [Tailscale and SSH setup](docs/tailscale-ssh.md) -- [Architecture and code organization](docs/architecture.md) -- [Storage and upload limits](docs/storage.md) +- [Using the portal, teams and jobs](docs/content/department-rollout.md) +- [k3s setup, accelerator allocation and diagnostics](docs/content/k3s-setup.md) +- [Tailscale and SSH setup](docs/content/tailscale-ssh.md) +- [Architecture and code organization](docs/content/architecture.md) +- [Storage and upload limits](docs/content/storage.md) - [Deployment and operations](deploy/private/README.md) -- [Developer setup](docs/development.md), [CLI](cli/docs/setup.md), [testing](docs/testing.md) +- [Developer setup](docs/content/development.md), [CLI](cli/docs/setup.md), [testing](docs/content/testing.md) Main contains the department release merged through [PR #110](https://github.com/UTMIST/Mist/pull/110). The earlier main and operating -snapshots are preserved in the [historical archive](docs/archive/README.md). +snapshots are preserved in the [historical archive](docs/content/archive/_index.md). ## Development and checks @@ -49,7 +49,7 @@ npm --prefix web-interface run build ``` For a local UI against the authenticated deployed API, follow -[developer setup](docs/development.md). For the existing installation, deploy with +[developer setup](docs/content/development.md). For the existing installation, deploy with `bash deploy/private/build-and-deploy.sh`; read its [prerequisites](deploy/private/README.md) before running it. Fresh-host provisioning is covered separately by the k3s guide. diff --git a/cli/docs/setup.md b/cli/docs/setup.md index 2377fda..789a050 100644 --- a/cli/docs/setup.md +++ b/cli/docs/setup.md @@ -65,4 +65,4 @@ To cancel that job while waiting/running, use `bin/mist-cli job cancel "$JOB_ID" read historical jobs; new work requires a team. `--scope common` requires common write permission. Custom tagged public images are allowed by team registry policy; TT custom images use `--tt-runtime container`. See the -[department guide](../../docs/department-rollout.md) for limits and storage. +[department guide](../../docs/content/department-rollout.md) for limits and storage. diff --git a/deploy/k3s/README.md b/deploy/k3s/README.md index cb60407..55ceeb1 100644 --- a/deploy/k3s/README.md +++ b/deploy/k3s/README.md @@ -1,7 +1,7 @@ # Kubernetes deployment assets Current installation: two k3s nodes, authenticated team jobs, scoped NFS storage -and private Tailscale portal. Start with the [k3s setup/handoff](../../docs/k3s-setup.md) +and private Tailscale portal. Start with the [k3s setup/handoff](../../docs/content/k3s-setup.md) and [deployment operations](../private/README.md). ## Manifest and tooling map @@ -29,6 +29,6 @@ The application image build uses [Dockerfile.api](Dockerfile.api). API/auth imag are imported into main's k3s containerd, not scheduled by a Docker supervisor. Use `bash deploy/private/build-and-deploy.sh` for the existing installation. -[Old deployment history](../../docs/archive/k3s-deployment-history.md) and -[original node setup](../../docs/archive/k3s-setup-history.md) preserve earlier +[Old deployment history](../../docs/content/archive/k3s-deployment-history.md) and +[original node setup](../../docs/content/archive/k3s-setup-history.md) preserve earlier observations; their unauthenticated/node-local instructions are superseded. diff --git a/deploy/k3s/evidence/department-2026-10-09/README.md b/deploy/k3s/evidence/department-2026-10-09/README.md index 9766550..a9596b2 100644 --- a/deploy/k3s/evidence/department-2026-10-09/README.md +++ b/deploy/k3s/evidence/department-2026-10-09/README.md @@ -2,7 +2,7 @@ Evidence from the actual two-node private pilot before repository cleanup. The original Go/Docker/logger counts below belong to that earlier release; -[current cleanup verification](../../../../docs/repository-cleanup.md) records +[current cleanup verification](../../../../docs/content/repository-cleanup.md) records the maintained suites after their obsolete implementations were removed. Reports contain synthetic verification data and job/device IDs; no passwords, cookies or auth secrets. diff --git a/deploy/k3s/setup-runbook.md b/deploy/k3s/setup-runbook.md index 5540d11..643eb3a 100644 --- a/deploy/k3s/setup-runbook.md +++ b/deploy/k3s/setup-runbook.md @@ -1,9 +1,9 @@ # Current k3s setup runbook -The maintained two-node runbook is [k3s setup and accelerator handoff](../../docs/k3s-setup.md). +The maintained two-node runbook is [k3s setup and accelerator handoff](../../docs/content/k3s-setup.md). It covers current versions, LAN/Tailscale roles, fresh server/worker boundaries, NVIDIA/TT operators, diagnostics and troubleshooting. -- [Tailscale and administrator SSH](../../docs/tailscale-ssh.md) +- [Tailscale and administrator SSH](../../docs/content/tailscale-ssh.md) - [Application deployment and service operations](../private/README.md) -- [Original single-node installation and worker-join history](../../docs/archive/k3s-setup-history.md) +- [Original single-node installation and worker-join history](../../docs/content/archive/k3s-setup-history.md) diff --git a/deploy/private/README.md b/deploy/private/README.md index 4d9ff64..5a8fe9e 100644 --- a/deploy/private/README.md +++ b/deploy/private/README.md @@ -1,9 +1,9 @@ # Private deployment operations -Current user/admin and architecture guide: [department rollout](../../docs/department-rollout.md). -Execution/evidence: [rollout checklist](../../docs/department-rollout-execution.md). -Setup: [k3s](../../docs/k3s-setup.md), [Tailscale/SSH](../../docs/tailscale-ssh.md), -[storage](../../docs/storage.md), and [developer setup](../../docs/development.md). +Current user/admin and architecture guide: [department rollout](../../docs/content/department-rollout.md). +Execution/evidence: [rollout checklist](../../docs/content/department-rollout-execution.md). +Setup: [k3s](../../docs/content/k3s-setup.md), [Tailscale/SSH](../../docs/content/tailscale-ssh.md), +[storage](../../docs/content/storage.md), and [developer setup](../../docs/content/development.md). ## Services and persistent state diff --git a/docs/README.md b/docs/README.md index ce6536a..2cad2d2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,43 +1,40 @@ -# Documentation +# Mist Documentation -Current main-branch documentation, verified against the two running machines on -**October 9, 2026**. Addresses and versions describe this installation; adjust -host-specific settings when provisioning another environment. +This directory contains the documentation site for Mist, powered by **Hugo** and the **Hextra** documentation theme. -## Find the right guide +## Viewing Documentation Locally -| Task | Guide | -|---|---| -| Sign in, manage teams, upload data, submit/download jobs | [Department operating guide](department-rollout.md) | -| Understand services and source layout | [Architecture](architecture.md) | -| Install/inspect k3s, join a worker, verify accelerator allocation | [k3s setup](k3s-setup.md) | -| Connect privately and administer either host | [Tailscale and SSH](tailscale-ssh.md) | -| Understand dataset/model pools and physical capacity | [Storage](storage.md) | -| Build, deploy, restart or troubleshoot services | [Deployment operations](../deploy/private/README.md) | -| Develop locally with the existing authenticated API | [Developer setup](development.md) | -| Authenticate and submit from a terminal | [CLI](../cli/docs/setup.md) | -| Run automated checks and read recorded live evidence | [Testing](testing.md) | -| Maintain UI components and interactions | [Design system](design-system.md) | -| Understand frontend architecture and backend offloading | [Frontend architecture](frontend-architecture.md) | -| Read completed scope, exclusions and release verification | [Execution record](department-rollout-execution.md) | -| See what repository cleanup removed and verified | [Cleanup record](repository-cleanup.md) | - -## Access and execution - -```text -Browser / CLI → Tailscale portal :8088 → Nginx → Go API → k3s - | | - Better Auth + team policy - | - CPU/NVIDIA or Tenstorrent Job - | - scoped dataset/model NFS mounts on QuietBox +To preview the documentation locally with live-reload: + +```bash +cd docs +hugo server +``` + +The site will be available at `http://localhost:1313/docs/`. + +To build the static site (outputting to `web-interface/public/docs` for the web portal): + +```bash +cd docs +hugo ``` -Mist login is separate from Tailscale membership and Linux SSH login. Researchers -use the portal; administrator SSH is restricted by the installed host rules. -The worker reaches the control plane and storage through LAN addresses. +## Documentation Guides -[Earlier plans and single-owner guides](archive/README.md) are historical only. -Current guidance does not require Redis, an SSH session to launch a job, a public -cloud load balancer, or permission to approve every tagged image. +Documentation source files are located in `content/`: + +| Guide | Description | +|---|---| +| [Department operating guide](content/department-rollout.md) | Sign in, manage teams, upload data, submit/download jobs | +| [Architecture](content/architecture.md) | Service topologies, boundaries, and codebase layout | +| [k3s setup](content/k3s-setup.md) | Cluster setup, worker nodes, and hardware acceleration | +| [Tailscale and SSH](content/tailscale-ssh.md) | Remote secure networking and administrator SSH | +| [Storage](content/storage.md) | Dataset and model shared storage pools | +| [Developer setup](content/development.md) | Local frontend and backend development | +| [Testing](content/testing.md) | Automated tests and evidence runs | +| [Design system](content/design-system.md) | Portal design principles and components | +| [Frontend architecture](content/frontend-architecture.md) | UI layout and backend offloading plan | +| [Execution record](content/department-rollout-execution.md) | Department rollout status and validation | +| [Cleanup record](content/repository-cleanup.md) | Repository maintenance history | +| [Historical archive](content/archive/_index.md) | Archived previous-generation guides and plans | diff --git a/docs/content/_index.md b/docs/content/_index.md new file mode 100644 index 0000000..c11a6df --- /dev/null +++ b/docs/content/_index.md @@ -0,0 +1,51 @@ +--- +type: docs +cascade: + type: docs +title: "Documentation Overview" +weight: 1 +--- + +# UTMIST Compute Platform Documentation + +Current main-branch documentation, verified against the two running machines on +**October 9, 2026**. Addresses and versions describe this installation; adjust +host-specific settings when provisioning another environment. + +## Find the right guide + +| Task | Guide | +|---|---| +| Sign in, manage teams, upload data, submit/download jobs | [Department operating guide](department-rollout.md) | +| Understand services and source layout | [Architecture](architecture.md) | +| Install/inspect k3s, join a worker, verify accelerator allocation | [k3s setup](k3s-setup.md) | +| Connect privately and administer either host | [Tailscale and SSH](tailscale-ssh.md) | +| Understand dataset/model pools and physical capacity | [Storage](storage.md) | +| Build, deploy, restart or troubleshoot services | [Deployment operations](../../deploy/private/README.md) | +| Develop locally with the existing authenticated API | [Developer setup](development.md) | +| Authenticate and submit from a terminal | [CLI](../../cli/docs/setup.md) | +| Run automated checks and read recorded live evidence | [Testing](testing.md) | +| Maintain UI components and interactions | [Design system](design-system.md) | +| Understand frontend architecture and backend offloading | [Frontend architecture](frontend-architecture.md) | +| Read completed scope, exclusions and release verification | [Execution record](department-rollout-execution.md) | +| See what repository cleanup removed and verified | [Cleanup record](repository-cleanup.md) | + +## Access and execution + +```text +Browser / CLI → Tailscale portal :8088 → Nginx → Go API → k3s + | | + Better Auth + team policy + | + CPU/NVIDIA or Tenstorrent Job + | + scoped dataset/model NFS mounts on QuietBox +``` + +Mist login is separate from Tailscale membership and Linux SSH login. Researchers +use the portal; administrator SSH is restricted by the installed host rules. +The worker reaches the control plane and storage through LAN addresses. + +[Earlier plans and single-owner guides](archive/_index.md) are historical only. +Current guidance does not require Redis, an SSH session to launch a job, a public +cloud load balancer, or permission to approve every tagged image. diff --git a/docs/architecture.md b/docs/content/architecture.md similarity index 98% rename from docs/architecture.md rename to docs/content/architecture.md index 7ce1f2d..8a3a755 100644 --- a/docs/architecture.md +++ b/docs/content/architecture.md @@ -1,3 +1,8 @@ +--- +title: "Architecture & Services" +weight: 20 +--- + # Architecture Current private department installation, verified October 9, 2026. diff --git a/docs/archive/README.md b/docs/content/archive/_index.md similarity index 89% rename from docs/archive/README.md rename to docs/content/archive/_index.md index 8f98a75..7c8bcdf 100644 --- a/docs/archive/README.md +++ b/docs/content/archive/_index.md @@ -1,3 +1,10 @@ +--- +title: "Historical Records" +weight: 200 +sidebar: + open: false +--- + # Historical records These files preserve earlier design decisions and verification phases. They are @@ -18,4 +25,4 @@ The old main is preserved on [`archive/main-before-k3s-20261009`](https://github.com/UTMIST/Mist/tree/archive/main-before-k3s-20261009). The pre-cleanup department release is commit [`3431ffd`](https://github.com/UTMIST/Mist/tree/3431ffdb713fac72881fc7f61f2750a82fe3f916). -Use the [current documentation](../README.md) for this installation. +Use the [current documentation](../_index.md) for this installation. diff --git a/docs/archive/complete-foundation.md b/docs/content/archive/complete-foundation.md similarity index 98% rename from docs/archive/complete-foundation.md rename to docs/content/archive/complete-foundation.md index f1d8875..fb0aa37 100644 --- a/docs/archive/complete-foundation.md +++ b/docs/content/archive/complete-foundation.md @@ -1,6 +1,6 @@ > **Historical record — superseded.** This text describes an earlier phase. > Commands and claims below are retained as history, not current setup instructions. -> Start with the [current documentation index](../README.md) and +> Start with the [current documentation index](../_index.md) and > [department operating guide](../department-rollout.md). > Current department storage and portal settings: [department rollout](../department-rollout.md). @@ -11,7 +11,7 @@ > **Historical account-only snapshot.** The [department rollout](../department-rollout.md) > supersedes these instructions: team membership, scoped folders, enforced quotas, > self-service images and the new portal are implemented. Use that guide and -> [current deployment operations](../../deploy/private/README.md) for current behavior. +> [current deployment operations](../../../deploy/private/README.md) for current behavior. > The remaining sections record the preceding foundation, including its limitations. Implemented on `feat/local-job-foundation`, October 8–9, 2026. This continues the @@ -196,7 +196,7 @@ inside `/outputs` if you want them to be listed/downloaded. ## Deployment, restart and rollback -See [deployment scripts](../../deploy/private/README.md). k3s server/agent, NFS and +See [deployment scripts](../../../deploy/private/README.md). k3s server/agent, NFS and Nginx are system services. Kubernetes restores API/auth deployments. The frontend is a static release, with `/srv/mist-web/current` pointing at the active build. Deployment configuration, RBAC, claim templates and image policies are checked diff --git a/docs/archive/foundation-execution-plan.md b/docs/content/archive/foundation-execution-plan.md similarity index 99% rename from docs/archive/foundation-execution-plan.md rename to docs/content/archive/foundation-execution-plan.md index aeceef5..48eef71 100644 --- a/docs/archive/foundation-execution-plan.md +++ b/docs/content/archive/foundation-execution-plan.md @@ -1,6 +1,6 @@ > **Historical record — superseded.** This text describes an earlier phase. > Commands and claims below are retained as history, not current setup instructions. -> Start with the [current documentation index](../README.md) and +> Start with the [current documentation index](../_index.md) and > [department operating guide](../department-rollout.md). # Mist foundation execution plan diff --git a/docs/archive/k3s-deployment-history.md b/docs/content/archive/k3s-deployment-history.md similarity index 95% rename from docs/archive/k3s-deployment-history.md rename to docs/content/archive/k3s-deployment-history.md index 4256a27..9f6cd55 100644 --- a/docs/archive/k3s-deployment-history.md +++ b/docs/content/archive/k3s-deployment-history.md @@ -1,6 +1,6 @@ > **Historical record — superseded.** This text describes an earlier phase. > Commands and claims below are retained as history, not current setup instructions. -> Start with the [current documentation index](../README.md) and +> Start with the [current documentation index](../_index.md) and > [department operating guide](../department-rollout.md). # Accelerator jobs on k3s @@ -16,7 +16,7 @@ join on 2026-10-03. The NVIDIA server runs Ubuntu 22.04, k3s 1.36, and two RTX A4000 GPUs. The host driver is pre-installed. NVIDIA GPU Operator v26.7.1 is configured with -[`nvidia-gpu-operator-values.yaml`](../../deploy/k3s/nvidia-gpu-operator-values.yaml). Pin the +[`nvidia-gpu-operator-values.yaml`](../../../deploy/k3s/nvidia-gpu-operator-values.yaml). Pin the chart version when installing or upgrading: ```bash @@ -64,7 +64,7 @@ Wormhole chips. The NVIDIA smoke Job remains specific to `utmist-z1opa08`. TT-Operator **0.3.0** installs Fabric Manager **0.2.30** and the DRA driver **0.0.59**, configured by -[`tenstorrent-operator-values.yaml`](../../deploy/k3s/tenstorrent-operator-values.yaml). +[`tenstorrent-operator-values.yaml`](../../../deploy/k3s/tenstorrent-operator-values.yaml). The existing host KMD/firmware and the existing NFD installation are retained; driver management, duplicate NFD, telemetry, JobSet, and PMIx are disabled. The manually verified `tenstorrent.com/has-tt=true` node label enables discovery. @@ -96,14 +96,14 @@ a ResourceClaimTemplate, and only its allocated device nodes are injected through CDI. Boards are released when the pod finishes, even while its completed Job and checkpoints are retained. -[`training-tenstorrent-job.yaml`](../../deploy/k3s/training-tenstorrent-job.yaml) now requests +[`training-tenstorrent-job.yaml`](../../../deploy/k3s/training-tenstorrent-job.yaml) now requests one board, runs without privileged access, and reserves 2 GiB of 1 GiB hugepages. The host still has 16 GiB of such hugepages. It mounts the existing TT-Metal checkout and Python environment read-only, so it is specific to this QuietBox rather than a portable training image. For uniquely named training checks and separate checkpoint directories, -use [`submit_tenstorrent_smoke.py`](../../deploy/k3s/submit_tenstorrent_smoke.py). It needs +use [`submit_tenstorrent_smoke.py`](../../../deploy/k3s/submit_tenstorrent_smoke.py). It needs Python PyYAML, available on the server. For example: ```bash @@ -161,7 +161,7 @@ the temporary PVC reader is deleted. The barrier times out after five minutes if interrupted. The checkpoint PVC is retained. Operator restart recovery and an NVIDIA CUDA regression check also passed. -See [`tenstorrent-allocation-results.json`](../../deploy/k3s/tenstorrent-allocation-results.json) +See [`tenstorrent-allocation-results.json`](../../../deploy/k3s/tenstorrent-allocation-results.json) for pinned image IDs and results. Logs, claim/queue snapshots, admission checks, and checkpoint archives are in `/home/utmist/k3s-allocation-results-2026-10-03` on the NVIDIA server. @@ -171,7 +171,7 @@ capacity for placement. Verify/correct it before adding memory-based selectors. ## Training smoke jobs -[`training_smoke.py`](../../deploy/k3s/training_smoke.py) learns a 32-input/32-output linear +[`training_smoke.py`](../../../deploy/k3s/training_smoke.py) learns a 32-input/32-output linear mapping from 128 synthetic samples and evaluates 64 held-out samples. Each device must complete 100 gradient updates, reduce training MSE by at least 98%, achieve held-out MSE below `0.0005`, change its weights, and reload its @@ -220,7 +220,7 @@ checkpoints. The archives were also checked after the Pods completed. Precision and implementations differ; these figures establish convergence and are not a performance comparison. Exact image IDs, the TT-Metal host commit, Job timestamps, artifact locations, and acceptance thresholds are in -[`training-smoke-results.json`](../../deploy/k3s/training-smoke-results.json). Local logs and +[`training-smoke-results.json`](../../../deploy/k3s/training-smoke-results.json). Local logs and checkpoint archives are retained under `/home/utmist/k3s-training-results-2026-10-03` on the NVIDIA server. The completed Kubernetes Jobs and their checkpoint PVCs are retained too. @@ -292,7 +292,7 @@ entrypoint/arguments, or choose **Small training check** for an accelerator. The page displays actual placement, queue messages, exit status and logs. Hardware availability is live; the Machines route uses the same inventory. Cancel stops a waiting/running workload and retains cancellation history. -CLI commands are in [the CLI guide](../../cli/docs/setup.md). +CLI commands are in [the CLI guide](../../../cli/docs/setup.md). The installed user units keep both local services running during the user's systemd session. Their paths target this server and need adjustment on another @@ -382,7 +382,7 @@ read by fresh containers on both nodes. Temporary read-only output probes are removed; test Jobs and result directories remain. The final markers are `FOUNDATION_BROWSER_PASSED` and `FOUNDATION_ALLOCATION_OUTPUTS_PASSED`. October 8 evidence is under `/home/utmist/mist-foundation-results-2026-10-08`. -See [the checked-in summary](../../deploy/k3s/foundation-results.json) for actual job IDs, +See [the checked-in summary](../../../deploy/k3s/foundation-results.json) for actual job IDs, deployed image ID, and completed checks. ### API acceptance test @@ -402,7 +402,7 @@ Successful output ends with `MIST_API_PASSED`. Jobs and checkpoint directories are retained. A barrier times out after five minutes if interrupted. The verification script uses the operator's administrative context to inspect claims/devices and release test barriers; the Mist API itself cannot exec. -Results from 2026-10-03 are in [mist-api-results.json](../../deploy/k3s/mist-api-results.json), +Results from 2026-10-03 are in [mist-api-results.json](../../../deploy/k3s/mist-api-results.json), with raw evidence and checkpoint archives under `/home/utmist/mist-api-results-2026-10-03`. These checks establish small model training and allocation, not large/distributed-model support or performance. diff --git a/docs/archive/k3s-setup-history.md b/docs/content/archive/k3s-setup-history.md similarity index 98% rename from docs/archive/k3s-setup-history.md rename to docs/content/archive/k3s-setup-history.md index 7b8b3af..d32e55d 100644 --- a/docs/archive/k3s-setup-history.md +++ b/docs/content/archive/k3s-setup-history.md @@ -1,6 +1,6 @@ > **Historical record — superseded.** This text describes an earlier phase. > Commands and claims below are retained as history, not current setup instructions. -> Start with the [current documentation index](../README.md) and +> Start with the [current documentation index](../_index.md) and > [department operating guide](../department-rollout.md). # Single-node k3s and NVIDIA GPU pilot: setup and handoff @@ -28,7 +28,7 @@ The NVIDIA GPU Operator added the Kubernetes integration: The operator was configured **not** to replace the existing host driver. Those values are in -[`nvidia-gpu-operator-values.yaml`](../../deploy/k3s/nvidia-gpu-operator-values.yaml). NVIDIA +[`nvidia-gpu-operator-values.yaml`](../../../deploy/k3s/nvidia-gpu-operator-values.yaml). NVIDIA states that its NRI plugin is not yet generally available; this is a pilot configuration. See the [GPU Operator installation guide](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) and [CDI/NRI guide](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/cdi.html). @@ -125,7 +125,7 @@ healthy. ### 4. Execute CUDA through k3s -The tracked [`gpu-smoke-job.yaml`](../../deploy/k3s/gpu-smoke-job.yaml) requests and limits one +The tracked [`gpu-smoke-job.yaml`](../../../deploy/k3s/gpu-smoke-job.yaml) requests and limits one `nvidia.com/gpu` device. Create the namespace if it does not already exist: ```bash diff --git a/docs/archive/kubernetes-pilot.md b/docs/content/archive/kubernetes-pilot.md similarity index 97% rename from docs/archive/kubernetes-pilot.md rename to docs/content/archive/kubernetes-pilot.md index eab19dd..8e5230a 100644 --- a/docs/archive/kubernetes-pilot.md +++ b/docs/content/archive/kubernetes-pilot.md @@ -1,6 +1,6 @@ > **Historical record — superseded.** This text describes an earlier phase. > Commands and claims below are retained as history, not current setup instructions. -> Start with the [current documentation index](../README.md) and +> Start with the [current documentation index](../_index.md) and > [department operating guide](../department-rollout.md). # Kubernetes execution pilot @@ -38,7 +38,7 @@ flowchart LR ``` See [installation, commands and acceptance tests](k3s-deployment-history.md#mist-api-cli-and-jobs-page) -and [job lifecycle and API endpoints](../../src/jobs.md). +and [job lifecycle and API endpoints](../../../src/jobs.md). ## Hardware and scheduling @@ -88,7 +88,7 @@ its allocated board while device cgroups denied the other three, and that allocation recovered after restarting the DRA driver. CLI and actual browser checks exercise submission, status, logs and cancellation; backend/CLI/web tests and a frontend production build cover the application changes. -See [recorded results](../../deploy/k3s/mist-api-results.json) for evidence paths. +See [recorded results](../../../deploy/k3s/mist-api-results.json) for evidence paths. ## Access and retention diff --git a/docs/archive/local-job-foundation.md b/docs/content/archive/local-job-foundation.md similarity index 98% rename from docs/archive/local-job-foundation.md rename to docs/content/archive/local-job-foundation.md index 1e09e53..12e48d7 100644 --- a/docs/archive/local-job-foundation.md +++ b/docs/content/archive/local-job-foundation.md @@ -1,6 +1,6 @@ > **Historical record — superseded.** This text describes an earlier phase. > Commands and claims below are retained as history, not current setup instructions. -> Start with the [current documentation index](../README.md) and +> Start with the [current documentation index](../_index.md) and > [department operating guide](../department-rollout.md). # Local job foundation @@ -87,7 +87,7 @@ autograd compatibility. Cross-machine/distributed training is separate work. ## API contracts -The [job lifecycle reference](../../src/jobs.md) describes state and retention. +The [job lifecycle reference](../../../src/jobs.md) describes state and retention. The browser uses `/api`; Vite proxies it to the API. The API itself has these unprefixed routes: @@ -306,7 +306,7 @@ Their final markers are `FOUNDATION_BROWSER_PASSED` and Today's detailed evidence is saved under `/home/utmist/mist-foundation-results-2026-10-08`. The checked-in -[verification summary](../../deploy/k3s/foundation-results.json) records actual +[verification summary](../../../deploy/k3s/foundation-results.json) records actual job IDs, the deployed image ID, and completed checks. Browser and allocation/ persistence checks passed on October 8 EDT (October 9 UTC). A separate live check verified image-pull error reporting, deadline failure, and retained diff --git a/docs/archive/shared-storage-deployment-plan.md b/docs/content/archive/shared-storage-deployment-plan.md similarity index 99% rename from docs/archive/shared-storage-deployment-plan.md rename to docs/content/archive/shared-storage-deployment-plan.md index 2233be7..385621c 100644 --- a/docs/archive/shared-storage-deployment-plan.md +++ b/docs/content/archive/shared-storage-deployment-plan.md @@ -1,6 +1,6 @@ > **Historical record — superseded.** This text describes an earlier phase. > Commands and claims below are retained as history, not current setup instructions. -> Start with the [current documentation index](../README.md) and +> Start with the [current documentation index](../_index.md) and > [department operating guide](../department-rollout.md). # Complete foundation: parts 4 and 5 diff --git a/docs/complete-foundation.md b/docs/content/complete-foundation.md similarity index 66% rename from docs/complete-foundation.md rename to docs/content/complete-foundation.md index d2ad790..f8009cf 100644 --- a/docs/complete-foundation.md +++ b/docs/content/complete-foundation.md @@ -1,7 +1,12 @@ +--- +title: "Complete Foundation (Historical)" +weight: 301 +--- + # Historical guide This phase has been superseded by the department platform on main. -- [Current documentation](README.md) +- [Current documentation](_index.md) - [Current operating guide](department-rollout.md) - [Archived original record](archive/complete-foundation.md) diff --git a/docs/department-rollout-execution.md b/docs/content/department-rollout-execution.md similarity index 97% rename from docs/department-rollout-execution.md rename to docs/content/department-rollout-execution.md index 7e532b0..acfec62 100644 --- a/docs/department-rollout-execution.md +++ b/docs/content/department-rollout-execution.md @@ -1,3 +1,8 @@ +--- +title: "Rollout Execution Record" +weight: 100 +--- + # Department rollout execution Authorized October 9, 2026: implement all seven agreed remaining parts on @@ -62,7 +67,7 @@ behavior; they do not add a separate monitoring service. Recorded October 9 on the private production portal. Reports and selected logs are checked in under -[department evidence](../deploy/k3s/evidence/department-2026-10-09/README.md). +[department evidence](../../deploy/k3s/evidence/department-2026-10-09/README.md). - 14 API/cluster/storage checks: membership, write denial, scoped read-only grants, running-job revocation, FIFO/fair turns, API restart, cancellation, limits, @@ -170,5 +175,5 @@ requires an independent reviewer for subsequent pull requests. The live private portal already runs the verified implementation. Git publication does not reset accounts, jobs or files, or switch the live services again. -See [the final release record](../deploy/k3s/evidence/department-2026-10-09/release-validation.json) +See [the final release record](../../deploy/k3s/evidence/department-2026-10-09/release-validation.json) and [testing instructions](testing.md). diff --git a/docs/department-rollout.md b/docs/content/department-rollout.md similarity index 97% rename from docs/department-rollout.md rename to docs/content/department-rollout.md index d20899e..7d13333 100644 --- a/docs/department-rollout.md +++ b/docs/content/department-rollout.md @@ -1,9 +1,14 @@ +--- +title: "Department Operating Guide" +weight: 10 +--- + # Mist department rollout Merged into **main** through [PR #110](https://github.com/UTMIST/Mist/pull/110) on October 9, 2026. This is the current operating guide. Original accounts, jobs, datasets and checkpoint PVCs are preserved; earlier instructions are in the -[historical archive](archive/README.md). +[historical archive](archive/_index.md). ## Use the portal @@ -255,7 +260,7 @@ A regular-member UI demonstration submitted a custom image containing `/app/train.py` and `CMD ["python", "/app/train.py"]`. No script or command override was injected. Its two-layer PyTorch network trained for 200 steps on one RTX A4000 using `cuda:0`, reduced loss from 14.088 to 0.004936, saved weights/metrics and -verified checkpoint reload. [Container mode evidence](../deploy/k3s/evidence/department-2026-10-09/container-mode-result.json). +verified checkpoint reload. [Container mode evidence](../../deploy/k3s/evidence/department-2026-10-09/container-mode-result.json). The demonstration image was preloaded on the main node; it was not pushed to a public registry. Normal team workflows build/push to an allowed registry. @@ -366,7 +371,7 @@ errors for rejected requests. The main-node build command is `bash deploy/private/build-and-deploy.sh`. It preserves bootstrap/internal auth credentials, builds the API/CLI and imports API/auth images, applies policy/RBAC manifests, waits for rollouts, type-checks and builds the SPA, -and installs an atomic Nginx release. See [private operations](../deploy/private/README.md) +and installs an atomic Nginx release. See [private operations](../../deploy/private/README.md) for host services and recovery of deployment changes. API/auth image tags are `mist-api:departments-20261009` and `mist-auth:departments-20261009`. @@ -374,8 +379,8 @@ Build the CLI with `go -C cli build -o ../bin/mist-cli .`. New developer tasks should retain the identity → team policy → queued Job → scoped storage boundary. Never accept owner/roles from a submission or mount an entire store into a job. -Tests: [department verifier](../deploy/k3s/verify_department.mjs), -[browser verifier](../deploy/k3s/verify_department_browser.mjs), [large-dataset verifier](../deploy/k3s/verify_large_dataset.py), Go team tests, +Tests: [department verifier](../../deploy/k3s/verify_department.mjs), +[browser verifier](../../deploy/k3s/verify_department_browser.mjs), [large-dataset verifier](../../deploy/k3s/verify_large_dataset.py), Go team tests, CLI tests and React job/control tests. Checked-in evidence is linked from the execution plan. Protected verification state contains credentials and is never checked in. diff --git a/docs/design-system.md b/docs/content/design-system.md similarity index 98% rename from docs/design-system.md rename to docs/content/design-system.md index a09ee98..2885488 100644 --- a/docs/design-system.md +++ b/docs/content/design-system.md @@ -1,3 +1,8 @@ +--- +title: "Portal Design System" +weight: 80 +--- + # Portal design October 9, 2026. The visual direction takes inspiration from Cursor's restrained diff --git a/docs/development.md b/docs/content/development.md similarity index 95% rename from docs/development.md rename to docs/content/development.md index e92e469..83c0345 100644 --- a/docs/development.md +++ b/docs/content/development.md @@ -1,3 +1,8 @@ +--- +title: "Developer Setup" +weight: 60 +--- + # Developer setup Use Go **1.25.1** and Node.js **22.16.0 or newer**. Run repository commands from @@ -72,5 +77,5 @@ bash deploy/private/build-and-deploy.sh This requires existing administrative kubeconfig, Docker/root access, auth Secrets, operators and mounted storage. It is not a fresh-machine installer. -Read [operations](../deploy/private/README.md) and [testing](testing.md) first. +Read [operations](../../deploy/private/README.md) and [testing](testing.md) first. Current user/admin behavior is in [the operating guide](department-rollout.md). diff --git a/docs/foundation-execution-plan.md b/docs/content/foundation-execution-plan.md similarity index 66% rename from docs/foundation-execution-plan.md rename to docs/content/foundation-execution-plan.md index 130c7c5..53c606b 100644 --- a/docs/foundation-execution-plan.md +++ b/docs/content/foundation-execution-plan.md @@ -1,7 +1,12 @@ +--- +title: "Foundation Execution Plan (Historical)" +weight: 302 +--- + # Historical guide This phase has been superseded by the department platform on main. -- [Current documentation](README.md) +- [Current documentation](_index.md) - [Current operating guide](department-rollout.md) - [Archived original record](archive/foundation-execution-plan.md) diff --git a/docs/content/frontend-architecture.md b/docs/content/frontend-architecture.md new file mode 100644 index 0000000..b1a81bf --- /dev/null +++ b/docs/content/frontend-architecture.md @@ -0,0 +1,130 @@ +--- +title: "Frontend Architecture" +weight: 90 +--- + +# Frontend Architecture & Backend Offloading + +Documentation of the current Mist frontend ([`web-interface`](../../web-interface)), its structural organization, and a roadmap of client-side processing to offload to the backend. + +## Architecture overview + +```mermaid +flowchart TD + User([User Browser]) --> Shell[Root Shell: __root.tsx] + Shell --> AG[AccountGate: auth.tsx] + AG --> TG[TeamGate: teams.tsx] + TG --> Nav[Navbar.tsx] + TG --> RouterOutlet[TanStack Router Outlet] + + RouterOutlet --> DPage[Dashboard: dashboard.tsx] + RouterOutlet --> JPage[Jobs Page: jobs.tsx] + RouterOutlet --> SPage[Datasets Page: datasets.tsx] + RouterOutlet --> MPage[Machines Page: machines.tsx] + RouterOutlet --> TPage[Teams Page: teams.tsx] + RouterOutlet --> PPage[Profile: profile.tsx] + + JPage --> JSF[JobSubmissionForm.tsx] + JPage --> JC[JobCard.tsx] + JC --> JL[JobLogs.tsx] + JC --> JF[JobFiles.tsx] + + SPage --> TSB[TeamStorageBrowser.tsx] +``` + +### Technology stack + +- **Vite 7.3**: Build tool and development proxy (`/api` and `/auth` forwards). +- **React 19.3**: Functional component architecture with custom hooks. +- **TanStack Router**: File-based routing located in `src/routes/` with code-splitting. +- **Tailwind CSS 4**: Modern styling via `@tailwindcss/vite` and customized Mist theme classes. +- **Better Auth Client**: Authentication state and admin member management. +- **Vitest 5 & Testing Library**: Unit and component tests. + +### Source layout + +| Location | Purpose | +|---|---| +| `src/main.tsx`, `router.tsx` | Router initialization and DOM hydration | +| `src/routes/__root.tsx` | Shell wrapper containing `AccountGate`, `TeamGate`, and `Navbar` | +| `src/routes/` | File-based routes (`dashboard.tsx`, `jobs.tsx`, `datasets.tsx`, `machines.tsx`, `teams.tsx`, `profile.tsx`) | +| `src/auth.tsx` | Better Auth client configuration, `AccountContext`, and `AccountGate` | +| `src/teams.tsx` | `TeamContext`, active team selection, workspace selector | +| `src/api.ts` | Typed HTTP client for jobs, datasets, hardware, and teams | +| `src/hooks/usePolling.ts` | In-flight request deduplication and abortable periodic polling | +| `src/components/jobs/` | `JobSubmissionForm`, `JobCard`, `JobLogs`, `JobFiles` | +| `src/components/` | `HardwarePanel`, `TeamStorageBrowser`, `Modal`, `Picker`, `Tabs`, `Card`, `Buttons` | + +--- + +## Backend offloading plan + +To keep the frontend as lightweight and simple as possible, the following responsibilities currently performed on the client should be handled by the backend: + +1. **Pagination and filtering**: + - `JobsPage`: Replace client-side slicing and array filtering with query parameters on `GET /jobs?status=...&q=...&page=...&limit=...`. + - `DatasetsPage`: Add `GET /datasets?q=...`. + - `AccountPage`: Replace Better Auth `listUsers({ limit: 1000 })` with a paginated endpoint `GET /members?page=...&q=...`. +2. **Metrics aggregation**: + - Provide `GET /metrics/summary` or summary counters in list endpoints to avoid transferring full item arrays just to compute badge totals. +3. **Image catalog and validation**: + - Move registry string parsing and image approval checks from `JobSubmissionForm` to server-side submission validation. +4. **Quota and limit calculation**: + - Replace client-side calculation of `maxDevices` and `maxTimeout` with authoritative constraints provided by the team policy API. +5. **Form parsing**: + - Accept raw multiline environment variable text and shell commands on `POST /jobs`, delegating parsing and validation to the server. +6. **Error status handling**: + - Replace English error string matching (e.g. `isPreparing`) with typed machine-readable status fields (`{ status: "provisioning" }`). +7. **Permission flags**: + - Return explicit capability booleans (`can_cancel`, `can_delete`, `can_manage`) on entities rather than parsing role strings on the client. +8. **Atomic operations**: + - Consolidate multi-call client sequences (e.g. team creation + creator enrollment, password reset + session revocation) into atomic backend operations. +9. **Event-driven updates**: + - Replace high-frequency polling with Server-Sent Events (SSE) or WebSockets for live job status and log streaming. + +--- + +## Testing with the fake backend + +A self-contained testing backend is provided in `scripts/fake-backend.mjs`. It simulates all Mist API endpoints and Better Auth handlers in memory. + +### Running locally + +```bash +# Terminal 1: Start fake backend (default port 3000) +npm --prefix web-interface run fake-backend +``` + +```bash +# Terminal 2: Start frontend dev server (port 3001) +npm --prefix web-interface run dev -- --host 127.0.0.1 --port 3001 --strictPort +``` + +Open http://127.0.0.1:3001 in your browser. Pre-seeded users include `ada@utmist.gitlab.io` (admin), `alan@utmist.gitlab.io`, and `grace@utmist.gitlab.io`. + +### Configuring hypothetical scenarios and fake information + +A companion configure tool is provided at `scripts/configure-fake-backend.mjs` to set up hypothetical cluster topologies, error conditions, and user setups: + +```bash +# Interactive menu (choose scenarios, add fake nodes, switch active user) +npm --prefix web-interface run configure-fake-backend + +# Switch to a preset scenario directly +node scripts/configure-fake-backend.mjs --scenario high-load +node scripts/configure-fake-backend.mjs --scenario unhealthy +node scripts/configure-fake-backend.mjs --scenario empty-slate +node scripts/configure-fake-backend.mjs --scenario large-cluster +node scripts/configure-fake-backend.mjs --scenario default + +# Inspect current cluster topology & status +node scripts/configure-fake-backend.mjs --status + +# Add custom fake machines, users, or teams +node scripts/configure-fake-backend.mjs --add-machine name=dgx-custom,ip=10.99.1.5,cpu=64,ram=256Gi,gpu=8 +node scripts/configure-fake-backend.mjs --add-user name="Marie Curie",email=curie@rad.edu,role=admin +node scripts/configure-fake-backend.mjs --set-user curie@rad.edu +``` + +If the fake backend is running, `configure-fake-backend.mjs` updates the live server immediately without requiring a restart, while also saving to `.fake-backend-config.json` for persistence. + diff --git a/docs/k3s-setup.md b/docs/content/k3s-setup.md similarity index 96% rename from docs/k3s-setup.md rename to docs/content/k3s-setup.md index 2c728f0..fb80ea3 100644 --- a/docs/k3s-setup.md +++ b/docs/content/k3s-setup.md @@ -1,8 +1,13 @@ +--- +title: "k3s Cluster & Accelerators" +weight: 30 +--- + # k3s setup and accelerator handoff Verified on **October 9, 2026**. This guide describes the existing two-node installation, fresh-install boundaries and diagnostic commands. Routine -application deployment is in [operations](../deploy/private/README.md). +application deployment is in [operations](../../deploy/private/README.md). ## Installed cluster @@ -182,7 +187,7 @@ PyTorch autograd/model compatibility. Use **Jobs → New job → NVIDIA/Tenstorrent → Training check** after signing in and selecting your team. This exercises authentication, quotas, queue, allocator, -training and persistent output. Use the [CLI](../cli/docs/setup.md) from either +training and persistent output. Use the [CLI](../../cli/docs/setup.md) from either host to submit a real script through the same API. The direct manifest below is an administrator infrastructure diagnostic in the @@ -215,7 +220,7 @@ retain needed evidence before deleting only that completed diagnostic Job. - **Storage unavailable:** verify QuietBox's NFS/team-volume services and both pool mounts. Do not delete PVCs or overwrite mounted backing files. -Current live evidence is in [the verification record](../deploy/k3s/evidence/department-2026-10-09/README.md). +Current live evidence is in [the verification record](../../deploy/k3s/evidence/department-2026-10-09/README.md). No new reboot, operator upgrade or rejoin was performed for this documentation audit. References: [K3s quick start](https://docs.k3s.io/quick-start), diff --git a/docs/kubernetes-pilot.md b/docs/content/kubernetes-pilot.md similarity index 67% rename from docs/kubernetes-pilot.md rename to docs/content/kubernetes-pilot.md index b778a4e..a91ce37 100644 --- a/docs/kubernetes-pilot.md +++ b/docs/content/kubernetes-pilot.md @@ -1,7 +1,12 @@ +--- +title: "Kubernetes Pilot (Historical)" +weight: 303 +--- + # Historical guide This phase has been superseded by the department platform on main. -- [Current documentation](README.md) +- [Current documentation](_index.md) - [Current operating guide](department-rollout.md) - [Archived original record](archive/kubernetes-pilot.md) diff --git a/docs/local-job-foundation.md b/docs/content/local-job-foundation.md similarity index 66% rename from docs/local-job-foundation.md rename to docs/content/local-job-foundation.md index a188e01..5cf0eda 100644 --- a/docs/local-job-foundation.md +++ b/docs/content/local-job-foundation.md @@ -1,7 +1,12 @@ +--- +title: "Local Job Foundation (Historical)" +weight: 304 +--- + # Historical guide This phase has been superseded by the department platform on main. -- [Current documentation](README.md) +- [Current documentation](_index.md) - [Current operating guide](department-rollout.md) - [Archived original record](archive/local-job-foundation.md) diff --git a/docs/repository-cleanup.md b/docs/content/repository-cleanup.md similarity index 93% rename from docs/repository-cleanup.md rename to docs/content/repository-cleanup.md index bc619bd..332e596 100644 --- a/docs/repository-cleanup.md +++ b/docs/content/repository-cleanup.md @@ -1,3 +1,8 @@ +--- +title: "Repository Cleanup" +weight: 110 +--- + # Repository cleanup — October 9, 2026 Based on main commit `3431ffd`, the department release merged through PR #110. @@ -38,7 +43,7 @@ from deploying services. ## Verification -- [Sanitized validation report](../deploy/k3s/evidence/department-2026-10-09/cleanup-validation.json). +- [Sanitized validation report](../../deploy/k3s/evidence/department-2026-10-09/cleanup-validation.json). - Full current Go workspace tests with race detection: **29 API tests + 6 CLI tests**, no skipped tests; CLI entrypoint has no test files. Vet passes. - API and CLI static builds, CLI/root/job help and retired-executor rejection pass. @@ -60,7 +65,7 @@ from deploying services. The existing portal continues to run the verified department release. These source changes do not redeploy it. Previous live CUDA/TT/browser/isolation -[release evidence](../deploy/k3s/evidence/department-2026-10-09/README.md) remains +[release evidence](../../deploy/k3s/evidence/department-2026-10-09/README.md) remains valid for that release; it is not claimed as a fresh training/browser run of the cleanup candidate. No large storage tests, production restarts, account/job/file mutations or full-disk cleanup were performed. @@ -73,7 +78,7 @@ legacy workspace. No blanket prune, formatting or history rewrite is used. ## Reproduce current source checks See [testing](testing.md), [developer setup](development.md) and -[documentation index](README.md). Raw local validation output is kept outside Git +[documentation index](_index.md). Raw local validation output is kept outside Git under `/home/utmist/mist-cleanup-validation-20261009`; publish only sanitized summaries. Credentials, session state, kubeconfig and SSH private keys stay out of committed documentation/evidence. diff --git a/docs/shared-storage-deployment-plan.md b/docs/content/shared-storage-deployment-plan.md similarity index 67% rename from docs/shared-storage-deployment-plan.md rename to docs/content/shared-storage-deployment-plan.md index 86916d7..f5252e5 100644 --- a/docs/shared-storage-deployment-plan.md +++ b/docs/content/shared-storage-deployment-plan.md @@ -1,7 +1,12 @@ +--- +title: "Shared Storage Plan (Historical)" +weight: 305 +--- + # Historical guide This phase has been superseded by the department platform on main. -- [Current documentation](README.md) +- [Current documentation](_index.md) - [Current operating guide](department-rollout.md) - [Archived original record](archive/shared-storage-deployment-plan.md) diff --git a/docs/storage.md b/docs/content/storage.md similarity index 97% rename from docs/storage.md rename to docs/content/storage.md index a116de1..2672e8e 100644 --- a/docs/storage.md +++ b/docs/content/storage.md @@ -1,3 +1,8 @@ +--- +title: "Storage & Upload Limits" +weight: 50 +--- + # Storage and upload limits Current department storage on **QuietBox**, verified October 9, 2026. Both @@ -91,7 +96,7 @@ findmnt -t ext4,nfs,nfs4 With root access, inspect `exportfs -v`, per-team mounts and the storage-agent journal when readiness fails. Migration/expansion and service installation are -in [deployment operations](../deploy/private/README.md). Never truncate or unlink +in [deployment operations](../../deploy/private/README.md). Never truncate or unlink mounted image files, recreate populated filesystems or treat team removal as cleanup. Small-file migration, quota bounds and real accelerator input/output paths passed. diff --git a/docs/tailscale-ssh.md b/docs/content/tailscale-ssh.md similarity index 97% rename from docs/tailscale-ssh.md rename to docs/content/tailscale-ssh.md index 611a5cf..19bd196 100644 --- a/docs/tailscale-ssh.md +++ b/docs/content/tailscale-ssh.md @@ -1,3 +1,8 @@ +--- +title: "Tailscale & Admin SSH" +weight: 40 +--- + # Tailscale and SSH setup Verified against both hosts on **October 9, 2026**. Tailscale **1.102.4** is @@ -112,7 +117,7 @@ The installed `mist-portal-firewall.service` permits ordinary Tailscale sources to main's TCP **8088**, denying direct SSH/backend/cluster access. Infrastructure IPs and the recorded administrator device **100.99.194.52** retain broader access. The script contains matching IPv6 entries. Adding an administrator device needs -an explicit update to [the host allowlist](../deploy/private/portal-firewall.sh), +an explicit update to [the host allowlist](../../deploy/private/portal-firewall.sh), plus any applicable tailnet policy. LAN rules are separate. A plain SSH public key does not bypass this firewall. Access from an external diff --git a/docs/testing.md b/docs/content/testing.md similarity index 94% rename from docs/testing.md rename to docs/content/testing.md index 275c3c3..2060c3d 100644 --- a/docs/testing.md +++ b/docs/content/testing.md @@ -1,3 +1,8 @@ +--- +title: "Automated Testing" +weight: 70 +--- + # Testing Use the pinned Go/Node runtimes in [developer setup](development.md). The root @@ -38,7 +43,7 @@ Current automated suites do not require Redis or Docker. The old supervisor, Docker manager/logger tests and fake CLI-config tests were removed with their retired implementations. This is not a test skip: those products are no longer part of the application. The earlier full legacy suite results remain as dated -[pre-cleanup evidence](../deploy/k3s/evidence/department-2026-10-09/release-validation.json). +[pre-cleanup evidence](../../deploy/k3s/evidence/department-2026-10-09/release-validation.json). ## Real installation checks @@ -66,7 +71,7 @@ verifiers have been retired because they no longer reflect current auth/storage. ## Evidence and limits -The [October 9 evidence](../deploy/k3s/evidence/department-2026-10-09/README.md) +The [October 9 evidence](../../deploy/k3s/evidence/department-2026-10-09/README.md) records real CUDA/TT training, separate board/GPU reservations, input/output mounts, checkpoint reload/download, grants/revocation, queue restart and browser flows. The regular-member Container demonstration preserved its own image CMD; diff --git a/docs/go.mod b/docs/go.mod new file mode 100644 index 0000000..1e7e5e9 --- /dev/null +++ b/docs/go.mod @@ -0,0 +1,5 @@ +module github.com/UTMIST/Mist/docs + +go 1.25.3 + +require github.com/imfing/hextra v0.13.0 // indirect diff --git a/docs/go.sum b/docs/go.sum new file mode 100644 index 0000000..6557292 --- /dev/null +++ b/docs/go.sum @@ -0,0 +1,2 @@ +github.com/imfing/hextra v0.13.0 h1:1zhCYy0tA9ZRErFBlU6NJEgR1HjK69bgd52EmpwnG00= +github.com/imfing/hextra v0.13.0/go.mod h1:vi+yhpq8YPp/aghvJlNKVnJKcPJ/VyAEcfC1BSV9ARo= diff --git a/docs/hugo.yaml b/docs/hugo.yaml new file mode 100644 index 0000000..94f90f1 --- /dev/null +++ b/docs/hugo.yaml @@ -0,0 +1,92 @@ +baseURL: "/docs/" +title: "Mist Documentation" +defaultContentLanguage: "en" +enableRobotsTXT: true + +module: + imports: + - path: "github.com/imfing/hextra" + +markup: + highlight: + noClasses: false + guessSyntax: true + goldmark: + renderer: + unsafe: true + extensions: + passthrough: + delimiters: + block: [['\[', '\]'], ['$$', '$$']] + inline: [['\(', '\)']] + enable: true + +menu: + main: + - name: "Overview" + pageRef: "/" + weight: 1 + - name: "Web Portal" + url: "/" + weight: 2 + - name: "Search" + weight: 5 + params: + type: search + - name: "GitHub" + url: "https://github.com/UTMIST/Mist" + weight: 10 + params: + icon: "github" + +params: + description: "Documentation and operational guides for UTMIST's research compute platform." + + externalLinkDecoration: true + + navbar: + displayTitle: true + displayLogo: true + logo: + text: "Mist Docs" + width: wide + + theme: + default: "system" + displayToggle: true + + footer: + enable: true + displayCopyright: true + displayPoweredBy: false + width: normal + + displayUpdatedDate: true + dateFormat: "January 2, 2006" + + search: + enable: true + type: flexsearch + flexsearch: + index: content + tokenize: forward + + editURL: + enable: true + base: "https://github.com/UTMIST/Mist/edit/main/docs/content" + + highlight: + copy: + enable: true + display: hover + + toc: + enable: true + displayTags: true + + page: + width: normal + displayPagination: true + + imageZoom: + enable: true diff --git a/pre_compute_platform.md b/pre_compute_platform.md index a23531c..6f67bdb 100644 --- a/pre_compute_platform.md +++ b/pre_compute_platform.md @@ -3,9 +3,9 @@ The earlier GCP jump-host, shared-password and manual reservation guide is superseded by the current private Mist installation. -- [Tailscale and SSH setup](docs/tailscale-ssh.md) -- [Research portal and team job submission](docs/department-rollout.md) -- [k3s cluster setup](docs/k3s-setup.md) +- [Tailscale and SSH setup](docs/content/tailscale-ssh.md) +- [Research portal and team job submission](docs/content/department-rollout.md) +- [k3s cluster setup](docs/content/k3s-setup.md) Passwords are not published in current documentation. The previous password remains in Git history; retire it if still valid. Researchers use Mist accounts diff --git a/scripts/check-docs.py b/scripts/check-docs.py index 531acb9..fcc3090 100644 --- a/scripts/check-docs.py +++ b/scripts/check-docs.py @@ -46,7 +46,7 @@ def slugs(path): fence = line[3:].strip() block = [] else: - if fence in ("bash", "sh") and "docs/archive/" not in name: + if fence in ("bash", "sh") and "/archive/" not in name: check = subprocess.run(["bash", "-n"], input="\n".join(block), text=True, capture_output=True) shells += 1 if check.returncode: diff --git a/src/jobs.md b/src/jobs.md index 8ad28da..df373c2 100644 --- a/src/jobs.md +++ b/src/jobs.md @@ -1,7 +1,7 @@ # Job lifecycle and API -Current authenticated team execution. See [the operating guide](../docs/department-rollout.md#developer-api-contract) -for the complete API and [architecture](../docs/architecture.md) for source layout. +Current authenticated team execution. See [the operating guide](../docs/content/department-rollout.md#developer-api-contract) +for the complete API and [architecture](../docs/content/architecture.md) for source layout. ## Lifecycle diff --git a/web-interface/.prettierignore b/web-interface/.prettierignore index 5322d7f..e820c72 100644 --- a/web-interface/.prettierignore +++ b/web-interface/.prettierignore @@ -1,3 +1,7 @@ package-lock.json pnpm-lock.yaml -yarn.lock \ No newline at end of file +yarn.lock +public/docs +dist +dist-ssr +.tanstack diff --git a/web-interface/README.md b/web-interface/README.md index 160d903..d965fb4 100644 --- a/web-interface/README.md +++ b/web-interface/README.md @@ -3,7 +3,7 @@ The authenticated React portal serves real team Jobs, hardware inventory, dataset uploads, common/member folder browsing, account administration and team policy/sharing controls. Production: **http://100.73.139.66:8088** through Tailscale. -See the [operating guide](../docs/department-rollout.md). +See the [operating guide](../docs/content/department-rollout.md). ## Start locally @@ -24,7 +24,7 @@ npm run dev -- --host 127.0.0.1 --port 3001 --strictPort Vite forwards `/api` and `/auth` to the API on port 3000. `MIST_API_PROXY` changes that target. Authenticate with a real Mist account and select your team; this -preview reaches real jobs/data. See [development](../docs/development.md) for +preview reaches real jobs/data. See [development](../docs/content/development.md) for other-host origins and backend work. Production uses a static Nginx release, not Vite or these per-user services. @@ -51,7 +51,7 @@ preserves image startup with blank overrides. CPU/RAM inventory shows allocation not live utilization. Inputs are read-only; save results in `/outputs`. The old fake account helpers/chart and unused `use-immer` dependency are retired. -Current behavior and components follow [the design system](../docs/design-system.md). +Current behavior and components follow [the design system](../docs/content/design-system.md). ## Verify @@ -62,4 +62,4 @@ npm run lint npm run build ``` -See [testing](../docs/testing.md) for browser-to-cluster checks and their scope. +See [testing](../docs/content/testing.md) for browser-to-cluster checks and their scope. diff --git a/web-interface/eslint.config.js b/web-interface/eslint.config.js index 3f272c0..7cdf373 100644 --- a/web-interface/eslint.config.js +++ b/web-interface/eslint.config.js @@ -15,6 +15,6 @@ export default [ }, }, { - ignores: ['eslint.config.js', 'prettier.config.js'], + ignores: ['eslint.config.js', 'prettier.config.js', 'public/**', 'dist/**'], }, ] diff --git a/web-interface/package.json b/web-interface/package.json index 0783ac3..2475954 100644 --- a/web-interface/package.json +++ b/web-interface/package.json @@ -11,6 +11,7 @@ "scripts": { "dev": "vite dev --port 3000", "build": "vite build", + "build:docs": "hugo -s ../docs", "preview": "vite preview", "test": "vitest run", "lint": "eslint", diff --git a/web-interface/src/auth.tsx b/web-interface/src/auth.tsx index 7f82822..546b417 100644 --- a/web-interface/src/auth.tsx +++ b/web-interface/src/auth.tsx @@ -4,6 +4,7 @@ import { createAuthClient } from 'better-auth/react' import { adminClient } from 'better-auth/client/plugins' import { jobsAPI } from '#/api.ts' import { errorMessage, usePolling } from '#/hooks/usePolling.ts' +import { BackendConnectionError } from '#/components/BackendConnectionError.tsx' export const authClient = createAuthClient({ basePath: '/auth', @@ -107,10 +108,11 @@ export function AccountGate({ children }: { children: ReactNode }) { ) if (session.error) return ( -
-

Account service unavailable: {session.error}

- -
+ ) if (session.data?.enabled && !session.data.user) return diff --git a/web-interface/src/components/BackendConnectionError.test.tsx b/web-interface/src/components/BackendConnectionError.test.tsx new file mode 100644 index 0000000..d074456 --- /dev/null +++ b/web-interface/src/components/BackendConnectionError.test.tsx @@ -0,0 +1,76 @@ +// @vitest-environment jsdom +import { afterEach, expect, test, vi } from 'vitest' +import { cleanup, fireEvent, render, screen } from '@testing-library/react' +import { BackendConnectionError } from './BackendConnectionError' + +afterEach(cleanup) + +test('renders connection error message and status badge', () => { + render( + , + ) + + const heading = screen.getByRole('heading', { level: 1 }) + expect(heading.textContent).toContain('Unable to reach the compute backend') + expect( + screen.getByText('Failed to fetch /api/session: connection refused'), + ).toBeTruthy() + expect(screen.getByText('Backend Offline')).toBeTruthy() +}) + +test('triggers onRetry when Try again button is clicked', () => { + const onRetry = vi.fn() + render( + , + ) + + const retryButton = screen.getByRole('button', { + name: 'Try again to connect to the backend', + }) + expect(retryButton).toBeTruthy() + expect(retryButton.textContent).toContain('Try again') + + fireEvent.click(retryButton) + expect(onRetry).toHaveBeenCalledTimes(1) +}) + +test('disables button and displays loading state when isRetrying is true', () => { + render( + {}} + isRetrying={true} + />, + ) + + const retryButton = screen.getByRole('button', { + name: 'Try again to connect to the backend', + }) + expect(retryButton.hasAttribute('disabled')).toBe(true) + expect(retryButton.textContent).toContain('Checking connection…') +}) + +test('provides links to static documentation and GitHub issues', () => { + render( + {}} + />, + ) + + const docsLink = screen.getByRole('link', { + name: /Documentation.*View documentation/s, + }) + expect(docsLink.getAttribute('href')).toBe('/docs/') + + const githubLink = screen.getByRole('link', { + name: /Report an Issue.*Open GitHub issues/s, + }) + expect(githubLink.getAttribute('href')).toBe( + 'https://github.com/UTMIST/Mist/issues', + ) + expect(githubLink.getAttribute('target')).toBe('_blank') +}) diff --git a/web-interface/src/components/BackendConnectionError.tsx b/web-interface/src/components/BackendConnectionError.tsx new file mode 100644 index 0000000..347cec4 --- /dev/null +++ b/web-interface/src/components/BackendConnectionError.tsx @@ -0,0 +1,241 @@ +import { useState } from 'react' +import { + AlertCircle, + BookOpen, + ExternalLink, + Github, + RefreshCw, +} from 'lucide-react' + +export interface BackendConnectionErrorProps { + error?: string + onRetry?: () => void | Promise + isRetrying?: boolean +} + +export function BackendConnectionError({ + error = 'Unable to establish connection to the Mist API service.', + onRetry, + isRetrying: externalRetrying, +}: BackendConnectionErrorProps) { + const [internalRetrying, setInternalRetrying] = useState(false) + const isRetrying = externalRetrying ?? internalRetrying + + async function handleRetry() { + if (isRetrying || !onRetry) return + setInternalRetrying(true) + try { + await onRetry() + } finally { + setInternalRetrying(false) + } + } + + return ( +
+
+ {/* Brand header & status badge */} +
+
+ +
+ + Mist + + Research Compute +
+
+ +
+
+
+ + {/* Primary Message */} +
+

SYSTEM STATUS / API DISCONNECTED

+

+ Unable to reach the compute backend +

+

+ The web portal is running, but cannot communicate with the Mist API + or account service. The backend server may be restarting, undergoing + maintenance, or temporarily unreachable over the private network. +

+
+ + {/* Error diagnostics banner */} +
+
+
+
+ + {/* Action button */} +
+ + + Automatic retries happen in the background. + +
+ + {/* Informational Cards: Docs & GitHub */} +
+ {/* Static Documentation link */} + +
+
+ + + Static & Active + +
+

+ Documentation + +

+

+ The static documentation site remains unaffected and fully + accessible even when the compute backend is down. Browse + operating guides, architecture, and setup manuals. +

+
+
+ View documentation + +
+
+ + {/* GitHub Repository Issue link */} + +
+
+ + + GitHub + +
+

+ Report an Issue +

+

+ Notice an unexpected service outage or API bug? Submit a report + or check active incidents on the UTMIST GitHub repository. +

+
+
+ Open GitHub issues +
+
+
+ + {/* Troubleshooting checklist */} +
+

+ Troubleshooting checklist +

+
    +
  • + Private mesh: Verify that your Tailscale client + is connected to the UTMIST tailnet. +
  • +
  • + Local dev: Ensure backend server is active at{' '} + + http://127.0.0.1:3000 + + . +
  • +
  • + Cluster admin: Check service logs with{' '} + + kubectl -n mist-system get pods + + . +
  • +
+
+
+
+ ) +} diff --git a/web-interface/src/components/JobsPage.test.tsx b/web-interface/src/components/JobsPage.test.tsx index ca397a0..3ace18f 100644 --- a/web-interface/src/components/JobsPage.test.tsx +++ b/web-interface/src/components/JobsPage.test.tsx @@ -270,7 +270,9 @@ test('keeps a cleared image blank while editing and uses the image startup defau const image = screen.getByLabelText('Container image') fireEvent.change(image, { target: { value: '' } }) expect(image.value).toBe('') - expect(screen.getByRole('button', { name: 'Submit job' }).hasAttribute('disabled')).toBe(true) + expect( + screen.getByRole('button', { name: 'Submit job' }).hasAttribute('disabled'), + ).toBe(true) fireEvent.change(image, { target: { value: 'cpu' } }) expect(image.value).toBe('cpu') fireEvent.change(image, { target: { value: 'cpu-runtime' } }) diff --git a/web-interface/src/components/Navbar.tsx b/web-interface/src/components/Navbar.tsx index 6814752..3b50311 100644 --- a/web-interface/src/components/Navbar.tsx +++ b/web-interface/src/components/Navbar.tsx @@ -1,7 +1,14 @@ import { Link } from '@tanstack/react-router' import { authClient, useAccount } from '#/auth.tsx' import { useState } from 'react' -import { LayoutDashboard, Play, Database, Server, Users } from 'lucide-react' +import { + LayoutDashboard, + Play, + Database, + Server, + Users, + BookOpen, +} from 'lucide-react' import { WorkspaceSelector } from '#/teams.tsx' export default function Navbar() { @@ -60,6 +67,12 @@ export default function Navbar() { {' '} Teams + + {' '} + Docs +
diff --git a/web-interface/src/components/jobs/JobSubmissionForm.tsx b/web-interface/src/components/jobs/JobSubmissionForm.tsx index 7bd51f3..21958ba 100644 --- a/web-interface/src/components/jobs/JobSubmissionForm.tsx +++ b/web-interface/src/components/jobs/JobSubmissionForm.tsx @@ -86,7 +86,7 @@ export function JobSubmissionForm({ const image = (mode === 'training-smoke' ? profile?.default_image - : imageOverride ?? profile?.default_image) || '' + : (imageOverride ?? profile?.default_image)) || '' const images = catalog?.images.filter((option) => option.accelerators.includes(compute)) ?? [] diff --git a/web-interface/src/format.ts b/web-interface/src/format.ts index e5e74c1..109eb12 100644 --- a/web-interface/src/format.ts +++ b/web-interface/src/format.ts @@ -1,6 +1,9 @@ export function formatBytes(bytes: number) { if (bytes === 0) return '0 B' const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'] - const power = Math.min(Math.floor(Math.log(bytes) / Math.log(1024)), units.length - 1) - return `${(bytes / 1024 ** power).toLocaleString(undefined, {maximumFractionDigits: power ? 1 : 0})} ${units[power]}` + const power = Math.min( + Math.floor(Math.log(bytes) / Math.log(1024)), + units.length - 1, + ) + return `${(bytes / 1024 ** power).toLocaleString(undefined, { maximumFractionDigits: power ? 1 : 0 })} ${units[power]}` } diff --git a/web-interface/src/routes/__root.tsx b/web-interface/src/routes/__root.tsx index ccda042..6df912b 100644 --- a/web-interface/src/routes/__root.tsx +++ b/web-interface/src/routes/__root.tsx @@ -6,9 +6,16 @@ import '../styles.css' import Navbar from '#/components/Navbar.tsx' import { AccountGate } from '#/auth.tsx' import { TeamGate } from '#/teams.tsx' +import { BackendConnectionError } from '#/components/BackendConnectionError.tsx' export const Route = createRootRoute({ component: RootComponent, + errorComponent: ({ error, reset }) => ( + + ), }) function RootComponent() { diff --git a/web-interface/vite.config.ts b/web-interface/vite.config.ts index dc69775..a52cb6c 100644 --- a/web-interface/vite.config.ts +++ b/web-interface/vite.config.ts @@ -1,12 +1,130 @@ import { defineConfig } from 'vite' +import type { Plugin } from 'vite' import { devtools } from '@tanstack/devtools-vite' import tsconfigPaths from 'vite-tsconfig-paths' +import fs from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { execSync } from 'node:child_process' import { tanstackRouter } from '@tanstack/router-plugin/vite' import viteReact from '@vitejs/plugin-react' import tailwindcss from '@tailwindcss/vite' +const __dirname = path.dirname(fileURLToPath(import.meta.url)) + +const MIME_TYPES: Record = { + '.html': 'text/html; charset=utf-8', + '.css': 'text/css; charset=utf-8', + '.js': 'application/javascript; charset=utf-8', + '.mjs': 'application/javascript; charset=utf-8', + '.json': 'application/json; charset=utf-8', + '.xml': 'application/xml; charset=utf-8', + '.svg': 'image/svg+xml', + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.ico': 'image/x-icon', + '.woff': 'font/woff', + '.woff2': 'font/woff2', + '.ttf': 'font/ttf', + '.txt': 'text/plain; charset=utf-8', +} + +function docsStaticPlugin(): Plugin { + const docsDir = path.resolve(__dirname, '../docs/public') + + function ensureDocsBuilt() { + if (!fs.existsSync(path.resolve(docsDir, 'index.html'))) { + try { + execSync('hugo -s ../docs', { stdio: 'ignore' }) + } catch { + // Hugo may not be installed in minimal test environments + } + } + } + + function serveFromDocs( + req: { url?: string }, + res: { + writeHead: (status: number, headers?: Record) => void + end: () => void + }, + next: () => void, + ) { + if (!req.url) return next() + const [pathname, search] = req.url.split('?') + const query = search ? `?${search}` : '' + + if (pathname === '/docs') { + res.writeHead(301, { Location: `/docs/${query}` }) + res.end() + return + } + + if (pathname.startsWith('/docs/')) { + const relPath = pathname.replace(/^\/docs\//, '') + let candidate = path.resolve(docsDir, relPath) + + // Prevent directory traversal outside docsDir + if (!candidate.startsWith(docsDir)) { + res.writeHead(403) + res.end() + return + } + + if (fs.existsSync(candidate)) { + const stat = fs.statSync(candidate) + if (stat.isDirectory()) { + if (!pathname.endsWith('/')) { + res.writeHead(301, { Location: `${pathname}/${query}` }) + res.end() + return + } + candidate = path.resolve(candidate, 'index.html') + } + } else { + const asHtml = `${candidate}.html` + if (fs.existsSync(asHtml)) candidate = asHtml + } + + if (fs.existsSync(candidate) && fs.statSync(candidate).isFile()) { + const ext = path.extname(candidate).toLowerCase() + const mime = MIME_TYPES[ext] ?? 'application/octet-stream' + res.writeHead(200, { + 'Content-Type': mime, + 'Cache-Control': 'no-cache', + }) + fs.createReadStream(candidate).pipe(res as any) + return + } + } + + next() + } + + return { + name: 'mist-docs-static', + buildStart() { + ensureDocsBuilt() + }, + closeBundle() { + ensureDocsBuilt() + const distDocs = path.resolve(__dirname, 'dist/docs') + if (fs.existsSync(docsDir)) { + fs.cpSync(docsDir, distDocs, { recursive: true }) + } + }, + configureServer(server) { + ensureDocsBuilt() + server.middlewares.use(serveFromDocs) + }, + } +} + const config = defineConfig({ server: { proxy: { @@ -15,7 +133,7 @@ const config = defineConfig({ }, '/api': { target: process.env.MIST_API_PROXY ?? 'http://127.0.0.1:3000', - rewrite: (path) => path.replace(/^\/api/, ''), + rewrite: (p) => p.replace(/^\/api/, ''), }, }, }, @@ -23,6 +141,7 @@ const config = defineConfig({ devtools(), tsconfigPaths({ projects: ['./tsconfig.json'] }), tailwindcss(), + docsStaticPlugin(), tanstackRouter({ target: 'react', autoCodeSplitting: true }), viteReact(), ],