From bbf87f602055351764ae4bcd9342c564bff6999c Mon Sep 17 00:00:00 2001 From: utmist-tt <339893541+utmist-tt@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:45:10 -0400 Subject: [PATCH 1/3] Retire unused execution prototypes and organize the Kubernetes API --- cli/cmd/config.go | 49 +- cli/cmd/config_test.go | 58 --- cli/cmd/job.go | 12 +- cli/cmd/root.go | 4 - cli/cmd/testutil.go | 72 --- cli/cmd/testutil_test.go | 24 + cli/docs/Mist.code-workspace | 10 - config/log.yaml | 12 - deploy/k3s/verify_complete_browser.mjs | 83 ---- deploy/k3s/verify_foundation.py | 204 -------- deploy/k3s/verify_foundation_browser.mjs | 247 ---------- deploy/k3s/verify_mist_api.py | 187 -------- deploy/k3s/verify_quietbox_submission.py | 64 --- docker-compose.yml | 15 - go | 0 go.mod | 3 - go.work | 6 + redis.conf | 55 --- scripts/check-docs.py | 75 +++ src/api.go | 406 ---------------- src/app.go | 108 +++++ src/app_test.go | 66 +++ src/container_job_test.go | 158 ------- src/docker/README.md | 143 ------ src/docker/docker.go | 178 ------- src/docker/docker_test.go | 300 ------------ src/docker/go.mod | 35 -- src/docker/go.sum | 132 ------ src/docker/pytorch-cpu/dockerfile | 8 - .../pytorch-cpu/pytorch-cpu_config.yaml | 12 - src/docker/pytorch-cpu/requirements.txt | 3 - src/docker/pytorch-cpu/run_test.sh | 2 - src/docker/pytorch-cuda/dockerfile | 8 - .../pytorch-cuda/pytorch-cuda_config.yaml | 12 - src/docker/pytorch-cuda/requirements.txt | 3 - src/docker/pytorch-cuda/run_test.sh | 2 - src/docker/pytorch-rocm/dockerfile | 8 - .../pytorch-rocm/pytorch-rocm_config.yaml | 11 - src/docker/pytorch-rocm/requirements.txt | 3 - src/docker/pytorch-rocm/run_test.sh | 2 - src/docker/run_container_job_test.sh | 23 - src/docker/run_tests.sh | 13 - src/go.mod | 35 +- src/go.sum | 83 ---- src/http.go | 35 ++ src/images.go | 3 +- src/int_test.go | 209 -------- src/job_scheduling_test.go | 45 -- src/job_types.go | 84 ++++ src/jobs_http.go | 109 +++++ src/kubernetes.go | 186 -------- src/main.go | 36 ++ src/multilogger/README.md | 44 -- src/multilogger/log.go | 195 -------- src/multilogger/log_test.go | 446 ------------------ src/outputs.go | 3 +- src/scheduler.go | 156 ------ src/status.go | 124 ----- src/supervisor.go | 308 ------------ src/team_runtime.go | 10 - src/util.go | 64 --- web-interface/package-lock.json | 20 +- web-interface/package.json | 3 +- web-interface/src/components/Chart.tsx | 142 ------ web-interface/src/util.ts | 25 - 65 files changed, 567 insertions(+), 4614 deletions(-) delete mode 100644 cli/cmd/config_test.go delete mode 100644 cli/cmd/testutil.go create mode 100644 cli/cmd/testutil_test.go delete mode 100644 cli/docs/Mist.code-workspace delete mode 100644 config/log.yaml delete mode 100644 deploy/k3s/verify_complete_browser.mjs delete mode 100644 deploy/k3s/verify_foundation.py delete mode 100644 deploy/k3s/verify_foundation_browser.mjs delete mode 100644 deploy/k3s/verify_mist_api.py delete mode 100644 deploy/k3s/verify_quietbox_submission.py delete mode 100644 docker-compose.yml delete mode 100644 go delete mode 100644 go.mod create mode 100644 go.work delete mode 100644 redis.conf create mode 100644 scripts/check-docs.py delete mode 100644 src/api.go create mode 100644 src/app.go create mode 100644 src/app_test.go delete mode 100644 src/container_job_test.go delete mode 100644 src/docker/README.md delete mode 100644 src/docker/docker.go delete mode 100644 src/docker/docker_test.go delete mode 100644 src/docker/go.mod delete mode 100644 src/docker/go.sum delete mode 100644 src/docker/pytorch-cpu/dockerfile delete mode 100644 src/docker/pytorch-cpu/pytorch-cpu_config.yaml delete mode 100644 src/docker/pytorch-cpu/requirements.txt delete mode 100644 src/docker/pytorch-cpu/run_test.sh delete mode 100644 src/docker/pytorch-cuda/dockerfile delete mode 100644 src/docker/pytorch-cuda/pytorch-cuda_config.yaml delete mode 100644 src/docker/pytorch-cuda/requirements.txt delete mode 100644 src/docker/pytorch-cuda/run_test.sh delete mode 100644 src/docker/pytorch-rocm/dockerfile delete mode 100644 src/docker/pytorch-rocm/pytorch-rocm_config.yaml delete mode 100644 src/docker/pytorch-rocm/requirements.txt delete mode 100644 src/docker/pytorch-rocm/run_test.sh delete mode 100755 src/docker/run_container_job_test.sh delete mode 100755 src/docker/run_tests.sh create mode 100644 src/http.go delete mode 100644 src/int_test.go delete mode 100644 src/job_scheduling_test.go create mode 100644 src/job_types.go create mode 100644 src/jobs_http.go create mode 100644 src/main.go delete mode 100644 src/multilogger/README.md delete mode 100644 src/multilogger/log.go delete mode 100644 src/multilogger/log_test.go delete mode 100644 src/scheduler.go delete mode 100644 src/status.go delete mode 100644 src/supervisor.go delete mode 100644 src/util.go delete mode 100644 web-interface/src/components/Chart.tsx delete mode 100644 web-interface/src/util.ts diff --git a/cli/cmd/config.go b/cli/cmd/config.go index 5869954..3ece81d 100644 --- a/cli/cmd/config.go +++ b/cli/cmd/config.go @@ -1,40 +1,15 @@ package cmd -import "fmt" - -// Config flags -type ConfigCmd struct { - DefaultCluster string `help:"Set the default compute cluster." optional: ""` - Show bool `help:"Show current configuration."` -} - -func (h *ConfigCmd) Run(ctx *AppContext) error { - // Some dummy config; Call API or something - defaultConfig := map[string]string{ - "defaultCluster": "AMD-cluster-1", - "region": "us-east", - } - - if h.Show && h.DefaultCluster != "" { - fmt.Printf("Cannot use --show and --default-cluster together") - return nil - } - - if h.DefaultCluster != "" { - // This is not actually set. - fmt.Printf("Setting default cluster to: %s\n", h.DefaultCluster) - return nil - } - - if h.Show { - fmt.Println("Current configuration: ") - for key, value := range defaultConfig { - fmt.Printf(" %s: %s \n", key, value) - } - return nil - } - - fmt.Println("No config action specified. Use --help for options.") - - return nil +import ( + "os" + "path/filepath" +) + +func defaultConfigPath() string { + dir, err := os.UserConfigDir() + if err != nil { + home, _ := os.UserHomeDir() + return filepath.Join(home, ".config", "mist", "config.json") + } + return filepath.Join(dir, "mist", "config.json") } diff --git a/cli/cmd/config_test.go b/cli/cmd/config_test.go deleted file mode 100644 index d3ecb15..0000000 --- a/cli/cmd/config_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package cmd - -import ( - "testing" - // "fmt" - // "bytes" -) - -// No Flag config -func TestConfigNoFlags(t *testing.T) { - cmd := &ConfigCmd{} - output := CaptureOutput(func() { - _ = cmd.Run(&AppContext{}) - }) - if want := "No config action specified. Use --help for options."; !contains(output, want) { - t.Errorf("expected output to contain %q, got %q", want, output) - } -} - -// Set Default Cluster to tt-gpu-cluster-1 -func TestConfigDefaultCluster(t *testing.T) { - cmd := &ConfigCmd{DefaultCluster: "tt-gpu-cluster-1"} // Create config object - - output := CaptureOutput(func() { - _ = cmd.Run(&AppContext{}) - }) - - // fmt.Printf("Captured the output: %s\n", output) - - if want := "Setting default cluster to: tt-gpu-cluster-1"; !contains(output, want) { - t.Errorf("expected output to contain %q, got %q", want, output) - } -} - -// Show Config -func TestConfigCmd_Show(t *testing.T) { - cmd := &ConfigCmd{Show: true} - output := CaptureOutput(func() { - _ = cmd.Run(&AppContext{}) - }) - - if want := "Current configuration:"; !contains(output, want) { - t.Errorf("expected output to contain %q, got %q", want, output) - } -} - -// Show Error message if both flags are sent -func TestConfigBothFlagError(t *testing.T) { - cmd := &ConfigCmd{DefaultCluster: "tt-gpu-cluster-1", Show: true} - output := CaptureOutput(func() { - _ = cmd.Run(&AppContext{}) - }) - - if want := "Cannot use --show and --default-cluster together"; !contains(output, want) { - t.Errorf("Expected the error message of \"%s\", got %q", want, output) - } - -} diff --git a/cli/cmd/job.go b/cli/cmd/job.go index 94f487a..e5d7faf 100644 --- a/cli/cmd/job.go +++ b/cli/cmd/job.go @@ -3,15 +3,7 @@ package cmd type JobCmd struct { Submit JobSubmitCmd `cmd:"" help:"Submit a new job"` Cancel JobCancelCmd `cmd:"" help:"Cancel an existing job"` - // Delete JobDeleteCmd `cmd: "" help: "Delete an existing job"` Status JobStatusCmd `cmd:"" help:"Check the status of a job"` - // Cancel CancelCmd `cmd:"" help:"Cancel a running job"` - Logs JobLogsCmd `cmd:"" help:"Read workload logs"` - List ListCmd `cmd:"" help:"List all jobs" default:1` -} - -func (j *JobCmd) Run() error { - // Possible fallback if no subcommand is providFAre yed - // fmt.Println("(job root) – try 'mist job submit|status|list|cancel' or mist help") - return nil + Logs JobLogsCmd `cmd:"" help:"Read workload logs"` + List ListCmd `cmd:"" help:"List jobs"` } diff --git a/cli/cmd/root.go b/cli/cmd/root.go index d655c8c..23a532d 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -14,7 +14,6 @@ type Config struct { SessionCookie string `json:"session_cookie,omitempty"` APIBaseURL string `json:"api_base_url,omitempty"` TeamID string `json:"team_id,omitempty"` - // maybe APIBaseURL, etc. } type AppContext struct { @@ -34,13 +33,10 @@ type Globals struct { type CLI struct { Globals - // Define your CLI structure here: Top Level Commands Team TeamCmd `cmd:"" help:"List and select team workspaces"` Auth AuthCmd `cmd:"" help:"Authentication commands"` Job JobCmd `cmd:"" help:"Job management commands"` - // Config ConfigCmd `cmd:"" help:"Configuration commands"` Help HelpCmd `cmd:"" help:"Show help information"` - // Config ConfigCmd `cmd:"" help: "Display Cluster Configuration"` } func loadConfig(path string) (*Config, error) { diff --git a/cli/cmd/testutil.go b/cli/cmd/testutil.go deleted file mode 100644 index 8838f12..0000000 --- a/cli/cmd/testutil.go +++ /dev/null @@ -1,72 +0,0 @@ -// cmd/testutil.go - -package cmd - -import ( - "bytes" - "fmt" - "io" - "os" - "path/filepath" - // "strings" -) - -// To simulate multiple consecutive user inputs, just put \n between your text ("Enter Key") -func MockInput(input string, f func()) { - old := os.Stdin - r, w, _ := os.Pipe() - w.Write([]byte(input)) - w.Close() - os.Stdin = r - f() - os.Stdin = old -} - -func CaptureOutput(f func()) string { - old := os.Stdout - r, w, _ := os.Pipe() - os.Stdout = w - - f() - - w.Close() - os.Stdout = old - - var buf bytes.Buffer - io.Copy(&buf, r) - return buf.String() -} - -func contains(s, substr string) bool { - return bytes.Contains([]byte(s), []byte(substr)) -} - -func jobExists(jobs []Job, id string) bool { - for _, job := range jobs { - if job.ID == id { - return true - } - } - return false -} - -func findJobByID(jobs []Job, id string) (Job, error) { - for _, job := range jobs { - if job.ID == id { - return job, nil - } - } - return Job{}, fmt.Errorf("job with ID %s not found", id) -} - -func defaultConfigPath() string { - dir, err := os.UserConfigDir() - if err != nil { - home, _ := os.UserHomeDir() - return filepath.Join(home, ".config", "mist", "config.json") - } - return filepath.Join(dir, "mist", "config.json") -} - -// Creating a helper function for capturing -// Stdout outputs diff --git a/cli/cmd/testutil_test.go b/cli/cmd/testutil_test.go new file mode 100644 index 0000000..6f60274 --- /dev/null +++ b/cli/cmd/testutil_test.go @@ -0,0 +1,24 @@ +package cmd + +import ( + "bytes" + "io" + "os" +) + +func CaptureOutput(f func()) string { + old := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + f() + w.Close() + os.Stdout = old + var buf bytes.Buffer + io.Copy(&buf, r) + r.Close() + return buf.String() +} + +func contains(s, substr string) bool { + return bytes.Contains([]byte(s), []byte(substr)) +} diff --git a/cli/docs/Mist.code-workspace b/cli/docs/Mist.code-workspace deleted file mode 100644 index 39f0155..0000000 --- a/cli/docs/Mist.code-workspace +++ /dev/null @@ -1,10 +0,0 @@ -{ - "folders": [ - { - "path": "../.." - }, - { - "path": "../.." - } - ] -} \ No newline at end of file diff --git a/config/log.yaml b/config/log.yaml deleted file mode 100644 index 367bea2..0000000 --- a/config/log.yaml +++ /dev/null @@ -1,12 +0,0 @@ -output: - types: - - type: file - level: INFO - - type: stdout - level: INFO - directory: ../logs/ - -components: - - app - - scheduler - - supervisor diff --git a/deploy/k3s/verify_complete_browser.mjs b/deploy/k3s/verify_complete_browser.mjs deleted file mode 100644 index 8ef4a80..0000000 --- a/deploy/k3s/verify_complete_browser.mjs +++ /dev/null @@ -1,83 +0,0 @@ -// Requires Playwright and an existing protected admin bootstrap JSON file. -// No credentials, session tokens or cookies are written into the result report. -import { readFileSync, writeFileSync, mkdirSync } from 'node:fs' -import { resolve } from 'node:path' -import { randomBytes } from 'node:crypto' -const { chromium } = await import(process.env.MIST_PLAYWRIGHT_MODULE ?? '/tmp/mist-browser-check/node_modules/playwright/index.mjs') -const base=process.env.MIST_SITE_URL ?? 'http://100.73.139.66:8088' -const credentials=JSON.parse(readFileSync(process.env.MIST_ADMIN_FILE ?? '/home/utmist/.config/mist/admin-bootstrap.json')) -const output=process.env.MIST_EVIDENCE_DIR ?? '/home/utmist/mist-complete-results-2026-10-09' -mkdirSync(output,{recursive:true}) -const browser=await chromium.launch({headless:true}) -const context=await browser.newContext({baseURL:base,viewport:{width:1440,height:1000}}) -const page=await context.newPage() -const errors=[];page.on('pageerror',e=>errors.push(e.message)) -const evidence={site:base,started:new Date().toISOString(),jobs:[],checks:{}} -const check=(value,message)=>{if(!value)throw new Error(message)} -async function login(target,email,password){await target.goto('/jobs');await target.getByLabel('Email',{exact:true}).fill(email);await target.getByLabel('Password',{exact:true}).fill(password);await target.getByRole('button',{name:'Sign in',exact:true}).click();await target.getByRole('heading',{name:'Jobs',exact:true}).waitFor()} -async function json(path,options){const r=await context.request.fetch(base+path,options);check(r.ok(),`${path} HTTP ${r.status()} ${await r.text()}`);return r.json()} -function matrix(n,m,phase){return Array.from({length:n},(_,i)=>Array.from({length:m},(_,j)=>Math.sin((i+1)*(j+2)+phase)))} -function multiply(x,w){return x.map(row=>w[0].map((_,k)=>row.reduce((sum,v,j)=>sum+v*w[j][k],0)))} -try{ - await login(page,credentials.email,credentials.password) - evidence.checks.browser_login=true - await page.screenshot({path:resolve(output,'01-jobs.png'),fullPage:true}) - const x=matrix(128,32,0.1),vx=matrix(64,32,1.2),truth=matrix(32,32,2.3).map(row=>row.map(v=>v*0.1)) - const dataset=Buffer.from(JSON.stringify({x,y:multiply(x,truth),validation_x:vx,validation_y:multiply(vx,truth)})) - await page.goto('/datasets') - await page.getByLabel('Dataset name (optional)').fill('Shared regression verification') - await page.getByLabel('File or ZIP archive').setInputFiles({name:'regression.json',mimeType:'application/json',buffer:dataset}) - await page.getByRole('button',{name:'Upload dataset',exact:true}).click() - await page.getByRole('status').filter({hasText:'is ready to use'}).waitFor({timeout:60000}) - const ds=(await json('/api/datasets')).datasets.find(d=>d.name==='Shared regression verification') - check(ds?.state==='Ready','Uploaded dataset not ready');evidence.dataset={id:ds.id,sha256:ds.sha256,size:ds.size} - await page.screenshot({path:resolve(output,'02-dataset.png'),fullPage:true}) - const training=readFileSync('deploy/k3s/examples/train_shared.py','utf8') - const cpuScript=`import json\nfrom pathlib import Path\np=Path('/inputs/regression.json')\nd=json.loads(p.read_text())\nassert len(d['x'])==128\nPath('/outputs/dataset-check.json').write_text(json.dumps({'rows':len(d['x']),'input_readonly':True}))\ntry:\n Path('/inputs/probe').write_text('bad')\n raise AssertionError('dataset mount is writable')\nexcept OSError:\n pass\nprint('SHARED_DATASET_PASSED CPU',flush=True)\n` - for(const backend of ['cpu','nvidia','tenstorrent']){ - await page.goto('/jobs');await page.getByLabel('Name',{exact:true}).fill(`Shared storage ${backend}`) - await page.getByLabel('Compute',{exact:true}).selectOption(backend) - await page.getByLabel('Dataset (optional)').selectOption(ds.id) - if(backend!=='cpu')await page.getByLabel(backend==='nvidia'?'GPUs':'Boards (two chips each)').fill(backend==='nvidia'?'2':'1') - // Custom Python script with argv exercises uploaded data on actual accelerator. - const script=backend==='cpu'?cpuScript:training.replace('args = parser.parse_args()',`args = parser.parse_args(['${backend}', '--expected-devices', '${backend==='nvidia'?2:2}', '--output', '/outputs'])`) - await page.getByLabel('Python script',{exact:true}).fill(script) - const responsePromise=page.waitForResponse(r=>r.request().method()==='POST'&&r.url().endsWith('/api/jobs')) - await page.getByRole('button',{name:'Submit job',exact:true}).click() - const response=await responsePromise;check(response.ok(),`Submit ${backend} failed: ${await response.text()}`) - const job=(await response.json()).job;evidence.jobs.push({id:job.id,backend}) - } - for(const entry of evidence.jobs){ - const deadline=Date.now()+10*60*1000 - while(Date.now()setTimeout(r,3000))} - const logs=(await json(`/api/jobs/${entry.id}/logs`)).logs - writeFileSync(resolve(output,`${entry.backend}.log`),logs) - check(entry.state==='Success',`${entry.backend} failed: ${logs.slice(-2000)}`) - check(logs.includes(entry.backend==='cpu'?'SHARED_DATASET_PASSED':'TRAINING_PASSED'),`Missing training marker ${entry.backend}`) - if(entry.backend!=='cpu')check(logs.includes('DATASET_LOADED file=/inputs/regression.json'),'Training did not consume uploaded data') - const listing=await json(`/api/jobs/${entry.id}/files`);check(listing.files.length>0,`No persisted outputs ${entry.backend}`) - entry.files=[] - for(const file of listing.files){const r=await context.request.get(`${base}/api/jobs/${entry.id}/files/download`,{params:{path:file.path}});check(r.status()===200,`Download failed ${file.path}`);const contents=await r.body();check(contents.length===file.size,'Output size mismatch');writeFileSync(resolve(output,`${entry.backend}-${file.path.replaceAll('/','_')}`),contents);entry.files.push({path:file.path,size:file.size})} - } - await page.goto('/jobs');await page.getByRole('button',{name:'Files',exact:true}).first().click();await page.getByRole('heading',{name:'Saved files'}).waitFor();await page.screenshot({path:resolve(output,'03-completed-files.png'),fullPage:true}) - const downloadPromise=page.waitForEvent('download');await page.locator('a[href*="/files/download"]').first().click();const download=await downloadPromise;await download.saveAs(resolve(output,'browser-download-'+download.suggestedFilename()));check(await download.failure()===null,'Browser download failed') - evidence.checks.browser_upload_select_train_download=true - // Real second account created using the administrative website. - const member={name:'Verification Member',email:`verify-${Date.now()}@mist.invalid`,password:randomBytes(20).toString('base64url')} - await page.goto('/profile');await page.getByLabel('Name',{exact:true}).fill(member.name);await page.getByLabel('Email',{exact:true}).fill(member.email);await page.getByLabel('Initial password').fill(member.password) - const createPromise=page.waitForResponse(r=>r.url().endsWith('/auth/admin/create-user'));await page.getByRole('button',{name:'Add member'}).click();const created=await createPromise;check(created.ok(),`Member creation failed: ${await created.text()}`) - const memberID=(await created.json()).user.id - const memberContext=await browser.newContext({baseURL:base});const memberPage=await memberContext.newPage();await login(memberPage,member.email,member.password) - await memberPage.goto('/profile');await memberPage.getByLabel('Current password').fill(member.password);member.password=randomBytes(20).toString('base64url');await memberPage.getByLabel('New password').fill(member.password);await memberPage.getByRole('button',{name:'Change password',exact:true}).click();await memberPage.getByRole('status').filter({hasText:'Password changed.'}).waitFor();evidence.checks.member_password_change=true - const foreignJob=evidence.jobs[0].id - for(const path of [`/api/jobs/${foreignJob}`,`/api/jobs/${foreignJob}/logs`,`/api/jobs/${foreignJob}/files`,`/api/datasets/${ds.id}`]){const r=await memberContext.request.get(base+path);check(r.status()===404,`Foreign resource exposed: ${path}`)} - check((await (await memberContext.request.get(base+'/api/jobs')).json()).count===0,'Member saw administrator jobs') - const forbidden=await memberContext.request.post(base+'/auth/admin/create-user',{headers:{Origin:base},data:{name:'bad',email:'bad@mist.invalid',password:member.password,role:'admin'}});check(forbidden.status()===403,'Regular member created admin') - const anon=await browser.newContext({baseURL:base});check((await anon.request.get(base+'/api/jobs')).status()===401,'Anonymous jobs accessible');check((await anon.request.post(base+'/auth/sign-up/email',{data:{name:'bad',email:'bad2@mist.invalid',password:member.password}})).status()>=400,'Public signup allowed');await anon.close() - const evil=await context.request.post(base+'/api/jobs',{headers:{Origin:'http://evil.invalid'},data:{command:['true']}});check(evil.status()===403,'Cross-origin submission accepted') - const ban=await context.request.post(base+'/auth/admin/ban-user',{headers:{Origin:base},data:{userId:memberID,banReason:'Completed verification'}});check(ban.ok(),`Ban failed HTTP ${ban.status()}: ${await ban.text()}`);check((await memberContext.request.get(base+'/api/jobs')).status()===401,'Deactivated session still accepted');await memberContext.close() - evidence.checks.member_creation_isolation_deactivation=true - evidence.checks.anonymous_signup_csrf_denied=true - check(errors.length===0,`Browser errors: ${errors.join('; ')}`);evidence.browser_errors=errors - evidence.finished=new Date().toISOString();writeFileSync(resolve(output,'complete-results.json'),JSON.stringify(evidence,null,2));console.log(JSON.stringify(evidence,null,2)) -}catch(err){evidence.error=err.message;writeFileSync(resolve(output,'complete-results.json'),JSON.stringify(evidence,null,2));throw err}finally{await browser.close()} diff --git a/deploy/k3s/verify_foundation.py b/deploy/k3s/verify_foundation.py deleted file mode 100644 index 7df1a67..0000000 --- a/deploy/k3s/verify_foundation.py +++ /dev/null @@ -1,204 +0,0 @@ -"""Live allocation/output checks for the local Mist foundation. - -Run after verify_foundation_browser.mjs. Requests use the Mist API; admin -kubectl only reads cluster state and creates temporary read-only output probes. -Jobs/results are retained. Holders are cancelled even if a check fails. -""" -import argparse -from datetime import datetime, timezone -import json -from pathlib import Path -import subprocess -import time -import urllib.error -import urllib.request - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--url', default='http://127.0.0.1:3000') - parser.add_argument('--artifacts', type=Path, required=True) - args = parser.parse_args() - args.artifacts.mkdir(parents=True, exist_ok=True) - evidence = {'started_at': datetime.now(timezone.utc).isoformat(), 'jobs': {}, 'checks': {}} - run = datetime.now(timezone.utc).strftime('foundation-%m%d-%H%M%S') - holders = set() - - def api(method, path, body=None, expected=200): - request = urllib.request.Request(args.url + path, method=method, - data=json.dumps(body).encode() if body is not None else None, - headers={'Content-Type': 'application/json'}) - try: - response = urllib.request.urlopen(request, timeout=30) - except urllib.error.HTTPError as error: - response = error - with response: - value = json.load(response) - assert response.code == expected, (path, response.code, value) - return value - - def kubectl(*options, body=None): - return subprocess.check_output(['k3s', 'kubectl', *options], - input=json.dumps(body) if body else None, text=True, timeout=45) - - def wait(check, description, timeout=180): - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - value = check() - if value: - return value - time.sleep(1) - raise AssertionError('Timed out: ' + description) - - def submit(label, **request): - value = api('POST', '/jobs', {'name': run + '-' + label, **request}, expected=201) - evidence['jobs'][label] = value['job_id'] - print('Submitted', label, value['job_id'], flush=True) - return value['job_id'] - - def status(job): - return api('GET', '/jobs/' + job) - - def logs(job): - return api('GET', '/jobs/' + job + '/logs')['logs'] - - def finished(job, desired='Success'): - def check(): - value = status(job) - if value['job_state'] in ('Success', 'Failure', 'Cancelled'): - assert value['job_state'] == desired, value - return value - value = wait(check, job + ' finished') - text = logs(job) - (args.artifacts / (job + '.log')).write_text(text) - evidence['checks'][job] = {'status': value, 'logs': text} - return value, text - - def pool(accelerator): - return next(p for p in api('GET', '/hardware')['pools'] if p['accelerator'] == accelerator) - - def inspect_outputs(label, job, pvc, filenames): - """Read files from a fresh container after the training container exits.""" - state = status(job) - assert state['job_state'] == 'Success', state - name = run + '-read-' + label - commands = [f'test -s /results/{job}/{file}' for file in filenames] - commands += [f'find /results/{job} -type f -exec sha256sum {{}} \\;', f'cat /results/{job}/results.json'] - pod = {'apiVersion': 'v1', 'kind': 'Pod', 'metadata': {'name': name, 'namespace': 'mist'}, - 'spec': {'restartPolicy': 'Never', 'automountServiceAccountToken': False, - 'nodeSelector': {'kubernetes.io/hostname': state['node']}, - 'containers': [{'name': 'inspect', 'image': 'busybox:1.36', 'imagePullPolicy': 'IfNotPresent', - 'command': ['sh', '-c', ' && '.join(commands)], - 'securityContext': {'allowPrivilegeEscalation': False, 'readOnlyRootFilesystem': True, - 'capabilities': {'drop': ['ALL']}}, - 'resources': {'requests': {'cpu': '10m', 'memory': '16Mi'}, 'limits': {'cpu': '100m', 'memory': '32Mi'}}, - 'volumeMounts': [{'name': 'results', 'mountPath': '/results', 'readOnly': True}]}], - 'volumes': [{'name': 'results', 'persistentVolumeClaim': {'claimName': pvc, 'readOnly': True}}]}} - kubectl('create', '-f', '-', body=pod) - try: - def done(): - value = json.loads(kubectl('-n', 'mist', 'get', 'pod', name, '-o', 'json')) - phase = value['status'].get('phase') - if phase in ('Succeeded', 'Failed'): - assert phase == 'Succeeded', value['status'] - return value - wait(done, 'saved outputs readable on ' + state['node'], timeout=60) - text = kubectl('-n', 'mist', 'logs', name) - lines = text.splitlines() - start = next(i for i, line in enumerate(lines) if line.startswith(('{', '['))) - results = json.loads('\n'.join(lines[start:])) - if label == 'nvidia': - assert len(results['results']) == 2 - assert all(result['final_loss'] < 0.001 for result in results['results']) - if label == 'tenstorrent': - assert len(results) == 4 and all(result['validation_mse'] < 5e-4 for result in results) - assert sum('/tenstorrent-' in line and '.pt' in line for line in lines[:start]) == 4 - (args.artifacts / (label + '-persisted-files.txt')).write_text(text) - evidence['checks']['persisted-' + label] = {'job_id': job, 'pvc': pvc, 'node': state['node'], 'files': filenames, 'sha256_and_results': text} - finally: - kubectl('-n', 'mist', 'delete', 'pod', name, '--wait=false') - - try: - before = api('GET', '/jobs')['count'] - for request in [ - {'image': 'unapproved/image:latest'}, - {'command': ['true'], 'accelerator': 'nvidia', 'device_count': 3}, - {'command': ['true'], 'accelerator': 'tenstorrent', 'device_count': 5}, - {'command': ['true'], 'privileged': True}, - {'command': ['true'], 'working_directory': 'relative'}, - {'command': ['true'], 'env': {'MIST_OUTPUT_DIR': '/override'}}, - ]: - api('POST', '/jobs', request, expected=400) - assert api('GET', '/jobs')['count'] == before - evidence['checks']['invalid_requests_rejected_without_jobs'] = True - - command = "import json,os,sys; from pathlib import Path; assert os.getcwd()=='/tmp'; assert sys.argv[1:]==['one argument with spaces','literal;argument']; p=Path(os.environ['MIST_OUTPUT_DIR']); (p/'results.json').write_text(json.dumps({'argv':sys.argv[1:],'cwd':os.getcwd()})); assert (Path(os.environ['MIST_CHECKPOINT_DIR'])/'results.json').read_text()==(p/'results.json').read_text(); print('COMMAND_ARGUMENTS_OUTPUT_ALIAS_PASSED',flush=True)" - literal = submit('literal-arguments', image='mist-training:cpu-v1', command=['python', '-c', command], - args=['one argument with spaces', 'literal;argument'], working_directory='/tmp') - _, text = finished(literal) - assert 'COMMAND_ARGUMENTS_OUTPUT_ALIAS_PASSED' in text - inspect_outputs('arguments', literal, 'mist-cpu-checkpoints', ['results.json']) - - for accelerator, total in [('nvidia', 2), ('tenstorrent', 4)]: - wait(lambda: pool(accelerator)['available'] == total, accelerator + ' idle') - script = "import time\nfrom pathlib import Path\n" - if accelerator == 'nvidia': - script += "import torch\nassert torch.cuda.is_available() and torch.cuda.device_count()==2\n" - else: - script += "assert len(list(Path('/dev/tenstorrent').glob('[0-9]*')))==4\n" - script += "print('DEVICE_HOLDER_READY',flush=True)\ntime.sleep(180)\n" - holder = submit(accelerator + '-holder', accelerator=accelerator, device_count=total, script=script, timeout_seconds=300) - holders.add(holder) - wait(lambda: 'DEVICE_HOLDER_READY' in logs(holder), accelerator + ' whole-machine reservation started') - snapshot = pool(accelerator) - assert snapshot['allocated'] == total and snapshot['available'] == 0, snapshot - if accelerator == 'tenstorrent': - assert len(status(holder).get('devices', [])) == 4 - follower_request = {'accelerator': accelerator, 'device_count': 1} - if accelerator == 'nvidia': - follower_request.update(image='mist-training:nvidia-v1', args=['--backend', 'nvidia', '--devices', '1']) - else: - follower_request.update(type='training-smoke') - follower = submit(accelerator + '-queued', **follower_request) - def queued(): - value = status(follower) - if value['job_state'] == 'Scheduled' and value.get('message'): - reason = 'cannot allocate' if accelerator == 'tenstorrent' else 'Insufficient nvidia.com/gpu' - assert reason in value['message'] and not value.get('node'), value - return value - queue = wait(queued, accelerator + ' queued behind holder', timeout=60) - assert api('POST', '/jobs/' + holder + '/cancel')['job_state'] == 'Cancelled' - assert api('POST', '/jobs/' + holder + '/cancel')['job_state'] == 'Cancelled' - holders.discard(holder) - _, text = finished(holder, 'Cancelled') - assert 'DEVICE_HOLDER_READY' in text - _, text = finished(follower) - marker = 'TRAINING_PASSED backend=tenstorrent devices=2' if accelerator == 'tenstorrent' else 'PACKAGED_TRAINING_PASSED backend=nvidia' - assert marker in text - wait(lambda: pool(accelerator)['available'] == total, accelerator + ' devices released', timeout=45) - evidence['checks'][accelerator + '-queue-cancel-reuse'] = {'occupied': snapshot, 'queued': queue, 'idle': pool(accelerator)} - print(accelerator + ': occupied/free counts, queue, cancellation, retained logs and automatic reuse passed', flush=True) - - browser = json.loads((args.artifacts / 'browser.json').read_text()) - assert 'error' not in browser and 'completed_at' in browser, 'Browser checks must finish first' - inspect_outputs('cpu', browser['jobs']['packaged-cpu']['id'], 'mist-cpu-checkpoints', ['model.json', 'results.json']) - inspect_outputs('nvidia', browser['jobs']['packaged-two-gpus']['id'], 'training-nvidia-checkpoints', ['nvidia-0.pt', 'nvidia-1.pt', 'results.json']) - tt_job = browser['jobs']['tt-two-boards']['id'] - inspect_outputs('tenstorrent', tt_job, 'training-tenstorrent-checkpoints', ['allocation.json', 'results.json']) - evidence['hardware_after'] = api('GET', '/hardware') - evidence['completed_at'] = datetime.now(timezone.utc).isoformat() - print('FOUNDATION_ALLOCATION_OUTPUTS_PASSED', flush=True) - except Exception as error: - evidence['error'] = repr(error) - raise - finally: - for holder in holders: - try: - api('POST', '/jobs/' + holder + '/cancel') - except Exception as error: - evidence['cleanup_error'] = repr(error) - (args.artifacts / 'foundation-acceptance.json').write_text(json.dumps(evidence, indent=2) + '\n') - - -if __name__ == '__main__': - main() diff --git a/deploy/k3s/verify_foundation_browser.mjs b/deploy/k3s/verify_foundation_browser.mjs deleted file mode 100644 index 5b2b89c..0000000 --- a/deploy/k3s/verify_foundation_browser.mjs +++ /dev/null @@ -1,247 +0,0 @@ -// Requires Playwright + its Chromium browser, installed outside the app. -// MIST_PLAYWRIGHT_MODULE may point to an existing Playwright index.mjs. -import fs from "node:fs"; -import path from "node:path"; - -const { chromium } = await import( - process.env.MIST_PLAYWRIGHT_MODULE || "playwright" -); -const artifacts = process.argv[2]; -if (!artifacts) - throw new Error( - "Usage: node verify_foundation_browser.mjs ARTIFACT_DIRECTORY", - ); -fs.mkdirSync(artifacts, { recursive: true }); -const url = process.env.MIST_WEB_URL || "http://127.0.0.1:3001"; -const browser = await chromium.launch({ headless: true }); -const page = await browser.newPage({ viewport: { width: 1440, height: 1080 } }); -const errors = []; -page.on("pageerror", (error) => errors.push(error.message)); -const evidence = { - started_at: new Date().toISOString(), - jobs: {}, - page_errors: errors, -}; -const run = `foundation-${Date.now()}`; - -async function api(method, route) { - const response = await page.request.fetch(url + "/api" + route, { method }); - if (!response.ok()) throw new Error(`${route}: ${await response.text()}`); - return response.json(); -} -async function waitFor(check, description, timeout = 240000) { - const deadline = Date.now() + timeout; - while (Date.now() < deadline) { - const value = await check(); - if (value) return value; - await page.waitForTimeout(1000); - } - throw new Error(`Timed out: ${description}`); -} -async function finished(id, desired = "Success") { - const job = await waitFor(async () => { - const value = await api("GET", "/jobs/" + id); - if (["Success", "Failure", "Cancelled"].includes(value.job_state)) { - if (value.job_state !== desired) throw new Error(JSON.stringify(value)); - return value; - } - }, id); - const { logs } = await api("GET", "/jobs/" + id + "/logs"); - fs.writeFileSync(path.join(artifacts, id + ".log"), logs); - return { job, logs }; -} -function card(name) { - return page - .getByRole("heading", { name, exact: true }) - .locator("..") - .locator(".."); -} -async function submit(label) { - const name = `${run}-${label}`; - await page.getByLabel("Name", { exact: true }).fill(name); - const response = page.waitForResponse( - (response) => - response.url().endsWith("/api/jobs") && - response.request().method() === "POST", - ); - await page.getByRole("button", { name: "Submit job", exact: true }).click(); - const result = await response; - if (!result.ok()) throw new Error(await result.text()); - const id = (await result.json()).job_id; - console.log(`Submitted through browser: ${label} ${id}`); - evidence.jobs[label] = { id, name }; - return { id, name }; -} -async function completed(entry, desired, marker) { - const result = await finished(entry.id, desired); - const label = - desired === "Success" - ? "Completed" - : desired === "Failure" - ? "Failed" - : "Cancelled"; - await card(entry.name) - .getByText(label, { exact: true }) - .waitFor({ timeout: 15000 }); - if (!result.logs.includes(marker)) - throw new Error("Missing log marker: " + marker); - await card(entry.name) - .getByRole("button", { name: "Logs", exact: true }) - .click(); - await page - .locator("pre") - .filter({ hasText: marker }) - .waitFor({ timeout: 15000 }); - await page.getByRole("button", { name: "Close", exact: true }).click(); - return result; -} - -try { - await page.goto(url + "/jobs"); - await page.getByRole("button", { name: "Submit job", exact: true }).waitFor(); - await waitFor( - () => - page.getByRole("button", { name: "Submit job", exact: true }).isEnabled(), - "catalog ready", - ); - evidence.hardware_before = await api("GET", "/hardware"); - - await page.getByLabel("Workload", { exact: true }).selectOption("container"); - await page - .getByLabel("Container image", { exact: true }) - .fill("mist-training:cpu-v1"); - await page.getByText("Resources and environment", { exact: true }).click(); - await page - .getByLabel("Environment variables (NAME=value, one per line)", { - exact: true, - }) - .fill("GREETING=browser environment preserved"); - const cpu = await submit("packaged-cpu"); - Object.assign( - evidence.jobs["packaged-cpu"], - await completed(cpu, "Success", "PACKAGED_TRAINING_PASSED backend=cpu"), - ); - if ( - !evidence.jobs["packaged-cpu"].logs.includes('"cwd": "/app"') || - !evidence.jobs["packaged-cpu"].logs.includes( - "browser environment preserved", - ) - ) - throw new Error("Image defaults or environment were lost"); - console.log("CPU packaged entrypoint, WORKDIR, training, and logs passed"); - - await page.getByLabel("Compute", { exact: true }).selectOption("tenstorrent"); - await page.getByLabel("Boards (two chips each)", { exact: true }).fill("2"); - await page - .getByLabel("Workload", { exact: true }) - .selectOption("training-smoke"); - const tt = await submit("tt-two-boards"); - Object.assign( - evidence.jobs["tt-two-boards"], - await completed( - tt, - "Success", - "TRAINING_PASSED backend=tenstorrent devices=4", - ), - ); - // DRA allocation details can disappear once a completed pod releases its claim. - if ( - evidence.jobs["tt-two-boards"].job.node !== "utmist-tt" || - evidence.jobs["tt-two-boards"].job.device_count !== 2 - ) - throw new Error("Incorrect TT placement/allocation"); - console.log("Tenstorrent two-board/four-chip training passed"); - - await page.getByLabel("Compute", { exact: true }).selectOption("nvidia"); - await page.getByLabel("GPUs", { exact: true }).fill("2"); - await page.getByLabel("Workload", { exact: true }).selectOption("container"); - await page - .getByLabel("Container image", { exact: true }) - .fill("mist-training:nvidia-v1"); - await page - .getByLabel("Arguments (one per line)", { exact: true }) - .fill("--backend\nnvidia\n--devices\n2"); - const nvidia = await submit("packaged-two-gpus"); - Object.assign( - evidence.jobs["packaged-two-gpus"], - await completed( - nvidia, - "Success", - "PACKAGED_TRAINING_PASSED backend=nvidia", - ), - ); - if (evidence.jobs["packaged-two-gpus"].job.node !== "utmist-z1opa08") - throw new Error("Incorrect NVIDIA placement"); - console.log("Both NVIDIA GPUs trained in a packaged image"); - - await page.getByLabel("Compute", { exact: true }).selectOption("cpu"); - await page.getByLabel("Workload", { exact: true }).selectOption("script"); - await page - .getByLabel("Python script", { exact: true }) - .fill( - "import sys\nprint('BROWSER_EXPECTED_FAILURE', flush=True)\nsys.exit(17)\n", - ); - const failed = await submit("failure"); - Object.assign( - evidence.jobs.failure, - await completed(failed, "Failure", "BROWSER_EXPECTED_FAILURE"), - ); - if (evidence.jobs.failure.job.exit_code !== 17) - throw new Error("Missing failed exit code"); - - await page - .getByLabel("Python script", { exact: true }) - .fill( - "import time\nprint('BROWSER_CANCEL_PRESERVED', flush=True)\ntime.sleep(120)\n", - ); - const cancelled = await submit("cancel"); - await waitFor( - async () => - (await api("GET", "/jobs/" + cancelled.id + "/logs")).logs.includes( - "BROWSER_CANCEL_PRESERVED", - ), - "cancel job running", - ); - await card(cancelled.name) - .getByRole("button", { name: "Cancel", exact: true }) - .click(); - Object.assign( - evidence.jobs.cancel, - await completed(cancelled, "Cancelled", "BROWSER_CANCEL_PRESERVED"), - ); - console.log("Browser failure and cancellation with retained logs passed"); - - await page.evaluate(() => window.scrollTo(0, 0)); - await page.screenshot({ path: path.join(artifacts, "jobs-browser.png") }); - await card(nvidia.name) - .getByRole("button", { name: "Logs", exact: true }) - .click(); - await page - .locator("pre") - .filter({ hasText: "PACKAGED_TRAINING_PASSED backend=nvidia" }) - .waitFor(); - await page - .locator("pre") - .screenshot({ path: path.join(artifacts, "nvidia-training-browser.png") }); - await page.goto(url + "/machines"); - await page - .getByRole("rowheader", { name: "Tenstorrent boards", exact: true }) - .waitFor(); - await page.screenshot({ path: path.join(artifacts, "machines-browser.png") }); - evidence.hardware_after = await api("GET", "/hardware"); - if (errors.length) throw new Error("Browser errors: " + errors.join("\n")); - evidence.completed_at = new Date().toISOString(); - console.log("FOUNDATION_BROWSER_PASSED"); -} catch (error) { - evidence.error = String(error); - await page - .screenshot({ path: path.join(artifacts, "browser-failure.png") }) - .catch(() => {}); - throw error; -} finally { - fs.writeFileSync( - path.join(artifacts, "browser.json"), - JSON.stringify(evidence, null, 2) + "\n", - ); - await browser.close(); -} diff --git a/deploy/k3s/verify_mist_api.py b/deploy/k3s/verify_mist_api.py deleted file mode 100644 index bf463a7..0000000 --- a/deploy/k3s/verify_mist_api.py +++ /dev/null @@ -1,187 +0,0 @@ -"""Live Mist API acceptance: real CPU/CUDA/TT jobs, queues, cancellation, logs. - -The API submits every workload. An admin kubectl connection is used only to -inspect reservations and release the training-test synchronization barriers. -No accelerator can fall back to CPU. Jobs and checkpoints are retained. -""" - -import argparse -import json -from pathlib import Path -import subprocess -import time -import urllib.error -import urllib.request -from datetime import datetime, timezone - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--url", default="http://127.0.0.1:3000") - parser.add_argument("--artifacts", type=Path, required=True) - args = parser.parse_args() - args.artifacts.mkdir(parents=True, exist_ok=True) - run = datetime.now(timezone.utc).strftime("api-%m%d-%H%M%S") - evidence = {"run": run, "checks": {}, "jobs": {}} - - def api(method, path, body=None, expected=200): - data = json.dumps(body).encode() if body is not None else None - request = urllib.request.Request(args.url + path, data=data, method=method, - headers={"Content-Type": "application/json"}) - try: - response = urllib.request.urlopen(request, timeout=45) - except urllib.error.HTTPError as error: - response = error - with response: - result = json.load(response) - assert response.code == expected, (path, response.code, result) - return result - - def kubectl(*options): - return subprocess.check_output(["k3s", "kubectl", *options], text=True, timeout=45) - - def wait(check, description, timeout=240): - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - value = check() - if value: - return value - time.sleep(1) - raise AssertionError("Timed out: " + description) - - def submit(label, **request): - result = api("POST", "/jobs", {"name": run + "-" + label, **request}, expected=201) - evidence["jobs"][label] = result["job_id"] - print(f"Submitted {label}: {result['job_id']}", flush=True) - return result["job_id"] - - def status(job): - return api("GET", "/jobs/" + job) - - def logs(job): - return api("GET", "/jobs/" + job + "/logs")["logs"] - - def finished(job, desired="Success", timeout=240): - def check(): - value = status(job) - if value["job_state"] in ["Success", "Failure", "Cancelled"]: - assert value["job_state"] == desired, value - return value - value = wait(check, job + " complete", timeout) - text = logs(job) - (args.artifacts / (job + ".log")).write_text(text) - evidence["checks"][job] = {"status": value, "logs": text} - return value, text - - def healthy(): - try: - return api("GET", "/healthz")["executor"] == "kubernetes" - except urllib.error.URLError: - return False - wait(healthy, "API reachable after deployment", timeout=60) - before = api("GET", "/jobs")["count"] - for request in [ - {"command": ["true"], "accelerator": "nvidia", "device_count": 3}, - {"command": ["true"], "accelerator": "tenstorrent", "device_count": 5}, - {"command": ["true"], "privileged": True}, - {"command": ["true"], "cpu": "9"}, - {"command": ["true"], "memory": "1Ti"}, - {"command": ["true"], "image": "unapproved/image:latest"}, - {"script": "print(1)", "script_name": "../escape.py"}, - ]: - api("POST", "/jobs", request, expected=400) - assert api("GET", "/jobs")["count"] == before - api("GET", "/jobs/training-tenstorrent", expected=404) - api("POST", "/jobs/training-tenstorrent/cancel", expected=404) - evidence["validation_rejected_without_jobs"] = True - - cpu = submit("cpu", accelerator="cpu", script="""import os,pathlib -p=pathlib.Path(os.environ['MIST_CHECKPOINT_DIR']); p.mkdir(parents=True,exist_ok=True) -(p/'output.txt').write_text('persistent CPU output') -print('CPU_API_PASSED',flush=True) -""") - value, text = finished(cpu) - assert value["exit_code"] == 0 and "CPU_API_PASSED" in text - failed = submit("failure", accelerator="cpu", script="import sys; print('EXPECTED_FAILURE',flush=True); sys.exit(17)") - value, text = finished(failed, "Failure") - assert value["exit_code"] == 17 and "EXPECTED_FAILURE" in text - cancelled = submit("cancel", accelerator="cpu", script="import time; print('CANCEL_LOG_PRESERVED',flush=True); time.sleep(120)") - wait(lambda: "CANCEL_LOG_PRESERVED" in logs(cancelled), "cancel workload started") - assert api("POST", "/jobs/" + cancelled + "/cancel")["job_state"] == "Cancelled" - assert api("POST", "/jobs/" + cancelled + "/cancel")["job_state"] == "Cancelled" - def stopped(): - pods = json.loads(kubectl("-n", "mist", "get", "pods", "-l", "batch.kubernetes.io/job-name=" + cancelled, "-o", "json"))["items"] - return not pods or all(p["status"]["phase"] not in ["Running", "Pending"] for p in pods) - wait(stopped, "cancelled pod stopped", timeout=45) - value, text = finished(cancelled, "Cancelled") - assert "CANCEL_LOG_PRESERVED" in text - deadline = submit("deadline", accelerator="cpu", command=["python", "-c", "import time; time.sleep(60)"], timeout_seconds=10) - wait(lambda: "DeadlineExceeded" in status(deadline).get("message", ""), "deadline reported by controller", timeout=60) - value, _ = finished(deadline, "Failure", timeout=60) - assert "DeadlineExceeded" in value["message"], value - print("CPU execution, failure, cancellation, retained logs, and deadline passed", flush=True) - - def training_group(accelerator, parallel): - # Each workload now mounts only its own output directory. Release each - # initialized pod's barrier individually, rather than sharing a PVC root. - barrier = "/outputs/start" - group = [submit(f"{accelerator}-{i}", type="training-smoke", accelerator=accelerator, - device_count=1, args=["--steps", "500", "--start-file", barrier]) for i in range(parallel)] - for job in group: - wait(lambda: "BARRIER_READY" in logs(job), job + " initialized") - snapshots = [status(job) for job in group] - assert all(s["job_state"] == "InProgress" and s["device_count"] == 1 for s in snapshots) - allocations = [] - for snapshot in snapshots: - allocation = json.loads(kubectl("-n", "mist", "exec", snapshot["pod"], "--", "cat", snapshot["checkpoint_directory"] + "/allocation.json")) - assert len(allocation["device_nodes"]) == 1 - allocations.append(allocation) - assert len({a["device_nodes"][0] for a in allocations}) == parallel, allocations - if accelerator == "tenstorrent": - devices = [s["devices"][0]["device"] for s in snapshots] - assert len(set(devices)) == 4 - fifth = submit(accelerator + "-queued", type="training-smoke", accelerator=accelerator, - device_count=1, args=["--steps", "500"]) - def queued(): - snapshot = status(fifth) - if snapshot["job_state"] == "Scheduled" and snapshot.get("message"): - assert not snapshot.get("node"), snapshot - reason = "cannot allocate" if accelerator == "tenstorrent" else "Insufficient nvidia.com/gpu" - assert reason in snapshot["message"], snapshot - return snapshot - queue = wait(queued, accelerator + " job queued", timeout=60) - for snapshot in snapshots: - kubectl("-n", "mist", "exec", snapshot["pod"], "--", "touch", barrier) - for job in group + [fifth]: - snapshot, text = finished(job) - chips = 2 if accelerator == "tenstorrent" else 1 - assert snapshot["exit_code"] == 0 and f"TRAINING_PASSED backend={accelerator} devices={chips}" in text - evidence[accelerator + "_concurrency"] = {"initial_status": snapshots, "allocations": allocations, - "queued_status": queue, "all_training_passed": True} - print(f"{accelerator}: {parallel} distinct reservations trained; queued job automatically reused hardware", flush=True) - - training_group("nvidia", 2) - training_group("tenstorrent", 4) - for accelerator, count, chips in [("nvidia", 2, 2), ("tenstorrent", 2, 4), ("tenstorrent", 4, 8)]: - job = submit(f"{accelerator}-full-{count}", type="training-smoke", accelerator=accelerator, device_count=count) - value, text = finished(job) - assert value["exit_code"] == 0 and f"TRAINING_PASSED backend={accelerator} devices={chips}" in text - # Cancelling a whole-QuietBox reservation must unblock another TT job. - holder = submit("tt-cancel-whole", accelerator="tenstorrent", device_count=4, - script="import time; print('WHOLE_TT_RESERVED',flush=True); time.sleep(120)") - wait(lambda: "WHOLE_TT_RESERVED" in logs(holder), "whole TT reservation running") - queued = submit("tt-after-cancel", type="training-smoke", accelerator="tenstorrent", device_count=1) - wait(lambda: "cannot allocate" in status(queued).get("message", ""), "TT job queued behind holder", timeout=60) - assert api("POST", "/jobs/" + holder + "/cancel")["job_state"] == "Cancelled" - finished(holder, "Cancelled") - finished(queued) - evidence["cancelled_tt_reservation_released"] = True - listed = api("GET", "/jobs")["jobs"] - assert set(evidence["jobs"].values()).issubset({job["id"] for job in listed}) - evidence["completed_at"] = datetime.now(timezone.utc).isoformat() - (args.artifacts / "api-acceptance.json").write_text(json.dumps(evidence, indent=2) + "\n") - print("MIST_API_PASSED: CPU, NVIDIA, Tenstorrent, queues, cancellation, logs, and checkpoints", flush=True) - - -if __name__ == "__main__": - main() diff --git a/deploy/k3s/verify_quietbox_submission.py b/deploy/k3s/verify_quietbox_submission.py deleted file mode 100644 index 89f613f..0000000 --- a/deploy/k3s/verify_quietbox_submission.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Authenticate from QuietBox, train on NVIDIA node, then download the result. - -Credentials arrive on standard input, never shell arguments or a remote file. -Run with a protected JSON credential file redirected into stdin. The caller -copies this script to QuietBox and passes only the source path to SSH. -""" -import http.cookiejar -import json -import sys -import time -import urllib.request - -BASE = 'http://100.73.139.66:8088' -credentials = json.load(sys.stdin) -jar = http.cookiejar.CookieJar() -client = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(jar)) - -def request(path, body=None): - data = None if body is None else json.dumps(body).encode() - req = urllib.request.Request(BASE + path, data=data, - headers={'Content-Type': 'application/json', 'Origin': BASE}) - with client.open(req, timeout=30) as response: - return json.load(response) - -request('/auth/sign-in/email', {'email': credentials['email'], 'password': credentials['password']}) -ds = request('/api/datasets')['datasets'][0] -script = """import json -from pathlib import Path -import torch -assert torch.cuda.is_available() -assert torch.cuda.device_count() == 1 -d=json.loads(Path('/inputs/regression.json').read_text()) -x=torch.tensor(d['x'],device='cuda') -y=torch.tensor(d['y'],device='cuda') -w=torch.zeros((32,32),device='cuda',requires_grad=True) -optimizer=torch.optim.SGD([w],lr=0.2) -initial=(x@w-y).square().mean().item() -for _ in range(100): - optimizer.zero_grad() - loss=(x@w-y).square().sum()/(2*x.shape[0]) - loss.backward() - assert w.grad.is_cuda - optimizer.step() -final=(x@w-y).square().mean().item() -assert final . -docker run -it --rm --gpus all bash -``` -## Test -``` -run_tests.sh -run_container_job_test.sh -``` -## Troubleshooting - -### Unknown or invalid runtime name: nvidia -- Add this do /etc/docker/daemon.json -``` -{ - "default-runtime": "runc", - "runtimes": { - "nvidia": { - "path": "/usr/bin/nvidia-container-runtime", - "runtimeArgs": [] - } - } -} -``` - -- restart docker engine: sudo systemctl restart docker - - -### nvidia-smi: command not found -- download [nvidia container toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html#installing-the-nvidia-container-toolkit) - -# mist/images Package Documentation - -## Overview - -The `mist/images` package provides a `DockerMgr` struct and related methods for managing Docker containers and volumes programmatically using the Docker Go SDK. It enforces limits on the number of containers and volumes, and provides safe creation, deletion, and lifecycle management. - ---- - -## Main APIs - -### `type DockerMgr struct` -Manages Docker containers and volumes, enforces resource limits, and tracks active resources. - -**Fields:** -- `ctx context.Context` — Context for Docker operations. -- `cli *client.Client` — Docker client. -- `containerLimit int` — Maximum allowed containers. -- `volumeLimit int` — Maximum allowed volumes. -- `containers map[string]struct{}` — Tracks active container IDs. -- `volumes map[string]struct{}` — Tracks active volume names. -- `mu sync.Mutex` — Mutex for concurrency protection. - ---- - -### `func NewDockerMgr(client *client.Client, containerLimit, volumeLimit int) *DockerMgr` -Creates a new `DockerMgr` with the specified Docker client and resource limits. - ---- - -### `func (mgr *DockerMgr) createVolume(volumeName string) (volume.Volume, error)` -Creates a Docker volume with the given name, enforcing the volume limit. -Returns the created volume or an error. - ---- - -### `func (mgr *DockerMgr) removeVolume(volumeName string, force bool) error` -Removes a Docker volume by name. -Returns an error if the volume does not exist or is in use (unless `force` is true). - ---- - -### `func (mgr *DockerMgr) runContainer(imageName string, runtimeName string, volumeName string) (string, error)` -Creates and starts a container with the specified image, runtime, and volume attached at `/data`. -Enforces the container limit. -Returns the container ID or an error. - ---- - -### `func (mgr *DockerMgr) stopContainer(containerID string) error` -Stops a running container by ID. -Returns an error if the operation fails. - ---- - -### `func (mgr *DockerMgr) removeContainer(containerID string) error` -Removes a container by ID and deletes it from the internal tracking map. -Returns an error if the operation fails. - ---- - -## Test Plan - -The test suite (`docker_test.go`) covers the following scenarios: - -- Create a volume, check it exists, delete it, check it no longer exists. -- Create a volume with the same name twice (should not fail). -- Remove a volume that doesn't exist (should fail or return error). -- Remove a volume in use (should fail or return error). -- Attach a volume that does not exist (should fail or return error). -- Two containers attach to the same volume (should succeed in Docker, but test for your policy). -- Two containers try to attach to the same volume at the same time (should succeed in Docker). -- Set a limit of 100 volumes (should fail on 101st if you enforce a limit). -- Set a limit of 10 containers (should fail on 11th if you enforce a limit). - ---- - -## Example Usage - -```go -cli, _ := client.NewClientWithOpts(client.FromEnv) -mgr := NewDockerMgr(cli, 10, 100) - -vol, err := mgr.createVolume("myvol") -if err != nil { /* handle error */ } - -cid, err := mgr.runContainerCuda("myvol") -if err != nil { /* handle error */ } - -_ = mgr.stopContainer(cid) -_ = mgr.removeContainer(cid) -_ = mgr.removeVolume("myvol", true) -``` - ---- - -## Notes - -- All resource-creating methods enforce limits and track resources in maps for accurate management. -- All destructive operations (`removeVolume`, `removeContainer`) return errors for non-existent or in-use resources. -- The package is designed for integration with Docker and expects a running Docker daemon. - ---- - -**For more details, see the source code and comments in `docker.go`. \ No newline at end of file diff --git a/src/docker/docker.go b/src/docker/docker.go deleted file mode 100644 index b8c19b9..0000000 --- a/src/docker/docker.go +++ /dev/null @@ -1,178 +0,0 @@ -package docker - -import ( - "context" - "fmt" - "log/slog" - "sync" - - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/mount" - "github.com/docker/docker/api/types/volume" - "github.com/docker/docker/client" -) - -// DockerMgr manages Docker containers and volumes, enforces resource limits, and tracks active resources. -type DockerMgr struct { - ctx context.Context - cli *client.Client - containerLimit int - volumeLimit int - containers map[string]struct{} - volumes map[string]struct{} - mu sync.Mutex -} - -// NewDockerMgr creates a new DockerMgr with the specified Docker client and resource limits. -func NewDockerMgr(client *client.Client, containerLimit, volumeLimit int) *DockerMgr { - return &DockerMgr{ - ctx: context.Background(), - cli: client, - containerLimit: containerLimit, - volumeLimit: volumeLimit, - containers: make(map[string]struct{}), - volumes: make(map[string]struct{}), - } -} - -// StopContainer stops a running container by its ID. -// Returns an error if the operation fails. -func (mgr *DockerMgr) StopContainer(containerID string) error { - ctx := mgr.ctx - cli := mgr.cli - - err := cli.ContainerStop(ctx, containerID, container.StopOptions{}) - if err != nil { - slog.Error("Failed to stop container", "containerID", containerID, "error", err) - return err - } - return nil -} - -// RemoveContainer removes a container by its ID and deletes it from the internal tracking map. -// Returns an error if the operation fails. -func (mgr *DockerMgr) RemoveContainer(containerID string) error { - mgr.mu.Lock() - defer mgr.mu.Unlock() - ctx := mgr.ctx - cli := mgr.cli - err := cli.ContainerRemove(ctx, containerID, container.RemoveOptions{RemoveVolumes: true}) - if err != nil { - slog.Error("Failed to remove container", "containerID", containerID, "error", err) - return err - } - delete(mgr.containers, containerID) - return nil -} - -// CreateVolume creates a Docker volume with the given name, enforcing the volume limit. -// Returns the created volume or an error. -func (mgr *DockerMgr) CreateVolume(volumeName string) (volume.Volume, error) { - mgr.mu.Lock() - defer mgr.mu.Unlock() - if len(mgr.volumes) >= mgr.volumeLimit { - slog.Warn("Volume limit reached", "limit", mgr.volumeLimit) - return volume.Volume{}, fmt.Errorf("volume limit reached") - } - ctx := mgr.ctx - cli := mgr.cli - - vol, err := cli.VolumeCreate(ctx, volume.CreateOptions{Name: volumeName}) - if err != nil { - slog.Error("Failed to create volume", "volumeName", volumeName, "error", err) - return volume.Volume{}, err - } - mgr.volumes[vol.Name] = struct{}{} - return vol, nil -} - -// RemoveVolume removes a Docker volume by name. -// Returns an error if the volume does not exist or is in use (unless force is true). -func (mgr *DockerMgr) RemoveVolume(volumeName string, force bool) error { - mgr.mu.Lock() - defer mgr.mu.Unlock() - ctx := mgr.ctx - cli := mgr.cli - - vols, _ := cli.VolumeList(ctx, volume.ListOptions{}) - found := false - for _, v := range vols.Volumes { - if v.Name == volumeName { - found = true - break - } - } - if !found { - slog.Warn("Volume does not exist", "volumeName", volumeName) - return fmt.Errorf("volume %s does not exist", volumeName) - } - - err := cli.VolumeRemove(ctx, volumeName, force) - if err != nil { - slog.Error("Failed to remove volume", "volumeName", volumeName, "error", err) - return err - } - delete(mgr.volumes, volumeName) - return nil -} - -// RunContainer creates and starts a container with the specified image, runtime, and volume attached at /data. -// Enforces the container limit and checks that the volume exists. -// Returns the container ID or an error. -func (mgr *DockerMgr) RunContainer(imageName string, runtimeName string, volumeName string) (string, error) { - mgr.mu.Lock() - defer mgr.mu.Unlock() - if len(mgr.containers) >= mgr.containerLimit { - slog.Warn("Container limit reached", "limit", mgr.containerLimit) - return "", fmt.Errorf("container limit reached") - } - ctx := mgr.ctx - cli := mgr.cli - - vols, _ := cli.VolumeList(ctx, volume.ListOptions{}) - found := false - for _, v := range vols.Volumes { - if v.Name == volumeName { - found = true - break - } - } - if !found { - slog.Error("Volume does not exist for container", "volumeName", volumeName) - return "", fmt.Errorf("volume %s does not exist", volumeName) - } - - resp, err := cli.ContainerCreate( - ctx, - &container.Config{ - Image: imageName, - Cmd: []string{"sleep", "1000"}, - }, - &container.HostConfig{ - Runtime: runtimeName, - Mounts: []mount.Mount{ - { - Type: mount.TypeVolume, - Source: volumeName, - Target: "/data", - }, - }, - }, - nil, - nil, - "", - ) - - if err != nil { - slog.Error("Failed to create container", "imageName", imageName, "runtimeName", runtimeName, "volumeName", volumeName, "error", err) - return "", err - } - - mgr.containers[resp.ID] = struct{}{} - if err := cli.ContainerStart(ctx, resp.ID, container.StartOptions{}); err != nil { - slog.Error("Failed to start container", "containerID", resp.ID, "error", err) - return "", err - } - - return resp.ID, nil -} diff --git a/src/docker/docker_test.go b/src/docker/docker_test.go deleted file mode 100644 index 63f897d..0000000 --- a/src/docker/docker_test.go +++ /dev/null @@ -1,300 +0,0 @@ -package docker - -import ( - "fmt" - "testing" - - "github.com/docker/docker/api/types/volume" - "github.com/docker/docker/client" -) - -func setupMgr(t *testing.T) *DockerMgr { - cli, err := client.NewClientWithOpts(client.FromEnv) - if err != nil { - t.Fatalf("Failed to create Docker client: %v", err) - } - return NewDockerMgr(cli, 10, 100) -} - -// cpuImageAndRuntime returns pytorch-cpu and runc for CPU-only tests. Skips if image not found. -func cpuImageAndRuntime(t *testing.T, mgr *DockerMgr) (imageName, runtimeName string) { - _, _, err := mgr.cli.ImageInspectWithRaw(mgr.ctx, "pytorch-cpu") - if err != nil { - t.Skipf("pytorch-cpu image not found. Build with: cd pytorch-cpu && docker build -t pytorch-cpu .") - } - return "pytorch-cpu", "runc" -} - -// Create a volume, check exists, delete, check not exists -func TestCreateDeleteVolume(t *testing.T) { - mgr := setupMgr(t) - volName := "test_volume_t1" - _, err := mgr.CreateVolume(volName) - if err != nil { - t.Errorf("Failed to create volume %s: %v", volName, err) - } - vols, _ := mgr.cli.VolumeList(mgr.ctx, volume.ListOptions{}) - found := false - for _, v := range vols.Volumes { - if v.Name == volName { - found = true - break - } - } - if !found { - t.Errorf("Volume %s not found after creation", volName) - } - err = mgr.RemoveVolume(volName, true) - if err != nil { - t.Errorf("Failed to remove volume %s: %v", volName, err) - } - vols, _ = mgr.cli.VolumeList(mgr.ctx, volume.ListOptions{}) - for _, v := range vols.Volumes { - if v.Name == volName { - t.Errorf("Volume %s still exists after deletion", volName) - } - } -} - -// Create a volume with same name twice (should not fail) -func TestCreateVolumeTwice(t *testing.T) { - mgr := setupMgr(t) - volName := "test_volume_t3" - _, err := mgr.CreateVolume(volName) - if err != nil { - t.Errorf("Failed to create volume %s: %v", volName, err) - } - defer mgr.RemoveVolume(volName, true) - _, err = mgr.CreateVolume(volName) - if err != nil { - t.Errorf("Failed to create volume %s a second time: %v", volName, err) - } -} - -// Remove volume that doesn't exist (should fail or panic) -func TestRemoveNonexistentVolume(t *testing.T) { - mgr := setupMgr(t) - err := mgr.RemoveVolume("nonexistent_volume_t4", true) - if err == nil { - t.Errorf("Expected error when removing nonexistent volume, but no error") - } else { - t.Logf("Correctly got error when removing nonexistent volume: %v", err) - } -} - -// TestRunContainerCPU verifies running a CPU container (pytorch-cpu + runc). -// Use this on machines without GPU. Requires: docker build -t pytorch-cpu ./pytorch-cpu -func TestRunContainerCPU(t *testing.T) { - mgr := setupMgr(t) - imageName, runtimeName := cpuImageAndRuntime(t, mgr) - volName := "test_volume_cpu" - _, err := mgr.CreateVolume(volName) - if err != nil { - t.Fatalf("Failed to create volume %s: %v", volName, err) - } - defer mgr.RemoveVolume(volName, true) - containerID, err := mgr.RunContainer(imageName, runtimeName, volName) - if err != nil { - t.Fatalf("Failed to start CPU container: %v", err) - } - defer func() { - _ = mgr.StopContainer(containerID) - _ = mgr.RemoveContainer(containerID) - }() - t.Logf("CPU container started: %s", containerID[:12]) -} - -// Remove volume in use (should fail or panic) -func TestRemoveVolumeInUse(t *testing.T) { - mgr := setupMgr(t) - imageName, runtimeName := cpuImageAndRuntime(t, mgr) - volName := "test_volume_t5" - _, err := mgr.CreateVolume(volName) - if err != nil { - t.Fatalf("Failed to create volume %s: %v", volName, err) - } - containerID, err := mgr.RunContainer(imageName, runtimeName, volName) - if err != nil { - t.Fatalf("Failed to start container: %v", err) - } - defer func() { - // Cleanup: stop and remove container, then remove volume - if err := mgr.StopContainer(containerID); err != nil { - t.Logf("Cleanup: failed to stop container %s: %v", containerID, err) - } else { - t.Logf("Cleanup: stopped container %s successfully", containerID) - } - if err := mgr.RemoveContainer(containerID); err != nil { - t.Logf("Cleanup: failed to remove container %s: %v", containerID, err) - } else { - t.Logf("Cleanup: removed container %s successfully", containerID) - } - if err := mgr.RemoveVolume(volName, true); err != nil { - t.Logf("Cleanup: failed to remove volume %s: %v", volName, err) - } else { - t.Logf("Cleanup: removed volume %s successfully", volName) - } - }() - err = mgr.RemoveVolume(volName, true) // Should error: volume is in use by a running container - if err == nil { - t.Errorf("Expected error when removing volume in use, but no error") - } else { - t.Logf("Correctly got error when removing volume in use: %v", err) - } -} - -// Attach a volume that does not exist (should fail or panic) -func TestAttachNonexistentVolume(t *testing.T) { - mgr := setupMgr(t) - imageName, runtimeName := cpuImageAndRuntime(t, mgr) - volName := "nonexistent_volume_t6" - id, err := mgr.RunContainer(imageName, runtimeName, volName) - // If Docker auto-creates the volume, this may not error; check your policy - if id != "" && err != nil { - t.Errorf("Expected error when attaching nonexistent volume, but got id=%v, err=%v", id, err) - } else if err != nil { - t.Logf("Correctly got error when attaching nonexistent volume: %v", err) - } -} - -// Two containers attach to the same volume (should succeed in Docker, but test for your policy) -func TestTwoContainersSameVolume(t *testing.T) { - mgr := setupMgr(t) - imageName, runtimeName := cpuImageAndRuntime(t, mgr) - volName := "test_volume_t7" - _, err := mgr.CreateVolume(volName) - if err != nil { - t.Fatalf("Failed to create volume %s: %v", volName, err) - } - id1, err := mgr.RunContainer(imageName, runtimeName, volName) - if err != nil { - t.Fatalf("Failed to start first container: %v", err) - } - id2, err := mgr.RunContainer(imageName, runtimeName, volName) - if err != nil { - t.Fatalf("Failed to start second container: %v", err) - } - // Both containers should be able to use the same volume - if err := mgr.StopContainer(id1); err != nil { - t.Logf("Failed to stop first container: %v", err) - } - if err := mgr.RemoveContainer(id1); err != nil { - t.Logf("Failed to remove first container: %v", err) - } - if err := mgr.StopContainer(id2); err != nil { - t.Logf("Failed to stop second container: %v", err) - } - if err := mgr.RemoveContainer(id2); err != nil { - t.Logf("Failed to remove second container: %v", err) - } - if err := mgr.RemoveVolume(volName, true); err != nil { - t.Logf("Failed to remove volume %s: %v", volName, err) - } -} - -// Two containers try to attach to the same volume at the same time (should succeed in Docker) -func TestTwoContainersSameVolumeConcurrent(t *testing.T) { - mgr := setupMgr(t) - imageName, runtimeName := cpuImageAndRuntime(t, mgr) - volName := "test_volume_t8" - _, err := mgr.CreateVolume(volName) - if err != nil { - t.Fatalf("Failed to create volume %s: %v", volName, err) - } - id1, err := mgr.RunContainer(imageName, runtimeName, volName) - if err != nil { - t.Fatalf("Failed to start first container: %v", err) - } - id2, err2 := mgr.RunContainer(imageName, runtimeName, volName) - if err2 != nil { - t.Fatalf("Failed to start second container: %v", err2) - } - // This test does not actually run containers concurrently, but checks Docker's shared volume support - if err := mgr.StopContainer(id1); err != nil { - t.Logf("Failed to stop first container: %v", err) - } - if err := mgr.RemoveContainer(id1); err != nil { - t.Logf("Failed to remove first container: %v", err) - } - if err := mgr.StopContainer(id2); err != nil { - t.Logf("Failed to stop second container: %v", err) - } - if err := mgr.RemoveContainer(id2); err != nil { - t.Logf("Failed to remove second container: %v", err) - } - if err := mgr.RemoveVolume(volName, true); err != nil { - t.Logf("Failed to remove volume %s: %v", volName, err) - } -} - -// Set a limit of 100 volumes (should fail on 101st if you enforce a limit) -func TestVolumeLimit(t *testing.T) { - mgr := setupMgr(t) - limit := 100 - created := []string{} - for i := 0; i < limit; i++ { - name := "test_volume_t9_" + fmt.Sprint(i) - _, err := mgr.CreateVolume(name) - if err != nil { - t.Fatalf("Failed to create volume %s: %v", name, err) - } - created = append(created, name) - } - name := "test_volume_fail" - _, err := mgr.CreateVolume(name) - if err == nil { - t.Errorf("Volume limit not enforced") - } else { - t.Logf("Correctly got error when exceeding volume limit: %v", err) - } - - defer func() { - // Cleanup: remove all created volumes - for _, name := range created { - if err := mgr.RemoveVolume(name, true); err != nil { - t.Logf("Cleanup: failed to remove volume %s: %v", name, err) - } - } - }() - // If your implementation doesn't enforce a limit, this test will fail -} - -// Set a limit of 10 containers (should fail on 11th if you enforce a limit) -func TestContainerLimit(t *testing.T) { - mgr := setupMgr(t) - imageName, runtimeName := cpuImageAndRuntime(t, mgr) - volName := "test_volume_t10" - _, err := mgr.CreateVolume(volName) - if err != nil { - t.Fatalf("Failed to create volume %s: %v", volName, err) - } - ids := []string{} - limit := 10 - for i := 0; i < limit; i++ { - id, err := mgr.RunContainer(imageName, runtimeName, volName) - if err != nil { - t.Fatalf("Failed to start container %d: %v", i, err) - } - ids = append(ids, id) - } - _, err = mgr.RunContainer(imageName, runtimeName, volName) - if err == nil { - t.Errorf("Container limit not enforced") - } else { - t.Logf("Correctly got error when exceeding container limit: %v", err) - } - defer func() { - // Cleanup: stop and remove all containers, then remove the volume - for _, id := range ids { - if err := mgr.StopContainer(id); err != nil { - t.Logf("Cleanup: failed to stop container %s: %v", id, err) - } - if err := mgr.RemoveContainer(id); err != nil { - t.Logf("Cleanup: failed to remove container %s: %v", id, err) - } - } - if err := mgr.RemoveVolume(volName, true); err != nil { - t.Logf("Cleanup: failed to remove volume %s: %v", volName, err) - } - }() -} diff --git a/src/docker/go.mod b/src/docker/go.mod deleted file mode 100644 index 4a29fc2..0000000 --- a/src/docker/go.mod +++ /dev/null @@ -1,35 +0,0 @@ -module mist/docker - -go 1.24.6 - -require github.com/docker/docker v28.2.2+incompatible - -require ( - github.com/Microsoft/go-winio v0.4.21 // indirect - github.com/containerd/errdefs v1.0.0 // indirect - github.com/containerd/errdefs/pkg v0.3.0 // indirect - github.com/containerd/log v0.1.0 // indirect - github.com/distribution/reference v0.6.0 // indirect - github.com/docker/go-connections v0.6.0 // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/go-logr/logr v1.4.3 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/sys/atomicwriter v0.1.0 // indirect - github.com/moby/term v0.5.2 // indirect - github.com/morikuni/aec v1.0.0 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/pkg/errors v0.9.1 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect - go.opentelemetry.io/otel v1.38.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 // indirect - go.opentelemetry.io/otel/metric v1.38.0 // indirect - go.opentelemetry.io/otel/trace v1.38.0 // indirect - golang.org/x/sys v0.35.0 // indirect - golang.org/x/time v0.12.0 // indirect - gotest.tools/v3 v3.5.2 // indirect -) diff --git a/src/docker/go.sum b/src/docker/go.sum deleted file mode 100644 index be7bb0e..0000000 --- a/src/docker/go.sum +++ /dev/null @@ -1,132 +0,0 @@ -github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= -github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Microsoft/go-winio v0.4.21 h1:+6mVbXh4wPzUrl1COX9A+ZCvEpYsOBZ6/+kwDnvLyro= -github.com/Microsoft/go-winio v0.4.21/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= -github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= -github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= -github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= -github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= -github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= -github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= -github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= -github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw= -github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= -github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= -github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= -github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= -github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= -github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= -github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= -github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= -go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= -go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4= -go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= -go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= -go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= -go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= -go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= -go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= -go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= -go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= -go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= -go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= -golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= -golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= -google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= -google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= -google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= -google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= -google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= -gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= diff --git a/src/docker/pytorch-cpu/dockerfile b/src/docker/pytorch-cpu/dockerfile deleted file mode 100644 index 0901357..0000000 --- a/src/docker/pytorch-cpu/dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM pytorch/pytorch:latest - -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - -RUN useradd -ms /bin/bash guest -USER guest -WORKDIR /home/guest diff --git a/src/docker/pytorch-cpu/pytorch-cpu_config.yaml b/src/docker/pytorch-cpu/pytorch-cpu_config.yaml deleted file mode 100644 index 6a5d0c4..0000000 --- a/src/docker/pytorch-cpu/pytorch-cpu_config.yaml +++ /dev/null @@ -1,12 +0,0 @@ -schemaVersion: '2.0.0' - -commandTests: - - name: "Check Sudo" - command: "which" - args: ["sudo"] - exitCode: 1 - - - name: "CUDA availability check" - command: "python" - args: ["-c", "import torch; print(torch.cuda.is_available())"] - expectedOutput: ["False"] \ No newline at end of file diff --git a/src/docker/pytorch-cpu/requirements.txt b/src/docker/pytorch-cpu/requirements.txt deleted file mode 100644 index b2a0911..0000000 --- a/src/docker/pytorch-cpu/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -pandas -scipy -matplotlib diff --git a/src/docker/pytorch-cpu/run_test.sh b/src/docker/pytorch-cpu/run_test.sh deleted file mode 100644 index 3963fbd..0000000 --- a/src/docker/pytorch-cpu/run_test.sh +++ /dev/null @@ -1,2 +0,0 @@ -image=$1 -container-structure-test test --image $image --config ${image}_config.yaml \ No newline at end of file diff --git a/src/docker/pytorch-cuda/dockerfile b/src/docker/pytorch-cuda/dockerfile deleted file mode 100644 index 68a8f34..0000000 --- a/src/docker/pytorch-cuda/dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM pytorch/pytorch:2.2.0-cuda12.1-cudnn8-runtime - -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - -RUN useradd -ms /bin/bash guest -USER guest -WORKDIR /home/guest diff --git a/src/docker/pytorch-cuda/pytorch-cuda_config.yaml b/src/docker/pytorch-cuda/pytorch-cuda_config.yaml deleted file mode 100644 index 5ae3ac3..0000000 --- a/src/docker/pytorch-cuda/pytorch-cuda_config.yaml +++ /dev/null @@ -1,12 +0,0 @@ -schemaVersion: '2.0.0' - -commandTests: - - name: "Check Sudo" - command: "which" - args: ["sudo"] - exitCode: 1 - - - name: "CUDA availability check" - command: "python" - args: ["-c", "import torch; print(torch.cuda.is_available())"] - expectedOutput: ["True"] \ No newline at end of file diff --git a/src/docker/pytorch-cuda/requirements.txt b/src/docker/pytorch-cuda/requirements.txt deleted file mode 100644 index b2a0911..0000000 --- a/src/docker/pytorch-cuda/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -pandas -scipy -matplotlib diff --git a/src/docker/pytorch-cuda/run_test.sh b/src/docker/pytorch-cuda/run_test.sh deleted file mode 100644 index 029c373..0000000 --- a/src/docker/pytorch-cuda/run_test.sh +++ /dev/null @@ -1,2 +0,0 @@ -image=$1 -container-structure-test test --image $image --config ${image}_config.yaml --runtime nvidia \ No newline at end of file diff --git a/src/docker/pytorch-rocm/dockerfile b/src/docker/pytorch-rocm/dockerfile deleted file mode 100644 index 5a8efb1..0000000 --- a/src/docker/pytorch-rocm/dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM rocm/pytorch - -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - -RUN useradd -ms /bin/bash guest -USER guest -WORKDIR /home/guest diff --git a/src/docker/pytorch-rocm/pytorch-rocm_config.yaml b/src/docker/pytorch-rocm/pytorch-rocm_config.yaml deleted file mode 100644 index 7cb0922..0000000 --- a/src/docker/pytorch-rocm/pytorch-rocm_config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -schemaVersion: '2.0.0' - -commandTests: - - name: "Check Sudo" - command: "whoami" - expectedOutput: ["guest"] - - - name: "CUDA availability check" - command: "python" - args: ["-c", "import torch; print(torch.cuda.is_available())"] - expectedOutput: ["False"] \ No newline at end of file diff --git a/src/docker/pytorch-rocm/requirements.txt b/src/docker/pytorch-rocm/requirements.txt deleted file mode 100644 index b2a0911..0000000 --- a/src/docker/pytorch-rocm/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -pandas -scipy -matplotlib diff --git a/src/docker/pytorch-rocm/run_test.sh b/src/docker/pytorch-rocm/run_test.sh deleted file mode 100644 index 3963fbd..0000000 --- a/src/docker/pytorch-rocm/run_test.sh +++ /dev/null @@ -1,2 +0,0 @@ -image=$1 -container-structure-test test --image $image --config ${image}_config.yaml \ No newline at end of file diff --git a/src/docker/run_container_job_test.sh b/src/docker/run_container_job_test.sh deleted file mode 100755 index 36ef5dc..0000000 --- a/src/docker/run_container_job_test.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Run CPU-only container tests. Requires Docker and Redis. -# Usage: ./run_container_job_test.sh - -set -e -cd "$(dirname "$0")/.." - -echo "Building pytorch-cpu image..." -docker build -t pytorch-cpu ./images/pytorch-cpu || { - echo "Failed to build pytorch-cpu. Ensure Docker is running." - exit 1 -} - -echo "Starting Redis..." -docker compose up -d redis 2>/dev/null || docker-compose up -d redis 2>/dev/null || true -sleep 2 - -echo "Running container job tests..." -go test -v -run 'TestContainerJobCPU|TestRunContainerCPUIntegration' -count=1 - -go test -v -run TestRunContainerCPU -count=1 - -echo "All container job tests passed." diff --git a/src/docker/run_tests.sh b/src/docker/run_tests.sh deleted file mode 100755 index 44ef9ed..0000000 --- a/src/docker/run_tests.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -images=("pytorch-cuda" "pytorch-rocm" "pytorch-cpu") -path=$(dirname $(readlink -f $0)) -for image in "${images[@]}"; do - cd $path/$image - docker build -t $image . -done - -for image in "${images[@]}"; do - cd $path/$image - sh ./run_test.sh $image -done \ No newline at end of file diff --git a/src/go.mod b/src/go.mod index d4b5a70..727c811 100644 --- a/src/go.mod +++ b/src/go.mod @@ -3,35 +3,17 @@ module mist go 1.24.6 require ( - github.com/docker/docker v28.5.2+incompatible - github.com/redis/go-redis/v9 v9.10.0 - gopkg.in/natefinch/lumberjack.v2 v2.2.1 - gopkg.in/yaml.v3 v3.0.1 + github.com/distribution/reference v0.6.0 k8s.io/api v0.34.1 k8s.io/apimachinery v0.34.1 k8s.io/client-go v0.34.1 - mist/docker v0.0.0 ) -replace mist/docker => ./docker - require ( - github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect - github.com/Microsoft/go-winio v0.4.21 // indirect - github.com/cenkalti/backoff/v5 v5.0.3 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/containerd/errdefs v1.0.0 // indirect - github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/distribution/reference v0.6.0 // indirect - github.com/docker/go-connections v0.6.0 // indirect - github.com/docker/go-units v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-logr/logr v1.4.3 // indirect - github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.23.0 // indirect @@ -41,25 +23,15 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/sys/sequential v0.6.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/spf13/pflag v1.0.6 // indirect + github.com/stretchr/testify v1.11.1 // indirect github.com/x448/float16 v0.8.4 // indirect - go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect - go.opentelemetry.io/otel v1.40.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect - go.opentelemetry.io/otel/metric v1.40.0 // indirect - go.opentelemetry.io/otel/sdk v1.40.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect - go.opentelemetry.io/otel/trace v1.40.0 // indirect - go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/net v0.50.0 // indirect @@ -71,6 +43,7 @@ require ( google.golang.org/protobuf v1.36.11 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect diff --git a/src/go.sum b/src/go.sum index 2938439..63a368f 100644 --- a/src/go.sum +++ b/src/go.sum @@ -1,46 +1,15 @@ -github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= -github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Microsoft/go-winio v0.4.21 h1:+6mVbXh4wPzUrl1COX9A+ZCvEpYsOBZ6/+kwDnvLyro= -github.com/Microsoft/go-winio v0.4.21/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= -github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= -github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= -github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= -github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= -github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= -github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= -github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= -github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= -github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= -github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= -github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= @@ -62,8 +31,6 @@ github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgY github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -79,22 +46,12 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= -github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= -github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= -github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= -github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= -github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= @@ -103,18 +60,12 @@ github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= -github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/redis/go-redis/v9 v9.10.0 h1:FxwK3eV8p/CQa0Ch276C7u2d0eNC9kCmAYQ7mCXCzVs= -github.com/redis/go-redis/v9 v9.10.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -122,7 +73,6 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -133,26 +83,6 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= -go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= -go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= -go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4= -go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= -go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= -go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= -go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= -go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= -go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= -go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= -go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= -go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= -go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= @@ -175,10 +105,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= @@ -199,12 +126,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= -google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= -google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= -google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -214,13 +135,9 @@ gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSP gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= -gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= -gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= k8s.io/api v0.34.1 h1:jC+153630BMdlFukegoEL8E/yT7aLyQkIVuwhmwDgJM= k8s.io/api v0.34.1/go.mod h1:SB80FxFtXn5/gwzCoN6QCtPD7Vbu5w2n1S0J5gFfTYk= k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= diff --git a/src/http.go b/src/http.go new file mode 100644 index 0000000..18e7e9c --- /dev/null +++ b/src/http.go @@ -0,0 +1,35 @@ +package main + +import ( + "encoding/json" + "errors" + "net/http" + "syscall" + + apierrors "k8s.io/apimachinery/pkg/api/errors" +) + +func writeJSON(w http.ResponseWriter, code int, value interface{}) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(value) +} + +func executorError(w http.ResponseWriter, err error) { + code := http.StatusBadGateway + var invalid *submissionError + if errors.As(err, &invalid) { + code = http.StatusBadRequest + } + if apierrors.IsNotFound(err) { + code = http.StatusNotFound + } + var denied *permissionError + if errors.As(err, &denied) { + code = http.StatusForbidden + } + if errors.Is(err, syscall.ENOSPC) || errors.Is(err, syscall.EDQUOT) { + code = 507 + } + writeJSON(w, code, map[string]string{"error": err.Error()}) +} diff --git a/src/images.go b/src/images.go index a66291a..cd9da95 100644 --- a/src/images.go +++ b/src/images.go @@ -27,7 +27,8 @@ type ImageCatalog struct { } // Accelerator lists describe server policy, not a guarantee that an image's -// code supports that hardware. The TT pilot requires its host runtime profile. +// code supports that hardware. Team TT jobs can select a compatible container +// runtime or the tested read-only host runtime profile. func (e *KubernetesExecutor) imageCatalog() ImageCatalog { catalog := ImageCatalog{Images: []ImageOption{}, Profiles: []ComputeProfile{ {Accelerator: "cpu", DefaultImage: cpuImage, DeviceUnit: "none", MaxDevices: 0}, diff --git a/src/int_test.go b/src/int_test.go deleted file mode 100644 index 3552145..0000000 --- a/src/int_test.go +++ /dev/null @@ -1,209 +0,0 @@ -package main - -import ( - "context" - "fmt" - "log/slog" - "mist/multilogger" - "os" - "os/signal" - "sync" - "syscall" - "testing" - "time" - - "github.com/redis/go-redis/v9" -) - -func addDummySupervisors(statusRegistry *StatusRegistry, log *slog.Logger) { - now := time.Now() - - dummySupervisors := []SupervisorStatus{ - { - ConsumerID: "worker_amd_001", - GPUType: "AMD", - Status: SupervisorStateActive, - LastSeen: now, // now - StartedAt: now.Add(-2 * time.Hour), // 2hours ago - }, - { - ConsumerID: "worker_nvidia_002", - GPUType: "NVIDIA", - Status: SupervisorStateActive, - LastSeen: now.Add(-30 * time.Second), // 30 seconds ago - StartedAt: now.Add(-1 * time.Hour), // 1 hour ago - }, - { - ConsumerID: "worker_tt_003", - GPUType: "TT", - Status: SupervisorStateInactive, - LastSeen: now.Add(-5 * time.Minute), // seen 5 minutes ago - StartedAt: now.Add(-3 * time.Hour), // 3 hours ago - }, - } - - for _, supervisor := range dummySupervisors { - if err := statusRegistry.UpdateStatus(supervisor.ConsumerID, supervisor); err != nil { - log.Error("failed to add dummy supervisor", "consumer_id", supervisor.ConsumerID, "error", err) - } else { - log.Info("added dummy supervisor", "consumer_id", supervisor.ConsumerID, "gpu_type", supervisor.GPUType, "status", supervisor.Status) - } - } -} - -func TestIntegration(t *testing.T) { - os.Setenv("ENV", "test") - defer os.Unsetenv("ENV") - - redisAddr := "localhost:6379" - config, _ := multilogger.GetLogConfig() - schedulerLog, err := multilogger.CreateLogger("scheduler", &config) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to create logger: %v\n", err) - os.Exit(1) - } - client := redis.NewClient(&redis.Options{Addr: redisAddr}) - defer client.Close() - if err := client.Ping(context.Background()).Err(); err != nil { - t.Errorf("Failed to connect to Redis: %v", err) - } - - scheduler := NewScheduler(redisAddr, schedulerLog) - defer scheduler.Close() - - supervisorLog, err := multilogger.CreateLogger("supervisor", &config) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to create logger: %v\n", err) - os.Exit(1) - } - - consumerID := fmt.Sprintf("worker_%d", os.Getpid()) - supervisor := NewSupervisor(redisAddr, consumerID, "AMD", supervisorLog) - - if err := supervisor.Start(); err != nil { - t.Errorf("Failed to start supervisor: %v", err) - } - - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - - // test jobs - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - jobTypes := []string{"a", "b", "c"} - for i := 0; i < 10; i++ { - jobType := jobTypes[i%len(jobTypes)] - payload := map[string]interface{}{ - "task_id": i, - "data": fmt.Sprintf("test_data_%d", i), - } - - if _, err := scheduler.Enqueue(jobType, "TT", payload); err != nil { - t.Errorf("Failed to enqueue job: %v", err) - } - } - }() - - wg.Wait() - supervisor.Stop() -} - -func TestDummySupervisors(t *testing.T) { - redisAddr := "localhost:6379" - log := slog.New(slog.NewJSONHandler(os.Stdout, nil)) - - // Test 1: Dummy supervisors should be added in test environment - os.Setenv("ENV", "test") - defer os.Unsetenv("ENV") - - // Clean up Redis data before test - client := redis.NewClient(&redis.Options{Addr: redisAddr}) - defer client.Close() - client.FlushDB(context.Background()) - - app := NewApp(redisAddr, "AMD", log) - defer app.redisClient.Close() - - // Manually add dummy supervisors for testing - addDummySupervisors(app.statusRegistry, log) - - supervisors, err := app.statusRegistry.GetAllSupervisors() - if err != nil { - t.Errorf("Failed to get supervisors: %v", err) - } - // Verify dummy supervisor IDs exist - dummyIDs := []string{"worker_amd_001", "worker_nvidia_002", "worker_tt_003"} - for _, dummyID := range dummyIDs { - found := false - for _, supervisor := range supervisors { - if supervisor.ConsumerID == dummyID { - found = true - break - } - } - if !found { - t.Errorf("Expected dummy supervisor %s not found", dummyID) - } - } -} - -// Unit tests for StatusRegistry -func TestStatusRegistry_BasicOperations(t *testing.T) { - redisAddr := "localhost:6379" - log := slog.New(slog.NewJSONHandler(os.Stdout, nil)) - - client := redis.NewClient(&redis.Options{Addr: redisAddr}) - defer client.Close() - client.FlushDB(context.Background()) - - registry := NewStatusRegistry(client, log) - - now := time.Now() - - // Test adding and retrieving a supervisor - status := SupervisorStatus{ - ConsumerID: "test_worker_001", - GPUType: "AMD", - Status: SupervisorStateActive, - LastSeen: now, - StartedAt: now.Add(-1 * time.Hour), - } - - // Add status - err := registry.UpdateStatus(status.ConsumerID, status) - if err != nil { - t.Errorf("UpdateStatus failed: %v", err) - } - - // Retrieve status - retrievedStatus, err := registry.GetSupervisor(status.ConsumerID) - if err != nil { - t.Errorf("GetSupervisor failed: %v", err) - } - - if retrievedStatus.Status != status.Status { - t.Errorf("Expected Status %s, got %s", status.Status, retrievedStatus.Status) - } - - // Test getting all supervisors - allSupervisors, err := registry.GetAllSupervisors() - if err != nil { - t.Errorf("GetAllSupervisors failed: %v", err) - } - - if len(allSupervisors) != 1 { - t.Errorf("Expected 1 supervisor, got %d", len(allSupervisors)) - } - - // Test getting active supervisors - activeSupervisors, err := registry.GetActiveSupervisors() - if err != nil { - t.Errorf("GetActiveSupervisors failed: %v", err) - } - - if len(activeSupervisors) != 1 { - t.Errorf("Expected 1 active supervisor, got %d", len(activeSupervisors)) - } -} diff --git a/src/job_scheduling_test.go b/src/job_scheduling_test.go deleted file mode 100644 index b7271f1..0000000 --- a/src/job_scheduling_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package main - -import ( - "context" - "log/slog" - "os" - "testing" - "time" - - "github.com/redis/go-redis/v9" -) - -func TestJobEnqueueAndSupervisor(t *testing.T) { - redisAddr := "localhost:6379" - log := slog.New(slog.NewJSONHandler(os.Stdout, nil)) - - client := redis.NewClient(&redis.Options{Addr: redisAddr}) - defer client.Close() - if err := client.Ping(context.Background()).Err(); err != nil { - t.Fatalf("Failed to connect to Redis: %v", err) - } - - client.FlushDB(context.Background()) - - // Scheduler - scheduler := NewScheduler(redisAddr, log) - defer scheduler.Close() - - // Supervisor - supervisor := NewSupervisor(redisAddr, "test_worker_001", "AMD", log) - if err := supervisor.Start(); err != nil { - t.Fatalf("Failed to start supervisor: %v", err) - } - defer supervisor.Stop() - - // Enqueue jobs - for i := 0; i < 3; i++ { - payload := map[string]interface{}{"task": i} - if _, err := scheduler.Enqueue("test_job_type", "AMD", payload); err != nil { - t.Errorf("Failed to enqueue job %d: %v", i, err) - } - } - - time.Sleep(3 * time.Second) -} diff --git a/src/job_types.go b/src/job_types.go new file mode 100644 index 0000000..e44b061 --- /dev/null +++ b/src/job_types.go @@ -0,0 +1,84 @@ +package main + +import "time" + +type JobState string + +const ( + JobStateScheduled JobState = "Scheduled" + JobStateInProgress JobState = "InProgress" + JobStateSuccess JobState = "Success" + JobStateFailure JobState = "Failure" + JobStateCancelled JobState = "Cancelled" +) + +type Job struct { + ID string `json:"id"` + Type string `json:"type"` + Payload map[string]interface{} `json:"payload"` + Retries int `json:"retries"` + Created time.Time `json:"created"` + RequiredGPU string `json:"required_gpu,omitempty"` + JobState JobState `json:"job_state"` + ConsumerID *string `json:"consumer_id,omitempty"` + TimeAssigned *time.Time `json:"time_assigned,omitempty"` + TimeStarted *time.Time `json:"time_started,omitempty"` + TimeCompleted *time.Time `json:"time_completed,omitempty"` + Result map[string]interface{} `json:"result,omitempty"` + Error *string `json:"error,omitempty"` +} + +type CreateJobRequest struct { + Type string `json:"type"` + DatasetID string `json:"dataset_id,omitempty"` + StorageScope string `json:"storage_scope,omitempty"` + TTRuntime string `json:"tt_runtime,omitempty"` + Payload map[string]interface{} `json:"payload"` + RequiredGPU string `json:"gpu,omitempty"` + Name string `json:"name,omitempty"` + Image string `json:"image,omitempty"` + Command []string `json:"command,omitempty"` + Args []string `json:"args,omitempty"` + WorkingDirectory string `json:"working_directory,omitempty"` + Script string `json:"script,omitempty"` + ScriptName string `json:"script_name,omitempty"` + Env map[string]string `json:"env,omitempty"` + CPU string `json:"cpu,omitempty"` + Memory string `json:"memory,omitempty"` + Accelerator string `json:"accelerator,omitempty"` + DeviceCount int `json:"device_count,omitempty"` + TimeoutSeconds int64 `json:"timeout_seconds,omitempty"` +} + +type KubernetesJobStatus struct { + Job + Name string `json:"name"` + Image string `json:"image"` + Accelerator string `json:"accelerator"` + DeviceCount int `json:"device_count"` + CPU string `json:"cpu"` + Memory string `json:"memory"` + Owner string `json:"owner"` + Node string `json:"node,omitempty"` + Pod string `json:"pod,omitempty"` + ExitCode *int32 `json:"exit_code,omitempty"` + Message string `json:"message,omitempty"` + Devices []AllocatedDevice `json:"devices,omitempty"` + CheckpointDirectory string `json:"checkpoint_directory"` + OutputDirectory string `json:"output_directory"` + WorkingDirectory string `json:"working_directory,omitempty"` + TimeoutSeconds int64 `json:"timeout_seconds"` + DatasetID string `json:"dataset_id,omitempty"` + TeamID string `json:"team_id,omitempty"` + CreatorID string `json:"creator_id,omitempty"` + CreatorName string `json:"creator_name,omitempty"` + StorageScope string `json:"storage_scope,omitempty"` + QueuePosition int `json:"queue_position,omitempty"` + CanCancel *bool `json:"can_cancel,omitempty"` +} + +type AllocatedDevice struct { + Driver string `json:"driver"` + Pool string `json:"pool"` + Device string `json:"device"` +} diff --git a/src/jobs_http.go b/src/jobs_http.go new file mode 100644 index 0000000..6044ce9 --- /dev/null +++ b/src/jobs_http.go @@ -0,0 +1,109 @@ +package main + +import ( + "encoding/json" + "io" + "net/http" + "strings" +) + +func (a *App) kubernetesJobs(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodPost: + var req CreateJobRequest + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 128*1024)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&req); err != nil { + writeJSON(w, 400, map[string]string{"error": "invalid submission: " + err.Error()}) + return + } + if err := decoder.Decode(new(interface{})); err != io.EOF { + writeJSON(w, 400, map[string]string{"error": "request must contain one JSON object"}) + return + } + status, err := a.requestExecutor(r).submit(r.Context(), req) + if err != nil { + executorError(w, err) + return + } + writeJSON(w, http.StatusCreated, map[string]interface{}{"job_id": status.ID, "job": status}) + case http.MethodGet: + jobs, err := a.requestExecutor(r).list(r.Context()) + if err != nil { + executorError(w, err) + return + } + writeJSON(w, 200, map[string]interface{}{"jobs": jobs, "count": len(jobs)}) + default: + w.Header().Set("Allow", "GET, POST") + writeJSON(w, 405, map[string]string{"error": "method not allowed"}) + } +} + +func (a *App) kubernetesJob(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/files") { + a.jobFiles(w, r) + return + } + parts := strings.Split(strings.TrimPrefix(r.URL.Path, "/jobs/"), "/") + if len(parts) == 0 || parts[0] == "" || len(parts) > 2 { + http.NotFound(w, r) + return + } + id := parts[0] + if len(parts) == 2 && parts[1] == "logs" && r.Method == http.MethodGet { + logs, err := a.requestExecutor(r).logs(r.Context(), id) + if err != nil { + executorError(w, err) + return + } + writeJSON(w, 200, map[string]string{"job_id": id, "logs": logs}) + return + } + if (len(parts) == 2 && parts[1] == "cancel" && r.Method == http.MethodPost) || (len(parts) == 1 && r.Method == http.MethodDelete) { + status, err := a.requestExecutor(r).cancel(r.Context(), id) + if err != nil { + executorError(w, err) + return + } + writeJSON(w, 200, status) + return + } + if len(parts) == 1 && r.Method == http.MethodGet { + status, err := a.requestExecutor(r).get(r.Context(), id) + if err != nil { + executorError(w, err) + return + } + writeJSON(w, 200, status) + return + } + writeJSON(w, 405, map[string]string{"error": "method not allowed"}) +} + +func (a *App) getJobStatus(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + jobID := r.URL.Query().Get("id") + if jobID == "" { + // Try to get from path if query param not provided + path := strings.TrimPrefix(r.URL.Path, "/jobs/status/") + if path != "" && path != "/jobs/status" { + jobID = path + } + } + + if jobID == "" { + http.Error(w, "Job ID is required", http.StatusBadRequest) + return + } + status, err := a.requestExecutor(r).get(r.Context(), jobID) + if err != nil { + executorError(w, err) + return + } + writeJSON(w, 200, status) +} diff --git a/src/kubernetes.go b/src/kubernetes.go index 89be5f1..7130dbe 100644 --- a/src/kubernetes.go +++ b/src/kubernetes.go @@ -8,14 +8,11 @@ import ( "errors" "fmt" "io" - "log/slog" - "net/http" "os" "path/filepath" "sort" "strconv" "strings" - "syscall" "time" batchv1 "k8s.io/api/batch/v1" @@ -58,57 +55,6 @@ type KubernetesExecutor struct { outputSubPath string } -func NewKubernetesApp(executor *KubernetesExecutor, log *slog.Logger) *App { - mux := http.NewServeMux() - a := &App{executor: executor, log: log, httpServer: &http.Server{ - Addr: envOr("MIST_HTTP_ADDR", "127.0.0.1:3000"), Handler: mux, - ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 30 * time.Second, WriteTimeout: 60 * time.Second, IdleTimeout: 60 * time.Second}} - mux.HandleFunc("/session", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, 200, map[string]interface{}{"enabled": false, "user": nil}) - }) - mux.HandleFunc("/datasets", a.datasets) - mux.HandleFunc("/datasets/", a.dataset) - mux.HandleFunc("GET /storage", a.storageInfo) - mux.HandleFunc("GET /storage/folders", a.storageFolders) - mux.HandleFunc("GET /storage/files", a.storageFolderFiles) - if raw := os.Getenv("MIST_AUTH_URL"); raw != "" { - var err error - a.auth, err = newAuthGateway(raw) - if err != nil { - panic(err) - } - if os.Getenv("MIST_TEAMS_ENABLED") == "true" { - if executor.storage == nil { - panic("team workspaces require shared storage") - } - a.teams = newTeamService(a) - mux.HandleFunc("/teams", a.teamEndpoint) - mux.HandleFunc("/teams/", a.teamEndpoint) - a.httpServer.Handler = a.auth.middleware(a.teams.middleware(mux)) - } else { - a.httpServer.Handler = a.auth.middleware(mux) - } - } - mux.HandleFunc("/jobs", a.kubernetesJobs) - mux.HandleFunc("/jobs/status", a.getJobStatus) - mux.HandleFunc("/jobs/", a.kubernetesJob) - mux.HandleFunc("GET /hardware", a.hardware) - mux.HandleFunc("GET /images", a.images) - mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - writeJSON(w, 405, map[string]string{"error": "method not allowed"}) - return - } - _, err := executor.client.BatchV1().Jobs(executor.namespace).List(r.Context(), metav1.ListOptions{Limit: 1, LabelSelector: managedLabel + "=mist"}) - if err != nil { - executorError(w, err) - return - } - writeJSON(w, 200, map[string]string{"status": "ok", "executor": "kubernetes"}) - }) - return a -} - func NewKubernetesExecutor() (*KubernetesExecutor, error) { var config *rest.Config var err error @@ -172,39 +118,6 @@ func envOr(key, fallback string) string { } func ptr[T any](value T) *T { return &value } -type KubernetesJobStatus struct { - Job - Name string `json:"name"` - Image string `json:"image"` - Accelerator string `json:"accelerator"` - DeviceCount int `json:"device_count"` - CPU string `json:"cpu"` - Memory string `json:"memory"` - Owner string `json:"owner"` - Node string `json:"node,omitempty"` - Pod string `json:"pod,omitempty"` - ExitCode *int32 `json:"exit_code,omitempty"` - Message string `json:"message,omitempty"` - Devices []AllocatedDevice `json:"devices,omitempty"` - CheckpointDirectory string `json:"checkpoint_directory"` - OutputDirectory string `json:"output_directory"` - WorkingDirectory string `json:"working_directory,omitempty"` - TimeoutSeconds int64 `json:"timeout_seconds"` - DatasetID string `json:"dataset_id,omitempty"` - TeamID string `json:"team_id,omitempty"` - CreatorID string `json:"creator_id,omitempty"` - CreatorName string `json:"creator_name,omitempty"` - StorageScope string `json:"storage_scope,omitempty"` - QueuePosition int `json:"queue_position,omitempty"` - CanCancel *bool `json:"can_cancel,omitempty"` -} - -type AllocatedDevice struct { - Driver string `json:"driver"` - Pool string `json:"pool"` - Device string `json:"device"` -} - func (e *KubernetesExecutor) normalize(req CreateJobRequest) (CreateJobRequest, error) { explicitImage := req.Image != "" if len(req.Payload) != 0 { @@ -900,102 +813,3 @@ func (e *KubernetesExecutor) cancel(ctx context.Context, id string) (*Kubernetes } return e.statusFor(ctx, job) } - -func writeJSON(w http.ResponseWriter, code int, value interface{}) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(code) - _ = json.NewEncoder(w).Encode(value) -} - -func executorError(w http.ResponseWriter, err error) { - code := http.StatusBadGateway - var invalid *submissionError - if errors.As(err, &invalid) { - code = http.StatusBadRequest - } - if apierrors.IsNotFound(err) { - code = http.StatusNotFound - } - var denied *permissionError - if errors.As(err, &denied) { - code = http.StatusForbidden - } - if errors.Is(err, syscall.ENOSPC) || errors.Is(err, syscall.EDQUOT) { - code = 507 - } - writeJSON(w, code, map[string]string{"error": err.Error()}) -} - -func (a *App) kubernetesJobs(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case http.MethodPost: - var req CreateJobRequest - decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 128*1024)) - decoder.DisallowUnknownFields() - if err := decoder.Decode(&req); err != nil { - writeJSON(w, 400, map[string]string{"error": "invalid submission: " + err.Error()}) - return - } - if err := decoder.Decode(new(interface{})); err != io.EOF { - writeJSON(w, 400, map[string]string{"error": "request must contain one JSON object"}) - return - } - status, err := a.requestExecutor(r).submit(r.Context(), req) - if err != nil { - executorError(w, err) - return - } - writeJSON(w, http.StatusCreated, map[string]interface{}{"job_id": status.ID, "job": status}) - case http.MethodGet: - jobs, err := a.requestExecutor(r).list(r.Context()) - if err != nil { - executorError(w, err) - return - } - writeJSON(w, 200, map[string]interface{}{"jobs": jobs, "count": len(jobs)}) - default: - w.Header().Set("Allow", "GET, POST") - writeJSON(w, 405, map[string]string{"error": "method not allowed"}) - } -} - -func (a *App) kubernetesJob(w http.ResponseWriter, r *http.Request) { - if strings.Contains(r.URL.Path, "/files") { - a.jobFiles(w, r) - return - } - parts := strings.Split(strings.TrimPrefix(r.URL.Path, "/jobs/"), "/") - if len(parts) == 0 || parts[0] == "" || len(parts) > 2 { - http.NotFound(w, r) - return - } - id := parts[0] - if len(parts) == 2 && parts[1] == "logs" && r.Method == http.MethodGet { - logs, err := a.requestExecutor(r).logs(r.Context(), id) - if err != nil { - executorError(w, err) - return - } - writeJSON(w, 200, map[string]string{"job_id": id, "logs": logs}) - return - } - if (len(parts) == 2 && parts[1] == "cancel" && r.Method == http.MethodPost) || (len(parts) == 1 && r.Method == http.MethodDelete) { - status, err := a.requestExecutor(r).cancel(r.Context(), id) - if err != nil { - executorError(w, err) - return - } - writeJSON(w, 200, status) - return - } - if len(parts) == 1 && r.Method == http.MethodGet { - status, err := a.requestExecutor(r).get(r.Context(), id) - if err != nil { - executorError(w, err) - return - } - writeJSON(w, 200, status) - return - } - writeJSON(w, 405, map[string]string{"error": "method not allowed"}) -} diff --git a/src/main.go b/src/main.go new file mode 100644 index 0000000..45a49df --- /dev/null +++ b/src/main.go @@ -0,0 +1,36 @@ +package main + +import ( + "context" + "log/slog" + "os" + "os/signal" + "syscall" + "time" +) + +func main() { + log := slog.New(slog.NewJSONHandler(os.Stdout, nil)) + if mode := envOr("MIST_EXECUTOR", "kubernetes"); mode != "kubernetes" { + log.Error("unsupported executor; the Docker/Redis executor has been retired", "executor", mode) + os.Exit(1) + } + executor, err := NewKubernetesExecutor() + if err != nil { + log.Error("Kubernetes configuration failed", "error", err) + os.Exit(1) + } + app := NewKubernetesApp(executor, log) + if err := app.Start(); err != nil { + log.Error("API startup failed", "error", err) + os.Exit(1) + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + <-ctx.Done() + shutdown, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + if err := app.Shutdown(shutdown); err != nil { + log.Error("API shutdown failed", "error", err) + } +} diff --git a/src/multilogger/README.md b/src/multilogger/README.md deleted file mode 100644 index 02dc76c..0000000 --- a/src/multilogger/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# multilogger - -`multilogger` lets you log to multiple io.Writer instances at once, with JSON structured logs, file rotation, and flexible metadata. - ---- - -## Overview - - -Multilogger allows you to write logs to multiple io.Writer targets (e.g. stdout, files, etc.) simultaneously, each with its own log level. -It’s ideal for tracking runtime activity across components such as schedulers, supervisors, and the main application. -If the configuration fails or a destination cannot be initialized, multilogger automatically falls back to a stderr logger. -File rotation is handled using [lumberjack](https://github.com/natefinch/lumberjack). Environment variables are able to override the log levels configured in the log.yaml file. - ---- - -## Example Usage -```go -// Create Multilogger -config := multilogger.GetLogConfig() -logger, _ := multilogger.CreateLogger("app", &config) -logger.Info("starting app") -``` - -```go -// Set Global Log Level -os.Setenv("LOG_LEVEL", "DEBUG") -``` - -```go -// Configure Log Levels By io.Writer -os.Setenv("FILE_LOG_LEVEL", "DEBUG") -os.Setenv("STDOUT_LOG_LEVEL", "INFO") -``` - -```go -// Add Metadata -config := multilogger.GetLogConfig() -logger, _ := multilogger.CreateLogger("app", &config) -logger = logger.WithGroup("jobInfo") -logger.Info("job started", "job_id", "abc123") -``` - ---- diff --git a/src/multilogger/log.go b/src/multilogger/log.go deleted file mode 100644 index 03a97d3..0000000 --- a/src/multilogger/log.go +++ /dev/null @@ -1,195 +0,0 @@ -package multilogger - -import ( - "context" - "errors" - "fmt" - "gopkg.in/natefinch/lumberjack.v2" - "gopkg.in/yaml.v3" - "io" - "log/slog" - "os" - "path/filepath" - "strings" -) - -const LogConfigFilePath = "../config/log.yaml" - -var levelMap = map[string]slog.Level{ - "DEBUG": slog.LevelDebug, - "INFO": slog.LevelInfo, - "WARN": slog.LevelWarn, - "ERROR": slog.LevelError, -} - -type MultiHandler struct { - subHandlers []slog.Handler -} - -type OutputType struct { - Type string `yaml:"type"` - Level string `yaml:"level"` -} - -type LogConfig struct { - Output struct { - Types []OutputType `yaml:"types"` - Directory string `yaml:"directory"` - } `yaml:"output"` - Components []string `yaml:"components"` -} - -func NewMultiHandler(writerLevels map[io.Writer]slog.Level) *MultiHandler { - var handlers []slog.Handler - - for writer, level := range writerLevels { - handlers = append(handlers, slog.NewJSONHandler(writer, &slog.HandlerOptions{Level: level})) - } - - m := MultiHandler{ - subHandlers: handlers, - } - return &m -} - -func (h *MultiHandler) Enabled(ctx context.Context, level slog.Level) bool { - for _, handler := range h.subHandlers { - if handler.Enabled(ctx, level) { - return true - } - } - return false - -} - -func (h *MultiHandler) WithAttrs(attrs []slog.Attr) slog.Handler { - newSubHandlers := make([]slog.Handler, len(h.subHandlers)) - for i, sub := range h.subHandlers { - newSubHandlers[i] = sub.WithAttrs(attrs) - } - return &MultiHandler{subHandlers: newSubHandlers} -} - -func (h *MultiHandler) WithGroup(name string) slog.Handler { - newSubHandlers := make([]slog.Handler, len(h.subHandlers)) - for i, sub := range h.subHandlers { - newSubHandlers[i] = sub.WithGroup(name) - } - return &MultiHandler{subHandlers: newSubHandlers} -} - -func (h *MultiHandler) Handle(ctx context.Context, record slog.Record) error { - var err error - for _, handler := range h.subHandlers { - if handler.Enabled(ctx, record.Level) { - if out := handler.Handle(ctx, record); out != nil { - err = out - } - } - } - return err -} - -func GetLogConfig() (LogConfig, error) { - var config LogConfig - - configFile, err := os.ReadFile(LogConfigFilePath) - if err != nil { - return config, err - } - - err = yaml.Unmarshal(configFile, &config) - if err != nil { - return config, err - } - - return config, nil -} - -func FallbackLogger(component string) *slog.Logger { - return slog.New( - slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}), - ).With("component", component) -} - -func CreateLogger(component string, config *LogConfig) (*slog.Logger, error) { - if err := OverrideYAMLConfig(config); err != nil { - fallback := FallbackLogger(component) - fallback.Warn("using fallback logger due to invalid env override") - return fallback, fmt.Errorf("failed to override YAML config: %w", err) - } - - writerLevels := make(map[io.Writer]slog.Level) - for _, t := range config.Output.Types { - lvl, ok := levelMap[t.Level] - if !ok { - fallback := FallbackLogger(component) - fallback.Warn("using fallback logger due to invalid log level") - return fallback, fmt.Errorf("invalid log level: %q", t.Level) - } - - switch t.Type { - case "stdout": - writerLevels[os.Stdout] = lvl - case "file": - directory := config.Output.Directory - - if err := os.MkdirAll(directory, 0755); err != nil { - fallback := FallbackLogger(component) - fallback.Warn("using fallback logger") - return fallback, fmt.Errorf("failed to create multilogger directory: %w", err) - } - - filePath := filepath.Join(directory, component+".log") - rotatingFileWriter := &lumberjack.Logger{ - Filename: filePath, - MaxSize: 10, - MaxBackups: 3, - MaxAge: 28, - Compress: true, - } - - writerLevels[rotatingFileWriter] = lvl - - default: - fallback := FallbackLogger(component) - fallback.Warn("using fallback logger due to unsupported output type", "type", t.Type) - return fallback, fmt.Errorf("unsupported output type: %q", t.Type) - } - } - - if len(writerLevels) == 0 { - fallback := FallbackLogger(component) - fallback.Warn("using fallback logger") - return fallback, errors.New("no valid multilogger outputs configured") - } - - handler := NewMultiHandler(writerLevels) - logger := slog.New(handler).With("component", component) - slog.Info("logger created successfully", "component", component, "outputs", len(writerLevels)) - return logger, nil -} - -func OverrideYAMLConfig(config *LogConfig) error { - if global := os.Getenv("LOG_LEVEL"); global != "" { - if _, ok := levelMap[global]; ok { - for i := range config.Output.Types { - config.Output.Types[i].Level = global - } - } else { - return fmt.Errorf("invalid global LOG_LEVEL: %q", global) - } - } - - for i, output := range config.Output.Types { - env := strings.ToUpper(output.Type) + "_LOG_LEVEL" - if lvl := strings.TrimSpace(os.Getenv(env)); lvl != "" { - if _, ok := levelMap[lvl]; ok { - config.Output.Types[i].Level = lvl - } else { - return fmt.Errorf("invalid multilogger level %q for %s", lvl, env) - } - } - } - return nil -} diff --git a/src/multilogger/log_test.go b/src/multilogger/log_test.go deleted file mode 100644 index 8c495f9..0000000 --- a/src/multilogger/log_test.go +++ /dev/null @@ -1,446 +0,0 @@ -package multilogger - -import ( - "bufio" - "bytes" - "encoding/json" - "io" - "log/slog" - "os" - "path/filepath" - "reflect" - "strings" - "testing" - "time" -) - -func getLogMessages(text *bytes.Buffer) []string { - scanner := bufio.NewScanner(text) - msgs := []string{} - - for scanner.Scan() { - obj := map[string]interface{}{} - line := scanner.Text() - json.Unmarshal([]byte(line), &obj) - msgs = append(msgs, obj["msg"].(string)) - } - return msgs -} - -func TestMultiHandler(t *testing.T) { - t.Run("test multiple writers with different log levels", func(t *testing.T) { - writer1 := &bytes.Buffer{} - writer2 := &bytes.Buffer{} - writer3 := &bytes.Buffer{} - - writerLevels := map[io.Writer]slog.Level{ - writer1: slog.LevelInfo, - writer2: slog.LevelDebug, - writer3: slog.LevelError, - } - - handler := NewMultiHandler(writerLevels) - logger := slog.New(handler) - - logger.Debug("debug") - logger.Info("info") - logger.Warn("warn") - logger.Error("error") - - expectedWriter1 := []string{"info", "warn", "error"} - expectedWriter2 := []string{"debug", "info", "warn", "error"} - expectedWriter3 := []string{"error"} - - if !reflect.DeepEqual(getLogMessages(writer1), expectedWriter1) { - t.Errorf("writer1: got %v, want %v", getLogMessages(writer1), expectedWriter1) - } - if !reflect.DeepEqual(getLogMessages(writer2), expectedWriter2) { - t.Errorf("writer2: got %v, want %v", getLogMessages(writer2), expectedWriter2) - } - if !reflect.DeepEqual(getLogMessages(writer3), expectedWriter3) { - t.Errorf("writer3: got %v, want %v", getLogMessages(writer3), expectedWriter3) - } - }) - - t.Run("test that WithGroup works with multiple handlers ", func(t *testing.T) { - writer1 := &bytes.Buffer{} - writer2 := &bytes.Buffer{} - - writerLevels := map[io.Writer]slog.Level{ - writer1: slog.LevelInfo, - writer2: slog.LevelDebug, - } - - handler := NewMultiHandler(writerLevels) - logger := slog.New(handler) - - groupLogger := logger.WithGroup("request") - groupLogger.Info("group test", "id", 123) - - obj1 := map[string]interface{}{} - obj2 := map[string]interface{}{} - json.Unmarshal(writer1.Bytes(), &obj1) - json.Unmarshal(writer2.Bytes(), &obj2) - - requestGroup1, ok := obj1["request"].(map[string]interface{}) - if !ok { - t.Fatal("expected 'request' to be a map") - } - - requestGroup2, ok := obj2["request"].(map[string]interface{}) - if !ok { - t.Fatal("expected 'request' to be a map") - } - - if requestGroup1["id"] != float64(123) { - t.Errorf("expected id 123, got %v", requestGroup1["id"]) - } - - if requestGroup2["id"] != float64(123) { - t.Errorf("expected id 123, got %v", requestGroup2["id"]) - } - - }) - - t.Run("test that WithGroup works where a sub handler with a higher level should not recieve a group", func(t *testing.T) { - writer1 := &bytes.Buffer{} - writer2 := &bytes.Buffer{} - - writerLevels := map[io.Writer]slog.Level{ - writer1: slog.LevelInfo, - writer2: slog.LevelError, - } - - handler := NewMultiHandler(writerLevels) - logger := slog.New(handler) - - groupLogger := logger.WithGroup("request") - groupLogger.Info("group test", "id", 123) - - obj1 := map[string]interface{}{} - obj2 := map[string]interface{}{} - json.Unmarshal(writer1.Bytes(), &obj1) - json.Unmarshal(writer2.Bytes(), &obj2) - - requestGroup1, ok := obj1["request"].(map[string]interface{}) - if !ok { - t.Fatal("expected 'request' to be a map") - } - - if _, ok := obj2["request"]; ok { - t.Fatal("did not expect 'request' group to exist for this writer") - } - - if requestGroup1["id"] != float64(123) { - t.Errorf("expected id 123, got %v", requestGroup1["id"]) - } - - }) - - t.Run("test WithAttrs with multiple sub handlers", func(t *testing.T) { - writer1 := &bytes.Buffer{} - writer2 := &bytes.Buffer{} - - writerLevels := map[io.Writer]slog.Level{ - writer1: slog.LevelInfo, - writer2: slog.LevelDebug, - } - - handler := NewMultiHandler(writerLevels) - newHandler := handler.WithAttrs([]slog.Attr{ - slog.String("exampleKey1", "testName"), - slog.Int("exampleKey2", 123), - }) - - logger := slog.New(newHandler) - logger.Info("info message") - - obj1 := map[string]interface{}{} - obj2 := map[string]interface{}{} - json.Unmarshal(writer1.Bytes(), &obj1) - json.Unmarshal(writer2.Bytes(), &obj2) - - if obj1["exampleKey1"] != "testName" { - t.Fatal("expected 'exampleKey1' to be 'testName'") - } - - if int(obj1["exampleKey2"].(float64)) != 123 { - t.Fatal("expected 'exampleKey2' to be '123'") - } - - if obj2["exampleKey1"] != "testName" { - t.Fatal("expected 'exampleKey1' to be 'testName'") - } - - if int(obj2["exampleKey2"].(float64)) != 123 { - t.Fatal("expected 'exampleKey2' to be '123'") - } - - }) -} - -func TestOverrideYAMLConfig(t *testing.T) { - t.Run("test with no env variables", func(t *testing.T) { - os.Clearenv() - config := &LogConfig{ - Output: struct { - Types []OutputType `yaml:"types"` - Directory string `yaml:"directory"` - }{ - Directory: t.TempDir(), - Types: []OutputType{ - {Type: "stdout", Level: "INFO"}, - {Type: "file", Level: "DEBUG"}, - }, - }, - Components: []string{"app"}, - } - - err := OverrideYAMLConfig(config) - if err != nil { - t.Fatal(err) - } - - if config.Output.Types[0].Level != "INFO" { - t.Fatal("expected 'stdout' to be 'INFO'") - } - - if config.Output.Types[1].Level != "DEBUG" { - t.Fatal("expected 'file' to be 'DEBUG'") - } - - }) - t.Run("test global env variable", func(t *testing.T) { - os.Clearenv() - config := &LogConfig{ - Output: struct { - Types []OutputType `yaml:"types"` - Directory string `yaml:"directory"` - }{ - Directory: t.TempDir(), - Types: []OutputType{ - {Type: "stdout", Level: "WARN"}, - {Type: "file", Level: "DEBUG"}, - }, - }, - Components: []string{"app"}, - } - - os.Setenv("LOG_LEVEL", "INFO") - - err := OverrideYAMLConfig(config) - if err != nil { - t.Fatal(err) - } - - if config.Output.Types[0].Level != "INFO" { - t.Fatal("expected 'stdout' to be 'INFO'") - } - - if config.Output.Types[1].Level != "INFO" { - t.Fatal("expected 'file' to be 'INFO'") - } - - }) - - t.Run("test env variables for different handlers", func(t *testing.T) { - os.Clearenv() - config := &LogConfig{ - Output: struct { - Types []OutputType `yaml:"types"` - Directory string `yaml:"directory"` - }{ - Directory: t.TempDir(), - Types: []OutputType{ - {Type: "stdout", Level: "WARN"}, - {Type: "file", Level: "DEBUG"}, - }, - }, - Components: []string{"app"}, - } - - os.Setenv("STDOUT_LOG_LEVEL", "DEBUG") - os.Setenv("FILE_LOG_LEVEL", "INFO") - - err := OverrideYAMLConfig(config) - if err != nil { - t.Fatal(err) - } - - if config.Output.Types[0].Level != "DEBUG" { - t.Fatal("expected 'stdout' to be 'DEBUG'") - } - - if config.Output.Types[1].Level != "INFO" { - t.Fatal("expected 'file' to be 'INFO'") - } - - }) - -} - -func TestFallbackLogger(t *testing.T) { - t.Run("invalid log level triggers fallback", func(t *testing.T) { - os.Clearenv() - config := &LogConfig{ - Output: struct { - Types []OutputType `yaml:"types"` - Directory string `yaml:"directory"` - }{ - Directory: t.TempDir(), - Types: []OutputType{{Type: "stdout", Level: "INVALID"}}, - }, - Components: []string{"app"}, - } - - logger, err := CreateLogger("app", config) - if err == nil { - t.Fatal("expected error due to invalid log level") - } - if logger == nil { - t.Fatal("expected fallback logger to be returned") - } - - }) - - t.Run("uncreatable directory triggers fallback", func(t *testing.T) { - os.Clearenv() - config := &LogConfig{ - Output: struct { - Types []OutputType `yaml:"types"` - Directory string `yaml:"directory"` - }{ - Directory: "/root/invaliddir", - Types: []OutputType{{Type: "file", Level: "INFO"}}, - }, - Components: []string{"app"}, - } - - logger, err := CreateLogger("app", config) - if err == nil { - t.Fatal("expected error due to directory creation failure") - } - if logger == nil { - t.Fatal("expected fallback logger to be returned") - } - - }) - - t.Run("no outputs configured triggers fallback", func(t *testing.T) { - os.Clearenv() - config := &LogConfig{ - Output: struct { - Types []OutputType `yaml:"types"` - Directory string `yaml:"directory"` - }{ - Directory: t.TempDir(), - Types: []OutputType{}, - }, - Components: []string{"app"}, - } - - logger, err := CreateLogger("app", config) - if err == nil { - t.Fatal("expected error due to no outputs configured") - } - if logger == nil { - t.Fatal("expected fallback logger to be returned") - } - - }) - - t.Run("unsupported output type triggers fallback", func(t *testing.T) { - os.Clearenv() - config := &LogConfig{ - Output: struct { - Types []OutputType `yaml:"types"` - Directory string `yaml:"directory"` - }{ - Directory: t.TempDir(), - Types: []OutputType{{Type: "network", Level: "INFO"}}, - }, - Components: []string{"app"}, - } - - logger, err := CreateLogger("app", config) - if err == nil { - t.Fatal("expected error due to unsupported output type") - } - if logger == nil { - t.Fatal("expected fallback logger to be returned") - } - }) - - t.Run("missing directory for file output triggers fallback", func(t *testing.T) { - os.Clearenv() - config := &LogConfig{ - Output: struct { - Types []OutputType `yaml:"types"` - Directory string `yaml:"directory"` - }{ - Directory: "", - Types: []OutputType{{Type: "file", Level: "INFO"}}, - }, - Components: []string{"app"}, - } - - logger, err := CreateLogger("app", config) - if err == nil { - t.Fatal("expected error due to missing directory") - } - if logger == nil { - t.Fatal("expected fallback logger to be returned") - } - }) - -} - -func TestCreateLogger(t *testing.T) { - t.Run("valid config creates multi-handler successfully", func(t *testing.T) { - os.Clearenv() - tmpDir := t.TempDir() - config := &LogConfig{ - Output: struct { - Types []OutputType `yaml:"types"` - Directory string `yaml:"directory"` - }{ - Directory: tmpDir, - Types: []OutputType{ - {Type: "stdout", Level: "INFO"}, - {Type: "file", Level: "DEBUG"}, - }, - }, - Components: []string{"app"}, - } - - logger, err := CreateLogger("app", config) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if logger == nil { - t.Fatal("expected logger to be created") - } - - testMsg := "this should go into the file" - logger.Info(testMsg) - - logFilePath := filepath.Join(tmpDir, "app.log") - time.Sleep(50 * time.Millisecond) - - if _, err := os.Stat(logFilePath); err != nil { - if os.IsNotExist(err) { - t.Fatalf("expected log file to exist at %s", logFilePath) - } - t.Fatalf("failed to stat log file: %v", err) - } - - content, err := os.ReadFile(logFilePath) - if err != nil { - t.Fatalf("failed to read log file: %v", err) - } - if !strings.Contains(string(content), testMsg) { - t.Fatalf("expected log file to contain %q, got:\n%s", testMsg, string(content)) - } - - }) -} diff --git a/src/outputs.go b/src/outputs.go index c07436f..9ea11f7 100644 --- a/src/outputs.go +++ b/src/outputs.go @@ -7,7 +7,8 @@ import ( // prepareOutputs creates the job's PVC subdirectory before kubelet mounts it // into the workload. Both supported root and nonroot images can write results. -// This is a trusted local pilot; filesystem permissions are not tenant auth. +// Retained node-local jobs use this initializer. Team jobs prepare their scoped +// NFS output directory through the API and do not mount the parent checkpoint PVC. func prepareOutputs(id string) corev1.Container { return corev1.Container{ Name: "prepare-outputs", Image: "busybox:1.36", ImagePullPolicy: corev1.PullIfNotPresent, diff --git a/src/scheduler.go b/src/scheduler.go deleted file mode 100644 index f7c8556..0000000 --- a/src/scheduler.go +++ /dev/null @@ -1,156 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "log/slog" - "time" - "errors" - - "github.com/redis/go-redis/v9" -) - -type Scheduler struct { - client *redis.Client - ctx context.Context - log *slog.Logger -} - -func NewScheduler(redisAddr string, log *slog.Logger) *Scheduler { - client := redis.NewClient(&redis.Options{ - Addr: redisAddr, - }) - - return &Scheduler{ - client: client, - ctx: context.Background(), - log: log, - } -} - -func (s *Scheduler) Enqueue(jobType string, requiredGPU string, payload map[string]interface{}) (string, error) { - // create a new job - job := Job{ - ID: generateJobID(), - Type: jobType, - Payload: payload, - Retries: 0, - Created: time.Now(), - RequiredGPU: requiredGPU, - JobState: JobStateScheduled, - } - - if ok, err := s.JobExists(job.ID); err != nil { - return "", err - } else if ok { - s.log.Warn("duplicate job skipped", "job_id", job.ID) - return job.ID, nil - } - - // marshal the payload - payloadJSON, err := json.Marshal(job.Payload) - if err != nil { - s.log.Error("failed to marshal job payload", "error", err) - return "", err - } - - // start redis pipeline - pipe := s.client.Pipeline() - - // add payload to redis stream - pipe.XAdd(s.ctx, &redis.XAddArgs{ - Stream: StreamName, - Values: map[string]interface{}{ - "job_id": job.ID, - "payload": string(payloadJSON), - "job_state": string(job.JobState), - }, - }) - - // store metadata in a redis hash - metadataKey := fmt.Sprintf("job:%s", job.ID) - pipe.HSet(s.ctx, metadataKey, map[string]interface{}{ - "type": job.Type, - "retries": job.Retries, - "created": job.Created.Format(time.RFC3339), - "required_gpu": job.RequiredGPU, - "job_state": string(job.JobState), - }) - - // execute pipeline - if _, err := pipe.Exec(s.ctx); err != nil { - s.log.Error("failed to enqueue job", "error", err) - return "", err - } - - s.log.Info("enqueued job", "job_id", job.ID, "job_type", job.Type, "gpu", requiredGPU) - return job.ID, nil -} - -func (s *Scheduler) Close() error { - return s.client.Close() -} - -func (s *Scheduler) JobExists(jobID string) (bool, error) { - exists, err := s.client.Exists(s.ctx, "job:"+jobID).Result() - if err != nil { - return false, err - } - return exists > 0, nil -} - -func (s *Scheduler) ListenForEvents() { - s.log.Info("listening for job events...", "stream", JobEventStream) - - lastID := "$" - - for { - result, err := s.client.XRead(s.ctx, &redis.XReadArgs{ - Streams: []string{JobEventStream, lastID}, - Count: 10, - Block: 5 * time.Second, - }).Result() - - if err != nil { - if errors.Is(err, redis.Nil) { - continue // no new messages - } - s.log.Error("error reading from event stream", "error", err) - time.Sleep(time.Second) - continue - } - - for _, stream := range result { - for _, msg := range stream.Messages { - s.handleEventMessage(msg) - lastID = msg.ID - } - } - } -} - -func (s *Scheduler) handleEventMessage(msg redis.XMessage) { - jobID, _ := msg.Values["job_id"].(string) - state, _ := msg.Values["state"].(string) - timestamp, _ := msg.Values["timestamp"].(string) - supervisor, _ := msg.Values["supervisor"].(string) - - if jobID == "" { - s.log.Warn("received event with missing job_id", "message_id", msg.ID) - return - } - - metadataKey := fmt.Sprintf("job:%s", jobID) - - // Update job state in Redis - if err := s.client.HSet(s.ctx, metadataKey, "job_state", state, "updated_at", timestamp).Err(); err != nil { - s.log.Error("failed to update job metadata", "job_id", jobID, "error", err) - return - } - - s.log.Info("job state updated", - "job_id", jobID, - "state", state, - "supervisor", supervisor) -} diff --git a/src/status.go b/src/status.go deleted file mode 100644 index bbb1292..0000000 --- a/src/status.go +++ /dev/null @@ -1,124 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "log/slog" - - "github.com/redis/go-redis/v9" -) - -type StatusRegistry struct { - redisClient *redis.Client - log *slog.Logger -} - -func NewStatusRegistry(redisClient *redis.Client, log *slog.Logger) *StatusRegistry { - return &StatusRegistry{ - redisClient: redisClient, - log: log, - } -} - -func (sr *StatusRegistry) GetAllSupervisors() ([]SupervisorStatus, error) { - ctx := context.Background() - result := sr.redisClient.HGetAll(ctx, SupervisorStatusKey) - if result.Err() != nil { - return nil, fmt.Errorf("failed to get supervisor status: %w", result.Err()) - } - - var supervisors []SupervisorStatus - for consumerID, statusJSON := range result.Val() { - var status SupervisorStatus - if err := json.Unmarshal([]byte(statusJSON), &status); err != nil { - sr.log.Error("failed to unmarshal supervisor status", "consumer_id", consumerID, "error", err) - continue - } - supervisors = append(supervisors, status) - } - - return supervisors, nil -} - -func (sr *StatusRegistry) GetSupervisor(consumerID string) (*SupervisorStatus, error) { - ctx := context.Background() - result := sr.redisClient.HGet(ctx, SupervisorStatusKey, consumerID) - if result.Err() != nil { - return nil, fmt.Errorf("failed to get supervisor status: %w", result.Err()) - } - - var status SupervisorStatus - if err := json.Unmarshal([]byte(result.Val()), &status); err != nil { - return nil, fmt.Errorf("failed to unmarshal supervisor status: %w", err) - } - - return &status, nil -} - -func (sr *StatusRegistry) GetActiveSupervisors() ([]SupervisorStatus, error) { - allSupervisors, err := sr.GetAllSupervisors() - if err != nil { - return nil, err - } - - var activeSupervisors []SupervisorStatus - for _, supervisor := range allSupervisors { - if supervisor.Status == SupervisorStateActive { - activeSupervisors = append(activeSupervisors, supervisor) - } - } - - return activeSupervisors, nil -} - -func (sr *StatusRegistry) UpdateStatus(consumerID string, status SupervisorStatus) error { - ctx := context.Background() - statusJSON, err := json.Marshal(status) - if err != nil { - return fmt.Errorf("failed to marshal supervisor status: %w", err) - } - - result := sr.redisClient.HSet(ctx, SupervisorStatusKey, consumerID, string(statusJSON)) - if result.Err() != nil { - return fmt.Errorf("failed to update supervisor status: %w", result.Err()) - } - - sr.log.Info("supervisor status updated", "consumer_id", consumerID, "status", status.Status) - return nil -} - -func (sr *StatusRegistry) GetJobStatus(jobID string) (*Job, error) { - ctx := context.Background() - result := sr.redisClient.HGet(ctx, JobStatusKey, jobID) - if result.Err() != nil { - if errors.Is(result.Err(), redis.Nil) { - return nil, fmt.Errorf("job not found: %s", jobID) - } - return nil, fmt.Errorf("failed to get job status: %w", result.Err()) - } - - var job Job - if err := json.Unmarshal([]byte(result.Val()), &job); err != nil { - return nil, fmt.Errorf("failed to unmarshal job status: %w", err) - } - - return &job, nil -} - -func (sr *StatusRegistry) UpdateJobStatus(jobID string, job Job) error { - ctx := context.Background() - statusJSON, err := json.Marshal(job) - if err != nil { - return fmt.Errorf("failed to marshal job status: %w", err) - } - - result := sr.redisClient.HSet(ctx, JobStatusKey, jobID, string(statusJSON)) - if result.Err() != nil { - return fmt.Errorf("failed to update job status: %w", result.Err()) - } - - sr.log.Info("job status updated", "job_id", jobID, "status", job.JobState) - return nil -} diff --git a/src/supervisor.go b/src/supervisor.go deleted file mode 100644 index 96da66a..0000000 --- a/src/supervisor.go +++ /dev/null @@ -1,308 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "log/slog" - "strconv" - "sync" - "time" - - "mist/docker" - - "github.com/docker/docker/client" - "github.com/redis/go-redis/v9" -) - -// CPUImage and CPURuntime are used when running CPU-only jobs (no GPU). -const ( - CPUImage = "pytorch-cpu" - CPURuntime = "runc" -) - -type Supervisor struct { - redisClient *redis.Client - ctx context.Context - cancel context.CancelFunc - consumerID string - gpuType string - dockerMgr *docker.DockerMgr - wg sync.WaitGroup - log *slog.Logger -} - -func NewSupervisor(redisAddr, consumerID, gpuType string, log *slog.Logger) *Supervisor { - redisClient := redis.NewClient(&redis.Options{ - Addr: redisAddr, - }) - - ctx, cancel := context.WithCancel(context.Background()) - - var dockerMgr *docker.DockerMgr - dockerCli, err := client.NewClientWithOpts(client.FromEnv) - if err != nil { - log.Warn("Docker client unavailable, containers will not be started", "error", err) - } else { - dockerMgr = docker.NewDockerMgr(dockerCli, 10, 100) - log.Info("Docker client initialized for container execution") - } - - return &Supervisor{ - redisClient: redisClient, - ctx: ctx, - cancel: cancel, - consumerID: consumerID, - gpuType: gpuType, - dockerMgr: dockerMgr, - log: log, - } -} - -func (s *Supervisor) Start() error { - // Create consumer group if it doesn't exist - err := s.createConsumerGroup() - if err != nil { - return fmt.Errorf("failed to create consumer group: %w", err) - } - - s.wg.Add(1) - go s.processJobs() - - s.log.Info("supervisor started", "consumer_id", s.consumerID, "gpu_type", s.gpuType) - return nil -} - -func (s *Supervisor) createConsumerGroup() error { - result := s.redisClient.XGroupCreateMkStream(s.ctx, StreamName, ConsumerGroup, "$") - if result.Err() != nil { - if result.Err().Error() != "BUSYGROUP Consumer Group name already exists" { - // in this case the group already exists - return result.Err() - } - } - return nil -} - -func (s *Supervisor) processJobs() { - defer s.wg.Done() - s.log.Info("job processor started", "consumer_id", s.consumerID) - defer s.log.Info("job processor stopped", "consumer_id", s.consumerID) - - for { - select { - case <-s.ctx.Done(): - return - default: - // Read from stream with blocking - result := s.redisClient.XReadGroup(s.ctx, &redis.XReadGroupArgs{ - Group: ConsumerGroup, - Consumer: s.consumerID, - Streams: []string{StreamName, ">"}, - Count: 1, - Block: time.Second * 5, - }) - - if result.Err() != nil { - if !errors.Is(result.Err(), redis.Nil) { - s.log.Error("error reading from stream", "error", result.Err()) - } - continue - } - - // Process each message - for _, stream := range result.Val() { - for _, message := range stream.Messages { - s.handleMessage(message) - } - } - } - } -} - -func (s *Supervisor) handleMessage(message redis.XMessage) { - jobID, ok := message.Values["job_id"].(string) - if !ok { - s.log.Error("invalid job_id in message", "message_id", message.ID) - s.ackMessage(message.ID) - return - } - - payloadData, ok := message.Values["payload"].(string) - if !ok { - s.log.Error("invalid payload in message", "message_id", message.ID) - s.ackMessage(message.ID) - return - } - - var payload map[string]interface{} - if err := json.Unmarshal([]byte(payloadData), &payload); err != nil { - s.log.Error("failed to unmarshal payload data", "error", err, "message_id", message.ID) - s.ackMessage(message.ID) - return - } - - jobKey := fmt.Sprintf("job:%s", jobID) - metadata, err := s.redisClient.HGetAll(s.ctx, jobKey).Result() - if err != nil { - s.log.Error("failed to fetch job metadata", "job_id", jobID, "error", err) - s.ackMessage(message.ID) - return - } - - if len(metadata) == 0 { - s.log.Error("job metadata not found", "job_id", jobID) - s.ackMessage(message.ID) - return - } - - jobType := metadata["type"] - requiredGPU := metadata["required_gpu"] - jobState := metadata["job_state"] - - createdTime, _ := time.Parse(time.RFC3339, metadata["created"]) - retries, _ := strconv.Atoi(metadata["retries"]) - - job := Job{ - ID: jobID, - Type: jobType, - Payload: payload, - Retries: retries, - Created: createdTime, - RequiredGPU: requiredGPU, - JobState: JobState(jobState), - } - - // certain jobs require a specific GPU - if !s.canHandleJob(job) { - s.log.Info("skipping job due to GPU mismatch", - "job_id", job.ID, "required_gpu", job.RequiredGPU, "supervisor_gpu", s.gpuType) - // let another supervisor can pick it up - return - } - - s.emitJobEvent(job.ID, JobStateInProgress) - - success := s.processJob(job) - - if success { - s.emitJobEvent(job.ID, JobStateSuccess) - s.updateJobState(job.ID, JobStateSuccess) - s.ackMessage(message.ID) - s.log.Info("job completed successfully", "job_id", job.ID) - } else { - s.emitJobEvent(job.ID, JobStateFailure) - s.updateJobState(job.ID, JobStateFailure) - s.ackMessage(message.ID) - s.log.Error("job failed", "job_id", job.ID) - } -} - -// canHandleJob checks if this supervisor can handle the given job based on GPU requirements -func (s *Supervisor) canHandleJob(job Job) bool { - // If job doesn't specify GPU requirement, any supervisor can handle it - if job.RequiredGPU == "" { - return true - } - - // Job must match supervisor's GPU type - return job.RequiredGPU == s.gpuType -} - -// processJob executes the job by starting a container. For CPU jobs only (no GPU). -// Returns true if the job completed successfully. -func (s *Supervisor) processJob(job Job) bool { - // Only run CPU containers on this machine (no GPU support) - if !s.isCPUJob(job) { - s.log.Info("skipping container start for GPU job on CPU-only machine", "job_id", job.ID) - return true // Ack without running - let GPU supervisor handle - } - - if s.dockerMgr == nil { - s.log.Warn("no container manager, simulating job success", "job_id", job.ID) - return true - } - - volumeName := fmt.Sprintf("job_%s_data", job.ID) - _, err := s.dockerMgr.CreateVolume(volumeName) - if err != nil { - s.log.Error("failed to create volume for job", "job_id", job.ID, "error", err) - return false - } - - containerID, err := s.dockerMgr.RunContainer(CPUImage, CPURuntime, volumeName) - if err != nil { - s.log.Error("failed to run container for job", "job_id", job.ID, "error", err) - _ = s.dockerMgr.RemoveVolume(volumeName, true) - return false - } - - // Run for a short time to simulate work, then clean up - time.Sleep(2 * time.Second) - - if err := s.dockerMgr.StopContainer(containerID); err != nil { - s.log.Error("failed to stop container", "job_id", job.ID, "container_id", containerID, "error", err) - } - if err := s.dockerMgr.RemoveContainer(containerID); err != nil { - s.log.Error("failed to remove container", "job_id", job.ID, "container_id", containerID, "error", err) - } - if err := s.dockerMgr.RemoveVolume(volumeName, true); err != nil { - s.log.Warn("failed to remove volume", "job_id", job.ID, "volume", volumeName, "error", err) - } - - s.log.Info("job container completed", "job_id", job.ID, "container_id", containerID) - return true -} - -// isCPUJob returns true if the job can run on CPU (no GPU required). -func (s *Supervisor) isCPUJob(job Job) bool { - switch job.RequiredGPU { - case "", "CPU": - return true - default: - return false - } -} - - -func (s *Supervisor) emitJobEvent(jobID string, state JobState) { - event := map[string]interface{}{ - "job_id": jobID, - "state": string(state), - "timestamp": time.Now().Format(time.RFC3339), - "supervisor": s.consumerID, - "gpu_type": s.gpuType, - } - - if err := s.redisClient.XAdd(s.ctx, &redis.XAddArgs{ - Stream: JobEventStream, - Values: event, - }).Err(); err != nil { - s.log.Error("failed to emit job event", "job_id", jobID, "state", state, "error", err) - } else { - s.log.Info("emitted job event", "job_id", jobID, "state", state) - } -} - - -func (s *Supervisor) updateJobState(jobID string, state JobState) { - jobKey := fmt.Sprintf("job:%s", jobID) - if err := s.redisClient.HSet(s.ctx, jobKey, "job_state", string(state)).Err(); err != nil { - s.log.Error("failed to update job state", "job_id", jobID, "error", err) - } -} - -func (s *Supervisor) ackMessage(messageID string) { - result := s.redisClient.XAck(s.ctx, StreamName, ConsumerGroup, messageID) - if result.Err() != nil { - s.log.Error("failed to ack message", "message_id", messageID, "error", result.Err()) - } -} - -func (s *Supervisor) Stop() { - s.log.Info("stopping supervisor", "consumer_id", s.consumerID) - s.cancel() - s.wg.Wait() - s.redisClient.Close() -} diff --git a/src/team_runtime.go b/src/team_runtime.go index d3c0e17..fe5362a 100644 --- a/src/team_runtime.go +++ b/src/team_runtime.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "net/http" "os" "path/filepath" "regexp" @@ -231,15 +230,6 @@ func (s *TeamService) ensureVolume(ctx context.Context, sourceTeam, namespace, c return err } } -func teamHTTPError(w http.ResponseWriter, err error) { - var denied *permissionError - if errors.As(err, &denied) { - writeJSON(w, 403, map[string]string{"error": err.Error()}) - return - } - executorError(w, err) -} - func quantity(value string) *resource.Quantity { if value == "" { value = "0" diff --git a/src/util.go b/src/util.go deleted file mode 100644 index 234ee16..0000000 --- a/src/util.go +++ /dev/null @@ -1,64 +0,0 @@ -package main - -import ( - "fmt" - "os" - "time" -) - -const ( - StreamName = "jobs:stream" - ConsumerGroup = "workers" - JobEventStream = "jobs:events" - SupervisorStatusKey = "supervisors:status" - JobStatusKey = "jobs:status" - MaxRetries = 3 - RetryDelay = 5 * time.Second -) - -type JobState string - -const ( - JobStateScheduled JobState = "Scheduled" - JobStateInProgress JobState = "InProgress" - JobStateSuccess JobState = "Success" - JobStateError JobState = "Error" - JobStateFailure JobState = "Failure" - JobStateCancelled JobState = "Cancelled" -) - -type Job struct { - ID string `json:"id"` - Type string `json:"type"` - Payload map[string]interface{} `json:"payload"` - Retries int `json:"retries"` - Created time.Time `json:"created"` - RequiredGPU string `json:"required_gpu,omitempty"` - JobState JobState `json:"job_state"` - ConsumerID *string `json:"consumer_id,omitempty"` - TimeAssigned *time.Time `json:"time_assigned,omitempty"` - TimeStarted *time.Time `json:"time_started,omitempty"` - TimeCompleted *time.Time `json:"time_completed,omitempty"` - Result map[string]interface{} `json:"result,omitempty"` - Error *string `json:"error,omitempty"` -} - -type SupervisorState string - -const ( - SupervisorStateActive SupervisorState = "active" - SupervisorStateInactive SupervisorState = "inactive" - SupervisorStateFailed SupervisorState = "failed" -) - -type SupervisorStatus struct { - ConsumerID string `json:"consumer_id"` - GPUType string `json:"gpu_type"` - Status SupervisorState `json:"status"` - LastSeen time.Time `json:"last_seen"` - StartedAt time.Time `json:"started_at"` -} - -func generateJobID() string { - return fmt.Sprintf("job_%d_%d", time.Now().UnixNano(), os.Getpid()) -} diff --git a/web-interface/package-lock.json b/web-interface/package-lock.json index 51c1767..cf7ce4f 100644 --- a/web-interface/package-lock.json +++ b/web-interface/package-lock.json @@ -15,8 +15,7 @@ "lucide-react": "^0.545.0", "react": "^19.2.0", "react-dom": "^19.2.0", - "tailwindcss": "^4.1.18", - "use-immer": "^0.11.0" + "tailwindcss": "^4.1.18" }, "devDependencies": { "@tailwindcss/typography": "^0.5.16", @@ -3162,15 +3161,6 @@ "node": ">= 4" } }, - "node_modules/immer": { - "version": "11.1.18", - "license": "MIT", - "peer": true, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/immer" - } - }, "node_modules/imurmurhash": { "version": "0.1.4", "dev": true, @@ -4317,14 +4307,6 @@ "punycode": "^2.1.0" } }, - "node_modules/use-immer": { - "version": "0.11.0", - "license": "MIT", - "peerDependencies": { - "immer": ">=8.0.0", - "react": "^16.8.0 || ^17.0.1 || ^18.0.0 || ^19.0.0" - } - }, "node_modules/use-sync-external-store": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.7.0.tgz", diff --git a/web-interface/package.json b/web-interface/package.json index d92fa88..dbb88b7 100644 --- a/web-interface/package.json +++ b/web-interface/package.json @@ -27,8 +27,7 @@ "lucide-react": "^0.545.0", "react": "^19.2.0", "react-dom": "^19.2.0", - "tailwindcss": "^4.1.18", - "use-immer": "^0.11.0" + "tailwindcss": "^4.1.18" }, "devDependencies": { "@tailwindcss/typography": "^0.5.16", diff --git a/web-interface/src/components/Chart.tsx b/web-interface/src/components/Chart.tsx deleted file mode 100644 index e51da0b..0000000 --- a/web-interface/src/components/Chart.tsx +++ /dev/null @@ -1,142 +0,0 @@ -import { ChevronLeft, ChevronRight } from 'lucide-react' -import type { UsageData } from '#/routes/jobs.tsx' - -export default function Chart({ - data, - index, - numComponents, - onPrev, - onNext, -}: { - data: UsageData - index: number - numComponents: number - onPrev: () => void - onNext: () => void -}) { - const max = 100 - const width = 600 - const height = 180 - const padLeft = 40 - const padRight = 10 - const padTop = 10 - const padBottom = 30 - const chartW = width - padLeft - padRight - const chartH = height - padTop - padBottom - - const points = data.observations.map((val, i) => { - const x = padLeft + (i / (data.observations.length - 1)) * chartW - const y = padTop + chartH - (val / max) * chartH - return `${x},${y}` - }) - - const areaPoints = [ - `${padLeft},${padTop + chartH}`, - ...points, - `${padLeft + chartW},${padTop + chartH}`, - ].join(' ') - - const linePoints = points.join(' ') - - const yLabels = [100, 80, 60, 40, 20, 0] - const xLabels = [ - '00:00', - '02:00', - '04:00', - '06:00', - '08:00', - '10:00', - '12:00', - '14:00', - '16:00', - '18:00', - '20:00', - '22:00', - '24:00', - ] - - return ( -
-

- {data.component} -

-
- - - {/* Y-axis labels and grid */} - {yLabels.map((label) => { - const y = padTop + chartH - (label / max) * chartH - return ( - - - {label}% - - - - ) - })} - {/* X-axis labels */} - {xLabels.map((label, i) => { - const x = padLeft + (i / (xLabels.length - 1)) * chartW - return ( - - {label} - - ) - })} - {/* Area fill */} - - {/* Line */} - - - -
- {/* Pagination dots */} -
- {Array.from({ length: numComponents }).map((_, i) => ( - - ))} -
-
- ) -} diff --git a/web-interface/src/util.ts b/web-interface/src/util.ts deleted file mode 100644 index 5d9dba1..0000000 --- a/web-interface/src/util.ts +++ /dev/null @@ -1,25 +0,0 @@ -export type User = { - username: string - role: string - profilePicture: string - email: string -} - -export function getUser(): User { - // Return user information - - // TODO: Call API - We use sample data for now - const user = { - username: 'TheArchons', - profilePicture: '/sample-avatar.png', // real avatars should probably be stored in a bucket - role: 'Software Developer', - email: 'thearchons@utmist.ca', - } - - return user -} - -export function logout() { - // TODO: Call API - we only log a message for now - console.log('logout called') -} From 50ddd1dae06025ba254e5efd11de52b9a67470ba Mon Sep 17 00:00:00 2001 From: utmist-tt <339893541+utmist-tt@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:45:11 -0400 Subject: [PATCH 2/3] Consolidate current cluster SSH storage and deployment documentation --- README.md | 141 ++---- auth-service/README.md | 5 +- cli/docs/setup.md | 37 +- deploy/k3s/Dockerfile.api | 2 +- deploy/k3s/README.md | 437 ++---------------- .../evidence/department-2026-10-09/README.md | 6 +- .../cleanup-validation.json | 56 +++ deploy/k3s/setup-runbook.md | 283 +----------- deploy/private/README.md | 36 +- docs/README.md | 42 ++ docs/architecture.md | 114 ++--- docs/archive/README.md | 21 + docs/archive/complete-foundation.md | 227 +++++++++ docs/archive/foundation-execution-plan.md | 390 ++++++++++++++++ docs/archive/k3s-deployment-history.md | 408 ++++++++++++++++ docs/archive/k3s-setup-history.md | 283 ++++++++++++ docs/archive/kubernetes-pilot.md | 139 ++++++ docs/archive/local-job-foundation.md | 334 +++++++++++++ .../archive/shared-storage-deployment-plan.md | 251 ++++++++++ docs/complete-foundation.md | 225 +-------- docs/department-rollout-execution.md | 8 +- docs/department-rollout.md | 14 +- docs/development.md | 76 +++ docs/foundation-execution-plan.md | 388 +--------------- docs/k3s-setup.md | 224 +++++++++ docs/kubernetes-pilot.md | 137 +----- docs/local-job-foundation.md | 332 +------------ docs/repository-cleanup.md | 78 ++++ docs/shared-storage-deployment-plan.md | 249 +--------- docs/storage.md | 100 ++++ docs/tailscale-ssh.md | 143 ++++++ docs/testing.md | 127 ++--- pre_compute_platform.md | 200 +------- src/jobs.md | 125 +++-- web-interface/README.md | 98 ++-- 35 files changed, 3196 insertions(+), 2540 deletions(-) create mode 100644 deploy/k3s/evidence/department-2026-10-09/cleanup-validation.json create mode 100644 docs/README.md create mode 100644 docs/archive/README.md create mode 100644 docs/archive/complete-foundation.md create mode 100644 docs/archive/foundation-execution-plan.md create mode 100644 docs/archive/k3s-deployment-history.md create mode 100644 docs/archive/k3s-setup-history.md create mode 100644 docs/archive/kubernetes-pilot.md create mode 100644 docs/archive/local-job-foundation.md create mode 100644 docs/archive/shared-storage-deployment-plan.md create mode 100644 docs/development.md create mode 100644 docs/k3s-setup.md create mode 100644 docs/repository-cleanup.md create mode 100644 docs/storage.md create mode 100644 docs/tailscale-ssh.md diff --git a/README.md b/README.md index 1c502b6..9d0f2f8 100644 --- a/README.md +++ b/README.md @@ -1,115 +1,62 @@ # Mist -UTMIST's compute platform. The Kubernetes pilot runs submitted CPU, NVIDIA, -and Tenstorrent workloads as real Jobs. The Jobs page and CLI use the same -API for submission, status, logs, and cancellation. The local foundation adds -live hardware inventory, self-service container images, team workspaces, -scoped shared storage and a durable fair job queue. +UTMIST's research compute platform. Submit Python scripts or tagged container +images, choose CPU, NVIDIA GPUs or Tenstorrent boards, upload datasets, and +collect persistent results through the same website or CLI. -## Current pilot +**Private website:** http://100.73.139.66:8088, accessible through authorized +Tailscale connectivity. A Mist account and team membership are also required. -- `utmist-z1opa08`: k3s server, CPU workloads, two NVIDIA RTX A4000 GPUs. -- `utmist-tt`: QuietBox worker, four n300 boards with two Wormhole chips each. -- Request one or two NVIDIA GPUs, or one to four Tenstorrent boards. Other - jobs can use the remaining devices; requests wait when capacity is occupied. -- Website: http://100.73.139.66:8088. API on that origin: `/api`; auth: `/auth`. +## Current installation -The department rollout includes admin/member login, team common/member folders, -explicit cross-team read/use grants, enforced storage/resource limits, durable -fair admission and revocation. The portal uses interactive compute choices, -clear feature panels and searchable, paginated job rows that expand for details. -Use **http://100.73.139.66:8088** privately through Tailscale. +| Machine | Role | LAN address | Tailscale address | +|---|---|---|---| +| `utmist-z1opa08` | k3s server, API/auth, website, CPU and two RTX A4000 GPUs | `10.0.0.175` | `100.73.139.66` | +| `utmist-tt` | k3s worker, four n300 boards/eight chips, shared storage | `10.0.0.112` | `100.95.175.37` | -See [the current department operating guide](docs/department-rollout.md), -[private deployment operations](deploy/private/README.md), -[completed execution plan and evidence](docs/department-rollout-execution.md), -[portal design](docs/design-system.md), and -[release testing](docs/testing.md). The earlier foundation guides record -historical phases. +The cluster and NFS use the existing LAN. Tailscale provides private portal and +administrator SSH access. Kubernetes/containerd run workload containers. -Credits, Jupyter, chaining, browser image archives/builds and arbitrary distributed -training remain outside this release. Custom TT code/images must use compatible -TT libraries. Automatic backups, HTTPS and monitoring/cleanup products were -explicitly excluded. Hosting remains private HTTP over Tailscale. +## Documentation -## Developer preview against this cluster +Start with the [documentation index](docs/README.md). -**Keep production authentication/team enforcement enabled for research users.** -The standalone trusted-owner mode below is only for development. +- [Using the portal, teams and jobs](docs/department-rollout.md) +- [k3s setup, accelerator allocation and diagnostics](docs/k3s-setup.md) +- [Tailscale and SSH setup](docs/tailscale-ssh.md) +- [Architecture and code organization](docs/architecture.md) +- [Storage and upload limits](docs/storage.md) +- [Deployment and operations](deploy/private/README.md) +- [Developer setup](docs/development.md), [CLI](cli/docs/setup.md), [testing](docs/testing.md) -Use Go 1.25.1 and Node.js 22.16.0 or newer. Apply the workload prerequisites -and claim templates described in the deployment guide first. Kubernetes is -now the default executor; Redis and Docker supervisors are unnecessary for -this path. This server has both runtimes under `~/.local/share/mist-runtimes`: +Main contains the department release merged through +[PR #110](https://github.com/UTMIST/Mist/pull/110). The earlier main and operating +snapshots are preserved in the [historical archive](docs/archive/README.md). -```bash -export PATH=/home/utmist/.local/share/mist-runtimes/go-1.25.1/bin:/home/utmist/.local/share/mist-runtimes/node-22.16.0/bin:$PATH -``` - -```bash -export KUBECONFIG=/home/utmist/.kube/config -# Standalone development defaults to a trusted local owner with auth disabled. -# For the deployed authenticated/shared API, use its existing port-forward. -export MIST_PILOT_OWNER=local-dev -cd src -go run . -``` - -If the deployed API's port-forward already occupies port 3000, use that API -or stop only `mist-api-forward.service` before starting a local backend. -Standalone `go run` uses the default base-image allowlist. Set -`MIST_ALLOWED_IMAGES` to the deployment's approved references when developing -with the packaged example images or other custom images. - -In another terminal: - -```bash -cd web-interface -npm ci -npm run dev -- --host 127.0.0.1 --port 3001 --strictPort -``` +## Development and checks -The Vite proxy forwards `/api` and `/auth` to the API on port 3000. The installed user -services already run both endpoints on this machine: +Use Go **1.25.1** and Node.js **22.16.0 or newer**. The Go workspace contains the +API (`src`) and CLI (`cli`). From the repository root: ```bash -systemctl --user status mist-api-forward mist-web +go test ./src/... ./cli/... +npm --prefix web-interface ci +npm --prefix web-interface test +npm --prefix web-interface run lint +npm --prefix web-interface run build ``` -## CLI - -From the repository root: - -```bash -go -C cli build -o ../bin/mist . -printf "print('Hello from a real Mist job')\n" > /tmp/hello.py -export MIST_API_URL=http://100.73.139.66:8088/api -bin/mist auth login --email your-member-email@example.org -bin/mist job submit /tmp/hello.py --compute CPU -bin/mist job list --all -bin/mist job status -bin/mist job logs -bin/mist job cancel -``` +For a local UI against the authenticated deployed API, follow +[developer setup](docs/development.md). For the existing installation, deploy with +`bash deploy/private/build-and-deploy.sh`; read its [prerequisites](deploy/private/README.md) +before running it. Fresh-host provisioning is covered separately by the k3s guide. -Use `--compute NVIDIA --devices 1` for one GPU, or -`--compute TT --devices 1` for one two-chip board. The submitted script must -actually use the chosen accelerator. `--api-url` or `MIST_API_URL` selects an -API endpoint. See [CLI setup](cli/docs/setup.md) for resource options. - -## Tests and legacy development - -```bash -go -C src test -run 'Test(Kubernetes|Hardware|GPURequest|CustomImage|TenstorrentCustom|Shared|Member)' ./... -go -C cli test ./... -cd web-interface -npm test -- --run -npm run build -npx tsc --noEmit -``` +The current API uses Kubernetes only. The unfinished Redis/Docker supervisor, +fake frontend account/chart and dummy CLI configuration have been retired. +Docker remains build tooling for images; it does not schedule Mist jobs. -The older Docker/Redis executor is available with `MIST_EXECUTOR=docker`. -Its supervisor execution is incomplete. For its existing tests, start a -local Redis instance and run `go -C src test ./...`; Docker integration tests -also need Docker permissions and their test image. This path is retained for -compatibility and is separate from the validated Kubernetes executor. +Custom accelerator images need compatible training libraries. Credits, Jupyter, +chaining, browser image archives/builds, private pull-credential management and +distributed multi-machine training are outside this release. Automatic backups, +HTTPS and monitoring/cleanup products were explicitly excluded. This is a private +HTTP installation with one control plane, one auth instance and one storage host. diff --git a/auth-service/README.md b/auth-service/README.md index 9f15490..d4c168e 100644 --- a/auth-service/README.md +++ b/auth-service/README.md @@ -21,5 +21,6 @@ Database-backed rate limiting survives restarts. Upload/storage metadata live in Go/NFS, not in this auth database. Member passwords use the library's password implementation; Go does not hash passwords or mint authentication tokens. -See [operations and recovery](../deploy/private/README.md) for online backups, -restoration, private HTTP/HTTPS configuration and deployment. +See [deployment operations](../deploy/private/README.md) for persistent-state +handling and private HTTP deployment. Automatic backups/recovery and HTTPS were +excluded from this release; the existing backup script is a manual utility. diff --git a/cli/docs/setup.md b/cli/docs/setup.md index 152ffb1..2377fda 100644 --- a/cli/docs/setup.md +++ b/cli/docs/setup.md @@ -3,8 +3,8 @@ Requires Go 1.25.1. Build from the repository root: ```bash -go -C cli build -o ../bin/mist . -bin/mist --help +go -C cli build -o ../bin/mist-cli . +bin/mist-cli --help ``` The default API URL is `http://127.0.0.1:3000`. Override it using @@ -14,38 +14,40 @@ Tailnet device or either SSH host, then authenticate: ```bash export MIST_API_URL=http://100.73.139.66:8088/api -bin/mist auth login --email your-member-email@example.org +bin/mist-cli auth login --email your-member-email@example.org ``` The password prompt does not echo. Automation can use `--password-stdin`. The CLI saves a real session cookie in a mode-0600 config; `--config` selects -an alternate file. Logout with `bin/mist auth logout` revokes the session. +an alternate file. Logout with `bin/mist-cli auth logout` revokes the session. Select your team before submitting new work: ```bash -bin/mist team list -bin/mist team use YOUR_TEAM_ID -bin/mist job submit /path/to/train.py --compute NVIDIA --devices 1 \ +bin/mist-cli team list +read -r -p "Paste your team ID from the list: " TEAM_ID +bin/mist-cli team use "$TEAM_ID" +bin/mist-cli job submit /path/to/train.py --compute NVIDIA --devices 1 \ --cpu 2 --memory 2Gi --timeout 1800 --name my-training -bin/mist job submit /path/to/tt_train.py --compute TT --devices 1 -bin/mist job submit /path/to/task.sh --compute CPU -bin/mist job list # Waiting and running jobs -bin/mist job list --all # Includes completed, failed, and cancelled jobs -bin/mist job status -bin/mist job logs -bin/mist job cancel +bin/mist-cli job submit /path/to/tt_train.py --compute TT --devices 1 +bin/mist-cli job submit /path/to/task.sh --compute CPU +bin/mist-cli job list # Waiting and running jobs +bin/mist-cli job list --all # Includes completed, failed, and cancelled jobs +read -r -p "Paste the job ID returned by submission: " JOB_ID +bin/mist-cli job status "$JOB_ID" +bin/mist-cli job logs "$JOB_ID" ``` Submission sends the actual file contents to the API. `.py` and `.sh` files are supported, up to 32 KiB. `--image` selects an image allowed by the API; otherwise the API chooses a runtime for the compute type. Upload datasets through the website, then pass `--dataset dataset-ID` to attach -one read-only at `/inputs`. Dependencies belong in an approved image. +one read-only at `/inputs`. Dependencies belong in an image allowed by the team registry policy. Result files are downloadable from the website's job Files panel. NVIDIA counts are whole GPUs (one or two); TT counts are whole n300 boards (one to four, two chips each). Scripts must use the corresponding runtime. -The TT profile uses QuietBox's existing TT-Metal installation. For a small +The default TT profile uses QuietBox's existing TT-Metal installation; a +compatible custom image can select the container runtime. For a small built-in accelerator training check, use the Jobs page or the API's `training-smoke` submission type. @@ -57,7 +59,8 @@ from the member account. Legacy fake-token authentication is removed. go -C cli test ./... ``` -The actual team ID comes from `team list`; do not paste `YOUR_TEAM_ID` literally. +The prompts above take actual IDs returned by `team list` and submission. +To cancel that job while waiting/running, use `bin/mist-cli job cancel "$JOB_ID"`. `--team` or `MIST_TEAM` overrides the saved selection. Use `team use legacy` to read historical jobs; new work requires a team. `--scope common` requires common write permission. Custom tagged public images are allowed by team registry policy; diff --git a/deploy/k3s/Dockerfile.api b/deploy/k3s/Dockerfile.api index 6e9766d..0cad2bf 100644 --- a/deploy/k3s/Dockerfile.api +++ b/deploy/k3s/Dockerfile.api @@ -1,6 +1,6 @@ # Build the static binary from the repository root: # CGO_ENABLED=0 go -C src build -o ../bin/mist-api . -# docker build -f deploy/k3s/Dockerfile.api -t mist-api:foundation-20261008 . +# docker build -f deploy/k3s/Dockerfile.api -t mist-api:departments-20261009 . FROM alpine:3.16.3 COPY bin/mist-api /usr/local/bin/mist-api USER 65532:65532 diff --git a/deploy/k3s/README.md b/deploy/k3s/README.md index 0454621..964cc37 100644 --- a/deploy/k3s/README.md +++ b/deploy/k3s/README.md @@ -1,403 +1,34 @@ -# Accelerator jobs on k3s - - -**October 9 update:** Team workspaces, scoped storage, enforced limits, durable queueing and private -production hosting are implemented. Earlier local pilot descriptions below record the -original phase. Use [the current operating guide](../../docs/department-rollout.md) for today's behavior. - -For the original NVIDIA-node setup and recovery procedure, read the -[setup runbook](setup-runbook.md). Its final section records the QuietBox worker -join on 2026-10-03. - -The NVIDIA server runs Ubuntu 22.04, k3s 1.36, and two RTX A4000 GPUs. The host -driver is pre-installed. NVIDIA GPU Operator v26.7.1 is configured with -[`nvidia-gpu-operator-values.yaml`](nvidia-gpu-operator-values.yaml). Pin the -chart version when installing or upgrading: - -```bash -sudo env KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm upgrade --install gpu-operator \ - oci://nvcr.io/nvidia/cloud-native-charts/gpu-operator \ - --version v26.7.1 --namespace gpu-operator --create-namespace \ - --values deploy/k3s/nvidia-gpu-operator-values.yaml --wait -``` - -Run from the repository root. Check that the operator pods are healthy and the -node advertises two GPUs before submitting a workload: - -```bash -sudo k3s kubectl -n gpu-operator get pods -sudo k3s kubectl describe node utmist-z1opa08 -``` - -The `Allocatable` section must include `nvidia.com/gpu: 2`. Then run the -single-GPU CUDA smoke test: - -```bash -sudo k3s kubectl create namespace mist -sudo k3s kubectl apply -f deploy/k3s/gpu-smoke-job.yaml -sudo k3s kubectl -n mist wait --for=condition=complete job/gpu-smoke --timeout=180s -sudo k3s kubectl -n mist logs job/gpu-smoke -``` - -The expected log contains `Test PASSED`. To rerun the Job, delete only the -finished smoke Job and apply the manifest again: - -```bash -sudo k3s kubectl -n mist delete job gpu-smoke -sudo k3s kubectl apply -f deploy/k3s/gpu-smoke-job.yaml -``` - -No Service or NodePort is needed for a batch Job. This test verifies that k3s -can allocate an NVIDIA GPU and execute CUDA. Mist now submits these resources -through its Kubernetes executor; see the [API setup](#mist-api-cli-and-jobs-page). - -The cluster also has `utmist-tt` as an agent at `10.0.0.112`. Tenstorrent -DRA allocation is installed there: four n300 boards, each containing two -Wormhole chips. The NVIDIA smoke Job remains specific to `utmist-z1opa08`. - -## Tenstorrent allocation - -TT-Operator **0.3.0** installs Fabric Manager **0.2.30** and the DRA driver -**0.0.59**, configured by -[`tenstorrent-operator-values.yaml`](tenstorrent-operator-values.yaml). -The existing host KMD/firmware and the existing NFD installation are retained; -driver management, duplicate NFD, telemetry, JobSet, and PMIx are disabled. -The manually verified `tenstorrent.com/has-tt=true` node label enables discovery. - -To reproduce the installation from the repo root: - -```bash -export KUBECONFIG=/home/utmist/.kube/config -k3s kubectl label node utmist-tt tenstorrent.com/has-tt=true --overwrite -helm upgrade --install tt-operator oci://ghcr.io/tenstorrent/helm/tt-operator \ - --version 0.3.0 --namespace tt-operator-system --create-namespace \ - --values deploy/k3s/tenstorrent-operator-values.yaml -k3s kubectl apply -f deploy/k3s/tenstorrent-workload-policy.yaml -k3s kubectl -n tt-operator-system get pods -k3s kubectl get deviceclasses,resourceslices -k3s kubectl get resourceslices -o yaml -``` - -All three operator pods must be Ready. The `tenstorrent.com` DeviceClass -and a ResourceSlice on `utmist-tt` must exist, with **four** `n300` devices -whose `chipCount` is **2**. ResourceSlices are the inventory; claims show -reservations. TT does not appear as an `nvidia.com/gpu` node resource. - -The supported reservation unit is a whole **n300 board**, including its -remote chip. Jobs can reserve one to four boards; separate jobs can use -the remaining boards. A request waits if insufficient boards or other -requested resources are free. Each pod gets its own ResourceClaim through -a ResourceClaimTemplate, and only its allocated device nodes are injected -through CDI. Boards are released when the pod finishes, even while its -completed Job and checkpoints are retained. - -[`training-tenstorrent-job.yaml`](training-tenstorrent-job.yaml) now requests -one board, runs without privileged access, and reserves 2 GiB of 1 GiB -hugepages. The host still has 16 GiB of such hugepages. It mounts the existing -TT-Metal checkout and Python environment read-only, so it is specific to -this QuietBox rather than a portable training image. - -For uniquely named training checks and separate checkpoint directories, -use [`submit_tenstorrent_smoke.py`](submit_tenstorrent_smoke.py). It needs -Python PyYAML, available on the server. For example: - -```bash -export KUBECONFIG=/home/utmist/.kube/config -python3 deploy/k3s/submit_tenstorrent_smoke.py my-tt-test-a --boards 1 -python3 deploy/k3s/submit_tenstorrent_smoke.py my-tt-test-b --boards 1 -k3s kubectl -n mist get jobs,pods,resourceclaims -k3s kubectl -n mist logs -f job/my-tt-test-a -``` - -Use `--boards 2` for four chips, or `--boards 4` for all eight. The script -submits the small regression test described below. A different model needs -compatible Tenstorrent training code. Claim count, expected chip count, -hugepage reservation, and checkpoint directory must all match the job. -These tests train on each allocated chip independently; they do not perform -distributed training of one model. For multiple boards, the check launches -one runtime per board using its PCI address in `TT_VISIBLE_DEVICES`. This -avoids assuming that an arbitrary selected subset forms a connected mesh. -The scheduler reserves boards but does not promise their fabric topology; -a model spanning boards needs an appropriate topology and training runtime. - -The admission policy applies to pods in `mist`, including init and ephemeral -containers. It rejects privileged access, added capabilities, host PID/IPC, -arbitrary host mounts, and writable mounts of the host runtime. This blocks -the previous privileged whole-machine test from being submitted again. -Individual character-device mounts remain allowed for the verification -probes, with access to unclaimed devices denied by device cgroups. The policy -is a guard for managed workloads, not a complete security boundary for -untrusted tenants or cluster administrators; hugepage mounts are host-shared. - -### Allocation acceptance test - -On 2026-10-03, four jobs reserved distinct boards and trained concurrently -on all eight chips. Their first-chip training intervals overlapped for -**3.924 seconds**. Each job could open its own device and received permission -errors opening the other three, even when the other character nodes were -mounted for the probe. A fifth job remained Pending with `cannot allocate -all claims`, then started automatically after a board was released. All five -jobs passed training, held-out-data, and checkpoint reload checks. Automatic -claim cleanup left all four boards free afterward. - -To repeat that live test (it requires all four boards to be free): - -```bash -export KUBECONFIG=/home/utmist/.kube/config -python3 deploy/k3s/verify_tenstorrent_allocation.py \ - --artifacts /home/utmist/k3s-allocation-results-new-run -``` - -The test holds initialized jobs at a shared-file barrier to inspect their -reservations, checks actual device access, observes the fifth job queuing, -and then releases the barrier. It verifies overlapping training timestamps, -loss thresholds, checkpoint files, and board reuse. Test jobs are retained; -the temporary PVC reader is deleted. The barrier times out after five -minutes if interrupted. The checkpoint PVC is retained. - -Operator restart recovery and an NVIDIA CUDA regression check also passed. -See [`tenstorrent-allocation-results.json`](tenstorrent-allocation-results.json) -for pinned image IDs and results. Logs, claim/queue snapshots, admission -checks, and checkpoint archives are in -`/home/utmist/k3s-allocation-results-2026-10-03` on the NVIDIA server. -The vendor currently advertises an incorrect board-memory capacity in the -ResourceSlice; these claims select board type/count and do not use that -capacity for placement. Verify/correct it before adding memory-based selectors. - -## Training smoke jobs - -[`training_smoke.py`](training_smoke.py) learns a 32-input/32-output linear -mapping from 128 synthetic samples and evaluates 64 held-out samples. Each -device must complete 100 gradient updates, reduce training MSE by at least -98%, achieve held-out MSE below `0.0005`, change its weights, and reload its -checkpoint successfully. NVIDIA uses PyTorch CUDA autograd. Tenstorrent uses -TT-NN for forward passes, explicit analytic gradients, and weight updates; -CPU Torch creates the data, reads metrics, and saves checkpoints. This does -not validate automatic differentiation or arbitrary models on Tenstorrent. - -The NVIDIA Job reserves both GPUs and tests each independently. The current -Tenstorrent manifest reserves one n300 board and tests its two chips -independently. The original eight-chip privileged Job, named -`training-tenstorrent`, is retained only as historical evidence; the new -Job is `training-tenstorrent-dra` and uses DRA/CDI reservations. - -Run from the repository root: - -```bash -sudo k3s kubectl -n mist create configmap training-smoke-script \ - --from-file=deploy/k3s/training_smoke.py --dry-run=client -o yaml \ - | sudo k3s kubectl apply -f - -sudo k3s kubectl apply -f deploy/k3s/training-nvidia-job.yaml -sudo k3s kubectl apply -f deploy/k3s/training-tenstorrent-job.yaml -sudo k3s kubectl -n mist logs -f job/training-nvidia -sudo k3s kubectl -n mist logs -f job/training-tenstorrent-dra -sudo k3s kubectl -n mist get jobs,pvc -``` - -Each successful log ends with `TRAINING_PASSED`. Checkpoints and -`results.json` are stored in the corresponding `training-*-checkpoints` PVC, -with the one-board TT check under its `dra-single` directory, -which uses storage local to the selected node. Preserve the PVCs when -removing a completed Job. Delete only the completed Job before applying its -manifest to rerun it; retained checkpoint filenames will be overwritten. -The TT manifest depends on QuietBox's existing host paths, hugepages, and -TT-Metal checkout and is not a portable training image. - -The original whole-node Jobs passed on 2026-10-03, with exit code zero. All ten devices changed -weights, met the held-out-data threshold, and reloaded their saved -checkpoints. The archives were also checked after the Pods completed. - -| Backend | Devices tested independently | Initial training MSE | Final training MSE | Held-out MSE | -| --- | --- | --- | --- | --- | -| NVIDIA RTX A4000, PyTorch FP32 | 2 GPUs | 0.319374 | 0.000000201866 | 0.000000932067 | -| Wormhole, TT-NN BF16 | 8 chips | 0.319374 | 0.0000623871 | 0.000175072 | - -Precision and implementations differ; these figures establish convergence -and are not a performance comparison. Exact image IDs, the TT-Metal host -commit, Job timestamps, artifact locations, and acceptance thresholds are in -[`training-smoke-results.json`](training-smoke-results.json). Local logs and -checkpoint archives are retained under -`/home/utmist/k3s-training-results-2026-10-03` on the NVIDIA server. The -completed Kubernetes Jobs and their checkpoint PVCs are retained too. - - -## Mist API, CLI and Jobs page - -The API now uses Kubernetes by default and runs in `mist-system`. Its service -account manages Jobs and script ConfigMaps in `mist`, reads Pod logs/events -and TT claims, and has no cluster administrator credentials. The configured -pilot owner is `utmist`. No authentication or public ingress is configured. -The foundation also lists node/pod/claim/slice state for live hardware inventory. -See [the foundation guide](../../docs/local-job-foundation.md) for contracts, -website instructions, persistent outputs, and the developer handoff. - -### Build and install on the current server - -Requires Go 1.25.1, Docker, k3s, and the private administrative kubeconfig. -Use the already installed GPU/TT operators described above. The API image is -built locally and imported into k3s containerd; `imagePullPolicy: Never` -means that it must be present on the selected server before deployment. - -```bash -export KUBECONFIG=/home/utmist/.kube/config -CGO_ENABLED=0 go -C src build -o ../bin/mist-api . -docker build -f deploy/k3s/Dockerfile.api -t mist-api:foundation-20261008 . -docker build -f deploy/k3s/examples/Dockerfile.training \ - -t mist-training:cpu-v1 deploy/k3s/examples -docker build -f deploy/k3s/examples/Dockerfile.training \ - --build-arg BASE_IMAGE=pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime \ - -t mist-training:nvidia-v1 deploy/k3s/examples -docker save -o /tmp/mist-foundation-images.tar \ - mist-api:foundation-20261008 mist-training:cpu-v1 mist-training:nvidia-v1 -sudo k3s ctr -n k8s.io images import --platform linux/amd64 /tmp/mist-foundation-images.tar -k3s kubectl apply -f deploy/k3s/mist-api.yaml -k3s kubectl apply -f deploy/k3s/mist-tenstorrent-claims.yaml -k3s kubectl apply -f deploy/k3s/tenstorrent-workload-policy.yaml -k3s kubectl -n mist create configmap training-smoke-script \ - --from-file=deploy/k3s/training_smoke.py --dry-run=client -o yaml \ - | k3s kubectl apply -f - -k3s kubectl -n mist-system rollout status deployment/mist-api --timeout=120s -``` - -The API manifest creates namespaces, RBAC and all three checkpoint PVCs -without submitting a training Job. If rebuilding the same local image tag, -import the new image and run `rollout restart deployment/mist-api` in -`mist-system`. Preserve Jobs and PVCs when restarting the API. - -For a manual local endpoint: - -```bash -k3s kubectl -n mist-system port-forward service/mist-api 3000:3000 --address 127.0.0.1 -``` - -Then in another terminal: - -```bash -curl http://127.0.0.1:3000/healthz -go -C cli build -o ../bin/mist . -bin/mist job list --all -cd web-interface -npm install --package-lock=false -npm run dev -- --host 127.0.0.1 --port 3001 --strictPort -``` - -Open http://127.0.0.1:3001/jobs. Choose CPU, NVIDIA GPUs or Tenstorrent boards; -submit a Python script, select an approved **Container image** with its own -entrypoint/arguments, or choose **Small training check** for an accelerator. -The page displays actual placement, queue messages, exit status and logs. -Hardware availability is live; the Machines route uses the same inventory. -Cancel stops a waiting/running workload and retains cancellation history. -CLI commands are in [the CLI guide](../../cli/docs/setup.md). - -The installed user units keep both local services running during the user's -systemd session. Their paths target this server and need adjustment on another -host; they do not enable user lingering or guarantee operation before login. - -```bash -mkdir -p ~/.config/systemd/user -cp deploy/k3s/mist-api-forward.service deploy/k3s/mist-web.service ~/.config/systemd/user/ -systemctl --user daemon-reload -systemctl --user enable --now mist-api-forward mist-web -systemctl --user status mist-api-forward mist-web -``` - -Do not run manual listeners on the same ports while these units are active. -Stop them with `systemctl --user stop mist-web mist-api-forward` if needed. - -### Submission examples - -```bash -curl -sS http://127.0.0.1:3000/jobs -H 'Content-Type: application/json' \ - -d '{"name":"hello","accelerator":"cpu","command":["python","-c","print(42)"]}' -curl -sS http://127.0.0.1:3000/jobs -H 'Content-Type: application/json' \ - -d '{"name":"one-board","type":"training-smoke","accelerator":"tenstorrent","device_count":1}' -curl -sS http://127.0.0.1:3000/jobs -H 'Content-Type: application/json' \ - -d '{"name":"one-gpu","type":"training-smoke","accelerator":"nvidia","device_count":1}' -``` - -CPU jobs request no accelerator. NVIDIA requests accept one or two GPUs; -TT requests accept one to four boards, each containing two chips. Defaults -are 250m CPU/256Mi RAM for CPU jobs, 2 CPU/2Gi for NVIDIA, and 2 CPU/4Gi plus -2Gi hugepages per TT board. CPU and memory limits equal requests. The API -accepts CPU up to 8, memory 64Mi to 32Gi (TT minimum 2Gi), and deadlines from -10 seconds to 24 hours, default 600 seconds. Deadlines include queue time. -Images must be approved by the API's allowlist; submissions cannot specify -privileged access or host mounts. Environment variables `MIST_JOB_ID`, -`MIST_OUTPUT_DIR=/outputs`, and `MIST_CHECKPOINT_DIR=/checkpoints/` -provide the job identity and two aliases for its persistent directory. - -Storage remains node-local with a shared backing PVC per compute profile. -New workloads mount only their job-specific subdirectory. Retained Jobs -provide history across API restarts, without TTL. Normal logs remain in pod -logs; cancellation preserves their last 32KiB in the Job metadata. Keep the -Pods/PVCs or archive them before cleanup. This is a local single-owner pilot, -not an authenticated multi-user deployment. - -### Custom images and private registries - -The deployment's `MIST_ALLOWED_IMAGES` approves exact image references; -`GET /images` exposes them to the website. For real workloads, build/push an -image containing code/dependencies, approve its immutable tag/digest, and let -the selected node pull it. Local example tags must be imported on the NVIDIA -server. An approved image is not guaranteed to be pullable or CUDA compatible. -The TT profile still requires its tested image and read-only host runtime. - -CPU/NVIDIA container jobs preserve image ENTRYPOINT/CMD and WORKDIR when no -override is supplied. Arguments are array entries, not a parsed shell command. -Scripts and training checks retain their tested runtime profiles. Write results -to `/outputs`; files elsewhere in the container are not persistent outputs. - -For private images, an administrator creates a `kubernetes.io/dockerconfigjson` -Secret in workload namespace `mist`, then adds `MIST_IMAGE_PULL_SECRETS` to -the API Deployment environment with the Secret name(s), comma-separated. -Jobs receive those names as `imagePullSecrets`. Mist does not read Secret -contents or accept registry passwords in the submission form. Keep Secret -manifests/credentials outside Git. K3s node registry configuration is another -option: [private registries](https://docs.k3s.io/installation/private-registry). - -### Foundation acceptance - -With both accelerators free and the example images imported, run the browser -story, then the allocation/output checks using the same artifact directory: - -```bash -export KUBECONFIG=/home/utmist/.kube/config -export MIST_PLAYWRIGHT_MODULE=/tmp/mist-browser-check/node_modules/playwright/index.mjs -node deploy/k3s/verify_foundation_browser.mjs /home/utmist/mist-foundation-results-new-run -python3 deploy/k3s/verify_foundation.py --artifacts /home/utmist/mist-foundation-results-new-run -``` - -Playwright and Chromium are test tooling, separate from the web application. -Set `MIST_PLAYWRIGHT_MODULE` to its installed `index.mjs`, or install Playwright -where Node can resolve it. `MIST_WEB_URL` changes the browser endpoint; -`--url` changes the API endpoint for the Python tool. -Checks cover packaged CPU/NVIDIA images, both GPUs, TT training, real UI -states/logs/cancellation, occupied/free counts, queues, reuse, and saved files -read by fresh containers on both nodes. Temporary read-only output probes are -removed; test Jobs and result directories remain. The final markers are -`FOUNDATION_BROWSER_PASSED` and `FOUNDATION_ALLOCATION_OUTPUTS_PASSED`. -October 8 evidence is under `/home/utmist/mist-foundation-results-2026-10-08`. -See [the checked-in summary](foundation-results.json) for actual job IDs, -deployed image ID, and completed checks. - -### API acceptance test - -This test needs both GPUs and all four TT boards available. It submits 17 -uniquely named jobs, holds initialized devices at barriers to prove separate -reservations, verifies queued jobs, training/checkpoint reload, CPU failures, -deadlines, retained cancellation logs, and cancellation freeing all TT boards. - -```bash -export KUBECONFIG=/home/utmist/.kube/config -python3 deploy/k3s/verify_mist_api.py \ - --artifacts /home/utmist/mist-api-results-new-run -``` - -Successful output ends with `MIST_API_PASSED`. Jobs and checkpoint directories -are retained. A barrier times out after five minutes if interrupted. The -verification script uses the operator's administrative context to inspect -claims/devices and release test barriers; the Mist API itself cannot exec. -Results from 2026-10-03 are in [mist-api-results.json](mist-api-results.json), -with raw evidence and checkpoint archives under -`/home/utmist/mist-api-results-2026-10-03`. These checks establish small model -training and allocation, not large/distributed-model support or performance. +# Kubernetes deployment assets + +Current installation: two k3s nodes, authenticated team jobs, scoped NFS storage +and private Tailscale portal. Start with the [k3s setup/handoff](../../docs/k3s-setup.md) +and [deployment operations](../private/README.md). + +## Manifest and tooling map + +| Assets | Purpose | +|---|---| +| `mist-api.yaml`, `mist-auth.yaml` | Single API/controller and Better Auth deployment | +| `team-rbac.yaml`, `team-workload-policy.yaml` | Team provisioning authority and workload admission | +| `nvidia-gpu-operator-values.yaml` | Pinned host-driver/CDI configuration | +| `tenstorrent-operator-values.yaml`, `mist-tenstorrent-claims.yaml` | TT operator configuration and retained legacy claims | +| `tenstorrent-workload-policy.yaml` | Admission protection for legacy `mist` diagnostics | +| `storage/` | NFS manifests, bounded pool provisioning, migration/growth | +| `training_smoke.py`, `examples/` | Accelerator checks and packaged-image examples | +| `gpu-smoke-job.yaml`, `training-*-job.yaml`, `submit_tenstorrent_smoke.py` | Administrator diagnostics in legacy namespace | +| `verify_department*`, `verify_split_storage.py`, `verify_large_dataset.py` | Current authenticated live verifiers | +| `verify_tenstorrent_allocation.py` | Infrastructure-level TT allocation diagnostic | +| `evidence/department-2026-10-09/` | Current release evidence | +| Earlier `*-results.json` | Historical dated test evidence | + +Direct manifest diagnostics require cluster administrator access and bypass +portal team admission. Researchers should use the authenticated portal or CLI. +Archived pre-team API/browser verifiers have been retired from executable source. + +The application image build uses [Dockerfile.api](Dockerfile.api). API/auth images +are imported into main's k3s containerd, not scheduled by a Docker supervisor. +Use `bash deploy/private/build-and-deploy.sh` for the existing installation. + +[Old deployment history](../../docs/archive/k3s-deployment-history.md) and +[original node setup](../../docs/archive/k3s-setup-history.md) preserve earlier +observations; their unauthenticated/node-local instructions are superseded. diff --git a/deploy/k3s/evidence/department-2026-10-09/README.md b/deploy/k3s/evidence/department-2026-10-09/README.md index 512a775..9766550 100644 --- a/deploy/k3s/evidence/department-2026-10-09/README.md +++ b/deploy/k3s/evidence/department-2026-10-09/README.md @@ -1,6 +1,10 @@ # Department verification — October 9, 2026 -Evidence from the actual two-node private pilot. Reports contain synthetic +Evidence from the actual two-node private pilot before repository cleanup. +The original Go/Docker/logger counts below belong to that earlier release; +[current cleanup verification](../../../../docs/repository-cleanup.md) records +the maintained suites after their obsolete implementations were removed. +Reports contain synthetic verification data and job/device IDs; no passwords, cookies or auth secrets. Scripts are in `deploy/k3s/verify_department*`. They require existing authorized local credentials and recorded machine addresses; inspect them before rerunning. diff --git a/deploy/k3s/evidence/department-2026-10-09/cleanup-validation.json b/deploy/k3s/evidence/department-2026-10-09/cleanup-validation.json new file mode 100644 index 0000000..59eeb18 --- /dev/null +++ b/deploy/k3s/evidence/department-2026-10-09/cleanup-validation.json @@ -0,0 +1,56 @@ +{ + "recorded_at": "2026-10-09T08:41:50.836023+00:00", + "base_main_commit": "3431ffdb713fac72881fc7f61f2750a82fe3f916", + "source_branch": "chore/main-cleanup-20261009", + "go": { + "race_detector": true, + "api_tests_passed": 29, + "cli_tests_passed": 6, + "skipped_tests": 0, + "vet": "passed", + "api_cli_static_builds": "passed", + "retired_executor_fail_fast": "passed", + "deadcode_tool": "golang.org/x/tools/cmd/deadcode v0.38.0, with tests", + "unreachable_functions_after_cleanup": 0 + }, + "frontend": { + "react_tests_passed": 10, + "typescript": "passed", + "lint": "passed", + "production_build": "passed", + "npm_audit_vulnerabilities": 0 + }, + "migration": { + "tests_passed": 6, + "skipped_tests": 0, + "environment": "disposable root container; read-only source, isolated tiny temporary files; no live storage" + }, + "candidate_api_smoke": { + "checks": { + "compiled_api_startup_real_kubernetes_health": true, + "real_auth_gateway_session": true, + "anonymous_jobs_denied": true, + "sigterm_graceful_shutdown": true + }, + "temporary_listener": "localhost only, removed after test", + "team_controller_enabled": false, + "cluster_or_account_mutations": false, + "production_redeployed": false + }, + "infrastructure_audit": { + "nodes_ready": 2, + "api_auth_available": true, + "operators_ready": true, + "ssh_quietbox_connection": "passed", + "tailscale_ssh_handler_enabled": false, + "live_host_service_enablement_checked": true + }, + "production_redeployed": false, + "large_storage_tests_run": false, + "fresh_training_or_browser_runs_claimed": false, + "notes": [ + "Prior live training/browser/isolation evidence is for the pre-cleanup department release", + "Main still requires independent pull-request review", + "Legacy source removal preserves existing Kubernetes jobs/PVCs and host files" + ] +} diff --git a/deploy/k3s/setup-runbook.md b/deploy/k3s/setup-runbook.md index 0bf7116..5540d11 100644 --- a/deploy/k3s/setup-runbook.md +++ b/deploy/k3s/setup-runbook.md @@ -1,278 +1,9 @@ -# Single-node k3s and NVIDIA GPU pilot: setup and handoff +# Current k3s setup runbook -This runbook records the configuration observed on **2026-09-28**, a repeatable -setup and validation procedure, and the failure encountered during setup. Run -commands from the Mist repository root unless noted. The -[Kubernetes execution pilot](../../docs/kubernetes-pilot.md) describes the -implemented Kubernetes executor and remaining product work. +The maintained two-node runbook is [k3s setup and accelerator handoff](../../docs/k3s-setup.md). +It covers current versions, LAN/Tailscale roles, fresh server/worker boundaries, +NVIDIA/TT operators, diagnostics and troubleshooting. -## What the pieces do - -The machine already had two NVIDIA GPUs and a working host driver. A **single -k3s server** was installed on it. A server includes a local kubelet and -container runtime and can run Jobs without a separate `k3s-agent` service. -The NVIDIA GPU Operator added the Kubernetes integration: - -1. Its **device plugin** registers `nvidia.com/gpu` with the kubelet, allowing - the scheduler to reserve a GPU for a Pod that requests one. -2. Its **container toolkit** prepares CDI device information that containerd - uses to expose the assigned GPU and host driver libraries to a workload. - The enabled NRI plugin handles GPU management containers and avoids - rewriting k3s's containerd runtime configuration. -3. A CUDA sample Job requests `nvidia.com/gpu: 1` and exercises the device. - -The operator was configured **not** to replace the existing host driver. Those -values are in -[`nvidia-gpu-operator-values.yaml`](nvidia-gpu-operator-values.yaml). NVIDIA -states that its NRI plugin is not yet generally available; this is a pilot -configuration. See the [GPU Operator installation guide](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) -and [CDI/NRI guide](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/cdi.html). - -## Observed state on 2026-09-28 - -| Item | Observed value | -| --- | --- | -| Node | `utmist-z1opa08`, one control-plane and worker node | -| OS | Ubuntu 22.04.5 LTS | -| k3s | `v1.36.4+k3s1`; `k3s.service` enabled and active | -| Container runtime | k3s containerd `2.3.4-k3s1.36` | -| GPUs | Two NVIDIA RTX A4000 cards | -| Host driver | NVIDIA `590.48.01`, already present before GPU Operator deployment | -| GPU Operator | Helm release `gpu-operator`, chart `v26.7.1`, namespace `gpu-operator` | -| Test namespace | `mist` | -| Node resource | Capacity and allocatable: `nvidia.com/gpu: 2` | - -The node registered at 19:23 EDT. GPU Operator resources appeared around 19:39 -EDT; Helm release revision 2 was deployed at 19:45 EDT. The `gpu-smoke` Job -completed at about 19:44 EDT. After the service fix, a new `gpu-confirm` Job -completed at 19:54 EDT. Both sample logs contained `Test PASSED`. Each Job -requested **one** GPU. The node advertised two, but these tests did not -independently identify or exercise each physical card. - -These times and versions come from systemd, Kubernetes object timestamps, -`helm list`, `nvidia-smi`, and the completed Job logs. The exact historical -k3s installation command was not retained, so the commands below are a -reproduction recipe rather than a transcript. - -## Reproduce or verify this pilot - -Check versions against the target host before rebuilding or upgrading. Have a -working NVIDIA driver and `helm` available. Never commit the kubeconfig, -cluster token, or private keys to this repository. - -### 1. Confirm the GPUs work on the host - -```bash -nvidia-smi -L -nvidia-smi --query-gpu=name,driver_version --format=csv,noheader -``` - -On this node the first command lists two RTX A4000 cards. Resolve host driver -or hardware failures before diagnosing Kubernetes GPU allocation. - -### 2. Run one k3s server - -For a fresh **single-node** install, pin the version used in this pilot: - -```bash -curl -sfL https://get.k3s.io | sudo env INSTALL_K3S_VERSION='v1.36.4+k3s1' sh -s - server -sudo systemctl is-active k3s -sudo k3s kubectl get nodes -o wide -``` - -Expect `active` and a `Ready` node. The installer writes an administrative -kubeconfig at `/etc/rancher/k3s/k3s.yaml`; `sudo k3s kubectl` uses it without -copying credentials. Any copied kubeconfig must be kept private. The -[k3s quick-start guide](https://docs.k3s.io/quick-start) explains that a -single server can host workloads and that agents join additional machines. - -This same host must **not** run a separate `k3s-agent` service: - -```bash -systemctl is-active k3s-agent -systemctl is-enabled k3s-agent -``` - -Expected outputs here are `inactive` and `disabled`. These commands can -return nonzero for those expected states. - -### 3. Install the GPU Operator - -The tracked Helm values disable Operator driver installation and enable its -NRI plugin for GPU management containers. Keep the chart version pinned when -reproducing this setup: - -```bash -sudo env KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm upgrade --install gpu-operator \ - oci://nvcr.io/nvidia/cloud-native-charts/gpu-operator \ - --version v26.7.1 --namespace gpu-operator --create-namespace \ - --values deploy/k3s/nvidia-gpu-operator-values.yaml --wait - -sudo k3s kubectl -n gpu-operator get pods -sudo k3s kubectl describe node utmist-z1opa08 -``` - -The device plugin and container toolkit Pods should be Running. In the node's -`Capacity` and `Allocatable` sections, expect `nvidia.com/gpu: 2` for this -hardware. A completed validator Pod is normal. The operator may need a little -time to register devices after k3s starts; check again after its Pods are -healthy. - -### 4. Execute CUDA through k3s - -The tracked [`gpu-smoke-job.yaml`](gpu-smoke-job.yaml) requests and limits one -`nvidia.com/gpu` device. Create the namespace if it does not already exist: - -```bash -sudo k3s kubectl create namespace mist -sudo k3s kubectl apply -f deploy/k3s/gpu-smoke-job.yaml -sudo k3s kubectl -n mist wait --for=condition=complete job/gpu-smoke --timeout=180s -sudo k3s kubectl -n mist describe job gpu-smoke -sudo k3s kubectl -n mist logs job/gpu-smoke -``` - -Expect `1 Succeeded`, `nvidia.com/gpu: 1` in the Job's requests and limits, -and `Test PASSED` in the logs. This establishes that k3s scheduled a GPU Pod, -the runtime exposed a device inside the container, and CUDA executed. Merely -seeing the host GPUs or the Kubernetes resource count is a weaker check. - -A completed Job does not rerun on `kubectl apply`. To repeat this test, delete -**only the completed smoke Job**, then apply the manifest again: - -```bash -sudo k3s kubectl -n mist delete job gpu-smoke -sudo k3s kubectl apply -f deploy/k3s/gpu-smoke-job.yaml -sudo k3s kubectl -n mist wait --for=condition=complete job/gpu-smoke --timeout=180s -sudo k3s kubectl -n mist logs job/gpu-smoke -``` - -Save logs first if they are needed for an incident review. This batch Job -does not need a Service or NodePort. - -## Failure encountered: duplicate agent on port 6444 - -Both `k3s.service` and `k3s-agent.service` had been enabled on the **same -host**. The agent listened on `127.0.0.1:6444`, which the server's API process -also needed. The server repeatedly exited with: - -```text -failed to listen on 127.0.0.1:6444: bind: address already in use -``` - -During the conflict, `kubectl` could not connect to `127.0.0.1:6443`. -`nvidia-smi` still saw the GPUs, but new Kubernetes workloads could not use -them because the control plane was unavailable. Agent logs also referenced -`https://myserver:6443`, an example server name rather than this node's -working endpoint. The fix on 2026-09-28 was: - -```bash -sudo systemctl stop k3s-agent.service -sudo systemctl disable k3s-agent.service -sudo systemctl is-active k3s -sudo k3s kubectl get nodes -``` - -The server recovered without replacing the host driver or reinstalling the -operator. If it does not recover, inspect logs before restarting it, -especially when other workloads are running: - -```bash -systemctl status k3s k3s-agent --no-pager -journalctl -u k3s -u k3s-agent -n 100 --no-pager -ss -ltn '( sport = :6443 or sport = :6444 )' -``` - -## Troubleshooting and limits - -- **API connection refused:** Check `k3s.service` and its journal first. On - this node, confirm the duplicate agent has not been enabled again. -- **Node Ready but no `nvidia.com/gpu`:** Check `nvidia-smi -L`, the device - plugin and toolkit Pods, and - `sudo k3s kubectl -n gpu-operator logs daemonset/nvidia-device-plugin-daemonset --tail=100`. - Compare live Helm values with the tracked file if the release has changed. -- **GPU Job Pending:** Use `sudo k3s kubectl -n mist describe pod ` - to inspect scheduler events and requested resources. Two GPUs are the - observed capacity, not a promise that both are free at every moment. -- **Pod starts but CUDA fails:** Inspect its logs, host `nvidia-smi`, toolkit - Pod logs, and device plugin logs. An advertised resource alone does not - prove CUDA works. - -The smoke test proves **one requested GPU worked through k3s on the test -date**. It does not prove independent operation of both cards, GPU sharing, -performance, reboot durability, multi-node behavior, or Mist API integration. -The subsequent 2026-10-03 allocation/training and Mist API checks are recorded -in [the current deployment guide](README.md#mist-api-cli-and-jobs-page). - -## QuietBox worker joined on 2026-10-03 - -`utmist-tt` joined the existing `utmist-z1opa08` cluster as an agent. Both -machines are on the `10.0.0.0/24` LAN as well as the same Tailscale network; -the cluster uses their LAN addresses and the default Flannel VXLAN backend. -The server was not changed to advertise its Tailscale address. - -| Node | Role | Kubernetes InternalIP | k3s version | -| --- | --- | --- | --- | -| `utmist-z1opa08` | server and NVIDIA worker | `10.0.0.175` | `v1.36.4+k3s1` | -| `utmist-tt` | Tenstorrent host and k3s agent | `10.0.0.112` | `v1.36.4+k3s1` | - -QuietBox previously ran an independent `v1.36.5+k3s1` server with only a test -Nginx workload, standard system pods, and no PVCs or user Jobs. Its server -state was backed up to the root-only -`/root/k3s-server-prejoin-2026-10-03.tar.gz` on QuietBox before uninstalling -that server. The agent is enabled at boot; no `k3s.service` remains there. - -The agent's `/etc/rancher/k3s/config.yaml` points to -`https://10.0.0.175:6443`, sets `node-ip: 10.0.0.112`, uses `enp201s0` for -Flannel, and reads its persistent, root-only bootstrap token from -`/etc/rancher/k3s/agent-token`. Do not commit or print that token. The -installation token expires after one hour; the persistent token was installed -and verified by restarting `k3s-agent`. - -QuietBox has UFW active with default deny for incoming and routed traffic. -The join added inbound UDP `8472` and TCP `10250` from `10.0.0.175` on -`enp201s0`, plus routed traffic between `10.42.1.0/24` on `cni0` and -`10.42.0.0/24` on `flannel.1` in both directions. The NVIDIA server's UFW is -inactive. A temporary BusyBox pod placed on QuietBox resolved -`kubernetes.default.svc.cluster.local` and fetched the Nginx Service whose -pod ran on the NVIDIA node; its log ended with `CROSS_NODE_OK`. The test pod -was then removed. - -QuietBox exposes four `/dev/tenstorrent` devices and has the `tenstorrent` -kernel module loaded. TT-Operator 0.3.0 now runs Fabric Manager 0.2.30 and -DRA driver 0.0.59 with host driver management disabled. It advertises four -n300 board reservations, each covering two chips, through the -`tenstorrent.com` DeviceClass and ResourceSlices. Host KMD and firmware are -retained. See [installation, allocation, and live tests](README.md#tenstorrent-allocation). - -Four nonprivileged training jobs passed concurrently, with device cgroups -denying access to the other jobs' boards. A fifth job queued and then -trained after automatic board release. The `mist` admission policy blocks -the former privileged whole-machine setup from bypassing reservations. -Distributed training and untrusted-tenant security remain separate work. - -To verify the current cluster from the server: - -```bash -sudo k3s kubectl get nodes -o wide -sudo k3s kubectl get pods -A -o wide -sudo k3s kubectl describe node utmist-tt -``` - -The administrative kubeconfig copy at `/home/utmist/.kube/config` is owned by -`utmist` and mode `0600` as of this join. - - -## Mist application integration on 2026-10-03 - -The deployed API now creates real CPU, NVIDIA and Tenstorrent Jobs. CLI and -browser checks exercised submission, actual status/logs, cancellation, and -TT board-count selection. Live acceptance checks covered distinct concurrent -reservations, queuing and reuse, whole-device requests, exit-17 failure, -deadline failure, persisted checkpoints, and cancellation freeing all boards. -The API uses a dedicated namespaced service account, not the administrative -kubeconfig used by local operators. See [API installation and live evidence](README.md#mist-api-cli-and-jobs-page). - -The user's `mist-api-forward` and `mist-web` systemd units expose only -`127.0.0.1:3000` and `127.0.0.1:3001`. These user services require a user -systemd session. Current authentication, credits, team quotas, shared storage -and large/distributed model support remain outside the validated pilot. +- [Tailscale and administrator SSH](../../docs/tailscale-ssh.md) +- [Application deployment and service operations](../private/README.md) +- [Original single-node installation and worker-join history](../../docs/archive/k3s-setup-history.md) diff --git a/deploy/private/README.md b/deploy/private/README.md index e25c6b2..24fd36a 100644 --- a/deploy/private/README.md +++ b/deploy/private/README.md @@ -2,6 +2,8 @@ Current user/admin and architecture guide: [department rollout](../../docs/department-rollout.md). Execution/evidence: [rollout checklist](../../docs/department-rollout-execution.md). +Setup: [k3s](../../docs/k3s-setup.md), [Tailscale/SSH](../../docs/tailscale-ssh.md), +[storage](../../docs/storage.md), and [developer setup](../../docs/development.md). ## Services and persistent state @@ -22,6 +24,14 @@ Never scale the current API above one replica: admission requires one controller ## Build and redeploy on main +Run from a checkout of the reviewed current main on the NVIDIA host. Required: +Go 1.25.1, Node 22.16+, working Docker access, the private kubeconfig, existing +auth Secrets, GPU/TT operators and mounted QuietBox storage. The script is for +this existing installation; it is not a fresh-cluster/bootstrap installer. +Checking out/merging Git does not restart or deploy services automatically. +The API is Kubernetes-only; Redis and the retired Docker supervisor are unnecessary. + + ```bash bash deploy/private/build-and-deploy.sh ``` @@ -54,7 +64,9 @@ rollback can repoint `/srv/mist-web/current` atomically to a previous complete release. Keep its API contract compatible. For a backend regression, rebuild a known department version with the same manifests/Secrets/state. The earlier account-only API does not enforce team policies and is not a safe ordinary -rollback for research users. +rollback for research users. The pre-cleanup department release `3431ffd` contains +the verified team architecture; rebuild a compatible department version rather +than restoring the archived pre-platform main for normal users. ## QuietBox host setup @@ -123,6 +135,28 @@ device. LAN rules and cloud Tailscale ACL/sharing settings are unchanged. synthetic untrusted source and removes its temporary namespace/rules afterward. It requires root in the main host network namespace. +## Services to restart and inspect + +Use a root-capable administrator session for service mutations. Logs are available +through `journalctl -u ` on the owning host and `kubectl logs` for pods. +Ordinary application redeployment should use the build script; restart k3s/NFS +only when resolving their own infrastructure problem because workloads depend on them. + +| Component | Owner | Status command | +|---|---|---| +| Portal | Main | `systemctl status nginx --no-pager` | +| API/controller | Cluster | `k3s kubectl -n mist-system rollout status deployment/mist-api` | +| Auth | Cluster | `k3s kubectl -n mist-system rollout status deployment/mist-auth` | +| Control plane | Main | `systemctl status k3s --no-pager` | +| Worker | QuietBox | `systemctl status k3s-agent --no-pager` | +| Shared storage | QuietBox | `systemctl status nfs-server mist-team-storage --no-pager` | +| Private network | Both | `systemctl status tailscaled --no-pager` | +| Portal restrictions | Both | `systemctl status mist-portal-firewall --no-pager` | + +The `mist-api-forward` and `mist-web` user services are local development helpers, +not requirements for the production portal. The default-namespace test Nginx and +k3s's Traefik also do not serve the current portal; it uses host Nginx on 8088. + ## Check after startup Main: diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..17a1067 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,42 @@ +# Documentation + +Current main-branch documentation, verified against the two running machines on +**October 9, 2026**. Addresses and versions describe this installation; adjust +host-specific settings when provisioning another environment. + +## Find the right guide + +| Task | Guide | +|---|---| +| Sign in, manage teams, upload data, submit/download jobs | [Department operating guide](department-rollout.md) | +| Understand services and source layout | [Architecture](architecture.md) | +| Install/inspect k3s, join a worker, verify accelerator allocation | [k3s setup](k3s-setup.md) | +| Connect privately and administer either host | [Tailscale and SSH](tailscale-ssh.md) | +| Understand dataset/model pools and physical capacity | [Storage](storage.md) | +| Build, deploy, restart or troubleshoot services | [Deployment operations](../deploy/private/README.md) | +| Develop locally with the existing authenticated API | [Developer setup](development.md) | +| Authenticate and submit from a terminal | [CLI](../cli/docs/setup.md) | +| Run automated checks and read recorded live evidence | [Testing](testing.md) | +| Maintain UI components and interactions | [Design system](design-system.md) | +| Read completed scope, exclusions and release verification | [Execution record](department-rollout-execution.md) | +| See what repository cleanup removed and verified | [Cleanup record](repository-cleanup.md) | + +## Access and execution + +```text +Browser / CLI → Tailscale portal :8088 → Nginx → Go API → k3s + | | + Better Auth + team policy + | + CPU/NVIDIA or Tenstorrent Job + | + scoped dataset/model NFS mounts on QuietBox +``` + +Mist login is separate from Tailscale membership and Linux SSH login. Researchers +use the portal; administrator SSH is restricted by the installed host rules. +The worker reaches the control plane and storage through LAN addresses. + +[Earlier plans and single-owner guides](archive/README.md) are historical only. +Current guidance does not require Redis, an SSH session to launch a job, a public +cloud load balancer, or permission to approve every tagged image. diff --git a/docs/architecture.md b/docs/architecture.md index 7cd8e0b..62ab764 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,66 +1,74 @@ # Architecture -Current private department pilot, October 9, 2026. See -[department rollout](department-rollout.md) for permissions, resource/storage -rules, the API contract and operating limits. +Current private department installation, verified October 9, 2026. +[User/admin policy](department-rollout.md) and [k3s details](k3s-setup.md) cover +its operating contracts. ```mermaid flowchart LR - Web[Browser / CLI] --> Portal[Nginx private Tailscale portal] - Portal --> API[Go API + single admission controller] - API --> Auth[Better Auth + SQLite PVC] - API --> State[Team policies + durable queued Jobs] + Client[Browser or authenticated CLI] --> Portal[Nginx private Tailscale portal :8088] + Portal --> API[Go API and single admission controller] + API --> Auth[Better Auth and local SQLite PVC] + API --> State[Team policy ConfigMaps and suspended Jobs] State --> K8s[k3s scheduler and device operators] - K8s --> Main[CPU / NVIDIA main node] - K8s --> TT[QuietBox Tenstorrent node] - Main --> Store[Scoped NFS team filesystems on QuietBox] + K8s --> Main[CPU and NVIDIA node] + K8s --> TT[QuietBox Tenstorrent worker] + Main --> Store[Scoped NFS dataset and model pools on QuietBox] TT --> Store ``` -The API authenticates and applies team policy before persisting a suspended Job. -Its single controller admits work fairly; Kubernetes/containerd place and run the -containers using NVIDIA device resources or TT DRA claims. The shared store has -one bounded filesystem per team, read-only input mounts and job-specific outputs. -SSH is for machine administration, not job execution. The current pilot has no -separate Redis scheduler, cloud load balancer or new monitoring product. +## Job flow -
-Historical architecture sketch — July 6, 2025 +1. Nginx serves the built SPA and forwards `/api` and `/auth` to Go. +2. Better Auth verifies the account. Go checks current team membership, image + registry rules, storage scope and resource/runtime/queue policy. +3. The API persists a suspended Job in the team's namespace and prepares its + scoped output directory. A single controller admits queued jobs fairly. +4. Kubernetes places the admitted pod on the appropriate node. NVIDIA's plugin + reserves whole GPUs; Tenstorrent DRA reserves whole n300 boards. +5. containerd starts the image. `/inputs` is read-only; `/outputs` is the job's + own directory in the model pool. Logs/status come from actual pods and Jobs. +6. Jobs and output files remain after exit. Cancellation or revocation stops + pods and frees reservations while retaining history and outputs. -```mermaid ---- -config: - layout: dagre ---- -flowchart LR - subgraph Ingress_Layer["Ingress Layer"] - APIGW["API Gateway"] - end - subgraph Control_Plane["Control Plane"] - Auth["Auth & Authorization Service"] - Queue["Job Queue"] - Scheduler["Scheduler Service"] - StateDB["State & Metadata"] - end - subgraph Compute_Plane["Compute Plane"] - Balance["Load Balancer"] - server1["Server #1"] - server2["Server #2"] - end - subgraph Observability["Observability"] - Metrics["Metrics"] - Logging["Logging"] - Alerts["Alerts"] - end - APIGW --> Auth - Auth --> Queue - Queue --> Scheduler - Scheduler --> Balance - Balance --> server1 & server2 - server1 --> APIGW & Observability - server2 --> APIGW & Observability -``` +Nginx currently proxies to one API service; the API must remain one replica +because it owns a single admission controller. Kubernetes placement uses +resource requests and device availability. It does not automatically turn one +submitted model into distributed training across both accelerator types. + +## Source organization + +| Location | Responsibility | +|---|---| +| `src/main.go`, `app.go` | Process lifecycle, HTTP routes and controller ownership | +| `src/jobs_http.go`, `http.go`, `job_types.go` | Job endpoints, shared responses and wire types | +| `src/kubernetes.go`, `job_states.go`, `outputs.go` | Container specification, execution/state and retained local-output initialization | +| `src/member_auth.go` | Session proxy, identity and authoritative authorization | +| `src/teams.go`, `team_queue.go`, `team_runtime.go`, `team_storage.go` | Membership/policy, fair admission, isolation and scoped storage | +| `src/shared_storage.go`, `hardware.go`, `images.go` | Uploads/results, inventory and image/runtime profiles | +| `auth-service/` | Better Auth and local SQLite persistence | +| `cli/` | Authenticated terminal client; no execution logic | +| `web-interface/` | React portal, typed API client, account/team state and UI components | +| `deploy/k3s/` | Manifests, accelerator diagnostics, live verifiers and evidence | +| `deploy/k3s/storage/` | QuietBox provisioning/migration/growth and service unit | +| `deploy/private/` | Image/web deployment, Nginx and host access rules | +| `docs/` | Current guides; historical records are under `docs/archive/` | + +API and CLI are separate Go modules joined by `go.work`. There is no Redis +scheduler or Docker supervisor in the current code. Historical account jobs and +local PVCs remain readable through explicit legacy-workspace compatibility. +Removing the old executor does not delete these Kubernetes resources. + +## State and boundaries -Architecture as of 7/6/2025 +- Better Auth uses a main-node local SQLite WAL PVC, not NFS. +- Team policies and queue state live in Kubernetes; no separate queue database. +- Every team has separate bounded dataset and model/result NFS filesystems. + Member/common folders share their corresponding team's pool. +- Workloads receive scoped mounts, no Kubernetes token, constrained host paths + and a network policy. Accelerator drivers still share the host kernel. +- Portal/SSH use Tailscale. Cluster API, Flannel and NFS use the LAN. -
+There is one control plane, one auth instance and one storage host. Availability +and arbitrary-model/distributed-training support are limited as described in the +[operating guide](department-rollout.md). diff --git a/docs/archive/README.md b/docs/archive/README.md new file mode 100644 index 0000000..8f98a75 --- /dev/null +++ b/docs/archive/README.md @@ -0,0 +1,21 @@ +# Historical records + +These files preserve earlier design decisions and verification phases. They are +not setup or operating instructions for the current main branch. Some commands +refer to removed prototypes; their original source remains in Git history. + +| Record | Phase | +|---|---| +| [Foundation execution plan](foundation-execution-plan.md) | Initial work plan and progress | +| [Shared storage/deployment plan](shared-storage-deployment-plan.md) | Original parts 4 and 5 | +| [Kubernetes pilot](kubernetes-pilot.md) | October 3 executor snapshot | +| [Local job foundation](local-job-foundation.md) | October 8 single-owner platform | +| [Complete foundation](complete-foundation.md) | Account-only platform before teams | +| [k3s setup history](k3s-setup-history.md) | Original NVIDIA installation and QuietBox join | +| [k3s deployment history](k3s-deployment-history.md) | Earlier accelerator/API acceptance | + +The old main is preserved on +[`archive/main-before-k3s-20261009`](https://github.com/UTMIST/Mist/tree/archive/main-before-k3s-20261009). +The pre-cleanup department release is commit +[`3431ffd`](https://github.com/UTMIST/Mist/tree/3431ffdb713fac72881fc7f61f2750a82fe3f916). +Use the [current documentation](../README.md) for this installation. diff --git a/docs/archive/complete-foundation.md b/docs/archive/complete-foundation.md new file mode 100644 index 0000000..f1d8875 --- /dev/null +++ b/docs/archive/complete-foundation.md @@ -0,0 +1,227 @@ +> **Historical record — superseded.** This text describes an earlier phase. +> Commands and claims below are retained as history, not current setup instructions. +> Start with the [current documentation index](../README.md) and +> [department operating guide](../department-rollout.md). + +> Current department storage and portal settings: [department rollout](../department-rollout.md). +> This document records the earlier foundation; its 100 GiB pilot was later expanded sparsely. + +# Mist foundation: historical operating snapshot + +> **Historical account-only snapshot.** The [department rollout](../department-rollout.md) +> supersedes these instructions: team membership, scoped folders, enforced quotas, +> self-service images and the new portal are implemented. Use that guide and +> [current deployment operations](../../deploy/private/README.md) for current behavior. +> The remaining sections record the preceding foundation, including its limitations. + +Implemented on `feat/local-job-foundation`, October 8–9, 2026. This continues the +[original foundation](local-job-foundation.md) with shared datasets, output +retrieval, separate member accounts and persistent private hosting. + +## Use the website + +Open **http://100.73.139.66:8088** from a device on the existing Tailscale network. +The alternative MagicDNS address is http://utmist.tail459b5e.ts.net:8088. + +1. Sign in with a Mist account, independent of the machine's Linux/SSH login. +2. Open **Datasets**, upload one file or a ZIP, and wait for **Ready**. +3. Open **Jobs**, choose CPU, NVIDIA GPUs or Tenstorrent boards, an approved + image, and optionally the uploaded dataset. Submit a script or container. +4. Follow the actual status, placement and logs. Save files under `/outputs`. +5. Click **Files** on the job to download results. Results remain after exit. + +The initial administrator's credentials are stored locally in +`/home/utmist/.config/mist/admin-bootstrap.json` (mode 0600), outside Git. View +that file locally, sign in, then change the password on **Account**. The file +contains a bootstrap password, not a password that automatically tracks changes. +Do not distribute its auth secret. Administrators add members on **Account**; +members can change passwords there. Deactivation revokes sessions. Public signup +is disabled. There is no email delivery/password-reset email service configured; +an administrator can reset a member password through Better Auth's authenticated +`/auth/admin/set-user-password` endpoint. Admin roles are managed by that service, +never by editing one's profile. + +**HTTPS status:** Tailscale Serve is disabled on this tailnet. Its CLI reports +that an account administrator must enable Serve in the Tailscale console. The +current HTTP endpoint is bound to localhost and the Tailscale IP only. Remote +traffic uses Tailscale's encrypted network, but browsers do not treat HTTP as a +secure origin. No public Funnel/domain was enabled. After enabling Serve, run +`tailscale serve --bg --https=443 http://127.0.0.1:8088`, change +`MIST_SITE_ORIGIN` in the auth deployment to `https://utmist.tail459b5e.ts.net`, +and restart auth so cookies use Secure. The HTTPS origin is already in the +trusted-origin lists. Check Serve status and browser login before distributing +that URL. Do not claim HTTPS is available before that verification. + +## Running architecture + +```text +Team browser / CLI / either SSH host + │ private Tailscale network + ▼ +Main node Nginx :8088 ── static production React build + │ /api/* and /auth/* + ▼ +Go Mist API, Kubernetes Deployment (mist-system) + ├── Better Auth service ── local SQLite PVC on main node + ├── dataset metadata/uploads/results ── shared NFS PVC + └── Kubernetes Jobs (mist namespace) + ├── Main node: CPU or 1–2 NVIDIA GPUs + └── QuietBox: 1–4 TT boards, two chips per board + │ + selected /inputs (read-only), own /outputs (writable) + │ + QuietBox /srv/mist-storage (NFSv4 over current LAN) +``` + +Nginx proxies directly to the existing API ClusterIP `10.43.1.116:3000`; +production does not rely on Vite, SSH sessions or the development port-forward. +The local preview at `127.0.0.1:3001` remains available and now uses real login. +Kubernetes schedules jobs using the requested resources and node profiles. +TT allocations use DRA boards. NVIDIA allocations use whole GPUs. Neither device +sharing within a GPU/board nor distributed mixed-backend training is implemented. + +## Storage layout and limits + +QuietBox exports `/srv/mist-storage` to only LAN IPs `10.0.0.175` and +`10.0.0.112`, using NFSv4 and root squash (anonymous UID/GID 65532). UFW admits +TCP 2049 from those two addresses. Its router Ethernet connection stays intact. +The main machine continues using its existing Wi-Fi connection. + +A dedicated **100 GiB sparse ext4 image** on QuietBox's existing NVMe backs the +store. No physical disk was formatted. Actual usable capacity is slightly smaller +because of filesystem overhead. `/etc/fstab` mounts it at boot; `nfs-server` +is enabled. Keep the NVMe's free space above this pilot budget. + +```text +/srv/mist-storage/ + metadata/datasets/.json owner, checksum, size, ready state + .uploads// staging, removed on failure/completion + datasets//content/ selected input files, read-only job mount + jobs//outputs/ job's own persistent results + legacy/{cpu,nvidia,tenstorrent}// copied original results +``` + +The upload limit is 2 GiB per file/archive and 2 GiB expanded. ZIPs reject unsafe +paths, symlinks, special files, duplicate files and more than 10,000 entries. +Uploads stream to staging, are checksummed and published atomically. A failed +upload never becomes Ready. Metadata, unrelated jobs and other datasets are not +mounted into a workload. No archive is executed by the upload service. + +`/inputs` and `MIST_INPUT_DIR` identify an attached dataset. `/outputs` and +`MIST_OUTPUT_DIR` identify persistent outputs; `/checkpoints/` remains an +alias. Jobs without a selected dataset have no `/inputs` mount. Image entrypoints +and working directories remain unchanged unless explicitly overridden. + +The budget applies to all shared files together, not separately to each PVC. +Two static RWX PV/PVCs expose the same backing store to different namespaces. +A PVC request alone is not a hard quota; the image's filesystem enforces capacity. +Upload admission leaves 1 GiB spare, but running jobs can still fill storage: +write errors must be handled by job code. Per-member/job disk quotas are future +work. Do not store SQLite on NFS. + +Original local checkpoint PVCs remain intact. Their completed outputs were +copied to `legacy/` and remain downloadable through retained job IDs. New jobs +use shared storage. QuietBox is now a storage dependency: if it is offline, mounts, +uploads, downloads and jobs that need those files may wait. NFS is not a backup. + +### Cleanup and retention + +Nothing deletes datasets or results automatically. Delete unneeded datasets +through the website; active-job references prevent deletion. For job results, +first verify the owning job has finished and no pod can write to it, then archive +and remove only that job's directory from `jobs//outputs` on QuietBox. +Keep the parent directory for retained Kubernetes Job history. Do not delete +storage metadata, the mounted image file, or entire legacy PVC directories. +Monitor `df -h /srv/mist-storage` on QuietBox. Remove only test output archives +that have already been saved elsewhere. Legacy copies use additional space; +original PVCs are intentionally not removed by this migration. + +## Authentication and ownership + +Better Auth 1.7.7 handles passwords, sessions, database migrations, role checks, +rate limiting and revocation. Go verifies each session with the auth service; +unavailable auth returns 503, anonymous access 401. Browser mutations require +trusted origins. Cookies are HttpOnly and SameSite=Lax; Secure requires the HTTPS +configuration described above. Session duration is seven days. + +New members receive stable owner labels derived from their auth user ID. Existing +`utmist` jobs are mapped to the initial administrator's configured email. Other +members cannot list, inspect, cancel, read logs from or download that owner's +jobs, and cannot use/delete its datasets. Hardware totals are visible to members +because those devices are shared. Administrators' membership permissions do not +automatically give access to another member's job files. + +This is a private, trusted team pilot. Approved containers, TT's read-only host +runtime mounts and NFS Unix permissions do not constitute a hardened hostile +multi-tenant sandbox. Public hosting, hostile code isolation, member quotas, +credits, priority/fair queues, Jupyter and job chaining require later work. + +## CLI + +```bash +export MIST_API_URL=http://100.73.139.66:8088/api +bin/mist-cli auth login --email your-member-email@example.org +bin/mist-cli job list --all +bin/mist-cli job submit train.py --compute NVIDIA --devices 1 +bin/mist-cli auth logout +``` + +Login prompts without echoing the password; `--password-stdin` supports controlled +automation. Cookies are saved atomically in a mode-0600 CLI config. Logout revokes +the session. Fake legacy tokens are no longer used. An alternate config can be +selected with `--config`; never paste a session cookie into a public document. + +## API additions + +All data routes require an authenticated cookie when `MIST_AUTH_URL` is configured. + +| Route | Function | +|---|---| +| `GET /session` | Auth enabled flag and current member; no session token | +| `/auth/*` | Same-origin proxy to maintained Better Auth endpoints | +| `GET /storage` | Actual capacity/free bytes and upload limit | +| `GET /datasets` | Current owner's ready datasets | +| `POST /datasets?filename=data.csv&name=Example` | Raw streaming file upload | +| `POST /datasets?filename=data.zip&format=zip` | Checked ZIP upload/extraction | +| `GET /datasets/` | Own ready dataset metadata | +| `DELETE /datasets/` | Delete own dataset if no active job references it | +| `POST /jobs` with `dataset_id` | Mount own ready dataset read-only | +| `GET /jobs//files` | Own job's regular result files | +| `GET /jobs//files/download?path=model.pt` | Stream attachment/download | + +The website prefixes Go routes with `/api`; auth stays at `/auth`. Listing limits +outputs to 5,000 files/10,000 entries/20 directory levels. Download paths use Go's +`os.Root` to prevent traversal and escaping symlinks. Save plain regular files +inside `/outputs` if you want them to be listed/downloaded. + +## Deployment, restart and rollback + +See [deployment scripts](../../deploy/private/README.md). k3s server/agent, NFS and +Nginx are system services. Kubernetes restores API/auth deployments. The frontend +is a static release, with `/srv/mist-web/current` pointing at the active build. +Deployment configuration, RBAC, claim templates and image policies are checked +in; bootstrap credentials and cluster tokens are not. + +Rollback frontend by repointing `current` to an earlier retained release and +reloading Nginx. Rollback an API image with `kubectl set image` after importing +that image. Keep auth, shared storage and login enabled during rollback; the old +unauthenticated foundation API is not a suitable private member-service rollback. +Database migrations require a pre-upgrade backup and version-specific review. + +## Verification + +Repeatable checks: + +- `deploy/k3s/verify_complete_browser.mjs`: actual login, dataset upload, CPU, + two NVIDIA GPUs and a TT board consuming the same uploaded dataset, downloads, + member creation/password change/isolation/deactivation, denied anonymous signup + and cross-origin writes. Uses protected local credentials; logs no secrets. +- `deploy/k3s/verify_quietbox_submission.py`: authenticated submission from + QuietBox, NVIDIA training assigned to the main machine, downloaded loss metrics. +- `src/shared_storage_test.go`: upload limits/atomic publication, ZIP attacks, + read-only dataset binding, escaping download symlinks, ownership and auth outage. +- Existing job/device/queue/cancellation/image tests remain in the repository. + +Detailed evidence is under `/home/utmist/mist-complete-results-2026-10-09`; +final verified results are recorded in `deploy/k3s/complete-results.json`. An accelerator smoke test proves this small +training workload; it does not prove arbitrary model/runtime compatibility. diff --git a/docs/archive/foundation-execution-plan.md b/docs/archive/foundation-execution-plan.md new file mode 100644 index 0000000..aeceef5 --- /dev/null +++ b/docs/archive/foundation-execution-plan.md @@ -0,0 +1,390 @@ +> **Historical record — superseded.** This text describes an earlier phase. +> Commands and claims below are retained as history, not current setup instructions. +> Start with the [current documentation index](../README.md) and +> [department operating guide](../department-rollout.md). + +# Mist foundation execution plan + +Recorded: October 8, 2026. This file preserves the agreed scope and should be +updated with implementation progress, test evidence, and remaining work. + +## Goal and decisions + +Make the local website usable for running container jobs on the existing +NVIDIA and Tenstorrent machines. Start with the working Kubernetes pilot, +rather than rebuilding execution. The user has requested implementation on +a new branch, organized code, and proper documentation. + +Today's scope is tasks 1–3 plus persistent output folders and verification. +Shared dataset storage, uploads, and result downloads are the next foundation +task. Website deployment and login follow the foundation. + +Decisions already made: + +- Develop and test locally. Team members can use the CLI over SSH or open the + local website through an SSH tunnel. A public website is unnecessary now. +- Mist API creates Kubernetes Jobs; k3s schedules them; containerd runs the + images. Normal job execution does not use SSH. +- Images are built and pushed to a registry, then selected by reference. A + browser image archive uploader or image build service is outside today's scope. +- CPU and NVIDIA support approved custom images. Tenstorrent initially uses + its existing tested image and host runtime profile. +- NVIDIA allocations are whole GPUs. Tenstorrent allocations are whole n300 + boards, with two accelerator chips per board. +- Start with persistent node-local results folders. Add shared storage later. +- For shared storage, the preferred initial host is Tenstorrent because it has + more free disk space. NFS over the existing network is the simplest proposal. +- Wi-Fi is adequate for initial testing. A future second Ethernet cable can + provide a dedicated storage link. Preserve Tenstorrent's existing router link: + its k3s configuration and network access currently depend on it. +- Preserve existing work and Git history. Implement on a new branch. Retiring + or merging the existing main branch is a later repository decision. +- No permission prompts are needed for the agreed implementation and tests. + Do not introduce unrelated public exposure, account changes, or disk formatting. + +## Architecture + +```text +Local website / CLI + | + v + Mist API + | + v + k3s cluster + / \ + v v +NVIDIA PC Tenstorrent +containerd containerd + | | + v v +Job container + allocated accelerator + persistent results folder +``` + +The full whiteboard plan also includes membership/authentication, credits, +priority policy, a storage box, Jupyter, job chaining, and optional agents. +Kubernetes supplies placement and device allocation. Mist supplies admission, +user policy, UI, dataset handling, and artifact access. Selecting an accelerator +does not convert CPU code into CUDA or Tenstorrent code, or distribute one model +across machines automatically. + +## Current baseline + +Repo: `/home/utmist/Mist`. Before this implementation, the branch is +`fix/local-runtime-setup` and contains substantial uncommitted k3s pilot work. +Preserve these changes when creating the new branch. + +An independent main preview exists at `/home/utmist/Mist-main-preview`. +Use main's reusable UI/components/types where helpful. Its Jobs data and several +actions are prototypes, and its legacy Docker/Redis executor is incomplete. + +| Machine | Kubernetes node | Tailnet IP | Cluster LAN IP | Resources | +| --- | --- | --- | --- | --- | +| Main/NVIDIA | `utmist-z1opa08` | `100.73.139.66` | `10.0.0.175` | 2 NVIDIA RTX A4000 GPUs | +| QuietBox | `utmist-tt` | `100.95.175.37` | `10.0.0.112` | 4 n300 boards / 8 Wormhole chips | + +Both nodes report Ready with k3s `v1.36.4+k3s1`. Workload namespace is `mist`; +API namespace is `mist-system`. + +- API already runs inside k3s. Local access is `http://127.0.0.1:3000`. +- Working Jobs page: `http://127.0.0.1:3001/jobs`. +- Vite proxies `/api` to the API; this proxy does not ship in the static build. +- User services: `mist-api-forward.service`, `mist-web.service`. +- Main preview is separate: UI port 3011, API port 3010. +- Existing Jobs page supports scripts, accelerator counts, training checks, + real states, logs, and cancellation. +- API is a trusted local, single-owner pilot. Owner is configuration, not an + authenticated identity. Authentication and tenant isolation remain work. +- Existing checkpoints are in three node-local `local-path` PVCs. Their declared + request is 256 MiB each; this is not a meaningful dataset storage budget. +- Shared storage and dataset upload/download services have not been configured. + +Storage readings taken during planning: + +| Machine | Filesystem capacity | Used | Available | +| --- | --- | --- | --- | +| NVIDIA | 439 GiB | 87 GiB | 330 GiB | +| Tenstorrent NVMe | 3.6 TiB | 1.6 TiB | 1.9 TiB | + +NVIDIA also has an unmounted ~447 GiB `/dev/sdb`. Its contents are unknown; +do not treat it as empty or format it as part of this plan. + +NVIDIA currently reaches QuietBox over Wi-Fi via the router. QuietBox's active +router Ethernet link negotiates 1 Gbps. Both machines have unused 10 Gbps +Ethernet hardware; a future direct link requires compatible ports/cable and +verification. Being on the same tailnet or k3s cluster does not share disks. + +## Implementation order + +### A. Establish the branch and API contracts + +- [x] Create a new foundation branch from the current working checkout. +- [x] Preserve existing pilot changes and inspect main for useful components. +- [x] Separate inventory and submission validation from HTTP/UI presentation. +- [x] Document new response/request shapes before connecting UI controls. +- [x] Keep changes reviewable and avoid unrelated rewrites. + +### B. Task 2: hardware discovery and availability + +Proposed endpoint: `GET /hardware`. Publish the final contract with the code. + +- [x] Report node readiness, accelerator type, allocation unit, total, + allocated, and currently available counts. +- [x] Count NVIDIA allocations from scheduled, nonterminal Kubernetes pods, + including workloads outside Mist. Do not subtract every submitted job. +- [x] Count Tenstorrent devices from ResourceSlices and actual DRA claim + allocations. Account for reservations while a workload starts. +- [x] Label TT counts as boards and show two chips per n300 board. +- [x] Treat machine/inventory failures explicitly; do not present unknown + resources as confidently available. +- [x] Give the API only the additional read permissions required for inventory. +- [x] Keep k3s responsible for actual allocation. UI availability is a snapshot; + a submitted job may wait even if devices looked free moments earlier. + +TT's current vendor ResourceSlice memory capacity is incorrect. Do not use that +value to display accelerator memory or determine placement. The current system +has neither GPU fractions nor arbitrary TT chip sharing. + +### C. Task 3: container image execution + +- [x] Expose approved image choices and explicit image references, command, + arguments, and CPU/memory/deadline controls through typed API/UI code. +- [x] Keep image approval on the server. An approved image can still fail to + pull or be incompatible; report the actual error. +- [x] Preserve the image's working directory. The current executor overrides + every workload with `/tmp`, which can break `python train.py` in an image + whose code lives in `/app`. +- [x] Define command/argument behavior, including whether the image's default + ENTRYPOINT/CMD may run. Kubernetes command/args are arrays; do not silently + treat every string as a shell program. +- [x] Retain script and training-check modes with their tested runtime profiles. +- [x] Support approved custom CPU/NVIDIA images. Keep the tested TT image/profile + restriction visible instead of promising arbitrary TT image compatibility. +- [x] Configure/document image pull credentials where needed; do not ask users + to paste registry credentials into a job form. +- [x] Verify an actual custom image with packaged code, rather than testing + only scripts injected into existing base images. A local fixture image may + be imported into containerd for the initial test. +- [x] Document resource limits and deadlines. Current Job deadline includes + waiting time as well as execution; make any change to this behavior explicit. + +### D. Persistent output folders + +- [x] Provide a predictable, writable per-job directory backed by the existing + checkpoint PVC on the selected machine. +- [x] Create the directory reliably and verify permissions for the supported + image profiles. +- [x] Preserve/document `MIST_CHECKPOINT_DIR` and `checkpoint_directory` so + workloads know where to save their results. +- [x] Verify a real output file remains after the container exits. +- [x] Describe how to retrieve files during local testing and where they live. +- [x] Keep job identifiers/storage references independent of physical disk + paths so shared storage can be added later. + +Files written only into the container's writable layer are not persistent +results. The existing shared checkpoint roots are also not a complete tenant +isolation boundary; do not represent this pilot as ready for untrusted users. + +### E. Task 1: finish website job controls + +- [x] Connect the website to the new hardware and image contracts. +- [x] Keep real job submission, polling, logs, assigned node, failure reasons, + exit codes, and cancellation. +- [x] Show requested resources and the persistent output location. +- [x] Make waiting-for-resources, machine unavailable, and image pull failures + understandable. +- [x] Preserve connection recovery and protect against duplicate submissions. +- [x] Use existing UI conventions and accessible labels. Keep frontend concerns + separate from the executor and typed API client. + +The prototype Dashboard/Machines/auth pages are not proof of working metrics or +authentication. Surface real hardware information in the relevant workflow. + +### F. Verify and hand off + +- [x] Run focused backend tests for execution semantics, inventory accounting, + validation, and cancellation where changed. +- [x] Run frontend type checking, production build, and meaningful interaction + checks for new behavior. +- [x] Run a real custom CPU image job through the website. +- [x] Run fresh NVIDIA and TT training checks through the website. +- [x] Verify occupied/free counts during execution and after completion. +- [x] Verify waiting/reuse and cancellation release for both allocation paths. +- [x] Verify failed commands and image pulls produce useful states/errors. +- [x] Verify saved outputs persist after pod completion. +- [x] Update the deployed API image if necessary; editing Go source does not + update the binary currently running in k3s. +- [x] Record actual results and date them. Older October 3 acceptance evidence + does not substitute for tests of today's changes. +- [x] Document startup, API contracts, image preparation, output access, tests, + and remaining work for the team. + +Avoid unrelated operator upgrades or repeat full infrastructure acceptance tests +without a concrete regression risk. Small synthetic training checks demonstrate +device execution and learning, but each team's actual model needs compatibility +validation; distributed training and TT autograd are not established by these checks. + +## Today's definition of done + +```text +Local website + | +Choose approved image + command + hardware + device count + | +Mist API creates a k3s Job + | +Selected machine runs the container + | +Real states/logs + correct device accounting + persistent output file +``` + +Implementation should be verifiable from the browser, with failures and +cancellation behaving correctly and documentation sufficient for another +developer to resume the work. + +## Next foundation task: shared datasets and downloads + +This is task 4, after today's execution scope: + +1. Create a storage folder on QuietBox with a bounded disk budget. +2. Configure private NFS access for both machines and Kubernetes workloads. +3. Add streaming dataset uploads, progress/error handling, a stable dataset ID, + and a ready state before jobs may use an upload. +4. Attach only the selected dataset read-only and the job's outputs writable. +5. Add result listing/downloads with ownership checks and retention rules. +6. Test one uploaded dataset with jobs on both machines. + +Proposed physical layout, not yet created: + +```text +Tenstorrent: /srv/mist-storage/ +├── datasets/ +│ └── dataset-id/ +└── jobs/ + └── job-id/ + └── outputs/ +``` + +Container paths may become `/inputs` and `/outputs`; preserve compatibility +with the existing checkpoint contract or document the migration explicitly. +NVIDIA reads TT-hosted files over the current LAN, which includes Wi-Fi. +Caching large datasets on NVIDIA and a dedicated Ethernet link can follow. +Central storage on one machine needs backup/retention planning and is unavailable +when that host fails. Archives need safe extraction and upload limits. + +## Later: deployment and product features + +Task 5 follows the foundation: + +- Serve a production frontend and route `/api` to the existing Mist API. +- Provide a team-accessible Tailnet address/hostname and persistent services. +- Implement login, job/file ownership, and appropriate member limits before + supporting multiple users. +- Add public domain/HTTPS only if public access is requested. + +Credits, fairness/priority policies, Jupyter, chaining, agents, detailed monitoring, +portable TT runtime images, and distributed model training are separate workstreams. + +## Resume notes and useful files + +### Code + +- `src/kubernetes.go`: executor, validation, resource requests, job status/logs/cancel. +- `src/kubernetes_test.go`: existing Kubernetes executor tests. +- `src/api.go`: API mode selection and request types. +- `web-interface/src/api.ts`: typed API client. +- `web-interface/src/components/JobsPage.tsx`: working job UI. +- `web-interface/src/components/JobsPage.test.tsx`: existing UI checks. +- `deploy/k3s/mist-api.yaml`: API Deployment, RBAC, Service, and checkpoint PVCs. +- `deploy/k3s/mist-tenstorrent-claims.yaml`: 1–4 board claim templates. +- `deploy/k3s/tenstorrent-workload-policy.yaml`: accelerator workload admission policy. +- `deploy/k3s/training_smoke.py`: actual NVIDIA/TT regression training check. +- `deploy/k3s/verify_mist_api.py`: earlier API allocation/queue/training acceptance tool. + +### Existing documentation and evidence + +- `docs/kubernetes-pilot.md`, `deploy/k3s/README.md`, `src/jobs.md`. +- `deploy/k3s/mist-api-results.json`: October 3 acceptance summary. +- `/home/utmist/mist-api-results-2026-10-03`: detailed prior acceptance artifacts. + +### Local tools and access + +- Go: `/home/utmist/.local/share/mist-runtimes/go-1.25.1/bin/go`. +- Node: `/home/utmist/.local/share/mist-runtimes/node-22.16.0/bin/node`. + Prepend that runtime's directory to PATH when running npm/npx. +- Kubeconfig: `/home/utmist/.kube/config`. +- QuietBox SSH: user `utmist-tt`, host `100.95.175.37`, existing identity file + `/home/utmist/.ssh/id_ed25519_quietbox_codex`; use IdentitiesOnly and BatchMode. +- Browser checks can use the temporary Playwright installation at + `/tmp/mist-browser-check/node_modules/playwright` and existing cached Chromium. +- User is in the Docker group. Sudo requires a password; prior authorized root + cluster operations used a Docker host bind/chroot. Do not print secret values. +- Do not include credentials, join tokens, registry secrets, or private keys in + documentation, Git, or tool output. + +Model recommendation for the user's remaining quota: GPT-6.1 Sol, Medium +reasoning, Standard speed. This was advice, not a settings change. Saved settings +were Max/Fast at the time of planning. Exact quota consumption cannot be predicted. + +## Progress log + +- Planning: agreed scope recorded; existing API health and both Ready nodes checked. +- Branch: `feat/local-job-foundation`. Commit `2f9b178` preserves the prior working + pilot and this plan. Commit identity uses the connected GitHub account's public + noreply address for this commit only; global Git settings were not changed. +- Backend additions: `hardware.go`, `images.go`, `outputs.go`, and `job_states.go`. + Inventory reads node/pod/claim/slice state; image defaults are preserved; + outputs have per-job mounts and an initialization step; history reads are batched. +- Frontend: typed inventory/catalog clients, a shared polling hook, separate + submission/card/log components, and live Machines information. +- Focused backend tests pass. Frontend interaction tests (5), type checking, and + production build pass. A test initially submitted before the catalog loaded; + it was corrected to wait for the form's actual readiness. Final lint also + passed after removing unnecessary test casts and shadowed callback names. +- Example images `mist-training:cpu-v1` and `mist-training:nvidia-v1` were built + and imported on the NVIDIA node. API image `mist-api:foundation-20261008` + is deployed; the final binary image ID is recorded in `foundation-results.json`. +- Fresh browser checks passed: packaged CPU training with image ENTRYPOINT/CMD + and `/app` WORKDIR, both NVIDIA GPUs, TT two-board/four-chip training, exit 17, + cancellation, real states/logs, and the live Machines page. +- Fresh allocation checks passed for NVIDIA and TT: whole-machine occupied/free + counts, queued follower, cancellation with retained logs, automatic device + reuse, and return to 2 GPUs / 4 boards available. +- Persistent output checks used new read-only containers after training exited. + CPU model/results, both NVIDIA checkpoints, and four TT chip checkpoints and + metrics were readable on the proper nodes. SHA256/file evidence was saved. +- A deliberately missing approved image exposed its real pull reason in the API + and browser, then failed with DeadlineExceeded. Temporary image approval was + removed, and prior job/cancellation history survived API restarts. +- Final focused backend tests: 12 passed. Frontend interaction tests: 5 passed; + type checking, production build, and lint of changed frontend files passed. + CLI tests passed. The full src suite passed in 33.325 seconds with isolated + Redis and the existing Docker test image. Its initial 15-second legacy CPU + test timed out during large-image disk activity; the suite passed after the + imports finished. No legacy executor behavior was changed. +- The first browser verifier expected a completed TT claim to retain allocations. + Kubernetes had correctly released them. The verifier now checks completed + request/log evidence and checks actual DRA devices during live reservations. +- Fresh RBAC checks passed: required inventory lists allowed; Secrets, node + writes, pod exec, and Jobs in other namespaces denied. Private registry + Secret wiring is implemented/documented; no private registry login was tested. +- Detailed evidence: `/home/utmist/mist-foundation-results-2026-10-08`. + Compact checked-in summary: `deploy/k3s/foundation-results.json`. + Final idle snapshot: both nodes Ready, 0 allocated / 2 available NVIDIA GPUs, + 0 allocated / 4 available TT boards. All 40 retained Mist Jobs were terminal. + Job history read measured 0.193 seconds, compared with the 3.586-second baseline. +- Today's agreed implementation and verification are complete: parts 1–3 plus + persistent local outputs. The percentage updates referred to today's scope. + The complete platform still needs part 4 (shared datasets/downloads) and + part 5 (deployment/login). The team handoff is `docs/local-job-foundation.md`. +- Screen question checked: automatic suspend on AC has timeout 0 (disabled), + so the display may turn off while work continues. Power settings were not changed. + +## References + +- [Kubernetes Jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) +- [Container images](https://kubernetes.io/docs/concepts/containers/images/) +- [K3s private registries](https://docs.k3s.io/installation/private-registry) +- [Kubernetes persistent volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) +- [Local path provisioner](https://github.com/rancher/local-path-provisioner) +- [Tenstorrent platform support](https://docs.tenstorrent.com/tt-operator/latest/platform-support.html) diff --git a/docs/archive/k3s-deployment-history.md b/docs/archive/k3s-deployment-history.md new file mode 100644 index 0000000..4256a27 --- /dev/null +++ b/docs/archive/k3s-deployment-history.md @@ -0,0 +1,408 @@ +> **Historical record — superseded.** This text describes an earlier phase. +> Commands and claims below are retained as history, not current setup instructions. +> Start with the [current documentation index](../README.md) and +> [department operating guide](../department-rollout.md). + +# Accelerator jobs on k3s + + +**October 9 update:** Team workspaces, scoped storage, enforced limits, durable queueing and private +production hosting are implemented. Earlier local pilot descriptions below record the +original phase. Use [the current operating guide](../department-rollout.md) for today's behavior. + +For the original NVIDIA-node setup and recovery procedure, read the +[setup runbook](k3s-setup-history.md). Its final section records the QuietBox worker +join on 2026-10-03. + +The NVIDIA server runs Ubuntu 22.04, k3s 1.36, and two RTX A4000 GPUs. The host +driver is pre-installed. NVIDIA GPU Operator v26.7.1 is configured with +[`nvidia-gpu-operator-values.yaml`](../../deploy/k3s/nvidia-gpu-operator-values.yaml). Pin the +chart version when installing or upgrading: + +```bash +sudo env KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm upgrade --install gpu-operator \ + oci://nvcr.io/nvidia/cloud-native-charts/gpu-operator \ + --version v26.7.1 --namespace gpu-operator --create-namespace \ + --values deploy/k3s/nvidia-gpu-operator-values.yaml --wait +``` + +Run from the repository root. Check that the operator pods are healthy and the +node advertises two GPUs before submitting a workload: + +```bash +sudo k3s kubectl -n gpu-operator get pods +sudo k3s kubectl describe node utmist-z1opa08 +``` + +The `Allocatable` section must include `nvidia.com/gpu: 2`. Then run the +single-GPU CUDA smoke test: + +```bash +sudo k3s kubectl create namespace mist +sudo k3s kubectl apply -f deploy/k3s/gpu-smoke-job.yaml +sudo k3s kubectl -n mist wait --for=condition=complete job/gpu-smoke --timeout=180s +sudo k3s kubectl -n mist logs job/gpu-smoke +``` + +The expected log contains `Test PASSED`. To rerun the Job, delete only the +finished smoke Job and apply the manifest again: + +```bash +sudo k3s kubectl -n mist delete job gpu-smoke +sudo k3s kubectl apply -f deploy/k3s/gpu-smoke-job.yaml +``` + +No Service or NodePort is needed for a batch Job. This test verifies that k3s +can allocate an NVIDIA GPU and execute CUDA. Mist now submits these resources +through its Kubernetes executor; see the [API setup](#mist-api-cli-and-jobs-page). + +The cluster also has `utmist-tt` as an agent at `10.0.0.112`. Tenstorrent +DRA allocation is installed there: four n300 boards, each containing two +Wormhole chips. The NVIDIA smoke Job remains specific to `utmist-z1opa08`. + +## Tenstorrent allocation + +TT-Operator **0.3.0** installs Fabric Manager **0.2.30** and the DRA driver +**0.0.59**, configured by +[`tenstorrent-operator-values.yaml`](../../deploy/k3s/tenstorrent-operator-values.yaml). +The existing host KMD/firmware and the existing NFD installation are retained; +driver management, duplicate NFD, telemetry, JobSet, and PMIx are disabled. +The manually verified `tenstorrent.com/has-tt=true` node label enables discovery. + +To reproduce the installation from the repo root: + +```bash +export KUBECONFIG=/home/utmist/.kube/config +k3s kubectl label node utmist-tt tenstorrent.com/has-tt=true --overwrite +helm upgrade --install tt-operator oci://ghcr.io/tenstorrent/helm/tt-operator \ + --version 0.3.0 --namespace tt-operator-system --create-namespace \ + --values deploy/k3s/tenstorrent-operator-values.yaml +k3s kubectl apply -f deploy/k3s/tenstorrent-workload-policy.yaml +k3s kubectl -n tt-operator-system get pods +k3s kubectl get deviceclasses,resourceslices +k3s kubectl get resourceslices -o yaml +``` + +All three operator pods must be Ready. The `tenstorrent.com` DeviceClass +and a ResourceSlice on `utmist-tt` must exist, with **four** `n300` devices +whose `chipCount` is **2**. ResourceSlices are the inventory; claims show +reservations. TT does not appear as an `nvidia.com/gpu` node resource. + +The supported reservation unit is a whole **n300 board**, including its +remote chip. Jobs can reserve one to four boards; separate jobs can use +the remaining boards. A request waits if insufficient boards or other +requested resources are free. Each pod gets its own ResourceClaim through +a ResourceClaimTemplate, and only its allocated device nodes are injected +through CDI. Boards are released when the pod finishes, even while its +completed Job and checkpoints are retained. + +[`training-tenstorrent-job.yaml`](../../deploy/k3s/training-tenstorrent-job.yaml) now requests +one board, runs without privileged access, and reserves 2 GiB of 1 GiB +hugepages. The host still has 16 GiB of such hugepages. It mounts the existing +TT-Metal checkout and Python environment read-only, so it is specific to +this QuietBox rather than a portable training image. + +For uniquely named training checks and separate checkpoint directories, +use [`submit_tenstorrent_smoke.py`](../../deploy/k3s/submit_tenstorrent_smoke.py). It needs +Python PyYAML, available on the server. For example: + +```bash +export KUBECONFIG=/home/utmist/.kube/config +python3 deploy/k3s/submit_tenstorrent_smoke.py my-tt-test-a --boards 1 +python3 deploy/k3s/submit_tenstorrent_smoke.py my-tt-test-b --boards 1 +k3s kubectl -n mist get jobs,pods,resourceclaims +k3s kubectl -n mist logs -f job/my-tt-test-a +``` + +Use `--boards 2` for four chips, or `--boards 4` for all eight. The script +submits the small regression test described below. A different model needs +compatible Tenstorrent training code. Claim count, expected chip count, +hugepage reservation, and checkpoint directory must all match the job. +These tests train on each allocated chip independently; they do not perform +distributed training of one model. For multiple boards, the check launches +one runtime per board using its PCI address in `TT_VISIBLE_DEVICES`. This +avoids assuming that an arbitrary selected subset forms a connected mesh. +The scheduler reserves boards but does not promise their fabric topology; +a model spanning boards needs an appropriate topology and training runtime. + +The admission policy applies to pods in `mist`, including init and ephemeral +containers. It rejects privileged access, added capabilities, host PID/IPC, +arbitrary host mounts, and writable mounts of the host runtime. This blocks +the previous privileged whole-machine test from being submitted again. +Individual character-device mounts remain allowed for the verification +probes, with access to unclaimed devices denied by device cgroups. The policy +is a guard for managed workloads, not a complete security boundary for +untrusted tenants or cluster administrators; hugepage mounts are host-shared. + +### Allocation acceptance test + +On 2026-10-03, four jobs reserved distinct boards and trained concurrently +on all eight chips. Their first-chip training intervals overlapped for +**3.924 seconds**. Each job could open its own device and received permission +errors opening the other three, even when the other character nodes were +mounted for the probe. A fifth job remained Pending with `cannot allocate +all claims`, then started automatically after a board was released. All five +jobs passed training, held-out-data, and checkpoint reload checks. Automatic +claim cleanup left all four boards free afterward. + +To repeat that live test (it requires all four boards to be free): + +```bash +export KUBECONFIG=/home/utmist/.kube/config +python3 deploy/k3s/verify_tenstorrent_allocation.py \ + --artifacts /home/utmist/k3s-allocation-results-new-run +``` + +The test holds initialized jobs at a shared-file barrier to inspect their +reservations, checks actual device access, observes the fifth job queuing, +and then releases the barrier. It verifies overlapping training timestamps, +loss thresholds, checkpoint files, and board reuse. Test jobs are retained; +the temporary PVC reader is deleted. The barrier times out after five +minutes if interrupted. The checkpoint PVC is retained. + +Operator restart recovery and an NVIDIA CUDA regression check also passed. +See [`tenstorrent-allocation-results.json`](../../deploy/k3s/tenstorrent-allocation-results.json) +for pinned image IDs and results. Logs, claim/queue snapshots, admission +checks, and checkpoint archives are in +`/home/utmist/k3s-allocation-results-2026-10-03` on the NVIDIA server. +The vendor currently advertises an incorrect board-memory capacity in the +ResourceSlice; these claims select board type/count and do not use that +capacity for placement. Verify/correct it before adding memory-based selectors. + +## Training smoke jobs + +[`training_smoke.py`](../../deploy/k3s/training_smoke.py) learns a 32-input/32-output linear +mapping from 128 synthetic samples and evaluates 64 held-out samples. Each +device must complete 100 gradient updates, reduce training MSE by at least +98%, achieve held-out MSE below `0.0005`, change its weights, and reload its +checkpoint successfully. NVIDIA uses PyTorch CUDA autograd. Tenstorrent uses +TT-NN for forward passes, explicit analytic gradients, and weight updates; +CPU Torch creates the data, reads metrics, and saves checkpoints. This does +not validate automatic differentiation or arbitrary models on Tenstorrent. + +The NVIDIA Job reserves both GPUs and tests each independently. The current +Tenstorrent manifest reserves one n300 board and tests its two chips +independently. The original eight-chip privileged Job, named +`training-tenstorrent`, is retained only as historical evidence; the new +Job is `training-tenstorrent-dra` and uses DRA/CDI reservations. + +Run from the repository root: + +```bash +sudo k3s kubectl -n mist create configmap training-smoke-script \ + --from-file=deploy/k3s/training_smoke.py --dry-run=client -o yaml \ + | sudo k3s kubectl apply -f - +sudo k3s kubectl apply -f deploy/k3s/training-nvidia-job.yaml +sudo k3s kubectl apply -f deploy/k3s/training-tenstorrent-job.yaml +sudo k3s kubectl -n mist logs -f job/training-nvidia +sudo k3s kubectl -n mist logs -f job/training-tenstorrent-dra +sudo k3s kubectl -n mist get jobs,pvc +``` + +Each successful log ends with `TRAINING_PASSED`. Checkpoints and +`results.json` are stored in the corresponding `training-*-checkpoints` PVC, +with the one-board TT check under its `dra-single` directory, +which uses storage local to the selected node. Preserve the PVCs when +removing a completed Job. Delete only the completed Job before applying its +manifest to rerun it; retained checkpoint filenames will be overwritten. +The TT manifest depends on QuietBox's existing host paths, hugepages, and +TT-Metal checkout and is not a portable training image. + +The original whole-node Jobs passed on 2026-10-03, with exit code zero. All ten devices changed +weights, met the held-out-data threshold, and reloaded their saved +checkpoints. The archives were also checked after the Pods completed. + +| Backend | Devices tested independently | Initial training MSE | Final training MSE | Held-out MSE | +| --- | --- | --- | --- | --- | +| NVIDIA RTX A4000, PyTorch FP32 | 2 GPUs | 0.319374 | 0.000000201866 | 0.000000932067 | +| Wormhole, TT-NN BF16 | 8 chips | 0.319374 | 0.0000623871 | 0.000175072 | + +Precision and implementations differ; these figures establish convergence +and are not a performance comparison. Exact image IDs, the TT-Metal host +commit, Job timestamps, artifact locations, and acceptance thresholds are in +[`training-smoke-results.json`](../../deploy/k3s/training-smoke-results.json). Local logs and +checkpoint archives are retained under +`/home/utmist/k3s-training-results-2026-10-03` on the NVIDIA server. The +completed Kubernetes Jobs and their checkpoint PVCs are retained too. + + +## Mist API, CLI and Jobs page + +The API now uses Kubernetes by default and runs in `mist-system`. Its service +account manages Jobs and script ConfigMaps in `mist`, reads Pod logs/events +and TT claims, and has no cluster administrator credentials. The configured +pilot owner is `utmist`. No authentication or public ingress is configured. +The foundation also lists node/pod/claim/slice state for live hardware inventory. +See [the foundation guide](local-job-foundation.md) for contracts, +website instructions, persistent outputs, and the developer handoff. + +### Build and install on the current server + +Requires Go 1.25.1, Docker, k3s, and the private administrative kubeconfig. +Use the already installed GPU/TT operators described above. The API image is +built locally and imported into k3s containerd; `imagePullPolicy: Never` +means that it must be present on the selected server before deployment. + +```bash +export KUBECONFIG=/home/utmist/.kube/config +CGO_ENABLED=0 go -C src build -o ../bin/mist-api . +docker build -f deploy/k3s/Dockerfile.api -t mist-api:foundation-20261008 . +docker build -f deploy/k3s/examples/Dockerfile.training \ + -t mist-training:cpu-v1 deploy/k3s/examples +docker build -f deploy/k3s/examples/Dockerfile.training \ + --build-arg BASE_IMAGE=pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime \ + -t mist-training:nvidia-v1 deploy/k3s/examples +docker save -o /tmp/mist-foundation-images.tar \ + mist-api:foundation-20261008 mist-training:cpu-v1 mist-training:nvidia-v1 +sudo k3s ctr -n k8s.io images import --platform linux/amd64 /tmp/mist-foundation-images.tar +k3s kubectl apply -f deploy/k3s/mist-api.yaml +k3s kubectl apply -f deploy/k3s/mist-tenstorrent-claims.yaml +k3s kubectl apply -f deploy/k3s/tenstorrent-workload-policy.yaml +k3s kubectl -n mist create configmap training-smoke-script \ + --from-file=deploy/k3s/training_smoke.py --dry-run=client -o yaml \ + | k3s kubectl apply -f - +k3s kubectl -n mist-system rollout status deployment/mist-api --timeout=120s +``` + +The API manifest creates namespaces, RBAC and all three checkpoint PVCs +without submitting a training Job. If rebuilding the same local image tag, +import the new image and run `rollout restart deployment/mist-api` in +`mist-system`. Preserve Jobs and PVCs when restarting the API. + +For a manual local endpoint: + +```bash +k3s kubectl -n mist-system port-forward service/mist-api 3000:3000 --address 127.0.0.1 +``` + +Then in another terminal: + +```bash +curl http://127.0.0.1:3000/healthz +go -C cli build -o ../bin/mist . +bin/mist job list --all +cd web-interface +npm install --package-lock=false +npm run dev -- --host 127.0.0.1 --port 3001 --strictPort +``` + +Open http://127.0.0.1:3001/jobs. Choose CPU, NVIDIA GPUs or Tenstorrent boards; +submit a Python script, select an approved **Container image** with its own +entrypoint/arguments, or choose **Small training check** for an accelerator. +The page displays actual placement, queue messages, exit status and logs. +Hardware availability is live; the Machines route uses the same inventory. +Cancel stops a waiting/running workload and retains cancellation history. +CLI commands are in [the CLI guide](../../cli/docs/setup.md). + +The installed user units keep both local services running during the user's +systemd session. Their paths target this server and need adjustment on another +host; they do not enable user lingering or guarantee operation before login. + +```bash +mkdir -p ~/.config/systemd/user +cp deploy/k3s/mist-api-forward.service deploy/k3s/mist-web.service ~/.config/systemd/user/ +systemctl --user daemon-reload +systemctl --user enable --now mist-api-forward mist-web +systemctl --user status mist-api-forward mist-web +``` + +Do not run manual listeners on the same ports while these units are active. +Stop them with `systemctl --user stop mist-web mist-api-forward` if needed. + +### Submission examples + +```bash +curl -sS http://127.0.0.1:3000/jobs -H 'Content-Type: application/json' \ + -d '{"name":"hello","accelerator":"cpu","command":["python","-c","print(42)"]}' +curl -sS http://127.0.0.1:3000/jobs -H 'Content-Type: application/json' \ + -d '{"name":"one-board","type":"training-smoke","accelerator":"tenstorrent","device_count":1}' +curl -sS http://127.0.0.1:3000/jobs -H 'Content-Type: application/json' \ + -d '{"name":"one-gpu","type":"training-smoke","accelerator":"nvidia","device_count":1}' +``` + +CPU jobs request no accelerator. NVIDIA requests accept one or two GPUs; +TT requests accept one to four boards, each containing two chips. Defaults +are 250m CPU/256Mi RAM for CPU jobs, 2 CPU/2Gi for NVIDIA, and 2 CPU/4Gi plus +2Gi hugepages per TT board. CPU and memory limits equal requests. The API +accepts CPU up to 8, memory 64Mi to 32Gi (TT minimum 2Gi), and deadlines from +10 seconds to 24 hours, default 600 seconds. Deadlines include queue time. +Images must be approved by the API's allowlist; submissions cannot specify +privileged access or host mounts. Environment variables `MIST_JOB_ID`, +`MIST_OUTPUT_DIR=/outputs`, and `MIST_CHECKPOINT_DIR=/checkpoints/` +provide the job identity and two aliases for its persistent directory. + +Storage remains node-local with a shared backing PVC per compute profile. +New workloads mount only their job-specific subdirectory. Retained Jobs +provide history across API restarts, without TTL. Normal logs remain in pod +logs; cancellation preserves their last 32KiB in the Job metadata. Keep the +Pods/PVCs or archive them before cleanup. This is a local single-owner pilot, +not an authenticated multi-user deployment. + +### Custom images and private registries + +The deployment's `MIST_ALLOWED_IMAGES` approves exact image references; +`GET /images` exposes them to the website. For real workloads, build/push an +image containing code/dependencies, approve its immutable tag/digest, and let +the selected node pull it. Local example tags must be imported on the NVIDIA +server. An approved image is not guaranteed to be pullable or CUDA compatible. +The TT profile still requires its tested image and read-only host runtime. + +CPU/NVIDIA container jobs preserve image ENTRYPOINT/CMD and WORKDIR when no +override is supplied. Arguments are array entries, not a parsed shell command. +Scripts and training checks retain their tested runtime profiles. Write results +to `/outputs`; files elsewhere in the container are not persistent outputs. + +For private images, an administrator creates a `kubernetes.io/dockerconfigjson` +Secret in workload namespace `mist`, then adds `MIST_IMAGE_PULL_SECRETS` to +the API Deployment environment with the Secret name(s), comma-separated. +Jobs receive those names as `imagePullSecrets`. Mist does not read Secret +contents or accept registry passwords in the submission form. Keep Secret +manifests/credentials outside Git. K3s node registry configuration is another +option: [private registries](https://docs.k3s.io/installation/private-registry). + +### Foundation acceptance + +With both accelerators free and the example images imported, run the browser +story, then the allocation/output checks using the same artifact directory: + +```bash +export KUBECONFIG=/home/utmist/.kube/config +export MIST_PLAYWRIGHT_MODULE=/tmp/mist-browser-check/node_modules/playwright/index.mjs +node deploy/k3s/verify_foundation_browser.mjs /home/utmist/mist-foundation-results-new-run +python3 deploy/k3s/verify_foundation.py --artifacts /home/utmist/mist-foundation-results-new-run +``` + +Playwright and Chromium are test tooling, separate from the web application. +Set `MIST_PLAYWRIGHT_MODULE` to its installed `index.mjs`, or install Playwright +where Node can resolve it. `MIST_WEB_URL` changes the browser endpoint; +`--url` changes the API endpoint for the Python tool. +Checks cover packaged CPU/NVIDIA images, both GPUs, TT training, real UI +states/logs/cancellation, occupied/free counts, queues, reuse, and saved files +read by fresh containers on both nodes. Temporary read-only output probes are +removed; test Jobs and result directories remain. The final markers are +`FOUNDATION_BROWSER_PASSED` and `FOUNDATION_ALLOCATION_OUTPUTS_PASSED`. +October 8 evidence is under `/home/utmist/mist-foundation-results-2026-10-08`. +See [the checked-in summary](../../deploy/k3s/foundation-results.json) for actual job IDs, +deployed image ID, and completed checks. + +### API acceptance test + +This test needs both GPUs and all four TT boards available. It submits 17 +uniquely named jobs, holds initialized devices at barriers to prove separate +reservations, verifies queued jobs, training/checkpoint reload, CPU failures, +deadlines, retained cancellation logs, and cancellation freeing all TT boards. + +```bash +export KUBECONFIG=/home/utmist/.kube/config +python3 deploy/k3s/verify_mist_api.py \ + --artifacts /home/utmist/mist-api-results-new-run +``` + +Successful output ends with `MIST_API_PASSED`. Jobs and checkpoint directories +are retained. A barrier times out after five minutes if interrupted. The +verification script uses the operator's administrative context to inspect +claims/devices and release test barriers; the Mist API itself cannot exec. +Results from 2026-10-03 are in [mist-api-results.json](../../deploy/k3s/mist-api-results.json), +with raw evidence and checkpoint archives under +`/home/utmist/mist-api-results-2026-10-03`. These checks establish small model +training and allocation, not large/distributed-model support or performance. diff --git a/docs/archive/k3s-setup-history.md b/docs/archive/k3s-setup-history.md new file mode 100644 index 0000000..7b8b3af --- /dev/null +++ b/docs/archive/k3s-setup-history.md @@ -0,0 +1,283 @@ +> **Historical record — superseded.** This text describes an earlier phase. +> Commands and claims below are retained as history, not current setup instructions. +> Start with the [current documentation index](../README.md) and +> [department operating guide](../department-rollout.md). + +# Single-node k3s and NVIDIA GPU pilot: setup and handoff + +This runbook records the configuration observed on **2026-09-28**, a repeatable +setup and validation procedure, and the failure encountered during setup. Run +commands from the Mist repository root unless noted. The +[Kubernetes execution pilot](kubernetes-pilot.md) describes the +implemented Kubernetes executor and remaining product work. + +## What the pieces do + +The machine already had two NVIDIA GPUs and a working host driver. A **single +k3s server** was installed on it. A server includes a local kubelet and +container runtime and can run Jobs without a separate `k3s-agent` service. +The NVIDIA GPU Operator added the Kubernetes integration: + +1. Its **device plugin** registers `nvidia.com/gpu` with the kubelet, allowing + the scheduler to reserve a GPU for a Pod that requests one. +2. Its **container toolkit** prepares CDI device information that containerd + uses to expose the assigned GPU and host driver libraries to a workload. + The enabled NRI plugin handles GPU management containers and avoids + rewriting k3s's containerd runtime configuration. +3. A CUDA sample Job requests `nvidia.com/gpu: 1` and exercises the device. + +The operator was configured **not** to replace the existing host driver. Those +values are in +[`nvidia-gpu-operator-values.yaml`](../../deploy/k3s/nvidia-gpu-operator-values.yaml). NVIDIA +states that its NRI plugin is not yet generally available; this is a pilot +configuration. See the [GPU Operator installation guide](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) +and [CDI/NRI guide](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/cdi.html). + +## Observed state on 2026-09-28 + +| Item | Observed value | +| --- | --- | +| Node | `utmist-z1opa08`, one control-plane and worker node | +| OS | Ubuntu 22.04.5 LTS | +| k3s | `v1.36.4+k3s1`; `k3s.service` enabled and active | +| Container runtime | k3s containerd `2.3.4-k3s1.36` | +| GPUs | Two NVIDIA RTX A4000 cards | +| Host driver | NVIDIA `590.48.01`, already present before GPU Operator deployment | +| GPU Operator | Helm release `gpu-operator`, chart `v26.7.1`, namespace `gpu-operator` | +| Test namespace | `mist` | +| Node resource | Capacity and allocatable: `nvidia.com/gpu: 2` | + +The node registered at 19:23 EDT. GPU Operator resources appeared around 19:39 +EDT; Helm release revision 2 was deployed at 19:45 EDT. The `gpu-smoke` Job +completed at about 19:44 EDT. After the service fix, a new `gpu-confirm` Job +completed at 19:54 EDT. Both sample logs contained `Test PASSED`. Each Job +requested **one** GPU. The node advertised two, but these tests did not +independently identify or exercise each physical card. + +These times and versions come from systemd, Kubernetes object timestamps, +`helm list`, `nvidia-smi`, and the completed Job logs. The exact historical +k3s installation command was not retained, so the commands below are a +reproduction recipe rather than a transcript. + +## Reproduce or verify this pilot + +Check versions against the target host before rebuilding or upgrading. Have a +working NVIDIA driver and `helm` available. Never commit the kubeconfig, +cluster token, or private keys to this repository. + +### 1. Confirm the GPUs work on the host + +```bash +nvidia-smi -L +nvidia-smi --query-gpu=name,driver_version --format=csv,noheader +``` + +On this node the first command lists two RTX A4000 cards. Resolve host driver +or hardware failures before diagnosing Kubernetes GPU allocation. + +### 2. Run one k3s server + +For a fresh **single-node** install, pin the version used in this pilot: + +```bash +curl -sfL https://get.k3s.io | sudo env INSTALL_K3S_VERSION='v1.36.4+k3s1' sh -s - server +sudo systemctl is-active k3s +sudo k3s kubectl get nodes -o wide +``` + +Expect `active` and a `Ready` node. The installer writes an administrative +kubeconfig at `/etc/rancher/k3s/k3s.yaml`; `sudo k3s kubectl` uses it without +copying credentials. Any copied kubeconfig must be kept private. The +[k3s quick-start guide](https://docs.k3s.io/quick-start) explains that a +single server can host workloads and that agents join additional machines. + +This same host must **not** run a separate `k3s-agent` service: + +```bash +systemctl is-active k3s-agent +systemctl is-enabled k3s-agent +``` + +Expected outputs here are `inactive` and `disabled`. These commands can +return nonzero for those expected states. + +### 3. Install the GPU Operator + +The tracked Helm values disable Operator driver installation and enable its +NRI plugin for GPU management containers. Keep the chart version pinned when +reproducing this setup: + +```bash +sudo env KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm upgrade --install gpu-operator \ + oci://nvcr.io/nvidia/cloud-native-charts/gpu-operator \ + --version v26.7.1 --namespace gpu-operator --create-namespace \ + --values deploy/k3s/nvidia-gpu-operator-values.yaml --wait + +sudo k3s kubectl -n gpu-operator get pods +sudo k3s kubectl describe node utmist-z1opa08 +``` + +The device plugin and container toolkit Pods should be Running. In the node's +`Capacity` and `Allocatable` sections, expect `nvidia.com/gpu: 2` for this +hardware. A completed validator Pod is normal. The operator may need a little +time to register devices after k3s starts; check again after its Pods are +healthy. + +### 4. Execute CUDA through k3s + +The tracked [`gpu-smoke-job.yaml`](../../deploy/k3s/gpu-smoke-job.yaml) requests and limits one +`nvidia.com/gpu` device. Create the namespace if it does not already exist: + +```bash +sudo k3s kubectl create namespace mist +sudo k3s kubectl apply -f deploy/k3s/gpu-smoke-job.yaml +sudo k3s kubectl -n mist wait --for=condition=complete job/gpu-smoke --timeout=180s +sudo k3s kubectl -n mist describe job gpu-smoke +sudo k3s kubectl -n mist logs job/gpu-smoke +``` + +Expect `1 Succeeded`, `nvidia.com/gpu: 1` in the Job's requests and limits, +and `Test PASSED` in the logs. This establishes that k3s scheduled a GPU Pod, +the runtime exposed a device inside the container, and CUDA executed. Merely +seeing the host GPUs or the Kubernetes resource count is a weaker check. + +A completed Job does not rerun on `kubectl apply`. To repeat this test, delete +**only the completed smoke Job**, then apply the manifest again: + +```bash +sudo k3s kubectl -n mist delete job gpu-smoke +sudo k3s kubectl apply -f deploy/k3s/gpu-smoke-job.yaml +sudo k3s kubectl -n mist wait --for=condition=complete job/gpu-smoke --timeout=180s +sudo k3s kubectl -n mist logs job/gpu-smoke +``` + +Save logs first if they are needed for an incident review. This batch Job +does not need a Service or NodePort. + +## Failure encountered: duplicate agent on port 6444 + +Both `k3s.service` and `k3s-agent.service` had been enabled on the **same +host**. The agent listened on `127.0.0.1:6444`, which the server's API process +also needed. The server repeatedly exited with: + +```text +failed to listen on 127.0.0.1:6444: bind: address already in use +``` + +During the conflict, `kubectl` could not connect to `127.0.0.1:6443`. +`nvidia-smi` still saw the GPUs, but new Kubernetes workloads could not use +them because the control plane was unavailable. Agent logs also referenced +`https://myserver:6443`, an example server name rather than this node's +working endpoint. The fix on 2026-09-28 was: + +```bash +sudo systemctl stop k3s-agent.service +sudo systemctl disable k3s-agent.service +sudo systemctl is-active k3s +sudo k3s kubectl get nodes +``` + +The server recovered without replacing the host driver or reinstalling the +operator. If it does not recover, inspect logs before restarting it, +especially when other workloads are running: + +```bash +systemctl status k3s k3s-agent --no-pager +journalctl -u k3s -u k3s-agent -n 100 --no-pager +ss -ltn '( sport = :6443 or sport = :6444 )' +``` + +## Troubleshooting and limits + +- **API connection refused:** Check `k3s.service` and its journal first. On + this node, confirm the duplicate agent has not been enabled again. +- **Node Ready but no `nvidia.com/gpu`:** Check `nvidia-smi -L`, the device + plugin and toolkit Pods, and + `sudo k3s kubectl -n gpu-operator logs daemonset/nvidia-device-plugin-daemonset --tail=100`. + Compare live Helm values with the tracked file if the release has changed. +- **GPU Job Pending:** Use `sudo k3s kubectl -n mist describe pod ` + to inspect scheduler events and requested resources. Two GPUs are the + observed capacity, not a promise that both are free at every moment. +- **Pod starts but CUDA fails:** Inspect its logs, host `nvidia-smi`, toolkit + Pod logs, and device plugin logs. An advertised resource alone does not + prove CUDA works. + +The smoke test proves **one requested GPU worked through k3s on the test +date**. It does not prove independent operation of both cards, GPU sharing, +performance, reboot durability, multi-node behavior, or Mist API integration. +The subsequent 2026-10-03 allocation/training and Mist API checks are recorded +in [the current deployment guide](k3s-deployment-history.md#mist-api-cli-and-jobs-page). + +## QuietBox worker joined on 2026-10-03 + +`utmist-tt` joined the existing `utmist-z1opa08` cluster as an agent. Both +machines are on the `10.0.0.0/24` LAN as well as the same Tailscale network; +the cluster uses their LAN addresses and the default Flannel VXLAN backend. +The server was not changed to advertise its Tailscale address. + +| Node | Role | Kubernetes InternalIP | k3s version | +| --- | --- | --- | --- | +| `utmist-z1opa08` | server and NVIDIA worker | `10.0.0.175` | `v1.36.4+k3s1` | +| `utmist-tt` | Tenstorrent host and k3s agent | `10.0.0.112` | `v1.36.4+k3s1` | + +QuietBox previously ran an independent `v1.36.5+k3s1` server with only a test +Nginx workload, standard system pods, and no PVCs or user Jobs. Its server +state was backed up to the root-only +`/root/k3s-server-prejoin-2026-10-03.tar.gz` on QuietBox before uninstalling +that server. The agent is enabled at boot; no `k3s.service` remains there. + +The agent's `/etc/rancher/k3s/config.yaml` points to +`https://10.0.0.175:6443`, sets `node-ip: 10.0.0.112`, uses `enp201s0` for +Flannel, and reads its persistent, root-only bootstrap token from +`/etc/rancher/k3s/agent-token`. Do not commit or print that token. The +installation token expires after one hour; the persistent token was installed +and verified by restarting `k3s-agent`. + +QuietBox has UFW active with default deny for incoming and routed traffic. +The join added inbound UDP `8472` and TCP `10250` from `10.0.0.175` on +`enp201s0`, plus routed traffic between `10.42.1.0/24` on `cni0` and +`10.42.0.0/24` on `flannel.1` in both directions. The NVIDIA server's UFW is +inactive. A temporary BusyBox pod placed on QuietBox resolved +`kubernetes.default.svc.cluster.local` and fetched the Nginx Service whose +pod ran on the NVIDIA node; its log ended with `CROSS_NODE_OK`. The test pod +was then removed. + +QuietBox exposes four `/dev/tenstorrent` devices and has the `tenstorrent` +kernel module loaded. TT-Operator 0.3.0 now runs Fabric Manager 0.2.30 and +DRA driver 0.0.59 with host driver management disabled. It advertises four +n300 board reservations, each covering two chips, through the +`tenstorrent.com` DeviceClass and ResourceSlices. Host KMD and firmware are +retained. See [installation, allocation, and live tests](k3s-deployment-history.md#tenstorrent-allocation). + +Four nonprivileged training jobs passed concurrently, with device cgroups +denying access to the other jobs' boards. A fifth job queued and then +trained after automatic board release. The `mist` admission policy blocks +the former privileged whole-machine setup from bypassing reservations. +Distributed training and untrusted-tenant security remain separate work. + +To verify the current cluster from the server: + +```bash +sudo k3s kubectl get nodes -o wide +sudo k3s kubectl get pods -A -o wide +sudo k3s kubectl describe node utmist-tt +``` + +The administrative kubeconfig copy at `/home/utmist/.kube/config` is owned by +`utmist` and mode `0600` as of this join. + + +## Mist application integration on 2026-10-03 + +The deployed API now creates real CPU, NVIDIA and Tenstorrent Jobs. CLI and +browser checks exercised submission, actual status/logs, cancellation, and +TT board-count selection. Live acceptance checks covered distinct concurrent +reservations, queuing and reuse, whole-device requests, exit-17 failure, +deadline failure, persisted checkpoints, and cancellation freeing all boards. +The API uses a dedicated namespaced service account, not the administrative +kubeconfig used by local operators. See [API installation and live evidence](k3s-deployment-history.md#mist-api-cli-and-jobs-page). + +The user's `mist-api-forward` and `mist-web` systemd units expose only +`127.0.0.1:3000` and `127.0.0.1:3001`. These user services require a user +systemd session. Current authentication, credits, team quotas, shared storage +and large/distributed model support remain outside the validated pilot. diff --git a/docs/archive/kubernetes-pilot.md b/docs/archive/kubernetes-pilot.md new file mode 100644 index 0000000..eab19dd --- /dev/null +++ b/docs/archive/kubernetes-pilot.md @@ -0,0 +1,139 @@ +> **Historical record — superseded.** This text describes an earlier phase. +> Commands and claims below are retained as history, not current setup instructions. +> Start with the [current documentation index](../README.md) and +> [department operating guide](../department-rollout.md). + +# Kubernetes execution pilot + + +**October 9 update:** Shared datasets/results, member login and private production +hosting are implemented. Earlier local pilot descriptions below record the +original phase. Use [the current operating guide](complete-foundation.md) for today's behavior. + +## Implemented on 2026-10-03 + +Mist uses Kubernetes `batch/v1` Jobs by default. The API submits actual +commands or uploaded scripts, reports real exit codes and scheduler messages, +retrieves pod logs, and cancels workloads. The CLI and Jobs page use these +endpoints. The optional `MIST_EXECUTOR=docker` path retains the older, +incomplete Docker/Redis supervisor for compatibility. + +Kubernetes owns placement, device accounting, and Job lifecycle. Mist validates +submission parameters and configured owner labels. Authentication, team +policy, credits, quotas, priority, and audit accounting are still application +work; the current API is a local single-owner pilot. + +```mermaid +flowchart LR + User[Jobs page / CLI] --> API[Mist API] + API --> Validate[Image, resources, script and deadline validation] + Validate --> Jobs[Kubernetes Jobs] + Jobs --> NVIDIA[CPU / NVIDIA server] + Jobs --> Claims[Tenstorrent DRA claims] + Claims --> TT[QuietBox boards via CDI] + Jobs --> State[Job and Pod status / logs] + State --> API + NVIDIA --> Artifacts[Node-local checkpoint PVCs] + TT --> Artifacts +``` + +See [installation, commands and acceptance tests](k3s-deployment-history.md#mist-api-cli-and-jobs-page) +and [job lifecycle and API endpoints](../../src/jobs.md). + +## Hardware and scheduling + +| Node | Workload resources | Allocation | +| --- | --- | --- | +| `utmist-z1opa08` | CPU and two NVIDIA RTX A4000 GPUs | `nvidia.com/gpu`, one or two whole GPUs | +| `utmist-tt` | Four n300 Wormhole boards, eight chips | DRA/CDI, one to four whole two-chip boards | + +CPU and NVIDIA jobs select the server because their pilot storage is local +to it. Tenstorrent jobs select QuietBox and reference one of four precreated +ResourceClaimTemplates. Each pod receives its own claim, not a shared claim +between jobs. Resources are released after completion or cancellation; +completed Jobs and checkpoints are retained. Requests wait when their +required CPU, RAM, hugepages, or devices are occupied. Scheduling does not +split a model or combine NVIDIA and Tenstorrent automatically. + +TT-Operator 0.3.0 runs Fabric Manager 0.2.30 and DRA driver 0.0.59 while +preserving host KMD 2.5.0 and firmware 19.4.2. Read-only host TT-Metal and +Python mounts make the runtime specific to this QuietBox. The vendor +ResourceSlice's memory capacity is incorrect; claims select board name, +chip count and board count, without using that capacity. + +The TT test trains each allocated chip independently using TT-NN forward +passes, analytic gradients and weight updates. For multiple boards, separate +runtimes select each board by PCI address through `TT_VISIBLE_DEVICES`. +Arbitrary board subsets need not form a usable connected mesh. Autograd, +distributed training of one model, and large production models are untested. + +## Verified behavior + +Live API acceptance checks passed for 17 jobs: + +- CPU success with actual output, exit-17 failure, deadline failure, and + cancellation with retained logs. +- Two simultaneous one-GPU NVIDIA jobs with distinct device nodes; a third + queued and automatically reused a GPU. A two-GPU job also trained. +- Four simultaneous one-board TT jobs with distinct claims and devices; + a fifth queued and trained after release. +- Two-board and four-board TT jobs trained on four and eight chips. +- Cancelling a four-board reservation released capacity for a waiting job. +- All successful accelerator runs met loss thresholds and reloaded their + checkpoints. Archives confirmed five NVIDIA and 24 TT checkpoint files + from those API checks after the pods completed. + +Separate infrastructure checks also verified that each TT job could open +its allocated board while device cgroups denied the other three, and that +allocation recovered after restarting the DRA driver. CLI and actual browser +checks exercise submission, status, logs and cancellation; backend/CLI/web +tests and a frontend production build cover the application changes. +See [recorded results](../../deploy/k3s/mist-api-results.json) for evidence paths. + +## Access and retention + +The API runs in `mist-system` with a dedicated service account and a Role +restricted to workloads in `mist`. The October 8 foundation adds a read-only +inventory ClusterRole for listing nodes, pods, ResourceClaims and ResourceSlices. +Secrets, node writes, pod exec, and creating jobs in other namespaces remain +outside its permissions. The October 3 checks predate this inventory role. Workload pods +receive no Kubernetes API token. The `mist` admission policy disallows +privileged execution and broad host device mounts. + +The service account can create workload Jobs, so RBAC alone does not make +it a complete security boundary. Admission policy, shared host hugepages, +shared checkpoint PVCs, and arbitrary script execution require further +hardening before accepting untrusted tenants. The local API has no +per-request authentication; `MIST_PILOT_OWNER` is configuration. + +API and web access bind to loopback, with no public ingress. A port-forward +exposes the deployed API. Retained Jobs carry submission/owner metadata and +cancelled logs across API restarts. There is no Job TTL. Normal logs depend +on retained pods and kubelet retention, and checkpoint PVCs are node-local. +Deleting those resources can remove history or artifacts. + +## Next milestones + +The October 8 local foundation implements approved custom CPU/NVIDIA container +execution, live Jobs/Machines inventory, and persistent per-job `/outputs` +mounts. See [the current guide and developer handoff](local-job-foundation.md) +and [execution plan](foundation-execution-plan.md) for fresh checks and the +agreed next foundation task: shared datasets and result downloads, followed +by team deployment/login. The milestones below cover the wider platform. + +1. Implement authentication and per-user/team authorization before remote + access; define quotas, concurrency limits, credits and priority policy. +2. Package the TT runtime into a reproducible training image and validate + real pilot-team models, topology and distributed training requirements. +3. Provide dataset upload and artifact access, shared/object storage, durable + metadata/log archival, cleanup policy, and datastore/PVC backups. +4. Add network restrictions, workload admission hardening, and operational + monitoring before opening the cluster to untrusted users. + +## References + +- [Kubernetes Job API](https://kubernetes.io/docs/concepts/workloads/controllers/job/) +- [Dynamic Resource Allocation](https://kubernetes.io/docs/concepts/resource-management/dynamic-resource-allocation/) +- [K3s storage](https://docs.k3s.io/storage) +- [Tenstorrent operator platform support](https://docs.tenstorrent.com/tt-operator/latest/platform-support.html) +- [Tenstorrent DRA driver](https://github.com/tenstorrent/tt-dra-driver) diff --git a/docs/archive/local-job-foundation.md b/docs/archive/local-job-foundation.md new file mode 100644 index 0000000..1e09e53 --- /dev/null +++ b/docs/archive/local-job-foundation.md @@ -0,0 +1,334 @@ +> **Historical record — superseded.** This text describes an earlier phase. +> Commands and claims below are retained as history, not current setup instructions. +> Start with the [current documentation index](../README.md) and +> [department operating guide](../department-rollout.md). + +# Local job foundation + + +**October 9 update:** Shared datasets/results, member login and private production +hosting are implemented. Earlier local pilot descriptions below record the +original phase. Use [the current operating guide](complete-foundation.md) for today's behavior. + +Implementation branch: `feat/local-job-foundation`. Recorded October 8, 2026 +(America/New_York). The preserved starting point is commit `2f9b178`. +The [execution plan](foundation-execution-plan.md) records scope and progress. + +## What runs now + +```text +Website :3001 / CLI + | + v +Mist API :3000 -- validates image, resources, and workload + | + v +k3s Job + pod -- Kubernetes schedules; containerd runs the image + / \ + v v +NVIDIA PC QuietBox +CPU / 2 GPUs 4 n300 boards / 8 chips + | | + v v +Per-job output folder on the selected node's persistent volume +``` + +Open **http://127.0.0.1:3001/jobs**. The **Machines** page shows the same real +inventory. Jobs support Python scripts, approved container images, and small +accelerator training checks. Dashboard/login and other prototype pages remain +separate work. Job execution uses Kubernetes, without SSH or a Docker daemon +inside the job. SSH is useful for administration and reaching the local website. + +The current API serves one configured owner, `utmist`. This is a trusted local +pilot. Membership, login, credits, quotas, public deployment, shared datasets, +and browser result downloads are subsequent tasks. + +## Use the website + +1. Check hardware availability at the top of **Jobs**. +2. Select **Compute**: CPU, NVIDIA, or Tenstorrent. +3. For accelerators, request whole GPUs or whole boards. A board has two chips. +4. Select **Workload** and an approved **Container image**. +5. For **Container image** workloads, optionally provide an executable and + arguments, one argument per line. Leaving them blank uses the image defaults. +6. **Resources and environment** contains CPU/RAM, deadline, working directory, + and `NAME=value` environment variables. Save files under `/outputs`. +7. Submit. The job card shows its ID, state, assigned machine, resources, + output path, and exit code. **Logs** reads actual container output. + **Cancel** stops a running or waiting job and preserves recent logs. + +An image must contain its own code and dependencies, or provide a runtime for +script mode. Requesting a GPU does not convert CPU code into accelerator code. +The form accepts an image reference; it does not upload/build Docker archives. + +### Included examples + +The current deployment approves and locally caches these two example images: + +| Image | Website settings | What it checks | +| --- | --- | --- | +| `mist-training:cpu-v1` | CPU; Container image; blank command/arguments | Packaged regression code and dataset, default entrypoint and `/app` working directory, saved/reloaded model | +| `mist-training:nvidia-v1` | NVIDIA; Container image; arguments below | CUDA training and checkpoint reload on each allocated GPU, without CPU fallback | + +For two GPUs, select **GPUs = 2** and enter these arguments, one per line: + +```text +--backend +nvidia +--devices +2 +``` + +For Tenstorrent, choose **Small training check** and one to four boards. This +uses the already tested TT image and installed host runtime. It trains a small +regression model on the chips of every reserved board. TT-NN gradients are +computed explicitly; this does not establish arbitrary PyTorch model or TT +autograd compatibility. Cross-machine/distributed training is separate work. + +## API contracts + +The [job lifecycle reference](../../src/jobs.md) describes state and retention. +The browser uses `/api`; Vite proxies it to the API. The API itself has these +unprefixed routes: + +| Route | Response | +| --- | --- | +| `GET /hardware` | `{observed_at, machines: [...], pools: [...]}` | +| `GET /images` | `{images: [{reference, accelerators}], profiles: [...]}` | +| `POST /jobs` | HTTP 201, `{job_id, job}` | +| `GET /jobs` | `{count, jobs: [...]}` | +| `GET /jobs/` | Actual job state, resources, placement and output paths | +| `GET /jobs//logs` | `{logs}` | +| `POST /jobs//cancel` | Retained job with state `Cancelled` | +| `GET /healthz` | Jobs API connectivity and executor | + +### Submission + +```json +{ + "name": "my-training-job", + "type": "command", + "accelerator": "nvidia", + "device_count": 1, + "image": "mist-training:nvidia-v1", + "args": ["--backend", "nvidia", "--devices", "1"], + "cpu": "2", + "memory": "2Gi", + "timeout_seconds": 600, + "env": {"GREETING": "my first job"} +} +``` + +| Field | Behavior | +| --- | --- | +| `name` | Optional display name, at most 128 bytes | +| `type` | `command` (default) or `training-smoke` | +| `accelerator` | `cpu` (default), `nvidia`, `tenstorrent` | +| `device_count` | CPU: 0; NVIDIA: 1–2; TT: 1–4 boards. Accelerator default is 1 | +| `image` | Exact server-approved reference. Required when using only image defaults | +| `command` / `args` | Optional arrays, at most 64 combined entries, each at most 8192 bytes. No implicit shell parsing | +| `working_directory` | Optional absolute container path. Custom CPU/NVIDIA containers preserve image WORKDIR when omitted | +| `script` / `script_name` | Optional Python/shell script, at most 32 KiB; filename only, `.py` or `.sh`. Cannot accompany `command` | +| `env` | Up to 32 entries, values at most 4096 bytes. Managed Mist/TT variables cannot be overridden | +| `cpu` | Kubernetes quantity, greater than zero and at most 8; request equals limit | +| `memory` | 64Mi–32Gi; TT minimum 2Gi; request equals limit | +| `timeout_seconds` | 10–86400; default 600; includes queue/startup time | + +`command` replaces the image ENTRYPOINT; `args` replaces CMD. With neither, +the image's entrypoint and arguments run. Scripts and training checks use +their runtime profile and default `/tmp` working directory. TT uses `/tmp` +and its host Python/TT-NN runtime unless explicitly configured otherwise. +To use shell syntax, request it explicitly, for example +`"command": ["sh", "-c"], "args": ["python train.py && echo done"]`. + +CPU defaults: 250m/256Mi. NVIDIA defaults: 2 CPU/2Gi. TT defaults: 2 CPU/4Gi +RAM plus 2Gi of 1Gi hugepages per board. Jobs have no retries. Images are +pulled with `IfNotPresent`; approval does not guarantee availability or +hardware compatibility. Invalid requests return HTTP 400 without creating +Jobs. Kubernetes failures include a message and actual exit code when available. + +### Hardware inventory + +Each machine reports `name`, internal `address`, `ready`, `schedulable`, +`cpu_allocatable`, and `memory_allocatable`. CPU/RAM are Kubernetes allocatable +capacity, not current utilization or remaining CPU/RAM. + +Each pool reports `accelerator`, `node`, `unit`, `total`, `allocated`, +`available`, `ready`, and optional `message` / `chips_per_device`. + +- NVIDIA: count scheduled nonterminal Pod reservations across all namespaces, + including bound Pending and Terminating pods and effective init/sidecar + requests. Unplaced queued pods do not consume a device. +- Tenstorrent: use current complete ResourceSlice generations and deduplicated + ResourceClaim allocations across all namespaces. Claims reserve boards + even before a pod starts. Administrative access claims do not consume devices. +- Unavailable/cordoned/tainted nodes or missing ready device drivers are not + advertised as ready. Incomplete TT inventory has `available: null`. +- Failed discovery returns HTTP 503. The UI marks availability unknown; + it does not invent zero allocations or free hardware. +- The known incorrect TT vendor memory capacity is ignored. + +The snapshot is assembled from separate Kubernetes reads and can change before +submission. Kubernetes remains the allocation authority. Active TT `devices` +details can disappear from a completed job once its claim is released; the +request count and saved `allocation.json` remain useful records. + +## Output persistence + +An init container creates a job-specific directory before the workload starts. +The workload mounts only this directory at two aliases: + +```text +Persistent volume / / + ├── model/checkpoint files + └── results.json + +Inside the workload: + /outputs MIST_OUTPUT_DIR + /checkpoints/ MIST_CHECKPOINT_DIR + MIST_JOB_ID +``` + +Both mount paths refer to the same files. `output_directory` is `/outputs` for +new jobs; old retained jobs keep their original `checkpoint_directory` contract. +The per-job directory is writable by the supported root/nonroot images. Its +permissions and shared backing PVC are not an authenticated tenant boundary. +Writes elsewhere in the container are not saved results. + +| Compute | PVC in `mist` | Physical node | +| --- | --- | --- | +| CPU | `mist-cpu-checkpoints` | `utmist-z1opa08` | +| NVIDIA | `training-nvidia-checkpoints` | `utmist-z1opa08` | +| Tenstorrent | `training-tenstorrent-checkpoints` | `utmist-tt` | + +Find the host directory using the administrative kubeconfig: + +```bash +k3s kubectl -n mist get pvc +k3s kubectl get pv +``` + +Read the selected PV's `spec.hostPath.path` and node affinity. On that node, +the files are under `//`. A read-only helper Pod +can also mount the corresponding PVC on its node. The verification tool below +does this after training has exited, records SHA256 hashes/results, and deletes +its temporary helper Pods. There is no browser file download endpoint yet. + +The local-path PVCs request 256Mi each; local-path does not enforce that as a +disk quota. Jobs/results are retained without TTL or automatic cleanup. +Normal logs are up to 64KiB and depend on pod/kubelet retention. Cancellation +stores up to 32KiB in Job annotations. API restarts preserve Kubernetes history. +Node/disk loss can lose outputs; shared storage and backups follow separately. + +## Development and deployment + +See the [deployment guide](k3s-deployment-history.md#mist-api-cli-and-jobs-page) +for image build/import and startup. Existing services on this machine are: + +```bash +systemctl --user status mist-api-forward mist-web +systemctl --user restart mist-api-forward mist-web +``` + +From a laptop with SSH access to the NVIDIA server, tunnel the local website: + +```bash +ssh -L 3001:127.0.0.1:3001 utmist@100.73.139.66 +``` + +Open `http://127.0.0.1:3001/jobs` on that laptop while the SSH connection is +open. A different laptop port, such as `-L 3002:127.0.0.1:3001`, avoids a local +port conflict. This does not require public ingress. + +The API runs in `mist-system` with namespaced workload permissions and a +read-only inventory ClusterRole: list nodes, pods, ResourceClaims and +ResourceSlices. It cannot read Secrets, exec into pods, update nodes, or create +workloads in other namespaces. The browser contains no Kubernetes credentials. + +Custom registry images require an administrator to update `MIST_ALLOWED_IMAGES` +with exact references, rebuild/reconfigure the API deployment as needed, and +verify image/code compatibility. Prefer immutable tags or digests. For a private +registry, create its image-pull Secret in workload namespace `mist`; set the +API's `MIST_IMAGE_PULL_SECRETS` to its name (or comma-separated names). +The API references the Secrets without reading their contents. Keep credentials +out of the job form, environment variables, and Git. K3s registry configuration +is a separate option for mirrors/private endpoints. + +### Code map + +| Area | Files | +| --- | --- | +| Request model and routing | `src/api.go`, `src/kubernetes.go` | +| Validation, Job construction, cancellation | `src/kubernetes.go` | +| Live inventory | `src/hardware.go` | +| Approved image/profile catalog | `src/images.go` | +| Output directory initialization | `src/outputs.go` | +| Batched job history reads | `src/job_states.go` | +| API types and client | `web-interface/src/api.ts` | +| Polling/recovery | `web-interface/src/hooks/usePolling.ts` | +| Jobs orchestration and form/cards/logs | `web-interface/src/components/JobsPage.tsx`, `components/jobs/*` | +| Shared inventory and Machines page | `components/HardwarePanel.tsx`, `components/MachinesPage.tsx` | +| Cluster deployment/RBAC | `deploy/k3s/mist-api.yaml` | +| Packaged examples | `deploy/k3s/examples/*` | + +### Verification + +```bash +go -C src test -run 'Test(Kubernetes|Hardware|GPURequest|CustomImage|TenstorrentCustom)' ./... +go -C cli test ./... +cd web-interface +npm test +npx tsc --noEmit +npm run build +``` + +The full legacy backend suite additionally requires its Docker images and +Redis. It is separate from the Kubernetes path. + +Fresh live checks require both accelerators initially free, the approved +example images imported, Playwright/Chromium installed, and admin kubectl access: + +```bash +export KUBECONFIG=/home/utmist/.kube/config +export MIST_PLAYWRIGHT_MODULE=/tmp/mist-browser-check/node_modules/playwright/index.mjs +node deploy/k3s/verify_foundation_browser.mjs /home/utmist/mist-foundation-results-new-run +python3 deploy/k3s/verify_foundation.py --artifacts /home/utmist/mist-foundation-results-new-run +``` + +Use one artifact directory for both commands and run them sequentially. +The browser check covers packaged CPU training, both NVIDIA GPUs, two TT +boards/four chips, real card states/logs, failure, and cancellation. The API +check covers occupied/free counts, queuing and reuse after cancellation for +both device types, command/argument semantics, and persistent files read from +new containers on both nodes. These tools submit real jobs and retain results. +Their final markers are `FOUNDATION_BROWSER_PASSED` and +`FOUNDATION_ALLOCATION_OUTPUTS_PASSED`. + +Today's detailed evidence is saved under +`/home/utmist/mist-foundation-results-2026-10-08`. The checked-in +[verification summary](../../deploy/k3s/foundation-results.json) records actual +job IDs, the deployed image ID, and completed checks. Browser and allocation/ +persistence checks passed on October 8 EDT (October 9 UTC). A separate live +check verified image-pull error reporting, deadline failure, and retained +history across API restarts. The normal image approval list was restored. + +Automated checks passed: 12 focused backend tests, the full src suite with its +local dependencies, CLI tests, 5 frontend interaction tests, type checking, +production build, and lint of changed frontend files. These are small model +execution checks, not a claim about every team model or distributed training. +See the execution plan's progress log and the developer handoff below. + +## Developer handoff: next work + +1. **Shared datasets and downloads (task 4):** storage on QuietBox, private + NFS/shared mount on both nodes, upload limits/progress, stable dataset IDs, + read-only inputs, job output listing/downloads, retention and backup rules. + Preserve the `/outputs` contract and migrate the checkpoint alias explicitly. +2. **Team deployment/login (task 5):** production web hosting with `/api` + routing, authenticated membership, job/file ownership, then team limits. +3. **Model/platform work:** portable TT image, real team models, scheduling + fairness/credits/priority, monitoring, Jupyter and chaining as separate work. + +Current Wi-Fi/LAN can support initial shared-storage tests. A second Ethernet +cable can improve throughput later. Do not remove QuietBox's existing router +link or format the NVIDIA machine's unmounted disk as part of this work. diff --git a/docs/archive/shared-storage-deployment-plan.md b/docs/archive/shared-storage-deployment-plan.md new file mode 100644 index 0000000..2233be7 --- /dev/null +++ b/docs/archive/shared-storage-deployment-plan.md @@ -0,0 +1,251 @@ +> **Historical record — superseded.** This text describes an earlier phase. +> Commands and claims below are retained as history, not current setup instructions. +> Start with the [current documentation index](../README.md) and +> [department operating guide](../department-rollout.md). + +# Complete foundation: parts 4 and 5 + +Authorized October 8, 2026 after completion of parts 1–3. Continue on +`feat/local-job-foundation` from `8f1b7f1`. The user requests finishing the +remaining foundation instead of handing it to other developers. No permission +prompts are needed for this work. Preserve existing jobs, checkpoints, network +links and unrelated disk contents. + +## Remaining scope + +### Part 4: shared datasets and result downloads + +- [x] Create `/srv/mist-storage` on QuietBox's existing NVMe filesystem. +- [x] Provide NFSv4 shared storage to the two current Kubernetes nodes, over + their existing LAN. Keep QuietBox's router Ethernet cable connected. +- [x] Install the NFS host client/server packages and persistent configuration. +- [x] Add shared persistent volumes and mount storage in the API. +- [x] Upload datasets through the API and website with size limits/progress, + stable dataset IDs, checksums, ready/error handling, and atomic publication. +- [x] Select a dataset for a job; attach only that dataset read-only at `/inputs`. +- [x] Store new job outputs on shared storage; preserve the `/outputs` and + checkpoint aliases. Keep old local checkpoints retrievable. +- [x] List and download saved job files with ownership checks, streaming, and + protection against traversal and escaping symlinks. Do not execute archives. +- [x] Verify an uploaded dataset is consumed by jobs on both physical machines, + with actual accelerator computation and persistent downloadable results. + +Use a bounded pilot budget (initially 100GiB for new datasets/outputs), an +upload limit (initially 2GiB per dataset), and explicit cleanup/retention docs. +NFS is central storage, not a backup. Preserve the existing checkpoint PVCs. +Metadata must not be writable by workload containers through output mounts. + +### Part 5: login and private deployment + +- [x] Use maintained Better Auth for separate Mist member accounts. Keep the + existing Go executor and add a small auth service with persistent SQLite on + a node-local volume, not SQLite on NFS. +- [x] Seed the first administrator from protected local credentials, outside + Git/tool output. Disable public signup; administrators add members. +- [x] Authenticate API access, derive job/dataset ownership from the session, + and verify isolation with two actual member sessions. +- [x] Add login/logout, current account, password change and member management + UI. Remove prototype profile/account behavior from the active workflow. +- [x] Serve the production frontend with same-origin API/auth proxying and + persistent services; avoid relying on Vite for the deployed website. +- [x] Make the site accessible through the existing private Tailscale network. + Use HTTPS through Tailscale Serve if already available; otherwise keep a + private Tailnet endpoint and record the HTTPS prerequisite accurately. +- [x] Add startup, restoration, membership, image approval, storage quota, + backup and cleanup documentation and repeatable deployment scripts. +- [x] Verify browser login → upload → select dataset/image/device → run → logs + → download, from both machines, plus denial of anonymous/other-owner access. +- [x] Commit tested changes and record exact deployed image/evidence paths. + +## Decisions + +- Keep image-reference submission and current NVIDIA/TT allocation units. + Arbitrary TT models, distributed training, credits, Jupyter, chaining and + image build/archive upload services are outside the five-part foundation. +- Deploy privately; no public domain, Funnel, paid service or external account + provisioning is implied. Use existing host and cluster access. +- A shared tailnet infrastructure account does not identify individual team + members; Mist accounts provide independent ownership. +- Existing owner `utmist` and retained jobs remain visible to the initial + administrator through an explicit migration mapping. +- HTTP cookies alone are not authorization. Go verifies sessions against the + auth service, and state-changing routes require trusted browser origins. +- Test first, then replace the deployed API/frontend with the verified build. + +## Resume information + +- Go/Node runtimes: `~/.local/share/mist-runtimes/go-1.25.1/bin/go` and + `~/.local/share/mist-runtimes/node-22.16.0/bin/node`. +- Kubeconfig: `/home/utmist/.kube/config`; k3s nodes `utmist-z1opa08`, `utmist-tt`. +- QuietBox SSH: `utmist-tt@100.95.175.37`, identity + `/home/utmist/.ssh/id_ed25519_quietbox_codex`, IdentitiesOnly/BatchMode. +- Authorized host operations can use Docker bind/chroot. Package installation + needs host networking because host systemd-resolved listens on loopback. +- Current API deployed in `mist-system`; workloads in `mist`. +- Main tailnet hostname: `utmist.tail459b5e.ts.net`; IP `100.73.139.66`. +- Baseline evidence: `deploy/k3s/foundation-results.json`, + `/home/utmist/mist-foundation-results-2026-10-08`. + +## Progress + +- Plan recorded; repository clean at start. QuietBox SSH and authorized Docker + host access work. QuietBox still has approximately 1.9TiB available. +- Tailscale is connected; no Serve configuration exists. Existing Nginx is + active. Preserve its current default service while adding the Mist endpoint. + +## Completed verification (October 9 UTC) + +- Actual production browser flow passed: login, dataset upload, CPU and both NVIDIA + GPUs, one TT board/two chips, logs, saved file download, account/member actions, + ownership isolation and deactivation. No browser runtime errors. +- QuietBox submitted an authenticated NVIDIA training job, which ran on the main + node and returned downloadable loss metrics. +- Focused Go tests (including race checks), CLI tests/login/logout, frontend + interaction tests, TypeScript and production build passed. +- Old checkpoint PVCs preserved; legacy outputs copied to shared storage. +- NFS/client firewall configuration and production restart/backup checks are + documented in `docs/complete-foundation.md` and `deploy/private/README.md`. +- Private HTTP website is live. Tailscale Serve is disabled; HTTPS requires the + tailnet administrator to enable Serve, accurately recorded as a prerequisite. + +Final review: frontend dependencies now use the checked-in npm lockfile; the +obsolete Bun lockfile was removed. CLI login/logout use real sessions, and +`job submit --dataset` attaches an owned dataset. SQLite backup integrity was +`ok`; restart checks preserved historical/new jobs, datasets and downloadable +results. The production website does not depend on a terminal session. + +## Department rollout scope decision — October 9, 2026 + +The user plans to introduce Mist to several research teams next week and has +explicitly excluded these additions from that rollout: + +- Automatic backups and additional recovery testing. +- HTTPS/Tailscale Serve enablement. +- Monitoring and storage cleanup features. + +The remaining proposed priorities are team membership/permissions and workload +isolation, team resource quotas, fair queuing, representative research workload +and capacity tests, and a documented self-service container-image workflow. +These are proposed follow-up work, not completed foundation features. + +The current access model remains private HTTP over the existing Tailscale +network. Existing manual operating procedures are documented in the foundation +guide. Credits, special job priorities, Jupyter, chaining, and browser image +upload/build services remain outside the immediate rollout scope. + +### Team self-service clarification + +The user clarified that administrators assign each team's conditions and limits; +teams normally do not request administrator permission for individual actions. +The intended rollout model is: + +- Administrators configure membership, permissions and team resource policies. +- Team members upload datasets, submit compatible container images/references, + run jobs and retrieve results independently within their assigned policies. +- The platform validates those policies and admits/queues requests automatically; + ordinary submissions do not require a human approval step. +- Replace the proposed per-image administrator approval process with a documented + self-service image policy. Define allowed image sources and runtime compatibility + as platform/team rules rather than approving each new image manually. + +Current foundation behavior remains distinct: individual owned jobs are already +self-service, but image eligibility uses a global exact-reference allowlist and +Tenstorrent uses the fixed tested runtime profile. Team policies and self-service +custom image eligibility are follow-up implementation work. + +### Team membership and storage requirements + +The user requires multiple teammates per team, explicitly enrolled as members. +Each team needs one team storage area containing a common folder and a folder +for each member. The user clarified that member folders are visible within the +team; they are not private from teammates: + +- Authorized team members can browse/read the team's common folder and the + individual member folders within that team. +- Member folders organize each teammate's files; they do not create a separate + privacy boundary within the team. Write/delete permissions need an explicit + policy and are not implied by permission to view files. +- Members of another team cannot discover, list, read, download, modify or use + the team's files through jobs unless explicitly granted access. +- Explicit cross-team grants must specify their recipients, storage scope and + permitted actions; grants must be revocable and checked on subsequent access. +- All dataset/file access and job input/output attachment must verify the + authenticated member, team membership or explicit grant, and selected scope. +- Workloads mount only their authorized selected input and output locations. + File and network access from running containers must enforce the same boundary; + a UI filter or a differently named folder is insufficient isolation. +- Administrators assign memberships and team conditions. Normal uploads and + submissions within those conditions remain self-service. + +These are new team features. The completed foundation currently provides +per-account ownership checks, rather than this team-sharing/membership model. + +Example intended organization: + +```text +storage/ + team-a/ # visible to Team A members + common/ + alice/ + bob/ + team-b/ # visible to Team B members + common/ + carol/ + dan/ +``` + +Folder names illustrate the organization; enforced authorization determines +access. Cross-team sharing exposes only the explicitly granted scope. + +### Portal login and administrator workspace + +The same portal and login page serve administrators and regular members. +Authenticated roles determine which controls appear and which API actions are +authorized; there is no separate administrator account required for ordinary +research work. + +- Regular members use their team workspace to upload files, submit jobs, view + logs and retrieve results within the team's policies. +- Administrators have the same research workflow plus administration controls + for teams, memberships, resource limits, image policies and access grants. +- An administrator can also be a team member. Ordinary uploads and jobs use the + selected team's membership, storage permissions and resource policies. +- Administrative actions must be explicitly authorized on the server; hiding + controls in the UI alone is insufficient. Administrative authority is distinct + from membership and does not implicitly make the account a member of every + team or bypass limits on its own research jobs. + +The foundation already supports login, basic member administration and ordinary +administrator jobs. The team workspace and additional team administration +controls described here remain follow-up work. + +### Rollout review: implementation details still to define + +The main workflow is covered by the proposed rollout scope. These details belong +within that work; they are not completed features or authorization to implement +every proposed default: + +- File permissions: distinguish team-wide read access from write/delete access. + Proposed default is members managing their own folders, with explicitly + assigned write permissions for the common folder and cross-team grants. +- Membership and grant revocation: deny subsequent access immediately and define + how to handle queued/running jobs that already hold access. Revoking a browser + permission does not remove a file mount from an existing running job. +- Storage quota enforcement: count datasets and generated results, including + writes from running containers. Upload-only checks do not enforce a team cap. +- Failure handling: expose useful reasons for image pull failure, full storage, + unavailable devices and offline nodes; define cancellation, runtime limits and + retry behavior so jobs release resources and do not silently duplicate work. +- Rollout capacity and migration: verify representative dataset/model sizes and + GPU/TT runtime compatibility, and preserve existing account-owned files/jobs + when introducing team ownership. Current pilot limits are 2GiB per upload and + 100GiB total new shared storage. +- Private container registries: decide whether launch supports private images; + if supported, credentials must be team-scoped and unavailable to other teams + and their workloads. Public image references do not need registry credentials. + +Account administration should also define an administrator-assisted password +reset path; self-service email recovery and external email provisioning are not +implied. Automatic backups, HTTPS, monitoring and cleanup remain excluded as +requested. Multi-machine distributed training is outside the existing foundation +and should not be represented as tested by the two separate accelerator smokes. diff --git a/docs/complete-foundation.md b/docs/complete-foundation.md index 7f91f6f..d2ad790 100644 --- a/docs/complete-foundation.md +++ b/docs/complete-foundation.md @@ -1,222 +1,7 @@ -> Current department storage and portal settings: [department rollout](department-rollout.md). -> This document records the earlier foundation; its 100 GiB pilot was later expanded sparsely. +# Historical guide -# Mist foundation: historical operating snapshot +This phase has been superseded by the department platform on main. -> **Historical account-only snapshot.** The [department rollout](department-rollout.md) -> supersedes these instructions: team membership, scoped folders, enforced quotas, -> self-service images and the new portal are implemented. Use that guide and -> [current deployment operations](../deploy/private/README.md) for current behavior. -> The remaining sections record the preceding foundation, including its limitations. - -Implemented on `feat/local-job-foundation`, October 8–9, 2026. This continues the -[original foundation](local-job-foundation.md) with shared datasets, output -retrieval, separate member accounts and persistent private hosting. - -## Use the website - -Open **http://100.73.139.66:8088** from a device on the existing Tailscale network. -The alternative MagicDNS address is http://utmist.tail459b5e.ts.net:8088. - -1. Sign in with a Mist account, independent of the machine's Linux/SSH login. -2. Open **Datasets**, upload one file or a ZIP, and wait for **Ready**. -3. Open **Jobs**, choose CPU, NVIDIA GPUs or Tenstorrent boards, an approved - image, and optionally the uploaded dataset. Submit a script or container. -4. Follow the actual status, placement and logs. Save files under `/outputs`. -5. Click **Files** on the job to download results. Results remain after exit. - -The initial administrator's credentials are stored locally in -`/home/utmist/.config/mist/admin-bootstrap.json` (mode 0600), outside Git. View -that file locally, sign in, then change the password on **Account**. The file -contains a bootstrap password, not a password that automatically tracks changes. -Do not distribute its auth secret. Administrators add members on **Account**; -members can change passwords there. Deactivation revokes sessions. Public signup -is disabled. There is no email delivery/password-reset email service configured; -an administrator can reset a member password through Better Auth's authenticated -`/auth/admin/set-user-password` endpoint. Admin roles are managed by that service, -never by editing one's profile. - -**HTTPS status:** Tailscale Serve is disabled on this tailnet. Its CLI reports -that an account administrator must enable Serve in the Tailscale console. The -current HTTP endpoint is bound to localhost and the Tailscale IP only. Remote -traffic uses Tailscale's encrypted network, but browsers do not treat HTTP as a -secure origin. No public Funnel/domain was enabled. After enabling Serve, run -`tailscale serve --bg --https=443 http://127.0.0.1:8088`, change -`MIST_SITE_ORIGIN` in the auth deployment to `https://utmist.tail459b5e.ts.net`, -and restart auth so cookies use Secure. The HTTPS origin is already in the -trusted-origin lists. Check Serve status and browser login before distributing -that URL. Do not claim HTTPS is available before that verification. - -## Running architecture - -```text -Team browser / CLI / either SSH host - │ private Tailscale network - ▼ -Main node Nginx :8088 ── static production React build - │ /api/* and /auth/* - ▼ -Go Mist API, Kubernetes Deployment (mist-system) - ├── Better Auth service ── local SQLite PVC on main node - ├── dataset metadata/uploads/results ── shared NFS PVC - └── Kubernetes Jobs (mist namespace) - ├── Main node: CPU or 1–2 NVIDIA GPUs - └── QuietBox: 1–4 TT boards, two chips per board - │ - selected /inputs (read-only), own /outputs (writable) - │ - QuietBox /srv/mist-storage (NFSv4 over current LAN) -``` - -Nginx proxies directly to the existing API ClusterIP `10.43.1.116:3000`; -production does not rely on Vite, SSH sessions or the development port-forward. -The local preview at `127.0.0.1:3001` remains available and now uses real login. -Kubernetes schedules jobs using the requested resources and node profiles. -TT allocations use DRA boards. NVIDIA allocations use whole GPUs. Neither device -sharing within a GPU/board nor distributed mixed-backend training is implemented. - -## Storage layout and limits - -QuietBox exports `/srv/mist-storage` to only LAN IPs `10.0.0.175` and -`10.0.0.112`, using NFSv4 and root squash (anonymous UID/GID 65532). UFW admits -TCP 2049 from those two addresses. Its router Ethernet connection stays intact. -The main machine continues using its existing Wi-Fi connection. - -A dedicated **100 GiB sparse ext4 image** on QuietBox's existing NVMe backs the -store. No physical disk was formatted. Actual usable capacity is slightly smaller -because of filesystem overhead. `/etc/fstab` mounts it at boot; `nfs-server` -is enabled. Keep the NVMe's free space above this pilot budget. - -```text -/srv/mist-storage/ - metadata/datasets/.json owner, checksum, size, ready state - .uploads// staging, removed on failure/completion - datasets//content/ selected input files, read-only job mount - jobs//outputs/ job's own persistent results - legacy/{cpu,nvidia,tenstorrent}// copied original results -``` - -The upload limit is 2 GiB per file/archive and 2 GiB expanded. ZIPs reject unsafe -paths, symlinks, special files, duplicate files and more than 10,000 entries. -Uploads stream to staging, are checksummed and published atomically. A failed -upload never becomes Ready. Metadata, unrelated jobs and other datasets are not -mounted into a workload. No archive is executed by the upload service. - -`/inputs` and `MIST_INPUT_DIR` identify an attached dataset. `/outputs` and -`MIST_OUTPUT_DIR` identify persistent outputs; `/checkpoints/` remains an -alias. Jobs without a selected dataset have no `/inputs` mount. Image entrypoints -and working directories remain unchanged unless explicitly overridden. - -The budget applies to all shared files together, not separately to each PVC. -Two static RWX PV/PVCs expose the same backing store to different namespaces. -A PVC request alone is not a hard quota; the image's filesystem enforces capacity. -Upload admission leaves 1 GiB spare, but running jobs can still fill storage: -write errors must be handled by job code. Per-member/job disk quotas are future -work. Do not store SQLite on NFS. - -Original local checkpoint PVCs remain intact. Their completed outputs were -copied to `legacy/` and remain downloadable through retained job IDs. New jobs -use shared storage. QuietBox is now a storage dependency: if it is offline, mounts, -uploads, downloads and jobs that need those files may wait. NFS is not a backup. - -### Cleanup and retention - -Nothing deletes datasets or results automatically. Delete unneeded datasets -through the website; active-job references prevent deletion. For job results, -first verify the owning job has finished and no pod can write to it, then archive -and remove only that job's directory from `jobs//outputs` on QuietBox. -Keep the parent directory for retained Kubernetes Job history. Do not delete -storage metadata, the mounted image file, or entire legacy PVC directories. -Monitor `df -h /srv/mist-storage` on QuietBox. Remove only test output archives -that have already been saved elsewhere. Legacy copies use additional space; -original PVCs are intentionally not removed by this migration. - -## Authentication and ownership - -Better Auth 1.7.7 handles passwords, sessions, database migrations, role checks, -rate limiting and revocation. Go verifies each session with the auth service; -unavailable auth returns 503, anonymous access 401. Browser mutations require -trusted origins. Cookies are HttpOnly and SameSite=Lax; Secure requires the HTTPS -configuration described above. Session duration is seven days. - -New members receive stable owner labels derived from their auth user ID. Existing -`utmist` jobs are mapped to the initial administrator's configured email. Other -members cannot list, inspect, cancel, read logs from or download that owner's -jobs, and cannot use/delete its datasets. Hardware totals are visible to members -because those devices are shared. Administrators' membership permissions do not -automatically give access to another member's job files. - -This is a private, trusted team pilot. Approved containers, TT's read-only host -runtime mounts and NFS Unix permissions do not constitute a hardened hostile -multi-tenant sandbox. Public hosting, hostile code isolation, member quotas, -credits, priority/fair queues, Jupyter and job chaining require later work. - -## CLI - -```bash -export MIST_API_URL=http://100.73.139.66:8088/api -bin/mist-cli auth login --email your-member-email@example.org -bin/mist-cli job list --all -bin/mist-cli job submit train.py --compute NVIDIA --devices 1 -bin/mist-cli auth logout -``` - -Login prompts without echoing the password; `--password-stdin` supports controlled -automation. Cookies are saved atomically in a mode-0600 CLI config. Logout revokes -the session. Fake legacy tokens are no longer used. An alternate config can be -selected with `--config`; never paste a session cookie into a public document. - -## API additions - -All data routes require an authenticated cookie when `MIST_AUTH_URL` is configured. - -| Route | Function | -|---|---| -| `GET /session` | Auth enabled flag and current member; no session token | -| `/auth/*` | Same-origin proxy to maintained Better Auth endpoints | -| `GET /storage` | Actual capacity/free bytes and upload limit | -| `GET /datasets` | Current owner's ready datasets | -| `POST /datasets?filename=data.csv&name=Example` | Raw streaming file upload | -| `POST /datasets?filename=data.zip&format=zip` | Checked ZIP upload/extraction | -| `GET /datasets/` | Own ready dataset metadata | -| `DELETE /datasets/` | Delete own dataset if no active job references it | -| `POST /jobs` with `dataset_id` | Mount own ready dataset read-only | -| `GET /jobs//files` | Own job's regular result files | -| `GET /jobs//files/download?path=model.pt` | Stream attachment/download | - -The website prefixes Go routes with `/api`; auth stays at `/auth`. Listing limits -outputs to 5,000 files/10,000 entries/20 directory levels. Download paths use Go's -`os.Root` to prevent traversal and escaping symlinks. Save plain regular files -inside `/outputs` if you want them to be listed/downloaded. - -## Deployment, restart and rollback - -See [deployment scripts](../deploy/private/README.md). k3s server/agent, NFS and -Nginx are system services. Kubernetes restores API/auth deployments. The frontend -is a static release, with `/srv/mist-web/current` pointing at the active build. -Deployment configuration, RBAC, claim templates and image policies are checked -in; bootstrap credentials and cluster tokens are not. - -Rollback frontend by repointing `current` to an earlier retained release and -reloading Nginx. Rollback an API image with `kubectl set image` after importing -that image. Keep auth, shared storage and login enabled during rollback; the old -unauthenticated foundation API is not a suitable private member-service rollback. -Database migrations require a pre-upgrade backup and version-specific review. - -## Verification - -Repeatable checks: - -- `deploy/k3s/verify_complete_browser.mjs`: actual login, dataset upload, CPU, - two NVIDIA GPUs and a TT board consuming the same uploaded dataset, downloads, - member creation/password change/isolation/deactivation, denied anonymous signup - and cross-origin writes. Uses protected local credentials; logs no secrets. -- `deploy/k3s/verify_quietbox_submission.py`: authenticated submission from - QuietBox, NVIDIA training assigned to the main machine, downloaded loss metrics. -- `src/shared_storage_test.go`: upload limits/atomic publication, ZIP attacks, - read-only dataset binding, escaping download symlinks, ownership and auth outage. -- Existing job/device/queue/cancellation/image tests remain in the repository. - -Detailed evidence is under `/home/utmist/mist-complete-results-2026-10-09`; -final verified results are recorded in `deploy/k3s/complete-results.json`. An accelerator smoke test proves this small -training workload; it does not prove arbitrary model/runtime compatibility. +- [Current documentation](README.md) +- [Current operating guide](department-rollout.md) +- [Archived original record](archive/complete-foundation.md) diff --git a/docs/department-rollout-execution.md b/docs/department-rollout-execution.md index 9dc8574..7e532b0 100644 --- a/docs/department-rollout-execution.md +++ b/docs/department-rollout-execution.md @@ -162,9 +162,11 @@ deployment, limits, design and verification. The historical `main` tip [`archive/main-before-k3s-20261009`](https://github.com/UTMIST/Mist/tree/archive/main-before-k3s-20261009). The department implementation is prepared for main in [PR #110](https://github.com/UTMIST/Mist/pull/110), using a squash merge. -The initial merge attempt was rejected by GitHub branch policy; main has not -yet been updated. GitHub requires one other reviewer to approve the latest push; -this account has maintain/push access and cannot bypass that rule. The live +The initial merge attempt required an independent review. PR #110 was then +merged on October 9 at 08:06:48 UTC as commit +`3431ffdb713fac72881fc7f61f2750a82fe3f916`; main now contains the verified release. +The account used by this agent has maintain/push access, and the repository still +requires an independent reviewer for subsequent pull requests. The live private portal already runs the verified implementation. Git publication does not reset accounts, jobs or files, or switch the live services again. diff --git a/docs/department-rollout.md b/docs/department-rollout.md index 2a42801..c30ea1c 100644 --- a/docs/department-rollout.md +++ b/docs/department-rollout.md @@ -1,9 +1,9 @@ # Mist department rollout -Implemented October 9, 2026 on `feat/local-job-foundation`. This guide supersedes -the account-only workspace instructions in [complete foundation](complete-foundation.md). -The older `main` preview and original accounts, jobs, datasets and checkpoint PVCs -are preserved. +Merged into **main** through [PR #110](https://github.com/UTMIST/Mist/pull/110) on +October 9, 2026. This is the current operating guide. Original accounts, jobs, +datasets and checkpoint PVCs are preserved; earlier instructions are in the +[historical archive](archive/README.md). ## Use the portal @@ -297,7 +297,7 @@ this service. The portal remains private HTTP; no Serve, Funnel or public hostin export MIST_API_URL=http://100.73.139.66:8088/api bin/mist-cli auth login --email your-mist-email bin/mist-cli team list -bin/mist-cli team use team-db3933031a6c2437 +bin/mist-cli team use TEAM_ID_FROM_LIST bin/mist-cli job submit train.py --compute NVIDIA --devices 1 bin/mist-cli job list --all bin/mist-cli auth logout @@ -358,8 +358,8 @@ The image must contain `/app/train.py`; inputs mount at `/inputs`, outputs at `/outputs`. Submit scripts with `script` and `script_name` instead. NVIDIA/TT requests also specify `device_count`; TT counts **boards**, not chips. The request and response types are in `web-interface/src/api.ts`, and authority lives in Go, -not browser validation. The server returns 201 for new teams/datasets, 200 for -submitted jobs, and JSON errors for rejected requests. +not browser validation. The server returns 201 for newly created teams, datasets and jobs, and JSON +errors for rejected requests. ## Deploy and hand off diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..e92e469 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,76 @@ +# Developer setup + +Use Go **1.25.1** and Node.js **22.16.0 or newer**. Run repository commands from +`/home/utmist/Mist` on main, or your own checkout on a development machine. + +On the existing main machine, the pinned runtimes are available at: + +```bash +export PATH=/home/utmist/.local/share/mist-runtimes/go-1.25.1/bin:/home/utmist/.local/share/mist-runtimes/node-22.16.0/bin:$PATH +go version +node --version +``` + +## Frontend against the existing authenticated API + +Main's `mist-api-forward` user service already forwards the API to +`127.0.0.1:3000`; `mist-web` runs a local Vite preview on `127.0.0.1:3001`. +These are development conveniences. The deployed Nginx portal runs independently. + +```bash +systemctl --user status mist-api-forward mist-web +``` + +If those ports are free, use two terminals on main: + +```bash +export KUBECONFIG=/home/utmist/.kube/config +k3s kubectl -n mist-system port-forward service/mist-api 3000:3000 --address 127.0.0.1 +``` + +```bash +npm --prefix web-interface ci +npm --prefix web-interface run dev -- --host 127.0.0.1 --port 3001 --strictPort +``` + +Open http://127.0.0.1:3001 and sign in with your Mist account. This points to real +accounts, team storage and hardware; submitting jobs or changing policies affects +the installation. Do not start duplicate listeners while the user services run. + +For a UI on another development machine, set `MIST_API_PROXY` to an API endpoint +reachable from it. Configure its exact browser origin in the API/auth trusted +origins before relying on session requests; Tailscale connectivity alone does +not add a trusted browser origin. + +## Backend and CLI + +The root `go.work` joins `src` and `cli`; it replaces the unused root module. + +```bash +go test ./src/... ./cli/... -count=1 +go vet ./src/... ./cli/... +go -C src build -o ../bin/mist-api . +go -C cli build -o ../bin/mist-cli . +``` + +`NewKubernetesApp` exposes the real API. Unit tests use fake Kubernetes clients +and temporary small files, so they need neither Redis nor Docker. Live verifiers +use the existing authenticated installation and protected credentials. + +For backend changes, use an isolated test cluster/namespace with its own auth +and storage, or the documented maintenance deployment of this installation. +Standalone auth-disabled mode remains an explicit trusted development option; +it does not reproduce department authentication/team isolation. Never deploy it +for research users. `MIST_EXECUTOR=docker` is no longer supported and fails at +startup instead of selecting the retired supervisor. + +## Deploying the existing installation + +```bash +bash deploy/private/build-and-deploy.sh +``` + +This requires existing administrative kubeconfig, Docker/root access, auth +Secrets, operators and mounted storage. It is not a fresh-machine installer. +Read [operations](../deploy/private/README.md) and [testing](testing.md) first. +Current user/admin behavior is in [the operating guide](department-rollout.md). diff --git a/docs/foundation-execution-plan.md b/docs/foundation-execution-plan.md index 9f35305..130c7c5 100644 --- a/docs/foundation-execution-plan.md +++ b/docs/foundation-execution-plan.md @@ -1,385 +1,7 @@ -# Mist foundation execution plan +# Historical guide -Recorded: October 8, 2026. This file preserves the agreed scope and should be -updated with implementation progress, test evidence, and remaining work. +This phase has been superseded by the department platform on main. -## Goal and decisions - -Make the local website usable for running container jobs on the existing -NVIDIA and Tenstorrent machines. Start with the working Kubernetes pilot, -rather than rebuilding execution. The user has requested implementation on -a new branch, organized code, and proper documentation. - -Today's scope is tasks 1–3 plus persistent output folders and verification. -Shared dataset storage, uploads, and result downloads are the next foundation -task. Website deployment and login follow the foundation. - -Decisions already made: - -- Develop and test locally. Team members can use the CLI over SSH or open the - local website through an SSH tunnel. A public website is unnecessary now. -- Mist API creates Kubernetes Jobs; k3s schedules them; containerd runs the - images. Normal job execution does not use SSH. -- Images are built and pushed to a registry, then selected by reference. A - browser image archive uploader or image build service is outside today's scope. -- CPU and NVIDIA support approved custom images. Tenstorrent initially uses - its existing tested image and host runtime profile. -- NVIDIA allocations are whole GPUs. Tenstorrent allocations are whole n300 - boards, with two accelerator chips per board. -- Start with persistent node-local results folders. Add shared storage later. -- For shared storage, the preferred initial host is Tenstorrent because it has - more free disk space. NFS over the existing network is the simplest proposal. -- Wi-Fi is adequate for initial testing. A future second Ethernet cable can - provide a dedicated storage link. Preserve Tenstorrent's existing router link: - its k3s configuration and network access currently depend on it. -- Preserve existing work and Git history. Implement on a new branch. Retiring - or merging the existing main branch is a later repository decision. -- No permission prompts are needed for the agreed implementation and tests. - Do not introduce unrelated public exposure, account changes, or disk formatting. - -## Architecture - -```text -Local website / CLI - | - v - Mist API - | - v - k3s cluster - / \ - v v -NVIDIA PC Tenstorrent -containerd containerd - | | - v v -Job container + allocated accelerator + persistent results folder -``` - -The full whiteboard plan also includes membership/authentication, credits, -priority policy, a storage box, Jupyter, job chaining, and optional agents. -Kubernetes supplies placement and device allocation. Mist supplies admission, -user policy, UI, dataset handling, and artifact access. Selecting an accelerator -does not convert CPU code into CUDA or Tenstorrent code, or distribute one model -across machines automatically. - -## Current baseline - -Repo: `/home/utmist/Mist`. Before this implementation, the branch is -`fix/local-runtime-setup` and contains substantial uncommitted k3s pilot work. -Preserve these changes when creating the new branch. - -An independent main preview exists at `/home/utmist/Mist-main-preview`. -Use main's reusable UI/components/types where helpful. Its Jobs data and several -actions are prototypes, and its legacy Docker/Redis executor is incomplete. - -| Machine | Kubernetes node | Tailnet IP | Cluster LAN IP | Resources | -| --- | --- | --- | --- | --- | -| Main/NVIDIA | `utmist-z1opa08` | `100.73.139.66` | `10.0.0.175` | 2 NVIDIA RTX A4000 GPUs | -| QuietBox | `utmist-tt` | `100.95.175.37` | `10.0.0.112` | 4 n300 boards / 8 Wormhole chips | - -Both nodes report Ready with k3s `v1.36.4+k3s1`. Workload namespace is `mist`; -API namespace is `mist-system`. - -- API already runs inside k3s. Local access is `http://127.0.0.1:3000`. -- Working Jobs page: `http://127.0.0.1:3001/jobs`. -- Vite proxies `/api` to the API; this proxy does not ship in the static build. -- User services: `mist-api-forward.service`, `mist-web.service`. -- Main preview is separate: UI port 3011, API port 3010. -- Existing Jobs page supports scripts, accelerator counts, training checks, - real states, logs, and cancellation. -- API is a trusted local, single-owner pilot. Owner is configuration, not an - authenticated identity. Authentication and tenant isolation remain work. -- Existing checkpoints are in three node-local `local-path` PVCs. Their declared - request is 256 MiB each; this is not a meaningful dataset storage budget. -- Shared storage and dataset upload/download services have not been configured. - -Storage readings taken during planning: - -| Machine | Filesystem capacity | Used | Available | -| --- | --- | --- | --- | -| NVIDIA | 439 GiB | 87 GiB | 330 GiB | -| Tenstorrent NVMe | 3.6 TiB | 1.6 TiB | 1.9 TiB | - -NVIDIA also has an unmounted ~447 GiB `/dev/sdb`. Its contents are unknown; -do not treat it as empty or format it as part of this plan. - -NVIDIA currently reaches QuietBox over Wi-Fi via the router. QuietBox's active -router Ethernet link negotiates 1 Gbps. Both machines have unused 10 Gbps -Ethernet hardware; a future direct link requires compatible ports/cable and -verification. Being on the same tailnet or k3s cluster does not share disks. - -## Implementation order - -### A. Establish the branch and API contracts - -- [x] Create a new foundation branch from the current working checkout. -- [x] Preserve existing pilot changes and inspect main for useful components. -- [x] Separate inventory and submission validation from HTTP/UI presentation. -- [x] Document new response/request shapes before connecting UI controls. -- [x] Keep changes reviewable and avoid unrelated rewrites. - -### B. Task 2: hardware discovery and availability - -Proposed endpoint: `GET /hardware`. Publish the final contract with the code. - -- [x] Report node readiness, accelerator type, allocation unit, total, - allocated, and currently available counts. -- [x] Count NVIDIA allocations from scheduled, nonterminal Kubernetes pods, - including workloads outside Mist. Do not subtract every submitted job. -- [x] Count Tenstorrent devices from ResourceSlices and actual DRA claim - allocations. Account for reservations while a workload starts. -- [x] Label TT counts as boards and show two chips per n300 board. -- [x] Treat machine/inventory failures explicitly; do not present unknown - resources as confidently available. -- [x] Give the API only the additional read permissions required for inventory. -- [x] Keep k3s responsible for actual allocation. UI availability is a snapshot; - a submitted job may wait even if devices looked free moments earlier. - -TT's current vendor ResourceSlice memory capacity is incorrect. Do not use that -value to display accelerator memory or determine placement. The current system -has neither GPU fractions nor arbitrary TT chip sharing. - -### C. Task 3: container image execution - -- [x] Expose approved image choices and explicit image references, command, - arguments, and CPU/memory/deadline controls through typed API/UI code. -- [x] Keep image approval on the server. An approved image can still fail to - pull or be incompatible; report the actual error. -- [x] Preserve the image's working directory. The current executor overrides - every workload with `/tmp`, which can break `python train.py` in an image - whose code lives in `/app`. -- [x] Define command/argument behavior, including whether the image's default - ENTRYPOINT/CMD may run. Kubernetes command/args are arrays; do not silently - treat every string as a shell program. -- [x] Retain script and training-check modes with their tested runtime profiles. -- [x] Support approved custom CPU/NVIDIA images. Keep the tested TT image/profile - restriction visible instead of promising arbitrary TT image compatibility. -- [x] Configure/document image pull credentials where needed; do not ask users - to paste registry credentials into a job form. -- [x] Verify an actual custom image with packaged code, rather than testing - only scripts injected into existing base images. A local fixture image may - be imported into containerd for the initial test. -- [x] Document resource limits and deadlines. Current Job deadline includes - waiting time as well as execution; make any change to this behavior explicit. - -### D. Persistent output folders - -- [x] Provide a predictable, writable per-job directory backed by the existing - checkpoint PVC on the selected machine. -- [x] Create the directory reliably and verify permissions for the supported - image profiles. -- [x] Preserve/document `MIST_CHECKPOINT_DIR` and `checkpoint_directory` so - workloads know where to save their results. -- [x] Verify a real output file remains after the container exits. -- [x] Describe how to retrieve files during local testing and where they live. -- [x] Keep job identifiers/storage references independent of physical disk - paths so shared storage can be added later. - -Files written only into the container's writable layer are not persistent -results. The existing shared checkpoint roots are also not a complete tenant -isolation boundary; do not represent this pilot as ready for untrusted users. - -### E. Task 1: finish website job controls - -- [x] Connect the website to the new hardware and image contracts. -- [x] Keep real job submission, polling, logs, assigned node, failure reasons, - exit codes, and cancellation. -- [x] Show requested resources and the persistent output location. -- [x] Make waiting-for-resources, machine unavailable, and image pull failures - understandable. -- [x] Preserve connection recovery and protect against duplicate submissions. -- [x] Use existing UI conventions and accessible labels. Keep frontend concerns - separate from the executor and typed API client. - -The prototype Dashboard/Machines/auth pages are not proof of working metrics or -authentication. Surface real hardware information in the relevant workflow. - -### F. Verify and hand off - -- [x] Run focused backend tests for execution semantics, inventory accounting, - validation, and cancellation where changed. -- [x] Run frontend type checking, production build, and meaningful interaction - checks for new behavior. -- [x] Run a real custom CPU image job through the website. -- [x] Run fresh NVIDIA and TT training checks through the website. -- [x] Verify occupied/free counts during execution and after completion. -- [x] Verify waiting/reuse and cancellation release for both allocation paths. -- [x] Verify failed commands and image pulls produce useful states/errors. -- [x] Verify saved outputs persist after pod completion. -- [x] Update the deployed API image if necessary; editing Go source does not - update the binary currently running in k3s. -- [x] Record actual results and date them. Older October 3 acceptance evidence - does not substitute for tests of today's changes. -- [x] Document startup, API contracts, image preparation, output access, tests, - and remaining work for the team. - -Avoid unrelated operator upgrades or repeat full infrastructure acceptance tests -without a concrete regression risk. Small synthetic training checks demonstrate -device execution and learning, but each team's actual model needs compatibility -validation; distributed training and TT autograd are not established by these checks. - -## Today's definition of done - -```text -Local website - | -Choose approved image + command + hardware + device count - | -Mist API creates a k3s Job - | -Selected machine runs the container - | -Real states/logs + correct device accounting + persistent output file -``` - -Implementation should be verifiable from the browser, with failures and -cancellation behaving correctly and documentation sufficient for another -developer to resume the work. - -## Next foundation task: shared datasets and downloads - -This is task 4, after today's execution scope: - -1. Create a storage folder on QuietBox with a bounded disk budget. -2. Configure private NFS access for both machines and Kubernetes workloads. -3. Add streaming dataset uploads, progress/error handling, a stable dataset ID, - and a ready state before jobs may use an upload. -4. Attach only the selected dataset read-only and the job's outputs writable. -5. Add result listing/downloads with ownership checks and retention rules. -6. Test one uploaded dataset with jobs on both machines. - -Proposed physical layout, not yet created: - -```text -Tenstorrent: /srv/mist-storage/ -├── datasets/ -│ └── dataset-id/ -└── jobs/ - └── job-id/ - └── outputs/ -``` - -Container paths may become `/inputs` and `/outputs`; preserve compatibility -with the existing checkpoint contract or document the migration explicitly. -NVIDIA reads TT-hosted files over the current LAN, which includes Wi-Fi. -Caching large datasets on NVIDIA and a dedicated Ethernet link can follow. -Central storage on one machine needs backup/retention planning and is unavailable -when that host fails. Archives need safe extraction and upload limits. - -## Later: deployment and product features - -Task 5 follows the foundation: - -- Serve a production frontend and route `/api` to the existing Mist API. -- Provide a team-accessible Tailnet address/hostname and persistent services. -- Implement login, job/file ownership, and appropriate member limits before - supporting multiple users. -- Add public domain/HTTPS only if public access is requested. - -Credits, fairness/priority policies, Jupyter, chaining, agents, detailed monitoring, -portable TT runtime images, and distributed model training are separate workstreams. - -## Resume notes and useful files - -### Code - -- `src/kubernetes.go`: executor, validation, resource requests, job status/logs/cancel. -- `src/kubernetes_test.go`: existing Kubernetes executor tests. -- `src/api.go`: API mode selection and request types. -- `web-interface/src/api.ts`: typed API client. -- `web-interface/src/components/JobsPage.tsx`: working job UI. -- `web-interface/src/components/JobsPage.test.tsx`: existing UI checks. -- `deploy/k3s/mist-api.yaml`: API Deployment, RBAC, Service, and checkpoint PVCs. -- `deploy/k3s/mist-tenstorrent-claims.yaml`: 1–4 board claim templates. -- `deploy/k3s/tenstorrent-workload-policy.yaml`: accelerator workload admission policy. -- `deploy/k3s/training_smoke.py`: actual NVIDIA/TT regression training check. -- `deploy/k3s/verify_mist_api.py`: earlier API allocation/queue/training acceptance tool. - -### Existing documentation and evidence - -- `docs/kubernetes-pilot.md`, `deploy/k3s/README.md`, `src/jobs.md`. -- `deploy/k3s/mist-api-results.json`: October 3 acceptance summary. -- `/home/utmist/mist-api-results-2026-10-03`: detailed prior acceptance artifacts. - -### Local tools and access - -- Go: `/home/utmist/.local/share/mist-runtimes/go-1.25.1/bin/go`. -- Node: `/home/utmist/.local/share/mist-runtimes/node-22.16.0/bin/node`. - Prepend that runtime's directory to PATH when running npm/npx. -- Kubeconfig: `/home/utmist/.kube/config`. -- QuietBox SSH: user `utmist-tt`, host `100.95.175.37`, existing identity file - `/home/utmist/.ssh/id_ed25519_quietbox_codex`; use IdentitiesOnly and BatchMode. -- Browser checks can use the temporary Playwright installation at - `/tmp/mist-browser-check/node_modules/playwright` and existing cached Chromium. -- User is in the Docker group. Sudo requires a password; prior authorized root - cluster operations used a Docker host bind/chroot. Do not print secret values. -- Do not include credentials, join tokens, registry secrets, or private keys in - documentation, Git, or tool output. - -Model recommendation for the user's remaining quota: GPT-6.1 Sol, Medium -reasoning, Standard speed. This was advice, not a settings change. Saved settings -were Max/Fast at the time of planning. Exact quota consumption cannot be predicted. - -## Progress log - -- Planning: agreed scope recorded; existing API health and both Ready nodes checked. -- Branch: `feat/local-job-foundation`. Commit `2f9b178` preserves the prior working - pilot and this plan. Commit identity uses the connected GitHub account's public - noreply address for this commit only; global Git settings were not changed. -- Backend additions: `hardware.go`, `images.go`, `outputs.go`, and `job_states.go`. - Inventory reads node/pod/claim/slice state; image defaults are preserved; - outputs have per-job mounts and an initialization step; history reads are batched. -- Frontend: typed inventory/catalog clients, a shared polling hook, separate - submission/card/log components, and live Machines information. -- Focused backend tests pass. Frontend interaction tests (5), type checking, and - production build pass. A test initially submitted before the catalog loaded; - it was corrected to wait for the form's actual readiness. Final lint also - passed after removing unnecessary test casts and shadowed callback names. -- Example images `mist-training:cpu-v1` and `mist-training:nvidia-v1` were built - and imported on the NVIDIA node. API image `mist-api:foundation-20261008` - is deployed; the final binary image ID is recorded in `foundation-results.json`. -- Fresh browser checks passed: packaged CPU training with image ENTRYPOINT/CMD - and `/app` WORKDIR, both NVIDIA GPUs, TT two-board/four-chip training, exit 17, - cancellation, real states/logs, and the live Machines page. -- Fresh allocation checks passed for NVIDIA and TT: whole-machine occupied/free - counts, queued follower, cancellation with retained logs, automatic device - reuse, and return to 2 GPUs / 4 boards available. -- Persistent output checks used new read-only containers after training exited. - CPU model/results, both NVIDIA checkpoints, and four TT chip checkpoints and - metrics were readable on the proper nodes. SHA256/file evidence was saved. -- A deliberately missing approved image exposed its real pull reason in the API - and browser, then failed with DeadlineExceeded. Temporary image approval was - removed, and prior job/cancellation history survived API restarts. -- Final focused backend tests: 12 passed. Frontend interaction tests: 5 passed; - type checking, production build, and lint of changed frontend files passed. - CLI tests passed. The full src suite passed in 33.325 seconds with isolated - Redis and the existing Docker test image. Its initial 15-second legacy CPU - test timed out during large-image disk activity; the suite passed after the - imports finished. No legacy executor behavior was changed. -- The first browser verifier expected a completed TT claim to retain allocations. - Kubernetes had correctly released them. The verifier now checks completed - request/log evidence and checks actual DRA devices during live reservations. -- Fresh RBAC checks passed: required inventory lists allowed; Secrets, node - writes, pod exec, and Jobs in other namespaces denied. Private registry - Secret wiring is implemented/documented; no private registry login was tested. -- Detailed evidence: `/home/utmist/mist-foundation-results-2026-10-08`. - Compact checked-in summary: `deploy/k3s/foundation-results.json`. - Final idle snapshot: both nodes Ready, 0 allocated / 2 available NVIDIA GPUs, - 0 allocated / 4 available TT boards. All 40 retained Mist Jobs were terminal. - Job history read measured 0.193 seconds, compared with the 3.586-second baseline. -- Today's agreed implementation and verification are complete: parts 1–3 plus - persistent local outputs. The percentage updates referred to today's scope. - The complete platform still needs part 4 (shared datasets/downloads) and - part 5 (deployment/login). The team handoff is `docs/local-job-foundation.md`. -- Screen question checked: automatic suspend on AC has timeout 0 (disabled), - so the display may turn off while work continues. Power settings were not changed. - -## References - -- [Kubernetes Jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) -- [Container images](https://kubernetes.io/docs/concepts/containers/images/) -- [K3s private registries](https://docs.k3s.io/installation/private-registry) -- [Kubernetes persistent volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) -- [Local path provisioner](https://github.com/rancher/local-path-provisioner) -- [Tenstorrent platform support](https://docs.tenstorrent.com/tt-operator/latest/platform-support.html) +- [Current documentation](README.md) +- [Current operating guide](department-rollout.md) +- [Archived original record](archive/foundation-execution-plan.md) diff --git a/docs/k3s-setup.md b/docs/k3s-setup.md new file mode 100644 index 0000000..2c728f0 --- /dev/null +++ b/docs/k3s-setup.md @@ -0,0 +1,224 @@ +# k3s setup and accelerator handoff + +Verified on **October 9, 2026**. This guide describes the existing two-node +installation, fresh-install boundaries and diagnostic commands. Routine +application deployment is in [operations](../deploy/private/README.md). + +## Installed cluster + +| Item | Main/NVIDIA | QuietBox original | +|---|---|---| +| Kubernetes node | `utmist-z1opa08` | `utmist-tt` | +| Linux user | `utmist` | `utmist-tt` | +| Role/service | server + worker, `k3s` | worker only, `k3s-agent` | +| k3s | `v1.36.4+k3s1` | `v1.36.4+k3s1` | +| OS/kernel | Ubuntu 22.04.5 LTS / 6.8.0-138-generic | Same | +| containerd | `2.3.4-k3s1.36` | Same | +| LAN/InternalIP | `10.0.0.175` | `10.0.0.112` | +| LAN interface | `wlxe0d362c98a69` (Wi-Fi) | `enp201s0` (Ethernet to router) | +| Tailscale IP | `100.73.139.66` | `100.95.175.37` | +| Accelerator | 2 × NVIDIA RTX A4000 | 4 × n300 boards, 2 Wormhole chips per board | +| Operator | GPU Operator `v26.7.1` | TT-Operator `0.3.0`, Fabric Manager `0.2.30`, DRA `0.0.59` | + +Main's host NVIDIA driver is **590.48.01**. QuietBox's `tenstorrent` kernel +module is loaded and `/dev/tenstorrent/0` through `3` exist. Operators retain +these host drivers. Both nodes were Ready and all operator deployments/daemonsets +were Ready at the audit. Service enablement was checked; a new reboot was not tested. + +```text +Researcher → Tailscale → main Nginx :8088 → Mist API/auth + | + k3s server :6443 at 10.0.0.175 + / \ + CPU/NVIDIA TT agent + 10.0.0.112 + \ / + NFS on QuietBox :2049 +``` + +The server already includes kubelet/containerd and can run workloads. Do not +start a separate agent on main: the earlier duplicate agent conflicted on port +6444. QuietBox has no independent `k3s` server service; its old test cluster was +retired before joining this server. + +## Verify from main + +Use the existing private admin kubeconfig. These commands do not print secrets: + +```bash +export KUBECONFIG=/home/utmist/.kube/config +k3s kubectl get nodes -o wide +k3s kubectl get deployments,daemonsets -A +k3s kubectl get pods -n mist-system -o wide +k3s kubectl get deviceclasses,resourceslices +helm list -A +``` + +The admin kubeconfig is private infrastructure access, not a researcher client +configuration. Keep it and node tokens out of Git and job containers. + +On main, expect `systemctl is-active k3s` to report active. On QuietBox, expect +`systemctl is-active k3s-agent` to report active. Both services are enabled at boot. + +## Fresh server installation + +This is for a new, prepared host. Do not reinstall the working control plane to +redeploy Mist or assume this reconstructs its existing accounts/storage. +Pin the observed version when reproducing it: + +```bash +curl -sfL https://get.k3s.io | sudo env INSTALL_K3S_VERSION='v1.36.4+k3s1' sh -s - server +sudo systemctl is-active k3s +sudo k3s kubectl get nodes -o wide +``` + +The installer creates `/etc/rancher/k3s/k3s.yaml`. Choose stable, reachable node +addresses and unique hostnames. Pin `node-ip`/`flannel-iface` in configuration +when multiple interfaces could select the wrong route. This installation uses +LAN addresses, not Tailscale addresses, for its Kubernetes nodes. + +## Join a fresh worker + +First verify the worker can reach **10.0.0.175:6443**. A machine on another LAN +cannot join this setup solely by being in the same tailnet; cluster advertising, +Flannel, NFS routes and firewalls would need a planned network change. + +For a clean worker, an administrator securely installs an appropriate persistent +join token in `/etc/rancher/k3s/agent-token`, owned by root with mode **0600**. +The persistent join token is obtained through authorized access to the server's +`/var/lib/rancher/k3s/server/node-token` or configured agent-token source; +do not paste it into Git, terminal transcripts or this guide. A temporary +bootstrap token is not a substitute for the persistent token used after restart. + +QuietBox's current `/etc/rancher/k3s/config.yaml` contains: + +```yaml +server: https://10.0.0.175:6443 +token-file: /etc/rancher/k3s/agent-token +node-ip: 10.0.0.112 +flannel-iface: enp201s0 +``` + +Use the new worker's own LAN IP/interface when provisioning another host. Then: + +```bash +curl -sfL https://get.k3s.io | sudo env INSTALL_K3S_VERSION='v1.36.4+k3s1' sh -s - agent +sudo systemctl is-active k3s-agent +sudo systemctl is-enabled k3s-agent +``` + +If the host already has another cluster, inspect its Jobs/PVCs and retain its +state before retiring it; the fresh-worker recipe does not migrate cluster data. +QuietBox's original test-server snapshot is root-only at +`/root/k3s-server-prejoin-2026-10-03.tar.gz`. It is historical, not a backup of the +current department platform. + +## Cluster and storage traffic + +| Destination/port | Purpose | Current network | +|---|---|---| +| Main TCP 6443 | Kubernetes API/agent registration | LAN, infrastructure access | +| Node UDP 8472 | Default Flannel VXLAN | LAN between nodes | +| Node TCP 10250 | kubelet access | LAN between nodes | +| QuietBox TCP 2049 | NFSv4 dataset/model mounts | LAN, main and QuietBox sources | +| Main TCP 8088 | Private portal | Tailscale and localhost listeners | +| Host TCP 22 | Ubuntu SSH | Authorized administrator access | + +Pods use `10.42.0.0/16`; Services use `10.43.0.0/16`. QuietBox's UFW allows +8472/UDP and 10250/TCP from main on `enp201s0`, NFS from both LAN host IPs and +forwarding between the node pod subnets. Main's installed host firewall +restricts ordinary Tailscale users to the portal. Keep VXLAN/kubelet/NFS access +scoped to trusted infrastructure sources. Wi-Fi works for the current route; +large transfers depend on LAN throughput. No direct host-to-host cable is required. + +## NVIDIA integration + +The operator's device plugin advertises `nvidia.com/gpu: 2`. CDI injects assigned +GPUs into workload containers. The tracked values retain the host driver and +enable the operator's NRI/CDI configuration. + +From the repository root on main, installation/upgrade for a prepared cluster: + +```bash +export KUBECONFIG=/home/utmist/.kube/config +helm upgrade --install gpu-operator oci://nvcr.io/nvidia/cloud-native-charts/gpu-operator \ + --version v26.7.1 --namespace gpu-operator --create-namespace \ + --values deploy/k3s/nvidia-gpu-operator-values.yaml --wait +k3s kubectl -n gpu-operator get pods +k3s kubectl get node utmist-z1opa08 -o jsonpath='{.status.allocatable.nvidia\.com/gpu}{"\n"}' +``` + +Expect 2 GPUs advertised; that is total schedulable capacity, not currently free +capacity. Each request reserves 1 or 2 whole GPUs. GPU partitioning/time slicing +is not configured. Inspect host `nvidia-smi -L` locally if devices disappear. + +## Tenstorrent integration + +The operator uses existing NFD and host KMD/firmware. It advertises four `n300` +DRA devices, each a two-chip board. TT capacity is in ResourceSlices, not the +NVIDIA extended resource field. + +```bash +export KUBECONFIG=/home/utmist/.kube/config +k3s kubectl label node utmist-tt tenstorrent.com/has-tt=true --overwrite +helm upgrade --install tt-operator oci://ghcr.io/tenstorrent/helm/tt-operator \ + --version 0.3.0 --namespace tt-operator-system --create-namespace \ + --values deploy/k3s/tenstorrent-operator-values.yaml +k3s kubectl -n tt-operator-system get pods +k3s kubectl get deviceclasses,resourceslices +``` + +Mist creates board-count ResourceClaimTemplates in managed team namespaces. +DRA allocates claims and CDI exposes only those boards. A one-board request can +use its two chips; other jobs can reserve remaining boards. Multi-board allocation +does not guarantee fabric topology or distributed model training. + +The default TT profile mounts the tested host checkout/environment read-only; +a compatible custom image can use `tt_runtime: container`. TT smoke checks use +TT-NN forward/analytic-gradient/update operations, not a proof of arbitrary +PyTorch autograd/model compatibility. + +## Test a workload + +Use **Jobs → New job → NVIDIA/Tenstorrent → Training check** after signing in and +selecting your team. This exercises authentication, quotas, queue, allocator, +training and persistent output. Use the [CLI](../cli/docs/setup.md) from either +host to submit a real script through the same API. + +The direct manifest below is an administrator infrastructure diagnostic in the +legacy `mist` namespace. It bypasses the portal's team admission and does not +establish team-policy enforcement. Run it only when the GPU is available: + +```bash +export KUBECONFIG=/home/utmist/.kube/config +k3s kubectl apply -f deploy/k3s/gpu-smoke-job.yaml +k3s kubectl -n mist wait --for=condition=complete job/gpu-smoke --timeout=180s +k3s kubectl -n mist logs job/gpu-smoke +``` + +Expected CUDA marker: `Test PASSED`. Applying a completed Job does not rerun it; +retain needed evidence before deleting only that completed diagnostic Job. + +## Troubleshooting + +- **Node missing/NotReady:** inspect `systemctl status k3s` on main or + `systemctl status k3s-agent` on QuietBox, then the matching journal and LAN route. +- **Main API down:** confirm the duplicate main agent stays inactive; inspect + port 6444 conflicts before restarting infrastructure. +- **Missing NVIDIA resource:** check host driver and GPU operator/plugin pods. +- **Missing TT devices:** check `/dev/tenstorrent`, the KMD and TT operator pods, + then ResourceSlices/ResourceClaims. +- **Job waiting:** inspect portal message, team limits and actual device capacity; + cluster admins can inspect pod/claim events in its team namespace. +- **Image pull fails:** image must be accessible and contain compatible code. + Locally imported Docker images must also be imported into k3s containerd. +- **Storage unavailable:** verify QuietBox's NFS/team-volume services and both pool + mounts. Do not delete PVCs or overwrite mounted backing files. + +Current live evidence is in [the verification record](../deploy/k3s/evidence/department-2026-10-09/README.md). +No new reboot, operator upgrade or rejoin was performed for this documentation audit. + +References: [K3s quick start](https://docs.k3s.io/quick-start), +[network requirements](https://docs.k3s.io/installation/requirements), +[GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html), +[TT-Operator](https://docs.tenstorrent.com/tt-operator/latest/). diff --git a/docs/kubernetes-pilot.md b/docs/kubernetes-pilot.md index 308ae37..b778a4e 100644 --- a/docs/kubernetes-pilot.md +++ b/docs/kubernetes-pilot.md @@ -1,134 +1,7 @@ -# Kubernetes execution pilot +# Historical guide +This phase has been superseded by the department platform on main. -**October 9 update:** Shared datasets/results, member login and private production -hosting are implemented. Earlier local pilot descriptions below record the -original phase. Use [the current operating guide](complete-foundation.md) for today's behavior. - -## Implemented on 2026-10-03 - -Mist uses Kubernetes `batch/v1` Jobs by default. The API submits actual -commands or uploaded scripts, reports real exit codes and scheduler messages, -retrieves pod logs, and cancels workloads. The CLI and Jobs page use these -endpoints. The optional `MIST_EXECUTOR=docker` path retains the older, -incomplete Docker/Redis supervisor for compatibility. - -Kubernetes owns placement, device accounting, and Job lifecycle. Mist validates -submission parameters and configured owner labels. Authentication, team -policy, credits, quotas, priority, and audit accounting are still application -work; the current API is a local single-owner pilot. - -```mermaid -flowchart LR - User[Jobs page / CLI] --> API[Mist API] - API --> Validate[Image, resources, script and deadline validation] - Validate --> Jobs[Kubernetes Jobs] - Jobs --> NVIDIA[CPU / NVIDIA server] - Jobs --> Claims[Tenstorrent DRA claims] - Claims --> TT[QuietBox boards via CDI] - Jobs --> State[Job and Pod status / logs] - State --> API - NVIDIA --> Artifacts[Node-local checkpoint PVCs] - TT --> Artifacts -``` - -See [installation, commands and acceptance tests](../deploy/k3s/README.md#mist-api-cli-and-jobs-page) -and [job lifecycle and API endpoints](../src/jobs.md). - -## Hardware and scheduling - -| Node | Workload resources | Allocation | -| --- | --- | --- | -| `utmist-z1opa08` | CPU and two NVIDIA RTX A4000 GPUs | `nvidia.com/gpu`, one or two whole GPUs | -| `utmist-tt` | Four n300 Wormhole boards, eight chips | DRA/CDI, one to four whole two-chip boards | - -CPU and NVIDIA jobs select the server because their pilot storage is local -to it. Tenstorrent jobs select QuietBox and reference one of four precreated -ResourceClaimTemplates. Each pod receives its own claim, not a shared claim -between jobs. Resources are released after completion or cancellation; -completed Jobs and checkpoints are retained. Requests wait when their -required CPU, RAM, hugepages, or devices are occupied. Scheduling does not -split a model or combine NVIDIA and Tenstorrent automatically. - -TT-Operator 0.3.0 runs Fabric Manager 0.2.30 and DRA driver 0.0.59 while -preserving host KMD 2.5.0 and firmware 19.4.2. Read-only host TT-Metal and -Python mounts make the runtime specific to this QuietBox. The vendor -ResourceSlice's memory capacity is incorrect; claims select board name, -chip count and board count, without using that capacity. - -The TT test trains each allocated chip independently using TT-NN forward -passes, analytic gradients and weight updates. For multiple boards, separate -runtimes select each board by PCI address through `TT_VISIBLE_DEVICES`. -Arbitrary board subsets need not form a usable connected mesh. Autograd, -distributed training of one model, and large production models are untested. - -## Verified behavior - -Live API acceptance checks passed for 17 jobs: - -- CPU success with actual output, exit-17 failure, deadline failure, and - cancellation with retained logs. -- Two simultaneous one-GPU NVIDIA jobs with distinct device nodes; a third - queued and automatically reused a GPU. A two-GPU job also trained. -- Four simultaneous one-board TT jobs with distinct claims and devices; - a fifth queued and trained after release. -- Two-board and four-board TT jobs trained on four and eight chips. -- Cancelling a four-board reservation released capacity for a waiting job. -- All successful accelerator runs met loss thresholds and reloaded their - checkpoints. Archives confirmed five NVIDIA and 24 TT checkpoint files - from those API checks after the pods completed. - -Separate infrastructure checks also verified that each TT job could open -its allocated board while device cgroups denied the other three, and that -allocation recovered after restarting the DRA driver. CLI and actual browser -checks exercise submission, status, logs and cancellation; backend/CLI/web -tests and a frontend production build cover the application changes. -See [recorded results](../deploy/k3s/mist-api-results.json) for evidence paths. - -## Access and retention - -The API runs in `mist-system` with a dedicated service account and a Role -restricted to workloads in `mist`. The October 8 foundation adds a read-only -inventory ClusterRole for listing nodes, pods, ResourceClaims and ResourceSlices. -Secrets, node writes, pod exec, and creating jobs in other namespaces remain -outside its permissions. The October 3 checks predate this inventory role. Workload pods -receive no Kubernetes API token. The `mist` admission policy disallows -privileged execution and broad host device mounts. - -The service account can create workload Jobs, so RBAC alone does not make -it a complete security boundary. Admission policy, shared host hugepages, -shared checkpoint PVCs, and arbitrary script execution require further -hardening before accepting untrusted tenants. The local API has no -per-request authentication; `MIST_PILOT_OWNER` is configuration. - -API and web access bind to loopback, with no public ingress. A port-forward -exposes the deployed API. Retained Jobs carry submission/owner metadata and -cancelled logs across API restarts. There is no Job TTL. Normal logs depend -on retained pods and kubelet retention, and checkpoint PVCs are node-local. -Deleting those resources can remove history or artifacts. - -## Next milestones - -The October 8 local foundation implements approved custom CPU/NVIDIA container -execution, live Jobs/Machines inventory, and persistent per-job `/outputs` -mounts. See [the current guide and developer handoff](local-job-foundation.md) -and [execution plan](foundation-execution-plan.md) for fresh checks and the -agreed next foundation task: shared datasets and result downloads, followed -by team deployment/login. The milestones below cover the wider platform. - -1. Implement authentication and per-user/team authorization before remote - access; define quotas, concurrency limits, credits and priority policy. -2. Package the TT runtime into a reproducible training image and validate - real pilot-team models, topology and distributed training requirements. -3. Provide dataset upload and artifact access, shared/object storage, durable - metadata/log archival, cleanup policy, and datastore/PVC backups. -4. Add network restrictions, workload admission hardening, and operational - monitoring before opening the cluster to untrusted users. - -## References - -- [Kubernetes Job API](https://kubernetes.io/docs/concepts/workloads/controllers/job/) -- [Dynamic Resource Allocation](https://kubernetes.io/docs/concepts/resource-management/dynamic-resource-allocation/) -- [K3s storage](https://docs.k3s.io/storage) -- [Tenstorrent operator platform support](https://docs.tenstorrent.com/tt-operator/latest/platform-support.html) -- [Tenstorrent DRA driver](https://github.com/tenstorrent/tt-dra-driver) +- [Current documentation](README.md) +- [Current operating guide](department-rollout.md) +- [Archived original record](archive/kubernetes-pilot.md) diff --git a/docs/local-job-foundation.md b/docs/local-job-foundation.md index 271eebb..a188e01 100644 --- a/docs/local-job-foundation.md +++ b/docs/local-job-foundation.md @@ -1,329 +1,7 @@ -# Local job foundation +# Historical guide +This phase has been superseded by the department platform on main. -**October 9 update:** Shared datasets/results, member login and private production -hosting are implemented. Earlier local pilot descriptions below record the -original phase. Use [the current operating guide](complete-foundation.md) for today's behavior. - -Implementation branch: `feat/local-job-foundation`. Recorded October 8, 2026 -(America/New_York). The preserved starting point is commit `2f9b178`. -The [execution plan](foundation-execution-plan.md) records scope and progress. - -## What runs now - -```text -Website :3001 / CLI - | - v -Mist API :3000 -- validates image, resources, and workload - | - v -k3s Job + pod -- Kubernetes schedules; containerd runs the image - / \ - v v -NVIDIA PC QuietBox -CPU / 2 GPUs 4 n300 boards / 8 chips - | | - v v -Per-job output folder on the selected node's persistent volume -``` - -Open **http://127.0.0.1:3001/jobs**. The **Machines** page shows the same real -inventory. Jobs support Python scripts, approved container images, and small -accelerator training checks. Dashboard/login and other prototype pages remain -separate work. Job execution uses Kubernetes, without SSH or a Docker daemon -inside the job. SSH is useful for administration and reaching the local website. - -The current API serves one configured owner, `utmist`. This is a trusted local -pilot. Membership, login, credits, quotas, public deployment, shared datasets, -and browser result downloads are subsequent tasks. - -## Use the website - -1. Check hardware availability at the top of **Jobs**. -2. Select **Compute**: CPU, NVIDIA, or Tenstorrent. -3. For accelerators, request whole GPUs or whole boards. A board has two chips. -4. Select **Workload** and an approved **Container image**. -5. For **Container image** workloads, optionally provide an executable and - arguments, one argument per line. Leaving them blank uses the image defaults. -6. **Resources and environment** contains CPU/RAM, deadline, working directory, - and `NAME=value` environment variables. Save files under `/outputs`. -7. Submit. The job card shows its ID, state, assigned machine, resources, - output path, and exit code. **Logs** reads actual container output. - **Cancel** stops a running or waiting job and preserves recent logs. - -An image must contain its own code and dependencies, or provide a runtime for -script mode. Requesting a GPU does not convert CPU code into accelerator code. -The form accepts an image reference; it does not upload/build Docker archives. - -### Included examples - -The current deployment approves and locally caches these two example images: - -| Image | Website settings | What it checks | -| --- | --- | --- | -| `mist-training:cpu-v1` | CPU; Container image; blank command/arguments | Packaged regression code and dataset, default entrypoint and `/app` working directory, saved/reloaded model | -| `mist-training:nvidia-v1` | NVIDIA; Container image; arguments below | CUDA training and checkpoint reload on each allocated GPU, without CPU fallback | - -For two GPUs, select **GPUs = 2** and enter these arguments, one per line: - -```text ---backend -nvidia ---devices -2 -``` - -For Tenstorrent, choose **Small training check** and one to four boards. This -uses the already tested TT image and installed host runtime. It trains a small -regression model on the chips of every reserved board. TT-NN gradients are -computed explicitly; this does not establish arbitrary PyTorch model or TT -autograd compatibility. Cross-machine/distributed training is separate work. - -## API contracts - -The [job lifecycle reference](../src/jobs.md) describes state and retention. -The browser uses `/api`; Vite proxies it to the API. The API itself has these -unprefixed routes: - -| Route | Response | -| --- | --- | -| `GET /hardware` | `{observed_at, machines: [...], pools: [...]}` | -| `GET /images` | `{images: [{reference, accelerators}], profiles: [...]}` | -| `POST /jobs` | HTTP 201, `{job_id, job}` | -| `GET /jobs` | `{count, jobs: [...]}` | -| `GET /jobs/` | Actual job state, resources, placement and output paths | -| `GET /jobs//logs` | `{logs}` | -| `POST /jobs//cancel` | Retained job with state `Cancelled` | -| `GET /healthz` | Jobs API connectivity and executor | - -### Submission - -```json -{ - "name": "my-training-job", - "type": "command", - "accelerator": "nvidia", - "device_count": 1, - "image": "mist-training:nvidia-v1", - "args": ["--backend", "nvidia", "--devices", "1"], - "cpu": "2", - "memory": "2Gi", - "timeout_seconds": 600, - "env": {"GREETING": "my first job"} -} -``` - -| Field | Behavior | -| --- | --- | -| `name` | Optional display name, at most 128 bytes | -| `type` | `command` (default) or `training-smoke` | -| `accelerator` | `cpu` (default), `nvidia`, `tenstorrent` | -| `device_count` | CPU: 0; NVIDIA: 1–2; TT: 1–4 boards. Accelerator default is 1 | -| `image` | Exact server-approved reference. Required when using only image defaults | -| `command` / `args` | Optional arrays, at most 64 combined entries, each at most 8192 bytes. No implicit shell parsing | -| `working_directory` | Optional absolute container path. Custom CPU/NVIDIA containers preserve image WORKDIR when omitted | -| `script` / `script_name` | Optional Python/shell script, at most 32 KiB; filename only, `.py` or `.sh`. Cannot accompany `command` | -| `env` | Up to 32 entries, values at most 4096 bytes. Managed Mist/TT variables cannot be overridden | -| `cpu` | Kubernetes quantity, greater than zero and at most 8; request equals limit | -| `memory` | 64Mi–32Gi; TT minimum 2Gi; request equals limit | -| `timeout_seconds` | 10–86400; default 600; includes queue/startup time | - -`command` replaces the image ENTRYPOINT; `args` replaces CMD. With neither, -the image's entrypoint and arguments run. Scripts and training checks use -their runtime profile and default `/tmp` working directory. TT uses `/tmp` -and its host Python/TT-NN runtime unless explicitly configured otherwise. -To use shell syntax, request it explicitly, for example -`"command": ["sh", "-c"], "args": ["python train.py && echo done"]`. - -CPU defaults: 250m/256Mi. NVIDIA defaults: 2 CPU/2Gi. TT defaults: 2 CPU/4Gi -RAM plus 2Gi of 1Gi hugepages per board. Jobs have no retries. Images are -pulled with `IfNotPresent`; approval does not guarantee availability or -hardware compatibility. Invalid requests return HTTP 400 without creating -Jobs. Kubernetes failures include a message and actual exit code when available. - -### Hardware inventory - -Each machine reports `name`, internal `address`, `ready`, `schedulable`, -`cpu_allocatable`, and `memory_allocatable`. CPU/RAM are Kubernetes allocatable -capacity, not current utilization or remaining CPU/RAM. - -Each pool reports `accelerator`, `node`, `unit`, `total`, `allocated`, -`available`, `ready`, and optional `message` / `chips_per_device`. - -- NVIDIA: count scheduled nonterminal Pod reservations across all namespaces, - including bound Pending and Terminating pods and effective init/sidecar - requests. Unplaced queued pods do not consume a device. -- Tenstorrent: use current complete ResourceSlice generations and deduplicated - ResourceClaim allocations across all namespaces. Claims reserve boards - even before a pod starts. Administrative access claims do not consume devices. -- Unavailable/cordoned/tainted nodes or missing ready device drivers are not - advertised as ready. Incomplete TT inventory has `available: null`. -- Failed discovery returns HTTP 503. The UI marks availability unknown; - it does not invent zero allocations or free hardware. -- The known incorrect TT vendor memory capacity is ignored. - -The snapshot is assembled from separate Kubernetes reads and can change before -submission. Kubernetes remains the allocation authority. Active TT `devices` -details can disappear from a completed job once its claim is released; the -request count and saved `allocation.json` remain useful records. - -## Output persistence - -An init container creates a job-specific directory before the workload starts. -The workload mounts only this directory at two aliases: - -```text -Persistent volume / / - ├── model/checkpoint files - └── results.json - -Inside the workload: - /outputs MIST_OUTPUT_DIR - /checkpoints/ MIST_CHECKPOINT_DIR - MIST_JOB_ID -``` - -Both mount paths refer to the same files. `output_directory` is `/outputs` for -new jobs; old retained jobs keep their original `checkpoint_directory` contract. -The per-job directory is writable by the supported root/nonroot images. Its -permissions and shared backing PVC are not an authenticated tenant boundary. -Writes elsewhere in the container are not saved results. - -| Compute | PVC in `mist` | Physical node | -| --- | --- | --- | -| CPU | `mist-cpu-checkpoints` | `utmist-z1opa08` | -| NVIDIA | `training-nvidia-checkpoints` | `utmist-z1opa08` | -| Tenstorrent | `training-tenstorrent-checkpoints` | `utmist-tt` | - -Find the host directory using the administrative kubeconfig: - -```bash -k3s kubectl -n mist get pvc -k3s kubectl get pv -``` - -Read the selected PV's `spec.hostPath.path` and node affinity. On that node, -the files are under `//`. A read-only helper Pod -can also mount the corresponding PVC on its node. The verification tool below -does this after training has exited, records SHA256 hashes/results, and deletes -its temporary helper Pods. There is no browser file download endpoint yet. - -The local-path PVCs request 256Mi each; local-path does not enforce that as a -disk quota. Jobs/results are retained without TTL or automatic cleanup. -Normal logs are up to 64KiB and depend on pod/kubelet retention. Cancellation -stores up to 32KiB in Job annotations. API restarts preserve Kubernetes history. -Node/disk loss can lose outputs; shared storage and backups follow separately. - -## Development and deployment - -See the [deployment guide](../deploy/k3s/README.md#mist-api-cli-and-jobs-page) -for image build/import and startup. Existing services on this machine are: - -```bash -systemctl --user status mist-api-forward mist-web -systemctl --user restart mist-api-forward mist-web -``` - -From a laptop with SSH access to the NVIDIA server, tunnel the local website: - -```bash -ssh -L 3001:127.0.0.1:3001 utmist@100.73.139.66 -``` - -Open `http://127.0.0.1:3001/jobs` on that laptop while the SSH connection is -open. A different laptop port, such as `-L 3002:127.0.0.1:3001`, avoids a local -port conflict. This does not require public ingress. - -The API runs in `mist-system` with namespaced workload permissions and a -read-only inventory ClusterRole: list nodes, pods, ResourceClaims and -ResourceSlices. It cannot read Secrets, exec into pods, update nodes, or create -workloads in other namespaces. The browser contains no Kubernetes credentials. - -Custom registry images require an administrator to update `MIST_ALLOWED_IMAGES` -with exact references, rebuild/reconfigure the API deployment as needed, and -verify image/code compatibility. Prefer immutable tags or digests. For a private -registry, create its image-pull Secret in workload namespace `mist`; set the -API's `MIST_IMAGE_PULL_SECRETS` to its name (or comma-separated names). -The API references the Secrets without reading their contents. Keep credentials -out of the job form, environment variables, and Git. K3s registry configuration -is a separate option for mirrors/private endpoints. - -### Code map - -| Area | Files | -| --- | --- | -| Request model and routing | `src/api.go`, `src/kubernetes.go` | -| Validation, Job construction, cancellation | `src/kubernetes.go` | -| Live inventory | `src/hardware.go` | -| Approved image/profile catalog | `src/images.go` | -| Output directory initialization | `src/outputs.go` | -| Batched job history reads | `src/job_states.go` | -| API types and client | `web-interface/src/api.ts` | -| Polling/recovery | `web-interface/src/hooks/usePolling.ts` | -| Jobs orchestration and form/cards/logs | `web-interface/src/components/JobsPage.tsx`, `components/jobs/*` | -| Shared inventory and Machines page | `components/HardwarePanel.tsx`, `components/MachinesPage.tsx` | -| Cluster deployment/RBAC | `deploy/k3s/mist-api.yaml` | -| Packaged examples | `deploy/k3s/examples/*` | - -### Verification - -```bash -go -C src test -run 'Test(Kubernetes|Hardware|GPURequest|CustomImage|TenstorrentCustom)' ./... -go -C cli test ./... -cd web-interface -npm test -npx tsc --noEmit -npm run build -``` - -The full legacy backend suite additionally requires its Docker images and -Redis. It is separate from the Kubernetes path. - -Fresh live checks require both accelerators initially free, the approved -example images imported, Playwright/Chromium installed, and admin kubectl access: - -```bash -export KUBECONFIG=/home/utmist/.kube/config -export MIST_PLAYWRIGHT_MODULE=/tmp/mist-browser-check/node_modules/playwright/index.mjs -node deploy/k3s/verify_foundation_browser.mjs /home/utmist/mist-foundation-results-new-run -python3 deploy/k3s/verify_foundation.py --artifacts /home/utmist/mist-foundation-results-new-run -``` - -Use one artifact directory for both commands and run them sequentially. -The browser check covers packaged CPU training, both NVIDIA GPUs, two TT -boards/four chips, real card states/logs, failure, and cancellation. The API -check covers occupied/free counts, queuing and reuse after cancellation for -both device types, command/argument semantics, and persistent files read from -new containers on both nodes. These tools submit real jobs and retain results. -Their final markers are `FOUNDATION_BROWSER_PASSED` and -`FOUNDATION_ALLOCATION_OUTPUTS_PASSED`. - -Today's detailed evidence is saved under -`/home/utmist/mist-foundation-results-2026-10-08`. The checked-in -[verification summary](../deploy/k3s/foundation-results.json) records actual -job IDs, the deployed image ID, and completed checks. Browser and allocation/ -persistence checks passed on October 8 EDT (October 9 UTC). A separate live -check verified image-pull error reporting, deadline failure, and retained -history across API restarts. The normal image approval list was restored. - -Automated checks passed: 12 focused backend tests, the full src suite with its -local dependencies, CLI tests, 5 frontend interaction tests, type checking, -production build, and lint of changed frontend files. These are small model -execution checks, not a claim about every team model or distributed training. -See the execution plan's progress log and the developer handoff below. - -## Developer handoff: next work - -1. **Shared datasets and downloads (task 4):** storage on QuietBox, private - NFS/shared mount on both nodes, upload limits/progress, stable dataset IDs, - read-only inputs, job output listing/downloads, retention and backup rules. - Preserve the `/outputs` contract and migrate the checkpoint alias explicitly. -2. **Team deployment/login (task 5):** production web hosting with `/api` - routing, authenticated membership, job/file ownership, then team limits. -3. **Model/platform work:** portable TT image, real team models, scheduling - fairness/credits/priority, monitoring, Jupyter and chaining as separate work. - -Current Wi-Fi/LAN can support initial shared-storage tests. A second Ethernet -cable can improve throughput later. Do not remove QuietBox's existing router -link or format the NVIDIA machine's unmounted disk as part of this work. +- [Current documentation](README.md) +- [Current operating guide](department-rollout.md) +- [Archived original record](archive/local-job-foundation.md) diff --git a/docs/repository-cleanup.md b/docs/repository-cleanup.md new file mode 100644 index 0000000..149081e --- /dev/null +++ b/docs/repository-cleanup.md @@ -0,0 +1,78 @@ +# Repository cleanup — October 9, 2026 + +Based on main commit `3431ffd`, the department release merged through PR #110. +Changes are prepared on `chore/main-cleanup-20261009`. The main ruleset still +requires a pull request and an independent review; Git publication is separate +from deploying services. + +## Removed and organized + +- Retired the unfinished Redis scheduler/supervisor, status registry, fake auth + handlers, Docker manager and prototype CPU/CUDA/ROCm images/scripts. Removed + their tests, Redis Compose/config and unused custom logger/config. Docker + remains image build/import tooling for the current platform. +- Removed the empty tracked `go` file and unused root Go module. `go.work` now + joins the active API and CLI modules. `go mod tidy` pruned retired dependencies. +- Split API lifecycle (`main.go`, `app.go`), HTTP job handlers (`jobs_http.go`), + shared errors/responses (`http.go`) and job types (`job_types.go`) from the + mixed prototype. Current wire fields, authentication/team enforcement and + Kubernetes state/history handling remain compatible. +- Removed unreachable `teamHTTPError` after compiler-based dead-code analysis. + A final analysis with tests reports no remaining unreachable Go functions. +- Removed the CLI dummy configuration command/tests, unused input/job helpers, + editor workspace artifact, malformed command tag and empty command fallback. + Persistent config-path handling is in production code; output helpers are + compiled only for tests. +- Removed the unused frontend fake-user/logout helper, obsolete chart and + `use-immer`/`immer` packages. Existing interactive components remain in use. +- Removed five pre-team API/browser verifiers that no longer reflect deployed + authentication/storage. Current department/storage/accelerator verifiers and + dated evidence remain. +- Archived seven superseded guides and preserved old document entry points. + Added a current documentation index, k3s/Tailscale/SSH/storage/development + handoff and a source/architecture map. Corrected submission HTTP status, + current TT runtime/image rules and main's merged release state. +- Removed the plaintext password from the old SSH guide. Git history retains it; + retire that credential if valid. This cleanup did not rotate account passwords. + +## Verification + +- [Sanitized validation report](../deploy/k3s/evidence/department-2026-10-09/cleanup-validation.json). +- Full current Go workspace tests with race detection: **29 API tests + 6 CLI + tests**, no skipped tests; CLI entrypoint has no test files. Vet passes. +- API and CLI static builds, CLI/root/job help and retired-executor rejection pass. +- **10 React tests**, TypeScript, full frontend lint and production build pass. + Removing the unused dependency reported **zero npm vulnerabilities**. +- **6 migration tests** pass as root in a disposable container with read-only + test source and tiny temporary files. An ordinary non-root invocation skips + these ownership tests; it is not represented as successful coverage. +- Both live cluster nodes, operator deployments/daemonsets and API/auth services + were inspected. Host service enablement, regular OpenSSH/Tailscale SSH settings, + LAN interfaces, token-file permissions, GPU/KMD inventory and storage usage + were checked on the owning hosts. No rejoin/reboot/operator upgrade was run. +- The compiled candidate API is checked on a temporary authenticated localhost + listener against real Kubernetes/auth: startup/health, anonymous session, + anonymous-job denial and graceful SIGTERM. Team admission is disabled for + this read-only probe, so it cannot run a second department controller. +- Local documentation links/anchors and current shell examples are checked by + `python3 scripts/check-docs.py`, without executing the examples. + +The existing portal continues to run the verified department release. These +source changes do not redeploy it. Previous live CUDA/TT/browser/isolation +[release evidence](../deploy/k3s/evidence/department-2026-10-09/README.md) remains +valid for that release; it is not claimed as a fresh training/browser run of the +cleanup candidate. No large storage tests, production restarts, account/job/file +mutations or full-disk cleanup were performed. + +Full-host storage and account data live outside the repository. Removing old +source/fixtures does not remove existing Docker containers/volumes or Kubernetes +Jobs/PVCs. Historical account jobs/files remain accessible through the current +legacy workspace. No blanket prune, formatting or history rewrite is used. + +## Reproduce current source checks + +See [testing](testing.md), [developer setup](development.md) and +[documentation index](README.md). Raw local validation output is kept outside Git +under `/home/utmist/mist-cleanup-validation-20261009`; publish only sanitized +summaries. Credentials, session state, kubeconfig and SSH private keys stay out +of committed documentation/evidence. diff --git a/docs/shared-storage-deployment-plan.md b/docs/shared-storage-deployment-plan.md index 3a034ec..86916d7 100644 --- a/docs/shared-storage-deployment-plan.md +++ b/docs/shared-storage-deployment-plan.md @@ -1,246 +1,7 @@ -# Complete foundation: parts 4 and 5 +# Historical guide -Authorized October 8, 2026 after completion of parts 1–3. Continue on -`feat/local-job-foundation` from `8f1b7f1`. The user requests finishing the -remaining foundation instead of handing it to other developers. No permission -prompts are needed for this work. Preserve existing jobs, checkpoints, network -links and unrelated disk contents. +This phase has been superseded by the department platform on main. -## Remaining scope - -### Part 4: shared datasets and result downloads - -- [x] Create `/srv/mist-storage` on QuietBox's existing NVMe filesystem. -- [x] Provide NFSv4 shared storage to the two current Kubernetes nodes, over - their existing LAN. Keep QuietBox's router Ethernet cable connected. -- [x] Install the NFS host client/server packages and persistent configuration. -- [x] Add shared persistent volumes and mount storage in the API. -- [x] Upload datasets through the API and website with size limits/progress, - stable dataset IDs, checksums, ready/error handling, and atomic publication. -- [x] Select a dataset for a job; attach only that dataset read-only at `/inputs`. -- [x] Store new job outputs on shared storage; preserve the `/outputs` and - checkpoint aliases. Keep old local checkpoints retrievable. -- [x] List and download saved job files with ownership checks, streaming, and - protection against traversal and escaping symlinks. Do not execute archives. -- [x] Verify an uploaded dataset is consumed by jobs on both physical machines, - with actual accelerator computation and persistent downloadable results. - -Use a bounded pilot budget (initially 100GiB for new datasets/outputs), an -upload limit (initially 2GiB per dataset), and explicit cleanup/retention docs. -NFS is central storage, not a backup. Preserve the existing checkpoint PVCs. -Metadata must not be writable by workload containers through output mounts. - -### Part 5: login and private deployment - -- [x] Use maintained Better Auth for separate Mist member accounts. Keep the - existing Go executor and add a small auth service with persistent SQLite on - a node-local volume, not SQLite on NFS. -- [x] Seed the first administrator from protected local credentials, outside - Git/tool output. Disable public signup; administrators add members. -- [x] Authenticate API access, derive job/dataset ownership from the session, - and verify isolation with two actual member sessions. -- [x] Add login/logout, current account, password change and member management - UI. Remove prototype profile/account behavior from the active workflow. -- [x] Serve the production frontend with same-origin API/auth proxying and - persistent services; avoid relying on Vite for the deployed website. -- [x] Make the site accessible through the existing private Tailscale network. - Use HTTPS through Tailscale Serve if already available; otherwise keep a - private Tailnet endpoint and record the HTTPS prerequisite accurately. -- [x] Add startup, restoration, membership, image approval, storage quota, - backup and cleanup documentation and repeatable deployment scripts. -- [x] Verify browser login → upload → select dataset/image/device → run → logs - → download, from both machines, plus denial of anonymous/other-owner access. -- [x] Commit tested changes and record exact deployed image/evidence paths. - -## Decisions - -- Keep image-reference submission and current NVIDIA/TT allocation units. - Arbitrary TT models, distributed training, credits, Jupyter, chaining and - image build/archive upload services are outside the five-part foundation. -- Deploy privately; no public domain, Funnel, paid service or external account - provisioning is implied. Use existing host and cluster access. -- A shared tailnet infrastructure account does not identify individual team - members; Mist accounts provide independent ownership. -- Existing owner `utmist` and retained jobs remain visible to the initial - administrator through an explicit migration mapping. -- HTTP cookies alone are not authorization. Go verifies sessions against the - auth service, and state-changing routes require trusted browser origins. -- Test first, then replace the deployed API/frontend with the verified build. - -## Resume information - -- Go/Node runtimes: `~/.local/share/mist-runtimes/go-1.25.1/bin/go` and - `~/.local/share/mist-runtimes/node-22.16.0/bin/node`. -- Kubeconfig: `/home/utmist/.kube/config`; k3s nodes `utmist-z1opa08`, `utmist-tt`. -- QuietBox SSH: `utmist-tt@100.95.175.37`, identity - `/home/utmist/.ssh/id_ed25519_quietbox_codex`, IdentitiesOnly/BatchMode. -- Authorized host operations can use Docker bind/chroot. Package installation - needs host networking because host systemd-resolved listens on loopback. -- Current API deployed in `mist-system`; workloads in `mist`. -- Main tailnet hostname: `utmist.tail459b5e.ts.net`; IP `100.73.139.66`. -- Baseline evidence: `deploy/k3s/foundation-results.json`, - `/home/utmist/mist-foundation-results-2026-10-08`. - -## Progress - -- Plan recorded; repository clean at start. QuietBox SSH and authorized Docker - host access work. QuietBox still has approximately 1.9TiB available. -- Tailscale is connected; no Serve configuration exists. Existing Nginx is - active. Preserve its current default service while adding the Mist endpoint. - -## Completed verification (October 9 UTC) - -- Actual production browser flow passed: login, dataset upload, CPU and both NVIDIA - GPUs, one TT board/two chips, logs, saved file download, account/member actions, - ownership isolation and deactivation. No browser runtime errors. -- QuietBox submitted an authenticated NVIDIA training job, which ran on the main - node and returned downloadable loss metrics. -- Focused Go tests (including race checks), CLI tests/login/logout, frontend - interaction tests, TypeScript and production build passed. -- Old checkpoint PVCs preserved; legacy outputs copied to shared storage. -- NFS/client firewall configuration and production restart/backup checks are - documented in `docs/complete-foundation.md` and `deploy/private/README.md`. -- Private HTTP website is live. Tailscale Serve is disabled; HTTPS requires the - tailnet administrator to enable Serve, accurately recorded as a prerequisite. - -Final review: frontend dependencies now use the checked-in npm lockfile; the -obsolete Bun lockfile was removed. CLI login/logout use real sessions, and -`job submit --dataset` attaches an owned dataset. SQLite backup integrity was -`ok`; restart checks preserved historical/new jobs, datasets and downloadable -results. The production website does not depend on a terminal session. - -## Department rollout scope decision — October 9, 2026 - -The user plans to introduce Mist to several research teams next week and has -explicitly excluded these additions from that rollout: - -- Automatic backups and additional recovery testing. -- HTTPS/Tailscale Serve enablement. -- Monitoring and storage cleanup features. - -The remaining proposed priorities are team membership/permissions and workload -isolation, team resource quotas, fair queuing, representative research workload -and capacity tests, and a documented self-service container-image workflow. -These are proposed follow-up work, not completed foundation features. - -The current access model remains private HTTP over the existing Tailscale -network. Existing manual operating procedures are documented in the foundation -guide. Credits, special job priorities, Jupyter, chaining, and browser image -upload/build services remain outside the immediate rollout scope. - -### Team self-service clarification - -The user clarified that administrators assign each team's conditions and limits; -teams normally do not request administrator permission for individual actions. -The intended rollout model is: - -- Administrators configure membership, permissions and team resource policies. -- Team members upload datasets, submit compatible container images/references, - run jobs and retrieve results independently within their assigned policies. -- The platform validates those policies and admits/queues requests automatically; - ordinary submissions do not require a human approval step. -- Replace the proposed per-image administrator approval process with a documented - self-service image policy. Define allowed image sources and runtime compatibility - as platform/team rules rather than approving each new image manually. - -Current foundation behavior remains distinct: individual owned jobs are already -self-service, but image eligibility uses a global exact-reference allowlist and -Tenstorrent uses the fixed tested runtime profile. Team policies and self-service -custom image eligibility are follow-up implementation work. - -### Team membership and storage requirements - -The user requires multiple teammates per team, explicitly enrolled as members. -Each team needs one team storage area containing a common folder and a folder -for each member. The user clarified that member folders are visible within the -team; they are not private from teammates: - -- Authorized team members can browse/read the team's common folder and the - individual member folders within that team. -- Member folders organize each teammate's files; they do not create a separate - privacy boundary within the team. Write/delete permissions need an explicit - policy and are not implied by permission to view files. -- Members of another team cannot discover, list, read, download, modify or use - the team's files through jobs unless explicitly granted access. -- Explicit cross-team grants must specify their recipients, storage scope and - permitted actions; grants must be revocable and checked on subsequent access. -- All dataset/file access and job input/output attachment must verify the - authenticated member, team membership or explicit grant, and selected scope. -- Workloads mount only their authorized selected input and output locations. - File and network access from running containers must enforce the same boundary; - a UI filter or a differently named folder is insufficient isolation. -- Administrators assign memberships and team conditions. Normal uploads and - submissions within those conditions remain self-service. - -These are new team features. The completed foundation currently provides -per-account ownership checks, rather than this team-sharing/membership model. - -Example intended organization: - -```text -storage/ - team-a/ # visible to Team A members - common/ - alice/ - bob/ - team-b/ # visible to Team B members - common/ - carol/ - dan/ -``` - -Folder names illustrate the organization; enforced authorization determines -access. Cross-team sharing exposes only the explicitly granted scope. - -### Portal login and administrator workspace - -The same portal and login page serve administrators and regular members. -Authenticated roles determine which controls appear and which API actions are -authorized; there is no separate administrator account required for ordinary -research work. - -- Regular members use their team workspace to upload files, submit jobs, view - logs and retrieve results within the team's policies. -- Administrators have the same research workflow plus administration controls - for teams, memberships, resource limits, image policies and access grants. -- An administrator can also be a team member. Ordinary uploads and jobs use the - selected team's membership, storage permissions and resource policies. -- Administrative actions must be explicitly authorized on the server; hiding - controls in the UI alone is insufficient. Administrative authority is distinct - from membership and does not implicitly make the account a member of every - team or bypass limits on its own research jobs. - -The foundation already supports login, basic member administration and ordinary -administrator jobs. The team workspace and additional team administration -controls described here remain follow-up work. - -### Rollout review: implementation details still to define - -The main workflow is covered by the proposed rollout scope. These details belong -within that work; they are not completed features or authorization to implement -every proposed default: - -- File permissions: distinguish team-wide read access from write/delete access. - Proposed default is members managing their own folders, with explicitly - assigned write permissions for the common folder and cross-team grants. -- Membership and grant revocation: deny subsequent access immediately and define - how to handle queued/running jobs that already hold access. Revoking a browser - permission does not remove a file mount from an existing running job. -- Storage quota enforcement: count datasets and generated results, including - writes from running containers. Upload-only checks do not enforce a team cap. -- Failure handling: expose useful reasons for image pull failure, full storage, - unavailable devices and offline nodes; define cancellation, runtime limits and - retry behavior so jobs release resources and do not silently duplicate work. -- Rollout capacity and migration: verify representative dataset/model sizes and - GPU/TT runtime compatibility, and preserve existing account-owned files/jobs - when introducing team ownership. Current pilot limits are 2GiB per upload and - 100GiB total new shared storage. -- Private container registries: decide whether launch supports private images; - if supported, credentials must be team-scoped and unavailable to other teams - and their workloads. Public image references do not need registry credentials. - -Account administration should also define an administrator-assisted password -reset path; self-service email recovery and external email provisioning are not -implied. Automatic backups, HTTPS, monitoring and cleanup remain excluded as -requested. Multi-machine distributed training is outside the existing foundation -and should not be represented as tested by the two separate accelerator smokes. +- [Current documentation](README.md) +- [Current operating guide](department-rollout.md) +- [Archived original record](archive/shared-storage-deployment-plan.md) diff --git a/docs/storage.md b/docs/storage.md new file mode 100644 index 0000000..a116de1 --- /dev/null +++ b/docs/storage.md @@ -0,0 +1,100 @@ +# Storage and upload limits + +Current department storage on **QuietBox**, verified October 9, 2026. Both +compute nodes mount scoped NFSv4 paths over their existing LAN. + +## Layout and authority + +```text +QuietBox /srv/mist-storage-volume.img (sparse base filesystem) +└── /srv/mist-storage/ + ├── metadata/ API-owned requests and metadata + ├── legacy/ preserved historical files + └── teams// bounded dataset filesystem + ├── common/datasets/ + ├── members//datasets/ + └── .models/ separate bounded model/result filesystem + ├── common/jobs//outputs/ + └── members//jobs//outputs/ +``` + +Member directory keys are derived internally; use API/UI scopes rather than +constructing filesystem paths. Teammates can read their team's common/member +folders. Members write their own folders; common writing needs assigned authority. +Other teams need an explicit read/use grant. Cross-team grants never permit writes. +Administrators must belong to a team to use its storage. + +Inputs mount read-only at `/inputs`; each job mounts only its own result directory +at `/outputs` and the checkpoint alias. Auth data lives on main's local SQLite +PVC. File ownership/permissions support the mounts; API and workload policy +provide authorization. They are not separate hard quotas for every member. + +## Capacity + +| Limit | Current value/behavior | +|---|---| +| New team datasets | **200 GiB total**, administrator configurable | +| New team models/results | **500 GiB total**, administrator configurable | +| Base logical capacity | **1,600 GiB** sparse ext4 image | +| Combined team allocation budget | **1,500 GiB** across dataset and model pools | +| Per-member hard quota | None; members share their team's corresponding pool | +| Expansion/shrink | Online expansion supported; shrink rejected | +| Team disable/removal | Preserves files and reserved allocation | + +At most two full 700-GiB default allocations fit within the budget before other +reserved allocations are counted. Existing teams retain their configured sizes; +new defaults do not silently resize them. Administrators can allocate smaller +pools for more teams. Policy overcommit is rejected before persistence. + +Dedicated bounded filesystems enforce both upload capacity and workload result +writes. PVC declarations alone do not enforce NFS capacity. The base/PVC sizes +may retain earlier declarative values; actual filesystem checks are authoritative. + +The image is sparse, so logical quota is not a physical disk reservation. During +the audit the base contained about **1.4 GiB** of data, the backing file consumed +about **5.9 GiB** physically and QuietBox had **1.9 TiB** free. These are dated +observations, not permanent capacity guarantees. The earlier unused bulk +reservation was released; unrelated research files were preserved. + +## Uploads + +Use **Datasets → Add dataset**, choose your own/common folder and upload a file +or ZIP. Bodies stream through Nginx and the API. Capacity is bounded by dataset +free space minus a **64 MiB metadata reserve**, with an optional lower +`MIST_MAX_DATASET_GIB` ceiling. There is no fixed 64 GiB upload limit. + +Transfers can run for up to **24 hours** and can be cancelled. An interrupted +transfer restarts from the beginning; resumable uploads are not implemented. +ZIP uploads require space for staging plus expanded content and reject unsafe +paths/links and expansion beyond the allowed limit. Dataset deletion is blocked +while any active/queued/terminating authorized job references it. + +Use the saved Files panel to download results. There is no automatic results +cleanup or retention service. Jobs that write outside `/outputs` do not retain +those container files. + +## Administration and inspection + +Set dataset/model sizes in **Teams → Manage → Limits**. The QuietBox provisioning +service checks manifests, mounts bounded images and publishes distinct NFS exports. +The API reports storage ready only when both mounts have the expected identities +and capacities. Missing model storage cannot borrow dataset capacity. + +On QuietBox: + +```bash +systemctl status nfs-server mist-team-storage --no-pager +mountpoint /srv/mist-storage +df -h / /srv/mist-storage +findmnt -t ext4,nfs,nfs4 +``` + +With root access, inspect `exportfs -v`, per-team mounts and the storage-agent +journal when readiness fails. Migration/expansion and service installation are +in [deployment operations](../deploy/private/README.md). Never truncate or unlink +mounted image files, recreate populated filesystems or treat team removal as cleanup. + +Small-file migration, quota bounds and real accelerator input/output paths passed. +The earlier 2 GiB + 4 MiB test is historical evidence; large transfers and +exhaustion were not repeated after the user's request. No 200/500 GiB stress, +automatic backup or disk-failure recovery is claimed. See [testing](testing.md). diff --git a/docs/tailscale-ssh.md b/docs/tailscale-ssh.md new file mode 100644 index 0000000..611a5cf --- /dev/null +++ b/docs/tailscale-ssh.md @@ -0,0 +1,143 @@ +# Tailscale and SSH setup + +Verified against both hosts on **October 9, 2026**. Tailscale **1.102.4** is +running on both. Both use Ubuntu OpenSSH on port **22**, with Tailscale's built-in +SSH handler **disabled**. The private portal uses the existing Tailscale network. + +## Machine addresses + +| Machine | Linux user | Tailscale IP | LAN IP | SSH command | +|---|---|---|---|---| +| NVIDIA/main | `utmist` | `100.73.139.66` | `10.0.0.175` | `ssh utmist@100.73.139.66` | +| QuietBox original | `utmist-tt` | `100.95.175.37` | `10.0.0.112` | `ssh utmist-tt@100.95.175.37` | + +Main's observed MagicDNS name is `utmist.tail459b5e.ts.net`. Its OS hostname has +mixed-case letters; Kubernetes normalizes the node name to `utmist-z1opa08`. +Use the IP commands above when DNS or shared-device name resolution is uncertain. + +## What each login grants + +- **Tailscale access:** reach a shared/authorized machine over the private network. +- **Linux SSH login:** administer that machine with its local account/password or + an authorized SSH key. This password is unrelated to Google/Tailscale login. +- **Mist login:** use the website/CLI with a Mist account and assigned team role. + +Researchers can use http://100.73.139.66:8088 without SSH. Machine sharing does +not create a Mist account or automatically permit administrator SSH. + +## Check the existing setup + +Run locally on either machine: + +```bash +tailscale version +tailscale status +tailscale ip -4 +systemctl is-active tailscaled ssh +``` + +From an authorized administrator device, connect with the table's SSH command. +An SSH key connection to QuietBox was checked during this documentation audit. +Both hosts' OpenSSH configurations enable public-key and password authentication. +Passwords are intentionally absent from this repository. + +## Set up a fresh Ubuntu host + +Run these steps from its local console or an existing administrator session. +For an already connected host, verify its current tailnet before changing login. + +```bash +curl -fsSL https://tailscale.com/install.sh | sh +sudo systemctl enable --now tailscaled +sudo tailscale up +``` + +Open the displayed authentication link and join the intended existing tailnet. +Use its administrator console to manage machine sharing or network access. +Do not log out an existing infrastructure node merely to repeat installation. + +Install Ubuntu's SSH server and choose regular SSH over Tailscale: + +```bash +sudo apt update +sudo apt install openssh-server +sudo /usr/sbin/sshd -t +sudo systemctl enable --now ssh +sudo tailscale set --ssh=false +tailscale ip -4 +``` + +Disabling the built-in SSH handler makes Ubuntu OpenSSH handle authentication. +Tailscale SSH has its own policy and can reject connections even when ordinary +network traffic is allowed. Changing handlers may interrupt a session that uses +Tailscale SSH; perform the switch from a local console when applicable. + +## SSH keys and convenient aliases + +On your administrator laptop, use an existing key or create one: + +```bash +ssh-keygen -t ed25519 +ssh-copy-id -i ~/.ssh/id_ed25519.pub utmist@100.73.139.66 +ssh-copy-id -i ~/.ssh/id_ed25519.pub utmist-tt@100.95.175.37 +``` + +`ssh-copy-id` requires an existing allowed login and adds only the public key. +Alternatively, a host administrator can install the `.pub` contents in that +Linux user's `~/.ssh/authorized_keys`. Keep private keys on their owner device. + +Optional entries in your laptop's `~/.ssh/config`: + +```sshconfig +Host mist-main + HostName 100.73.139.66 + User utmist + IdentityFile ~/.ssh/id_ed25519 + IdentitiesOnly yes + +Host mist-quietbox + HostName 100.95.175.37 + User utmist-tt + IdentityFile ~/.ssh/id_ed25519 + IdentitiesOnly yes +``` + +Connect with `ssh mist-main` or `ssh mist-quietbox`. Adjust the IdentityFile when +using a different key. A successful network connection still needs valid Linux +credentials and access permitted by the host rules. + +## Current host restrictions + +The installed `mist-portal-firewall.service` permits ordinary Tailscale sources +to main's TCP **8088**, denying direct SSH/backend/cluster access. Infrastructure +IPs and the recorded administrator device **100.99.194.52** retain broader access. +The script contains matching IPv6 entries. Adding an administrator device needs +an explicit update to [the host allowlist](../deploy/private/portal-firewall.sh), +plus any applicable tailnet policy. LAN rules are separate. + +A plain SSH public key does not bypass this firewall. Access from an external +researcher's device has not been verified by this host-side documentation audit. + +The GCP jump-host, port-2025 and manual Discord-reservation instructions from the +old guide are superseded for this installation. Its plaintext password has been +removed from current documentation; Git history still contains that old value. +Retire that credential if it remains valid. No passwords were changed by this audit. + +## Troubleshooting + +| Symptom | Check | +|---|---| +| Timeout or host unreachable | Tailscale connectivity, machine online, host allowlist and tailnet access | +| `tailnet policy does not permit you to SSH` | Built-in Tailscale SSH handler/policy; current hosts use regular OpenSSH | +| `Permission denied` | Correct Linux user, authorized key or Linux password | +| Browser loads but cannot sign in | Mist account/session and trusted browser origin; SSH credentials are unrelated | +| SSH works but worker cannot join | k3s currently uses LAN `10.0.0.175:6443`; tailnet membership alone does not supply that route | + +The current website is private HTTP over Tailscale. Public hosting, Serve/Funnel +and HTTPS setup were excluded from this release. Tailscale subscription limits +should be checked in the account console; machine count and user-seat count are +different and no fixed seat limit is asserted here. + +References: [Tailscale Linux installation](https://tailscale.com/docs/install/linux), +[Tailscale SSH](https://tailscale.com/docs/features/tailscale-ssh), +[Ubuntu OpenSSH](https://documentation.ubuntu.com/server/how-to/security/openssh-server/). diff --git a/docs/testing.md b/docs/testing.md index 9659d1e..275c3c3 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1,67 +1,82 @@ # Testing -## Current Kubernetes department path - -Use the Go and Node runtimes recorded in `deploy/private/build-and-deploy.sh`. +Use the pinned Go/Node runtimes in [developer setup](development.md). The root +workspace joins the API and CLI. From the repository root: ```bash -go -C src test ./... -run 'Test(Department|Kubernetes|Shared|Dataset|Auth|Member|Image|Hardware|Result)' -count=1 -go -C cli test ./... +go test ./src/... ./cli/... -count=1 +go test -race ./src/... ./cli/... -count=1 +go vet ./src/... ./cli/... +npm --prefix web-interface ci npm --prefix web-interface test npx --prefix web-interface tsc --noEmit -p web-interface/tsconfig.json +npm --prefix web-interface run lint +npm --prefix web-interface run build +python3 scripts/check-docs.py ``` -The Go suite uses fake Kubernetes clients for permissions, scoped storage, queued -admission/restarts and pending TT allocation accounting. React tests cover job -submission/logs/cancellation/pagination and search/keyboard/focus behavior. - -The real pilot checks are in `deploy/k3s/verify_department.mjs`, -`verify_department_browser.mjs`, `verify_department_runtime.py`, -`verify_department_remote_cli.py`, `verify_department_admission.py`, and -`verify_large_dataset.py`, and `verify_split_storage.py`. They use protected -local credentials and synthetic fixtures. API restart, deactivation, quota exhaustion and firewall checks should -run in an idle maintenance window. They preserve historical user data; the large -file verifier removes only its own uploaded probe after its Job exits. - -The browser verifier reuses its fixtures by default. Optional -`MIST_VERIFY_NEW_MEMBER=1 MIST_VERIFY_NEW_TEAM=1` exercises fresh creation dialogs, -deactivating/renaming its old fixtures and retaining their files/history. Each new -team fixture uses 1 GiB datasets + 1 GiB models; disabling retains both allocations. - -[Recorded evidence](../deploy/k3s/evidence/department-2026-10-09/README.md) covers -actual accelerator training, isolation, hard capacity, grants/revocation, CLI -submission from TT to NVIDIA, the larger dataset and browser workflows. - -## Legacy Redis/Docker integration fixtures - -The full historical Go integration suite needs the original Docker/Redis -fixtures. The release check compiles the test binaries, runs them against a -separate Redis container with no published ports, and mounts the host Docker -socket for the existing CPU container fixtures. It never flushes the preview or -production Redis data. See the [release validation record](../deploy/k3s/evidence/department-2026-10-09/release-validation.json) -for the full results. -The current Kubernetes execution path does not use Redis. - -### Setup +## Automated scope -For testing, docker engine and docker-compose should be installed - see [here](https://docs.docker.com/engine/install/). -Additionally, golang should be [installed](https://go.dev/doc/install). +Go tests use fake Kubernetes clients and small temporary filesystem fixtures for +job state/cancellation/validation, hardware allocation, authentication/isolation, +team grants/quotas/admission/restarts, scoped storage and uploads/ZIP checks. +Application tests cover unavailable-cluster startup and the scoped compatibility +status endpoint. CLI tests exercise real request/response behavior through test +servers. React tests cover submission, image defaults/editing, pagination, +logs/cancellation and accessible shared controls. Migration tests use tiny files. They require root to verify ownership; a normal +user run reports skips. Run them as root in an isolated environment, for example: -### Running the historical tests - -Start containers by running `docker-compose up` in the main directory (docker must be installed). - -Run tests on the main application by running `go test` in the `src` directory. Note that `TestIntegration` will fail without the containers in [`docker-compose.yml`](../docker-compose.yml) up and running. - -## Scope of release verification - -Full backend, Docker module, CLI and logger tests are checked before publication, -along with React tests, TypeScript, frontend lint/build, migration tests and the -real browser flow. Real CUDA and Tenstorrent training checks cover allocated -accelerators, input/model mounts and checkpoint downloads. The visible regular -member Container demonstration verifies a packaged image's own startup command. +```bash +sudo python3 -m unittest discover -s deploy/k3s/storage -p test_migration.py -v +``` -Large uploads/exhaustion are not repeated at the user's request. The follow-up -uses a 260,729-byte dataset. This does not claim 200/500 GiB transfer stress, -every possible edge case, arbitrary research model compatibility, HA, physical -disk-failure recovery, or the features explicitly excluded from the rollout. +The tests replace the store root with a temporary directory and do not access +live storage. Docker can provide an isolated root environment when sudo is +unavailable. + +Current automated suites do not require Redis or Docker. The old supervisor, +Docker manager/logger tests and fake CLI-config tests were removed with their +retired implementations. This is not a test skip: those products are no longer +part of the application. The earlier full legacy suite results remain as dated +[pre-cleanup evidence](../deploy/k3s/evidence/department-2026-10-09/release-validation.json). + +## Real installation checks + +| Verifier | Purpose | +|---|---| +| `deploy/k3s/verify_department.mjs` | API, auth, teams, grants, queue/revocation, storage and training | +| `deploy/k3s/verify_department_browser.mjs` | Real admin/member UI, custom image flow, pagination and responsive layout | +| `deploy/k3s/verify_department_runtime.py` | Concurrent TT allocations, failure and execution deadline | +| `deploy/k3s/verify_department_remote_cli.py` | SSH to TT, submit through CLI, execute on NVIDIA | +| `deploy/k3s/verify_department_admission.py` | Admission denials for unsafe workload specifications | +| `deploy/k3s/verify_split_storage.py` | Separate dataset/model pools, migration preservation and small-file training | +| `deploy/k3s/verify_large_dataset.py` | Earlier large-upload fixture; not part of ordinary checks | +| `deploy/private/verify-portal-firewall.sh` | Installed host chains with a synthetic untrusted source | + +These require existing protected credentials, node addresses and operators. +Inspect the script before rerunning: API restart, revocation/deactivation, quota +exhaustion and firewall tests belong in an idle maintenance window. Verification +fixtures preserve history/files; new browser teams reserve 1 GiB per pool even +after being disabled. Do not repeatedly create fixtures without accounting for +reserved storage. + +The retained `verify_tenstorrent_allocation.py` is an administrator infrastructure +diagnostic; it bypasses the portal's team admission. Pre-team browser/API +verifiers have been retired because they no longer reflect current auth/storage. + +## Evidence and limits + +The [October 9 evidence](../deploy/k3s/evidence/department-2026-10-09/README.md) +records real CUDA/TT training, separate board/GPU reservations, input/output +mounts, checkpoint reload/download, grants/revocation, queue restart and browser +flows. The regular-member Container demonstration preserved its own image CMD; +that image was locally preloaded, not registry-pushed. + +The [cleanup record](repository-cleanup.md) separates new source checks from +previous live-release evidence. A running service is not silently redeployed by +checking out a branch or editing documentation. + +Large uploads and exhaustion are not repeated at the user's request. The +separate-pool follow-up used a 260,729-byte dataset. No 200/500 GiB transfer stress, +every possible edge case, arbitrary research model, HA or disk-failure recovery +is claimed. Excluded features remain outside the release. diff --git a/pre_compute_platform.md b/pre_compute_platform.md index 870ef40..a23531c 100644 --- a/pre_compute_platform.md +++ b/pre_compute_platform.md @@ -1,194 +1,12 @@ -# Quick guide — how to SSH into the Tenstorrent “quiet box” (shared access) +# Machine access -Below is a practical, copy-pasteable guide you can share with your team so people can connect to the Tenstorrent quiet box through the GCP SSH jump proxy. It covers key generation, how to share your public key, the SSH `config` entries (exactly what you asked to include), connection steps, the temporary manual reservation process via Discord, rules about containers, troubleshooting, and good safety practices. +The earlier GCP jump-host, shared-password and manual reservation guide is +superseded by the current private Mist installation. ---- +- [Tailscale and SSH setup](docs/tailscale-ssh.md) +- [Research portal and team job submission](docs/department-rollout.md) +- [k3s cluster setup](docs/k3s-setup.md) -# 1) What we’re doing (short) - -We expose the quiet box only behind an SSH jump host on GCP. Users get access by giving us their SSH **public key**; access is time-limited until a compute reservation system exists. People must work inside their own container while using the quiet box. Reservations are handled manually through a Discord channel for now — see section 4. - ---- - -# 2) How to create and send your SSH public key - -1. Generate an SSH key pair locally (if you don’t already have one). Recommended: ed25519 (modern, compact, secure). - -```bash -# create a new key (press enter to accept default file and optionally add a passphrase) -ssh-keygen -t ed25519 -C "your.email@domain.com" -``` - -2. Print your public key and copy it: - -```bash -cat ~/.ssh/id_ed25519.pub -``` - -3. Send **only** the public key string (the `.pub` contents) to the admins which are Ambrose Ling and Kenny Cui (e.g., paste into the Discord message or via whatever secure channel the team uses). Example public key looks like: - -``` -ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI... user@host -``` - -(Do **not** send your private key — file `~/.ssh/id_ed25519`.) - -If you already use `~/.ssh/id_rsa` / other keys, you can send whichever public key you prefer — ed25519 is preferred. - ---- - -# 3) Add this to your `~/.ssh/config` - -Put the following in your SSH config file (`~/.ssh/config`) **exactly** as shown (or append if entries already exist): - -``` -Host mist-jump - HostName 34.170.88.115 - User utorontomist - -Host mist - Hostname localhost - User utmist-tt - Port 2025 - ProxyJump mist-jump -``` - -Notes: - -* After the admins add your public key to the jump host / target server, you’ll connect with: - -```bash -ssh mist -``` - -Login in with server password: `welovett2025!` - -* The `ProxyJump` causes your SSH client to first connect to `mist-jump` (the GCP host), then forward into the quiet box on port `2025` as user `utmist-tt`. - ---- - -# 4) Manual reservation (Discord workflow) - -Because there is no automated reservation system yet, we use a Discord channel to manage access. Suggested workflow to keep things fair and safe: - -1. **Channel name**: `#quietbox-reservations` (or similar). -2. **How to claim time**: Post a short message like: - -``` -Claim: @your-name — Project: — From: 2025-10-29 14:00 EDT — To: 2025-10-29 16:00 EDT -``` - -3. **Required fields**: who, project, start & end times (use local timezone, include timezone), and container image or name you’ll use. Until there is clear demand of machine usage times we will stick to this. If we see that multiple projects need to use the same machine -4. **Admins confirm**: An admin reacts/acknowledges the claim (e.g., ✅). Until a claim is acknowledged, you should not start work. -5. **During your slot**: If you finish early, post an “ended” message so others can update the slot. If you need to extend, post in the channel and wait for confirmation. -6. **Enforcement**: If someone is using the box outside their claimed time, ping the channel owner/admin to resolve. - -Suggested message template to claim: - -``` -Claiming quietbox: -- User: @ambrose -- Project: example-ml -- Start: 2025-10-29 14:00 EDT -- End: 2025-10-29 16:00 EDT -- Container Name: -``` - ---- - -# 5) Rules while using the quiet box (must-follow) - -* **Work only inside your container.** Do not run experiments on the host OS. Use Docker / Podman / the approved container runtime. -* **No background or persistent processes** outside your container. Kill any long-running jobs when your slot ends. -* **Follow agreed resource limits.** If admins set GPU/CPU limits, stay within them. -* **Clean up**: remove temporary large files from shared storage after your run, or move them to a location designated for your project. -* **Be respectful**: If another user needs time urgently, coordinate in the Discord channel. -* **Security**: never share private keys or credentials in Discord or public chat. - ---- - -# 6) Example container usage patterns - -Generic Docker example (adjust to your image and options): - -```bash -# pull image -docker pull ghcr.io/tenstorrent/tt-metal/tt-metalium/ubuntu-22.04-dev-amd64:latest - -# run container with GPU access and mount a workspace -docker run -it \ - --name your_name:project_name\ - -v /dev/hugepages-1G:/dev/hugepages-1G \ - --device /dev/tenstorrent \ - -v $(pwd):/home/utmist-tt/UTMIST \ - -v /mnt:/mnt \ - ghcr.io/tenstorrent/tt-metal/tt-metalium/ubuntu-22.04-dev-amd64:latest \ - /bin/bash -``` - -Inside the container you run your jobs. `--rm` ensures the container is removed when you exit. - -If your setup uses a different runtime (Podman, singularity), follow the project's standard container instructions. - ---- - -# 7) Connection checklist & verification - -Before connecting: - -* You’ve sent your public key to admins and they confirmed it was installed. -* Your `~/.ssh/config` contains the `mist-jump` and `mist` entries shown above. - -To connect: - -```bash -ssh mist -``` - -If you want verbose logs (helpful for troubleshooting): - -```bash -ssh -vvv mist -``` - -Quick verification steps: - -* `ssh -G mist` prints the resolved config for debugging. -* Confirm you land inside the correct user (prompt or `whoami`). -* Once connected, check you’re inside a container (e.g., `cat /proc/1/cgroup` or ask admins about the container launch process). - ---- - -# 8) Common troubleshooting - -* **Permission denied (publickey)** - - * Ensure the admin actually installed your `~/.pub` key on the target account. - * Confirm you’re using the right private key (`ssh -i ~/.ssh/id_ed25519 mist` to force it). - * Check file permissions: `~/.ssh` should be `700`, private key `600`. -* **ProxyJump failing / connection to jump host refused** - - * Confirm `mist-jump` HostName IP is reachable: `ssh -vvv mist-jump`. - * Ensure your local firewall or corporate network allows outbound SSH (port 22). -* **Port 2025 connection refused** - - * That implies the service on the jump host isn’t forwarding correctly yet; ask admins to confirm the tunnel on GCP and the quiet box service are up. -* **Agent forwarding issues** - - * If you rely on agent forwarding, make sure you started `ssh-agent` and added keys with `ssh-add`. -* **If you get unexpected host key warnings** - - * Verify fingerprints with an admin before accepting. Do **not** blindly accept host key changes. - -Use `ssh -vvv mist` and share the logs with admins if you need help. - ---- - -# 9) Session end & cleanup checklist - -Before you finish: - -* Stop any running jobs inside your container (`Ctrl+C` or `pkill` inside container). -* `exit` the container and ensure the container is stopped/removed if required. -* Log out of the SSH session (`exit` or `logout`). -* Post in Discord that your slot ended, or update the reservation message. +Passwords are not published in current documentation. The previous password +remains in Git history; retire it if still valid. Researchers use Mist accounts +and team membership, while administrator SSH is separately restricted. diff --git a/src/jobs.md b/src/jobs.md index a70e6d9..8ad28da 100644 --- a/src/jobs.md +++ b/src/jobs.md @@ -1,71 +1,54 @@ -# Mist job lifecycle - -## Kubernetes executor (default) - -1. The API validates the image, script or command, accelerator count, CPU, - memory, and deadline. It accepts Python/shell scripts up to 32 KiB or an - explicit command/argument arrays, or an explicitly selected image's default - entrypoint. Custom CPU/NVIDIA containers preserve image WORKDIR by default; - `working_directory` provides an absolute override. An image allowlist is enforced. -2. Submission creates a unique `mist-...` Kubernetes Job in `mist`. The Job - stores its submission and authenticated member owner in metadata. Scripts use - a ConfigMap owned by the Job. Retries are disabled; the deadline is enforced - by Kubernetes. Workload pods receive no service account token. -3. Kubernetes places CPU/NVIDIA jobs on the NVIDIA server and Tenstorrent - jobs on QuietBox. NVIDIA requests reserve whole GPUs. Tenstorrent requests - reference a board-count ResourceClaimTemplate; each pod gets a separate - claim and CDI injects only its allocated board devices. One board contains - two chips. Insufficient resources leave the job `Scheduled`. -4. Mist reads actual Job/Pod state and logs. States are `Scheduled`, - `InProgress`, `Success`, `Failure`, and `Cancelled`. Exit status comes from - the workload container; a nonzero exit is never reported as success. -5. Cancellation suspends the Job, stops its pods, and releases reservations. - Cancellation time and the last 32 KiB of logs are kept in Job annotations. - Repeated cancellation is safe. Finished jobs cannot be cancelled. -6. The API creates a job directory on shared NFS storage before submission. The - workload mounts only its own directory at `/outputs` and - `/checkpoints/`. A selected owned ready dataset is mounted read-only - at `/inputs`. Managed environment variables identify these locations. - Legacy jobs retain original local PVCs; copied results remain downloadable. - - -There is no Job TTL. Retained Jobs preserve history across API restarts; -normal logs depend on retained pods and kubelet log retention. Removing Jobs, -pods, or PVCs can remove history, logs, or checkpoints. External metadata/log -archival remains future work; shared datasets/results are implemented. - -## API - -| Endpoint | Action | -| --- | --- | -| `POST /jobs` | Submit a validated job; returns `job_id` and initial job state | -| `GET /jobs` | List this member's managed jobs | -| `GET /jobs/` | Actual status, placement, resources, exit code and active TT allocation | -| `GET /jobs//logs` | JSON containing recent stdout/stderr, up to 64 KiB | -| `POST /jobs//cancel` | Cancel a running or waiting job | -| `DELETE /jobs/` | Alias for cancellation, preserving history | -| `GET /jobs/status?id=` | Compatibility status endpoint | -| `GET /healthz` | Verify access to the Kubernetes Jobs API | -| `GET /hardware` | Live node readiness and whole-device allocations/availability | -| `GET /images` | Server-approved references and compute profiles | - -When `MIST_AUTH_URL` is configured, Better Auth sessions identify each member. -Go authorizes every data request and rejects anonymous/other-owner access. -`MIST_PILOT_OWNER` remains for explicitly local development and legacy owner -migration. Namespaced workload RBAC and the read-only inventory ClusterRole -limit the API's cluster permissions. Credits, team quotas, priority admission -and distributed training are subsequent work. See -[the complete guide](../docs/complete-foundation.md) for login and storage routes. - -Full request fields, response semantics, image ENTRYPOINT/CMD behavior, -resource limits, private registry setup, and output access are documented in -[the local foundation guide](../docs/local-job-foundation.md#api-contracts). -Active TT allocation details may disappear once the DRA claim is released; -saved training allocation files remain on the corresponding PVC. - -## Legacy Docker/Redis executor - -`MIST_EXECUTOR=docker` selects the older Scheduler/Supervisor implementation. -It stores jobs as Redis hashes and consumes Redis streams. Its execution -logic is incomplete and does not use the Kubernetes allocation described -above. Do not use its placeholder success results as proof of training. +# Job lifecycle and API + +Current authenticated team execution. See [the operating guide](../docs/department-rollout.md#developer-api-contract) +for the complete API and [architecture](../docs/architecture.md) for source layout. + +## Lifecycle + +1. Authenticate the Better Auth session and select an active team. Validate image + registry, workload, device count, CPU/memory/runtime and writing authority. +2. Create a suspended `mist-...` Job in `mist-team-` and persist its + submission/creator metadata. Scripts use a Job-owned ConfigMap. +3. A single durable controller applies fair team admission and resource limits. + Kubernetes then schedules the pod using whole GPU resources or TT DRA claims. +4. Read actual Job/pod status, placement, logs and exit code. API states remain + `Scheduled`, `InProgress`, `Success`, `Failure` and `Cancelled`. Scheduled + includes queued/admitted/pending work; inspect the message/queue metadata. +5. Cancellation/revocation suspends the Job and removes pods, freeing allocated + devices. It preserves outputs/history; explicit cancellation saves recent logs. +6. Dataset mounts are read-only `/inputs`. Outputs are the job's own model-pool + directory at `/outputs` and `/checkpoints/`. Finished outputs remain. + +A team's timeout starts after admission, not while waiting in the Mist queue. +The controller is single-replica. Jobs have no automatic retry or TTL. Retained +pods/kubelet retention determine normal log availability; cancellation keeps the +last 32 KiB. Workloads receive no service-account token or arbitrary host mounts. + +Historical account jobs and original local PVCs remain available through the +explicit legacy workspace. New deployed submissions/uploads require a team. +The former Redis/Docker executor and fake supervisor endpoints are retired. + +## Job endpoints + +Browser/CLI calls use the portal prefix `/api`; Nginx strips it before Go. +Team work sends `X-Mist-Team` with a real team ID. Go derives identity and authority +from the session, not request ownership fields. + +| Method/path after `/api` | Action | +|---|---| +| `POST /jobs` | Submit; **201** with `job_id` and job metadata | +| `GET /jobs` | List permitted team jobs, or selected legacy account history | +| `GET /jobs/{id}` | Actual state, placement, resources, exit and active TT allocation | +| `GET /jobs/{id}/logs` | Recent stdout/stderr JSON, up to 64 KiB | +| `POST /jobs/{id}/cancel` | Authorized cancellation | +| `DELETE /jobs/{id}` | Cancellation alias, preserves history | +| `GET /jobs/{id}/files` | Output listing; `/files/download?path=...` downloads | +| `GET /jobs/status?id=...` | Compatibility status route using the same scoped executor | +| `GET /hardware`, `/images` | Capacity and image/runtime policy profiles | +| `GET /healthz` | Kubernetes Jobs API availability | + +Scripts are limited to 32 KiB; request JSON is bounded and rejects unknown fields +or multiple objects. An explicit image with no command/arguments preserves its +ENTRYPOINT/CMD. CPU/NVIDIA preserve image WORKDIR without an override. TT requires +compatible libraries and an explicit host/container runtime choice when applicable. +Custom tagged images are self-service within allowed registry rules. diff --git a/web-interface/README.md b/web-interface/README.md index e67bc71..160d903 100644 --- a/web-interface/README.md +++ b/web-interface/README.md @@ -1,77 +1,65 @@ # Mist web interface -The `/jobs` page submits real Kubernetes jobs. `/machines` shows live hardware -inventory. `/datasets` uploads owned datasets; `/profile` manages the real account -and (for admins) members. The production site is http://100.73.139.66:8088. -See [the complete operating guide](../docs/complete-foundation.md). +The authenticated React portal serves real team Jobs, hardware inventory, +dataset uploads, common/member folder browsing, account administration and team +policy/sharing controls. Production: **http://100.73.139.66:8088** through Tailscale. +See the [operating guide](../docs/department-rollout.md). ## Start locally -Requires Node.js 22.16.0 or newer. Start the Mist API or its port-forward on -`127.0.0.1:3000`, then run: +Use Node.js **22.16.0 or newer**. The main machine's existing API port-forward +listens on `127.0.0.1:3000`; its Vite development service uses port 3001. +Check those services before starting duplicate listeners: ```bash -npm ci -npm run dev -- --host 127.0.0.1 --port 3001 --strictPort +systemctl --user status mist-api-forward mist-web ``` -Open http://127.0.0.1:3001/jobs. Existing user services on this server already -run these endpoints: `systemctl --user status mist-api-forward mist-web`. - -Vite proxies `/api` and `/auth` to port 3000; `MIST_API_PROXY` changes the target. -For a production build, configure same-origin `/api` and `/auth` reverse proxies or set -`VITE_API_URL` at build time. The development proxy is not part of the static build. - -## Jobs and hardware - -- Python scripts, approved container image references, and small training checks. -- Whole NVIDIA GPUs and Tenstorrent boards; each board contains two chips. -- Image defaults, executable/argument overrides, environment variables, - optional working directory, CPU/RAM requests, and deadline. -- Real waiting/running/completed/failed/cancelled states, placement, logs, - exit codes, and cancellation. -- Live whole-device totals, allocations, available counts, and node readiness. -- Persistent shared output location `/outputs`, browser result downloads, - streaming dataset uploads, ZIP validation and read-only `/inputs` selection. - -Image eligibility is server policy, not a guarantee of CUDA/TT compatibility. -TT currently uses its tested image and host runtime. Availability is a snapshot; -Kubernetes controls allocation. CPU/RAM are allocatable capacity, not live usage. - -The deployed service uses maintained Better Auth sessions and per-member ownership. -The UI gates its routes on `/api/session`; Go independently authorizes data routes. -See [the guide](../docs/complete-foundation.md) for API/storage/membership contracts. -A standalone API without `MIST_AUTH_URL` remains an explicitly local developer mode. +With the API/port-forward available and port 3001 free, from this directory: -## Code organization +```bash +npm ci +npm run dev -- --host 127.0.0.1 --port 3001 --strictPort +``` -- `src/auth.tsx`: member gate and maintained auth client. -- `src/components/AccountPage.tsx`: password change/admin membership. -- `src/components/DatasetsPage.tsx`: uploads, progress, owned dataset list. -- `src/api.ts`: typed API requests and response models. +Vite forwards `/api` and `/auth` to the API on port 3000. `MIST_API_PROXY` changes +that target. Authenticate with a real Mist account and select your team; this +preview reaches real jobs/data. See [development](../docs/development.md) for +other-host origins and backend work. Production uses a static Nginx release, +not Vite or these per-user services. + +## Responsibilities + +- `src/auth.tsx`, `teams.tsx`: account/team context; server authorization remains + authoritative for every operation. +- `src/api.ts`: typed API models/requests and upload progress. +- `src/components/JobsPage.tsx`, `components/jobs/`: paginated history, focused + submission dialog, actual logs, cancellation and saved files. +- `src/components/DatasetsPage.tsx`, `TeamStorageBrowser.tsx`: uploads and + authorized common/member/shared storage browsing. +- `src/components/TeamsPage.tsx`, `AccountPage.tsx`: team policy/membership/sharing, + member administration and password controls. +- `src/components/Picker.tsx`, `Tabs.tsx`, `Modal.tsx`, `Card.tsx`: shared controls. +- `src/components/HardwarePanel.tsx`, `MachinesPage.tsx`: live node/device inventory. - `src/hooks/usePolling.ts`: independent polling, cancellation and recovery. -- `src/components/JobsPage.tsx`: page data and actions. -- `src/components/jobs/`: submission form, job cards, log viewer and saved-file downloads. -- `src/components/HardwarePanel.tsx`: shared live inventory table. -- `src/components/MachinesPage.tsx`: machine inventory route content. -- `src/routes/`: TanStack Router file routes; `routeTree.gen.ts` is generated. +- `src/routes/`: TanStack file routes; `routeTree.gen.ts` is generated. + +Compute choices request whole GPUs or TT boards. One TT board has two chips. +Custom tagged images follow team registry policy and need compatible libraries; +TT may use the tested host profile or an image-provided runtime. Container mode +preserves image startup with blank overrides. CPU/RAM inventory shows allocation, +not live utilization. Inputs are read-only; save results in `/outputs`. -Keep hardware policy/execution in the API. Components should describe the -response and request fields without reproducing the Kubernetes scheduler. +The old fake account helpers/chart and unused `use-immer` dependency are retired. +Current behavior and components follow [the design system](../docs/design-system.md). ## Verify ```bash npm test npx tsc --noEmit +npm run lint npm run build ``` -Interaction tests cover state/log/cancel flows, TT board submission, connection -recovery, literal arguments and working-directory settings, and invalid image/ -count rejection. Browser-to-cluster checks are documented in -[deployment verification](../deploy/k3s/README.md#foundation-acceptance). - -The app uses React, TypeScript, TanStack Router, Vite, and Tailwind CSS. -Follow the existing Card/Button styles and explicit form labels. Check only -changed files with Prettier/ESLint rather than rewriting unrelated prototypes. +See [testing](../docs/testing.md) for browser-to-cluster checks and their scope. From c9c356390d4c46440dc8751e5e0a299fedf282fe Mon Sep 17 00:00:00 2001 From: utmist-tt <339893541+utmist-tt@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:49:20 -0400 Subject: [PATCH 3/3] Record cleanup pull request for handoff --- docs/repository-cleanup.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/repository-cleanup.md b/docs/repository-cleanup.md index 149081e..bc619bd 100644 --- a/docs/repository-cleanup.md +++ b/docs/repository-cleanup.md @@ -1,7 +1,8 @@ # Repository cleanup — October 9, 2026 Based on main commit `3431ffd`, the department release merged through PR #110. -Changes are prepared on `chore/main-cleanup-20261009`. The main ruleset still +Changes are published in [PR #111](https://github.com/UTMIST/Mist/pull/111) +on `chore/main-cleanup-20261009`. The main ruleset still requires a pull request and an independent review; Git publication is separate from deploying services.