From a8f0ba2cc110ebd0507e9eef5738d99211e4c9ab Mon Sep 17 00:00:00 2001 From: monkeydioude Date: Sat, 10 Oct 2026 13:39:20 +0200 Subject: [PATCH] jobs API: serve the latest log lines GET /jobs/logs returns the last 1000 log lines, numbered per boot, so a dashboard can follow josuke live without access to journald. Callers pass the previous last line number and boot id to get new lines only. Co-Authored-By: Claude Opus 5.5 --- README.md | 8 +++++ bin/josuke/main.go | 9 +++++- jobsapi.go | 22 ++++++++++++-- jobsapi_test.go | 31 ++++++++++++++++++- logbuffer.go | 75 ++++++++++++++++++++++++++++++++++++++++++++++ logbuffer_test.go | 75 ++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 215 insertions(+), 5 deletions(-) create mode 100644 logbuffer.go create mode 100644 logbuffer_test.go diff --git a/README.md b/README.md index e99e5d0..1429154 100644 --- a/README.md +++ b/README.md @@ -223,6 +223,14 @@ Enabled with `jobs_api`: ] ``` `status` is `running`, `stopping` or `waiting`. +- `GET /jobs/logs` returns the last 1000 log lines, as written to stderr. Lines are numbered from 1 at each boot, and `boot_id` changes at each boot: + ```sh + curl -H 'Authorization: Bearer ' 'http://localhost:8082/jobs/logs?after=41&boot_id=dm14nqdyb9cw' + ``` + ```json + {"boot_id": "dm14nqdyb9cw", "last": 42, "lines": [{"seq": 42, "text": "2026/10/10 13:39:07 job#1 [INFO] sh | build line 1"}]} + ``` + `after` and `boot_id` are optional: pass the previous `last` and `boot_id` to get the new lines only. Every line kept is returned if `boot_id` is not the current one, as josuke restarted in between. - `POST /jobs/{id}/stop` stops a job: ```sh curl -X POST -H 'Authorization: Bearer ' http://localhost:8082/jobs/3/stop diff --git a/bin/josuke/main.go b/bin/josuke/main.go index 8905ea5..fc63e8b 100644 --- a/bin/josuke/main.go +++ b/bin/josuke/main.go @@ -3,8 +3,11 @@ package main import ( "flag" "fmt" + "io" "log" "net/http" + "os" + "time" "github.com/monkeydioude/josuke" ) @@ -28,6 +31,10 @@ func main() { configFileName := flag.String("c", "config.json", "Path to config file, yml or json format") flag.Parse() + // Set before josuke.New: the job loggers write where the log package writes when they are created. + logs := josuke.NewLogBuffer(josuke.LogBufferSize, time.Now()) + log.SetOutput(io.MultiWriter(os.Stderr, logs)) + j, err := josuke.New(*configFileName) if err != nil { @@ -35,7 +42,7 @@ func main() { } j.HandleHooks() - j.HandleJobs() + j.HandleJobs(logs) if j.HealthcheckRoute == "" { j.HealthcheckRoute = "/healthcheck" diff --git a/jobsapi.go b/jobsapi.go index 6f086ca..657c722 100644 --- a/jobsapi.go +++ b/jobsapi.go @@ -20,23 +20,25 @@ type JobsAPI struct { // HandleJobs declares the HTTP handlers of the jobs API, if configured: // - GET lists the running and waiting jobs +// - GET /logs returns the latest log lines // - POST /{id}/stop stops a job -func (j *Josuke) HandleJobs() { +func (j *Josuke) HandleJobs(logs *LogBuffer) { if j.JobsAPI == nil { return } if j.JobsAPI.Token == "" { log.Fatal("[ERR ] jobs_api requires a token") } - route := j.registerJobsAPI(http.DefaultServeMux) + route := j.registerJobsAPI(http.DefaultServeMux, logs) if j.LogEnabled(InfoLevel) { log.Printf("[INFO] jobs API available on %s\n", route) } } -func (j *Josuke) registerJobsAPI(mux *http.ServeMux) string { +func (j *Josuke) registerJobsAPI(mux *http.ServeMux, logs *LogBuffer) string { route := cmp.Or(strings.TrimSuffix(j.JobsAPI.Route, "/"), "/jobs") mux.HandleFunc("GET "+route, j.authorized(j.listJobs)) + mux.HandleFunc("GET "+route+"/logs", j.authorized(listLogs(logs))) mux.HandleFunc("POST "+route+"/{id}/stop", j.authorized(j.stopJob)) return route } @@ -57,6 +59,20 @@ func (j *Josuke) listJobs(rw http.ResponseWriter, req *http.Request) { writeJSON(rw, http.StatusOK, j.queue.list()) } +// listLogs returns the lines logged after the "after" line number of the "boot_id" boot, +// or every line kept if they are missing. +func listLogs(logs *LogBuffer) http.HandlerFunc { + return func(rw http.ResponseWriter, req *http.Request) { + query := req.URL.Query() + after, err := strconv.ParseUint(cmp.Or(query.Get("after"), "0"), 10, 64) + if err != nil { + http.Error(rw, "invalid after line number", http.StatusBadRequest) + return + } + writeJSON(rw, http.StatusOK, logs.Since(query.Get("boot_id"), after)) + } +} + func (j *Josuke) stopJob(rw http.ResponseWriter, req *http.Request) { id, err := strconv.ParseUint(req.PathValue("id"), 10, 64) if err != nil { diff --git a/jobsapi_test.go b/jobsapi_test.go index d2580b6..2addc98 100644 --- a/jobsapi_test.go +++ b/jobsapi_test.go @@ -17,7 +17,7 @@ func Test_Jobs_API_Lists_And_Stops_Jobs(t *testing.T) { queue: newTestQueue(t, 2, "", blockUntilStopped(started)), } mux := http.NewServeMux() - assert.Equal(t, "/jobs", j.registerJobsAPI(mux)) + assert.Equal(t, "/jobs", j.registerJobsAPI(mux, NewLogBuffer(10, testBoot))) call := func(method, path, token string) *httptest.ResponseRecorder { req := httptest.NewRequest(method, path, nil) if token != "" { @@ -58,3 +58,32 @@ func Test_Jobs_API_Lists_And_Stops_Jobs(t *testing.T) { assert.Equal(t, http.StatusBadRequest, call("POST", "/jobs/abc/stop", "s3cr3t").Code) assert.Equal(t, http.StatusMethodNotAllowed, call("GET", "/jobs/1/stop", "s3cr3t").Code) } + +func Test_Jobs_API_Returns_The_Latest_Log_Lines(t *testing.T) { + j := &Josuke{JobsAPI: &JobsAPI{Token: "s3cr3t", Route: "/deploys/"}} + logs := NewLogBuffer(10, testBoot) + writeLines(t, logs, "one", "two", "three") + mux := http.NewServeMux() + j.registerJobsAPI(mux, logs) + call := func(path, token string) *httptest.ResponseRecorder { + req := httptest.NewRequest("GET", path, nil) + req.Header.Set("Authorization", "Bearer "+token) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + return rec + } + bootID := logs.Since("", 0).BootID + + assert.Equal(t, http.StatusUnauthorized, call("/deploys/logs", "wrong").Code) + assert.Equal(t, http.StatusBadRequest, call("/deploys/logs?after=abc", "s3cr3t").Code) + + rec := call("/deploys/logs", "s3cr3t") + require.Equal(t, http.StatusOK, rec.Code) + assert.JSONEq(t, `{"boot_id": "`+bootID+`", "last": 3, "lines": [ + {"seq": 1, "text": "one"}, {"seq": 2, "text": "two"}, {"seq": 3, "text": "three"} + ]}`, rec.Body.String()) + + rec = call("/deploys/logs?after=2&boot_id="+bootID, "s3cr3t") + require.Equal(t, http.StatusOK, rec.Code) + assert.JSONEq(t, `{"boot_id": "`+bootID+`", "last": 3, "lines": [{"seq": 3, "text": "three"}]}`, rec.Body.String()) +} diff --git a/logbuffer.go b/logbuffer.go new file mode 100644 index 0000000..3766179 --- /dev/null +++ b/logbuffer.go @@ -0,0 +1,75 @@ +package josuke + +import ( + "cmp" + "slices" + "strconv" + "strings" + "sync" + "time" +) + +// LogBufferSize is the number of log lines kept in memory for the jobs API. +const LogBufferSize = 1000 + +// LogLine is a log entry, numbered from 1 since josuke started. +type LogLine struct { + Seq uint64 `json:"seq"` + Text string `json:"text"` +} + +// LogPage is the answer to a request for the latest log lines. +type LogPage struct { + // BootID identifies the josuke process: line numbers start over when it changes. + BootID string `json:"boot_id"` + // Last is the number of the last line written, to ask for the next lines. + Last uint64 `json:"last"` + Lines []LogLine `json:"lines"` +} + +// LogBuffer is an io.Writer keeping the last lines written to it, so they can be read through the jobs API. +// It receives the output of the log package, each Write is one log entry. +type LogBuffer struct { + bootID string + size int + + mu sync.Mutex + lines []LogLine + last uint64 +} + +func NewLogBuffer(size int, bootedAt time.Time) *LogBuffer { + return &LogBuffer{ + bootID: strconv.FormatInt(bootedAt.UnixNano(), 36), + size: size, + } +} + +func (b *LogBuffer) Write(p []byte) (int, error) { + text := strings.TrimSuffix(string(p), "\n") + + b.mu.Lock() + defer b.mu.Unlock() + b.last++ + if len(b.lines) == b.size { + // The dropped line is freed when append moves the lines to a new array. + b.lines = b.lines[1:] + } + b.lines = append(b.lines, LogLine{Seq: b.last, Text: text}) + return len(p), nil +} + +// Since returns the lines written after the line numbered after, oldest first. +// Line numbers of another boot mean nothing to this one, every line kept is returned then. +func (b *LogBuffer) Since(bootID string, after uint64) LogPage { + b.mu.Lock() + defer b.mu.Unlock() + + if bootID != b.bootID { + after = 0 + } + start, _ := slices.BinarySearchFunc(b.lines, after+1, func(l LogLine, seq uint64) int { + return cmp.Compare(l.Seq, seq) + }) + return LogPage{BootID: b.bootID, Last: b.last, Lines: slices.Clone(b.lines[start:])} +} diff --git a/logbuffer_test.go b/logbuffer_test.go new file mode 100644 index 0000000..3311dea --- /dev/null +++ b/logbuffer_test.go @@ -0,0 +1,75 @@ +package josuke + +import ( + "fmt" + "log" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var testBoot = time.Date(2026, 10, 10, 13, 0, 0, 0, time.UTC) + +func writeLines(t *testing.T, b *LogBuffer, texts ...string) { + t.Helper() + for _, text := range texts { + n, err := b.Write([]byte(text + "\n")) + require.NoError(t, err) + require.Equal(t, len(text)+1, n) + } +} + +func Test_LogBuffer_Returns_The_Lines_After_A_Line_Number(t *testing.T) { + b := NewLogBuffer(10, testBoot) + writeLines(t, b, "one", "two", "three") + bootID := b.Since("", 0).BootID + + page := b.Since(bootID, 1) + + assert.Equal(t, uint64(3), page.Last) + assert.Equal(t, []LogLine{{2, "two"}, {3, "three"}}, page.Lines) + assert.Empty(t, b.Since(bootID, 3).Lines) +} + +func Test_LogBuffer_Keeps_The_Last_Lines_Only(t *testing.T) { + b := NewLogBuffer(2, testBoot) + writeLines(t, b, "one", "two", "three") + + page := b.Since("", 0) + + assert.Equal(t, uint64(3), page.Last) + assert.Equal(t, []LogLine{{2, "two"}, {3, "three"}}, page.Lines) +} + +func Test_LogBuffer_Returns_Every_Line_To_A_Reader_Of_Another_Boot(t *testing.T) { + previous := NewLogBuffer(10, testBoot.Add(-time.Hour)) + b := NewLogBuffer(10, testBoot) + writeLines(t, b, "one", "two") + require.NotEqual(t, previous.Since("", 0).BootID, b.Since("", 0).BootID) + + page := b.Since(previous.Since("", 0).BootID, 1) + + assert.Equal(t, []LogLine{{1, "one"}, {2, "two"}}, page.Lines) +} + +func Test_LogBuffer_Keeps_A_Multiline_Entry_Whole(t *testing.T) { + b := NewLogBuffer(10, testBoot) + + log.New(b, "job#1 ", log.Lmsgprefix).Printf("[ERR ] panic: boom\n%s", "stack line 1\nstack line 2\n") + + assert.Equal(t, []LogLine{{1, "job#1 [ERR ] panic: boom\nstack line 1\nstack line 2"}}, b.Since("", 0).Lines) +} + +func Test_LogBuffer_Does_Not_Share_Its_Lines(t *testing.T) { + b := NewLogBuffer(3, testBoot) + writeLines(t, b, "one") + page := b.Since("", 0) + + for i := range 5 { + writeLines(t, b, fmt.Sprint(i)) + } + + assert.Equal(t, []LogLine{{1, "one"}}, page.Lines) +}