cli: empty lists print to stderr only; scrape --extract keeps the API's field order

CHANGELOG.md

@@ -9,6 +9,7 @@ Bug fixes from a review of 0.1.3, and live progress on a terminal. No new comman
 - **Ctrl-C exits 130** with "interrupted", including while `scrape -` waits on stdin. A mistyped subcommand (`spicrawl batch wiat`) is a usage error with a suggestion, not help and exit 0.
 - **Keys:** a trailing CR/LF or space on the key (flag, env or file) is trimmed. The config file is written via a private temp file, short keys are fully masked, and commands that get the key and base URL from flags work without `$HOME`.
 - **scrape:** `--body null` is a usage error, not a panic (also `batch submit`). Big integers survive `--json` (**key order now follows the API**, as `--jsonl` already did). Stdout write errors stop the run instead of exiting 0. `-o` writes atomically and **a target error page no longer overwrites the file**. `-o -` means stdout. Screenshot labels never overwrite each other. `--body` merges deeply. A BOM on stdin is stripped. Human output no longer HTML-escapes `<>&`. The multi-URL summary counts correctly. `--impersonate` help says it is on by default.
+- **Empty results and `--extract` order** (**behaviour change**, human mode only). `batch list` ("no batch jobs") and `batch results` ("no finished items yet") now print that message on stderr, as `sessions list` and `logs` already did, so stdout is empty and `spicrawl batch list | wc -l` is 0; `-q` silences it. `scrape --extract` (and the file `-o` writes for it) lists the extracted fields in the API's order instead of sorted alphabetically. `--json` and `--jsonl` are unchanged.
 - **batch:** `wait` and `submit --wait` exit 4 when the job failed or was cancelled. Polling survives transient 5xx and network errors. `results -o` no longer empties an existing file on failure.
 - **browser:** a server without the cloud browser says so (exit 8) instead of `NOT_FOUND`. `$(spicrawl browser url -q)` yields the bare URL.
 - **usage summary** shows the enforced monthly allowance. **status** fails when the key check cannot reach the API.

DESIGN.md

@@ -16,6 +16,11 @@ never invent a field the API does not accept (the API rejects unknown fields).
    `TERM=dumb` or `CI`. Switched off, the output is the plain text it always was.
    It never prints an engine or browser name; it says what the request asked for
    ("rendering JavaScript"), not which tier the API picked.
+   An empty result is not data either: in human mode an empty list
+   (`sessions list`, `logs`, `batch list`, `batch results`) prints nothing on
+   stdout and its one-line message ("no sessions") on stderr, so
+   `spicrawl sessions list | wc -l` is 0 and `-q` silences the message. JSON
+   mode prints the empty collection on stdout.
 2. **JSON when not a terminal.** `--json` forces it. Human mode prints tables
    or the document itself.
 3. **Never prompts without a TTY.** Anything interactive has a flag form.

internal/cmd/batch.go

@@ -388,7 +388,7 @@ func batchRunList(cmd *cobra.Command, _ []string) error {
 				fmt.Sprintf("%d/%d", j.Progress.Completed, j.Progress.Total), j.SubmittedAt})
 		}
 		if len(rows) == 0 {
-			fmt.Fprintln(w, "no batch jobs")
+			p.Info("no batch jobs") // not data: stderr, so stdout stays empty
 		} else {
 			p.Table([]string{"ID", "STATUS", "NAME", "DONE", "SUBMITTED"}, rows)
 		}

internal/cmd/batch_golden_test.go

@@ -153,7 +153,7 @@ func init() {
 				gJSON(200, `{"batches":[`+open.JSON()+`]}`)(w, r)
 			}})},
 		goldenCase{name: "batch_human/list", human: true, args: []string{"batch", "list"}, server: list(two)},
-		goldenCase{name: "batch_human/list_empty", human: true, args: []string{"batch", "list"}, server: list(`{"batches":[]}`)},
+		goldenCase{name: "batch_human/list_empty", human: true, stderr: true, args: []string{"batch", "list"}, server: list(`{"batches":[]}`)},
 		goldenCase{name: "batch_human/list_next_cursor", human: true, args: []string{"batch", "list"}, server: list(`{"batches":[` + completed.JSON() + `],"next_cursor":"1790000000000000000.ABC"}`)},
 
 		// results and content
@@ -166,7 +166,7 @@ func init() {
 		goldenCase{name: "batch/results_expired", args: []string{"batch", "results", gBatchID}, stderr: true,
 			server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}/results": gProblem(410, "ERR::REQUEST::BEYOND_RETENTION", "results expired")})},
 		goldenCase{name: "batch_human/results", human: true, args: []string{"batch", "results", gBatchID}, server: results("")},
-		goldenCase{name: "batch_human/results_empty", human: true, args: []string{"batch", "results", gBatchID},
+		goldenCase{name: "batch_human/results_empty", human: true, stderr: true, args: []string{"batch", "results", gBatchID},
 			server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}/results": func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(200) }})},
 		goldenCase{name: "batch/content", args: []string{"batch", "content", gBatchID, "0"},
 			server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}/tasks/{seq}/content": gJSON(200, `"<h1>A</h1>\n"`)})},

internal/cmd/batch_results.go

@@ -138,7 +138,7 @@ func batchRunResults(cmd *cobra.Command, args []string) error {
 
 	if !raw {
 		if len(rows) == 0 {
-			fmt.Fprintln(p.Out, "no finished items yet")
+			p.Info("no finished items yet") // not data: stderr, so stdout stays empty
 		} else {
 			p.Table([]string{"SEQ", "STATUS", "HTTP", "URL", "BYTES"}, rows)
 		}

internal/cmd/batch_test.go

@@ -17,17 +17,17 @@ import (
 // and cobra only hands a fresh one to commands whose ctx is nil.
 func batchRunCLI(t *testing.T, baseURL string, args ...string) (string, string, int) {
 	t.Helper()
-	var clear func(c *cobra.Command)
-	clear = func(c *cobra.Command) {
-		c.SetContext(nil) //nolint:staticcheck // nil lets cobra propagate the new root context
-		for _, sub := range c.Commands() {
-			clear(sub)
-		}
-	}
-	clear(rootCmd)
+	clearContexts(rootCmd)
 	return runCLI(t, baseURL, args...)
 }
 
+func clearContexts(c *cobra.Command) {
+	c.SetContext(nil) //nolint:staticcheck // nil lets cobra propagate the new root context
+	for _, sub := range c.Commands() {
+		clearContexts(sub)
+	}
+}
+
 func batchWriteFile(t *testing.T, name, content string) string {
 	t.Helper()
 	p := filepath.Join(t.TempDir(), name)

internal/cmd/empty_results_test.go

@@ -0,0 +1,43 @@
+package cmd
+
+import (
+	"net/http"
+	"strings"
+	"testing"
+)
+
+// An empty list in human mode is not data (DESIGN.md: stdout is data only):
+// stdout stays empty, so `cmd | wc -l` is 0, and the message goes to stderr.
+// JSON mode is unchanged: it prints the empty collection.
+func TestHumanEmptyResultsAreStderrOnly(t *testing.T) {
+	cases := []struct {
+		name string
+		args []string
+		body string
+		want string
+	}{
+		{"sessions list", []string{"sessions", "list"}, `{"sessions":[]}`, "no sessions"},
+		{"logs", []string{"logs"}, `{"data":[],"page":{"has_more":false,"retention_hours":24}}`, "no requests in the last 24 hours"},
+		{"batch list", []string{"batch", "list"}, `{"batches":[]}`, "no batch jobs"},
+		{"batch results", []string{"batch", "results", "B1"}, ``, "no finished items yet"},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			srv, _ := newRecordingServer(t, func(w http.ResponseWriter, _ *http.Request) {
+				w.Header().Set("Content-Type", "application/json")
+				_, _ = w.Write([]byte(c.body))
+			})
+			clearContexts(rootCmd)
+			out, errb, code := runCLIHuman(t, srv.URL, c.args...)
+			if code != 0 || out != "" || strings.TrimSpace(errb) != c.want {
+				t.Errorf("exit %d, stdout %q, stderr %q; want 0, empty, %q", code, out, errb, c.want)
+			}
+
+			clearContexts(rootCmd)
+			out, errb, code = runCLIHuman(t, srv.URL, append([]string{"-q"}, c.args...)...)
+			if code != 0 || out != "" || errb != "" {
+				t.Errorf("-q: exit %d, stdout %q, stderr %q; want 0, empty, empty", code, out, errb)
+			}
+		})
+	}
+}

internal/cmd/ops_golden_test.go

@@ -120,7 +120,7 @@ func init() {
 		goldenCase{name: "logs_human/list", human: true, args: []string{"logs"}, server: logs(both)},
 		goldenCase{name: "logs_human/all_projects", human: true, args: []string{"logs", "--all-projects", "--errors"}, server: logs(both)},
 		goldenCase{name: "logs_human/has_more", human: true, args: []string{"logs"}, server: logs(`{"data":[` + gLogOK + `],` + gLogPageMore + `}`)},
-		goldenCase{name: "logs_human/empty", human: true, args: []string{"logs"}, server: logs(`{"data":[],` + gLogPageEnd + `}`)},
+		goldenCase{name: "logs_human/empty", human: true, stderr: true, args: []string{"logs"}, server: logs(`{"data":[],` + gLogPageEnd + `}`)},
 		goldenCase{name: "logs_human/get_full", human: true, args: []string{"logs", "get", "x"}, server: logGet(gLogOK)},
 		goldenCase{name: "logs_human/get_blocked", human: true, args: []string{"logs", "get", "x"}, server: logGet(gLogBlocked)},
 

internal/cmd/scrape.go

@@ -456,7 +456,7 @@ func (r *scrapeResult) document(body map[string]any) ([]byte, string) {
 		return r.resp.Body, "content_base64"
 	}
 	if scrapeWantsData(body) {
-		if data, ok := r.obj["data"]; ok && data != nil {
+		if data := r.orderedData(); data != nil {
 			var buf bytes.Buffer
 			enc := json.NewEncoder(&buf)
 			enc.SetEscapeHTML(false) // a human reads <, > and & as they are
@@ -470,6 +470,25 @@ func (r *scrapeResult) document(body map[string]any) ([]byte, string) {
 	return []byte(s), "content"
 }
 
+// orderedData is the envelope's "data" with its object members in the order
+// the API sent them. r.obj is a map, which would print them sorted. nil when
+// there is no data (absent or null).
+func (r *scrapeResult) orderedData() any {
+	var env struct {
+		Data json.RawMessage `json:"data"`
+	}
+	if json.Unmarshal(r.resp.Body, &env) != nil || len(env.Data) == 0 {
+		return nil
+	}
+	dec := json.NewDecoder(bytes.NewReader(env.Data))
+	dec.UseNumber()
+	v, err := decodeOrdered(dec)
+	if err != nil {
+		return nil
+	}
+	return v
+}
+
 func scrapeWantsData(body map[string]any) bool {
 	for _, k := range []string{"extract", "ai_extract", "autoparse"} {
 		if v, ok := body[k]; ok && v != nil && v != false {

internal/cmd/scrape_golden_test.go

@@ -107,6 +107,8 @@ func init() {
 		goldenCase{name: "scrape_human/markdown_meta", human: true, args: []string{"scrape", u, "--format", "markdown", "--meta"}, server: gScrape(md)},
 		goldenCase{name: "scrape_human/text_no_trailing_newline", human: true, args: []string{"scrape", u, "--format", "text"}, server: gScrape(gScrapeRaw("text/plain", "Example Domain", 200))},
 		goldenCase{name: "scrape_human/extract_data", human: true, args: []string{"scrape", u, "--extract", `{"title":"h1"}`}, server: gScrape(gScrapeEnv(`,"data":{"title":"Anvil","price":"$19.99 & free shipping"}`))},
+		goldenCase{name: "scrape_human/extract_data_nested", human: true, args: []string{"scrape", u, "--extract", `{"title":"h1"}`},
+			server: gScrape(gScrapeEnv(`,"data":{"title":"Anvil","sku":"A-1","specs":{"weight":"50kg","color":"black","dims":{"w":1.0,"h":2e3}},"tags":["z","a"],"price":19.99,"empty":{},"none":null}`))},
 		goldenCase{name: "scrape_human/links", human: true, args: []string{"scrape", u, "--links", "--format", "markdown"}, server: gScrape(gScrapeEnv(`,"links":["https://example.com/a","https://example.com/b?x=1&y=2"]`))},
 		goldenCase{name: "scrape_human/network", human: true, args: []string{"scrape", u, "--render", "--network-capture", `{}`},
 			server: gScrape(gScrapeEnv(`,"network":[{"url":"https://example.com/api/price","status":200,"resource_type":"xhr","content_type":"application/json"},{"url":"https://example.com/x.png","resource_type":"image"}]`))},

internal/cmd/scrape_ordered.go

@@ -0,0 +1,80 @@
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+)
+
+// ordObj is a JSON object that keeps its members in the order they arrived.
+// Decoding into map[string]any loses that: encoding/json writes a map's keys
+// sorted, so a human reading `--extract` output saw the fields shuffled.
+type ordObj []ordMember
+
+type ordMember struct {
+	key string
+	val any
+}
+
+// MarshalJSON writes the members in order, without HTML escaping (a person
+// reads <, > and & as they are). Nested ordObj values come back through here.
+func (o ordObj) MarshalJSON() ([]byte, error) {
+	var b bytes.Buffer
+	enc := json.NewEncoder(&b)
+	enc.SetEscapeHTML(false)
+	b.WriteByte('{')
+	for i, m := range o {
+		if i > 0 {
+			b.WriteByte(',')
+		}
+		for j, v := range []any{m.key, m.val} {
+			if j == 1 {
+				b.WriteByte(':')
+			}
+			if err := enc.Encode(v); err != nil {
+				return nil, err
+			}
+			b.Truncate(b.Len() - 1) // Encode ends with a newline
+		}
+	}
+	b.WriteByte('}')
+	return b.Bytes(), nil
+}
+
+// decodeOrdered reads the next JSON value from dec: objects as ordObj, arrays
+// as []any, scalars as Token returns them (json.Number if dec.UseNumber).
+func decodeOrdered(dec *json.Decoder) (any, error) {
+	t, err := dec.Token()
+	if err != nil {
+		return nil, err
+	}
+	d, ok := t.(json.Delim)
+	if !ok {
+		return t, nil
+	}
+	if d == '{' {
+		obj := ordObj{}
+		for dec.More() {
+			k, err := dec.Token()
+			if err != nil {
+				return nil, err
+			}
+			v, err := decodeOrdered(dec)
+			if err != nil {
+				return nil, err
+			}
+			obj = append(obj, ordMember{k.(string), v})
+		}
+		_, err = dec.Token() // }
+		return obj, err
+	}
+	arr := []any{}
+	for dec.More() {
+		v, err := decodeOrdered(dec)
+		if err != nil {
+			return nil, err
+		}
+		arr = append(arr, v)
+	}
+	_, err = dec.Token() // ]
+	return arr, err
+}

internal/cmd/scrape_ordered_test.go

@@ -0,0 +1,66 @@
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestDecodeOrderedKeepsMemberOrderAndNumbers(t *testing.T) {
+	in := `{"z":1.0,"a":{"y":[{"b":1e3,"a":null}],"x":{}},"m":[],"s":"a<b>&c"}`
+	dec := json.NewDecoder(strings.NewReader(in))
+	dec.UseNumber()
+	v, err := decodeOrdered(dec)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var buf bytes.Buffer
+	enc := json.NewEncoder(&buf)
+	enc.SetEscapeHTML(false)
+	if err := enc.Encode(v); err != nil {
+		t.Fatal(err)
+	}
+	if got := strings.TrimSpace(buf.String()); got != in {
+		t.Errorf("round trip changed it:\n got %s\nwant %s", got, in)
+	}
+}
+
+// Human mode prints --extract data in the API's order, and so does the file
+// -o writes (both come from document()).
+func TestScrapeHumanDataKeepsAPIOrder(t *testing.T) {
+	scrapeHumanMode(t)
+	srv, _ := newRecordingServer(t, func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"url":"u","status":200,"content":"","data":{"title":"T","price":9,"list":[{"z":1,"a":2}]}}`))
+	})
+	want := "{\n  \"title\": \"T\",\n  \"price\": 9,\n  \"list\": [\n    {\n      \"z\": 1,\n      \"a\": 2\n    }\n  ]\n}\n"
+	out, stderr, code := scrapeRunCLI(t, srv.URL, "scrape", "https://example.com", "--extract", `{"title":"h1"}`)
+	if code != 0 || out != want {
+		t.Fatalf("exit %d, stderr %s\n got %q\nwant %q", code, stderr, out, want)
+	}
+
+	file := filepath.Join(t.TempDir(), "data.json")
+	if _, stderr, code := scrapeRunCLI(t, srv.URL, "scrape", "https://example.com", "--extract", `{"title":"h1"}`, "-o", file); code != 0 {
+		t.Fatalf("exit %d, stderr %s", code, stderr)
+	}
+	if b, _ := os.ReadFile(file); strings.TrimSpace(string(b)) != strings.TrimSpace(want) {
+		t.Errorf("-o file = %q, want %q", b, want)
+	}
+}
+
+// JSON mode prints the API's bytes (re-indented) when the envelope is
+// untouched, so the order was always the API's there.
+func TestScrapeJSONDataKeepsAPIOrder(t *testing.T) {
+	srv, _ := newRecordingServer(t, func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"url":"u","status":200,"content":"","data":{"title":"T","price":9}}`))
+	})
+	out, _, code := scrapeRunCLI(t, srv.URL, "scrape", "https://example.com", "--extract", `{"title":"h1"}`)
+	if i, j := strings.Index(out, `"title"`), strings.Index(out, `"price"`); code != 0 || i < 0 || i > j {
+		t.Errorf("exit %d, data out of order:\n%s", code, out)
+	}
+}

internal/cmd/sessions_golden_test.go

@@ -58,7 +58,7 @@ func init() {
 
 		goldenCase{name: "sessions_human/create", human: true, args: []string{"sessions", "create"}, server: one("POST", "/v1/sessions", gSessionNew, 201)},
 		goldenCase{name: "sessions_human/list", human: true, args: []string{"sessions", "list"}, server: list(page)},
-		goldenCase{name: "sessions_human/list_empty", human: true, args: []string{"sessions", "list"}, server: list(`{"sessions":[]}`)},
+		goldenCase{name: "sessions_human/list_empty", human: true, stderr: true, args: []string{"sessions", "list"}, server: list(`{"sessions":[]}`)},
 		goldenCase{name: "sessions_human/list_next_cursor", human: true, args: []string{"sessions", "list"}, server: list(more)},
 		goldenCase{name: "sessions_human/get", human: true, args: []string{"sessions", "get", gSessionID}, server: one("GET", "/v1/sessions/{id}", gSession, 200)},
 		goldenCase{name: "sessions_human/get_released", human: true, args: []string{"sessions", "get", gSessionID}, server: one("GET", "/v1/sessions/{id}", gSessionReleased, 200)},

internal/cmd/testdata/golden/batch_human/list_empty.stderr

@@ -0,0 +1 @@
+no batch jobs

internal/cmd/testdata/golden/batch_human/list_empty.stdout

@@ -1 +0,0 @@
-no batch jobs

internal/cmd/testdata/golden/batch_human/results_empty.stderr

@@ -0,0 +1 @@
+no finished items yet

internal/cmd/testdata/golden/batch_human/results_empty.stdout

@@ -1 +0,0 @@
-no finished items yet

internal/cmd/testdata/golden/logs_human/empty.stderr

@@ -0,0 +1 @@
+no requests in the last 24 hours

internal/cmd/testdata/golden/scrape_human/extract_data.stdout

@@ -1,4 +1,4 @@
 {
-  "price": "$19.99 & free shipping",
-  "title": "Anvil"
+  "title": "Anvil",
+  "price": "$19.99 & free shipping"
 }

internal/cmd/testdata/golden/scrape_human/extract_data_nested.exit

@@ -0,0 +1 @@
+0

internal/cmd/testdata/golden/scrape_human/extract_data_nested.stdout

@@ -0,0 +1,19 @@
+{
+  "title": "Anvil",
+  "sku": "A-1",
+  "specs": {
+    "weight": "50kg",
+    "color": "black",
+    "dims": {
+      "w": 1.0,
+      "h": 2e3
+    }
+  },
+  "tags": [
+    "z",
+    "a"
+  ],
+  "price": 19.99,
+  "empty": {},
+  "none": null
+}

internal/cmd/testdata/golden/sessions_human/list_empty.stderr

@@ -0,0 +1 @@
+no sessions