From c595d62898d0bdebaf5201b642f6a3a105fb362d Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Sat, 14 Nov 2026 16:41:24 -0800 Subject: [PATCH 1/2] Accept uploaded data files for import into the notebook root PUT /api/v1/import saves the request body under .sqlnb-imports in the notebook root and returns its absolute path; DELETE removes one saved file. Names are reduced to a safe base name with a data-file extension, existing files are never replaced, symlinks are not followed, and the size limit advertised in the bootstrap response is enforced while the body streams, removing the partial file on failure or abort. --- docs/notebooks.md | 12 + http/src/imports.rs | 363 ++++++++++++++++++++++++++++++ http/src/imports/tests.rs | 418 +++++++++++++++++++++++++++++++++++ http/src/lib.rs | 6 + http/src/platform/unix.rs | 17 ++ http/src/platform/windows.rs | 10 + 6 files changed, 826 insertions(+) create mode 100644 http/src/imports.rs create mode 100644 http/src/imports/tests.rs diff --git a/docs/notebooks.md b/docs/notebooks.md index 29b7542..e91ccaa 100644 --- a/docs/notebooks.md +++ b/docs/notebooks.md @@ -6,6 +6,18 @@ The authenticated `GET /api/v1/recent-notebook` endpoint returns `{ "path": "relative.sqlnb" }`, or `{ "path": null }` when no readable recent notebook is available. The UI records a successful open or save using `PUT` to the same endpoint with `{ "path": "relative.sqlnb" }`. The path must identify an existing notebook within the configured root. This preference survives server restarts and port changes and is isolated by canonical notebook root under the OS cache's `sidequery/sqlnb-extension/notebooks` directory. It contains only the relative path; unavailable or corrupt preferences do not affect normal notebook file operations. +## Uploaded data files + +A browser does not tell a page where a dropped file is stored, so the notebook UI imports a dropped CSV, Parquet or JSON file by uploading a copy. The bootstrap response advertises the endpoint as `"imports": { "endpoint": "/api/v1/import", "maxBytes": 1073741824, "folder": "/absolute/notebook/root/.sqlnb-imports" }`; servers without the field have no upload endpoint. + +`PUT /api/v1/import?name=` takes the raw file bytes as the request body and returns `201 { "path": "", "name": "", "bytes": }`. It uses the same capability and origin checks as the file API. Files are saved only in `.sqlnb-imports` directly under the notebook root, which is created on first use with a `.gitignore` of `*` and left out of the notebook listing: + +- The name must not contain a path separator or control character. Other characters outside letters, digits, spaces and `-_.()` become `_`, leading dots are removed, and the stem is cut to 120 characters. The extension must be `.csv`, `.tsv`, `.txt`, `.parquet`, `.json`, `.jsonl` or `.ndjson`, optionally followed by `.gz` or `.zst`. +- An existing name is never replaced: the upload is saved as `name (2).csv`, `name (3).csv` and so on. The folder and file names are opened relative to the notebook root's descriptor without following symlinks, so a symlinked `.sqlnb-imports` is refused and a symlink inside it is neither written through nor deleted. +- A declared `Content-Length` above `maxBytes` is refused with `413` before reading; the body is also counted while it streams. Bytes go to a temporary file that is renamed into place only after the whole body arrives and is synced; a refused, failed or aborted upload removes it. + +`DELETE /api/v1/import?name=` removes one uploaded file by the name the upload returned. Uploaded files otherwise stay until the user deletes them. + ## DuckDB UI notebooks The local DuckDB UI launched by `duckdb -ui` stores its notebooks in a DuckDB diff --git a/http/src/imports.rs b/http/src/imports.rs new file mode 100644 index 0000000..5f91e5d --- /dev/null +++ b/http/src/imports.rs @@ -0,0 +1,363 @@ +//! Data files uploaded for import into DuckDB. +//! +//! A browser does not tell a page where a dropped file is stored, so the page +//! sends the file's bytes here and imports the saved copy by its absolute path. +//! Uploads go only into one folder of the notebook root, under a sanitized name +//! with a data-file extension, never replace an existing file, and are bounded +//! while they stream. They stay until the user deletes them. +use crate::*; +use axum::{body::Body, routing::MethodRouter}; +use bytes::Bytes; +use http_body_util::BodyExt; +use std::ffi::OsStr; + +/// Folder of the notebook root that holds uploaded files. It is left out of the +/// notebook listing and ignored by Git. +pub(crate) const IMPORTS_FOLDER: &str = ".sqlnb-imports"; +/// Largest upload accepted, advertised in the bootstrap response. +pub(crate) const MAX_IMPORT_BYTES: u64 = 1024 * 1024 * 1024; +const MAX_NAME_CHARS: usize = 120; +const MAX_NUMBERED_NAMES: u32 = 10_000; +const DATA_EXTENSIONS: [&str; 7] = ["csv", "tsv", "txt", "parquet", "json", "jsonl", "ndjson"]; +const COMPRESSED_EXTENSIONS: [&str; 2] = ["gz", "zst"]; + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct Capability { + endpoint: &'static str, + max_bytes: u64, + folder: String, +} + +pub(crate) fn capability(state: &AppState) -> Capability { + Capability { + endpoint: "/api/v1/import", + max_bytes: MAX_IMPORT_BYTES, + folder: state + .notebooks + .root + .join(IMPORTS_FOLDER) + .display() + .to_string(), + } +} + +/// `PUT` saves the request body as a new file; `DELETE` removes one saved file. +/// The handler enforces its own streaming limit instead of the router default. +pub(crate) fn routes() -> MethodRouter { + axum::routing::put(upload) + .delete(remove) + .layer(DefaultBodyLimit::disable()) +} + +/// Whether a listing entry lies in the imports folder. +pub(crate) fn is_imports_entry(path: &str) -> bool { + path == IMPORTS_FOLDER + || path + .strip_prefix(IMPORTS_FOLDER) + .is_some_and(|rest| rest.starts_with('/')) +} + +#[derive(Deserialize)] +struct NameQuery { + name: String, +} + +#[derive(Debug, Serialize)] +pub(crate) struct UploadResponse { + pub(crate) path: String, + pub(crate) name: String, + pub(crate) bytes: u64, +} + +fn invalid_name() -> ApiError { + ApiError::new( + StatusCode::BAD_REQUEST, + "invalid_name", + "name must be a file name without path separators", + ) +} + +fn unsupported_type() -> ApiError { + ApiError::new( + StatusCode::BAD_REQUEST, + "unsupported_file_type", + "only .csv, .tsv, .txt, .parquet, .json, .jsonl and .ndjson files, optionally .gz or .zst compressed, can be imported", + ) +} + +fn too_large(limit: u64) -> ApiError { + ApiError::new( + StatusCode::PAYLOAD_TOO_LARGE, + "file_too_large", + format!("uploads are limited to {limit} bytes"), + ) +} + +fn aborted() -> ApiError { + ApiError::new( + StatusCode::BAD_REQUEST, + "upload_aborted", + "the upload did not complete", + ) +} + +fn task_failed() -> ApiError { + file_io(std::io::Error::other("upload task failed")) +} + +/// Splits a lower-case data extension, with an optional compression suffix, off a name. +fn split_data_extension(name: &str) -> Option<(&str, String)> { + let lower = name.to_ascii_lowercase(); + let mut stem_length = name.len(); + let mut extension = String::new(); + for compressed in COMPRESSED_EXTENSIONS { + if let Some(rest) = lower.strip_suffix(&format!(".{compressed}")) { + stem_length = rest.len(); + extension = format!(".{compressed}"); + break; + } + } + let base = &lower[..stem_length]; + let data = DATA_EXTENSIONS + .iter() + .find(|data| base.ends_with(&format!(".{data}")))?; + stem_length -= data.len() + 1; + Some((&name[..stem_length], format!(".{data}{extension}"))) +} + +/// The name a supplied file name is saved under: a single component of letters, +/// digits, spaces and `-_.()`, other characters replaced, no leading dots, with an +/// allowed data extension. Names with separators or control characters are refused. +pub(crate) fn safe_name(supplied: &str) -> Result { + if supplied.is_empty() + || supplied + .chars() + .any(|c| c == '/' || c == '\\' || c.is_control()) + { + return Err(invalid_name()); + } + let mapped: String = supplied + .chars() + .map(|c| { + if c.is_alphanumeric() || " -_.()".contains(c) { + c + } else { + '_' + } + }) + .collect(); + let trimmed = mapped + .trim_start_matches(['.', ' ']) + .trim_end_matches(['.', ' ']); + let (stem, extension) = split_data_extension(trimmed).ok_or_else(unsupported_type)?; + let stem: String = stem.chars().take(MAX_NAME_CHARS).collect(); + let stem = stem.trim_end_matches(['.', ' ']); + if stem.is_empty() { + return Err(invalid_name()); + } + let name = format!("{stem}{extension}"); + if !valid_component(OsStr::new(&name)) { + return Err(invalid_name()); + } + Ok(name) +} + +/// `name`, then `stem (2).ext`, `stem (3).ext`, ... +fn numbered(name: &str, number: u32) -> String { + if number == 1 { + return name.to_owned(); + } + let (stem, extension) = split_data_extension(name).expect("safe names keep their extension"); + format!("{stem} ({number}){extension}") +} + +fn open_imports_folder(store: &NotebookStore, create: bool) -> Result { + let name = CString::new(IMPORTS_FOLDER).expect("constant has no NUL"); + let root = store.root_fd.platform_handle(); + if create { + mkdir_at(root, &name).map_err(file_io)?; + } + let folder = open_directory_at(root, &name).map_err(|error| { + if error.kind() == std::io::ErrorKind::NotFound { + ApiError::new(StatusCode::NOT_FOUND, "not_found", "no uploaded files") + } else { + ApiError::new( + StatusCode::CONFLICT, + "invalid_imports_folder", + format!("{IMPORTS_FOLDER} must be a real folder of the notebook root"), + ) + } + })?; + if create { + // A Git-backed notebook root must not pick up uploaded data. + match create_new_file_at(folder.platform_handle(), c".gitignore") { + Ok(mut ignore) => ignore.write_all(b"*\n").map_err(file_io)?, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(file_io(error)), + } + } + Ok(folder) +} + +enum Chunk { + Data(Bytes), + Finish, +} + +/// Writes chunks to a temporary file and gives it the first free numbered name +/// once `Finish` arrives. If the sender goes away first, or anything fails, the +/// temporary file is removed. +fn write_upload( + store: &NotebookStore, + name: &str, + mut chunks: tokio::sync::mpsc::Receiver, +) -> Result<(String, u64), ApiError> { + let folder = open_imports_folder(store, true)?; + let parent = folder.platform_handle(); + let temporary = CString::new(format!(".upload-{}.part", uuid::Uuid::new_v4())) + .expect("UUID contains no NUL"); + let mut file = create_new_file_at(parent, &temporary).map_err(file_io)?; + let mut written = 0u64; + let result = (|| { + loop { + match chunks.blocking_recv() { + Some(Chunk::Data(data)) => { + file.write_all(&data).map_err(file_io)?; + written += data.len() as u64; + } + Some(Chunk::Finish) => break, + None => return Err(aborted()), + } + } + file.sync_all().map_err(file_io)?; + for number in 1..=MAX_NUMBERED_NAMES { + let candidate = numbered(name, number); + let destination = CString::new(candidate.as_str()).map_err(|_| invalid_name())?; + match rename_no_replace(parent, &temporary, parent, &destination) { + Ok(()) => { + sync_directory(parent).map_err(file_io)?; + return Ok(candidate); + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(file_io(error)), + } + } + Err(ApiError::new( + StatusCode::CONFLICT, + "name_unavailable", + "too many uploaded files share this name", + )) + })(); + drop(file); + match result { + Ok(saved) => Ok((saved, written)), + Err(error) => { + let _ = remove_file_at(parent, &temporary); + Err(error) + } + } +} + +/// Streams `body` into the imports folder, refusing it once it passes `limit` bytes. +pub(crate) async fn receive( + store: Arc, + name: String, + mut body: Body, + limit: u64, +) -> Result { + let (sender, receiver) = tokio::sync::mpsc::channel(4); + let writer_store = store.clone(); + let writer_name = name.clone(); + let writer = + tokio::task::spawn_blocking(move || write_upload(&writer_store, &writer_name, receiver)); + let mut received = 0u64; + let streamed = async { + while let Some(frame) = body.frame().await { + let frame = frame.map_err(|_| aborted())?; + let Ok(data) = frame.into_data() else { + continue; + }; + received += data.len() as u64; + if received > limit { + return Err(too_large(limit)); + } + if sender.send(Chunk::Data(data)).await.is_err() { + // The writer failed; its error is the response. + return Ok(()); + } + } + let _ = sender.send(Chunk::Finish).await; + Ok(()) + } + .await; + drop(sender); + // Wait for the writer even when streaming failed, so the partial file is gone + // before the response is sent. + let written = writer.await.map_err(|_| task_failed())?; + streamed?; + let (saved, bytes) = written?; + Ok(UploadResponse { + path: store + .root + .join(IMPORTS_FOLDER) + .join(&saved) + .display() + .to_string(), + name: saved, + bytes, + }) +} + +async fn upload( + State(state): State, + headers: HeaderMap, + Query(query): Query, + body: Body, +) -> Result { + authenticate(&headers, &state)?; + let name = safe_name(&query.name)?; + let declared = headers + .get(header::CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()); + if declared.is_some_and(|length| length > MAX_IMPORT_BYTES) { + return Err(too_large(MAX_IMPORT_BYTES)); + } + let response = receive(state.notebooks.clone(), name, body, MAX_IMPORT_BYTES).await?; + Ok((StatusCode::CREATED, Json(response))) +} + +/// Deletes one uploaded file by the name the upload returned. +async fn remove( + State(state): State, + headers: HeaderMap, + Query(query): Query, +) -> Result { + authenticate(&headers, &state)?; + let name = safe_name(&query.name)?; + if name != query.name { + return Err(invalid_name()); + } + let store = state.notebooks.clone(); + let removed = name.clone(); + tokio::task::spawn_blocking(move || { + let folder = open_imports_folder(&store, false)?; + let parent = folder.platform_handle(); + let file = CString::new(name).map_err(|_| invalid_name())?; + let not_found = + || ApiError::new(StatusCode::NOT_FOUND, "not_found", "no such uploaded file"); + let identity = metadata_at(parent, &file).map_err(|_| not_found())?; + if !identity.regular { + return Err(not_found()); + } + remove_file_at(parent, &file).map_err(file_io)?; + sync_directory(parent).map_err(file_io) + }) + .await + .map_err(|_| task_failed())??; + Ok(Json(serde_json::json!({ "name": removed }))) +} + +#[cfg(test)] +mod tests; diff --git a/http/src/imports/tests.rs b/http/src/imports/tests.rs new file mode 100644 index 0000000..00d83b3 --- /dev/null +++ b/http/src/imports/tests.rs @@ -0,0 +1,418 @@ +use super::*; +use axum::http::Request; +use serde_json::Value; +use std::io::{BufRead, BufReader}; +use std::net::TcpStream; +use tempfile::TempDir; +use tower::ServiceExt; + +struct Fixture { + temp: TempDir, + root: PathBuf, + state: AppState, +} + +const PORT: u16 = 4778; + +fn config(temp: &TempDir, port: u16) -> ServerConfig { + let assets = temp.path().join("assets"); + fs::create_dir_all(&assets).unwrap(); + fs::write(assets.join("index.html"), "sidequery").unwrap(); + ServerConfig { + port, + local_asset_directory: assets.display().to_string(), + remote_asset_url: String::new(), + quack_url: "http://127.0.0.1:49999".into(), + token: "test-capability".into(), + duckdb_version: "test".into(), + notebook_root: temp.path().join("notebooks").display().to_string(), + mcp_token: String::new(), + } +} + +impl Fixture { + fn new() -> Self { + let temp = tempfile::tempdir().unwrap(); + let state = build_state(config(&temp, PORT), PORT).unwrap(); + let root = state.notebooks.root.clone(); + Self { temp, root, state } + } + + fn imports(&self) -> PathBuf { + self.root.join(IMPORTS_FOLDER) + } + + fn request(&self, method: &str, uri: &str, body: &[u8]) -> Request { + Request::builder() + .method(method) + .uri(uri) + .header(header::HOST, format!("127.0.0.1:{PORT}")) + .header(header::AUTHORIZATION, "Bearer test-capability") + .header(header::ORIGIN, format!("http://127.0.0.1:{PORT}")) + .body(Body::from(body.to_vec())) + .unwrap() + } + + async fn send(&self, request: Request) -> (StatusCode, Value) { + let response = router(self.state.clone()).oneshot(request).await.unwrap(); + let status = response.status(); + let bytes = to_bytes(response.into_body(), 1024 * 1024).await.unwrap(); + ( + status, + serde_json::from_slice(&bytes).unwrap_or(Value::Null), + ) + } + + async fn upload(&self, name: &str, body: &[u8]) -> (StatusCode, Value) { + let uri = format!("/api/v1/import?name={}", encode(name)); + self.send(self.request("PUT", &uri, body)).await + } + + /// Files in the imports folder other than its `.gitignore`. + fn saved(&self) -> Vec { + let mut names: Vec = fs::read_dir(self.imports()) + .map(|entries| { + entries + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|name| name != ".gitignore") + .collect() + }) + .unwrap_or_default(); + names.sort(); + names + } +} + +fn encode(value: &str) -> String { + url::form_urlencoded::byte_serialize(value.as_bytes()).collect() +} + +#[tokio::test] +async fn saves_the_upload_under_the_notebook_root_and_advertises_it() { + let fixture = Fixture::new(); + let (status, body) = fixture.upload("Q3 sales.csv", b"id,total\n1,2\n").await; + assert_eq!(status, StatusCode::CREATED); + let saved = fixture.imports().join("Q3 sales.csv"); + assert_eq!(body["name"], "Q3 sales.csv"); + assert_eq!(body["bytes"], 13); + assert_eq!(body["path"], saved.display().to_string()); + assert!(Path::new(body["path"].as_str().unwrap()).is_absolute()); + assert_eq!(fs::read(&saved).unwrap(), b"id,total\n1,2\n"); + // Uploaded data is ignored by Git and kept out of the notebook listing. + assert_eq!( + fs::read_to_string(fixture.imports().join(".gitignore")).unwrap(), + "*\n" + ); + fs::write(fixture.imports().join("stray.sql"), "select 1").unwrap(); + let (_, listing) = fixture + .send(fixture.request("GET", "/api/v1/files", b"")) + .await; + assert_eq!(listing["files"], serde_json::json!([])); + assert_eq!(listing["directories"], serde_json::json!([])); + + let (_, bootstrap) = fixture + .send(fixture.request("GET", "/api/v1/bootstrap", b"")) + .await; + assert_eq!(bootstrap["imports"]["endpoint"], "/api/v1/import"); + assert_eq!(bootstrap["imports"]["maxBytes"], MAX_IMPORT_BYTES); + assert_eq!( + bootstrap["imports"]["folder"], + fixture.imports().display().to_string() + ); +} + +#[tokio::test] +async fn never_overwrites_and_numbers_the_name_instead() { + let fixture = Fixture::new(); + for body in [b"one".as_slice(), b"two", b"three"] { + assert_eq!( + fixture.upload("events.json.gz", body).await.0, + StatusCode::CREATED + ); + } + let (_, body) = fixture.upload("Events.JSON.GZ", b"four").await; + assert_eq!( + fixture.saved(), + [ + "events (2).json.gz", + "events (3).json.gz", + "events.json.gz", + body["name"].as_str().unwrap() + ] + .into_iter() + .map(String::from) + .collect::>() + .into_iter() + .collect::>() + ); + assert_eq!( + fs::read(fixture.imports().join("events.json.gz")).unwrap(), + b"one" + ); + assert_eq!( + fs::read(fixture.imports().join("events (3).json.gz")).unwrap(), + b"three" + ); +} + +#[test] +fn reduces_names_to_a_safe_base_name_with_a_data_extension() { + assert_eq!(safe_name("..hidden.CSV").unwrap(), "hidden.csv"); + assert_eq!( + safe_name("we:ird*name?.Parquet").unwrap(), + "we_ird_name_.parquet" + ); + assert_eq!(safe_name("logs.ndjson.zst").unwrap(), "logs.ndjson.zst"); + assert_eq!(safe_name("ventes été.tsv").unwrap(), "ventes été.tsv"); + let long = format!("{}.txt", "a".repeat(400)); + assert_eq!( + safe_name(&long).unwrap(), + format!("{}.txt", "a".repeat(120)) + ); + for name in [ + "../escape.csv", + "nested/file.csv", + "..\\escape.csv", + "/etc/passwd.csv", + "line\nbreak.csv", + "", + ] { + assert_eq!( + safe_name(name).unwrap_err().code, + "invalid_name", + "{name:?}" + ); + } + for name in [".csv", "...csv", " . .json"] { + assert!(safe_name(name).is_err(), "{name:?}"); + } + for name in [ + "notes.sql", + "book.sqlnb", + "run.sh", + "data.csv.exe", + "data.gz", + "data.xlsx", + ] { + assert_eq!( + safe_name(name).unwrap_err().code, + "unsupported_file_type", + "{name:?}" + ); + } +} + +#[tokio::test] +async fn refuses_traversal_separators_and_unsupported_types_without_writing() { + let fixture = Fixture::new(); + for (name, code) in [ + ("../escape.csv", "invalid_name"), + ("a/b.csv", "invalid_name"), + ("a\\b.csv", "invalid_name"), + ("payload.sh", "unsupported_file_type"), + ("notebook.sqlnb", "unsupported_file_type"), + ] { + let (status, body) = fixture.upload(name, b"x").await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{name}"); + assert_eq!(body["error"]["code"], code, "{name}"); + } + assert!(!fixture.imports().exists()); + assert!(!fixture.temp.path().join("escape.csv").exists()); +} + +#[tokio::test] +async fn requires_the_file_api_capability_and_origin() { + let fixture = Fixture::new(); + let uri = "/api/v1/import?name=data.csv"; + let unauthenticated = Request::builder() + .method("PUT") + .uri(uri) + .header(header::HOST, format!("127.0.0.1:{PORT}")) + .body(Body::from("x")) + .unwrap(); + let mut wrong_origin = fixture.request("PUT", uri, b"x"); + wrong_origin + .headers_mut() + .insert(header::ORIGIN, HeaderValue::from_static("http://evil.test")); + let mut wrong_host = fixture.request("PUT", uri, b"x"); + wrong_host + .headers_mut() + .insert(header::HOST, HeaderValue::from_static("evil.test")); + let mut delete = fixture.request("DELETE", uri, b""); + delete.headers_mut().remove(header::AUTHORIZATION); + for (request, status) in [ + (unauthenticated, StatusCode::UNAUTHORIZED), + (wrong_origin, StatusCode::UNAUTHORIZED), + (wrong_host, StatusCode::BAD_REQUEST), + (delete, StatusCode::UNAUTHORIZED), + ] { + assert_eq!(fixture.send(request).await.0, status); + } + assert!(!fixture.imports().exists()); +} + +#[cfg(unix)] +#[tokio::test] +async fn does_not_follow_symlinks_out_of_the_root() { + let fixture = Fixture::new(); + let outside = fixture.temp.path().join("outside"); + fs::create_dir_all(&outside).unwrap(); + std::os::unix::fs::symlink(&outside, fixture.imports()).unwrap(); + let (status, body) = fixture.upload("data.csv", b"x").await; + assert_eq!(status, StatusCode::CONFLICT); + assert_eq!(body["error"]["code"], "invalid_imports_folder"); + assert_eq!(fs::read_dir(&outside).unwrap().count(), 0); + let (status, _) = fixture + .send(fixture.request("DELETE", "/api/v1/import?name=data.csv", b"")) + .await; + assert_eq!(status, StatusCode::CONFLICT); + + // A symlink inside the real folder is never written through or deleted. + fs::remove_file(fixture.imports()).unwrap(); + fs::create_dir(fixture.imports()).unwrap(); + fs::write(outside.join("target.csv"), "outside").unwrap(); + std::os::unix::fs::symlink( + outside.join("target.csv"), + fixture.imports().join("data.csv"), + ) + .unwrap(); + let (status, body) = fixture.upload("data.csv", b"inside").await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(body["name"], "data (2).csv"); + assert_eq!( + fs::read_to_string(outside.join("target.csv")).unwrap(), + "outside" + ); + let (status, _) = fixture + .send(fixture.request("DELETE", "/api/v1/import?name=data.csv", b"")) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert!(outside.join("target.csv").exists()); + assert!(fixture.imports().join("data.csv").is_symlink()); +} + +#[tokio::test] +async fn refuses_a_declared_length_over_the_limit() { + let fixture = Fixture::new(); + let mut request = fixture.request("PUT", "/api/v1/import?name=big.parquet", b""); + request.headers_mut().insert( + header::CONTENT_LENGTH, + HeaderValue::from_str(&(MAX_IMPORT_BYTES + 1).to_string()).unwrap(), + ); + let (status, body) = fixture.send(request).await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(body["error"]["code"], "file_too_large"); + assert!(fixture.saved().is_empty()); +} + +#[tokio::test] +async fn enforces_the_limit_while_streaming_and_removes_the_partial_file() { + let fixture = Fixture::new(); + let store = fixture.state.notebooks.clone(); + let error = receive( + store.clone(), + "big.csv".into(), + Body::from(vec![b'x'; 4096]), + 1024, + ) + .await + .unwrap_err(); + assert_eq!(error.status, StatusCode::PAYLOAD_TOO_LARGE); + assert!(fixture.saved().is_empty()); + // Exactly at the limit is accepted. + let saved = receive(store, "fits.csv".into(), Body::from(vec![b'x'; 1024]), 1024) + .await + .unwrap(); + assert_eq!(saved.bytes, 1024); + assert_eq!(fixture.saved(), ["fits.csv"]); +} + +#[test] +fn removes_the_partial_file_when_the_client_aborts() { + let temp = tempfile::tempdir().unwrap(); + let mut handle = start(config(&temp, 0)).unwrap(); + let port = handle.port; + let imports = temp.path().join("notebooks").join(IMPORTS_FOLDER); + let mut stream = TcpStream::connect(("127.0.0.1", port)).unwrap(); + write!( + stream, + "PUT /api/v1/import?name=partial.csv HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\n\ + Authorization: Bearer test-capability\r\nContent-Length: 1000000\r\n\r\n" + ) + .unwrap(); + stream.write_all(&[b'x'; 4096]).unwrap(); + stream.flush().unwrap(); + // The temporary file exists while the upload is in flight. + let in_flight = (0..200).any(|_| { + thread::sleep(Duration::from_millis(10)); + fs::read_dir(&imports).is_ok_and(|entries| { + entries + .flatten() + .any(|entry| entry.file_name().to_string_lossy().ends_with(".part")) + }) + }); + assert!(in_flight); + drop(stream); + let cleaned = (0..200).any(|_| { + thread::sleep(Duration::from_millis(10)); + fs::read_dir(&imports) + .unwrap() + .flatten() + .all(|entry| entry.file_name() == ".gitignore") + }); + assert!(cleaned, "partial upload left behind"); + handle.stop(); +} + +#[test] +fn serves_uploads_over_a_real_connection() { + let temp = tempfile::tempdir().unwrap(); + let mut handle = start(config(&temp, 0)).unwrap(); + let port = handle.port; + let mut stream = TcpStream::connect(("127.0.0.1", port)).unwrap(); + write!( + stream, + "PUT /api/v1/import?name=real.csv HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\n\ + Authorization: Bearer test-capability\r\nContent-Length: 4\r\nConnection: close\r\n\r\na,b\n" + ) + .unwrap(); + let mut status = String::new(); + BufReader::new(&stream).read_line(&mut status).unwrap(); + assert!(status.starts_with("HTTP/1.1 201"), "{status}"); + handle.stop(); + assert_eq!( + fs::read( + temp.path() + .join("notebooks") + .join(IMPORTS_FOLDER) + .join("real.csv") + ) + .unwrap(), + b"a,b\n" + ); +} + +#[tokio::test] +async fn deletes_only_an_uploaded_file_by_its_saved_name() { + let fixture = Fixture::new(); + let (status, _) = fixture + .send(fixture.request("DELETE", "/api/v1/import?name=none.csv", b"")) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + fixture.upload("keep.csv", b"keep").await; + fixture.upload("drop.csv", b"drop").await; + fs::write(fixture.root.join("outside.csv"), "root file").unwrap(); + for name in ["../outside.csv", ".gitignore", "we:ird.csv", "drop.sql"] { + let uri = format!("/api/v1/import?name={}", encode(name)); + let (status, _) = fixture.send(fixture.request("DELETE", &uri, b"")).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{name}"); + } + let (status, body) = fixture + .send(fixture.request("DELETE", "/api/v1/import?name=drop.csv", b"")) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["name"], "drop.csv"); + assert_eq!(fixture.saved(), ["keep.csv"]); + assert!(fixture.root.join("outside.csv").exists()); + assert!(fixture.imports().join(".gitignore").exists()); +} diff --git a/http/src/lib.rs b/http/src/lib.rs index 8620158..de50cef 100644 --- a/http/src/lib.rs +++ b/http/src/lib.rs @@ -18,6 +18,7 @@ mod windows_process; use platform::*; mod duckdb_ui; mod history; +mod imports; mod mcp; mod project_git; use serde::{Deserialize, Serialize}; @@ -243,6 +244,7 @@ struct BootstrapResponse { engine: &'static str, duckdb_version: String, mcp_bridge: bool, + imports: imports::Capability, } #[derive(Serialize)] @@ -1126,6 +1128,8 @@ impl NotebookStore { &mut files, &mut directories, )?; + files.retain(|entry| !imports::is_imports_entry(&entry.path)); + directories.retain(|path| !imports::is_imports_entry(path)); files.sort_by(|left, right| left.path.cmp(&right.path)); directories.sort(); Ok(NotebookListing { files, directories }) @@ -1723,6 +1727,7 @@ async fn bootstrap( engine: "native-duckdb", duckdb_version: state.duckdb_version.clone(), mcp_bridge: !state.mcp_token.is_empty(), + imports: imports::capability(&state), }), )) } @@ -2301,6 +2306,7 @@ fn router(state: AppState) -> Router { .route("/api/v1/file/move", post(move_file)) .route("/api/v1/file/duplicate", post(duplicate_file)) .route("/api/v1/folder", post(create_folder)) + .route("/api/v1/import", imports::routes()) .route("/quack", post(quack).options(options_quack)); if !state.mcp_token.is_empty() { use rmcp::transport::streamable_http_server::{ diff --git a/http/src/platform/unix.rs b/http/src/platform/unix.rs index dc7cfc6..969524e 100644 --- a/http/src/platform/unix.rs +++ b/http/src/platform/unix.rs @@ -363,6 +363,23 @@ pub(crate) fn atomic_write_at( result } +/// Creates a new private file in `parent`; fails if the name exists, including as a symlink. +pub(crate) fn create_new_file_at(parent: libc::c_int, name: &CStr) -> std::io::Result { + let fd = unsafe { + libc::openat( + parent, + name.as_ptr(), + libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL | libc::O_NOFOLLOW | libc::O_CLOEXEC, + 0o600, + ) + }; + if fd < 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(unsafe { File::from_raw_fd(fd) }) + } +} + pub(crate) fn create_directory_at(parent: libc::c_int, name: &CStr) -> std::io::Result<()> { if unsafe { libc::mkdirat(parent, name.as_ptr(), 0o755) } == 0 { Ok(()) diff --git a/http/src/platform/windows.rs b/http/src/platform/windows.rs index 3f08ebb..edcb9bf 100644 --- a/http/src/platform/windows.rs +++ b/http/src/platform/windows.rs @@ -208,6 +208,16 @@ pub(crate) fn open_file_at(parent: HANDLE, name: &CStr) -> Result { FILE_NON_DIRECTORY_FILE, ) } +/// Creates a new file in `parent`; fails if the name exists, including as a reparse point. +pub(crate) fn create_new_file_at(parent: HANDLE, name: &CStr) -> Result { + open_at( + parent, + name, + FILE_WRITE_DATA | FILE_READ_ATTRIBUTES | DELETE, + FILE_CREATE, + FILE_NON_DIRECTORY_FILE, + ) +} pub(crate) fn create_directory_at(parent: HANDLE, name: &CStr) -> Result<()> { open_at( parent, From 91e6fe36166c86abcae872b0242b6d05cc8e82de Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Sat, 14 Nov 2026 16:49:55 -0800 Subject: [PATCH 2/2] Let a client cancel an upload by id after its body has arrived An aborted request cannot stop an upload the server has already received in full and saved. PUT /api/v1/import takes an optional client-chosen id, and DELETE /api/v1/import?id= deletes that upload's file if it was saved or makes it fail before it is saved; both outcomes are decided under one lock. --- docs/notebooks.md | 2 + http/src/imports.rs | 167 +++++++++++++++++++++++++++++++------- http/src/imports/tests.rs | 63 +++++++++++++- 3 files changed, 199 insertions(+), 33 deletions(-) diff --git a/docs/notebooks.md b/docs/notebooks.md index e91ccaa..902e572 100644 --- a/docs/notebooks.md +++ b/docs/notebooks.md @@ -16,6 +16,8 @@ A browser does not tell a page where a dropped file is stored, so the notebook U - An existing name is never replaced: the upload is saved as `name (2).csv`, `name (3).csv` and so on. The folder and file names are opened relative to the notebook root's descriptor without following symlinks, so a symlinked `.sqlnb-imports` is refused and a symlink inside it is neither written through nor deleted. - A declared `Content-Length` above `maxBytes` is refused with `413` before reading; the body is also counted while it streams. Bytes go to a temporary file that is renamed into place only after the whole body arrives and is synced; a refused, failed or aborted upload removes it. +`PUT` also takes an optional `id`, a lowercase hyphenated UUID chosen by the client. An aborted request cannot cancel an upload whose whole body the server already received and saved, so a client that cancels also sends `DELETE /api/v1/import?id=`: it deletes the file if that upload was saved, and otherwise makes the upload fail before it is saved. Publishing and cancelling are decided under one lock, so one of the two always wins cleanly. Ids are used once; the server remembers the latest 1024. + `DELETE /api/v1/import?name=` removes one uploaded file by the name the upload returned. Uploaded files otherwise stay until the user deletes them. ## DuckDB UI notebooks diff --git a/http/src/imports.rs b/http/src/imports.rs index 5f91e5d..51d2a05 100644 --- a/http/src/imports.rs +++ b/http/src/imports.rs @@ -59,8 +59,64 @@ pub(crate) fn is_imports_entry(path: &str) -> bool { } #[derive(Deserialize)] -struct NameQuery { +struct UploadQuery { name: String, + /// Client-chosen UUID by which the client can cancel this upload. + id: Option, +} + +#[derive(Deserialize)] +struct RemoveQuery { + name: Option, + id: Option, +} + +/// Uploads a client may still cancel, by notebook root and upload id. Cancelling +/// by id covers the race an aborted request cannot: the server may have received +/// the whole body, and saved it, before the client's abort arrives. +#[derive(Default)] +struct Uploads { + states: std::collections::HashMap<(PathBuf, String), UploadState>, + order: std::collections::VecDeque<(PathBuf, String)>, +} + +enum UploadState { + Receiving, + Cancelled, + Saved(String), +} + +const MAX_TRACKED_UPLOADS: usize = 1024; +static UPLOADS: std::sync::LazyLock> = std::sync::LazyLock::new(Default::default); + +fn uploads() -> std::sync::MutexGuard<'static, Uploads> { + UPLOADS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +impl Uploads { + fn set(&mut self, key: &(PathBuf, String), state: UploadState) { + if self.states.insert(key.clone(), state).is_none() { + self.order.push_back(key.clone()); + } + while self.order.len() > MAX_TRACKED_UPLOADS { + if let Some(oldest) = self.order.pop_front() { + self.states.remove(&oldest); + } + } + } +} + +fn upload_id(value: &str) -> Result { + match uuid::Uuid::parse_str(value) { + Ok(id) if id.hyphenated().to_string() == value => Ok(value.to_owned()), + _ => Err(ApiError::new( + StatusCode::BAD_REQUEST, + "invalid_upload_id", + "id must be a lowercase hyphenated UUID", + )), + } } #[derive(Debug, Serialize)] @@ -206,11 +262,12 @@ enum Chunk { } /// Writes chunks to a temporary file and gives it the first free numbered name -/// once `Finish` arrives. If the sender goes away first, or anything fails, the -/// temporary file is removed. +/// once `Finish` arrives. If the sender goes away first, the upload was cancelled +/// by id, or anything fails, the temporary file is removed. fn write_upload( store: &NotebookStore, name: &str, + tracked: Option<&(PathBuf, String)>, mut chunks: tokio::sync::mpsc::Receiver, ) -> Result<(String, u64), ApiError> { let folder = open_imports_folder(store, true)?; @@ -231,11 +288,22 @@ fn write_upload( } } file.sync_all().map_err(file_io)?; + // Publishing and cancelling by id are decided under one lock: either the + // cancel comes first and nothing is published, or it finds the saved name. + let mut registry = tracked.map(|key| (key, uploads())); + if let Some((key, registry)) = ®istry { + if matches!(registry.states.get(*key), Some(UploadState::Cancelled)) { + return Err(aborted()); + } + } for number in 1..=MAX_NUMBERED_NAMES { let candidate = numbered(name, number); let destination = CString::new(candidate.as_str()).map_err(|_| invalid_name())?; match rename_no_replace(parent, &temporary, parent, &destination) { Ok(()) => { + if let Some((key, registry)) = registry.as_mut() { + registry.set(key, UploadState::Saved(candidate.clone())); + } sync_directory(parent).map_err(file_io)?; return Ok(candidate); } @@ -263,14 +331,24 @@ fn write_upload( pub(crate) async fn receive( store: Arc, name: String, + id: Option, mut body: Body, limit: u64, ) -> Result { + let tracked = id.map(|id| (store.root.clone(), id)); + if let Some(key) = &tracked { + let mut registry = uploads(); + // An id is used once; one cancelled before its upload began stays cancelled. + if registry.states.contains_key(key) { + return Err(aborted()); + } + registry.set(key, UploadState::Receiving); + } let (sender, receiver) = tokio::sync::mpsc::channel(4); let writer_store = store.clone(); - let writer_name = name.clone(); - let writer = - tokio::task::spawn_blocking(move || write_upload(&writer_store, &writer_name, receiver)); + let writer = tokio::task::spawn_blocking(move || { + write_upload(&writer_store, &name, tracked.as_ref(), receiver) + }); let mut received = 0u64; let streamed = async { while let Some(frame) = body.frame().await { @@ -312,11 +390,12 @@ pub(crate) async fn receive( async fn upload( State(state): State, headers: HeaderMap, - Query(query): Query, + Query(query): Query, body: Body, ) -> Result { authenticate(&headers, &state)?; let name = safe_name(&query.name)?; + let id = query.id.as_deref().map(upload_id).transpose()?; let declared = headers .get(header::CONTENT_LENGTH) .and_then(|value| value.to_str().ok()) @@ -324,39 +403,67 @@ async fn upload( if declared.is_some_and(|length| length > MAX_IMPORT_BYTES) { return Err(too_large(MAX_IMPORT_BYTES)); } - let response = receive(state.notebooks.clone(), name, body, MAX_IMPORT_BYTES).await?; + let response = receive(state.notebooks.clone(), name, id, body, MAX_IMPORT_BYTES).await?; Ok((StatusCode::CREATED, Json(response))) } -/// Deletes one uploaded file by the name the upload returned. +fn delete_saved(store: &NotebookStore, name: &str) -> Result<(), ApiError> { + let folder = open_imports_folder(store, false)?; + let parent = folder.platform_handle(); + let file = CString::new(name).map_err(|_| invalid_name())?; + let not_found = || ApiError::new(StatusCode::NOT_FOUND, "not_found", "no such uploaded file"); + let identity = metadata_at(parent, &file).map_err(|_| not_found())?; + if !identity.regular { + return Err(not_found()); + } + remove_file_at(parent, &file).map_err(file_io)?; + sync_directory(parent).map_err(file_io) +} + +/// Deletes one uploaded file by the name the upload returned (`?name=`), or +/// cancels an upload by its id (`?id=`), deleting its file if it was already saved. async fn remove( State(state): State, headers: HeaderMap, - Query(query): Query, + Query(query): Query, ) -> Result { authenticate(&headers, &state)?; - let name = safe_name(&query.name)?; - if name != query.name { - return Err(invalid_name()); - } let store = state.notebooks.clone(); - let removed = name.clone(); - tokio::task::spawn_blocking(move || { - let folder = open_imports_folder(&store, false)?; - let parent = folder.platform_handle(); - let file = CString::new(name).map_err(|_| invalid_name())?; - let not_found = - || ApiError::new(StatusCode::NOT_FOUND, "not_found", "no such uploaded file"); - let identity = metadata_at(parent, &file).map_err(|_| not_found())?; - if !identity.regular { - return Err(not_found()); + match (query.name, query.id) { + (Some(name), None) => { + if safe_name(&name)? != name { + return Err(invalid_name()); + } + let removed = name.clone(); + tokio::task::spawn_blocking(move || delete_saved(&store, &name)) + .await + .map_err(|_| task_failed())??; + Ok(Json(serde_json::json!({ "name": removed }))) } - remove_file_at(parent, &file).map_err(file_io)?; - sync_directory(parent).map_err(file_io) - }) - .await - .map_err(|_| task_failed())??; - Ok(Json(serde_json::json!({ "name": removed }))) + (None, Some(id)) => { + let key = (store.root.clone(), upload_id(&id)?); + tokio::task::spawn_blocking(move || { + let mut registry = uploads(); + if let Some(UploadState::Saved(name)) = registry.states.get(&key) { + match delete_saved(&store, &name.clone()) { + Ok(()) => {} + Err(error) if error.status == StatusCode::NOT_FOUND => {} + Err(error) => return Err(error), + } + } + registry.set(&key, UploadState::Cancelled); + Ok(()) + }) + .await + .map_err(|_| task_failed())??; + Ok(Json(serde_json::json!({ "id": id, "cancelled": true }))) + } + _ => Err(ApiError::new( + StatusCode::BAD_REQUEST, + "invalid_request", + "pass either name or id", + )), + } } #[cfg(test)] diff --git a/http/src/imports/tests.rs b/http/src/imports/tests.rs index 00d83b3..2044926 100644 --- a/http/src/imports/tests.rs +++ b/http/src/imports/tests.rs @@ -312,6 +312,7 @@ async fn enforces_the_limit_while_streaming_and_removes_the_partial_file() { let error = receive( store.clone(), "big.csv".into(), + None, Body::from(vec![b'x'; 4096]), 1024, ) @@ -320,9 +321,15 @@ async fn enforces_the_limit_while_streaming_and_removes_the_partial_file() { assert_eq!(error.status, StatusCode::PAYLOAD_TOO_LARGE); assert!(fixture.saved().is_empty()); // Exactly at the limit is accepted. - let saved = receive(store, "fits.csv".into(), Body::from(vec![b'x'; 1024]), 1024) - .await - .unwrap(); + let saved = receive( + store, + "fits.csv".into(), + None, + Body::from(vec![b'x'; 1024]), + 1024, + ) + .await + .unwrap(); assert_eq!(saved.bytes, 1024); assert_eq!(fixture.saved(), ["fits.csv"]); } @@ -416,3 +423,53 @@ async fn deletes_only_an_uploaded_file_by_its_saved_name() { assert!(fixture.root.join("outside.csv").exists()); assert!(fixture.imports().join(".gitignore").exists()); } + +#[tokio::test] +async fn cancelling_by_upload_id_discards_a_saved_or_later_upload() { + let fixture = Fixture::new(); + let first = "6f1c1e8e-3c55-4f0a-9d8e-0a6b9a6c1f01"; + let second = "6f1c1e8e-3c55-4f0a-9d8e-0a6b9a6c1f02"; + // The whole body arrived and was saved before the client's cancel. + let uri = format!("/api/v1/import?name=a.csv&id={first}"); + let (status, _) = fixture.send(fixture.request("PUT", &uri, b"a")).await; + assert_eq!(status, StatusCode::CREATED); + fixture.upload("keep.csv", b"keep").await; + let uri = format!("/api/v1/import?id={first}"); + let (status, body) = fixture.send(fixture.request("DELETE", &uri, b"")).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["cancelled"], true); + assert_eq!(fixture.saved(), ["keep.csv"]); + // Cancelling again, or an id never seen, succeeds without touching files. + assert_eq!( + fixture.send(fixture.request("DELETE", &uri, b"")).await.0, + StatusCode::OK + ); + let uri = format!("/api/v1/import?id={second}"); + assert_eq!( + fixture.send(fixture.request("DELETE", &uri, b"")).await.0, + StatusCode::OK + ); + // An upload whose id was cancelled before it began is refused. + let uri = format!("/api/v1/import?name=b.csv&id={second}"); + let (status, body) = fixture.send(fixture.request("PUT", &uri, b"b")).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(body["error"]["code"], "upload_aborted"); + assert_eq!(fixture.saved(), ["keep.csv"]); + + for uri in [ + "/api/v1/import?name=c.csv&id=not-a-uuid", + "/api/v1/import?name=c.csv&id=6F1C1E8E-3C55-4F0A-9D8E-0A6B9A6C1F03", + ] { + let (status, body) = fixture.send(fixture.request("PUT", uri, b"c")).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{uri}"); + assert_eq!(body["error"]["code"], "invalid_upload_id"); + } + for uri in [ + "/api/v1/import", + "/api/v1/import?name=keep.csv&id=6f1c1e8e-3c55-4f0a-9d8e-0a6b9a6c1f04", + ] { + let (status, _) = fixture.send(fixture.request("DELETE", uri, b"")).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{uri}"); + } + assert_eq!(fixture.saved(), ["keep.csv"]); +}