From 2d777a4958d6e8096c9b38806f12993fd085218e Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Sat, 14 Nov 2026 16:26:44 -0800 Subject: [PATCH 1/5] Let Instant SQL preview CTEs, casts, windows, views and qualified names The read policy now verifies CTE bodies, cast target types, window functions, view definitions (recursively, as DuckDB binds them) and qualified tables and columns, resolving every name through a catalog retriever configured like the binder's. Refusals state their reason. Adds the policy contract to the runtime smoke test and documents it. --- README.md | 1 + docs/instant-sql.md | 103 ++++++ scripts/read-policy-smoke.mjs | 199 +++++++++++ scripts/runtime-smoke.mjs | 2 + src/include/sqlnb_read_policy.hpp | 557 ++++++++++++++++++++++++++---- 5 files changed, 800 insertions(+), 62 deletions(-) create mode 100644 docs/instant-sql.md create mode 100644 scripts/read-policy-smoke.mjs diff --git a/README.md b/README.md index 1ef006e..0dfcb86 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ To connect an MCP client, start with `mcp := true` and follow the - [Build and test](docs/development.md) — native prerequisites and version-specific builds. - [Notebook storage](docs/notebooks.md) — files, DuckDB UI notebooks, history, and project Git. - [MCP reference](docs/mcp.md) — authentication, tools, and execution behavior. +- [Instant SQL read policy](docs/instant-sql.md) — what previews automatically and why the rest needs Run. - [UI assets](docs/ui-assets.md) — custom frontends, updates, and the deployment contract. - [Release procedures](docs/releases.md) — maintainer builds, signing, and publication. diff --git a/docs/instant-sql.md b/docs/instant-sql.md new file mode 100644 index 0000000..f5b4a47 --- /dev/null +++ b/docs/instant-sql.md @@ -0,0 +1,103 @@ +# Instant SQL read policy + +[Back to README](../README.md) + +Instant SQL previews the statement under the cursor while the user types, without +an explicit Run. The browser asks the native extension to run it through +`sqlnb_read_query(sql)`, which applies the read policy in +[`sqlnb_read_policy.hpp`](../src/include/sqlnb_read_policy.hpp). `sqlnb_is_read_only(sql)` +reports the same decision as a boolean; it is a hint, never an authorization. + +## How a statement is verified + +`sqlnb_read_query` parses the SQL with DuckDB's built-in parser (no parser +extensions) and verifies the parsed statement inside the bind of the outer +query, in the same transaction that then binds and runs it. Nothing is +prepared, explained or executed to classify it. + +Verification happens before DuckDB binds the statement because binding itself +can run code: AT clauses, type parameters, table function binds, replacement +scans of files and URLs, and macro expansion. Every name is resolved through a +catalog entry retriever configured the way DuckDB's binder configures its own: +the caller's search path at the top level and, inside a view, the view's own +catalog and schema first. Names are checked against the allowlists below before +any catalog lookup, so verification never autoloads or installs an extension. + +Anything the verifier does not positively recognise is refused. A refusal is an +error that starts with `SQLNB_READ_ONLY: Instant SQL does not preview` followed by +the reason and, for definitions inside views, the chain of views. The statement +remains available through explicit Run. + +## Accepted + +- Exactly one `SELECT` statement, including set operations, subqueries, `VALUES` + and `FROM`-first syntax. +- Non-recursive `WITH` clauses whose bodies are themselves accepted. A CTE is + visible to later CTEs of the same clause, to the query and to its subqueries, + never inside its own body. When a CTE shares its name with a table or view, that + relation must be acceptable too. +- Tables in attached native DuckDB databases (including attached database files + and temporary tables), unqualified or qualified as `schema.table` or + `database.schema.table`, without generated columns, whose column types are + `BOOLEAN`, signed and unsigned integers, `FLOAT`, `DOUBLE`, `DECIMAL`, `DATE`, + `TIME`, `TIMETZ`, `TIMESTAMP` (all precisions and `WITH TIME ZONE`), `INTERVAL`, + `BLOB`, `UUID` or `VARCHAR` without a collation, or `LIST`, `ARRAY`, `STRUCT` and + `MAP` of those. +- Views in native DuckDB databases whose definitions pass this policy, + recursively, up to 8 nested levels and 64 view expansions per statement. +- The table functions `range`, `generate_series`, `unnest` and `repeat`. +- Constants, column references (including relation-qualified columns such as + `t.col` or `db.schema.t.col` and struct and map field access such as `col.field`), + comparisons, `AND`/`OR`/`NOT`, `CASE`, `BETWEEN`, `IN`, `IS [NOT] NULL`, + `COALESCE`, `TRY`, subscripts, slices, list literals and `*` or `COLUMNS(...)` + without lambdas. +- Calls to an allowlist of side-effect-free scalar and aggregate built-ins + (arithmetic, rounding, string, pattern-matching, list, struct and date/time + helpers, and the common aggregates). +- Window functions: `row_number`, `rank`, `dense_rank`, `percent_rank`, + `cume_dist`, `ntile`, `lag`, `lead`, `first_value`, `last_value`, `nth_value` + and the allowlisted aggregates, with partitions, ordering, frames, `FILTER`, + named windows and `QUALIFY`. +- `CAST`, `TRY_CAST`, `::` and typed literals to the built-in types listed above, + with integer literal parameters (for example `DECIMAL(5, 2)` or `INTEGER[3]`). + +Every function name, function behind an operator or column reference (field +access, implicit struct packing, SQL value functions such as `current_date`) and +type name must resolve, through the binder's search path, to a DuckDB built-in: +an internal entry from DuckDB itself, its `core_functions` or the ICU extension +SQLNB loads at startup. A user macro, user function or user type that shadows a +built-in name is refused. + +## Refused + +- Every statement other than a single `SELECT`: DML, DDL, `PRAGMA`, `ATTACH`, + `SET`, `COPY`, `CALL`, `EXPORT`, PIVOT statements and multiple statements. +- `INSERT`, `UPDATE`, `DELETE`, `MERGE` or `COPY` inside a query or `WITH` clause. +- Recursive CTEs (including `USING KEY`), which can run unbounded before a row + limit applies. +- Names that are not tables or views in an attached native DuckDB database: files, + URLs and other replacement scans, and tables or views in attached databases of + other types. +- Any other table function, including file, network and catalog scans + (`read_csv`, `read_parquet`, `glob`, `query`, `duckdb_*`), table macros and + `PIVOT`/`UNPIVOT`, `DESCRIBE`, `SHOW` and `SUMMARIZE` in `FROM`. +- Functions outside the allowlist, including volatile or state-changing ones such + as `nextval`, `setval` and `random`; user-defined functions; macros, whose bodies + are not verified; qualified and method-style calls such as `schema.f()` or + `x.lower()`. +- Casts to user-defined, extension (`JSON`, `GEOMETRY`) or other unlisted types + (`ENUM`, `UNION`, `VARIANT`, `BIT`), qualified type names and type parameters + that are not integer literals (such as a collation). +- Lambdas, `COLLATE`, prepared-statement parameters, `AT` time-travel clauses, + tables with generated columns and columns of unlisted types. +- Views that reach anything refused here, nest more than 8 levels deep or expand + more than 64 views. +- A name that the surrounding query defines as a CTE, and any statement while a + loaded extension rewrites column names during binding. + +## Tests + +[`read-policy-smoke.mjs`](../scripts/read-policy-smoke.mjs) runs inside the +runtime smoke test of `scripts/build-release.sh`. It checks each accepted shape +against its rows and each refusal against its reason, and asserts that refused +statements leave sequences, tables, files and a local HTTP server untouched. diff --git a/scripts/read-policy-smoke.mjs b/scripts/read-policy-smoke.mjs new file mode 100644 index 0000000..72e7a4b --- /dev/null +++ b/scripts/read-policy-smoke.mjs @@ -0,0 +1,199 @@ +// Instant SQL read policy contract. runtime-smoke.mjs calls this with its +// DuckDB CLI helpers once the native policy functions are registered. +// +// Accepted shapes must return the same rows as the plain query. Refused +// shapes must name their reason and must leave no trace: sequences, tables, +// files and the HTTP server below observe every side effect they could cause. +import assert from 'node:assert/strict'; +import { existsSync, mkdtempSync } from 'node:fs'; +import { createServer } from 'node:http'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +export async function readPolicySmoke({ runSQL, expectSQLError, sqlLiteral, firstValue }) { + let remoteRequests = 0; + const server = createServer((_request, response) => { + remoteRequests += 1; + response.writeHead(200, { 'content-type': 'text/csv' }); + response.end('value\n1\n'); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const remoteCSV = `http://127.0.0.1:${server.address().port}/remote.csv`; + const scratch = mkdtempSync(join(tmpdir(), 'sqlnb-read-policy.')); + const copyTarget = join(scratch, 'copy.csv'); + + // The CLI harness reads one output line per statement, so the rows travel + // as one list value. Aggregation does not promise order: compare as a set. + const canonical = (rows) => (rows ?? []).map((row) => JSON.stringify(row)).sort(); + async function accepts(sql, expected) { + assert.equal(firstValue(await runSQL(`select sqlnb_is_read_only(${sqlLiteral(sql)}) as allowed`)), true, sql); + const rows = firstValue(await runSQL(`select list(r) as rows from sqlnb_read_query(${sqlLiteral(sql)}) as r`)); + assert.deepEqual(canonical(rows), canonical(expected), sql); + } + async function refuses(sql, reason) { + assert.equal(firstValue(await runSQL(`select sqlnb_is_read_only(${sqlLiteral(sql)}) as allowed`)), false, sql); + await expectSQLError( + `select * from sqlnb_read_query(${sqlLiteral(sql)})`, + new RegExp(`SQLNB_READ_ONLY: Instant SQL does not preview ${reason}`), + ); + } + const sequenceValue = async () => firstValue(await runSQL('select currval(\'policy.counter\') as value')); + const policyRows = async () => firstValue(await runSQL('select count(*)::integer as count from policy.items')); + + await runSQL(` + create schema policy; + create table policy.items as + select i::integer as id, (i % 2)::integer as grp, ('item' || i)::varchar as label, + {x: i::integer, y: 'k' || i} as info, map {'k': i::integer} as tags + from range(1, 5) as r(i); + create sequence policy.counter; + create view policy.items_view as select id, label from policy.items where id > 1; + create view policy.items_view_view as select id * 10 as id10 from policy.items_view; + create view policy.cte_view as with base as (select id from policy.items) select max(id) as top from base; + create view policy.remote_view as select * from read_csv(${sqlLiteral(remoteCSV)}); + create view policy.remote_view_view as select * from policy.remote_view; + create view policy.glob_view as select * from glob(${sqlLiteral(join(scratch, '*'))}); + create view policy.sequence_view as select nextval('policy.counter') as n; + create macro policy.bump() as nextval('policy.counter'); + create macro policy.bump_rows() as table select nextval('policy.counter') as n; + create type policy.mood as enum ('calm', 'loud'); + attach ':memory:' as policy_db; + create schema policy_db.inner_schema; + create table policy_db.inner_schema.remote_name as select 42::integer as answer; + create view policy_db.inner_schema.local_view as select answer from remote_name; + create view policy_db.inner_schema.safe_view as select answer + 1 as next from remote_name; + -- View bodies bind their own catalog and schema first. Each name below + -- resolves to a safe relation one way and a URL scan the other way. + create view main.remote_name as select * from read_csv(${sqlLiteral(remoteCSV)}); + create table main.trap_name as select 1::integer as value; + create view policy_db.inner_schema.trap_name as select * from read_csv(${sqlLiteral(remoteCSV)}); + create view policy_db.inner_schema.trap_view as select * from trap_name; + select nextval('policy.counter') as initialized`); + for (let level = 0; level <= 8; level += 1) { + const source = level === 0 ? 'policy.items' : `policy.depth_${level - 1}`; + await runSQL(`create view policy.depth_${level} as select id from ${source}; select ${level} as created`); + } + // Creating the URL-backed views bound them once; only later requests count. + const requestsAfterSetup = remoteRequests; + assert.equal(await sequenceValue(), 1); + + // 1. CTEs, casts and window functions over otherwise-allowed inputs. + await accepts( + 'with picked as (select id, label from policy.items where id < 3) select * from picked order by id', + [{ id: 1, label: 'item1' }, { id: 2, label: 'item2' }], + ); + await accepts( + `with first_step as (select id from policy.items), + second_step as (select id * 2 as doubled from first_step) + select sum(doubled)::integer as total from second_step where doubled > (select min(id) from first_step)`, + [{ total: 20 }], + ); + // A CTE may shadow a table name; the table it shadows is verified as well. + await accepts('with items as (select 5 as id) select id from items', [{ id: 5 }]); + // A CTE named like a URL is a CTE: nothing is fetched. + await accepts(`with "${remoteCSV}" as (select 1 as one) select one from "${remoteCSV}"`, [{ one: 1 }]); + await accepts( + `select id::varchar as text_id, cast(id as decimal(5, 2)) as amount, try_cast(label as integer) as parsed, + [id]::integer[] as ids, {'a': id}::struct(a bigint) as boxed, date '2024-02-03' as day, + '2024-02-03 04:05:06'::timestamp as moment + from policy.items where id = 1`, + [{ + text_id: '1', amount: 1, parsed: null, ids: [1], boxed: { a: 1 }, day: '2024-02-03', + moment: '2024-02-03 04:05:06', + }], + ); + await accepts( + `select id, row_number() over (partition by grp order by id) as position, + sum(id) over (order by id rows between 1 preceding and current row)::integer as running, + lag(id, 1, 0) over (order by id) as previous, rank() over w as ranked + from policy.items window w as (order by grp) qualify position <= 2 order by id`, + [ + { id: 1, position: 1, running: 1, previous: 0, ranked: 3 }, + { id: 2, position: 1, running: 3, previous: 1, ranked: 1 }, + { id: 3, position: 2, running: 5, previous: 2, ranked: 3 }, + { id: 4, position: 2, running: 7, previous: 3, ranked: 1 }, + ], + ); + + // 2. Views, verified through their definitions with DuckDB's view search path. + await accepts('select * from policy.items_view_view order by id10', [{ id10: 20 }, { id10: 30 }, { id10: 40 }]); + await accepts('select top from policy.cte_view', [{ top: 4 }]); + await accepts('select count(*)::integer as rows from policy.depth_7', [{ rows: 4 }]); + // Inside these views unqualified names bind policy_db.inner_schema first, not main. + await accepts('select answer from policy_db.inner_schema.local_view', [{ answer: 42 }]); + await refuses('select * from policy_db.inner_schema.trap_view', 'the table function \'read_csv\'.*inside view'); + + // 3. Dotted references: catalogs, schemas, relation-qualified columns and struct fields. + await accepts('select next from policy_db.inner_schema.safe_view', [{ next: 43 }]); + await accepts('select answer from policy_db.inner_schema.remote_name', [{ answer: 42 }]); + await accepts( + `select items.id, items.info.x as x, info.y as y, items.tags['k'] as tag, v.label + from policy.items as items join policy.items_view as v on items.id = v.id where items.id = 2`, + [{ id: 2, x: 2, y: 'k2', tag: 2, label: 'item2' }], + ); + await accepts( + 'select policy_db.inner_schema.remote_name.answer from policy_db.inner_schema.remote_name', + [{ answer: 42 }], + ); + + // 4. Refusals, each with its reason and without side effects. + await refuses('insert into policy.items values (9, 1, \'x\', null, null)', 'INSERT statements'); + await refuses('update policy.items set id = 0', 'UPDATE statements'); + await refuses('delete from policy.items', 'DELETE statements'); + await refuses('create table policy.created (a integer)', 'CREATE statements'); + await refuses('drop table policy.items', 'DROP statements'); + await refuses('pragma version', 'PRAGMA statements'); + await refuses('attach \':memory:\' as policy_other', 'ATTACH statements'); + await refuses('set threads = 1', 'SET statements'); + await refuses(`copy (select 1) to ${sqlLiteral(copyTarget)}`, 'COPY statements'); + await refuses('select 1; select 2', 'more than one statement'); + await refuses( + 'with added as (insert into policy.items values (9, 1, \'x\', null, null) returning id) select * from added', + 'INSERT, UPDATE, DELETE or MERGE inside a query', + ); + await refuses('select nextval(\'policy.counter\')', 'the function \'nextval\''); + await refuses( + 'with c as (select id from policy.items) select id, nextval(\'policy.counter\') over () from c', + 'the window function \'nextval\'', + ); + await refuses('select policy.bump()', 'the qualified or method-style call .*bump'); + await runSQL('set search_path = \'policy,main\'; select true as ok'); + await refuses('select bump()', 'the function \'bump\''); + await refuses('select * from bump_rows()', 'the table function \'bump_rows\''); + await refuses('select \'calm\'::mood', 'casts to the type \'mood\''); + await runSQL('reset search_path; select true as ok'); + await runSQL('create schema shadow; create macro shadow.lower(x) as nextval(\'policy.counter\'); select true as ok'); + await runSQL('set search_path = \'shadow,main\'; select true as ok'); + await refuses('select lower(\'A\')', 'the macro \'lower\''); + await runSQL('reset search_path; drop schema shadow cascade; select true as ok'); + await refuses('select \'a\'::varchar(\'nocase\')', 'casts to the type'); + await refuses('select * from policy.sequence_view', 'the function \'nextval\'.*inside view'); + await refuses(`select * from read_csv(${sqlLiteral(remoteCSV)})`, 'the table function \'read_csv\''); + await refuses(`select * from '${remoteCSV}'`, '.*which is not a table or view'); + await refuses('select * from policy.remote_view', 'the table function \'read_csv\'.*inside view'); + await refuses('select * from policy.remote_view_view', 'the table function \'read_csv\'.*inside view .* > '); + await refuses('select * from policy.glob_view', 'the table function \'glob\'.*inside view'); + await refuses('select count(*) from policy.depth_8', 'the view .*more than 8 levels deep'); + await refuses( + 'with recursive r(n) as (select 1 union all select n + 1 from r) select * from r', + 'recursive CTEs', + ); + await refuses('with loop as (select * from loop) select * from loop', '\'loop\', which is not a table or view'); + // A CTE is not in scope inside its own body, where DuckDB would fall back to a replacement scan. + await refuses( + `with "${remoteCSV}" as (select * from "${remoteCSV}") select * from "${remoteCSV}"`, + '.*which is not a table or view', + ); + await refuses('select list_transform([1], x -> x + 1)', 'the function \'list_transform\''); + await refuses('select random()', 'the function \'random\''); + await refuses('select * from policy.items at (version => 1)', 'AT \\(time travel\\) clauses'); + await refuses('select * from (pivot policy.items on grp using sum(id))', 'statements that DuckDB expands into several statements'); + + assert.equal(await sequenceValue(), 1, 'refused statements must not advance sequences'); + assert.equal(await policyRows(), 4, 'refused statements must not write rows'); + assert.equal(remoteRequests, requestsAfterSetup, 'refused statements must not reach the network'); + assert.equal(existsSync(copyTarget), false, 'refused statements must not write files'); + + await runSQL('detach policy_db; drop schema policy cascade; drop view main.remote_name; drop table main.trap_name; select true as cleaned'); + await new Promise((resolve) => server.close(resolve)); +} diff --git a/scripts/runtime-smoke.mjs b/scripts/runtime-smoke.mjs index 1a2dc0a..f5d6fe1 100644 --- a/scripts/runtime-smoke.mjs +++ b/scripts/runtime-smoke.mjs @@ -15,6 +15,7 @@ import { createServer } from 'node:http'; import { createHash } from 'node:crypto'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { readPolicySmoke } from './read-policy-smoke.mjs'; const values = new Map(); for (let index = 2; index < process.argv.length; index += 2) { @@ -228,6 +229,7 @@ if (hasReadPolicy) { await runSQL("select * from sqlnb_read_query('select * from shared_table')"), [{ value: 7 }], ); + await readPolicySmoke({ runSQL, expectSQLError, sqlLiteral, firstValue }); } const startRows = await runSQL( `select sqlnb_start(assets := ${sqlLiteral(fixture)}, open_browser := false, notebook_root := ${sqlLiteral(notebookRoot)}, mcp := true) as status`, diff --git a/src/include/sqlnb_read_policy.hpp b/src/include/sqlnb_read_policy.hpp index 9f8c32c..3f9be68 100644 --- a/src/include/sqlnb_read_policy.hpp +++ b/src/include/sqlnb_read_policy.hpp @@ -1,22 +1,40 @@ #pragma once +// Instant SQL read policy. +// // This policy deliberately accepts a subset of SQL. It parses without extension // hooks and never PREPAREs, EXPLAINs or executes unverified SQL to classify it. -// Unknown constructs remain available through explicit Run, never Instant SQL. +// DuckDB binding itself can run code (AT clauses, type parameters, table +// function binds, replacement scans, macro expansion), so the parsed statement +// is verified before binding, in the binding transaction, resolving every name +// through a CatalogEntryRetriever configured exactly as DuckDB's binder +// configures its own (including the search path DuckDB uses inside views). +// Anything not positively recognised is refused with a reason; refused +// statements remain available through explicit Run, never Instant SQL. #if __has_include("duckdb/main/query_result_stream.hpp") #include "duckdb/catalog/catalog.hpp" #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/type_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry_retriever.hpp" +#include "duckdb/catalog/catalog_search_path.hpp" #include "duckdb/catalog/entry_lookup_info.hpp" +#include "duckdb/common/enums/statement_type.hpp" #include "duckdb/function/scalar_function.hpp" #include "duckdb/function/table_function.hpp" +#include "duckdb/main/database_manager.hpp" #include "duckdb/main/extension/extension_loader.hpp" #include "duckdb/parser/parser.hpp" #include "duckdb/parser/column_list.hpp" +#include "duckdb/parser/expression/cast_expression.hpp" #include "duckdb/parser/expression/columnref_expression.hpp" +#include "duckdb/parser/expression/constant_expression.hpp" #include "duckdb/parser/common_table_expression_info.hpp" #include "duckdb/parser/parsed_expression_iterator.hpp" #include "duckdb/parser/expression/function_expression.hpp" #include "duckdb/parser/expression/subquery_expression.hpp" +#include "duckdb/parser/expression/type_expression.hpp" +#include "duckdb/parser/expression/window_expression.hpp" #include "duckdb/parser/query_node/select_node.hpp" #include "duckdb/parser/query_node/set_operation_node.hpp" #include "duckdb/parser/statement/select_statement.hpp" @@ -25,19 +43,49 @@ #include "duckdb/parser/tableref/joinref.hpp" #include "duckdb/parser/tableref/subqueryref.hpp" #include "duckdb/parser/tableref/table_function_ref.hpp" +#include "duckdb/planner/binder.hpp" +#include "duckdb/planner/binding_alias.hpp" +#include "duckdb/planner/planner_extension.hpp" namespace duckdb { namespace sqlnb_read_policy { +//! Views may read other views. Past this nesting depth a statement needs an explicit Run. +static constexpr idx_t MAX_VIEW_DEPTH = 8; +//! Bounds the total number of view bodies verified for one statement, so views +//! that share (diamond) references cannot make verification exponential. +static constexpr idx_t MAX_VIEW_EXPANSIONS = 64; + class Policy { public: - explicit Policy(ClientContext &context_p) : context(context_p) {} + //! `outer_binder` is the binder that will bind the returned statement. Its + //! catalog search path and the CTEs of the query around it apply to the + //! statement, so verification uses them too. + explicit Policy(ClientContext &context_p, optional_ptr outer_binder_p = nullptr) + : context(context_p), outer_binder(outer_binder_p) { + auto retriever = make_uniq(context); + if (outer_binder) retriever->Inherit(outer_binder->EntryRetriever()); + retrievers.push_back(std::move(retriever)); + } unique_ptr Parse(const string &sql) { auto parser = Parser::GetBuiltinParser(); parser.ParseQuery(sql); - if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::SELECT_STATEMENT) { - Reject(); + if (parser.statements.empty()) Refuse("an empty statement"); + if (parser.statements.size() != 1) Refuse("more than one statement at a time"); + auto type = parser.statements[0]->type; + if (type == StatementType::MULTI_STATEMENT) { + // PIVOT expands into DDL that creates types before the query runs. + Refuse("statements that DuckDB expands into several statements (such as PIVOT)"); + } + if (type != StatementType::SELECT_STATEMENT) { + // DML, DDL, PRAGMA, ATTACH, SET, COPY, CALL, EXPORT and every other kind. + Refuse(StatementTypeToString(type) + " statements"); + } + // A planner extension can turn any unresolved column name into arbitrary + // SQL during binding; none ships with DuckDB itself. + for (auto &extension : PlannerExtension::Iterate(context)) { + if (extension.get_sql_value_function) Refuse("statements while an extension rewrites column names during binding"); } auto statement = unique_ptr_cast(std::move(parser.statements[0])); CheckNode(*statement->node); @@ -45,15 +93,64 @@ class Policy { } private: - [[noreturn]] static void Reject() { - throw InvalidInputException("SQLNB_READ_ONLY: this statement cannot be verified for Instant SQL; use Run explicitly"); + [[noreturn]] void Refuse(const string &reason) const { + string message = "SQLNB_READ_ONLY: Instant SQL does not preview " + reason; + if (!view_path.empty()) message += " (inside view " + StringUtil::Join(view_path, " > ") + ")"; + // Single-argument constructor: the message is never used as a format string. + throw InvalidInputException(message + ". Use Run to execute this statement explicitly."); } - void CheckFunction(FunctionExpression &function, bool table_function) { - const auto name = StringUtil::Lower(function.FunctionName().GetIdentifierName()); - // Names are only a first filter. Resolve the actual entry to exclude user - // macros/UDFs shadowing an allowed built-in and extension-provided overloads. - static const unordered_set scans = {"range", "generate_series", "unnest", "repeat"}; + static string Quote(const string &name) { + return "'" + name + "'"; + } + + CatalogEntryRetriever &Retriever() { + return *retrievers.back(); + } + + //! Resolves a function name exactly as the expression binder does and + //! requires a DuckDB built-in (never a macro, UDF or third-party extension). + void RequireBuiltin(const string &name, CatalogType lookup_type, const string &display, bool allow_window = false) { + optional_ptr entry; + try { + EntryLookupInfo lookup(lookup_type, QualifiedName(Identifier(name))); + auto bound = Binder::BindTableName(Retriever(), QualifiedName(Identifier(name))); + entry = Retriever().GetEntry(EntryLookupInfo(lookup, bound), OnEntryNotFound::RETURN_NULL); + } catch (const Exception &) { + Refuse(display + ", which could not be resolved"); + } + if (!entry) Refuse(display + ", which does not exist"); + if (entry->type == CatalogType::MACRO_ENTRY || entry->type == CatalogType::TABLE_MACRO_ENTRY) { + Refuse("the macro " + Quote(name) + " (macro bodies are not verified)"); + } + if (!entry->internal) Refuse("the user-defined function " + Quote(name)); + auto extension = entry->extension_name.GetIdentifierName(); + // SQLNB loads DuckDB's ICU extension at startup; it replaces the date/time + // value functions (current_date, ...) with time-zone-aware built-ins. + if (!extension.empty() && extension != "core_functions" && extension != "icu") { + Refuse("the function " + Quote(name) + " from the " + Quote(extension) + " extension"); + } + bool accepted = false; + switch (entry->type) { + case CatalogType::SCALAR_FUNCTION_ENTRY: + accepted = lookup_type == CatalogType::SCALAR_FUNCTION_ENTRY; + break; + case CatalogType::AGGREGATE_FUNCTION_ENTRY: + accepted = lookup_type == CatalogType::SCALAR_FUNCTION_ENTRY; + break; + case CatalogType::WINDOW_FUNCTION_ENTRY: + accepted = allow_window; + break; + case CatalogType::TABLE_FUNCTION_ENTRY: + accepted = lookup_type == CatalogType::TABLE_FUNCTION_ENTRY; + break; + default: + break; + } + if (!accepted) Refuse(display + ", which is not a built-in function of the expected kind"); + } + + static const unordered_set &ScalarAllowlist() { static const unordered_set scalars = { "+", "-", "*", "/", "//", "%", "**", "||", "abs", "ceil", "ceiling", "floor", "round", "trunc", "sqrt", "pow", "power", "exp", "ln", "log", "log10", "sign", "greatest", "least", @@ -66,31 +163,106 @@ class Policy { "array_slice", "list_slice", "array_length", "list_count", "list_sum", "list_min", "list_max", "date_part", "date_trunc", "date_diff", "datediff", "strftime", "strptime", "try_strptime" }; - if (!(table_function ? scans : scalars).count(name)) Reject(); - auto entry = Catalog::GetEntry(context, - EntryLookupInfo(table_function ? CatalogType::TABLE_FUNCTION_ENTRY : CatalogType::SCALAR_FUNCTION_ENTRY, - function.GetQualifiedName()), OnEntryNotFound::RETURN_NULL); - if (!entry || !entry->internal || - (entry->type != CatalogType::SCALAR_FUNCTION_ENTRY && entry->type != CatalogType::AGGREGATE_FUNCTION_ENTRY && - entry->type != CatalogType::TABLE_FUNCTION_ENTRY)) Reject(); - auto extension = entry->extension_name.GetIdentifierName(); - if (!extension.empty() && extension != "core_functions") Reject(); + return scalars; + } + + void CheckFunction(FunctionExpression &function, bool table_function) { + const auto name = StringUtil::Lower(function.FunctionName().GetIdentifierName()); + // A qualified call is either a catalog function or DuckDB's method-call + // rewrite ("x.lower()"); the binder chooses between them by lookup. + if (function.GetQualifiedName().Path().size() != 1) { + Refuse("the qualified or method-style call " + Quote(function.GetQualifiedName().ToString())); + } + // Names are a first filter, checked before any catalog lookup: DuckDB + // autoloads (and may install) extensions for unknown function names. + static const unordered_set scans = {"range", "generate_series", "unnest", "repeat"}; + if (table_function && !scans.count(name)) { + Refuse("the table function " + Quote(name) + " (only range, generate_series, unnest and repeat are previewed; " + "file, network and catalog scans need Run)"); + } + if (!table_function && !ScalarAllowlist().count(name)) { + Refuse("the function " + Quote(name) + " (only approved side-effect-free built-ins are previewed)"); + } + // Resolve the actual entry to exclude user macros/UDFs shadowing an allowed + // built-in and extension-provided overloads. + RequireBuiltin(name, table_function ? CatalogType::TABLE_FUNCTION_ENTRY : CatalogType::SCALAR_FUNCTION_ENTRY, + (table_function ? "the table function " : "the function ") + Quote(name)); ParsedExpressionIterator::EnumerateChildren(function, [this](ParsedExpression &child) { CheckExpression(child); }); } + void CheckWindow(WindowExpression &window) { + static const unordered_set window_functions = { + "row_number", "rank", "dense_rank", "percent_rank", "cume_dist", "ntile", + "lag", "lead", "first_value", "last_value", "nth_value" + }; + const auto name = StringUtil::Lower(window.FunctionName().GetIdentifierName()); + if (window.GetQualifiedName().Path().size() != 1) { + Refuse("the qualified window function " + Quote(window.GetQualifiedName().ToString())); + } + if (!window_functions.count(name) && !ScalarAllowlist().count(name)) { + Refuse("the window function " + Quote(name)); + } + // The binder resolves window names in the function namespace and expands a + // macro of that name; require a built-in aggregate or window function. + RequireBuiltin(name, CatalogType::SCALAR_FUNCTION_ENTRY, "the window function " + Quote(name), true); + // Arguments, partitions, orders, frame offsets and FILTER are all children. + // RANGE offsets bind +/- from the system catalog, never the search path. + ParsedExpressionIterator::EnumerateChildren(window, [this](ParsedExpression &child) { CheckExpression(child); }); + } + + //! A column reference may bind to something other than a column: an implicit + //! struct_pack of a whole relation, a struct/map field extraction, or a SQL + //! value function. Each is a function the binder resolves through the search + //! path, so each must resolve to a built-in. + void CheckColumnRef(ColumnRefExpression &reference) { + RequireBuiltin("struct_pack", CatalogType::SCALAR_FUNCTION_ENTRY, "the column reference " + Quote(reference.ToString())); + if (reference.ColumnNames().size() == 1) { + static const unordered_map value_functions = { + {"current_catalog", "current_catalog"}, {"current_date", "current_date"}, + {"current_schema", "current_schema"}, {"current_role", "current_role"}, + {"current_time", "get_current_time"}, {"current_timestamp", "get_current_timestamp"}, + {"current_user", "current_user"}, {"localtime", "current_localtime"}, + {"localtimestamp", "current_localtimestamp"}, {"session_user", "session_user"}, {"user", "user"} + }; + auto value_function = value_functions.find(StringUtil::Lower(reference.GetColumnName().GetIdentifierName())); + if (value_function != value_functions.end()) { + RequireBuiltin(value_function->second, CatalogType::SCALAR_FUNCTION_ENTRY, + "the SQL value function " + Quote(value_function->first)); + } + return; + } + // "a.b" is a relation-qualified column or a field of column a. Values of + // verified inputs are never JSON, VARIANT, GEOMETRY or UNION (types are + // checked at every source), so extraction binds struct_extract or + // map_extract_value. + RequireStructExtraction(reference.ToString()); + } + + void RequireStructExtraction(const string &display) { + RequireBuiltin("struct_extract", CatalogType::SCALAR_FUNCTION_ENTRY, "the field access " + Quote(display)); + RequireBuiltin("map_extract_value", CatalogType::SCALAR_FUNCTION_ENTRY, "the field access " + Quote(display)); + } + void CheckExpression(ParsedExpression &expression) { switch (expression.GetExpressionClass()) { case ExpressionClass::FUNCTION: CheckFunction(expression.Cast(), false); return; + case ExpressionClass::WINDOW: + CheckWindow(expression.Cast()); + return; + case ExpressionClass::CAST: { + auto &cast = expression.Cast(); + // The generic iterator does not visit the target type expression. + CheckTypeExpression(cast.TargetType()); + CheckExpression(*cast.ChildMutable()); + return; + } case ExpressionClass::SUBQUERY: CheckNode(*expression.Cast().SubqueryMutable()->node); break; case ExpressionClass::COLUMN_REF: - // A dotted reference can be a relation-qualified column or an implicit - // struct/JSON extraction that binds a scalar through the search path. - // Without binding we cannot distinguish them safely. - if (expression.Cast().ColumnNames().size() != 1) Reject(); + CheckColumnRef(expression.Cast()); break; case ExpressionClass::OPERATOR: switch (expression.GetExpressionType()) { @@ -102,10 +274,23 @@ class Policy { case ExpressionType::COMPARE_IN: case ExpressionType::COMPARE_NOT_IN: break; + // These operators bind the named scalar function through the search + // path. Verified inputs are never JSON or VARIANT (see CheckColumnRef). + case ExpressionType::ARRAY_EXTRACT: + RequireBuiltin("array_extract", CatalogType::SCALAR_FUNCTION_ENTRY, "the subscript " + Quote(expression.ToString())); + RequireBuiltin("map_extract_value", CatalogType::SCALAR_FUNCTION_ENTRY, "the subscript " + Quote(expression.ToString())); + break; + case ExpressionType::ARRAY_SLICE: + RequireBuiltin("array_slice", CatalogType::SCALAR_FUNCTION_ENTRY, "the slice " + Quote(expression.ToString())); + break; + case ExpressionType::ARRAY_CONSTRUCTOR: + RequireBuiltin("list_value", CatalogType::SCALAR_FUNCTION_ENTRY, "the list " + Quote(expression.ToString())); + break; + case ExpressionType::STRUCT_EXTRACT: + RequireStructExtraction(expression.ToString()); + break; default: - // Array/struct extraction and other operators can dispatch to hidden - // scalar calls. Require an explicitly checked function call instead. - Reject(); + Refuse("the operator in " + Quote(expression.ToString())); } break; case ExpressionClass::CASE: @@ -116,18 +301,140 @@ class Policy { case ExpressionClass::POSITIONAL_REFERENCE: case ExpressionClass::BETWEEN: break; + case ExpressionClass::LAMBDA: + case ExpressionClass::LAMBDA_REF: + Refuse("lambda expressions"); + case ExpressionClass::COLLATE: + Refuse("COLLATE expressions"); + case ExpressionClass::PARAMETER: + Refuse("prepared statement parameters"); default: - // Casts, collations, lambdas and window functions can resolve additional - // user-defined code at bind time. No permissive default for new AST kinds. - Reject(); + // No permissive default for new AST kinds. + Refuse("the expression " + Quote(expression.ToString())); } ParsedExpressionIterator::EnumerateChildren(expression, [this](ParsedExpression &child) { CheckExpression(child); }); } - static void CheckType(const LogicalType &type) { + //! Types bind from the catalog in DuckDB 2: a user type can shadow a built-in + //! name and type parameters are evaluated during binding. + void CheckTypeExpression(const TypeExpression &type) { + enum class Parameters { NONE, LENGTH, DECIMAL, LIST, ARRAY, STRUCT, MAP }; + static const unordered_map types = { + {"boolean", Parameters::NONE}, {"bool", Parameters::NONE}, {"logical", Parameters::NONE}, + {"tinyint", Parameters::NONE}, {"int1", Parameters::NONE}, {"smallint", Parameters::NONE}, + {"int2", Parameters::NONE}, {"int16", Parameters::NONE}, {"short", Parameters::NONE}, + {"integer", Parameters::NONE}, {"int", Parameters::NONE}, {"int4", Parameters::NONE}, + {"int32", Parameters::NONE}, {"signed", Parameters::NONE}, {"bigint", Parameters::NONE}, + {"int8", Parameters::NONE}, {"int64", Parameters::NONE}, {"long", Parameters::NONE}, + {"hugeint", Parameters::NONE}, {"int128", Parameters::NONE}, {"utinyint", Parameters::NONE}, + {"uint8", Parameters::NONE}, {"usmallint", Parameters::NONE}, {"uint16", Parameters::NONE}, + {"uinteger", Parameters::NONE}, {"uint32", Parameters::NONE}, {"ubigint", Parameters::NONE}, + {"uint64", Parameters::NONE}, {"uhugeint", Parameters::NONE}, {"uint128", Parameters::NONE}, + {"float", Parameters::NONE}, {"float4", Parameters::NONE}, {"real", Parameters::NONE}, + {"double", Parameters::NONE}, {"float8", Parameters::NONE}, {"date", Parameters::NONE}, + {"time", Parameters::NONE}, {"timestamp", Parameters::NONE}, {"datetime", Parameters::NONE}, + {"timestamp_s", Parameters::NONE}, {"timestamp_ms", Parameters::NONE}, {"timestamp_us", Parameters::NONE}, + {"timestamp_ns", Parameters::NONE}, {"timestamptz", Parameters::NONE}, + {"timestamp with time zone", Parameters::NONE}, {"timetz", Parameters::NONE}, + {"time with time zone", Parameters::NONE}, {"interval", Parameters::NONE}, {"blob", Parameters::NONE}, + {"bytea", Parameters::NONE}, {"binary", Parameters::NONE}, {"varbinary", Parameters::NONE}, + {"uuid", Parameters::NONE}, {"guid", Parameters::NONE}, + {"varchar", Parameters::LENGTH}, {"text", Parameters::LENGTH}, {"string", Parameters::LENGTH}, + {"char", Parameters::LENGTH}, {"bpchar", Parameters::LENGTH}, {"nvarchar", Parameters::LENGTH}, + {"decimal", Parameters::DECIMAL}, {"numeric", Parameters::DECIMAL}, {"dec", Parameters::DECIMAL}, + {"list", Parameters::LIST}, {"array", Parameters::ARRAY}, {"struct", Parameters::STRUCT}, + {"row", Parameters::STRUCT}, {"map", Parameters::MAP} + }; + const auto &type_name = type.GetTypeName(); + const auto display = "casts to the type " + Quote(type.ToString()); + if (type.GetQualifiedName().Path().size() != 1) Refuse(display); + // Filter by name before any lookup: DuckDB autoloads extensions for some type names. + auto kind = types.find(StringUtil::Lower(type_name.GetIdentifierName())); + if (kind == types.end()) Refuse(display); + // Type parameters are folded during binding: only literals and nested + // built-in types, in the shape each type takes. ARRAY is (type[, size]). + auto &children = type.GetChildren(); + for (idx_t i = 0; i < children.size(); i++) { + bool nested_type = false; + switch (kind->second) { + case Parameters::NONE: Refuse(display); + case Parameters::LENGTH: + case Parameters::DECIMAL: nested_type = false; break; + case Parameters::LIST: + case Parameters::STRUCT: + case Parameters::MAP: nested_type = true; break; + case Parameters::ARRAY: nested_type = i == 0; break; + } + auto expected = nested_type ? ExpressionClass::TYPE : ExpressionClass::CONSTANT; + if (children[i]->GetExpressionClass() != expected) Refuse(display); + // Integer literals only: a VARCHAR string parameter selects a collation. + if (!nested_type && children[i]->Cast().GetLiteral().kind != LiteralKind::INTEGER) { + Refuse(display); + } + if (nested_type) CheckTypeExpression(children[i]->Cast()); + } + bool shape = false; + switch (kind->second) { + case Parameters::NONE: shape = children.empty(); break; + case Parameters::LENGTH: shape = children.size() <= 1; break; + case Parameters::DECIMAL: shape = children.size() <= 2; break; + case Parameters::LIST: shape = children.size() == 1; break; + case Parameters::ARRAY: shape = children.size() == 1 || children.size() == 2; break; + case Parameters::STRUCT: shape = !children.empty(); break; + case Parameters::MAP: shape = children.size() == 2; break; + } + if (!shape) Refuse(display); + // Built-in types are generated, internal entries in each DuckDB catalog's + // main schema; a user type that shadows the name is never internal. + auto entry = ResolveType(type); + if (!entry || !entry->internal || entry->Cast().user_type.HasAlias()) { + Refuse("casts to the user-defined type " + Quote(type.ToString())); + } + } + + //! Mirrors ExpressionBinder::BindExpression(TypeExpression &) for an + //! unqualified type name. + optional_ptr ResolveType(const TypeExpression &type) { + auto valid = [](optional_ptr entry) { + return entry && entry->type == CatalogType::TYPE_ENTRY && + entry->Cast().user_type.id() != LogicalTypeId::INVALID; + }; + try { + const auto &type_name = type.GetTypeName(); + EntryLookupInfo type_lookup(CatalogType::TYPE_ENTRY, QualifiedName(type_name)); + auto bound_name = Binder::BindTableName(Retriever(), type.GetQualifiedName()); + auto &type_catalog = bound_name.Catalog(); + if (type_catalog.empty() && !DatabaseManager::Get(context).HasAttachedDatabase()) { + auto entry = Retriever().GetEntry(EntryLookupInfo(type_lookup, bound_name.WithCatalog(Identifier::SystemCatalog())), + OnEntryNotFound::RETURN_NULL); + return valid(entry) ? entry : nullptr; + } + auto entry = Retriever().GetEntry(EntryLookupInfo(type_lookup, bound_name), OnEntryNotFound::RETURN_NULL); + if (!valid(entry)) { + entry = Retriever().GetEntry( + EntryLookupInfo(type_lookup, QualifiedName(type_catalog, Identifier::InvalidSchema(), type_name)), + OnEntryNotFound::RETURN_NULL); + } + if (!valid(entry)) { + entry = Retriever().GetEntry( + EntryLookupInfo(type_lookup, QualifiedName(Identifier::InvalidCatalog(), Identifier::InvalidSchema(), type_name)), + OnEntryNotFound::RETURN_NULL); + } + if (!valid(entry)) { + entry = Retriever().GetEntry( + EntryLookupInfo(type_lookup, QualifiedName(Identifier::SystemCatalog(), Identifier::DefaultSchema(), type_name)), + OnEntryNotFound::RETURN_NULL); + } + return valid(entry) ? entry : nullptr; + } catch (const Exception &) { + return nullptr; + } + } + + static void CheckType(const LogicalType &type, const std::function &refuse) { // Aliased/custom types can introduce user-defined implicit casts, even when // the SQL itself contains no explicit function or CAST expression. - if (type.HasAlias()) Reject(); + if (type.HasAlias()) refuse(); switch (type.id()) { case LogicalTypeId::BOOLEAN: case LogicalTypeId::TINYINT: @@ -156,45 +463,131 @@ class Policy { case LogicalTypeId::UUID: return; case LogicalTypeId::VARCHAR: - if (!StringType::GetCollation(type).empty()) Reject(); + if (!StringType::GetCollation(type).empty()) refuse(); return; case LogicalTypeId::LIST: - CheckType(ListType::GetChildType(type)); + CheckType(ListType::GetChildType(type), refuse); return; case LogicalTypeId::ARRAY: - CheckType(ArrayType::GetChildType(type)); + CheckType(ArrayType::GetChildType(type), refuse); return; case LogicalTypeId::STRUCT: - for (const auto &child : StructType::GetChildTypes(type)) CheckType(child.second); + for (const auto &child : StructType::GetChildTypes(type)) CheckType(child.second, refuse); return; case LogicalTypeId::MAP: - CheckType(MapType::KeyType(type)); - CheckType(MapType::ValueType(type)); + CheckType(MapType::KeyType(type), refuse); + CheckType(MapType::ValueType(type), refuse); return; default: - Reject(); + refuse(); + } + } + + bool InCTEScope(const Identifier &name) const { + for (idx_t i = cte_floor; i < cte_names.size(); i++) { + if (cte_names[i] == name) return true; + } + return false; + } + + //! Mirrors Binder::Bind(BaseTableRef &) for the catalog lookup. + optional_ptr LookupRelation(BaseTableRef &table) { + try { + EntryLookupInfo table_lookup(CatalogType::TABLE_ENTRY, QualifiedName(table.Table())); + auto bound_name = Binder::BindTableName(Retriever(), table.GetQualifiedName()); + return Retriever().GetEntry(EntryLookupInfo(table_lookup, bound_name), OnEntryNotFound::RETURN_NULL); + } catch (const Exception &) { + Refuse(Quote(table.GetQualifiedName().ToString()) + ", which could not be resolved"); } } + void CheckRelation(CatalogEntry &entry, const string &display) { + auto catalog_type = entry.ParentCatalog().GetCatalogType(); + if (catalog_type != "duckdb") { + Refuse(display + " from an attached " + Quote(catalog_type) + " database (only native DuckDB tables are previewed)"); + } + if (entry.type == CatalogType::TABLE_ENTRY) { + auto &table = entry.Cast(); + // Generated columns are expressions DuckDB binds from the table definition. + if (table.HasGeneratedColumns()) Refuse("the table " + display + ", which has generated columns"); + for (const auto &column : table.GetColumns().Logical()) { + CheckType(column.Type(), [&]() { + Refuse("the column " + Quote(column.Name().GetIdentifierName()) + " of " + display + " with type " + + column.Type().ToString()); + }); + } + return; + } + if (entry.type == CatalogType::VIEW_ENTRY) { + CheckView(entry.Cast(), display); + return; + } + Refuse(display + ", which is not a table or view"); + } + + //! Applies this policy to the view's definition, binding names the way + //! DuckDB binds view bodies: the view's own catalog and schema first, and no + //! CTEs from the referencing query. + void CheckView(ViewCatalogEntry &view, const string &display) { + if (view_path.size() >= MAX_VIEW_DEPTH) { + Refuse("the view " + display + ", which nests views more than " + to_string(MAX_VIEW_DEPTH) + " levels deep"); + } + if (++view_expansions > MAX_VIEW_EXPANSIONS) { + Refuse("statements that expand more than " + to_string(MAX_VIEW_EXPANSIONS) + " views"); + } + auto query = unique_ptr_cast(view.GetQuery().Copy()); + // Binder::GetSearchPath(view catalog, view schema, true), as Bind(BaseTableRef &) applies it. + vector search_path; + search_path.emplace_back(view.ParentCatalog().GetName(), view.ParentSchema().name); + search_path.emplace_back(view.ParentCatalog().GetName(), Identifier::InvalidSchema(), true); + auto retriever = make_uniq(context); + retriever->Inherit(Retriever()); + retriever->SetSearchPath(std::move(search_path)); + retrievers.push_back(std::move(retriever)); + const auto previous_floor = cte_floor; + cte_floor = cte_names.size(); + view_path.push_back(display); + CheckNode(*query->node); + view_path.pop_back(); + cte_floor = previous_floor; + retrievers.pop_back(); + } + + void CheckBaseTable(BaseTableRef &table) { + // DuckDB evaluates AT expressions during binding, even when the catalog + // subsequently rejects time travel. + if (table.at_clause) Refuse("AT (time travel) clauses"); + const auto &name = table.GetQualifiedName(); + const auto display = Quote(name.ToString()); + if (name.Path().size() == 1 && InCTEScope(table.Table())) { + // DuckDB binds the innermost visible CTE first. If a relation with the + // same name exists, require it to be acceptable as well, so the + // decision never depends on that precedence. + if (auto entry = LookupRelation(table)) CheckRelation(*entry, display); + return; + } + // Outside views, the statement is bound beneath the caller's query, whose + // CTEs would take precedence over the catalog. + if (view_path.empty() && outer_binder && outer_binder->GetCTEBinding(BindingAlias(name.Schema(), table.Table()))) { + Refuse(display + ", which names a CTE of the surrounding query"); + } + // Missing relations are refused before replacement scans (including files + // and URLs) can run. + auto entry = LookupRelation(table); + if (!entry) { + Refuse(display + ", which is not a table or view in an attached DuckDB database (file, URL and replacement " + "scans need Run)"); + } + CheckRelation(*entry, display); + } + void CheckTable(TableRef &reference) { switch (reference.type) { case TableReferenceType::EMPTY_FROM: return; - case TableReferenceType::BASE_TABLE: { - auto &table = reference.Cast(); - // DuckDB evaluates AT expressions during binding, even when the catalog - // subsequently rejects time travel. Native catalogs admitted here do not - // support it, so reject before any hidden expression can execute. - if (table.at_clause) Reject(); - // Missing tables are rejected before replacement scans (including files) - // can run. Views and generated columns can conceal calls: fail closed. - auto entry = Catalog::GetEntry(context, EntryLookupInfo(CatalogType::TABLE_ENTRY, table.GetQualifiedName()), - OnEntryNotFound::RETURN_NULL); - if (!entry || entry->type != CatalogType::TABLE_ENTRY || entry->ParentCatalog().GetCatalogType() != "duckdb" || - entry->Cast().HasGeneratedColumns()) Reject(); - for (const auto &column : entry->Cast().GetColumns().Logical()) CheckType(column.Type()); + case TableReferenceType::BASE_TABLE: + CheckBaseTable(reference.Cast()); return; - } case TableReferenceType::SUBQUERY: CheckNode(*reference.Cast().subquery->node); return; @@ -214,20 +607,36 @@ class Policy { return; case TableReferenceType::TABLE_FUNCTION: { auto &table = reference.Cast(); - if (!table.function || table.function->GetExpressionClass() != ExpressionClass::FUNCTION || table.subquery) Reject(); + if (!table.function || table.function->GetExpressionClass() != ExpressionClass::FUNCTION || table.subquery) { + Refuse("this table function call"); + } CheckFunction(table.function->Cast(), true); return; } + case TableReferenceType::PIVOT: + // PIVOT can run a query during binding to find its columns. + Refuse("PIVOT and UNPIVOT"); + case TableReferenceType::SHOW_REF: + Refuse("DESCRIBE, SHOW and SUMMARIZE"); default: - Reject(); + Refuse("this kind of FROM clause"); } } void CheckNode(QueryNode &node) { - // CTEs can contain modifying query nodes in DuckDB 2. Keep them explicit - // until their lexical scope and every node kind have a dedicated policy. - if (!node.cte_map.map.empty()) Reject(); - if (node.type == QueryNodeType::SELECT_NODE) { + // CTEs are visible to later CTEs of the same WITH clause, to the query and + // to its subqueries. A CTE never sees itself or later siblings here, so this + // scope is never wider than DuckDB's. + const auto scope_size = cte_names.size(); + for (auto &cte : node.cte_map.map) { + auto &info = *cte.second; + if (!info.key_targets.empty() || !info.payload_aggregates.empty()) Refuse("recursive CTEs"); + if (!info.query_node) Refuse("this WITH clause"); + CheckNode(*info.query_node); + cte_names.push_back(cte.first); + } + switch (node.type) { + case QueryNodeType::SELECT_NODE: { auto &select = node.Cast(); for (auto &expression : select.select_list) CheckExpression(*expression); for (auto &expression : select.groups.group_expressions) CheckExpression(*expression); @@ -235,14 +644,38 @@ class Policy { if (select.having) CheckExpression(*select.having); if (select.qualify) CheckExpression(*select.qualify); if (select.from_table) CheckTable(*select.from_table); - } else if (node.type == QueryNodeType::SET_OPERATION_NODE) { + break; + } + case QueryNodeType::SET_OPERATION_NODE: for (auto &child : node.Cast().children) CheckNode(*child); - } else Reject(); + break; + case QueryNodeType::RECURSIVE_CTE_NODE: + // A recursive CTE can run unbounded before the row limit applies. + Refuse("recursive CTEs"); + case QueryNodeType::INSERT_QUERY_NODE: + case QueryNodeType::UPDATE_QUERY_NODE: + case QueryNodeType::DELETE_QUERY_NODE: + case QueryNodeType::MERGE_QUERY_NODE: + Refuse("INSERT, UPDATE, DELETE or MERGE inside a query"); + case QueryNodeType::COPY_QUERY_NODE: + Refuse("COPY inside a query"); + default: + Refuse("this query form"); + } ParsedExpressionIterator::EnumerateQueryNodeModifiers(node, [this](unique_ptr &expression) { CheckExpression(*expression); }); + cte_names.resize(scope_size); } ClientContext &context; + optional_ptr outer_binder; + //! The innermost entry resolves names: the caller's search path, then one per view being verified. + vector> retrievers; + //! CTE names visible at the current point; entries below cte_floor belong to a query outside the current view. + vector cte_names; + idx_t cte_floor = 0; + vector view_path; + idx_t view_expansions = 0; }; static void IsReadOnly(DataChunk &input, ExpressionState &state, Vector &result) { @@ -260,7 +693,7 @@ static unique_ptr ReadQuery(ClientContext &context, TableFunctionBindI if (input.inputs[0].IsNull()) throw InvalidInputException("SQLNB_READ_ONLY: SQL cannot be null"); // Validate and return the same parsed tree, in the outer query's binding // transaction. A separate prior classification cannot authorize this path. - return make_uniq(Policy(context).Parse(input.inputs[0].GetValue())); + return make_uniq(Policy(context, input.binder).Parse(input.inputs[0].GetValue())); } } // namespace sqlnb_read_policy From a9bf20e533c1942c01aac5ab6cc2aa78521558be Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Sat, 14 Nov 2026 16:56:17 -0800 Subject: [PATCH 2/5] Let Instant SQL preview LIKE, GLOB, bitwise and list operators These operators parse to function names that bind through the search path, so they go through the same built-in check as named functions and a macro that shadows one is refused. --- docs/instant-sql.md | 7 +++++++ scripts/read-policy-smoke.mjs | 27 +++++++++++++++++++++++++++ src/include/sqlnb_read_policy.hpp | 6 ++++++ 3 files changed, 40 insertions(+) diff --git a/docs/instant-sql.md b/docs/instant-sql.md index f5b4a47..3f1550a 100644 --- a/docs/instant-sql.md +++ b/docs/instant-sql.md @@ -54,6 +54,13 @@ remains available through explicit Run. - Calls to an allowlist of side-effect-free scalar and aggregate built-ins (arithmetic, rounding, string, pattern-matching, list, struct and date/time helpers, and the common aggregates). +- The operators that parse to built-in functions: arithmetic and unary minus, + `LIKE`, `NOT LIKE`, `ILIKE`, `NOT ILIKE` (with or without `ESCAPE`), `GLOB`, + `SIMILAR TO`, regular-expression `~` and `!~`, `^@` (starts with), the bitwise + `&`, `|`, `<<`, `>>`, `~` and `xor`, `^` (power), `||`, and the list operators + `@>`, `<@` and `&&`. DuckDB binds each of these by its function name (`LIKE` is + `~~`, `GLOB` is `~~~`), so a macro of that name on the search path would replace + the operator; such a macro is refused like any other. - Window functions: `row_number`, `rank`, `dense_rank`, `percent_rank`, `cume_dist`, `ntile`, `lag`, `lead`, `first_value`, `last_value`, `nth_value` and the allowlisted aggregates, with partitions, ordering, frames, `FILTER`, diff --git a/scripts/read-policy-smoke.mjs b/scripts/read-policy-smoke.mjs index 72e7a4b..34a42fa 100644 --- a/scripts/read-policy-smoke.mjs +++ b/scripts/read-policy-smoke.mjs @@ -115,6 +115,22 @@ export async function readPolicySmoke({ runSQL, expectSQLError, sqlLiteral, firs ], ); + // Pattern matching, bitwise and list operators bind built-in functions. + await accepts( + `select id, label like 'item%' as like_match, label not like '%2' as not_like, label ilike 'ITEM_' as ilike_match, + label not ilike 'x%' as not_ilike, label glob 'item[13]' as glob_match, label similar to 'item[0-9]' as similar, + label ~ '3$' as regex, label ^@ 'it' as prefix, label like 'item!%' escape '!' as escaped, + (id & 1) as bit_and, (id | 8) as bit_or, (id << 2) as shifted_left, (id >> 1) as shifted_right, + xor(id, 1) as bit_xor, ~id as bit_not, -id as negated, (2 ^ id)::integer as raised, + [1, 2, id] @> [id] as list_contains, [id] <@ [1, 2, 3] as list_contained, [id] && [2, 4] as list_overlap + from policy.items where label like 'item%' and id = 3`, + [{ + id: 3, like_match: true, not_like: true, ilike_match: true, not_ilike: true, glob_match: true, similar: true, + regex: true, prefix: true, escaped: false, bit_and: 1, bit_or: 11, shifted_left: 12, shifted_right: 1, + bit_xor: 2, bit_not: -4, negated: -3, raised: 8, list_contains: true, list_contained: true, list_overlap: false, + }], + ); + // 2. Views, verified through their definitions with DuckDB's view search path. await accepts('select * from policy.items_view_view order by id10', [{ id10: 20 }, { id10: 30 }, { id10: 40 }]); await accepts('select top from policy.cte_view', [{ top: 4 }]); @@ -166,6 +182,17 @@ export async function readPolicySmoke({ runSQL, expectSQLError, sqlLiteral, firs await runSQL('set search_path = \'shadow,main\'; select true as ok'); await refuses('select lower(\'A\')', 'the macro \'lower\''); await runSQL('reset search_path; drop schema shadow cascade; select true as ok'); + // An operator is a function name: a macro on the search path replaces it. + await runSQL(`create schema shadow; + create macro shadow."~~"(a, b) as nextval('policy.counter') > 0; + create macro shadow."&"(a, b) as nextval('policy.counter'); + create macro shadow."@>"(a, b) as nextval('policy.counter') > 0; + select true as ok`); + await runSQL('set search_path = \'shadow,main\'; select true as ok'); + await refuses('select label from policy.items where label like \'item%\'', 'the macro \'~~\''); + await refuses('select 1 & 2', 'the macro \'&\''); + await refuses('select [1] @> [1]', 'the macro \'@>\''); + await runSQL('reset search_path; drop schema shadow cascade; select true as ok'); await refuses('select \'a\'::varchar(\'nocase\')', 'casts to the type'); await refuses('select * from policy.sequence_view', 'the function \'nextval\'.*inside view'); await refuses(`select * from read_csv(${sqlLiteral(remoteCSV)})`, 'the table function \'read_csv\''); diff --git a/src/include/sqlnb_read_policy.hpp b/src/include/sqlnb_read_policy.hpp index 3f9be68..9e7f53f 100644 --- a/src/include/sqlnb_read_policy.hpp +++ b/src/include/sqlnb_read_policy.hpp @@ -157,6 +157,12 @@ class Policy { "lower", "upper", "length", "char_length", "octet_length", "trim", "ltrim", "rtrim", "substring", "substr", "left", "right", "replace", "repeat", "concat", "concat_ws", "starts_with", "ends_with", "contains", "like_escape", "not_like_escape", "ilike_escape", "not_ilike_escape", "regexp_matches", "regexp_full_match", + // Operators parse to these function names and bind like any function, so + // a macro of the same name on the search path would replace them: + // LIKE ~~, NOT LIKE !~~, ILIKE ~~*, NOT ILIKE !~~*, GLOB ~~~, ^@ starts_with, + // bitwise & | << >> ~ xor, ^ power, and list containment @> <@ &&. + // SIMILAR TO and ~ (regex) parse to regexp_full_match and regexp_matches. + "~~", "!~~", "~~*", "!~~*", "~~~", "^@", "&", "|", "<<", ">>", "~", "xor", "^", "@>", "<@", "&&", "count", "count_star", "sum", "avg", "min", "max", "median", "stddev", "stddev_samp", "stddev_pop", "variance", "var_samp", "var_pop", "bool_and", "bool_or", "first", "last", "any_value", "string_agg", "list", "array_agg", "list_value", "struct_pack", "struct_extract", "array_extract", "list_extract", From 77a06ac6e94d19f438c6de3848d919d712c19302 Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Sat, 14 Nov 2026 17:00:09 -0800 Subject: [PATCH 3/5] Refuse Instant SQL previews of DuckDB files attached from remote locations A DuckDB database attached over HTTP or S3 still reports the catalog type duckdb, so the policy now also requires the database to be in memory or at a local path. Reading a remote database is network I/O, which a preview would repeat every time typing pauses. --- docs/instant-sql.md | 12 +++++++-- scripts/read-policy-smoke.mjs | 45 ++++++++++++++++++++++++++----- src/include/sqlnb_read_policy.hpp | 21 +++++++++++++++ 3 files changed, 69 insertions(+), 9 deletions(-) diff --git a/docs/instant-sql.md b/docs/instant-sql.md index 3f1550a..aceeafb 100644 --- a/docs/instant-sql.md +++ b/docs/instant-sql.md @@ -36,8 +36,9 @@ remains available through explicit Run. visible to later CTEs of the same clause, to the query and to its subqueries, never inside its own body. When a CTE shares its name with a table or view, that relation must be acceptable too. -- Tables in attached native DuckDB databases (including attached database files - and temporary tables), unqualified or qualified as `schema.table` or +- Tables in attached native DuckDB databases that are in memory or stored at a + local path (including attached database files and temporary tables), + unqualified or qualified as `schema.table` or `database.schema.table`, without generated columns, whose column types are `BOOLEAN`, signed and unsigned integers, `FLOAT`, `DOUBLE`, `DECIMAL`, `DATE`, `TIME`, `TIMETZ`, `TIMESTAMP` (all precisions and `WITH TIME ZONE`), `INTERVAL`, @@ -85,6 +86,13 @@ built-in name is refused. - Names that are not tables or views in an attached native DuckDB database: files, URLs and other replacement scans, and tables or views in attached databases of other types. +- Tables and views in a native DuckDB database file attached from a remote + location (`http://`, `https://`, `s3://`, `hf://` and the other remote + prefixes DuckDB recognises, or any path with a non-`file:` scheme). Such a + database still reports the catalog type `duckdb`, but every read is network I/O + and a preview would repeat it whenever typing pauses. The check uses the + database's storage path (`Catalog::GetDBPath`), not its name, and applies inside + views too. - Any other table function, including file, network and catalog scans (`read_csv`, `read_parquet`, `glob`, `query`, `duckdb_*`), table macros and `PIVOT`/`UNPIVOT`, `DESCRIBE`, `SHOW` and `SUMMARIZE` in `FROM`. diff --git a/scripts/read-policy-smoke.mjs b/scripts/read-policy-smoke.mjs index 34a42fa..b5158ac 100644 --- a/scripts/read-policy-smoke.mjs +++ b/scripts/read-policy-smoke.mjs @@ -5,22 +5,41 @@ // shapes must name their reason and must leave no trace: sequences, tables, // files and the HTTP server below observe every side effect they could cause. import assert from 'node:assert/strict'; -import { existsSync, mkdtempSync } from 'node:fs'; +import { existsSync, mkdtempSync, readFileSync } from 'node:fs'; import { createServer } from 'node:http'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; export async function readPolicySmoke({ runSQL, expectSQLError, sqlLiteral, firstValue }) { + const scratch = mkdtempSync(join(tmpdir(), 'sqlnb-read-policy.')); + const copyTarget = join(scratch, 'copy.csv'); + const servedDatabase = join(scratch, 'served.duckdb'); let remoteRequests = 0; - const server = createServer((_request, response) => { + const server = createServer((request, response) => { remoteRequests += 1; - response.writeHead(200, { 'content-type': 'text/csv' }); - response.end('value\n1\n'); + if (!request.url.endsWith('.duckdb')) { + response.writeHead(200, { 'content-type': 'text/csv' }); + response.end('value\n1\n'); + return; + } + // A DuckDB database file, served with the HEAD and range support httpfs uses. + const body = readFileSync(servedDatabase); + const range = /^bytes=(\d+)-(\d*)$/.exec(request.headers.range ?? ''); + if (request.method === 'HEAD' || !range) { + response.writeHead(200, { 'content-length': body.length, 'accept-ranges': 'bytes' }); + response.end(request.method === 'HEAD' ? undefined : body); + return; + } + const start = Number(range[1]); + const end = range[2] ? Math.min(Number(range[2]), body.length - 1) : body.length - 1; + response.writeHead(206, { + 'content-range': `bytes ${start}-${end}/${body.length}`, 'content-length': end - start + 1, 'accept-ranges': 'bytes', + }); + response.end(body.subarray(start, end + 1)); }); await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); const remoteCSV = `http://127.0.0.1:${server.address().port}/remote.csv`; - const scratch = mkdtempSync(join(tmpdir(), 'sqlnb-read-policy.')); - const copyTarget = join(scratch, 'copy.csv'); + const remoteDatabase = `http://127.0.0.1:${server.address().port}/served.duckdb`; // The CLI harness reads one output line per statement, so the rows travel // as one list value. Aggregation does not promise order: compare as a set. @@ -68,6 +87,13 @@ export async function readPolicySmoke({ runSQL, expectSQLError, sqlLiteral, firs create table main.trap_name as select 1::integer as value; create view policy_db.inner_schema.trap_name as select * from read_csv(${sqlLiteral(remoteCSV)}); create view policy_db.inner_schema.trap_view as select * from trap_name; + attach ${sqlLiteral(servedDatabase)} as served_source; + create table served_source.facts as select 7::integer as fact; + detach served_source; + -- The same file attached from disk and over HTTP: both are native DuckDB catalogs. + attach ${sqlLiteral(servedDatabase)} as local_file_db (read_only); + attach ${sqlLiteral(remoteDatabase)} as remote_file_db (read_only); + create view policy.remote_file_view as select fact from remote_file_db.facts; select nextval('policy.counter') as initialized`); for (let level = 0; level <= 8; level += 1) { const source = level === 0 ? 'policy.items' : `policy.depth_${level - 1}`; @@ -151,6 +177,7 @@ export async function readPolicySmoke({ runSQL, expectSQLError, sqlLiteral, firs 'select policy_db.inner_schema.remote_name.answer from policy_db.inner_schema.remote_name', [{ answer: 42 }], ); + await accepts('select fact from local_file_db.facts', [{ fact: 7 }]); // 4. Refusals, each with its reason and without side effects. await refuses('insert into policy.items values (9, 1, \'x\', null, null)', 'INSERT statements'); @@ -201,6 +228,9 @@ export async function readPolicySmoke({ runSQL, expectSQLError, sqlLiteral, firs await refuses('select * from policy.remote_view_view', 'the table function \'read_csv\'.*inside view .* > '); await refuses('select * from policy.glob_view', 'the table function \'glob\'.*inside view'); await refuses('select count(*) from policy.depth_8', 'the view .*more than 8 levels deep'); + // A DuckDB file attached over HTTP is still a "duckdb" catalog; reading it is network I/O. + await refuses('select fact from remote_file_db.facts', '\'remote_file_db.facts\' from the database \'remote_file_db\', which is stored at a remote location'); + await refuses('select fact from policy.remote_file_view', '\'remote_file_db.facts\' .*which is stored at a remote location.*inside view'); await refuses( 'with recursive r(n) as (select 1 union all select n + 1 from r) select * from r', 'recursive CTEs', @@ -221,6 +251,7 @@ export async function readPolicySmoke({ runSQL, expectSQLError, sqlLiteral, firs assert.equal(remoteRequests, requestsAfterSetup, 'refused statements must not reach the network'); assert.equal(existsSync(copyTarget), false, 'refused statements must not write files'); - await runSQL('detach policy_db; drop schema policy cascade; drop view main.remote_name; drop table main.trap_name; select true as cleaned'); + await runSQL(`detach policy_db; detach local_file_db; detach remote_file_db; drop schema policy cascade; + drop view main.remote_name; drop table main.trap_name; select true as cleaned`); await new Promise((resolve) => server.close(resolve)); } diff --git a/src/include/sqlnb_read_policy.hpp b/src/include/sqlnb_read_policy.hpp index 9e7f53f..03d9b37 100644 --- a/src/include/sqlnb_read_policy.hpp +++ b/src/include/sqlnb_read_policy.hpp @@ -20,6 +20,8 @@ #include "duckdb/catalog/catalog_search_path.hpp" #include "duckdb/catalog/entry_lookup_info.hpp" #include "duckdb/common/enums/statement_type.hpp" +#include "duckdb/common/file_system.hpp" +#include "duckdb/common/path.hpp" #include "duckdb/function/scalar_function.hpp" #include "duckdb/function/table_function.hpp" #include "duckdb/main/database_manager.hpp" @@ -512,6 +514,7 @@ class Policy { if (catalog_type != "duckdb") { Refuse(display + " from an attached " + Quote(catalog_type) + " database (only native DuckDB tables are previewed)"); } + CheckLocalStorage(entry.ParentCatalog(), display); if (entry.type == CatalogType::TABLE_ENTRY) { auto &table = entry.Cast(); // Generated columns are expressions DuckDB binds from the table definition. @@ -531,6 +534,24 @@ class Policy { Refuse(display + ", which is not a table or view"); } + //! A native DuckDB database file can be attached from a URL (http, s3, hf, ...) + //! and still reports the catalog type "duckdb"; reading it issues network + //! requests. Only in-memory databases and files on a local path are previewed. + void CheckLocalStorage(Catalog &catalog, const string &display) { + if (catalog.InMemory()) return; + const auto path = catalog.GetDBPath(); + bool local = false; + try { + local = !FileSystem::IsRemoteFile(path) && Path::FromString(path).IsLocal(); + } catch (const Exception &) { + local = false; + } + if (!local) { + Refuse(display + " from the database " + Quote(catalog.GetName().GetIdentifierName()) + + ", which is stored at a remote location (previews never read over the network)"); + } + } + //! Applies this policy to the view's definition, binding names the way //! DuckDB binds view bodies: the view's own catalog and schema first, and no //! CTEs from the referencing query. From 69f888e759a1013272b2badd4afa09bf7ab2201c Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Sat, 14 Nov 2026 17:06:22 -0800 Subject: [PATCH 4/5] Refuse the scalar repeat in Instant SQL and document preview cost bounds One scalar repeat call can build a value of any size within the notebook's preview time budget, before the result byte cap applies. The repeat table function stays accepted because its rows are bounded. The docs now describe how the host bounds a preview's rows, bytes and execution time. --- docs/instant-sql.md | 15 ++++++++++++++- scripts/read-policy-smoke.mjs | 3 +++ src/include/sqlnb_read_policy.hpp | 6 +++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/instant-sql.md b/docs/instant-sql.md index aceeafb..8d1eb9d 100644 --- a/docs/instant-sql.md +++ b/docs/instant-sql.md @@ -97,7 +97,9 @@ built-in name is refused. (`read_csv`, `read_parquet`, `glob`, `query`, `duckdb_*`), table macros and `PIVOT`/`UNPIVOT`, `DESCRIBE`, `SHOW` and `SUMMARIZE` in `FROM`. - Functions outside the allowlist, including volatile or state-changing ones such - as `nextval`, `setval` and `random`; user-defined functions; macros, whose bodies + as `nextval`, `setval` and `random`, and the scalar `repeat(string, n)`, which + can build a value of any size in one call (the `repeat` table function is + accepted: its rows are bounded); user-defined functions; macros, whose bodies are not verified; qualified and method-style calls such as `schema.f()` or `x.lower()`. - Casts to user-defined, extension (`JSON`, `GEOMETRY`) or other unlisted types @@ -110,6 +112,17 @@ built-in name is refused. - A name that the surrounding query defines as a CTE, and any statement while a loaded extension rewrites column names during binding. +## Cost + +The policy bounds what a preview can do, not how long it runs. The notebook +host bounds that: it fetches at most 200 rows or 1 MiB, and interrupts a preview +that has executed for 2 seconds (Quack `CANCEL_REQUEST` for that query's own id), +showing "Preview stopped after 2,000 ms. Use Run." An explicit Run never has +this budget. Aggregates and window functions over hundreds of millions of rows +finish in tens of milliseconds on current hardware; unbounded shapes such as +`count(*)` over `range(1e12)` stop at the budget. Aggregates that build one large +value (`string_agg` or `list` over a large input) are bounded only by that time. + ## Tests [`read-policy-smoke.mjs`](../scripts/read-policy-smoke.mjs) runs inside the diff --git a/scripts/read-policy-smoke.mjs b/scripts/read-policy-smoke.mjs index b5158ac..b6af877 100644 --- a/scripts/read-policy-smoke.mjs +++ b/scripts/read-policy-smoke.mjs @@ -243,6 +243,9 @@ export async function readPolicySmoke({ runSQL, expectSQLError, sqlLiteral, firs ); await refuses('select list_transform([1], x -> x + 1)', 'the function \'list_transform\''); await refuses('select random()', 'the function \'random\''); + // A single scalar call can build a value of any size; the table function is row-bounded. + await refuses('select repeat(\'x\', 1000000000)', 'the function \'repeat\''); + await accepts('select count(*)::integer as n from repeat(7, 3)', [{ n: 3 }]); await refuses('select * from policy.items at (version => 1)', 'AT \\(time travel\\) clauses'); await refuses('select * from (pivot policy.items on grp using sum(id))', 'statements that DuckDB expands into several statements'); diff --git a/src/include/sqlnb_read_policy.hpp b/src/include/sqlnb_read_policy.hpp index 03d9b37..fed3265 100644 --- a/src/include/sqlnb_read_policy.hpp +++ b/src/include/sqlnb_read_policy.hpp @@ -153,11 +153,15 @@ class Policy { } static const unordered_set &ScalarAllowlist() { + // The scalar repeat(string, n) is absent on purpose: one call builds a + // value of any size within the preview time budget, and the result byte + // cap applies only after the server has serialized it. The repeat table + // function stays: its rows are bounded by the row limit and the budget. static const unordered_set scalars = { "+", "-", "*", "/", "//", "%", "**", "||", "abs", "ceil", "ceiling", "floor", "round", "trunc", "sqrt", "pow", "power", "exp", "ln", "log", "log10", "sign", "greatest", "least", "lower", "upper", "length", "char_length", "octet_length", "trim", "ltrim", "rtrim", "substring", - "substr", "left", "right", "replace", "repeat", "concat", "concat_ws", "starts_with", "ends_with", "contains", + "substr", "left", "right", "replace", "concat", "concat_ws", "starts_with", "ends_with", "contains", "like_escape", "not_like_escape", "ilike_escape", "not_ilike_escape", "regexp_matches", "regexp_full_match", // Operators parse to these function names and bind like any function, so // a macro of the same name on the search path would replace them: From fe5482d4856e276b8c30518468788b773ea8478c Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Sat, 14 Nov 2026 17:20:13 -0800 Subject: [PATCH 5/5] Add an opt-in sampled tier to Instant SQL previews sqlnb_sample_plan names the one table a sampled preview would sample and the percentage. sqlnb_read_query takes that table and percentage, verifies the statement as before, and puts a seeded Bernoulli sample on the verified reference only while that table is still the statement's sole read of a table of at least 1,000,000 rows. --- docs/instant-sql.md | 53 +++++++++ scripts/read-policy-smoke.mjs | 71 ++++++++++++ src/include/sqlnb_read_policy.hpp | 178 +++++++++++++++++++++++++++++- 3 files changed, 301 insertions(+), 1 deletion(-) diff --git a/docs/instant-sql.md b/docs/instant-sql.md index 8d1eb9d..e2ce98b 100644 --- a/docs/instant-sql.md +++ b/docs/instant-sql.md @@ -123,6 +123,59 @@ finish in tens of milliseconds on current hardware; unbounded shapes such as `count(*)` over `range(1e12)` stop at the budget. Aggregates that build one large value (`string_agg` or `list` over a large input) are bounded only by that time. +## Sampled previews + +The notebook offers an opt-in setting, "Instant SQL previews: Sampled +(approximate)". With it, a preview that the exact path cannot finish quickly runs +on a sample of one table instead. Previews are exact by default. + +- `sqlnb_sample_plan(sql)` returns JSON naming the table a sampled preview would + sample and the percentage, for example + `{"catalog":"big","schema":"main","table":"f","rows":50000000,"percent":0.2,"method":"bernoulli"}`, + or `NULL` when the preview stays exact or the policy refuses the statement. Like + `sqlnb_is_read_only`, it is a hint. +- `sqlnb_read_query(sql, sample_catalog := …, sample_schema := …, sample_table := …, + sample_percent := …)` verifies the statement exactly as above, then puts a + Bernoulli sample with a fixed seed on the verified reference to that table and + binds the rewritten statement. It refuses the sample, with the reason, unless + that table is still the statement's sample candidate. Sampling never changes + what the policy accepts. +- The candidate is the statement's largest table, and only when it has at least + 1,000,000 rows, the whole statement (including the views it reads) reads it + exactly once and directly in the statement rather than inside a view, and + nothing in the statement or its views already samples (`TABLESAMPLE`, + `USING SAMPLE`). One table is sampled; the other inputs of a join are read in + full, so the disclosed percentage is the one applied. +- The percentage is the largest of 10, 5, 2, 1, 0.5, 0.2, 0.1, 0.05, 0.02, 0.01, + 0.005, 0.002 and 0.001 percent that keeps at least 100,000 expected rows: 10% + of 1,200,000 rows, 0.2% of 50,000,000. +- The seed is fixed (42), so the same statement samples the same rows on every + pause in typing, in memory, in attached files and in Parquet-backed tables + alike. A sample on a base table applies before `WHERE`. + +The method was chosen by measurement on DuckDB 2.0.0 (36-core M5 Ultra, a +50,000,000-row table in an attached database file): + +| Shape, 4 threads | Exact | Seeded Bernoulli | Unseeded Bernoulli | `SYSTEM` | +| --- | --- | --- | --- | --- | +| Full-table window function | 2.53 s | 0.95 s | 0.30 s | n/a | +| Aggregate over 200,000 groups | 0.38 s | 0.85 s | 0.24 s | n/a | +| Aggregate | 0.53 s | 0.92 s | n/a | 0.03 s | +| Top-N | 0.03 s | 0.88 s | n/a | 0.03 s | + +- Seeded Bernoulli is repeatable but runs on one thread: about 0.9 s per + 50,000,000 rows at any thread count. It beats the exact preview only for shapes + that are slow exactly, which is why the notebook tries the exact preview first + and samples only when that does not finish within one second. Above about + 110,000,000 rows the sampled scan itself exceeds the 2-second budget. +- Unseeded Bernoulli is parallel but samples different rows each time, so a + preview would change on every pause. +- `SYSTEM` sampling keeps or drops whole vectors of 2,048 rows. Even with a seed + it was not repeatable (38,912 and 59,392 rows for the same 1% sample of + 5,000,000 rows), drew those rows from 19 to 29 vectors where Bernoulli spread + over all 2,442, and returned no rows from a 500-row table. It is not used. +- `RESERVOIR` sampling is repeatable but needs the whole input (about 1.2 s). + ## Tests [`read-policy-smoke.mjs`](../scripts/read-policy-smoke.mjs) runs inside the diff --git a/scripts/read-policy-smoke.mjs b/scripts/read-policy-smoke.mjs index b6af877..0ef3789 100644 --- a/scripts/read-policy-smoke.mjs +++ b/scripts/read-policy-smoke.mjs @@ -94,6 +94,10 @@ export async function readPolicySmoke({ runSQL, expectSQLError, sqlLiteral, firs attach ${sqlLiteral(servedDatabase)} as local_file_db (read_only); attach ${sqlLiteral(remoteDatabase)} as remote_file_db (read_only); create view policy.remote_file_view as select fact from remote_file_db.facts; + -- Sampled previews: only tables of at least 1,000,000 rows are sampled. + create table policy.big as select i::integer as id, (i % 7)::integer as grp from range(1200000) as r(i); + create view policy.big_view as select id from policy.big; + create view policy.sampled_view as select id from policy.items using sample 50%; select nextval('policy.counter') as initialized`); for (let level = 0; level <= 8; level += 1) { const source = level === 0 ? 'policy.items' : `policy.depth_${level - 1}`; @@ -249,6 +253,73 @@ export async function readPolicySmoke({ runSQL, expectSQLError, sqlLiteral, firs await refuses('select * from policy.items at (version => 1)', 'AT \\(time travel\\) clauses'); await refuses('select * from (pivot policy.items on grp using sum(id))', 'statements that DuckDB expands into several statements'); + // 5. Sampled previews. The plan is a hint; sqlnb_read_query verifies the + // statement again and applies a seeded Bernoulli sample to the verified tree. + const database = firstValue(await runSQL('select current_database() as name')); + const plan = async (sql) => { + const value = firstValue(await runSQL(`select sqlnb_sample_plan(${sqlLiteral(sql)}) as plan`)); + return value === null ? null : JSON.parse(value); + }; + const sampleArguments = (table, percent, catalog = database, schema = 'policy') => + `sample_catalog := ${sqlLiteral(catalog)}, sample_schema := ${sqlLiteral(schema)}, ` + + `sample_table := ${sqlLiteral(table)}, sample_percent := ${percent}`; + const sampled = async (sql, table, percent) => (await runSQL( + `select count(*)::integer as rows, sum(id)::bigint as total from sqlnb_read_query(${sqlLiteral(sql)}, ${sampleArguments(table, percent)})`, + ))[0]; + async function refusesSampled(sql, reason, table = 'big', percent = 10) { + await expectSQLError( + `select * from sqlnb_read_query(${sqlLiteral(sql)}, ${sampleArguments(table, percent)})`, + new RegExp(`SQLNB_READ_ONLY: ${reason}`), + ); + } + + assert.deepEqual(await plan('select count(*) from policy.big'), { + catalog: database, schema: 'policy', table: 'big', rows: 1200000, percent: 10, method: 'bernoulli', + }); + // One relation is sampled; the other inputs of a join are read in full. + assert.equal((await plan('select count(*) from policy.big join policy.items using (id)'))?.table, 'big'); + assert.equal((await plan('with b as (select id from policy.big) select max(id) from b'))?.table, 'big'); + for (const exact of [ + 'select count(*) from policy.items', + 'select count(*) from policy.big a join policy.big b using (id)', + 'select count(*) from policy.big_view', + 'select count(*) from policy.big join policy.big_view using (id)', + 'select count(*) from policy.big using sample 1%', + 'select count(*) from policy.big tablesample 1%', + 'select count(*) from (select id from policy.big) using sample 1%', + 'select count(*) from policy.big join policy.sampled_view using (id)', + 'insert into policy.items select id, 0, \'x\', null, null from policy.big', + 'select nextval(\'policy.counter\'), id from policy.big', + `select * from policy.big, read_csv(${sqlLiteral(remoteCSV)})`, + ]) { + assert.equal(await plan(exact), null, exact); + } + + const first = await sampled('select id from policy.big', 'big', 10); + assert.ok(first.rows > 110000 && first.rows < 130000, `a 10% sample of 1,200,000 rows kept ${first.rows}`); + assert.deepEqual(await sampled('select id from policy.big', 'big', 10), first, 'the fixed seed repeats the sample'); + assert.deepEqual(await sampled('select id from policy.big where id >= 0', 'big', 10), first, 'filters see the same sample'); + const joined = (await runSQL( + `select count(*)::integer as rows from sqlnb_read_query('select id from policy.big join policy.items using (id)', ${sampleArguments('big', 10)})`, + ))[0].rows; + assert.ok(joined <= 4, 'only the sampled table loses rows in a join'); + + // Sampling never widens what the policy accepts. + await refusesSampled('insert into policy.big values (1, 1)', 'Instant SQL does not preview INSERT statements'); + await refusesSampled('select nextval(\'policy.counter\'), id from policy.big', 'Instant SQL does not preview the function \'nextval\''); + await refusesSampled(`select * from read_csv(${sqlLiteral(remoteCSV)})`, 'Instant SQL does not preview the table function \'read_csv\''); + await refusesSampled('select fact from remote_file_db.facts', 'Instant SQL does not preview .*stored at a remote location', 'facts'); + // A sample applies only to the statement's sole read of a large table. + await refusesSampled('select id from policy.items', 'Instant SQL does not preview a sample of .*policy.big\', which must be this statement\'s only read'); + await refusesSampled('select count(*) from policy.big a join policy.big b using (id)', 'Instant SQL does not preview a sample of .*which must be this statement\'s only read'); + await refusesSampled('select count(*) from policy.big_view', 'Instant SQL does not preview a sample of .*which must be this statement\'s only read'); + await refusesSampled('select count(*) from policy.big tablesample 1%', 'Instant SQL does not preview a sample of .*which must be this statement\'s only read'); + await refusesSampled('select id from policy.big', 'Instant SQL does not preview a sample of 100', 'big', 100); + await expectSQLError( + 'select * from sqlnb_read_query(\'select id from policy.big\', sample_table := \'big\')', + /SQLNB_READ_ONLY: a sampled preview needs a catalog, schema, table and percent/, + ); + assert.equal(await sequenceValue(), 1, 'refused statements must not advance sequences'); assert.equal(await policyRows(), 4, 'refused statements must not write rows'); assert.equal(remoteRequests, requestsAfterSetup, 'refused statements must not reach the network'); diff --git a/src/include/sqlnb_read_policy.hpp b/src/include/sqlnb_read_policy.hpp index fed3265..4a2649d 100644 --- a/src/include/sqlnb_read_policy.hpp +++ b/src/include/sqlnb_read_policy.hpp @@ -27,6 +27,8 @@ #include "duckdb/main/database_manager.hpp" #include "duckdb/main/extension/extension_loader.hpp" #include "duckdb/parser/parser.hpp" +#include "duckdb/parser/parsed_data/sample_options.hpp" +#include "duckdb/storage/data_table.hpp" #include "duckdb/parser/column_list.hpp" #include "duckdb/parser/expression/cast_expression.hpp" #include "duckdb/parser/expression/columnref_expression.hpp" @@ -57,9 +59,22 @@ static constexpr idx_t MAX_VIEW_DEPTH = 8; //! Bounds the total number of view bodies verified for one statement, so views //! that share (diamond) references cannot make verification exponential. static constexpr idx_t MAX_VIEW_EXPANSIONS = 64; +//! Sampled previews leave smaller tables exact: below this an exact preview is +//! already fast, and a sample would be a large share of the table anyway. +static constexpr idx_t MIN_SAMPLE_ROWS = 1000000; +//! A sampled preview keeps about this many rows of the sampled table. +static constexpr idx_t SAMPLE_TARGET_ROWS = 100000; +//! Fixed so the same statement samples the same rows on every pause in typing. +static constexpr idx_t SAMPLE_SEED = 42; class Policy { public: + struct TableUse { + optional_ptr table; + //! The reference in the verified statement; null inside a view definition. + optional_ptr reference; + }; + //! `outer_binder` is the binder that will bind the returned statement. Its //! catalog search path and the CTEs of the query around it apply to the //! statement, so verification uses them too. @@ -94,7 +109,89 @@ class Policy { return statement; } + //! The table a sampled preview reads a Bernoulli sample of: the verified + //! statement's largest table, when it has at least MIN_SAMPLE_ROWS rows and + //! the whole statement (including the views it reads) reads it exactly once, + //! directly, and has no sample clause anywhere: a sample of a sample would + //! disclose a percentage that was not the one applied. Sampling one table only + //! keeps the disclosure exact: in a join, the rows of that table are sampled + //! and every other input is read in full. Call after Parse. + struct SamplePlan { + string catalog; + string schema; + string table; + idx_t rows = 0; + double percent = 0; + }; + optional_ptr SampleCandidate() { + optional_ptr largest; + idx_t largest_rows = 0; + for (auto &use : table_uses) { + auto rows = TableRows(*use.table); + if (!largest || rows > largest_rows) { + largest = &use; + largest_rows = rows; + } + } + if (!largest || largest_rows < MIN_SAMPLE_ROWS || !largest->reference || has_sample_clause) return nullptr; + for (auto &use : table_uses) { + if (&use != largest.get() && use.table.get() == largest->table.get()) return nullptr; + } + return largest; + } + + bool PlanSample(SamplePlan &plan) { + auto use = SampleCandidate(); + if (!use) return false; + plan.catalog = use->table->ParentCatalog().GetName().GetIdentifierName(); + plan.schema = use->table->ParentSchema().name.GetIdentifierName(); + plan.table = use->table->name.GetIdentifierName(); + plan.rows = TableRows(*use->table); + plan.percent = SamplePercent(plan.rows); + return true; + } + + //! Applies a Bernoulli sample of `percent` with a fixed seed to the table the + //! plan named, in the verified statement that is then bound. Refuses when that + //! table is no longer the statement's sample candidate. + void ApplySample(const string &catalog, const string &schema, const string &table, double percent) { + if (!(percent > 0 && percent < 100)) Refuse("a sample of " + to_string(percent) + "% (it must be between 0 and 100)"); + auto use = SampleCandidate(); + if (!use || use->table->ParentCatalog().GetName().GetIdentifierName() != catalog || + use->table->ParentSchema().name.GetIdentifierName() != schema || use->table->name.GetIdentifierName() != table) { + Refuse("a sample of " + Quote(catalog + "." + schema + "." + table) + + ", which must be this statement's only read of a table of at least 1,000,000 rows, in a statement without a sample clause"); + } + auto sample = make_uniq(); + sample->sample_size = Value::DOUBLE(percent); + sample->is_percentage = true; + sample->method = SampleMethod::BERNOULLI_SAMPLE; + sample->SetSeed(SAMPLE_SEED); + sample->repeatable = true; + use->reference->sample = std::move(sample); + } + private: + static idx_t TableRows(TableCatalogEntry &table) { + try { + return table.GetStorage().GetTotalRows(); + } catch (const Exception &) { + return 0; + } + } + + //! The largest of 10, 5, 2, 1, 0.5, ... 0.001 percent that keeps at least + //! SAMPLE_TARGET_ROWS rows, so the disclosed percentage is a round number. + static double SamplePercent(idx_t rows) { + static const double ladder[] = {10, 5, 2, 1, 0.5, 0.2, 0.1, 0.05, 0.02, 0.01, 0.005, 0.002, 0.001}; + double percent = ladder[0]; + for (auto candidate : ladder) { + if (double(rows) * candidate / 100 < double(SAMPLE_TARGET_ROWS)) break; + percent = candidate; + } + return percent; + } + [[noreturn]] void Refuse(const string &reason) const { string message = "SQLNB_READ_ONLY: Instant SQL does not preview " + reason; if (!view_path.empty()) message += " (inside view " + StringUtil::Join(view_path, " > ") + ")"; @@ -610,9 +707,15 @@ class Policy { "scans need Run)"); } CheckRelation(*entry, display); + if (entry->type == CatalogType::TABLE_ENTRY) { + // References inside view definitions are verified on a copy and bound + // from the catalog later, so they count but can never be sampled. + table_uses.push_back({&entry->Cast(), view_path.empty() ? &table : nullptr}); + } } void CheckTable(TableRef &reference) { + if (reference.sample) has_sample_clause = true; switch (reference.type) { case TableReferenceType::EMPTY_FROM: return; @@ -669,6 +772,7 @@ class Policy { switch (node.type) { case QueryNodeType::SELECT_NODE: { auto &select = node.Cast(); + if (select.sample) has_sample_clause = true; for (auto &expression : select.select_list) CheckExpression(*expression); for (auto &expression : select.groups.group_expressions) CheckExpression(*expression); if (select.where_clause) CheckExpression(*select.where_clause); @@ -707,6 +811,10 @@ class Policy { idx_t cte_floor = 0; vector view_path; idx_t view_expansions = 0; + //! Every read of a table, in the statement and in the views it reads. + vector table_uses; + //! Whether the statement or a view it reads already samples anything. + bool has_sample_clause = false; }; static void IsReadOnly(DataChunk &input, ExpressionState &state, Vector &result) { @@ -720,11 +828,70 @@ static void IsReadOnly(DataChunk &input, ExpressionState &state, Vector &result) }); } +static string JSONString(const string &value) { + string out = "\""; + for (unsigned char c : value) { + if (c == '"' || c == '\\') { + out += '\\'; + out += char(c); + } else if (c < 0x20) { + static const char *hex = "0123456789abcdef"; + out += "\\u00"; + out += hex[c >> 4]; + out += hex[c & 15]; + } else { + out += char(c); + } + } + return out + "\""; +} + +//! A hint for the notebook: which table a sampled preview of this statement +//! would sample and at what percentage, as JSON, or NULL when it would stay +//! exact (or the policy refuses it). Never an authorization: sqlnb_read_query +//! verifies again and applies only a sample that still matches. +static void SamplePlanFunction(DataChunk &input, ExpressionState &state, Vector &result) { + UnaryExecutor::Execute(input.data[0], result, input.size(), + [&](string_t sql) -> optional { + Policy::SamplePlan plan; + bool planned = false; + try { + Policy policy(state.GetContext()); + policy.Parse(sql.GetString()); + planned = policy.PlanSample(plan); + } catch (const Exception &) { + planned = false; + } + if (!planned) return optional(); + auto json = "{\"catalog\":" + JSONString(plan.catalog) + ",\"schema\":" + JSONString(plan.schema) + + ",\"table\":" + JSONString(plan.table) + ",\"rows\":" + to_string(plan.rows) + + ",\"percent\":" + Value::DOUBLE(plan.percent).ToString() + ",\"method\":\"bernoulli\"}"; + return StringVector::AddString(result, json); + }); +} + static unique_ptr ReadQuery(ClientContext &context, TableFunctionBindInput &input) { if (input.inputs[0].IsNull()) throw InvalidInputException("SQLNB_READ_ONLY: SQL cannot be null"); // Validate and return the same parsed tree, in the outer query's binding // transaction. A separate prior classification cannot authorize this path. - return make_uniq(Policy(context, input.binder).Parse(input.inputs[0].GetValue())); + Policy policy(context, input.binder); + auto statement = policy.Parse(input.inputs[0].GetValue()); + // A sampled preview names the table and percentage it asked for, so what it + // discloses is exactly what is applied; the sample goes on the verified tree. + static const char *sample_parameters[] = {"sample_catalog", "sample_schema", "sample_table", "sample_percent"}; + idx_t given = 0; + for (auto name : sample_parameters) { + auto entry = input.named_parameters.find(name); + if (entry != input.named_parameters.end() && !entry->second.IsNull()) given++; + } + if (given != 0) { + if (given != 4) throw InvalidInputException("SQLNB_READ_ONLY: a sampled preview needs a catalog, schema, table and percent"); + policy.ApplySample(input.named_parameters["sample_catalog"].GetValue(), + input.named_parameters["sample_schema"].GetValue(), + input.named_parameters["sample_table"].GetValue(), + input.named_parameters["sample_percent"].GetValue()); + } + return make_uniq(std::move(statement)); } } // namespace sqlnb_read_policy @@ -733,8 +900,17 @@ static void RegisterSQLNBReadPolicy(ExtensionLoader &loader) { ScalarFunction policy("sqlnb_is_read_only", {LogicalType::VARCHAR}, LogicalType::BOOLEAN, sqlnb_read_policy::IsReadOnly); policy.SetVolatile(); loader.RegisterFunction(policy); + ScalarFunction plan("sqlnb_sample_plan", {LogicalType::VARCHAR}, LogicalType::VARCHAR, sqlnb_read_policy::SamplePlanFunction); + plan.SetVolatile(); + loader.RegisterFunction(plan); TableFunction read("sqlnb_read_query", {LogicalType::VARCHAR}, nullptr, nullptr); read.bind_replace = sqlnb_read_policy::ReadQuery; + read.GetSignature().WithTypedKwargs("options", [](TypedKwargs &options) { + options.Add("sample_catalog", LogicalType::VARCHAR) + .Add("sample_schema", LogicalType::VARCHAR) + .Add("sample_table", LogicalType::VARCHAR) + .Add("sample_percent", LogicalType::DOUBLE); + }); loader.RegisterFunction(read); } } // namespace duckdb