From b0fbdf26f0c3cf096bd3469be549724b5b55f6be Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Thu, 5 Nov 2026 14:09:39 -0800 Subject: [PATCH] Update Poached for DuckDB 2.0 --- .../workflows/MainDistributionPipeline.yml | 7 +- .github/workflows/Release.yml | 7 +- .gitignore | 1 + Makefile | 3 + README.md | 10 +- duckdb | 2 +- extension-ci-tools | 2 +- src/parser.cpp | 144 +++++++++--------- test/sql/duckdb_2.test | 85 +++++++++++ 9 files changed, 184 insertions(+), 77 deletions(-) create mode 100644 test/sql/duckdb_2.test diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index c5a0810..d1ffb5c 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -14,8 +14,9 @@ concurrency: jobs: duckdb-stable-build: name: Build extension binaries - uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.4.2 + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@6fcd0cc249ae03afe8599031201fe353d013aab1 with: - duckdb_version: v1.4.2 - ci_tools_version: v1.4.2 + duckdb_version: v2.0.0 + duckdb_commit: d54cbbe3df17e8e5ce68312f483b4155836112aa + ci_tools_version: 6fcd0cc249ae03afe8599031201fe353d013aab1 extension_name: poached diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index 491f78f..43016b0 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -69,10 +69,11 @@ jobs: build: name: Build extension binaries needs: create-tag - uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.4.2 + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@6fcd0cc249ae03afe8599031201fe353d013aab1 with: - duckdb_version: v1.4.2 - ci_tools_version: v1.4.2 + duckdb_version: v2.0.0 + duckdb_commit: d54cbbe3df17e8e5ce68312f483b4155836112aa + ci_tools_version: 6fcd0cc249ae03afe8599031201fe353d013aab1 extension_name: poached release: diff --git a/.gitignore b/.gitignore index b9f264b..cbc0a36 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ build +.cache/ .idea cmake-build-debug duckdb_unittest_tempdir/ diff --git a/Makefile b/Makefile index adf0cdd..5cadfeb 100644 --- a/Makefile +++ b/Makefile @@ -4,5 +4,8 @@ PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) EXT_NAME=poached EXT_CONFIG=${PROJ_DIR}extension_config.cmake +# DuckDB 2.0 source builds need an explicit release version for extension metadata. +OVERRIDE_GIT_DESCRIBE ?= v2.0.0 + # Include the Makefile from extension-ci-tools include extension-ci-tools/makefiles/duckdb_extension.Makefile diff --git a/README.md b/README.md index 68f6e96..65b1b4e 100644 --- a/README.md +++ b/README.md @@ -110,9 +110,17 @@ SELECT parse_table_names('SELECT * FROM users JOIN orders ON true'); ## Building from Source +The current source targets DuckDB **v2.0.0**. The DuckDB and extension-ci-tools +submodules are pinned to the revisions used by CI and the release workflow. +Binaries built against older DuckDB versions must be rebuilt for 2.0. + ```shell # Clone with submodules -git clone --recurse-submodules +git clone --recurse-submodules https://github.com/sidequery/poached.git +cd poached + +# After updating an existing checkout +git submodule update --init --recursive # Build release make release diff --git a/duckdb b/duckdb index d1dc88f..d54cbbe 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit d1dc88f950d456d72493df452dabdcd13aa413dd +Subproject commit d54cbbe3df17e8e5ce68312f483b4155836112aa diff --git a/extension-ci-tools b/extension-ci-tools index 5b96f63..6fcd0cc 160000 --- a/extension-ci-tools +++ b/extension-ci-tools @@ -1 +1 @@ -Subproject commit 5b96f6390b7eb51ef395c81e26b3627c2049ec28 +Subproject commit 6fcd0cc249ae03afe8599031201fe353d013aab1 diff --git a/src/parser.cpp b/src/parser.cpp index 8f7b634..5d2627e 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -15,6 +15,7 @@ #include "duckdb/function/table_function.hpp" #include "duckdb/function/scalar_function.hpp" #include "duckdb/common/exception.hpp" +#include "duckdb/common/vector/list_vector.hpp" #include "duckdb/common/vector_operations/binary_executor.hpp" #include @@ -38,7 +39,7 @@ struct TokenizeSqlState : public GlobalTableFunctionState { }; static unique_ptr TokenizeSqlBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto result = make_uniq(); result->query = input.inputs[0].GetValue(); result->tokens = Parser::Tokenize(result->query); @@ -79,7 +80,7 @@ static void TokenizeSqlFunc(ClientContext &context, TableFunctionInput &data_p, count++; state.current_idx++; } - output.SetCardinality(count); + output.SetChildCardinality(count); } // ============================================================================ @@ -89,7 +90,7 @@ static void TokenizeSqlFunc(ClientContext &context, TableFunctionInput &data_p, static void IsValidSqlFunc(DataChunk &args, ExpressionState &state, Vector &result) { UnaryExecutor::Execute(args.data[0], result, args.size(), [&](string_t query) { try { - Parser parser; + auto parser = Parser::GetBuiltinParser(); parser.ParseQuery(query.GetString()); return true; } catch (...) { @@ -103,12 +104,13 @@ static void SqlErrorMessageFunc(DataChunk &args, ExpressionState &state, Vector auto count = args.size(); UnifiedVectorFormat vdata; - input.ToUnifiedFormat(count, vdata); + input.ToUnifiedFormat(vdata); auto input_data = UnifiedVectorFormat::GetData(vdata); result.SetVectorType(VectorType::FLAT_VECTOR); - auto result_data = FlatVector::GetData(result); - auto &result_validity = FlatVector::Validity(result); + FlatVector::SetSize(result, count); + auto result_data = FlatVector::GetDataMutable(result); + auto &result_validity = FlatVector::ValidityMutable(result); for (idx_t i = 0; i < count; i++) { auto idx = vdata.sel->get_index(i); @@ -119,7 +121,7 @@ static void SqlErrorMessageFunc(DataChunk &args, ExpressionState &state, Vector auto query_str = input_data[idx].GetString(); try { - Parser parser; + auto parser = Parser::GetBuiltinParser(); parser.ParseQuery(query_str); // Valid SQL - return NULL result_validity.SetInvalid(i); @@ -134,7 +136,7 @@ static void SqlErrorMessageFunc(DataChunk &args, ExpressionState &state, Vector static void NumStatementsFunc(DataChunk &args, ExpressionState &state, Vector &result) { UnaryExecutor::Execute(args.data[0], result, args.size(), [&](string_t query) { try { - Parser parser; + auto parser = Parser::GetBuiltinParser(); parser.ParseQuery(query.GetString()); return static_cast(parser.statements.size()); } catch (...) { @@ -162,7 +164,7 @@ struct SqlKeywordsState : public GlobalTableFunctionState { }; static unique_ptr SqlKeywordsBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto result = make_uniq(); auto keywords = Parser::KeywordList(); for (auto &kw : keywords) { @@ -189,7 +191,7 @@ static void SqlKeywordsFunc(ClientContext &context, TableFunctionInput &data_p, count++; state.current_idx++; } - output.SetCardinality(count); + output.SetChildCardinality(count); } // ============================================================================ @@ -201,8 +203,9 @@ static void ParseKeywordNamesFunc(DataChunk &args, ExpressionState &state, Vecto auto count = args.size(); result.SetVectorType(VectorType::FLAT_VECTOR); - auto result_data = FlatVector::GetData(result); - auto &child = ListVector::GetEntry(result); + FlatVector::SetSize(result, count); + auto result_data = FlatVector::GetDataMutable(result); + auto &child = ListVector::GetChildMutable(result); idx_t current_size = ListVector::GetListSize(result); idx_t keyword_count = keywords.size(); @@ -233,12 +236,12 @@ struct ParseStatementsState : public GlobalTableFunctionState { }; static unique_ptr ParseStatementsBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto result = make_uniq(); result->query = input.inputs[0].GetValue(); try { - Parser parser; + auto parser = Parser::GetBuiltinParser(); parser.ParseQuery(result->query); result->statements = std::move(parser.statements); } catch (const Exception &e) { @@ -274,7 +277,7 @@ static void ParseStatementsFunc(ClientContext &context, TableFunctionInput &data output.data[3].SetValue(count, Value::BIGINT(0)); count++; state.current_idx++; - output.SetCardinality(count); + output.SetChildCardinality(count); return; } @@ -287,7 +290,7 @@ static void ParseStatementsFunc(ClientContext &context, TableFunctionInput &data count++; state.current_idx++; } - output.SetCardinality(count); + output.SetChildCardinality(count); } // ============================================================================ @@ -314,8 +317,8 @@ static void ExtractTablesFromQueryNode(QueryNode *node, vector & // Handle CTEs for (auto &cte : node->cte_map.map) { - if (cte.second && cte.second->query) { - ExtractTablesFromQueryNode(cte.second->query->node.get(), tables); + if (cte.second && cte.second->query_node) { + ExtractTablesFromQueryNode(cte.second->query_node.get(), tables); } } } @@ -327,8 +330,8 @@ static void ExtractTablesFromTableRef(duckdb::TableRef *tref, vectorCast(); ExtractedTable t; - t.schema = base.schema_name; - t.table = base.table_name; + t.schema = base.GetQualifiedName().Schema().GetIdentifierName(); + t.table = base.Table().GetIdentifierName(); t.context = ctx; tables.push_back(t); break; @@ -351,8 +354,8 @@ static void ExtractTablesFromTableRef(duckdb::TableRef *tref, vectorCast(); ExtractedTable t; - t.schema = fn.schema; - t.table = fn.function_name; + t.schema = fn.GetQualifiedName().Schema().GetIdentifierName(); + t.table = fn.FunctionName().GetIdentifierName(); t.context = "TABLE_FUNCTION"; tables.push_back(t); } @@ -388,12 +391,12 @@ struct ParseTablesState : public GlobalTableFunctionState { }; static unique_ptr ParseTablesBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto result = make_uniq(); string query = input.inputs[0].GetValue(); try { - Parser parser; + auto parser = Parser::GetBuiltinParser(); parser.ParseQuery(query); for (auto &stmt : parser.statements) { ExtractTablesFromStatement(stmt.get(), result->tables); @@ -429,7 +432,7 @@ static void ParseTablesFunc(ClientContext &context, TableFunctionInput &data_p, count++; state.current_idx++; } - output.SetCardinality(count); + output.SetChildCardinality(count); } // ============================================================================ @@ -440,7 +443,7 @@ static void ParseTableNamesFunc(DataChunk &args, ExpressionState &state, Vector UnaryExecutor::Execute(args.data[0], result, args.size(), [&](string_t query) { vector tables; try { - Parser parser; + auto parser = Parser::GetBuiltinParser(); parser.ParseQuery(query.GetString()); for (auto &stmt : parser.statements) { ExtractTablesFromStatement(stmt.get(), tables); @@ -448,9 +451,10 @@ static void ParseTableNamesFunc(DataChunk &args, ExpressionState &state, Vector } catch (...) { } - auto &child = ListVector::GetEntry(result); + auto &child = ListVector::GetChildMutable(result); auto current_size = ListVector::GetListSize(result); + ListVector::Reserve(result, current_size + tables.size()); for (auto &t : tables) { child.SetValue(current_size++, Value(t.table)); } @@ -491,18 +495,18 @@ static bool IsAggregateFunction(const string &name) { return aggregates.find(lower_name) != aggregates.end(); } -static void ExtractFunctionsFromExpression(ParsedExpression *expr, vector &functions) { +static void ExtractFunctionsFromExpression(const ParsedExpression *expr, vector &functions) { if (!expr) return; - if (expr->type == ExpressionType::FUNCTION) { + if (expr->GetExpressionType() == ExpressionType::FUNCTION) { auto &fn = expr->Cast(); FunctionRef f; - f.name = fn.function_name; - f.type = fn.is_operator ? "operator" : (IsAggregateFunction(fn.function_name) ? "aggregate" : "scalar"); + f.name = fn.FunctionName().GetIdentifierName(); + f.type = fn.IsOperator() ? "operator" : (IsAggregateFunction(f.name) ? "aggregate" : "scalar"); functions.push_back(f); // Recurse into function arguments - for (auto &child : fn.children) { - ExtractFunctionsFromExpression(child.get(), functions); + for (auto &argument : fn.GetArguments()) { + ExtractFunctionsFromExpression(&argument.GetExpression(), functions); } } } @@ -549,12 +553,12 @@ struct ParseFunctionsState : public GlobalTableFunctionState { }; static unique_ptr ParseFunctionsBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto result = make_uniq(); string query = input.inputs[0].GetValue(); try { - Parser parser; + auto parser = Parser::GetBuiltinParser(); parser.ParseQuery(query); for (auto &stmt : parser.statements) { ExtractFunctionsFromStatement(stmt.get(), result->functions); @@ -587,7 +591,7 @@ static void ParseFunctionsFunc(ClientContext &context, TableFunctionInput &data_ count++; state.current_idx++; } - output.SetCardinality(count); + output.SetChildCardinality(count); } // ============================================================================ @@ -598,7 +602,7 @@ static void ParseFunctionNamesFunc(DataChunk &args, ExpressionState &state, Vect UnaryExecutor::Execute(args.data[0], result, args.size(), [&](string_t query) { vector functions; try { - Parser parser; + auto parser = Parser::GetBuiltinParser(); parser.ParseQuery(query.GetString()); for (auto &stmt : parser.statements) { ExtractFunctionsFromStatement(stmt.get(), functions); @@ -606,9 +610,10 @@ static void ParseFunctionNamesFunc(DataChunk &args, ExpressionState &state, Vect } catch (...) { } - auto &child = ListVector::GetEntry(result); + auto &child = ListVector::GetChildMutable(result); auto current_size = ListVector::GetListSize(result); + ListVector::Reserve(result, current_size + functions.size()); for (auto &f : functions) { child.SetValue(current_size++, Value(f.name)); } @@ -642,12 +647,13 @@ static string ComparisonTypeToOperator(ExpressionType type) { } } -static void ExtractWhereConditions(ParsedExpression *expr, vector &conditions) { +static void ExtractWhereConditions(const ParsedExpression *expr, vector &conditions) { if (!expr) return; - if (expr->type == ExpressionType::CONJUNCTION_AND || expr->type == ExpressionType::CONJUNCTION_OR) { + if (expr->GetExpressionType() == ExpressionType::CONJUNCTION_AND || + expr->GetExpressionType() == ExpressionType::CONJUNCTION_OR) { auto &conj = expr->Cast(); - for (auto &child : conj.children) { + for (auto &child : conj.GetChildren()) { ExtractWhereConditions(child.get(), conditions); } return; @@ -656,22 +662,24 @@ static void ExtractWhereConditions(ParsedExpression *expr, vectorGetExpressionClass() == ExpressionClass::COMPARISON) { auto &cmp = expr->Cast(); WhereCondition cond; - cond.op = ComparisonTypeToOperator(cmp.type); + cond.op = ComparisonTypeToOperator(cmp.GetExpressionType()); // Try to get column name from left side - if (cmp.left && cmp.left->type == ExpressionType::COLUMN_REF) { - auto &col = cmp.left->Cast(); - cond.column_name = col.GetColumnName(); - } else if (cmp.left) { - cond.column_name = cmp.left->ToString(); + auto &left = cmp.Left(); + if (left.GetExpressionType() == ExpressionType::COLUMN_REF) { + auto &col = left.Cast(); + cond.column_name = col.GetColumnName().GetIdentifierName(); + } else { + cond.column_name = left.ToString(); } // Try to get value from right side - if (cmp.right && cmp.right->type == ExpressionType::VALUE_CONSTANT) { - auto &val = cmp.right->Cast(); - cond.value = val.value.ToString(); - } else if (cmp.right) { - cond.value = cmp.right->ToString(); + auto &right = cmp.Right(); + if (right.GetExpressionType() == ExpressionType::VALUE_CONSTANT) { + auto &val = right.Cast(); + cond.value = val.GetLiteral().ToValue().ToString(); + } else { + cond.value = right.ToString(); } conditions.push_back(cond); @@ -692,12 +700,12 @@ struct ParseWhereState : public GlobalTableFunctionState { }; static unique_ptr ParseWhereBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto result = make_uniq(); string query = input.inputs[0].GetValue(); try { - Parser parser; + auto parser = Parser::GetBuiltinParser(); parser.ParseQuery(query); for (auto &stmt : parser.statements) { if (stmt->type == StatementType::SELECT_STATEMENT) { @@ -741,7 +749,7 @@ static void ParseWhereFunc(ClientContext &context, TableFunctionInput &data_p, D count++; state.current_idx++; } - output.SetCardinality(count); + output.SetChildCardinality(count); } // ============================================================================ @@ -828,23 +836,23 @@ struct ParseColumnsState : public GlobalTableFunctionState { idx_t current_idx = 0; }; -static string GetExpressionName(ParsedExpression *expr) { +static string GetExpressionName(const ParsedExpression *expr) { if (!expr) return ""; // If there's an alias, use it - if (!expr->alias.empty()) { - return expr->alias; + if (expr->HasAlias()) { + return expr->GetAlias().GetIdentifierName(); } // Otherwise, generate a name based on expression type - switch (expr->type) { + switch (expr->GetExpressionType()) { case ExpressionType::COLUMN_REF: { auto &col = expr->Cast(); - return col.GetColumnName(); + return col.GetColumnName().GetIdentifierName(); } case ExpressionType::FUNCTION: { auto &fn = expr->Cast(); - return fn.function_name; + return fn.FunctionName().GetIdentifierName(); } default: return expr->ToString(); @@ -852,13 +860,13 @@ static string GetExpressionName(ParsedExpression *expr) { } static unique_ptr ParseColumnsBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto result = make_uniq(); string query = input.inputs[0].GetValue(); idx_t stmt_index = input.inputs[1].GetValue(); try { - Parser parser; + auto parser = Parser::GetBuiltinParser(); parser.ParseQuery(query); if (stmt_index < parser.statements.size()) { @@ -900,7 +908,7 @@ static void ParseColumnsFunc(ClientContext &context, TableFunctionInput &data_p, count++; state.current_idx++; } - output.SetCardinality(count); + output.SetChildCardinality(count); } // ============================================================================ @@ -910,7 +918,7 @@ static void ParseColumnsFunc(ClientContext &context, TableFunctionInput &data_p, static void ParseColumnNamesFunc(DataChunk &args, ExpressionState &state, Vector &result) { auto &query_vec = args.data[0]; auto &idx_vec = args.data[1]; - auto &child = ListVector::GetEntry(result); + auto &child = ListVector::GetChildMutable(result); idx_t current_size = ListVector::GetListSize(result); BinaryExecutor::Execute(query_vec, idx_vec, result, args.size(), @@ -918,7 +926,7 @@ static void ParseColumnNamesFunc(DataChunk &args, ExpressionState &state, Vector vector col_names; try { if (stmt_index >= 0) { - Parser parser; + auto parser = Parser::GetBuiltinParser(); parser.ParseQuery(query.GetString()); auto idx = static_cast(stmt_index); if (idx < parser.statements.size()) { @@ -973,7 +981,7 @@ static string EscapeJsonString(const string &s) { static void SqlParseJsonFunc(DataChunk &args, ExpressionState &state, Vector &result) { UnaryExecutor::Execute(args.data[0], result, args.size(), [&](string_t query) { try { - Parser parser; + auto parser = Parser::GetBuiltinParser(); parser.ParseQuery(query.GetString()); std::ostringstream json; @@ -1036,7 +1044,7 @@ void RegisterParserFunctions(ExtensionLoader &loader) { loader.RegisterFunction(is_valid_sql); ScalarFunction sql_error_message("sql_error_message", {LogicalType::VARCHAR}, LogicalType::VARCHAR, SqlErrorMessageFunc); - sql_error_message.null_handling = FunctionNullHandling::SPECIAL_HANDLING; + sql_error_message.SetNullHandling(FunctionNullHandling::SPECIAL_HANDLING); loader.RegisterFunction(sql_error_message); ScalarFunction num_statements("num_statements", {LogicalType::VARCHAR}, LogicalType::BIGINT, NumStatementsFunc); diff --git a/test/sql/duckdb_2.test b/test/sql/duckdb_2.test new file mode 100644 index 0000000..5c8d866 --- /dev/null +++ b/test/sql/duckdb_2.test @@ -0,0 +1,85 @@ +# name: test/sql/duckdb_2.test +# description: parser and vector API compatibility with DuckDB 2.0 +# group: [parser] + +require poached + +# Identifier conversions preserve spelling and schema qualification. +query TTT +select * from parse_tables('select * from "Sales"."Orders"') +---- +Sales Orders FROM + +query TTT +select * from parse_tables('select * from main.range(3)') +---- +main range TABLE_FUNCTION + +# CTEs now own their QueryNode directly. +query TT rowsort +select schema_name, table_name from parse_tables('with recent as (select * from sales.orders) select * from recent') +---- +NULL recent +sales orders + +query T +select parse_column_names('select "CustomerID", 1 as "Total", upper(name) from orders', 0) +---- +[CustomerID, Total, upper] + +# Function arguments use the new argument wrapper, including named arguments. +query TT rowsort +select * from parse_functions('select my_func(label := upper(name)), sum(amount), amount + 1 from orders') +---- ++ operator +my_func scalar +sum aggregate +upper scalar + +# Literals still return decoded values, without SQL string quoting. +query TTT rowsort +select * from parse_where('select * from t where "Name" = ''O''''Brien'' and amount >= 12.50 or n < -3') +---- +Name = O'Brien +amount >= 12.50 +n < -3 + +# Non-literal comparison operands still render as SQL expressions. +query TTT +select * from parse_where('select * from t where a > b') +---- +a > b + +# Flat result vectors must carry their size and preserve per-row NULLs. +query ITI +select id, is_valid_sql(sql), sql_error_message(sql) is null +from (values (1, 'select 1'), (2, 'SELEC 1'), (3, NULL), (4, 'select 2')) inputs(id, sql) +order by id +---- +1 true true +2 false false +3 NULL true +4 true true + +# List children must grow beyond one vector's initial capacity. +query III +with inputs as ( + select 'select upper(a), lower(b), length(c) from t' || i::varchar || ' join u on true' as sql + from range(3000) r(i) +) +select sum(len(parse_table_names(sql))), sum(len(parse_function_names(sql))), + sum(len(parse_column_names(sql, 0))) +from inputs +---- +6000 9000 9000 + +# Table functions must emit all chunks, including the final partial chunk. +query I +select count(*) from parse_tokens(repeat('select 1; ', 3000)) +---- +9000 + +query I +select count(*) from parse_statements(repeat('select 1; ', 3000)) +---- +3000