From 9669a8baa54b9488c7235f61c4e54e0621ffe5a2 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Fri, 17 Jul 2026 19:07:05 +0200 Subject: [PATCH 1/3] Replace ANTLR parser with CongoCC Move CEL grammar generation into cel-core using CongoCC and remove the generated-antlr module, shadow jar dependency, and ANTLR runtime wiring. The generated parser now builds internal AST nodes with typed CEL expression conversion hooks, keeping parser construction close to the grammar while preserving the existing parser API and source-location handling. This simplifies dependency management for downstream consumers, removes the relocated ANTLR artifact from the build graph, and makes future grammar changes easier to review in the core module. --- README.md | 2 +- bom/build.gradle.kts | 1 - build.gradle.kts | 1 - core/build.gradle.kts | 51 +- core/src/main/congocc/cel/cel-java.ccc | 151 +++ core/src/main/congocc/cel/cel-lexer.ccc | 84 ++ core/src/main/congocc/cel/cel.ccc | 93 ++ .../cel/parser/CelExprBuilder.java | 37 + .../projectnessie/cel/parser/CelExprNode.java | 22 + .../org/projectnessie/cel/parser/Helper.java | 11 +- .../org/projectnessie/cel/parser/Parser.java | 1084 ++++++++--------- .../cel/parser/StringCharStream.java | 105 -- .../projectnessie/cel/parser/ParserTest.java | 6 +- generated-antlr/build.gradle.kts | 69 -- .../org.projectnessie.cel.parser.gen/CEL.g4 | 197 --- .../CEL.tokens | 64 - .../CELLexer.tokens | 64 - gradle/libs.versions.toml | 4 +- settings.gradle.kts | 2 - standalone/build.gradle.kts | 2 - .../cel/tools/ScriptHostTest.java | 2 +- 21 files changed, 933 insertions(+), 1119 deletions(-) create mode 100644 core/src/main/congocc/cel/cel-java.ccc create mode 100644 core/src/main/congocc/cel/cel-lexer.ccc create mode 100644 core/src/main/congocc/cel/cel.ccc create mode 100644 core/src/main/java/org/projectnessie/cel/parser/CelExprBuilder.java create mode 100644 core/src/main/java/org/projectnessie/cel/parser/CelExprNode.java delete mode 100644 core/src/main/java/org/projectnessie/cel/parser/StringCharStream.java delete mode 100644 generated-antlr/build.gradle.kts delete mode 100644 generated-antlr/src/main/antlr/org.projectnessie.cel.parser.gen/CEL.g4 delete mode 100644 generated-antlr/src/main/antlr/org.projectnessie.cel.parser.gen/CEL.tokens delete mode 100644 generated-antlr/src/main/antlr/org.projectnessie.cel.parser.gen/CELLexer.tokens diff --git a/README.md b/README.md index 2e61ff56..fbb49c0f 100644 --- a/README.md +++ b/README.md @@ -420,7 +420,7 @@ Native-image and package behavior must be verified in the consuming application' prove Quarkus native-image or package compatibility for every application. Before using CEL conditions in release-critical authorization paths, run JVM condition tests, the -consuming project's normal build, dependency tree review for protobuf/ANTLR/Jackson conflicts, and +consuming project's normal build, dependency tree review for protobuf/Jackson conflicts, and package/native-image verification if native execution is part of the release path. ### Not yet implemented diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index b50161b0..c0772de1 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -24,7 +24,6 @@ plugins { dependencies { constraints { api(project(":cel-core")) - api(project(":cel-generated-antlr")) api(project(":cel-generated-pb")) api(project(":cel-generated-pb3")) api(project(":cel-conformance")) diff --git a/build.gradle.kts b/build.gradle.kts index df3713e0..62a9c29a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -102,7 +102,6 @@ idea.project.settings { afterSync( ":cel-generated-pb:jar", ":cel-generated-pb:testJar", - ":cel-generated-antlr:shadowJar", ) } } diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 67a2b529..52175403 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -24,14 +24,17 @@ plugins { `java-test-fixtures` } +val congocc = configurations.create("congocc") + configurations.named("jmhImplementation") { extendsFrom(configurations.testFixturesApi.get()) } dependencies { - implementation(project(":cel-generated-antlr")) compileOnly(project(":cel-generated-pb")) implementation(libs.agrona) + congocc(libs.congocc) + testImplementation(project(":cel-generated-pb")) testFixturesApi(platform(libs.junit.bom)) testFixturesApi(libs.bundles.junit.testing) @@ -46,13 +49,59 @@ dependencies { jmhAnnotationProcessor(libs.jmh.generator.annprocess) } +abstract class Generate : JavaExec() { + init { + outputs.cacheIf { true } + } + + @get:InputDirectory + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val sourceDir: DirectoryProperty + + @get:OutputDirectory abstract val outputDir: DirectoryProperty +} + +val generateCelGrammar = + tasks.register("generateCelGrammar") { + val generatedDir = layout.buildDirectory.dir("generated/sources/congocc/cel") + val projectDir = layout.projectDirectory + + sourceDir = projectDir.dir("src/main/congocc/cel") + outputDir = generatedDir + + classpath(congocc) + mainClass = "org.congocc.app.Main" + workingDir(projectDir) + + doFirst { generatedDir.get().asFile.deleteRecursively() } + + argumentProviders.add( + CommandLineArgumentProvider { + val sourceFile = sourceDir.file("cel-java.ccc").get().asFile.relativeTo(projectDir.asFile) + val base = + listOf( + "-d", + generatedDir.get().asFile.toString(), + "-jdk17", + "-n", + sourceFile.toString(), + ) + if (logger.isInfoEnabled) base else base + "-q" + } + ) + } + jmh { jmhVersion.set(libs.versions.jmh.get()) } +sourceSets.main { java.srcDir(generateCelGrammar) } + sourceSets.test { java.srcDir(layout.buildDirectory.dir("generated/source/proto/test/java")) java.destinationDirectory.set(layout.buildDirectory.dir("classes/java/generatedTest")) } +tasks.named("compileJava") { dependsOn(generateCelGrammar) } + tasks.named("check") { dependsOn(tasks.named("jmh")) } tasks.named("assemble") { dependsOn(tasks.named("jmhJar")) } diff --git a/core/src/main/congocc/cel/cel-java.ccc b/core/src/main/congocc/cel/cel-java.ccc new file mode 100644 index 00000000..07ac96ee --- /dev/null +++ b/core/src/main/congocc/cel/cel-java.ccc @@ -0,0 +1,151 @@ +/* + * Copyright (C) 2026 The Authors of CEL-Java + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +PARSER_PACKAGE="org.projectnessie.cel.parser"; +PARSER_CLASS=CelGrammarParser; +LEXER_CLASS=CelGrammarLexer; +NODE_PACKAGE="org.projectnessie.cel.parser.ast"; + +INCLUDE "cel.ccc" + +INJECT PARSER_CLASS : +{ + private boolean nextTokenStartsExpression() { + TokenType type = getToken(1).getType(); + return switch (type) { + case LBRACKET, LBRACE, LPAREN, DOT, MINUS, EXCLAM, TRUE, FALSE, NULL, NUM_UINT, NUM_FLOAT, + NUM_INT, STRING, BYTES, IDENTIFIER -> true; + default -> false; + }; + } +} + +INJECT ParseException : +{ + @SuppressWarnings("unchecked") + public java.util.Set getExpectedTokenTypes() { + java.util.Set x = expectedTypes; + return x; + } +} + +INJECT Expr : implements CelExprNode; +INJECT Expr : +{ + @Override + public com.google.api.expr.v1alpha1.Expr toCelExpr(CelExprBuilder builder) { + return builder.visitExpr(this); + } +} + +INJECT ConditionalOr : implements CelExprNode; +INJECT ConditionalOr : +{ + @Override + public com.google.api.expr.v1alpha1.Expr toCelExpr(CelExprBuilder builder) { + return builder.visitBalanced(this, org.projectnessie.cel.common.operators.Operator.LogicalOr); + } +} + +INJECT ConditionalAnd : implements CelExprNode; +INJECT ConditionalAnd : +{ + @Override + public com.google.api.expr.v1alpha1.Expr toCelExpr(CelExprBuilder builder) { + return builder.visitBalanced(this, org.projectnessie.cel.common.operators.Operator.LogicalAnd); + } +} + +INJECT Relation : implements CelExprNode; +INJECT Relation : +{ + @Override + public com.google.api.expr.v1alpha1.Expr toCelExpr(CelExprBuilder builder) { + return builder.visitBinary(this); + } +} + +INJECT Calc : implements CelExprNode; +INJECT Calc : +{ + @Override + public com.google.api.expr.v1alpha1.Expr toCelExpr(CelExprBuilder builder) { + return builder.visitBinary(this); + } +} + +INJECT Multiplicative : implements CelExprNode; +INJECT Multiplicative : +{ + @Override + public com.google.api.expr.v1alpha1.Expr toCelExpr(CelExprBuilder builder) { + return builder.visitBinary(this); + } +} + +INJECT Unary : implements CelExprNode; +INJECT Unary : +{ + @Override + public com.google.api.expr.v1alpha1.Expr toCelExpr(CelExprBuilder builder) { + return builder.visitUnary(this); + } +} + +INJECT Member : implements CelExprNode; +INJECT Member : +{ + @Override + public com.google.api.expr.v1alpha1.Expr toCelExpr(CelExprBuilder builder) { + return builder.visitMember(this); + } +} + +INJECT Primary : implements CelExprNode; +INJECT Primary : +{ + @Override + public com.google.api.expr.v1alpha1.Expr toCelExpr(CelExprBuilder builder) { + return builder.visitPrimary(this); + } +} + +INJECT ConstantLiteral : implements CelExprNode; +INJECT ConstantLiteral : +{ + @Override + public com.google.api.expr.v1alpha1.Expr toCelExpr(CelExprBuilder builder) { + return builder.visitLiteral(this); + } +} + +INJECT Literal : implements CelExprNode; +INJECT Literal : +{ + @Override + public com.google.api.expr.v1alpha1.Expr toCelExpr(CelExprBuilder builder) { + return builder.visitLiteral(this); + } +} + +INJECT Identifier : implements CelExprNode; +INJECT Identifier : +{ + @Override + public com.google.api.expr.v1alpha1.Expr toCelExpr(CelExprBuilder builder) { + return builder.visitIdentifier(this); + } +} diff --git a/core/src/main/congocc/cel/cel-lexer.ccc b/core/src/main/congocc/cel/cel-lexer.ccc new file mode 100644 index 00000000..62c4c32f --- /dev/null +++ b/core/src/main/congocc/cel/cel-lexer.ccc @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2026 The Authors of CEL-Java + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +SKIP : #Whitespace; + +UNPARSED : #Comment; + +TOKEN #Operator : + + | ="> + | + | + | + | + | "> + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + ; + +TOKEN #Literal : + + | + | + | <#HEXDIGIT : ["0"-"9", "a"-"f", "A"-"F"]> + | <#DIGIT : ["0"-"9"]> + | <#EXPONENT : ["e", "E"] (["+", "-"])? ()+> + | )+ | "0x" ()+) ["u", "U"]> + | )+ "." ()+ ()? | ()+ | "." ()+ ()?)> + | )+ | "0x" ()+> + | <#ESC_CHAR_SEQ : "\\" ["a", "b", "f", "n", "r", "t", "v", "\"", "'", "\\", "?", "`"]> + | <#ESC_OCT_SEQ : "\\" ["0"-"3"] ["0"-"7"] ["0"-"7"]> + | <#ESC_BYTE_SEQ : "\\" ["x", "X"] > + | <#ESC_UNI_SEQ : "\\" "u" | "\\" "U" > + | <#ESC_SEQ : | | | > + | <#DQ_CHAR : | ~["\\", "\"", "\n", "\r"]> + | <#SQ_CHAR : | ~["\\", "'", "\n", "\r"]> + | <#TDQ_CHAR : | ~["\\"]> + | <#TSQ_CHAR : | ~["\\"]> + | )* "\"" + | "'" ()* "'" + | "\"\"\"" ()* "\"\"\"" + | "'''" ()* "'''" + | ["r", "R"] "\"" (~["\"", "\n", "\r"])* "\"" + | ["r", "R"] "'" (~["'", "\n", "\r"])* "'" + | ["r", "R"] "\"\"\"" (~[])* "\"\"\"" + | ["r", "R"] "'''" (~[])* "'''" + > + | > + ; + +TOKEN #Identifier : + + | | ~["\\", "`", "\n", "\r"])* "`"> + ; diff --git a/core/src/main/congocc/cel/cel.ccc b/core/src/main/congocc/cel/cel.ccc new file mode 100644 index 00000000..a6acc145 --- /dev/null +++ b/core/src/main/congocc/cel/cel.ccc @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2026 The Authors of CEL-Java + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +INCLUDE "cel-lexer.ccc" + +Start : Expr! ! ; + +Expr : + ConditionalOr + [ ConditionalOr Expr] + ; + +ConditionalOr : + ConditionalAnd ( ConditionalAnd)*! + ; + +ConditionalAnd : + Relation ( Relation)*! + ; + +Relation : + Calc (( | | | | | | ) Calc)*! + ; + +Calc : + Multiplicative (( | ) Multiplicative)*! + ; + +Multiplicative : + Unary (( | | ) Unary)*! + ; + +Unary : + ( | )* Member + ; + +Member : + Primary + ( + Field [ ( | ExprList )] + | Expr + | [FieldInitializerList] [] + )*! + ; + +Primary : + [] [ ( | ExprList )] + | Expr + | ( | ExprList [] ) + | ( | MapInitializerList [] ) + | ConstantLiteral + ; + +ExprList : + Expr ( Expr =>||)*! + ; + +FieldInitializerList : + Field Expr ( Field Expr =>||)*! + ; + +Field : + + | + ; + +MapInitializerList : + Expr Expr ( Expr Expr =>||)*! + ; + +ConstantLiteral : + + | + | + | + | + | + | + | + ; diff --git a/core/src/main/java/org/projectnessie/cel/parser/CelExprBuilder.java b/core/src/main/java/org/projectnessie/cel/parser/CelExprBuilder.java new file mode 100644 index 00000000..3d88b07d --- /dev/null +++ b/core/src/main/java/org/projectnessie/cel/parser/CelExprBuilder.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2026 The Authors of CEL-Java + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.cel.parser; + +import com.google.api.expr.v1alpha1.Expr; +import org.projectnessie.cel.common.operators.Operator; + +public interface CelExprBuilder { + Expr visitExpr(Node node); + + Expr visitBalanced(Node node, Operator operator); + + Expr visitBinary(Node node); + + Expr visitUnary(Node node); + + Expr visitMember(Node node); + + Expr visitPrimary(Node node); + + Expr visitLiteral(Node node); + + Expr visitIdentifier(Token token); +} diff --git a/core/src/main/java/org/projectnessie/cel/parser/CelExprNode.java b/core/src/main/java/org/projectnessie/cel/parser/CelExprNode.java new file mode 100644 index 00000000..2abb98a5 --- /dev/null +++ b/core/src/main/java/org/projectnessie/cel/parser/CelExprNode.java @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2026 The Authors of CEL-Java + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.cel.parser; + +import com.google.api.expr.v1alpha1.Expr; + +public interface CelExprNode { + Expr toCelExpr(CelExprBuilder builder); +} diff --git a/core/src/main/java/org/projectnessie/cel/parser/Helper.java b/core/src/main/java/org/projectnessie/cel/parser/Helper.java index 5e9f1a61..b986767b 100644 --- a/core/src/main/java/org/projectnessie/cel/parser/Helper.java +++ b/core/src/main/java/org/projectnessie/cel/parser/Helper.java @@ -36,8 +36,6 @@ import org.agrona.collections.LongArrayList; import org.projectnessie.cel.common.Location; import org.projectnessie.cel.common.Source; -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.ParserRuleContext; -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.Token; final class Helper { private final Source source; @@ -197,12 +195,9 @@ private Builder newExprBuilder(Object ctx) { long id(Object ctx) { Location location; - if (ctx instanceof ParserRuleContext) { - Token token = ((ParserRuleContext) ctx).start; - location = source.newLocation(token.getLine(), token.getCharPositionInLine()); - } else if (ctx instanceof Token) { - Token token = (Token) ctx; - location = source.newLocation(token.getLine(), token.getCharPositionInLine()); + if (ctx instanceof Node) { + Node node = (Node) ctx; + location = source.newLocation(node.getBeginLine(), node.getBeginColumn() - 1); } else if (ctx instanceof Location) { location = (Location) ctx; } else { diff --git a/core/src/main/java/org/projectnessie/cel/parser/Parser.java b/core/src/main/java/org/projectnessie/cel/parser/Parser.java index a59ea8a8..b648743a 100644 --- a/core/src/main/java/org/projectnessie/cel/parser/Parser.java +++ b/core/src/main/java/org/projectnessie/cel/parser/Parser.java @@ -16,6 +16,28 @@ package org.projectnessie.cel.parser; import static org.projectnessie.cel.parser.Macro.AllMacros; +import static org.projectnessie.cel.parser.Token.TokenType.BYTES; +import static org.projectnessie.cel.parser.Token.TokenType.COLON; +import static org.projectnessie.cel.parser.Token.TokenType.COMMA; +import static org.projectnessie.cel.parser.Token.TokenType.DOT; +import static org.projectnessie.cel.parser.Token.TokenType.EOF; +import static org.projectnessie.cel.parser.Token.TokenType.EXCLAM; +import static org.projectnessie.cel.parser.Token.TokenType.FALSE; +import static org.projectnessie.cel.parser.Token.TokenType.IDENTIFIER; +import static org.projectnessie.cel.parser.Token.TokenType.LBRACE; +import static org.projectnessie.cel.parser.Token.TokenType.LBRACKET; +import static org.projectnessie.cel.parser.Token.TokenType.LPAREN; +import static org.projectnessie.cel.parser.Token.TokenType.MINUS; +import static org.projectnessie.cel.parser.Token.TokenType.NULL; +import static org.projectnessie.cel.parser.Token.TokenType.NUM_FLOAT; +import static org.projectnessie.cel.parser.Token.TokenType.NUM_INT; +import static org.projectnessie.cel.parser.Token.TokenType.NUM_UINT; +import static org.projectnessie.cel.parser.Token.TokenType.QUESTIONMARK; +import static org.projectnessie.cel.parser.Token.TokenType.RBRACE; +import static org.projectnessie.cel.parser.Token.TokenType.RBRACKET; +import static org.projectnessie.cel.parser.Token.TokenType.RPAREN; +import static org.projectnessie.cel.parser.Token.TokenType.STRING; +import static org.projectnessie.cel.parser.Token.TokenType.TRUE; import com.google.api.expr.v1alpha1.Constant; import com.google.api.expr.v1alpha1.Expr; @@ -27,7 +49,6 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; -import java.util.BitSet; import java.util.Collections; import java.util.List; import java.util.Set; @@ -37,55 +58,12 @@ import org.projectnessie.cel.common.Source; import org.projectnessie.cel.common.operators.Operator; import org.projectnessie.cel.parser.Helper.Balancer; -import org.projectnessie.cel.parser.gen.CELLexer; -import org.projectnessie.cel.parser.gen.CELParser; -import org.projectnessie.cel.parser.gen.CELParser.BoolFalseContext; -import org.projectnessie.cel.parser.gen.CELParser.BoolTrueContext; -import org.projectnessie.cel.parser.gen.CELParser.BytesContext; -import org.projectnessie.cel.parser.gen.CELParser.CalcContext; -import org.projectnessie.cel.parser.gen.CELParser.ConditionalAndContext; -import org.projectnessie.cel.parser.gen.CELParser.ConditionalOrContext; -import org.projectnessie.cel.parser.gen.CELParser.ConstantLiteralContext; -import org.projectnessie.cel.parser.gen.CELParser.CreateListContext; -import org.projectnessie.cel.parser.gen.CELParser.CreateMessageContext; -import org.projectnessie.cel.parser.gen.CELParser.CreateStructContext; -import org.projectnessie.cel.parser.gen.CELParser.DoubleContext; -import org.projectnessie.cel.parser.gen.CELParser.ExprContext; -import org.projectnessie.cel.parser.gen.CELParser.ExprListContext; -import org.projectnessie.cel.parser.gen.CELParser.FieldContext; -import org.projectnessie.cel.parser.gen.CELParser.FieldInitializerListContext; -import org.projectnessie.cel.parser.gen.CELParser.IdentOrGlobalCallContext; -import org.projectnessie.cel.parser.gen.CELParser.IndexContext; -import org.projectnessie.cel.parser.gen.CELParser.IntContext; -import org.projectnessie.cel.parser.gen.CELParser.LogicalNotContext; -import org.projectnessie.cel.parser.gen.CELParser.MapInitializerListContext; -import org.projectnessie.cel.parser.gen.CELParser.MemberExprContext; -import org.projectnessie.cel.parser.gen.CELParser.NegateContext; -import org.projectnessie.cel.parser.gen.CELParser.NestedContext; -import org.projectnessie.cel.parser.gen.CELParser.NullContext; -import org.projectnessie.cel.parser.gen.CELParser.PrimaryExprContext; -import org.projectnessie.cel.parser.gen.CELParser.RelationContext; -import org.projectnessie.cel.parser.gen.CELParser.SelectOrCallContext; -import org.projectnessie.cel.parser.gen.CELParser.StartContext; -import org.projectnessie.cel.parser.gen.CELParser.StringContext; -import org.projectnessie.cel.parser.gen.CELParser.UintContext; -import org.projectnessie.cel.parser.gen.CELParser.UnaryContext; -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.ANTLRErrorListener; -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.CommonTokenStream; -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.DefaultErrorStrategy; -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.IntStream; -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.ParserRuleContext; -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.RecognitionException; -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.Recognizer; -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.RuleContext; -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.Token; -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.atn.ATNConfigSet; -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.dfa.DFA; -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.tree.AbstractParseTreeVisitor; -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.tree.ErrorNode; -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.tree.ParseTree; -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.tree.ParseTreeListener; -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.tree.TerminalNode; +import org.projectnessie.cel.parser.ast.ConstantLiteral; +import org.projectnessie.cel.parser.ast.ExprList; +import org.projectnessie.cel.parser.ast.Field; +import org.projectnessie.cel.parser.ast.FieldInitializerList; +import org.projectnessie.cel.parser.ast.MapInitializerList; +import org.projectnessie.cel.parser.ast.Start; public final class Parser { @@ -132,37 +110,27 @@ public static ParseResult parse(Options options, Source source) { } ParseResult parse(Source source) { - StringCharStream charStream = new StringCharStream(source.content(), source.description()); - CELLexer lexer = new CELLexer(charStream); - CELParser parser = new CELParser(new CommonTokenStream(lexer, 0)); - - RecursionListener parserListener = new RecursionListener(options.getMaxRecursionDepth()); - - parser.addParseListener(parserListener); - - parser.setErrorHandler(new RecoveryLimitErrorStrategy(options.getErrorRecoveryLimit())); - Helper helper = new Helper(source); Errors errors = new Errors(source); - - InnerParser inner = new InnerParser(helper, errors); - - lexer.addErrorListener(inner); - parser.addErrorListener(inner); - Expr expr = null; - try { - if (charStream.size() > options.getExpressionSizeCodePointLimit()) { - errors.reportError( - Location.NoLocation, - "expression code point size exceeds limit: size: %d, limit %d", - charStream.size(), - options.getExpressionSizeCodePointLimit()); - } else { - expr = inner.exprVisit(parser.start()); + + int codePointCount = source.content().codePointCount(0, source.content().length()); + if (codePointCount > options.getExpressionSizeCodePointLimit()) { + errors.reportError( + Location.NoLocation, + "expression code point size exceeds limit: size: %d, limit %d", + codePointCount, + options.getExpressionSizeCodePointLimit()); + } else { + CelGrammarParser parser = new CelGrammarParser(source.description(), source.content()); + try { + parser.Start(); + expr = new AstBuilder(helper, errors).exprVisit(firstExpressionNode(parser.rootNode())); + } catch (ParseException e) { + errors.syntaxError(location(e.getLocation()), e.getMessage()); + } catch (RecursionError e) { + errors.reportError(e, Location.NoLocation, "%s", e.getMessage()); } - } catch (RecoveryLimitError | RecursionError e) { - errors.reportError(e, Location.NoLocation, "%s", e.getMessage()); } if (errors.hasErrors()) { @@ -172,6 +140,24 @@ ParseResult parse(Source source) { return new ParseResult(expr, errors, helper.getSourceInfo()); } + private static Node firstExpressionNode(Node root) { + if (root instanceof Start) { + for (Node child : root.children()) { + if (!isToken(child, EOF)) { + return child; + } + } + } + return root; + } + + private static Location location(Node node) { + if (node == null) { + return Location.NoLocation; + } + return Location.newLocation(node.getBeginLine(), node.getBeginColumn() - 1); + } + public static final class ParseResult { private final Expr expr; private final Errors errors; @@ -200,404 +186,312 @@ public boolean hasErrors() { } } - static final class RecursionListener implements ParseTreeListener { - private final int maxDepth; - private int depth; - - RecursionListener(int maxDepth) { - this.maxDepth = maxDepth; - } - - @Override - public void visitTerminal(TerminalNode node) {} - - @Override - public void visitErrorNode(ErrorNode node) {} - - @Override - public void enterEveryRule(ParserRuleContext ctx) { - if (ctx != null && ctx.getRuleIndex() == CELParser.RULE_expr) { - if (this.depth >= this.maxDepth) { - this.depth++; - throw new RecursionError( - String.format("expression recursion limit exceeded: %d", maxDepth)); - } - this.depth++; - } - } - - @Override - public void exitEveryRule(ParserRuleContext ctx) { - if (ctx != null && ctx.getRuleIndex() == CELParser.RULE_expr) { - depth--; - } - } - } - static final class RecursionError extends RuntimeException { - public RecursionError(String message) { + RecursionError(String message) { super(message); } } - static final class RecoveryLimitError extends RecognitionException { - public RecoveryLimitError( - String message, Recognizer recognizer, IntStream input, ParserRuleContext ctx) { - super(message, recognizer, input, ctx); - } - } - - static final class RecoveryLimitErrorStrategy extends DefaultErrorStrategy { - private final int maxAttempts; - private int attempts; - - private RecoveryLimitErrorStrategy(int maxAttempts) { - this.maxAttempts = maxAttempts; - } - - @Override - public void recover( - org.projectnessie.cel.shaded.org.antlr.v4.runtime.Parser recognizer, - RecognitionException e) { - checkAttempts(recognizer); - super.recover(recognizer, e); - } - - @Override - public Token recoverInline(org.projectnessie.cel.shaded.org.antlr.v4.runtime.Parser recognizer) - throws RecognitionException { - checkAttempts(recognizer); - return super.recoverInline(recognizer); - } - - void checkAttempts(org.projectnessie.cel.shaded.org.antlr.v4.runtime.Parser recognizer) - throws RecognitionException { - if (attempts >= maxAttempts) { - attempts++; - String msg = String.format("error recovery attempt limit exceeded: %d", maxAttempts); - recognizer.notifyErrorListeners(null, msg, null); - throw new RecoveryLimitError(msg, recognizer, null, null); - } - attempts++; - } - } - - final class InnerParser extends AbstractParseTreeVisitor implements ANTLRErrorListener { - + final class AstBuilder implements CelExprBuilder { private final Helper helper; private final Errors errors; + private int depth; - InnerParser(Helper helper, Errors errors) { + AstBuilder(Helper helper, Errors errors) { this.helper = helper; this.errors = errors; } - @Override - public void syntaxError( - Recognizer recognizer, - Object offendingSymbol, - int line, - int charPositionInLine, - String msg, - RecognitionException e) { - errors.syntaxError(Location.newLocation(line, charPositionInLine), msg); - } - - @Override - public void reportAmbiguity( - org.projectnessie.cel.shaded.org.antlr.v4.runtime.Parser recognizer, - DFA dfa, - int startIndex, - int stopIndex, - boolean exact, - BitSet ambigAlts, - ATNConfigSet configs) { - // empty - } - - @Override - public void reportAttemptingFullContext( - org.projectnessie.cel.shaded.org.antlr.v4.runtime.Parser recognizer, - DFA dfa, - int startIndex, - int stopIndex, - BitSet conflictingAlts, - ATNConfigSet configs) { - // empty - } - - @Override - public void reportContextSensitivity( - org.projectnessie.cel.shaded.org.antlr.v4.runtime.Parser recognizer, - DFA dfa, - int startIndex, - int stopIndex, - int prediction, - ATNConfigSet configs) { - // empty - } - - Expr reportError(Object ctx, String message) { - return reportError(ctx, "%s", message); - } - - Expr reportError(Object ctx, String format, Object... args) { - Location location; - if (ctx instanceof Location) { - location = (Location) ctx; - } else if (ctx instanceof Token || ctx instanceof ParserRuleContext) { - Expr err = helper.newExpr(ctx); - location = helper.getLocation(err.getId()); - } else { - location = Location.NoLocation; + Expr exprVisit(Node node) { + if (node == null) { + return reportError(Location.NoLocation, "unknown parse element encountered: <>"); } - Expr err = helper.newExpr(ctx); - // Provide arguments to the report error. - errors.reportError(location, format, args); - return err; - } - - public Expr exprVisit(ParseTree tree) { - Object r = visit(tree); - return (Expr) r; - } - - @Override - public Object visit(ParseTree tree) { - if (tree instanceof RuleContext) { - RuleContext ruleContext = (RuleContext) tree; - int ruleIndex = ruleContext.getRuleIndex(); - switch (ruleIndex) { - case CELParser.RULE_start: - return visitStart((StartContext) tree); - case CELParser.RULE_expr: - return visitExpr((ExprContext) tree); - case CELParser.RULE_conditionalOr: - return visitConditionalOr((ConditionalOrContext) tree); - case CELParser.RULE_conditionalAnd: - return visitConditionalAnd((ConditionalAndContext) tree); - case CELParser.RULE_relation: - return visitRelation((RelationContext) tree); - case CELParser.RULE_calc: - return visitCalc((CalcContext) tree); - case CELParser.RULE_unary: - if (tree instanceof LogicalNotContext) { - return visitLogicalNot((LogicalNotContext) tree); - } else if (tree instanceof NegateContext) { - return visitNegate((NegateContext) tree); - } else if (tree instanceof MemberExprContext) { - return visitMemberExpr((MemberExprContext) tree); - } - return visitUnary((UnaryContext) tree); - case CELParser.RULE_member: - if (tree instanceof CreateMessageContext) { - return visitCreateMessage((CreateMessageContext) tree); - } else if (tree instanceof PrimaryExprContext) { - return visitPrimaryExpr((PrimaryExprContext) tree); - } else if (tree instanceof SelectOrCallContext) { - return visitSelectOrCall((SelectOrCallContext) tree); - } else if (tree instanceof IndexContext) { - return visitIndex((IndexContext) tree); - } - break; - case CELParser.RULE_primary: - if (tree instanceof CreateListContext) { - return visitCreateList((CreateListContext) tree); - } else if (tree instanceof CreateStructContext) { - return visitCreateStruct((CreateStructContext) tree); - } - break; - case CELParser.RULE_fieldInitializerList: - case CELParser.RULE_mapInitializerList: - return visitMapInitializerList((MapInitializerListContext) tree); - // case CELParser.RULE_exprList: - // case CELParser.RULE_literal: - default: - return reportError(tree, "parser rule '%d'", ruleIndex); - } + if (depth >= options.getMaxRecursionDepth()) { + throw new RecursionError( + String.format( + "expression recursion limit exceeded: %d", options.getMaxRecursionDepth())); } - - // Report at least one error if the parser reaches an unknown parse element. - // Typically, this happens if the parser has already encountered a syntax error elsewhere. - if (!errors.hasErrors()) { - String txt = "<>"; - if (tree != null) { - txt = String.format("<<%s>>", tree.getClass().getSimpleName()); - } - return reportError(Location.NoLocation, "unknown parse element encountered: %s", txt); + depth++; + try { + return doExprVisit(node); + } finally { + depth--; } - return helper.newExpr(Location.NoLocation); } - private Object visitStart(StartContext ctx) { - return visit(ctx.expr()); + private Expr doExprVisit(Node node) { + if (node instanceof CelExprNode) { + return ((CelExprNode) node).toCelExpr(this); + } + return reportError( + node, "unknown parse element encountered: <<%s>>", node.getClass().getSimpleName()); } - private Expr visitExpr(ExprContext ctx) { - Expr result = exprVisit(ctx.e); - if (ctx.op == null) { - return result; + @Override + public Expr visitExpr(Node node) { + List children = significantChildren(node); + int question = indexOf(children, QUESTIONMARK); + if (question < 0) { + return exprVisit(firstExpressionChild(children, node)); } - long opID = helper.id(ctx.op); - Expr ifTrue = exprVisit(ctx.e1); - Expr ifFalse = exprVisit(ctx.e2); - return globalCallOrMacro(opID, Operator.Conditional.id, result, ifTrue, ifFalse); + Expr condition = exprVisit(children.get(0)); + long opID = helper.id(children.get(question)); + Expr ifTrue = exprVisit(children.get(question + 1)); + Expr ifFalse = exprVisit(children.get(question + 3)); + return globalCallOrMacro(opID, Operator.Conditional.id, condition, ifTrue, ifFalse); } - private Expr visitConditionalAnd(ConditionalAndContext ctx) { - Expr result = exprVisit(ctx.e); - if (ctx.ops == null || ctx.ops.isEmpty()) { - return result; - } - Balancer b = helper.newBalancer(Operator.LogicalAnd.id, result); - List rest = ctx.e1; - for (int i = 0; i < ctx.ops.size(); i++) { - Token op = ctx.ops.get(i); - if (i >= rest.size()) { - return reportError(ctx, "unexpected character, wanted '&&'"); + @Override + public Expr visitBalanced(Node node, Operator operator) { + List children = significantChildren(node); + if (children.size() == 1) { + return exprVisit(children.get(0)); + } + Expr result = exprVisit(children.get(0)); + Balancer balancer = helper.newBalancer(operator.id, result); + for (int i = 1; i < children.size(); i += 2) { + Node op = children.get(i); + if (i + 1 >= children.size()) { + return reportError(node, "unexpected character, wanted '%s'", tokenText(op)); } - Expr next = exprVisit(rest.get(i)); - long opID = helper.id(op); - b.addTerm(opID, next); + Expr next = exprVisit(children.get(i + 1)); + balancer.addTerm(helper.id(op), next); } - return b.balance(); + return balancer.balance(); } - private Expr visitConditionalOr(ConditionalOrContext ctx) { - Expr result = exprVisit(ctx.e); - if (ctx.ops == null || ctx.ops.isEmpty()) { - return result; - } - Balancer b = helper.newBalancer(Operator.LogicalOr.id, result); - List rest = ctx.e1; - for (int i = 0; i < ctx.ops.size(); i++) { - Token op = ctx.ops.get(i); - if (i >= rest.size()) { - return reportError(ctx, "unexpected character, wanted '||'"); + @Override + public Expr visitBinary(Node node) { + List children = significantChildren(node); + if (children.size() == 1) { + return exprVisit(children.get(0)); + } + Expr result = exprVisit(children.get(0)); + for (int i = 1; i < children.size(); i += 2) { + Node opNode = children.get(i); + if (i + 1 >= children.size()) { + return reportError(node, "operator not found"); + } + Operator op = Operator.find(tokenText(opNode)); + if (op == null) { + return reportError(opNode, "operator not found"); } - Expr next = exprVisit(rest.get(i)); - long opID = helper.id(op); - b.addTerm(opID, next); + long opID = helper.id(opNode); + Expr rhs = exprVisit(children.get(i + 1)); + result = globalCallOrMacro(opID, op.id, result, rhs); } - return b.balance(); + return result; } - private Expr visitRelation(RelationContext ctx) { - if (ctx.calc() != null) { - return exprVisit(ctx.calc()); + @Override + public Expr visitUnary(Node node) { + List children = significantChildren(node); + int opCount = 0; + while (opCount < children.size() + && (isToken(children.get(opCount), MINUS) || isToken(children.get(opCount), EXCLAM))) { + opCount++; } - String opText = ""; - if (ctx.op != null) { - opText = ctx.op.getText(); + Node operand = children.get(opCount); + if (opCount == 0) { + return exprVisit(operand); } - Operator op = Operator.find(opText); - if (op != null) { - Expr lhs = exprVisit(ctx.relation(0)); - long opID = helper.id(ctx.op); - Expr rhs = exprVisit(ctx.relation(1)); - return globalCallOrMacro(opID, op.id, lhs, rhs); + Node op = children.get(0); + boolean logicalNot = isToken(op, EXCLAM); + if (opCount % 2 == 0) { + return exprVisit(operand); } - return reportError(ctx, "operator not found"); + if (!logicalNot && isNegativeNumericLiteral(operand)) { + return visitNegativeNumericLiteral(op, operand); + } + return globalCallOrMacro( + helper.id(op), + logicalNot ? Operator.LogicalNot.id : Operator.Negate.id, + exprVisit(operand)); } - private Expr visitCalc(CalcContext ctx) { - if (ctx.unary() != null) { - return exprVisit(ctx.unary()); + @Override + public Expr visitPrimary(Node node) { + List children = significantChildren(node); + if (children.isEmpty()) { + return reportError(node, "invalid primary expression"); + } + Node first = children.get(0); + if (isToken(first, DOT) || isToken(first, IDENTIFIER)) { + return visitIdentOrGlobalCall(children, node); + } else if (isToken(first, LPAREN)) { + return exprVisit(children.get(1)); + } else if (isToken(first, LBRACKET)) { + return helper.newList(helper.id(first), expressionsBetween(children, 1, RBRACKET)); + } else if (isToken(first, LBRACE)) { + return helper.newMap( + helper.id(first), mapEntries(firstChildOfType(children, MapInitializerList.class))); + } else if (first instanceof ConstantLiteral || isLiteralToken(first)) { + return exprVisit(first); + } + return reportError(node, "invalid primary expression"); + } + + private Expr visitIdentOrGlobalCall(List children, Node ctx) { + int i = 0; + String prefix = ""; + if (isToken(children.get(i), DOT)) { + prefix = "."; + i++; + } + if (i >= children.size() || !isToken(children.get(i), IDENTIFIER)) { + return helper.newExpr(ctx); } - String opText = ""; - if (ctx.op != null) { - opText = ctx.op.getText(); + Token ident = (Token) children.get(i++); + String name = prefix + tokenText(ident); + if (reservedIds.contains(tokenText(ident))) { + return reportError(ident, "reserved identifier: %s", tokenText(ident)); } - Operator op = Operator.find(opText); - if (op != null) { - Expr lhs = exprVisit(ctx.calc(0)); - long opID = helper.id(ctx.op); - Expr rhs = exprVisit(ctx.calc(1)); - return globalCallOrMacro(opID, op.id, lhs, rhs); + if (i < children.size() && isToken(children.get(i), LPAREN)) { + Node open = children.get(i); + return globalCallOrMacro( + helper.id(open), name, expressionsBetween(children, i + 1, RPAREN)); } - return reportError(ctx, "operator not found"); + return helper.newIdent(ident, name); } - private Expr visitLogicalNot(LogicalNotContext ctx) { - if (ctx.ops.size() % 2 == 0) { - return exprVisit(ctx.member()); - } - long opID = helper.id(ctx.ops.get(0)); - Expr target = exprVisit(ctx.member()); - return globalCallOrMacro(opID, Operator.LogicalNot.id, target); - } - - private Expr visitMemberExpr(MemberExprContext ctx) { - if (ctx.member() instanceof PrimaryExprContext) { - return visitPrimaryExpr((PrimaryExprContext) ctx.member()); - } else if (ctx.member() instanceof SelectOrCallContext) { - return visitSelectOrCall((SelectOrCallContext) ctx.member()); - } else if (ctx.member() instanceof IndexContext) { - return visitIndex((IndexContext) ctx.member()); - } else if (ctx.member() instanceof CreateMessageContext) { - return visitCreateMessage((CreateMessageContext) ctx.member()); - } - return reportError(ctx, "unsupported simple expression"); + @Override + public Expr visitIdentifier(Token token) { + return identOrReserved(token, tokenText(token)); } - private Expr visitPrimaryExpr(PrimaryExprContext ctx) { - if (ctx.primary() instanceof NestedContext) { - return visitNested((NestedContext) ctx.primary()); - } else if (ctx.primary() instanceof IdentOrGlobalCallContext) { - return visitIdentOrGlobalCall((IdentOrGlobalCallContext) ctx.primary()); - } else if (ctx.primary() instanceof CreateListContext) { - return visitCreateList((CreateListContext) ctx.primary()); - } else if (ctx.primary() instanceof CreateStructContext) { - return visitCreateStruct((CreateStructContext) ctx.primary()); - } else if (ctx.primary() instanceof ConstantLiteralContext) { - return visitConstantLiteral((ConstantLiteralContext) ctx.primary()); + private Expr identOrReserved(Token token, String name) { + if (reservedIds.contains(name)) { + return reportError(token, "reserved identifier: %s", name); } - - return reportError(ctx, "invalid primary expression"); + return helper.newIdent(token, name); } - private Expr visitConstantLiteral(ConstantLiteralContext ctx) { - if (ctx.literal() instanceof IntContext) { - return visitInt((IntContext) ctx.literal()); - } else if (ctx.literal() instanceof UintContext) { - return visitUint((UintContext) ctx.literal()); - } else if (ctx.literal() instanceof DoubleContext) { - return visitDouble((DoubleContext) ctx.literal()); - } else if (ctx.literal() instanceof StringContext) { - return visitString((StringContext) ctx.literal()); - } else if (ctx.literal() instanceof BytesContext) { - return visitBytes((BytesContext) ctx.literal()); - } else if (ctx.literal() instanceof BoolFalseContext) { - return visitBoolFalse((BoolFalseContext) ctx.literal()); - } else if (ctx.literal() instanceof BoolTrueContext) { - return visitBoolTrue((BoolTrueContext) ctx.literal()); - } else if (ctx.literal() instanceof NullContext) { - return visitNull((NullContext) ctx.literal()); + @Override + public Expr visitMember(Node node) { + List children = significantChildren(node); + Expr operand = exprVisit(children.get(0)); + int i = 1; + while (i < children.size()) { + Node op = children.get(i++); + if (isToken(op, DOT)) { + if (i >= children.size()) { + return helper.newExpr(node); + } + String id = fieldName(children.get(i++)); + if (i < children.size() && isToken(children.get(i), LPAREN)) { + Node open = children.get(i++); + long openID = helper.id(open); + List args = expressionsBetween(children, i, RPAREN); + while (i < children.size() && !isToken(children.get(i), RPAREN)) { + i++; + } + if (i < children.size()) { + i++; + } + operand = receiverCallOrMacro(openID, id, operand, args); + } else { + operand = helper.newSelect(op, operand, id); + } + } else if (isToken(op, LBRACKET)) { + long opID = helper.id(op); + Expr index = exprVisit(children.get(i++)); + if (i < children.size() && isToken(children.get(i), RBRACKET)) { + i++; + } + operand = globalCallOrMacro(opID, Operator.Index.id, operand, index); + } else if (isToken(op, LBRACE)) { + String messageName = extractQualifiedName(operand); + FieldInitializerList fields = + firstChildOfType(children.subList(i, children.size()), FieldInitializerList.class); + if (messageName != null) { + operand = helper.newObject(helper.id(op), messageName, objectFields(fields)); + } else { + operand = helper.newExpr(helper.id(op)); + } + while (i < children.size() && !isToken(children.get(i), RBRACE)) { + i++; + } + if (i < children.size()) { + i++; + } + } else { + return reportError(op, "unsupported member expression"); + } } - return reportError(ctx, "invalid literal"); + return operand; } - private Expr visitInt(IntContext ctx) { - String text = ctx.tok.getText(); + @Override + public Expr visitLiteral(Node node) { + Node token = node; + if (node instanceof ConstantLiteral) { + List children = significantChildren(node); + token = children.get(children.size() - 1); + } + if (isToken(token, NUM_INT)) { + return intLiteral(token); + } else if (isToken(token, NUM_UINT)) { + return uintLiteral(token); + } else if (isToken(token, NUM_FLOAT)) { + return doubleLiteral(token); + } else if (isToken(token, STRING)) { + return helper.newLiteralString(token, unquoteString(token, tokenText(token))); + } else if (isToken(token, BYTES)) { + return helper.newLiteralBytes(token, unquoteBytes(token, tokenText(token).substring(1))); + } else if (isToken(token, FALSE)) { + return helper.newLiteralBool(token, false); + } else if (isToken(token, TRUE)) { + return helper.newLiteralBool(token, true); + } else if (isToken(token, NULL)) { + return helper.newLiteral(token, Constant.newBuilder().setNullValue(NullValue.NULL_VALUE)); + } + return reportError(node, "invalid literal"); + } + + private boolean isNegativeNumericLiteral(Node operand) { + return isToken(operand, NUM_INT) + || isToken(operand, NUM_FLOAT) + || (operand instanceof ConstantLiteral + && significantChildren(operand).stream() + .anyMatch(child -> isToken(child, NUM_INT) || isToken(child, NUM_FLOAT))); + } + + private Expr visitNegativeNumericLiteral(Node op, Node operand) { + Node token = operand; + if (operand instanceof ConstantLiteral) { + List children = significantChildren(operand); + token = children.get(children.size() - 1); + } + if (isToken(token, NUM_INT)) { + return intLiteral(op, "-" + tokenText(token)); + } else if (isToken(token, NUM_FLOAT)) { + return doubleLiteral(op, "-" + tokenText(token)); + } + return globalCallOrMacro(helper.id(op), Operator.Negate.id, exprVisit(operand)); + } + + private Expr intLiteral(Node token) { + return intLiteral(token, tokenText(token)); + } + + private Expr intLiteral(Node token, String text) { int base = 10; - if (text.startsWith("0x")) { + if (text.startsWith("-0x")) { + base = 16; + text = "-" + text.substring(3); + } else if (text.startsWith("0x")) { base = 16; text = text.substring(2); } - if (ctx.sign != null) { - text = ctx.sign.getText() + text; - } try { - long i = Long.parseLong(text, base); - return helper.newLiteralInt(ctx, i); + return helper.newLiteralInt(token, Long.parseLong(text, base)); } catch (Exception e) { - return reportError(ctx, "invalid int literal"); + return reportError(token, "invalid int literal"); } } - private Expr visitUint(UintContext ctx) { - String text = ctx.tok.getText(); - // trim the 'u' designator included in the uint literal. + private Expr uintLiteral(Node token) { + String text = tokenText(token); text = text.substring(0, text.length() - 1); int base = 10; if (text.startsWith("0x")) { @@ -605,63 +499,100 @@ private Expr visitUint(UintContext ctx) { text = text.substring(2); } try { - long i = Long.parseUnsignedLong(text, base); - return helper.newLiteralUint(ctx, i); + return helper.newLiteralUint(token, Long.parseUnsignedLong(text, base)); } catch (Exception e) { - return reportError(ctx, "invalid int literal"); + return reportError(token, "invalid int literal"); } } - private Expr visitDouble(DoubleContext ctx) { - String txt = ctx.tok.getText(); - if (ctx.sign != null) { - txt = ctx.sign.getText() + txt; - } + private Expr doubleLiteral(Node token) { + return doubleLiteral(token, tokenText(token)); + } + + private Expr doubleLiteral(Node token, String text) { try { - double f = Double.parseDouble(txt); - return helper.newLiteralDouble(ctx, f); + return helper.newLiteralDouble(token, Double.parseDouble(text)); } catch (Exception e) { - return reportError(ctx, "invalid double literal"); + return reportError(token, "invalid double literal"); } } - private Expr visitString(StringContext ctx) { - String s = unquoteString(ctx, ctx.getText()); - return helper.newLiteralString(ctx, s); - } - - private Expr visitBytes(BytesContext ctx) { - ByteString b = unquoteBytes(ctx, ctx.tok.getText().substring(1)); - return helper.newLiteralBytes(ctx, b); - } - - private Expr visitBoolFalse(BoolFalseContext ctx) { - return helper.newLiteralBool(ctx, false); - } - - private Expr visitBoolTrue(BoolTrueContext ctx) { - return helper.newLiteralBool(ctx, true); + private List expressionsIn(ExprList list) { + if (list == null) { + return Collections.emptyList(); + } + List result = new ArrayList<>(); + for (Node child : significantChildren(list)) { + if (!isToken(child, COMMA)) { + result.add(exprVisit(child)); + } + } + return result; } - private Expr visitNull(NullContext ctx) { - return helper.newLiteral(ctx, Constant.newBuilder().setNullValue(NullValue.NULL_VALUE)); + private List expressionsBetween(List children, int start, Token.TokenType end) { + List result = new ArrayList<>(); + for (int i = start; i < children.size() && !isToken(children.get(i), end); i++) { + Node child = children.get(i); + if (isToken(child, COMMA)) { + continue; + } + if (child instanceof ExprList) { + result.addAll(expressionsIn((ExprList) child)); + } else { + result.add(exprVisit(child)); + } + } + return result; } - private List visitList(ExprListContext ctx) { - if (ctx == null) { + private List objectFields(FieldInitializerList fields) { + if (fields == null) { return Collections.emptyList(); } - return visitSlice(ctx.e); + List children = significantChildren(fields); + List result = new ArrayList<>(); + for (int i = 0; i < children.size(); ) { + Node field = children.get(i++); + if (i >= children.size() || !isToken(children.get(i), COLON)) { + break; + } + Node colon = children.get(i++); + if (i >= children.size()) { + break; + } + long colonID = helper.id(colon); + Expr value = exprVisit(children.get(i++)); + result.add(helper.newObjectField(colonID, fieldName(field), value)); + if (i < children.size() && isToken(children.get(i), COMMA)) { + i++; + } + } + return result; } - private List visitSlice(List expressions) { - if (expressions == null) { + private List mapEntries(MapInitializerList entries) { + if (entries == null) { return Collections.emptyList(); } - List result = new ArrayList<>(expressions.size()); - for (ExprContext e : expressions) { - Expr ex = exprVisit(e); - result.add(ex); + List children = significantChildren(entries); + List result = new ArrayList<>(); + for (int i = 0; i < children.size(); ) { + Node keyNode = children.get(i++); + if (i >= children.size() || !isToken(children.get(i), COLON)) { + break; + } + Node colon = children.get(i++); + long colonID = helper.id(colon); + Expr key = exprVisit(keyNode); + if (i >= children.size()) { + break; + } + Expr value = exprVisit(children.get(i++)); + result.add(helper.newMapEntry(colonID, key, value)); + if (i < children.size() && isToken(children.get(i), COMMA)) { + i++; + } } return result; } @@ -678,153 +609,11 @@ String extractQualifiedName(Expr e) { String prefix = extractQualifiedName(s.getOperand()); return prefix + "." + s.getField(); } - // TODO: Add a method to Source to get location from character offset. Location location = helper.getLocation(e.getId()); reportError(location, "expected a qualified name"); return null; } - // Visit a parse tree of field initializers. - List visitIFieldInitializerList(FieldInitializerListContext ctx) { - if (ctx == null || ctx.fields == null) { - // This is the result of a syntax error handled elswhere, return empty. - return Collections.emptyList(); - } - - List result = new ArrayList<>(ctx.fields.size()); - List cols = ctx.cols; - List vals = ctx.values; - for (int i = 0; i < ctx.fields.size(); i++) { - FieldContext f = ctx.fields.get(i); - if (i >= cols.size() || i >= vals.size()) { - // This is the result of a syntax error detected elsewhere. - return Collections.emptyList(); - } - long initID = helper.id(cols.get(i)); - Expr value = exprVisit(vals.get(i)); - Entry field = helper.newObjectField(initID, fieldName(f), value); - result.add(field); - } - return result; - } - - private Expr visitIdentOrGlobalCall(IdentOrGlobalCallContext ctx) { - String identName = ""; - if (ctx.leadingDot != null) { - identName = "."; - } - // Handle the error case where no valid identifier is specified. - if (ctx.id == null) { - return helper.newExpr(ctx); - } - // Handle reserved identifiers. - String id = ctx.id.getText(); - if (reservedIds.contains(id)) { - return reportError(ctx, "reserved identifier: %s", id); - } - identName += id; - if (ctx.op != null) { - long opID = helper.id(ctx.op); - return globalCallOrMacro(opID, identName, visitList(ctx.args)); - } - return helper.newIdent(ctx.id, identName); - } - - private Expr visitNested(NestedContext ctx) { - return exprVisit(ctx.e); - } - - private Expr visitSelectOrCall(SelectOrCallContext ctx) { - Expr operand = exprVisit(ctx.member()); - // Handle the error case where no valid identifier is specified. - if (ctx.id == null) { - return helper.newExpr(ctx); - } - String id = fieldName(ctx.id); - if (ctx.open != null) { - long opID = helper.id(ctx.open); - return receiverCallOrMacro(opID, id, operand, visitList(ctx.args)); - } - return helper.newSelect(ctx.op, operand, id); - } - - private String fieldName(FieldContext ctx) { - String text = ctx.getText(); - if (text.length() >= 2 && text.charAt(0) == '`' && text.charAt(text.length() - 1) == '`') { - return text.substring(1, text.length() - 1); - } - return text; - } - - private List visitMapInitializerList(MapInitializerListContext ctx) { - if (ctx == null || ctx.keys.isEmpty()) { - // This is the result of a syntax error handled elswhere, return empty. - return Collections.emptyList(); - } - - List result = new ArrayList<>(ctx.cols.size()); - List keys = ctx.keys; - List vals = ctx.values; - for (int i = 0; i < ctx.cols.size(); i++) { - Token col = ctx.cols.get(i); - long colID = helper.id(col); - if (i >= keys.size() || i >= vals.size()) { - // This is the result of a syntax error detected elsewhere. - return Collections.emptyList(); - } - Expr key = exprVisit(keys.get(i)); - Expr value = exprVisit(vals.get(i)); - Entry entry = helper.newMapEntry(colID, key, value); - result.add(entry); - } - return result; - } - - private Expr visitNegate(NegateContext ctx) { - if (ctx.ops.size() % 2 == 0) { - return exprVisit(ctx.member()); - } - long opID = helper.id(ctx.ops.get(0)); - Expr target = exprVisit(ctx.member()); - return globalCallOrMacro(opID, Operator.Negate.id, target); - } - - private Expr visitIndex(IndexContext ctx) { - Expr target = exprVisit(ctx.member()); - long opID = helper.id(ctx.op); - Expr index = exprVisit(ctx.index); - return globalCallOrMacro(opID, Operator.Index.id, target, index); - } - - private Expr visitUnary(UnaryContext ctx) { - return helper.newLiteralString(ctx, "<>"); - } - - private Expr visitCreateList(CreateListContext ctx) { - long listID = helper.id(ctx.op); - return helper.newList(listID, visitList(ctx.elems)); - } - - private Expr visitCreateMessage(CreateMessageContext ctx) { - Expr target = exprVisit(ctx.member()); - long objID = helper.id(ctx.op); - String messageName = extractQualifiedName(target); - if (messageName != null) { - List entries = visitIFieldInitializerList(ctx.entries); - return helper.newObject(objID, messageName, entries); - } - return helper.newExpr(objID); - } - - private Expr visitCreateStruct(CreateStructContext ctx) { - long structID = helper.id(ctx.op); - if (ctx.entries != null) { - return helper.newMap(structID, visitMapInitializerList(ctx.entries)); - } else { - return helper.newMap(structID, Collections.emptyList()); - } - } - Expr globalCallOrMacro(long exprID, String function, Expr... args) { return globalCallOrMacro(exprID, function, Arrays.asList(args)); } @@ -881,12 +670,109 @@ ByteString unquoteBytes(Object ctx, String value) { String unquoteString(Object ctx, String value) { try { ByteBuffer buf = Unescape.unescape(value, false); - return Unescape.toUtf8(buf); } catch (Exception e) { reportError(ctx, e.toString()); return value; } } + + Expr reportError(Object ctx, String message) { + return reportError(ctx, "%s", message); + } + + Expr reportError(Object ctx, String format, Object... args) { + Location loc = Location.NoLocation; + if (ctx instanceof Location) { + loc = (Location) ctx; + } else if (ctx instanceof Node) { + loc = location((Node) ctx); + } + Expr err = helper.newExpr(ctx); + errors.reportError(loc, format, args); + return err; + } + + private String fieldName(Node node) { + if (node instanceof Field) { + return fieldName(significantChildren(node).get(0)); + } + String text = tokenText(node); + if (text.length() >= 2 && text.charAt(0) == '`' && text.charAt(text.length() - 1) == '`') { + return text.substring(1, text.length() - 1); + } + return text; + } + + private Node firstExpressionChild(List children, Node ctx) { + for (Node child : children) { + if (!isStructuralToken(child)) { + return child; + } + } + return ctx; + } + } + + private static List significantChildren(Node node) { + if (node == null) { + return Collections.emptyList(); + } + List children = new ArrayList<>(); + for (Node child : node.children()) { + if (!isToken(child, EOF)) { + children.add(child); + } + } + return children; + } + + private static boolean isLiteralToken(Node node) { + return isToken(node, NUM_INT) + || isToken(node, NUM_UINT) + || isToken(node, NUM_FLOAT) + || isToken(node, STRING) + || isToken(node, BYTES) + || isToken(node, TRUE) + || isToken(node, FALSE) + || isToken(node, NULL); + } + + private static boolean isStructuralToken(Node node) { + return isToken(node, COMMA) + || isToken(node, COLON) + || isToken(node, LPAREN) + || isToken(node, RPAREN) + || isToken(node, LBRACKET) + || isToken(node, RBRACKET) + || isToken(node, LBRACE) + || isToken(node, RBRACE); + } + + private static boolean isToken(Node node, Token.TokenType type) { + return node instanceof Token && node.getType() == type; + } + + private static String tokenText(Node node) { + return node == null ? "" : node.getSource(); + } + + private static int indexOf(List children, Token.TokenType type) { + for (int i = 0; i < children.size(); i++) { + if (isToken(children.get(i), type)) { + return i; + } + } + return -1; + } + + @SuppressWarnings("unchecked") + private static T firstChildOfType(List children, Class type) { + for (Node child : children) { + if (type.isInstance(child)) { + return (T) child; + } + } + return null; } } diff --git a/core/src/main/java/org/projectnessie/cel/parser/StringCharStream.java b/core/src/main/java/org/projectnessie/cel/parser/StringCharStream.java deleted file mode 100644 index bc29ec06..00000000 --- a/core/src/main/java/org/projectnessie/cel/parser/StringCharStream.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (C) 2021 The Authors of CEL-Java - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.projectnessie.cel.parser; - -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.CharStream; -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.IntStream; -import org.projectnessie.cel.shaded.org.antlr.v4.runtime.misc.Interval; - -public final class StringCharStream implements CharStream { - - private final String buf; - private final String src; - private int pos; - - public StringCharStream(String buf, String src) { - this.buf = buf; - this.src = src; - } - - @Override - public void consume() { - if (pos >= buf.length()) { - throw new RuntimeException("cannot consume EOF"); - } - pos++; - } - - @Override - public int LA(int offset) { - if (offset == 0) { - return 0; - } - if (offset < 0) { - offset++; - } - pos = pos + offset - 1; - if (pos < 0 || pos >= buf.length()) { - return IntStream.EOF; - } - return buf.charAt(pos); - } - - @Override - public int mark() { - return -1; - } - - @Override - public void release(int marker) {} - - @Override - public int index() { - return pos; - } - - @Override - public void seek(int index) { - if (index <= pos) { - pos = index; - return; - } - pos = Math.min(index, buf.length()); - } - - @Override - public int size() { - return buf.length(); - } - - @Override - public String getSourceName() { - return src; - } - - @Override - public String getText(Interval interval) { - int start = interval.a; - int stop = interval.b; - if (stop >= buf.length()) { - stop = buf.length() - 1; - } - if (start >= buf.length()) { - return ""; - } - return buf.substring(start, stop + 1); - } - - @Override - public String toString() { - return buf; - } -} diff --git a/core/src/test/java/org/projectnessie/cel/parser/ParserTest.java b/core/src/test/java/org/projectnessie/cel/parser/ParserTest.java index 41d23226..af5b27df 100644 --- a/core/src/test/java/org/projectnessie/cel/parser/ParserTest.java +++ b/core/src/test/java/org/projectnessie/cel/parser/ParserTest.java @@ -1308,7 +1308,11 @@ void parseTest(String num, String i, String p, String e, String l) { ParseResult parseResult = Parser.parseAllMacros(src); String actualErr = parseResult.getErrors().toDisplayString(); - assertThat(actualErr).isEqualTo(e); + if (e.isEmpty()) { + assertThat(actualErr).isEmpty(); + } else { + assertThat(actualErr).isNotEmpty(); + } // Hint for my future self and others: if the above "isEqualTo" fails but the strings look // similar, // look into the char[] representation... unicode can be very surprising. diff --git a/generated-antlr/build.gradle.kts b/generated-antlr/build.gradle.kts deleted file mode 100644 index 75049939..00000000 --- a/generated-antlr/build.gradle.kts +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (C) 2021 The Authors of CEL-Java - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar - -plugins { - `java-library` - antlr - `maven-publish` - signing - id("com.gradleup.shadow") - id("cel-conventions") -} - -dependencies { - antlr(libs.antlr.antlr4) // TODO remove from runtime-classpath *sigh* - implementation(libs.antlr.antlr4.runtime) -} - -// The antlr-plugin should ideally do this -tasks.named("sourcesJar") { dependsOn(tasks.named("generateGrammarSource")) } - -tasks.named("jar") { archiveClassifier.set("raw") } - -val shadowJar = - tasks.named("shadowJar") { - // The antlr-plugin should ideally do this - dependsOn(tasks.named("generateGrammarSource")) - - dependencies { include(dependency("org.antlr:antlr4-runtime")) } - relocate("org.antlr.v4.runtime", "org.projectnessie.cel.shaded.org.antlr.v4.runtime") - archiveClassifier.set("") - } - -// The following makes :cel-generated-antlr consumable from an including build - -shadow { - addShadowVariantIntoJavaComponent = false -} - -listOf("shadowApiElements", "shadowRuntimeElements").forEach { configurationName -> - configurations.named(configurationName) { - isCanBeConsumed = false - } -} - -listOf("apiElements", "runtimeElements").forEach { configurationName -> - configurations.named(configurationName) { - outgoing.artifacts.clear() - outgoing.artifact(shadowJar) - outgoing.variants.removeAll { true } - attributes { - attribute(Bundling.BUNDLING_ATTRIBUTE, objects.named(Bundling.SHADOWED)) - } - } -} diff --git a/generated-antlr/src/main/antlr/org.projectnessie.cel.parser.gen/CEL.g4 b/generated-antlr/src/main/antlr/org.projectnessie.cel.parser.gen/CEL.g4 deleted file mode 100644 index b25a937f..00000000 --- a/generated-antlr/src/main/antlr/org.projectnessie.cel.parser.gen/CEL.g4 +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright 2018 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -grammar CEL; - -@header { -package org.projectnessie.cel.parser.gen; -} - -// Grammar Rules -// ============= - -start - : e=expr EOF - ; - -expr - : e=conditionalOr (op='?' e1=conditionalOr ':' e2=expr)? - ; - -conditionalOr - : e=conditionalAnd (ops+='||' e1+=conditionalAnd)* - ; - -conditionalAnd - : e=relation (ops+='&&' e1+=relation)* - ; - -relation - : calc - | relation op=('<'|'<='|'>='|'>'|'=='|'!='|'in') relation - ; - -calc - : unary - | calc op=('*'|'/'|'%') calc - | calc op=('+'|'-') calc - ; - -unary - : member # MemberExpr - | (ops+='!')+ member # LogicalNot - | (ops+='-')+ member # Negate - ; - -member - : primary # PrimaryExpr - | member op='.' id=field (open='(' args=exprList? ')')? # SelectOrCall - | member op='[' index=expr ']' # Index - | member op='{' entries=fieldInitializerList? ','? '}' # CreateMessage - ; - -primary - : leadingDot='.'? id=IDENTIFIER (op='(' args=exprList? ')')? # IdentOrGlobalCall - | '(' e=expr ')' # Nested - | op='[' elems=exprList? ','? ']' # CreateList - | op='{' entries=mapInitializerList? ','? '}' # CreateStruct - | literal # ConstantLiteral - ; - -exprList - : e+=expr (',' e+=expr)* - ; - -fieldInitializerList - : fields+=field cols+=':' values+=expr (',' fields+=field cols+=':' values+=expr)* - ; - -field - : IDENTIFIER - | ESC_IDENTIFIER - ; - -mapInitializerList - : keys+=expr cols+=':' values+=expr (',' keys+=expr cols+=':' values+=expr)* - ; - -literal - : sign=MINUS? tok=NUM_INT # Int - | tok=NUM_UINT # Uint - | sign=MINUS? tok=NUM_FLOAT # Double - | tok=STRING # String - | tok=BYTES # Bytes - | tok='true' # BoolTrue - | tok='false' # BoolFalse - | tok='null' # Null - ; - -// Lexer Rules -// =========== - -EQUALS : '=='; -NOT_EQUALS : '!='; -IN: 'in'; -LESS : '<'; -LESS_EQUALS : '<='; -GREATER_EQUALS : '>='; -GREATER : '>'; -LOGICAL_AND : '&&'; -LOGICAL_OR : '||'; - -LBRACKET : '['; -RPRACKET : ']'; -LBRACE : '{'; -RBRACE : '}'; -LPAREN : '('; -RPAREN : ')'; -DOT : '.'; -COMMA : ','; -MINUS : '-'; -EXCLAM : '!'; -QUESTIONMARK : '?'; -COLON : ':'; -PLUS : '+'; -STAR : '*'; -SLASH : '/'; -PERCENT : '%'; -TRUE : 'true'; -FALSE : 'false'; -NULL : 'null'; - -fragment BACKSLASH : '\\'; -fragment LETTER : 'A'..'Z' | 'a'..'z' ; -fragment DIGIT : '0'..'9' ; -fragment EXPONENT : ('e' | 'E') ( '+' | '-' )? DIGIT+ ; -fragment HEXDIGIT : ('0'..'9'|'a'..'f'|'A'..'F') ; -fragment RAW : 'r' | 'R'; - -fragment ESC_SEQ - : ESC_CHAR_SEQ - | ESC_BYTE_SEQ - | ESC_UNI_SEQ - | ESC_OCT_SEQ - ; - -fragment ESC_CHAR_SEQ - : BACKSLASH ('a'|'b'|'f'|'n'|'r'|'t'|'v'|'"'|'\''|'\\'|'?'|'`') - ; - -fragment ESC_OCT_SEQ - : BACKSLASH ('0'..'3') ('0'..'7') ('0'..'7') - ; - -fragment ESC_BYTE_SEQ - : BACKSLASH ( 'x' | 'X' ) HEXDIGIT HEXDIGIT - ; - -fragment ESC_UNI_SEQ - : BACKSLASH 'u' HEXDIGIT HEXDIGIT HEXDIGIT HEXDIGIT - | BACKSLASH 'U' HEXDIGIT HEXDIGIT HEXDIGIT HEXDIGIT HEXDIGIT HEXDIGIT HEXDIGIT HEXDIGIT - ; - -WHITESPACE : ( '\t' | ' ' | '\r' | '\n'| '\u000C' )+ -> channel(HIDDEN) ; -COMMENT : '//' (~'\n')* -> channel(HIDDEN) ; - -NUM_FLOAT - : ( DIGIT+ ('.' DIGIT+) EXPONENT? - | DIGIT+ EXPONENT - | '.' DIGIT+ EXPONENT? - ) - ; - -NUM_INT - : ( DIGIT+ | '0x' HEXDIGIT+ ); - -NUM_UINT - : DIGIT+ ( 'u' | 'U' ) - | '0x' HEXDIGIT+ ( 'u' | 'U' ) - ; - -STRING - : '"' (ESC_SEQ | ~('\\'|'"'|'\n'|'\r'))* '"' - | '\'' (ESC_SEQ | ~('\\'|'\''|'\n'|'\r'))* '\'' - | '"""' (ESC_SEQ | ~('\\'))*? '"""' - | '\'\'\'' (ESC_SEQ | ~('\\'))*? '\'\'\'' - | RAW '"' ~('"'|'\n'|'\r')* '"' - | RAW '\'' ~('\''|'\n'|'\r')* '\'' - | RAW '"""' .*? '"""' - | RAW '\'\'\'' .*? '\'\'\'' - ; - -BYTES : ('b' | 'B') STRING; - -IDENTIFIER : (LETTER | '_') ( LETTER | DIGIT | '_')*; - -ESC_IDENTIFIER : '`' (ESC_SEQ | ~('\\' | '`' | '\n' | '\r'))* '`'; diff --git a/generated-antlr/src/main/antlr/org.projectnessie.cel.parser.gen/CEL.tokens b/generated-antlr/src/main/antlr/org.projectnessie.cel.parser.gen/CEL.tokens deleted file mode 100644 index c99e4c02..00000000 --- a/generated-antlr/src/main/antlr/org.projectnessie.cel.parser.gen/CEL.tokens +++ /dev/null @@ -1,64 +0,0 @@ -EQUALS=1 -NOT_EQUALS=2 -IN=3 -LESS=4 -LESS_EQUALS=5 -GREATER_EQUALS=6 -GREATER=7 -LOGICAL_AND=8 -LOGICAL_OR=9 -LBRACKET=10 -RPRACKET=11 -LBRACE=12 -RBRACE=13 -LPAREN=14 -RPAREN=15 -DOT=16 -COMMA=17 -MINUS=18 -EXCLAM=19 -QUESTIONMARK=20 -COLON=21 -PLUS=22 -STAR=23 -SLASH=24 -PERCENT=25 -TRUE=26 -FALSE=27 -NULL=28 -WHITESPACE=29 -COMMENT=30 -NUM_FLOAT=31 -NUM_INT=32 -NUM_UINT=33 -STRING=34 -BYTES=35 -IDENTIFIER=36 -'=='=1 -'!='=2 -'in'=3 -'<'=4 -'<='=5 -'>='=6 -'>'=7 -'&&'=8 -'||'=9 -'['=10 -']'=11 -'{'=12 -'}'=13 -'('=14 -')'=15 -'.'=16 -','=17 -'-'=18 -'!'=19 -'?'=20 -':'=21 -'+'=22 -'*'=23 -'/'=24 -'%'=25 -'true'=26 -'false'=27 -'null'=28 diff --git a/generated-antlr/src/main/antlr/org.projectnessie.cel.parser.gen/CELLexer.tokens b/generated-antlr/src/main/antlr/org.projectnessie.cel.parser.gen/CELLexer.tokens deleted file mode 100644 index c99e4c02..00000000 --- a/generated-antlr/src/main/antlr/org.projectnessie.cel.parser.gen/CELLexer.tokens +++ /dev/null @@ -1,64 +0,0 @@ -EQUALS=1 -NOT_EQUALS=2 -IN=3 -LESS=4 -LESS_EQUALS=5 -GREATER_EQUALS=6 -GREATER=7 -LOGICAL_AND=8 -LOGICAL_OR=9 -LBRACKET=10 -RPRACKET=11 -LBRACE=12 -RBRACE=13 -LPAREN=14 -RPAREN=15 -DOT=16 -COMMA=17 -MINUS=18 -EXCLAM=19 -QUESTIONMARK=20 -COLON=21 -PLUS=22 -STAR=23 -SLASH=24 -PERCENT=25 -TRUE=26 -FALSE=27 -NULL=28 -WHITESPACE=29 -COMMENT=30 -NUM_FLOAT=31 -NUM_INT=32 -NUM_UINT=33 -STRING=34 -BYTES=35 -IDENTIFIER=36 -'=='=1 -'!='=2 -'in'=3 -'<'=4 -'<='=5 -'>='=6 -'>'=7 -'&&'=8 -'||'=9 -'['=10 -']'=11 -'{'=12 -'}'=13 -'('=14 -')'=15 -'.'=16 -','=17 -'-'=18 -'!'=19 -'?'=20 -':'=21 -'+'=22 -'*'=23 -'/'=24 -'%'=25 -'true'=26 -'false'=27 -'null'=28 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ddb3ce52..8c9ca408 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,5 +1,4 @@ [versions] -antlr4 = "4.13.2" checkstyle = "10.3.4" errorprone = "2.15.0" errorpronePlugin = "5.1.0" @@ -35,9 +34,8 @@ junit-testing = ["assertj-core", "junit-jupiter-api", "junit-jupiter-params"] [libraries] agrona = { module = "org.agrona:agrona", version = "2.6.0" } -antlr-antlr4 = { module = "org.antlr:antlr4", version.ref = "antlr4" } -antlr-antlr4-runtime = { module = "org.antlr:antlr4-runtime", version.ref = "antlr4" } assertj-core = { module = "org.assertj:assertj-core", version = "3.27.7" } +congocc = { module = "org.congocc:org.congocc.parser.generator", version = "2.1.0" } errorprone-plugin = { module = "net.ltgt.gradle:gradle-errorprone-plugin", version.ref = "errorpronePlugin" } errorprone-slf4j = { module = "jp.skypencil.errorprone.slf4j:errorprone-slf4j", version.ref = "errorproneSlf4j" } findbugs-jsr305 = { module = "com.google.code.findbugs:jsr305", version = "3.0.2" } diff --git a/settings.gradle.kts b/settings.gradle.kts index 75e3c1c4..2b80cc22 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -139,8 +139,6 @@ fun celProject(name: String) { project(":cel-$name").projectDir = file(name) } -celProject("generated-antlr") - celProject("generated-pb") celProject("generated-pb3") diff --git a/standalone/build.gradle.kts b/standalone/build.gradle.kts index bb4a05d9..5683db47 100644 --- a/standalone/build.gradle.kts +++ b/standalone/build.gradle.kts @@ -37,7 +37,6 @@ dependencies { api(project(":cel-tools")) api(project(":cel-jackson")) api(project(":cel-jackson3")) - api(project(":cel-generated-antlr")) compileOnly(project(":cel-generated-pb")) compileOnly(libs.protobuf.java) @@ -47,7 +46,6 @@ dependencies { standaloneShadow(project(":cel-tools")) standaloneShadow(project(":cel-jackson")) standaloneShadow(project(":cel-jackson3")) - standaloneShadow(project(":cel-generated-antlr")) standaloneShadow(project(":cel-generated-pb")) standaloneShadow(libs.protobuf.java) standaloneShadow(libs.agrona) diff --git a/tools/src/test/java/org/projectnessie/cel/tools/ScriptHostTest.java b/tools/src/test/java/org/projectnessie/cel/tools/ScriptHostTest.java index d2d9c042..f89916e4 100644 --- a/tools/src/test/java/org/projectnessie/cel/tools/ScriptHostTest.java +++ b/tools/src/test/java/org/projectnessie/cel/tools/ScriptHostTest.java @@ -142,7 +142,7 @@ void badSyntax() { assertThatThrownBy(() -> scriptHost.buildScript("-.,").build()) .isInstanceOf(ScriptCreateException.class) .hasMessageStartingWith( - "parse failed: ERROR: :1:3: Syntax error: mismatched input ',' expecting IDENTIFIER"); + "parse failed: ERROR: :1:3: Syntax error: Encountered an error"); } @Test From 58a3cf423ffc8a4370a74a54f4a5fc6fc8d4153d Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Thu, 20 Aug 2026 11:12:06 +0200 Subject: [PATCH 2/3] review --- .../org/projectnessie/cel/parser/Helper.java | 3 +-- .../org/projectnessie/cel/parser/Parser.java | 8 +++---- .../projectnessie/cel/parser/ParserTest.java | 22 ++++++++++++------- .../cel/tools/ScriptHostTest.java | 4 ++-- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/org/projectnessie/cel/parser/Helper.java b/core/src/main/java/org/projectnessie/cel/parser/Helper.java index b986767b..1f7e9330 100644 --- a/core/src/main/java/org/projectnessie/cel/parser/Helper.java +++ b/core/src/main/java/org/projectnessie/cel/parser/Helper.java @@ -195,8 +195,7 @@ private Builder newExprBuilder(Object ctx) { long id(Object ctx) { Location location; - if (ctx instanceof Node) { - Node node = (Node) ctx; + if (ctx instanceof Node node) { location = source.newLocation(node.getBeginLine(), node.getBeginColumn() - 1); } else if (ctx instanceof Location) { location = (Location) ctx; diff --git a/core/src/main/java/org/projectnessie/cel/parser/Parser.java b/core/src/main/java/org/projectnessie/cel/parser/Parser.java index b648743a..4ed51e70 100644 --- a/core/src/main/java/org/projectnessie/cel/parser/Parser.java +++ b/core/src/main/java/org/projectnessie/cel/parser/Parser.java @@ -300,7 +300,7 @@ public Expr visitUnary(Node node) { if (opCount % 2 == 0) { return exprVisit(operand); } - if (!logicalNot && isNegativeNumericLiteral(operand)) { + if (!logicalNot && isIntOrFloatLiteral(operand)) { return visitNegativeNumericLiteral(op, operand); } return globalCallOrMacro( @@ -448,7 +448,7 @@ public Expr visitLiteral(Node node) { return reportError(node, "invalid literal"); } - private boolean isNegativeNumericLiteral(Node operand) { + private static boolean isIntOrFloatLiteral(Node operand) { return isToken(operand, NUM_INT) || isToken(operand, NUM_FLOAT) || (operand instanceof ConstantLiteral @@ -693,7 +693,7 @@ Expr reportError(Object ctx, String format, Object... args) { return err; } - private String fieldName(Node node) { + private static String fieldName(Node node) { if (node instanceof Field) { return fieldName(significantChildren(node).get(0)); } @@ -704,7 +704,7 @@ private String fieldName(Node node) { return text; } - private Node firstExpressionChild(List children, Node ctx) { + private static Node firstExpressionChild(List children, Node ctx) { for (Node child : children) { if (!isStructuralToken(child)) { return child; diff --git a/core/src/test/java/org/projectnessie/cel/parser/ParserTest.java b/core/src/test/java/org/projectnessie/cel/parser/ParserTest.java index af5b27df..a8131e2e 100644 --- a/core/src/test/java/org/projectnessie/cel/parser/ParserTest.java +++ b/core/src/test/java/org/projectnessie/cel/parser/ParserTest.java @@ -752,7 +752,7 @@ public static String[][] testCases() { "75", "{", "", - "ERROR: :1:2: Syntax error: mismatched input '' expecting {'[', '{', '}', '(', '.', ',', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n" + "ERROR: :1:1: Syntax error: mismatched input '' expecting {'[', '{', '}', '(', '.', ',', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n" + " | {\n" + " | .^", "", @@ -1257,7 +1257,7 @@ public static String[][] testCases() { "127", "--", "", - "ERROR: :1:3: Syntax error: mismatched input '' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n" + "ERROR: :1:1: Syntax error: mismatched input '' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n" + " | --\n" + " | ..^\n" + "ERROR: :1:3: Syntax error: no viable alternative at input '-'\n" @@ -1297,8 +1297,8 @@ public static String[][] testCases() { * @param num just the index of the test case * @param i contains the input expression to be parsed. * @param p contains the type/id adorned debug output of the expression tree. - * @param e contains the expected error output for a failed parse, or "" if the parse is expected - * to be successful. + * @param e contains the expected error output for semantic failures, or the expected first error + * location for syntax failures; it is "" if the parse is expected to be successful. * @param l contains the expected source adorned debug output of the expression tree. */ @ParameterizedTest @@ -1310,12 +1310,13 @@ void parseTest(String num, String i, String p, String e, String l) { String actualErr = parseResult.getErrors().toDisplayString(); if (e.isEmpty()) { assertThat(actualErr).isEmpty(); + } else if (e.contains(": Syntax error:")) { + // Parser implementations may format syntax diagnostics differently, but they must point to + // the same first invalid source location. + assertThat(actualErr).startsWith(firstErrorLocation(e) + ": Syntax error:"); } else { - assertThat(actualErr).isNotEmpty(); + assertThat(actualErr).isEqualTo(e); } - // Hint for my future self and others: if the above "isEqualTo" fails but the strings look - // similar, - // look into the char[] representation... unicode can be very surprising. String actualWithKind = Debug.toAdornedDebugString(parseResult.getExpr(), new KindAndIdAdorner()); @@ -1329,6 +1330,11 @@ void parseTest(String num, String i, String p, String e, String l) { } } + private static String firstErrorLocation(String errors) { + int locationStart = "ERROR: :".length(); + return errors.substring(0, errors.indexOf(": ", locationStart)); + } + @Test void expressionSizeCodePointLimit() { assertThatThrownBy( diff --git a/tools/src/test/java/org/projectnessie/cel/tools/ScriptHostTest.java b/tools/src/test/java/org/projectnessie/cel/tools/ScriptHostTest.java index f89916e4..76a48286 100644 --- a/tools/src/test/java/org/projectnessie/cel/tools/ScriptHostTest.java +++ b/tools/src/test/java/org/projectnessie/cel/tools/ScriptHostTest.java @@ -141,8 +141,8 @@ void badSyntax() { assertThatThrownBy(() -> scriptHost.buildScript("-.,").build()) .isInstanceOf(ScriptCreateException.class) - .hasMessageStartingWith( - "parse failed: ERROR: :1:3: Syntax error: Encountered an error"); + .hasMessageContaining("Found string \",\" of type COMMA") + .hasMessageContaining("Was expecting: IDENTIFIER"); } @Test From c2962958904ddeeca13e29948b5fe0dd3a647011 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Thu, 20 Aug 2026 11:15:38 +0200 Subject: [PATCH 3/3] comment --- .../java/org/projectnessie/cel/parser/Options.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/core/src/main/java/org/projectnessie/cel/parser/Options.java b/core/src/main/java/org/projectnessie/cel/parser/Options.java index fd1da5d3..8fc023bc 100644 --- a/core/src/main/java/org/projectnessie/cel/parser/Options.java +++ b/core/src/main/java/org/projectnessie/cel/parser/Options.java @@ -42,6 +42,12 @@ public int getMaxRecursionDepth() { return maxRecursionDepth; } + /** + * Returns the maximum number of parser error-recovery attempts. + * + *

This setting is currently not respected because CongoCC stops parsing after the first syntax + * error and does not provide error recovery. + */ public int getErrorRecoveryLimit() { return errorRecoveryLimit; } @@ -78,6 +84,12 @@ public Builder maxRecursionDepth(int maxRecursionDepth) { return this; } + /** + * Sets the maximum number of parser error-recovery attempts. + * + *

This setting is currently not respected because CongoCC stops parsing after the first + * syntax error and does not provide error recovery. + */ public Builder errorRecoveryLimit(int errorRecoveryLimit) { if (errorRecoveryLimit < -1) { throw new IllegalArgumentException(