diff --git a/src/main/java/io/github/treesitter/jtreesitter/Parser.java b/src/main/java/io/github/treesitter/jtreesitter/Parser.java index bf51cc5..41d4933 100644 --- a/src/main/java/io/github/treesitter/jtreesitter/Parser.java +++ b/src/main/java/io/github/treesitter/jtreesitter/Parser.java @@ -3,6 +3,8 @@ import static io.github.treesitter.jtreesitter.internal.TreeSitter.*; import io.github.treesitter.jtreesitter.internal.*; + +import java.io.ByteArrayOutputStream; import java.lang.foreign.Arena; import java.lang.foreign.MemoryLayout; import java.lang.foreign.MemorySegment; @@ -64,7 +66,6 @@ public Parser setLanguage(Language language) { * (type, message) -> logger.info("%s - %s".formatted(type.name(), message))); * } */ - @SuppressWarnings("unused") public Parser setLogger(@Nullable Logger logger) { if (logger == null) { ts_parser_set_logger(self, TSLogger.allocate(arena)); @@ -216,7 +217,7 @@ public Optional parse(String source, InputEncoding encoding, @Nullable Tre var old = oldTree == null ? MemorySegment.NULL : oldTree.segment(); var tree = ts_parser_parse_string_encoding(self, old, string, bytes.length, encoding.ordinal()); if (tree.equals(MemorySegment.NULL)) return Optional.empty(); - return Optional.of(new Tree(tree, language, source, encoding.charset())); + return Optional.of(new Tree(tree, language, bytes, encoding.charset())); } } @@ -253,7 +254,6 @@ public Optional parse(ParseCallback parseCallback, InputEncoding encoding, * @return An optional {@linkplain Tree} which is empty if parsing was halted. * @throws IllegalStateException If the parser does not have a language assigned. */ - @SuppressWarnings("unused") public Optional parse( ParseCallback parseCallback, InputEncoding encoding, @Nullable Tree oldTree, @Nullable Options options) throws IllegalStateException { @@ -261,6 +261,7 @@ public Optional parse( throw new IllegalStateException("The parser has no language assigned"); } + var collected = new ByteArrayOutputStream(); var input = TSInput.allocate(arena); TSInput.payload(input, MemorySegment.NULL); TSInput.encoding(input, encoding.ordinal()); @@ -272,6 +273,15 @@ public Optional parse( return MemorySegment.NULL; } var buffer = result.getBytes(encoding.charset()); + var collectedSize = collected.size(); + if (index >= collectedSize) { + collected.write(buffer, 0, buffer.length); + } else { + var overlap = collectedSize - index; + if (overlap < buffer.length) { + collected.write(buffer, overlap, buffer.length - overlap); + } + } bytes.set(C_INT, 0, buffer.length); return arena.allocateFrom(C_CHAR, buffer); }, @@ -295,7 +305,7 @@ public Optional parse( tree = ts_parser_parse_with_options(self, old, input, parseOptions); } if (tree.equals(MemorySegment.NULL)) return Optional.empty(); - return Optional.of(new Tree(tree, language, null, null)); + return Optional.of(new Tree(tree, language, collected.toByteArray(), encoding.charset())); } /** diff --git a/src/main/java/io/github/treesitter/jtreesitter/Tree.java b/src/main/java/io/github/treesitter/jtreesitter/Tree.java index 2088a8b..19440fa 100644 --- a/src/main/java/io/github/treesitter/jtreesitter/Tree.java +++ b/src/main/java/io/github/treesitter/jtreesitter/Tree.java @@ -17,16 +17,16 @@ public final class Tree implements AutoCloseable, Cloneable { private final MemorySegment self; private byte[] source; - private @Nullable Charset charset; + private final Charset charset; private final Arena arena; private final Language language; private @Nullable List includedRanges; - Tree(MemorySegment self, Language language, @Nullable String source, @Nullable Charset charset) { + Tree(MemorySegment self, Language language, byte[] source, Charset charset) { arena = Arena.ofShared(); this.self = self.reinterpret(arena, TreeSitter::ts_tree_delete); this.language = language; - this.source = source != null && charset != null ? source.getBytes(charset) : new byte[0]; + this.source = source; this.charset = charset; } @@ -45,13 +45,13 @@ MemorySegment segment() { } boolean hasText() { - return charset != null; + return source.length > 0; } @Nullable String getRegion(@Unsigned int start, @Unsigned int end) { var length = Math.min(end, source.length) - start; - return charset != null ? new String(source, start, length, charset) : null; + return hasText() ? new String(source, start, length, charset) : null; } /** Get the language that was used to parse the syntax tree. */ @@ -137,7 +137,6 @@ public void edit(InputEdit edit) { ts_tree_edit(self, edit.into(alloc)); } finally { source = new byte[0]; - charset = null; } } diff --git a/src/test/java/io/github/treesitter/jtreesitter/ParserTest.java b/src/test/java/io/github/treesitter/jtreesitter/ParserTest.java index cbfc5ab..1732722 100644 --- a/src/test/java/io/github/treesitter/jtreesitter/ParserTest.java +++ b/src/test/java/io/github/treesitter/jtreesitter/ParserTest.java @@ -107,10 +107,10 @@ void parseLogger() { @DisplayName("parse(callback)") void parseCallback() { var source = "class Foo {}"; - ParseCallback callback = (offset, _) -> source.substring(offset, Integer.min(offset, source.length())); + ParseCallback callback = (offset, _) -> source.substring(offset, Integer.min(source.length(), offset + 1)); parser.setLanguage(language); try (var tree = parser.parse(callback, InputEncoding.UTF_8).orElseThrow()) { - assertNull(tree.getText()); + assertEquals(source, tree.getText()); assertEquals("program", tree.getRootNode().getType()); } }