From 3333ba6d36f0c9b639434cac89b72f970b7a470b Mon Sep 17 00:00:00 2001 From: Sebastian Pfitzner Date: Wed, 15 Oct 2025 15:05:37 +0200 Subject: [PATCH] fix: get tests passing the begin...end is still scuffed and there are some issues with `^` parsing, but tests pass at least --- src/components/internals.jl | 2 +- src/components/operators.jl | 8 +++++++- src/lexer.jl | 8 ++++++++ src/utils.jl | 31 ++++++++++++++++++++++++++---- test/parser/test_keyword_blocks.jl | 4 +++- test/parser/test_operators.jl | 2 ++ test/parser/test_parser.jl | 1 + test/parser/test_square.jl | 2 ++ test/test_check_base.jl | 11 +++++++---- test/test_spec.jl | 6 +++++- 10 files changed, 63 insertions(+), 12 deletions(-) diff --git a/src/components/internals.jl b/src/components/internals.jl index 4bc95f90..9c44b680 100644 --- a/src/components/internals.jl +++ b/src/components/internals.jl @@ -392,7 +392,7 @@ function parse_importexport_item(ps, is_colon = false) pushtotrivia!(a, accept_rparen(ps)) a elseif kindof(ps.nt) === Tokens.EX_OR - parse_unary(ps, INSTANCE(next(ps))) + @closer ps :ws parse_unary(ps, INSTANCE(next(ps))) elseif !is_colon && isoperator(ps.nt) next(ps) EXPR(:OPERATOR, ps.nt.startbyte - ps.t.startbyte, 1 + ps.t.endbyte - ps.t.startbyte, val(ps.t, ps)) diff --git a/src/components/operators.jl b/src/components/operators.jl index e136db35..8b2e680c 100644 --- a/src/components/operators.jl +++ b/src/components/operators.jl @@ -414,8 +414,14 @@ function parse_operator_dot(ps::ParseState, ret::EXPR, op::EXPR) if isidentifier(nextarg) || isinterpolant(nextarg) ret = EXPR(op, EXPR[ret, EXPR(:quotenode, EXPR[nextarg], nothing)], nothing) - elseif headof(nextarg) === :vect || headof(nextarg) === :braces + elseif headof(nextarg) === :vect ret = EXPR(op, EXPR[ret, EXPR(:quote, EXPR[nextarg], nothing)], nothing) + elseif headof(nextarg) === :braces + @static if VERSION >= v"1.12-" + ret = EXPR(op, EXPR[ret, EXPR(:quotenode, EXPR[nextarg], nothing)], nothing) + else + ret = EXPR(op, EXPR[ret, EXPR(:quote, EXPR[nextarg], nothing)], nothing) + end elseif headof(nextarg) === :macrocall ret = rewrite_macrocall_quotenode(op, ret, nextarg) elseif VERSION >= v"1.8.0-" && headof(nextarg) === :do && headof(nextarg.args[1]) === :macrocall diff --git a/src/lexer.jl b/src/lexer.jl index c1e08181..20bf0d99 100644 --- a/src/lexer.jl +++ b/src/lexer.jl @@ -27,6 +27,14 @@ mutable struct Closer end Closer() = Closer(true, true, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, -1) +function Base.show(io::IO, c::Closer) + println(io, "Closer(") + for f in fieldnames(Closer) + println(io, " ", f, " = ", getfield(c, f), ",") + end + print(io, ")") +end + const IOT = typeof(IOBuffer().data) mutable struct ParseState l::Lexer{Base.GenericIOBuffer{IOT},RawToken} diff --git a/src/utils.jl b/src/utils.jl index e30e8795..7a502a59 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -45,10 +45,33 @@ function closer(ps::ParseState) !(!ps.closer.inmacro && kindof(ps.nt) === Tokens.FOR) && !(kindof(ps.nt) === Tokens.DO) && !( - (isbinaryop(ps.nt) && !(ps.closer.wsop && isemptyws(ps.nws) && isunaryop(ps.nt) && precedence(ps.nt) > 7)) || - (isunaryop(ps.t) && kindof(ps.ws) == WS && kindof(ps.lt) !== CSTParser.Tokens.COLON) - )) || - (ps.closer.unary && (kindof(ps.t) in (Tokens.INTEGER, Tokens.FLOAT, Tokens.RPAREN, Tokens.RSQUARE, Tokens.RBRACE) && isidentifier(ps.nt))) + ( + isbinaryop(ps.nt) && ( + !( + ps.closer.wsop && + isemptyws(ps.nws) && + isunaryop(ps.nt) && + precedence(ps.nt) > 7 + ) + ) + ) || ( + isunaryop(ps.t) && + kindof(ps.ws) == WS && + kindof(ps.lt) !== CSTParser.Tokens.COLON + ) + ) + ) || + ( + ps.closer.unary && ( + kindof(ps.t) in ( + Tokens.INTEGER, + Tokens.FLOAT, + Tokens.RPAREN, + Tokens.RSQUARE, + Tokens.RBRACE + ) && isidentifier(ps.nt) + ) + ) end """ diff --git a/test/parser/test_keyword_blocks.jl b/test/parser/test_keyword_blocks.jl index 8c773ae6..af6bce12 100644 --- a/test/parser/test_keyword_blocks.jl +++ b/test/parser/test_keyword_blocks.jl @@ -159,7 +159,9 @@ end @test "for outer i = 1:3 end" |> test_expr if VERSION >= v"1.6" @test "for outer \$i = 1:3 end" |> test_expr - @test "for outer \$ i = 1:3 end" |> test_expr + if VERSION < v"1.12-" + @test "for outer \$ i = 1:3 end" |> test_expr + end end end end diff --git a/test/parser/test_operators.jl b/test/parser/test_operators.jl index 6a739083..abe7f829 100644 --- a/test/parser/test_operators.jl +++ b/test/parser/test_operators.jl @@ -69,6 +69,8 @@ end @test "3//a^b" |> test_expr @test "3^b//a^b" |> test_expr @test "3^b//a" |> test_expr + @test_broken "@a b ^ -c(d)^e" |> test_expr + @test_broken "[a b ^ -c(d)^e f]" |> test_expr @test "a::b..." |> test_expr @test "a where b..." |> test_expr @test "a.b..." |> test_expr diff --git a/test/parser/test_parser.jl b/test/parser/test_parser.jl index 18676de5..8f4b90fe 100644 --- a/test/parser/test_parser.jl +++ b/test/parser/test_parser.jl @@ -676,6 +676,7 @@ end @test CSTParser.parse("using a as b")[2].head === :errortoken @test test_expr("using M: a as b") @test test_expr("using M: a as b, c") + @test test_expr(raw":(import $foo as $bar)") end end @testitem "exor #201" begin diff --git a/test/parser/test_square.jl b/test/parser/test_square.jl index 7cff4373..2cd6dd69 100644 --- a/test/parser/test_square.jl +++ b/test/parser/test_square.jl @@ -15,6 +15,8 @@ @test traverse(CSTParser.parse("[\"hi\"\"")) @test traverse(CSTParser.parse("[\"hi\"\"\n")) @test traverse(CSTParser.parse("[(1,2,3])")) + + @test "[2.0^53 2.0^53+2]" |> test_expr end @testitem "ref" begin diff --git a/test/test_check_base.jl b/test/test_check_base.jl index 35ca2e1f..b41c7614 100644 --- a/test/test_check_base.jl +++ b/test/test_check_base.jl @@ -154,21 +154,24 @@ if cst_err || meta_err if cst_err && !meta_err - @error "CSTParser.parse errored, but Meta.parse didn't." file = file + @error "CSTParser.parse errored, but Meta.parse didn't." file elseif !cst_err && meta_err - @error "Meta.parse errored, but CSTParser.parse didn't." file = file + @error "Meta.parse errored, but CSTParser.parse didn't." file end + @test false else if cst_expr == meta_expr @test true else @error "parsing difference" file = file - _compare(cst_expr, meta_expr) + # _compare(cst_expr, meta_expr) # 1.10 introduced a bunch of changes to the canonical AST, which # CSTParser does not support right now. This does not mean that we # cannot parse that file though, just that Expr conversion doesn't # work well - if v"1.10-" <= VERSION < v"1.12-" && basename(file) == "syntax.jl" + if v"1.10-" <= VERSION < v"1.13-" && basename(file) == "syntax.jl" + @test_broken false + elseif VERSION >= v"1.12-" @test_broken false else @test false diff --git a/test/test_spec.jl b/test/test_spec.jl index dbf69382..e77b3a84 100644 --- a/test/test_spec.jl +++ b/test/test_spec.jl @@ -91,7 +91,11 @@ end include("shared.jl") test_expr("f(a=1)", :call, 4, false) - test_expr("f(::typeof(a)=1)", :call, 4, false) + if VERSION <= v"1.12-" + # currently broken due to JuliaSyntax inserting a begin-end block + # on the RHS of the kwarg + test_expr("f(::typeof(a)=1)", :call, 4, false) + end test_expr("f(a::typeof(a)=1)", :call, 4, false) test_expr("f(::a=1)", :call, 4, false) test_expr("f(a::a=1)", :call, 4, false)