Merge pull request #356 from julia-vscode/sp/juliacon-fixes

Various fixes

Author: pfitzseb

src/CSTParser.jl

@@ -38,7 +38,7 @@ Acceptable starting tokens are:
 + An `@`.
 
 """
-function parse_expression(ps::ParseState, esc_on_error = false; allow_const_field = false)
+function parse_expression(ps::ParseState, esc_on_error = false)
     if kindof(ps.nt) === Tokens.ENDMARKER
         ret = mErrorToken(ps, UnexpectedToken)
     elseif (esc_on_error && ps.nt.kind == Tokens.ERROR)
@@ -48,7 +48,7 @@ function parse_expression(ps::ParseState, esc_on_error = false; allow_const_fiel
     else
         next(ps)
         if iskeyword(kindof(ps.t)) && kindof(ps.t) != Tokens.DO
-            ret = parse_kw(ps; allow_const_field = allow_const_field)
+            ret = parse_kw(ps)
         elseif kindof(ps.t) === Tokens.LPAREN
             ret = parse_paren(ps)
         elseif kindof(ps.t) === Tokens.LSQUARE

src/components/internals.jl

@@ -4,7 +4,7 @@ const term_c = (Tokens.RPAREN, Tokens.RSQUARE, Tokens.RBRACE, Tokens.END, Tokens
 Continue parsing statements until an element of `closers` is hit (usually
 `end`). Statements are grouped in a `Block` EXPR.
 """
-function parse_block(ps::ParseState, ret::Vector{EXPR}=EXPR[], closers=(Tokens.END,), docable=false; allow_const_field = false)
+function parse_block(ps::ParseState, ret::Vector{EXPR}=EXPR[], closers=(Tokens.END,), docable=false)
     prevpos = position(ps)
     while kindof(ps.nt) ∉ closers # loop until an expected closer is hit
         if kindof(ps.nt) ∈ term_c # error handling if an unexpected closer is hit
@@ -17,7 +17,7 @@ function parse_block(ps::ParseState, ret::Vector{EXPR}=EXPR[], closers=(Tokens.E
             if docable
                 a = parse_doc(ps)
             else
-                a = parse_expression(ps; allow_const_field = allow_const_field)
+                a = parse_expression(ps)
             end
             push!(ret, a)
         end

src/components/keywords.jl

@@ -3,7 +3,7 @@
 
 Dispatch function for when the parser has reached a keyword.
 """
-function parse_kw(ps::ParseState; allow_const_field = false)
+function parse_kw(ps::ParseState)
     k = kindof(ps.t)
     if ps.closer.precedence == 20 && ps.lt.kind === Tokens.EX_OR && k !== Tokens.END
         return EXPR(:IDENTIFIER, ps)
@@ -52,7 +52,7 @@ function parse_kw(ps::ParseState; allow_const_field = false)
     elseif k === Tokens.BAREMODULE
         return @default ps @closer ps :block parse_blockexpr(ps, :baremodule)
     elseif k === Tokens.CONST
-        return @default ps parse_const(ps; allow_const_field = allow_const_field)
+        return @default ps parse_const(ps)
     elseif k === Tokens.GLOBAL
         return @default ps parse_local_global(ps, false)
     elseif k === Tokens.LOCAL
@@ -75,7 +75,10 @@ function parse_kw(ps::ParseState; allow_const_field = false)
     elseif k === Tokens.TYPE
         return EXPR(:IDENTIFIER, ps)
     elseif k === Tokens.STRUCT
-        return @default ps @closer ps :block parse_blockexpr(ps, :struct, allow_const_field = true)
+        enable!(ps, ParserFlags.AllowConstWithoutAssignment)
+        ret = @default ps @closer ps :block parse_blockexpr(ps, :struct)
+        disable!(ps, ParserFlags.AllowConstWithoutAssignment)
+        return ret
     elseif k === Tokens.MUTABLE
         return @default ps @closer ps :block parse_mutable(ps)
     elseif k === Tokens.OUTER
@@ -85,10 +88,14 @@ function parse_kw(ps::ParseState; allow_const_field = false)
     end
 end
 
-function parse_const(ps::ParseState; allow_const_field = false)
+function parse_const(ps::ParseState)
     kw = EXPR(ps)
+    lt = ps.lt
+    nt = ps.nt
     arg = parse_expression(ps)
-    if !allow_const_field && !(isassignment(unwrapbracket(arg)) || (headof(arg) === :global && length(arg.args) > 0 && isassignment(unwrapbracket(arg.args[1]))))
+    allow_no_assignment = has_flag(ps, ParserFlags.AllowConstWithoutAssignment) ||
+        has_flag(ps, ParserFlags.InQuote) && (kindof(nt) === Tokens.GLOBAL || kindof(lt) === Tokens.GLOBAL)
+    if !allow_no_assignment && !(isassignment(unwrapbracket(arg)) || (headof(arg) === :global && length(arg.args) > 0 && isassignment(unwrapbracket(arg.args[1]))))
         arg = mErrorToken(ps, arg, ExpectedAssignment)
     end
     ret = EXPR(:const, EXPR[arg], EXPR[kw])
@@ -154,7 +161,11 @@ function parse_mutable(ps::ParseState)
     if kindof(ps.nt) === Tokens.STRUCT
         kw = EXPR(ps)
         next(ps)
-        ret = parse_blockexpr(ps, :mutable, allow_const_field = true)
+
+        enable!(ps, ParserFlags.AllowConstWithoutAssignment)
+        ret = parse_blockexpr(ps, :mutable)
+        disable!(ps, ParserFlags.AllowConstWithoutAssignment)
+
         pushfirst!(ret.trivia, setparent!(kw, ret))
         update_span!(ret)
     else
@@ -308,15 +319,15 @@ function parse_do(ps::ParseState, pre::EXPR)
 end
 
 """
-    parse_blockexpr(ps::ParseState, head; allow_const_field = false)
+    parse_blockexpr(ps::ParseState, head)
 
 General function for parsing block expressions comprised of a series of statements
 terminated by an `end`.
 """
-function parse_blockexpr(ps::ParseState, head; allow_const_field = false)
+function parse_blockexpr(ps::ParseState, head)
     kw = EXPR(ps)
     sig = parse_blockexpr_sig(ps, head)
-    blockargs = parse_block(ps, EXPR[], (Tokens.END,), docable(head); allow_const_field = allow_const_field)
+    blockargs = parse_block(ps, EXPR[], (Tokens.END,), docable(head))
     if head === :begin
         EXPR(:block, blockargs, EXPR[kw, accept_end(ps)])
     elseif sig === nothing

src/components/lists.jl

@@ -118,8 +118,7 @@ function parse_array_outer(ps::ParseState, trivia, isref)
     max_bp = -typemax(Int)
     is_start = true
     while kindof(ps.nt) !== Tokens.RSQUARE && kindof(ps.nt) !== Tokens.ENDMARKER
-        a = @nocloser ps :newline @nocloser ps :newline @closesquare ps @closer ps :insquare @closer ps :ws @closer ps :wsop @closer ps :comma parse_expression(ps)
-
+        a = @nocloser ps :semicolon @nocloser ps :newline @closesquare ps @closer ps :insquare @closer ps :ws @closer ps :wsop @closer ps :comma parse_expression(ps)
         if is_start
             args = EXPR[]
             if isref && _do_kw_convert(ps, a)

src/components/operators.jl

@@ -249,7 +249,9 @@ function parse_unary_colon(ps::ParseState, op::EXPR)
         ret = op
     else
         prev_errored = ps.errored
+        enable!(ps, ParserFlags.InQuote)
         arg = @precedence ps 20 @nocloser ps :inref parse_expression(ps)
+        disable!(ps, ParserFlags.InQuote)
         if isbracketed(arg) && headof(arg.args[1]) === :errortoken && errorof(arg.args[1]) === UnexpectedAssignmentOp
             ps.errored = prev_errored
             arg.args[1] = arg.args[1].args[1]
@@ -471,7 +473,11 @@ function parse_operator(ps::ParseState, ret::EXPR, op::EXPR)
         ret = EXPR(op, EXPR[ret], nothing)
     elseif is_prime(op)
         if isidentifier(ret) || isliteral(ret) ||
-                headof(ret) in (:call, :tuple, :brackets, :ref, :vect, :vcat, :hcat, :typed_vcat, :typed_hcat, :comprehension, :typed_comprehension, :curly, :braces, :braces_cat) ||
+                headof(ret) in (
+                    :call, :tuple, :brackets, :ref, :vect, :vcat, :hcat, :ncat, :typed_vcat,
+                    :typed_hcat, :typed_ncat, :comprehension, :typed_comprehension, :curly,
+                    :braces, :braces_cat
+                ) ||
                 headof(ret) === :do ||
                 is_dot(headof(ret)) ||
                 is_prime(headof(ret))

src/iterate.jl

@@ -257,10 +257,20 @@ function _const(x, i)
 end
 
 function _global(x, i)
-    if hastrivia(x) && (headof(first(x.trivia)) === :GLOBAL || headof(first(x.trivia)) === :LOCAL)
-        oddt_evena(x, i)
+    if length(x.trivia) > length(x.args) # probably :tuple
+        if i <= 2
+            x.trivia[i]
+        elseif isodd(i)
+            x.args[div(i - 1, 2)]
+        else
+            x.trivia[div(i - 2, 2) + 2]
+        end
     else
-        odda_event(x, i)
+        if hastrivia(x) && (headof(first(x.trivia)) === :GLOBAL || headof(first(x.trivia)) === :LOCAL)
+            oddt_evena(x, i)
+        else
+            odda_event(x, i)
+        end
     end
 end
 

src/lexer.jl

@@ -42,9 +42,10 @@ mutable struct ParseState
     nnws::RawToken
     closer::Closer
     errored::Bool
+    flags::UInt64
 end
 function ParseState(str::Union{IOBuffer,String})
-    ps = ParseState(tokenize(str, RawToken), false, RawToken(), RawToken(), RawToken(), RawToken(), RawToken(), RawToken(), RawToken(), RawToken(), Closer(), false)
+    ps = ParseState(tokenize(str, RawToken), false, RawToken(), RawToken(), RawToken(), RawToken(), RawToken(), RawToken(), RawToken(), RawToken(), Closer(), false, 0)
     return next(next(ps))
 end
 
@@ -58,6 +59,15 @@ function ParseState(str::Union{IOBuffer,String}, loc::Int)
     return ps
 end
 
+module ParserFlags
+    const AllowConstWithoutAssignment = 0x1
+    const InQuote = 0x2
+end
+
+enable!(ps::ParseState, flag::Integer) = ps.flags |= flag
+disable!(ps::ParseState, flag::Integer) = ps.flags &= ~flag
+has_flag(ps::ParseState, flag::Integer) = ps.flags & flag > 0
+
 function Base.show(io::IO, ps::ParseState)
     println(io, "ParseState at $(position(ps.l.io))")
     println(io, "last    : ", kindof(ps.lt), " ($(ps.lt))", "    ($(wstype(ps.lws)))")
@@ -66,21 +76,18 @@ function Base.show(io::IO, ps::ParseState)
 end
 peekchar(ps::ParseState) = peekchar(ps.l)
 if !applicable(Base.peek, Tuple{IOBuffer, Char})
-function _peek(s::IO, ::Type{T}) where T
-    mark(s)
-    try read(s, T)::T
-    finally
-        reset(s)
+    function _peek(s::IO, ::Type{T}) where T
+        mark(s)
+        try read(s, T)::T
+        finally
+            reset(s)
+        end
     end
-end
-peekchar(io) = _peek(io, Char)
+    peekchar(io) = _peek(io, Char)
 else
-peekchar(io) = peek(io, Char)
+    peekchar(io) = peek(io, Char)
 end
 
-
-
-
 wstype(t::AbstractToken) = kindof(t) == EmptyWS ? "empty" :
                     kindof(t) == NewLineWS ? "ws w/ newline" :
                     kindof(t) == SemiColonWS ? "ws w/ semicolon" : "ws"

src/spec.jl

@@ -280,8 +280,8 @@ end
 function Base.length(x::EXPR)
     headof(x) === :NONSTDIDENTIFIER && return 0
     headof(x) === :flatten && return length(Iterating._flatten_lhs(x))
-    n = x.args isa Nothing ? 0 : length(x.args)
-    n += x.trivia !== nothing ? length(x.trivia) : 0
+    n = x.args === nothing ? 0 : length(x.args)
+    n += x.trivia === nothing ? 0 : length(x.trivia)
     x.head isa EXPR && !(x.head.span === 0) && (n += 1)
     return n
 end

test/iterate.jl

@@ -29,6 +29,18 @@ end
             @test x[1] === x.trivia[1]
             @test x[2] === x.args[1]
         end
+
+        @testset "global tuple" begin
+            x = cst"global (a = 1,b = 2)"
+            @test length(x) == 6
+            @test x[1] === x.trivia[1]
+            @test x[2] === x.trivia[2]
+            @test x[3] === x.args[1]
+            @test x[4] === x.trivia[3]
+            @test x[5] === x.args[2]
+            @test x[6] === x.trivia[4]
+        end
+
         @testset "const" begin
             x = cst"const a = 1"
             @test length(x) == 2

test/parser.jl

@@ -179,6 +179,14 @@ end
             @test ":(;;;)" |> test_expr
         end
 
+        # this errors during *lowering*, not parsing.
+        @testset "parse const without assignment in quote" begin
+            @test ":(global const x)" |> test_expr
+            @test ":(global const x::Int)" |> test_expr
+            @test ":(const global x)" |> test_expr
+            @test ":(const global x::Int)" |> test_expr
+        end
+
         @testset "where precedence" begin
             @test "a = b where c = d" |> test_expr
             @test "a = b where c" |> test_expr
@@ -511,11 +519,13 @@ end
                 @test "[x;;]" |> test_expr
                 @test "[x;; y;;    z]" |> test_expr
                 @test "[x;;; y;;;z]" |> test_expr
-            end
-
-            @testset "ncat" begin
+                @test "[x;;; y;;;z]'" |> test_expr
                 @test "[1 2; 3 4]" |> test_expr
                 @test "[1;2;;3;4;;5;6;;;;9]" |> test_expr
+                if VERSION > v"1.7-"
+                    @test "[let; x; end;; y]" |> test_expr
+                    @test "[let; x; end;;;; y]" |> test_expr
+                end
             end
 
             @testset "typed_ncat" begin
@@ -527,20 +537,40 @@ end
                 @test "t[x;;; y;;;z]" |> test_expr
                 @test "t[x;;\ny]" |> test_expr
                 @test "t[x y;;\nz a]" |> test_expr
+                @test "t[x y;;\nz a]'" |> test_expr
+                @test "t[let; x; end;; y]" |> test_expr
+                @test "t[let; x; end;;;; y]" |> test_expr
             end
         end
 
         @testset "hcat" begin
             @test "[x y]" |> test_expr
+            @test "[let; x; end y]" |> test_expr
+            @test "[let; x; end; y]" |> test_expr
         end
 
         @testset "typed_hcat" begin
             @test "t[x y]" |> test_expr
+            @test "t[let; x; end y]" |> test_expr
+            @test "t[let; x; end; y]" |> test_expr
         end
 
         @testset "Comprehension" begin
             @test "[i for i = 1:10]" |> test_expr
             @test "Int[i for i = 1:10]" |> test_expr
+            @test "[let;x;end for x in x]" |> test_expr
+            @test "[let; x; end for x in x]" |> test_expr
+            @test "[let x=x; x+x; end for x in x]" |> test_expr
+            if VERSION > v"1.7-"
+                @test """[
+                    [
+                        let l = min((d-k),k);
+                            binomial(d-l,l);
+                        end; for k in 1:d-1
+                    ] for d in 2:9
+                ]
+                """ |> test_expr
+            end
         end
     end