From c70b231f31c9d7d7353aa1c3feef09230ad053f6 Mon Sep 17 00:00:00 2001 From: Fengyang Wang Date: Thu, 26 Jan 2017 18:27:46 -0500 Subject: [PATCH 1/6] Consider all whitespace in space-sensitive context --- src/julia-parser.scm | 11 +++++++---- test/parse.jl | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/julia-parser.scm b/src/julia-parser.scm index deacd61d90f70..370aca386a562 100644 --- a/src/julia-parser.scm +++ b/src/julia-parser.scm @@ -529,6 +529,9 @@ (begin0 (ts:last-tok s) (ts:set-tok! s #f)))) +(define (space-before-next-token? s) + (or (skip-ws (ts:port s) #f) (eqv? #\newline (peek-char (ts:port s))))) + ;; --- misc --- (define (syntax-deprecation s what instead) @@ -703,7 +706,7 @@ ((and range-colon-enabled (eq? t ':)) (take-token s) (if (and space-sensitive spc - (or (peek-token s) #t) (not (ts:space? s))) + (not (space-before-next-token? s))) ;; "a :b" in space sensitive mode (begin (ts:put-back! s ':) ex) @@ -751,7 +754,7 @@ (begin (take-token s) (if (eq? t '~) (if (and space-sensitive (ts:space? s) - (not (eqv? (peek-char (ts:port s)) #\ ))) + (not (space-before-next-token? s))) (begin (ts:put-back! s t) ex) (list 'call t ex (parse-assignment s down))) @@ -785,7 +788,7 @@ (begin (take-token s) (cond ((and space-sensitive spc (memq t unary-and-binary-ops) - (not (eqv? (peek-char (ts:port s)) #\ ))) + (not (space-before-next-token? s))) ;; here we have "x -y" (ts:put-back! s t) (reverse! chain)) @@ -803,7 +806,7 @@ (begin (take-token s) (cond ((and space-sensitive spc (memq t unary-and-binary-ops) - (not (eqv? (peek-char (ts:port s)) #\ ))) + (not (space-before-next-token? s))) ;; here we have "x -y" (ts:put-back! s t) ex) diff --git a/test/parse.jl b/test/parse.jl index 35b0e5f96a56f..bb27fa63fd12f 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -933,3 +933,24 @@ end # a :block for its body short_where_call = :(f(x::T) where T = T) @test short_where_call.args[2].head == :block + +# issue #16594 +@test length(:(@test 1 + + 1 == 2).args) == 2 +@test [1 + + 1] == [2] +@test [1 +1] == [1 1] +@test length(:(@x 1 +1 -1).args) == 4 +@test length(:(@x 1 + 1 -1).args) == 3 +@test length(:(@x 1 + 1 - 1).args) == 2 +@test length(:(@x 1 + + 1 - + 1).args) == 2 +@test length(:(@x 1 + + 1 + + 1).args) == 2 +@test length(:([x .+ + y]).args) == 1 + +# line break in : expression disallowed +@test_throws ParseError parse("[1 :\n2] == [1:2]""") From 9f286b959b712837a52f62dcde6fb18d6b91fb24 Mon Sep 17 00:00:00 2001 From: Fengyang Wang Date: Sun, 10 Sep 2017 20:05:30 -0400 Subject: [PATCH 2/6] oops --- src/julia-parser.scm | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/julia-parser.scm b/src/julia-parser.scm index 9694daf33063a..37e18bdbb9ede 100644 --- a/src/julia-parser.scm +++ b/src/julia-parser.scm @@ -1309,8 +1309,7 @@ ((let) (let ((binds (if (memv (peek-token s) '(#\newline #\;)) '() - (parse-comma-separated- - s s)))) + (parse-comma-separated-assignments s)))) (if (not (or (eof-object? (peek-token s)) (memv (peek-token s) '(#\newline #\; end)))) (error "let variables should end in \";\" or newline")) From 2a1cb92acab20c50085fbd78817015f2d86faedb Mon Sep 17 00:00:00 2001 From: Fengyang Wang Date: Thu, 19 Jul 2018 20:00:54 -0500 Subject: [PATCH 3/6] Fix bitrotted parse tests and odd string literal --- test/parse.jl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/parse.jl b/test/parse.jl index 58e823bc6b497..1c03f3635b7c2 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -228,24 +228,24 @@ end # issue #16594 @test length(:(@test 1 + - 1 == 2).args) == 2 + 1 == 2).args) == 3 @test [1 + 1] == [2] @test [1 +1] == [1 1] -@test length(:(@x 1 +1 -1).args) == 4 -@test length(:(@x 1 + 1 -1).args) == 3 -@test length(:(@x 1 + 1 - 1).args) == 2 +@test length(:(@x 1 +1 -1).args) == 5 +@test length(:(@x 1 + 1 -1).args) == 4 +@test length(:(@x 1 + 1 - 1).args) == 3 @test length(:(@x 1 + 1 - - 1).args) == 2 + 1).args) == 3 @test length(:(@x 1 + 1 + - 1).args) == 2 + 1).args) == 3 @test length(:([x .+ - y]).args) == 1 + y]).args) == 2 # line break in : expression disallowed -@test_throws ParseError Meta.parse("[1 :\n2] == [1:2]""") +@test_throws ParseError Meta.parse("[1 :\n2] == [1:2]") @test tryparse(Float64, "1.23") === 1.23 @test tryparse(Float32, "1.23") === 1.23f0 From 17c7eb224a236245ad1407e617265db856721620 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Fri, 20 Jul 2018 09:49:53 -0400 Subject: [PATCH 4/6] fix another test --- test/parse.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/parse.jl b/test/parse.jl index 1c03f3635b7c2..4debc1e4fe775 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -241,8 +241,8 @@ end @test length(:(@x 1 + 1 + 1).args) == 3 -@test length(:([x .+ - y]).args) == 2 +@test :([x .+ + y]) == :([x .+ y]) # line break in : expression disallowed @test_throws ParseError Meta.parse("[1 :\n2] == [1:2]") From e3fed413447b2b51bd76951939bd9dc7afe67dd6 Mon Sep 17 00:00:00 2001 From: Fengyang Wang Date: Sat, 21 Jul 2018 01:39:38 -0500 Subject: [PATCH 5/6] ParseError to Meta.ParseError --- test/parse.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/parse.jl b/test/parse.jl index 4debc1e4fe775..e89f305abf017 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -245,7 +245,7 @@ end y]) == :([x .+ y]) # line break in : expression disallowed -@test_throws ParseError Meta.parse("[1 :\n2] == [1:2]") +@test_throws Meta.ParseError Meta.parse("[1 :\n2] == [1:2]") @test tryparse(Float64, "1.23") === 1.23 @test tryparse(Float32, "1.23") === 1.23f0 From 0ebee060280d20fdc5b0186d23dd44744acd0e1f Mon Sep 17 00:00:00 2001 From: Fengyang Wang Date: Sun, 22 Jul 2018 17:25:58 -0500 Subject: [PATCH 6/6] Improve parse tests added --- test/parse.jl | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/test/parse.jl b/test/parse.jl index e89f305abf017..af3a26d582f5c 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -227,20 +227,25 @@ end @test_throws ArgumentError Base.tryparse_internal(Bool, "foo", 1, 2, 10, true) # issue #16594 -@test length(:(@test 1 + - 1 == 2).args) == 3 +@test Meta.parse("@x a + \nb") == Meta.parse("@x a +\nb") @test [1 + 1] == [2] @test [1 +1] == [1 1] -@test length(:(@x 1 +1 -1).args) == 5 -@test length(:(@x 1 + 1 -1).args) == 4 -@test length(:(@x 1 + 1 - 1).args) == 3 -@test length(:(@x 1 + - 1 - - 1).args) == 3 -@test length(:(@x 1 + - 1 + - 1).args) == 3 + +# issue #16594, note for the macro tests, order is important +# because the line number is included as part of the expression +# (i.e. both macros must start on the same line) +@test :(@test((1+1) == 2)) == :(@test 1 + + 1 == 2) +@test :(@x 1 +1 -1) == :(@x(1, +1, -1)) +@test :(@x 1 + 1 -1) == :(@x(1+1, -1)) +@test :(@x 1 + 1 - 1) == :(@x(1 + 1 - 1)) +@test :(@x(1 + 1 - 1)) == :(@x 1 + + 1 - + 1) +@test :(@x(1 + 1 + 1)) == :(@x 1 + + 1 + + 1) @test :([x .+ y]) == :([x .+ y])