From 6a0e87eb387c13f1d6f78e1736c23cdb4a06b504 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 27 Feb 2020 01:43:13 -0800 Subject: [PATCH 01/77] cleanup Ordinal (#13501) --- compiler/condsyms.nim | 1 + lib/pure/hashes.nim | 2 +- lib/system.nim | 12 ++++++++++++ lib/system/arithmetics.nim | 4 ++-- lib/system/basic_types.nim | 6 +----- tests/system/tsystem_misc.nim | 17 +++++++++++++++++ tests/varres/tprevent_forloopvar_mutations.nim | 4 ++-- 7 files changed, 36 insertions(+), 10 deletions(-) diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 344f6fd5aeb50..d82f4811dd04e 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -104,6 +104,7 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasCursor") defineSymbol("nimHasExceptionsQuery") defineSymbol("nimHasIsNamedTuple") + defineSymbol("nimHashOrdinalFixed") when defined(nimHasLibFFI): # Renaming as we can't conflate input vs output define flags; e.g. this diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index b55a7865dd909..52a724d7bbd69 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -112,7 +112,7 @@ proc hash*[T: proc](x: T): Hash {.inline.} = else: result = hash(pointer(x)) -proc hash*(x: int|int64|uint|uint64|char|Ordinal): Hash {.inline.} = +proc hash*[T: Ordinal](x: T): Hash {.inline.} = ## Efficient hashing of integers. cast[Hash](ord(x)) diff --git a/lib/system.nim b/lib/system.nim index 69083673bebe0..51ff3af400341 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -87,6 +87,18 @@ proc defined*(x: untyped): bool {.magic: "Defined", noSideEffect, compileTime.} ## # Do here programmer friendly expensive sanity checks. ## # Put here the normal code +when defined(nimHashOrdinalFixed): + type + Ordinal*[T] {.magic: Ordinal.} ## Generic ordinal type. Includes integer, + ## bool, character, and enumeration types + ## as well as their subtypes. See also + ## `SomeOrdinal`. +else: + # bootstrap <= 0.20.0 + type + OrdinalImpl[T] {.magic: Ordinal.} + Ordinal* = OrdinalImpl | uint | uint64 + when defined(nimHasRunnableExamples): proc runnableExamples*(body: untyped) {.magic: "RunnableExamples".} ## A section you should use to mark `runnable example`:idx: code with. diff --git a/lib/system/arithmetics.nim b/lib/system/arithmetics.nim index ba9ade192dd3d..757a813e8619e 100644 --- a/lib/system/arithmetics.nim +++ b/lib/system/arithmetics.nim @@ -22,7 +22,7 @@ proc pred*[T: Ordinal](x: T, y = 1): T {.magic: "Pred", noSideEffect.} ## echo pred(5) # => 4 ## echo pred(5, 3) # => 2 -proc inc*[T: Ordinal|uint|uint64](x: var T, y = 1) {.magic: "Inc", noSideEffect.} +proc inc*[T: Ordinal](x: var T, y = 1) {.magic: "Inc", noSideEffect.} ## Increments the ordinal ``x`` by ``y``. ## ## If such a value does not exist, ``OverflowError`` is raised or a compile @@ -33,7 +33,7 @@ proc inc*[T: Ordinal|uint|uint64](x: var T, y = 1) {.magic: "Inc", noSideEffect. ## inc(i) # i <- 3 ## inc(i, 3) # i <- 6 -proc dec*[T: Ordinal|uint|uint64](x: var T, y = 1) {.magic: "Dec", noSideEffect.} +proc dec*[T: Ordinal](x: var T, y = 1) {.magic: "Dec", noSideEffect.} ## Decrements the ordinal ``x`` by ``y``. ## ## If such a value does not exist, ``OverflowError`` is raised or a compile diff --git a/lib/system/basic_types.nim b/lib/system/basic_types.nim index 39ad0a76c5207..a6bf69ebc2f5d 100644 --- a/lib/system/basic_types.nim +++ b/lib/system/basic_types.nim @@ -20,10 +20,6 @@ const off* = false ## Alias for ``false``. type - Ordinal*[T] {.magic: Ordinal.} ## Generic ordinal type. Includes integer, - ## bool, character, and enumeration types - ## as well as their subtypes. - SomeSignedInt* = int|int8|int16|int32|int64 ## Type class matching all signed integer types. @@ -35,7 +31,7 @@ type SomeOrdinal* = int|int8|int16|int32|int64|bool|enum|uint|uint8|uint16|uint32|uint64 ## Type class matching all ordinal types; however this includes enums with - ## holes. + ## holes. See also `Ordinal` BiggestInt* = int64 ## is an alias for the biggest signed integer type the Nim compiler diff --git a/tests/system/tsystem_misc.nim b/tests/system/tsystem_misc.nim index 56af97f36c52f..fff8b7022ef50 100644 --- a/tests/system/tsystem_misc.nim +++ b/tests/system/tsystem_misc.nim @@ -193,3 +193,20 @@ block: a = {k1} b = {k1,k2} doAssert a < b + + +block: # Ordinal + doAssert int is Ordinal + doAssert uint is Ordinal + doAssert int64 is Ordinal + doAssert uint64 is Ordinal + doAssert char is Ordinal + type Foo = enum k1, k2 + doAssert Foo is Ordinal + doAssert Foo is SomeOrdinal + doAssert enum is SomeOrdinal + + # these fail: + # doAssert enum is Ordinal # fails + # doAssert Ordinal is SomeOrdinal + # doAssert SomeOrdinal is Ordinal diff --git a/tests/varres/tprevent_forloopvar_mutations.nim b/tests/varres/tprevent_forloopvar_mutations.nim index 39819165880fc..ac62608aff07e 100644 --- a/tests/varres/tprevent_forloopvar_mutations.nim +++ b/tests/varres/tprevent_forloopvar_mutations.nim @@ -3,9 +3,9 @@ discard """ line: 17 nimout: '''type mismatch: got but expected one of: -proc inc[T: Ordinal | uint | uint64](x: var T; y = 1) +proc inc[T: Ordinal](x: var T; y = 1) first type mismatch at position: 1 - required type for x: var T: Ordinal or uint or uint64 + required type for x: var T: Ordinal but expression 'i' is immutable, not 'var' expression: inc i From e84e01cb8cf8196ce70d9944f1262440a45e552d Mon Sep 17 00:00:00 2001 From: solo989 Date: Thu, 27 Feb 2020 02:08:57 -0800 Subject: [PATCH 02/77] Update pegs.nim to work at compiletime. No range errors. (#13459) --- lib/pure/pegs.nim | 350 ++++++++++++++++++++++++---------------------- 1 file changed, 186 insertions(+), 164 deletions(-) diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 3c3941285c83c..4f723b4af8e9f 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -1420,7 +1420,7 @@ type PegLexer {.inheritable.} = object ## the lexer object. bufpos: int ## the current position within the buffer - buf: cstring ## the buffer itself + buf: string ## the buffer itself lineNumber: int ## the current line number lineStart: int ## index of last line start in buffer colOffset: int ## column to add @@ -1481,6 +1481,9 @@ proc handleHexChar(c: var PegLexer, xi: var int) = proc getEscapedChar(c: var PegLexer, tok: var Token) = inc(c.bufpos) + if c.bufpos >= len(c.buf): + tok.kind = tkInvalid + return case c.buf[c.bufpos] of 'r', 'R', 'c', 'C': add(tok.literal, '\c') @@ -1508,6 +1511,9 @@ proc getEscapedChar(c: var PegLexer, tok: var Token) = inc(c.bufpos) of 'x', 'X': inc(c.bufpos) + if c.bufpos >= len(c.buf): + tok.kind = tkInvalid + return var xi = 0 handleHexChar(c, xi) handleHexChar(c, xi) @@ -1517,7 +1523,7 @@ proc getEscapedChar(c: var PegLexer, tok: var Token) = var val = ord(c.buf[c.bufpos]) - ord('0') inc(c.bufpos) var i = 1 - while (i <= 3) and (c.buf[c.bufpos] in {'0'..'9'}): + while (c.bufpos < len(c.buf)) and (i <= 3) and (c.buf[c.bufpos] in {'0'..'9'}): val = val * 10 + ord(c.buf[c.bufpos]) - ord('0') inc(c.bufpos) inc(i) @@ -1571,7 +1577,7 @@ proc getString(c: var PegLexer, tok: var Token) = proc getDollar(c: var PegLexer, tok: var Token) = var pos = c.bufpos + 1 - if c.buf[pos] in {'0'..'9'}: + if pos < c.buf.len and c.buf[pos] in {'0'..'9'}: tok.kind = tkBackref tok.index = 0 while pos < c.buf.len and c.buf[pos] in {'0'..'9'}: @@ -1586,54 +1592,55 @@ proc getCharSet(c: var PegLexer, tok: var Token) = tok.charset = {} var pos = c.bufpos + 1 var caret = false - if c.buf[pos] == '^': - inc(pos) - caret = true - while pos < c.buf.len: - var ch: char - case c.buf[pos] - of ']': - if pos < c.buf.len: inc(pos) - break - of '\\': - c.bufpos = pos - getEscapedChar(c, tok) - pos = c.bufpos - ch = tok.literal[tok.literal.len-1] - of '\C', '\L', '\0': - tok.kind = tkInvalid - break - else: - ch = c.buf[pos] + if pos < c.buf.len: + if c.buf[pos] == '^': inc(pos) - incl(tok.charset, ch) - if c.buf[pos] == '-': - if pos+1 < c.buf.len and c.buf[pos+1] == ']': - incl(tok.charset, '-') - inc(pos) + caret = true + while pos < c.buf.len: + var ch: char + case c.buf[pos] + of ']': + if pos < c.buf.len: inc(pos) + break + of '\\': + c.bufpos = pos + getEscapedChar(c, tok) + pos = c.bufpos + ch = tok.literal[tok.literal.len-1] + of '\C', '\L', '\0': + tok.kind = tkInvalid + break else: - if pos+1 < c.buf.len: + ch = c.buf[pos] + inc(pos) + incl(tok.charset, ch) + if c.buf[pos] == '-': + if pos+1 < c.buf.len and c.buf[pos+1] == ']': + incl(tok.charset, '-') inc(pos) - else: - break - var ch2: char - case c.buf[pos] - of '\\': - c.bufpos = pos - getEscapedChar(c, tok) - pos = c.bufpos - ch2 = tok.literal[tok.literal.len-1] - of '\C', '\L', '\0': - tok.kind = tkInvalid - break else: if pos+1 < c.buf.len: - ch2 = c.buf[pos] inc(pos) else: break - for i in ord(ch)+1 .. ord(ch2): - incl(tok.charset, chr(i)) + var ch2: char + case c.buf[pos] + of '\\': + c.bufpos = pos + getEscapedChar(c, tok) + pos = c.bufpos + ch2 = tok.literal[tok.literal.len-1] + of '\C', '\L', '\0': + tok.kind = tkInvalid + break + else: + if pos+1 < c.buf.len: + ch2 = c.buf[pos] + inc(pos) + else: + break + for i in ord(ch)+1 .. ord(ch2): + incl(tok.charset, chr(i)) c.bufpos = pos if caret: tok.charset = {'\1'..'\xFF'} - tok.charset @@ -1661,6 +1668,13 @@ proc getTok(c: var PegLexer, tok: var Token) = setLen(tok.literal, 0) skip(c) + if c.bufpos >= c.buf.len: + tok.kind = tkEof + tok.literal = "[EOF]" + add(tok.literal, '\0') + inc(c.bufpos) + return + case c.buf[c.bufpos] of '{': inc(c.bufpos) @@ -1700,6 +1714,8 @@ proc getTok(c: var PegLexer, tok: var Token) = of '$': getDollar(c, tok) of 'a'..'z', 'A'..'Z', '\128'..'\255': getSymbol(c, tok) + if c.bufpos >= c.buf.len: + return if c.buf[c.bufpos] in {'\'', '"'} or c.buf[c.bufpos] == '$' and c.bufpos+1 < c.buf.len and c.buf[c.bufpos+1] in {'0'..'9'}: @@ -1768,7 +1784,9 @@ proc arrowIsNextTok(c: PegLexer): bool = # the only look ahead we need var pos = c.bufpos while pos < c.buf.len and c.buf[pos] in {'\t', ' '}: inc(pos) - result = c.buf[pos] == '<' and (pos+1 < c.buf.len) and c.buf[pos+1] == '-' + if pos+1 >= c.buf.len: + return + result = c.buf[pos] == '<' and c.buf[pos+1] == '-' # ----------------------------- parser ---------------------------------------- @@ -2038,141 +2056,145 @@ proc escapePeg*(s: string): string = if inQuote: result.add('\'') when isMainModule: - assert escapePeg("abc''def'") == r"'abc'\x27\x27'def'\x27" - assert match("(a b c)", peg"'(' @ ')'") - assert match("W_HI_Le", peg"\y 'while'") - assert(not match("W_HI_L", peg"\y 'while'")) - assert(not match("W_HI_Le", peg"\y v'while'")) - assert match("W_HI_Le", peg"y'while'") - - assert($ +digits == $peg"\d+") - assert "0158787".match(peg"\d+") - assert "ABC 0232".match(peg"\w+\s+\d+") - assert "ABC".match(peg"\d+ / \w+") - - var accum: seq[string] = @[] - for word in split("00232this02939is39an22example111", peg"\d+"): - accum.add(word) - assert(accum == @["this", "is", "an", "example"]) - - assert matchLen("key", ident) == 3 - - var pattern = sequence(ident, *whitespace, term('='), *whitespace, ident) - assert matchLen("key1= cal9", pattern) == 11 - - var ws = newNonTerminal("ws", 1, 1) - ws.rule = *whitespace - - var expr = newNonTerminal("expr", 1, 1) - expr.rule = sequence(capture(ident), *sequence( - nonterminal(ws), term('+'), nonterminal(ws), nonterminal(expr))) - - var c: Captures - var s = "a+b + c +d+e+f" - assert rawMatch(s, expr.rule, 0, c) == len(s) - var a = "" - for i in 0..c.ml-1: - a.add(substr(s, c.matches[i][0], c.matches[i][1])) - assert a == "abcdef" - #echo expr.rule - - #const filename = "lib/devel/peg/grammar.txt" - #var grammar = parsePeg(newFileStream(filename, fmRead), filename) - #echo "a <- [abc]*?".match(grammar) - assert find("_____abc_______", term("abc"), 2) == 5 - assert match("_______ana", peg"A <- 'ana' / . A") - assert match("abcs%%%", peg"A <- ..A / .A / '%'") - - var matches: array[0..MaxSubpatterns-1, string] - if "abc" =~ peg"{'a'}'bc' 'xyz' / {\ident}": - assert matches[0] == "abc" - else: - assert false - - var g2 = peg"""S <- A B / C D - A <- 'a'+ - B <- 'b'+ - C <- 'c'+ - D <- 'd'+ - """ - assert($g2 == "((A B) / (C D))") - assert match("cccccdddddd", g2) - assert("var1=key; var2=key2".replacef(peg"{\ident}'='{\ident}", "$1<-$2$2") == - "var1<-keykey; var2<-key2key2") - assert("var1=key; var2=key2".replace(peg"{\ident}'='{\ident}", "$1<-$2$2") == - "$1<-$2$2; $1<-$2$2") - assert "var1=key; var2=key2".endsWith(peg"{\ident}'='{\ident}") - - if "aaaaaa" =~ peg"'aa' !. / ({'a'})+": - assert matches[0] == "a" - else: - assert false + proc pegsTest() = + assert escapePeg("abc''def'") == r"'abc'\x27\x27'def'\x27" + assert match("(a b c)", peg"'(' @ ')'") + assert match("W_HI_Le", peg"\y 'while'") + assert(not match("W_HI_L", peg"\y 'while'")) + assert(not match("W_HI_Le", peg"\y v'while'")) + assert match("W_HI_Le", peg"y'while'") + + assert($ +digits == $peg"\d+") + assert "0158787".match(peg"\d+") + assert "ABC 0232".match(peg"\w+\s+\d+") + assert "ABC".match(peg"\d+ / \w+") + + var accum: seq[string] = @[] + for word in split("00232this02939is39an22example111", peg"\d+"): + accum.add(word) + assert(accum == @["this", "is", "an", "example"]) + + assert matchLen("key", ident) == 3 + + var pattern = sequence(ident, *whitespace, term('='), *whitespace, ident) + assert matchLen("key1= cal9", pattern) == 11 + + var ws = newNonTerminal("ws", 1, 1) + ws.rule = *whitespace + + var expr = newNonTerminal("expr", 1, 1) + expr.rule = sequence(capture(ident), *sequence( + nonterminal(ws), term('+'), nonterminal(ws), nonterminal(expr))) + + var c: Captures + var s = "a+b + c +d+e+f" + assert rawMatch(s, expr.rule, 0, c) == len(s) + var a = "" + for i in 0..c.ml-1: + a.add(substr(s, c.matches[i][0], c.matches[i][1])) + assert a == "abcdef" + #echo expr.rule + + #const filename = "lib/devel/peg/grammar.txt" + #var grammar = parsePeg(newFileStream(filename, fmRead), filename) + #echo "a <- [abc]*?".match(grammar) + assert find("_____abc_______", term("abc"), 2) == 5 + assert match("_______ana", peg"A <- 'ana' / . A") + assert match("abcs%%%", peg"A <- ..A / .A / '%'") + + var matches: array[0..MaxSubpatterns-1, string] + if "abc" =~ peg"{'a'}'bc' 'xyz' / {\ident}": + assert matches[0] == "abc" + else: + assert false + + var g2 = peg"""S <- A B / C D + A <- 'a'+ + B <- 'b'+ + C <- 'c'+ + D <- 'd'+ + """ + assert($g2 == "((A B) / (C D))") + assert match("cccccdddddd", g2) + assert("var1=key; var2=key2".replacef(peg"{\ident}'='{\ident}", "$1<-$2$2") == + "var1<-keykey; var2<-key2key2") + assert("var1=key; var2=key2".replace(peg"{\ident}'='{\ident}", "$1<-$2$2") == + "$1<-$2$2; $1<-$2$2") + assert "var1=key; var2=key2".endsWith(peg"{\ident}'='{\ident}") + + if "aaaaaa" =~ peg"'aa' !. / ({'a'})+": + assert matches[0] == "a" + else: + assert false - if match("abcdefg", peg"c {d} ef {g}", matches, 2): - assert matches[0] == "d" - assert matches[1] == "g" - else: - assert false + if match("abcdefg", peg"c {d} ef {g}", matches, 2): + assert matches[0] == "d" + assert matches[1] == "g" + else: + assert false - accum = @[] - for x in findAll("abcdef", peg".", 3): - accum.add(x) - assert(accum == @["d", "e", "f"]) + accum = @[] + for x in findAll("abcdef", peg".", 3): + accum.add(x) + assert(accum == @["d", "e", "f"]) - for x in findAll("abcdef", peg"^{.}", 3): - assert x == "d" + for x in findAll("abcdef", peg"^{.}", 3): + assert x == "d" - if "f(a, b)" =~ peg"{[0-9]+} / ({\ident} '(' {@} ')')": - assert matches[0] == "f" - assert matches[1] == "a, b" - else: - assert false + if "f(a, b)" =~ peg"{[0-9]+} / ({\ident} '(' {@} ')')": + assert matches[0] == "f" + assert matches[1] == "a, b" + else: + assert false - assert match("eine übersicht und außerdem", peg"(\letter \white*)+") - # ß is not a lower cased letter?! - assert match("eine übersicht und auerdem", peg"(\lower \white*)+") - assert match("EINE ÜBERSICHT UND AUSSERDEM", peg"(\upper \white*)+") - assert(not match("456678", peg"(\letter)+")) + assert match("eine übersicht und außerdem", peg"(\letter \white*)+") + # ß is not a lower cased letter?! + assert match("eine übersicht und auerdem", peg"(\lower \white*)+") + assert match("EINE ÜBERSICHT UND AUSSERDEM", peg"(\upper \white*)+") + assert(not match("456678", peg"(\letter)+")) - assert("var1 = key; var2 = key2".replacef( - peg"\skip(\s*) {\ident}'='{\ident}", "$1<-$2$2") == - "var1<-keykey;var2<-key2key2") + assert("var1 = key; var2 = key2".replacef( + peg"\skip(\s*) {\ident}'='{\ident}", "$1<-$2$2") == + "var1<-keykey;var2<-key2key2") - assert match("prefix/start", peg"^start$", 7) + assert match("prefix/start", peg"^start$", 7) - if "foo" =~ peg"{'a'}?.*": - assert matches[0].len == 0 - else: assert false + if "foo" =~ peg"{'a'}?.*": + assert matches[0].len == 0 + else: assert false - if "foo" =~ peg"{''}.*": - assert matches[0] == "" - else: assert false + if "foo" =~ peg"{''}.*": + assert matches[0] == "" + else: assert false - if "foo" =~ peg"{'foo'}": - assert matches[0] == "foo" - else: assert false + if "foo" =~ peg"{'foo'}": + assert matches[0] == "foo" + else: assert false - let empty_test = peg"^\d*" - let str = "XYZ" + let empty_test = peg"^\d*" + let str = "XYZ" - assert(str.find(empty_test) == 0) - assert(str.match(empty_test)) + assert(str.find(empty_test) == 0) + assert(str.match(empty_test)) - proc handleMatches*(m: int, n: int, c: openArray[string]): string = - result = "" + proc handleMatches(m: int, n: int, c: openArray[string]): string = + result = "" - if m > 0: - result.add ", " + if m > 0: + result.add ", " - result.add case n: - of 2: toLowerAscii(c[0]) & ": '" & c[1] & "'" - of 1: toLowerAscii(c[0]) & ": ''" - else: "" + result.add case n: + of 2: toLowerAscii(c[0]) & ": '" & c[1] & "'" + of 1: toLowerAscii(c[0]) & ": ''" + else: "" - assert("Var1=key1;var2=Key2; VAR3". - replace(peg"{\ident}('='{\ident})* ';'* \s*", - handleMatches) == "var1: 'key1', var2: 'Key2', var3: ''") + assert("Var1=key1;var2=Key2; VAR3". + replace(peg"{\ident}('='{\ident})* ';'* \s*", + handleMatches) == "var1: 'key1', var2: 'Key2', var3: ''") - doAssert "test1".match(peg"""{@}$""") - doAssert "test2".match(peg"""{(!$ .)*} $""") + doAssert "test1".match(peg"""{@}$""") + doAssert "test2".match(peg"""{(!$ .)*} $""") + pegsTest() + static: + pegsTest() From 81b0718a8fca1ea3fa619ab75bb4cc9275026a4f Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 24 Feb 2020 01:24:47 -0800 Subject: [PATCH 03/77] make CI tests faster + more precise --- tests/gc/gcleak2.nim | 77 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 5 deletions(-) diff --git a/tests/gc/gcleak2.nim b/tests/gc/gcleak2.nim index fe1718aef6b45..eb71fe55f88c3 100644 --- a/tests/gc/gcleak2.nim +++ b/tests/gc/gcleak2.nim @@ -2,6 +2,11 @@ discard """ outputsub: "no leak: " """ +## this test makes sure getOccupiedMem() is constant after a few iterations +## for --gc:bohem, --gc:markAndSweep, --gc:arc, --gc:orc. +## and for other gc, it consists of cycles with constant min/max after a few cycles. +## This ensures we have no leaks. + when defined(GC_setMaxPause): GC_setMaxPause 2_000 @@ -14,13 +19,75 @@ proc makeObj(): TTestObj = result.x = "Hello" result.s = @[1,2,3] +const collectAlways = defined(gcMarkAndSweep) or defined(boehmgc) +const isDeterministic = collectAlways or defined(gcArc) or defined(gcOrc) + ## when isDeterministic, we expect memory to reach a fixed point + ## after `numIterStable` iterations + +var memMax = 0 # peak + +when isDeterministic: + const numIterStable = 3 + # stabilize after this many iterations +else: + const numCycleStable = when defined(useRealtimeGC): 6 else: 4 + # stabilize after this many cycles; empirically determined + # after running all combinations of gc's with / without -d:release + var memPrevious = 0 + var memMin = int.high # right after a peak + var numCollections = 0 + +let numIter = when isDeterministic: 1_000 else: 1_000_000 + ## full collection is expensive, and the memory doesn't change after + ## each iteration so there's no point in a large `numIter` (this was taking + ## 350s for `nim c -r -d:release --gc:boehm` + `nim c -r --gc:boehm`, 50% + ## of running time of `testament/testament all`. + proc inProc() = - for i in 1 .. 1_000_000: - when defined(gcMarkAndSweep) or defined(boehmgc): - GC_fullcollect() + for i in 1 .. numIter: + when collectAlways: GC_fullcollect() var obj: TTestObj obj = makeObj() - if getOccupiedMem() > 300_000: quit("still a leak!") + let mem = getOccupiedMem() + when isDeterministic: + if i <= numIterStable: + memMax = mem + doAssert memMax <= 50_000 # adjust as needed + else: + # memory shouldn't increase after 1st few iterations + # on linux 386 it somehow takes 3 iters to converge + doAssert mem <= memMax + else: + if mem < memPrevious: + # a collection happened, it peaked at memPrevious + # echo (mem, memMin, memMax, numCollections, numIter, i) # for debugging + numCollections.inc + if numCollections <= numCycleStable: + # this is the 1st few collections, we update the min/max + doAssert memPrevious < 300_000 # adjust as needed + if memMin < mem: # `<` intentional; the valley may increase + memMin = mem + if memMax < memPrevious: + memMax = memPrevious + else: + # after a collection, we always go back to same level + doAssert mem <= memMin, $(mem, memMin) + + if numCollections >= numCycleStable: + # after a few cycles, the max stabilizes + doAssert mem <= memMax, $(mem, memMax) + + memPrevious = mem inProc() -echo "no leak: ", getOccupiedMem() +let mem = getOccupiedMem() +var msg = "no leak: " +when isDeterministic: + msg.add $(mem, memMax) + echo msg +else: + msg.add $(mem, memMin, memMax, numCollections, numIter) + echo msg + # make sure some collections did happen, otherwise the previous tests + # are meaningless + doAssert numCollections > 1000 # 3999 on local OSX; leaving some slack From 9b8c9abeade5f37b56f1c3824ea179efa9ce47e5 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 27 Feb 2020 01:33:06 -0800 Subject: [PATCH 04/77] revert changes to tests/gc/gcleak2.nim --- tests/gc/gcleak2.nim | 77 +++----------------------------------------- 1 file changed, 5 insertions(+), 72 deletions(-) diff --git a/tests/gc/gcleak2.nim b/tests/gc/gcleak2.nim index eb71fe55f88c3..fe1718aef6b45 100644 --- a/tests/gc/gcleak2.nim +++ b/tests/gc/gcleak2.nim @@ -2,11 +2,6 @@ discard """ outputsub: "no leak: " """ -## this test makes sure getOccupiedMem() is constant after a few iterations -## for --gc:bohem, --gc:markAndSweep, --gc:arc, --gc:orc. -## and for other gc, it consists of cycles with constant min/max after a few cycles. -## This ensures we have no leaks. - when defined(GC_setMaxPause): GC_setMaxPause 2_000 @@ -19,75 +14,13 @@ proc makeObj(): TTestObj = result.x = "Hello" result.s = @[1,2,3] -const collectAlways = defined(gcMarkAndSweep) or defined(boehmgc) -const isDeterministic = collectAlways or defined(gcArc) or defined(gcOrc) - ## when isDeterministic, we expect memory to reach a fixed point - ## after `numIterStable` iterations - -var memMax = 0 # peak - -when isDeterministic: - const numIterStable = 3 - # stabilize after this many iterations -else: - const numCycleStable = when defined(useRealtimeGC): 6 else: 4 - # stabilize after this many cycles; empirically determined - # after running all combinations of gc's with / without -d:release - var memPrevious = 0 - var memMin = int.high # right after a peak - var numCollections = 0 - -let numIter = when isDeterministic: 1_000 else: 1_000_000 - ## full collection is expensive, and the memory doesn't change after - ## each iteration so there's no point in a large `numIter` (this was taking - ## 350s for `nim c -r -d:release --gc:boehm` + `nim c -r --gc:boehm`, 50% - ## of running time of `testament/testament all`. - proc inProc() = - for i in 1 .. numIter: - when collectAlways: GC_fullcollect() + for i in 1 .. 1_000_000: + when defined(gcMarkAndSweep) or defined(boehmgc): + GC_fullcollect() var obj: TTestObj obj = makeObj() - let mem = getOccupiedMem() - when isDeterministic: - if i <= numIterStable: - memMax = mem - doAssert memMax <= 50_000 # adjust as needed - else: - # memory shouldn't increase after 1st few iterations - # on linux 386 it somehow takes 3 iters to converge - doAssert mem <= memMax - else: - if mem < memPrevious: - # a collection happened, it peaked at memPrevious - # echo (mem, memMin, memMax, numCollections, numIter, i) # for debugging - numCollections.inc - if numCollections <= numCycleStable: - # this is the 1st few collections, we update the min/max - doAssert memPrevious < 300_000 # adjust as needed - if memMin < mem: # `<` intentional; the valley may increase - memMin = mem - if memMax < memPrevious: - memMax = memPrevious - else: - # after a collection, we always go back to same level - doAssert mem <= memMin, $(mem, memMin) - - if numCollections >= numCycleStable: - # after a few cycles, the max stabilizes - doAssert mem <= memMax, $(mem, memMax) - - memPrevious = mem + if getOccupiedMem() > 300_000: quit("still a leak!") inProc() -let mem = getOccupiedMem() -var msg = "no leak: " -when isDeterministic: - msg.add $(mem, memMax) - echo msg -else: - msg.add $(mem, memMin, memMax, numCollections, numIter) - echo msg - # make sure some collections did happen, otherwise the previous tests - # are meaningless - doAssert numCollections > 1000 # 3999 on local OSX; leaving some slack +echo "no leak: ", getOccupiedMem() From fdc5925cbdd53ea8b2877fb56de5a1e5a3025a0c Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 27 Feb 2020 01:36:06 -0800 Subject: [PATCH 05/77] CI tests run faster: save 120s in azure machines, 335s on local OSX --- tests/gc/gcleak2.nim | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/gc/gcleak2.nim b/tests/gc/gcleak2.nim index fe1718aef6b45..bc943dbe71757 100644 --- a/tests/gc/gcleak2.nim +++ b/tests/gc/gcleak2.nim @@ -14,8 +14,20 @@ proc makeObj(): TTestObj = result.x = "Hello" result.s = @[1,2,3] +const numIter = + when defined(boehmgc): + # super slow because GC_fullcollect() at each iteration; especially + # on OSX 10.15 where it takes ~170s + # `getOccupiedMem` should be constant after each iteration for i >= 3 + 1_000 + elif defined(gcMarkAndSweep): + # likewise, somewhat slow, 1_000_000 would run for 8s + # and same remark as above + 100_000 + else: 1_000_000 + proc inProc() = - for i in 1 .. 1_000_000: + for i in 1 .. numIter: when defined(gcMarkAndSweep) or defined(boehmgc): GC_fullcollect() var obj: TTestObj From 73f5f1e80c9baf29c4d49dda7fa41351bb2b2400 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 27 Feb 2020 02:30:31 -0800 Subject: [PATCH 06/77] save another 33s of CI for tests/gc/gcleak.nim --- tests/gc/gcleak.nim | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/gc/gcleak.nim b/tests/gc/gcleak.nim index 0b2e6e14dbf50..0bf993968ef84 100644 --- a/tests/gc/gcleak.nim +++ b/tests/gc/gcleak.nim @@ -12,7 +12,14 @@ type proc makeObj(): TTestObj = result.x = "Hello" -for i in 1 .. 100_000: +const numIter = + # see tests/gc/gcleak2.nim + when defined(boehmgc): + 1_000 + elif defined(gcMarkAndSweep): 10_000 + else: 100_000 + +for i in 1 .. numIter: when defined(gcMarkAndSweep) or defined(boehmgc): GC_fullcollect() var obj = makeObj() From 96bffadf6045835ddefbf2903e135d50a93f7016 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 22 Feb 2020 15:09:37 -0600 Subject: [PATCH 07/77] Fix #9405 - cfg and nims run in sync --- compiler/cmdlinehelper.nim | 29 ++--------------------------- compiler/nimconf.nim | 30 +++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/compiler/cmdlinehelper.nim b/compiler/cmdlinehelper.nim index b3c0c46c2be00..a038990715064 100644 --- a/compiler/cmdlinehelper.nim +++ b/compiler/cmdlinehelper.nim @@ -10,7 +10,7 @@ ## Helpers for binaries that use compiler passes, eg: nim, nimsuggest, nimfix import - options, idents, nimconf, scriptconfig, extccomp, commands, msgs, + options, idents, nimconf, extccomp, commands, msgs, lineinfos, modulegraphs, condsyms, os, pathutils from strutils import normalize @@ -43,32 +43,13 @@ proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) = conf.projectPath = AbsoluteDir canonicalizePath(conf, AbsoluteFile getCurrentDir()) proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: ConfigRef): bool = - loadConfigs(DefaultConfig, cache, conf) # load all config files if self.suggestMode: conf.command = "nimsuggest" + loadConfigs(DefaultConfig, cache, conf) # load all config files - template runNimScriptIfExists(path: AbsoluteFile) = - let p = path # eval once - if fileExists(p): - runNimScript(cache, p, freshDefines = false, conf) - - # Caution: make sure this stays in sync with `loadConfigs` - if optSkipSystemConfigFile notin conf.globalOptions: - runNimScriptIfExists(getSystemConfigPath(conf, DefaultConfigNims)) - - if optSkipUserConfigFile notin conf.globalOptions: - runNimScriptIfExists(getUserConfigPath(DefaultConfigNims)) - - if optSkipParentConfigFiles notin conf.globalOptions: - for dir in parentDirs(conf.projectPath.string, fromRoot = true, inclusive = false): - runNimScriptIfExists(AbsoluteDir(dir) / DefaultConfigNims) - - if optSkipProjConfigFile notin conf.globalOptions: - runNimScriptIfExists(conf.projectPath / DefaultConfigNims) block: let scriptFile = conf.projectFull.changeFileExt("nims") if not self.suggestMode: - runNimScriptIfExists(scriptFile) # 'nim foo.nims' means to just run the NimScript file and do nothing more: if fileExists(scriptFile) and scriptFile == conf.projectFull: if conf.command == "": @@ -76,12 +57,6 @@ proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: Confi return false elif conf.command.normalize == "e": return false - else: - if scriptFile != conf.projectFull: - runNimScriptIfExists(scriptFile) - else: - # 'nimsuggest foo.nims' means to just auto-complete the NimScript file - discard # now process command line arguments again, because some options in the # command line can overwrite the config file's settings diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index 93cc215734f3d..df1ceb6106efb 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -11,7 +11,7 @@ import llstream, commands, os, strutils, msgs, lexer, - options, idents, wordrecg, strtabs, lineinfos, pathutils + options, idents, wordrecg, strtabs, lineinfos, pathutils, scriptconfig # ---------------- configuration file parser ----------------------------- # we use Nim's scanner here to save space and work @@ -248,17 +248,31 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef) = if readConfigFile(configPath, cache, conf): configFiles.add(configPath) + template runNimScriptIfExists(path: AbsoluteFile) = + let p = path # eval once + if fileExists(p): + runNimScript(cache, p, freshDefines = false, conf) + if optSkipSystemConfigFile notin conf.globalOptions: readConfigFile(getSystemConfigPath(conf, cfg)) + if cfg == DefaultConfig: + runNimScriptIfExists(getSystemConfigPath(conf, DefaultConfigNims)) + if optSkipUserConfigFile notin conf.globalOptions: readConfigFile(getUserConfigPath(cfg)) + if cfg == DefaultConfig: + runNimScriptIfExists(getUserConfigPath(DefaultConfigNims)) + let pd = if not conf.projectPath.isEmpty: conf.projectPath else: AbsoluteDir(getCurrentDir()) if optSkipParentConfigFiles notin conf.globalOptions: for dir in parentDirs(pd.string, fromRoot=true, inclusive=false): readConfigFile(AbsoluteDir(dir) / cfg) + if cfg == DefaultConfig: + runNimScriptIfExists(AbsoluteDir(dir) / DefaultConfigNims) + if optSkipProjConfigFile notin conf.globalOptions: readConfigFile(pd / cfg) @@ -269,6 +283,20 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef) = projectConfig = changeFileExt(conf.projectFull, "nim.cfg") readConfigFile(projectConfig) + if cfg == DefaultConfig: + runNimScriptIfExists(pd / DefaultConfigNims) + for filename in configFiles: # delayed to here so that `hintConf` is honored rawMessage(conf, hintConf, filename.string) + + block: + let scriptFile = conf.projectFull.changeFileExt("nims") + if conf.command != "nimsuggest": + runNimScriptIfExists(scriptFile) + else: + if scriptFile != conf.projectFull: + runNimScriptIfExists(scriptFile) + else: + # 'nimsuggest foo.nims' means to just auto-complete the NimScript file + discard From ce36fdc8974b843b2fb0404196c2778e8b0b91dd Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 24 Feb 2020 19:20:52 -0800 Subject: [PATCH 08/77] correctly honor cmdline --hint:conf:on/off ; correctly show Conf hints in order --- compiler/cmdlinehelper.nim | 2 ++ compiler/commands.nim | 6 ++++++ compiler/msgs.nim | 5 ++++- compiler/nimconf.nim | 1 + compiler/options.nim | 4 ++++ compiler/scriptconfig.nim | 3 +-- 6 files changed, 18 insertions(+), 3 deletions(-) diff --git a/compiler/cmdlinehelper.nim b/compiler/cmdlinehelper.nim index a038990715064..9df6d37e0d5f7 100644 --- a/compiler/cmdlinehelper.nim +++ b/compiler/cmdlinehelper.nim @@ -27,7 +27,9 @@ proc initDefinesProg*(self: NimProg, conf: ConfigRef, name: string) = defineSymbol conf.symbols, name proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) = + conf.isCmdLine = true self.processCmdLine(passCmd1, "", conf) + conf.isCmdLine = false if self.supportsStdinFile and conf.projectName == "-": handleStdinInput(conf) elif conf.projectName != "": diff --git a/compiler/commands.nim b/compiler/commands.nim index 638444efeb7b0..76203762cfd60 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -204,11 +204,17 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass, incl(conf.notes, n) incl(conf.mainPackageNotes, n) incl(conf.enableNotes, n) + if conf.isCmdLine: + incl(conf.cmdLineNotes, n) + excl(conf.cmdLineDisabledNotes, n) of "off": excl(conf.notes, n) excl(conf.mainPackageNotes, n) incl(conf.disableNotes, n) excl(conf.foreignPackageNotes, n) + if conf.isCmdLine: + incl(conf.cmdLineDisabledNotes, n) + excl(conf.cmdLineNotes, n) else: localError(conf, info, errOnOrOffExpectedButXFound % arg) proc processCompile(conf: ConfigRef; filename: string) = diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 6d1ecb2b14e4e..7fa8e8aef8181 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -418,7 +418,10 @@ proc rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) = inc(conf.warnCounter) of hintMin..hintMax: sev = Severity.Hint - if not conf.hasHint(msg): return + if msg in conf.cmdLineDisabledNotes: return # eg: `--hints:conf:off` passed on cmdline + # handle `--hints:off` (regardless of cmdline/cfg file) + # handle `--hints:conf:on` on cmdline + if not conf.hasHint(msg) and not (optHints in conf.options and msg in conf.cmdLineNotes)): return title = HintTitle color = HintColor if msg != hintUserRaw: kind = HintsToStr[ord(msg) - ord(hintMin)] diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index df1ceb6106efb..bd00832be7151 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -251,6 +251,7 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef) = template runNimScriptIfExists(path: AbsoluteFile) = let p = path # eval once if fileExists(p): + configFiles.add(p) runNimScript(cache, p, freshDefines = false, conf) if optSkipSystemConfigFile notin conf.globalOptions: diff --git a/compiler/options.nim b/compiler/options.nim index 7a7ab0bcd8cdf..fc37979e297ff 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -231,6 +231,8 @@ type foreignPackageNotes*: TNoteKinds notes*: TNoteKinds mainPackageNotes*: TNoteKinds + cmdLineNotes*: TNoteKinds + cmdLineDisabledNotes*: TNoteKinds mainPackageId*: int errorCounter*: int hintCounter*: int @@ -286,6 +288,7 @@ type structuredErrorHook*: proc (config: ConfigRef; info: TLineInfo; msg: string; severity: Severity) {.closure, gcsafe.} cppCustomNamespace*: string + isCmdLine*: bool # whether we are currently processing cmdline args, not cfg files proc hasHint*(conf: ConfigRef, note: TNoteKind): bool = optHints in conf.options and note in conf.notes @@ -391,6 +394,7 @@ proc newConfigRef*(): ConfigRef = arguments: "", suggestMaxResults: 10_000, maxLoopIterationsVM: 10_000_000, + isCmdLine: false, ) setTargetFromSystem(result.target) # enable colors by default on terminals diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim index e9e720b1b71bd..6aad1b25ff975 100644 --- a/compiler/scriptconfig.nim +++ b/compiler/scriptconfig.nim @@ -199,7 +199,6 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string; proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile; freshDefines=true; conf: ConfigRef) = - rawMessage(conf, hintConf, scriptName.string) let oldSymbolFiles = conf.symbolFiles conf.symbolFiles = disabledSf @@ -224,7 +223,7 @@ proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile; incl(m.flags, sfMainModule) graph.vm = setupVM(m, cache, scriptName.string, graph) - graph.compileSystemModule() # TODO: see why this unsets hintConf in conf.notes + graph.compileSystemModule() discard graph.processModule(m, llStreamOpen(scriptName, fmRead)) # watch out, "newruntime" can be set within NimScript itself and then we need From c1cbf94e2d41911fe11ea98e86000f9a851fc01a Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 26 Feb 2020 03:10:11 -0800 Subject: [PATCH 09/77] remove isCmdLine; use passCmd1 --- compiler/cmdlinehelper.nim | 2 -- compiler/commands.nim | 4 ++-- compiler/msgs.nim | 2 +- compiler/options.nim | 2 -- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/compiler/cmdlinehelper.nim b/compiler/cmdlinehelper.nim index 9df6d37e0d5f7..a038990715064 100644 --- a/compiler/cmdlinehelper.nim +++ b/compiler/cmdlinehelper.nim @@ -27,9 +27,7 @@ proc initDefinesProg*(self: NimProg, conf: ConfigRef, name: string) = defineSymbol conf.symbols, name proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) = - conf.isCmdLine = true self.processCmdLine(passCmd1, "", conf) - conf.isCmdLine = false if self.supportsStdinFile and conf.projectName == "-": handleStdinInput(conf) elif conf.projectName != "": diff --git a/compiler/commands.nim b/compiler/commands.nim index 76203762cfd60..d2f1f78b37a36 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -204,7 +204,7 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass, incl(conf.notes, n) incl(conf.mainPackageNotes, n) incl(conf.enableNotes, n) - if conf.isCmdLine: + if pass == passCmd1: incl(conf.cmdLineNotes, n) excl(conf.cmdLineDisabledNotes, n) of "off": @@ -212,7 +212,7 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass, excl(conf.mainPackageNotes, n) incl(conf.disableNotes, n) excl(conf.foreignPackageNotes, n) - if conf.isCmdLine: + if pass == passCmd1: incl(conf.cmdLineDisabledNotes, n) excl(conf.cmdLineNotes, n) else: localError(conf, info, errOnOrOffExpectedButXFound % arg) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 7fa8e8aef8181..1972952493da2 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -421,7 +421,7 @@ proc rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) = if msg in conf.cmdLineDisabledNotes: return # eg: `--hints:conf:off` passed on cmdline # handle `--hints:off` (regardless of cmdline/cfg file) # handle `--hints:conf:on` on cmdline - if not conf.hasHint(msg) and not (optHints in conf.options and msg in conf.cmdLineNotes)): return + if not conf.hasHint(msg) and not (optHints in conf.options and msg in conf.cmdLineNotes): return title = HintTitle color = HintColor if msg != hintUserRaw: kind = HintsToStr[ord(msg) - ord(hintMin)] diff --git a/compiler/options.nim b/compiler/options.nim index fc37979e297ff..9d073ce4574eb 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -288,7 +288,6 @@ type structuredErrorHook*: proc (config: ConfigRef; info: TLineInfo; msg: string; severity: Severity) {.closure, gcsafe.} cppCustomNamespace*: string - isCmdLine*: bool # whether we are currently processing cmdline args, not cfg files proc hasHint*(conf: ConfigRef, note: TNoteKind): bool = optHints in conf.options and note in conf.notes @@ -394,7 +393,6 @@ proc newConfigRef*(): ConfigRef = arguments: "", suggestMaxResults: 10_000, maxLoopIterationsVM: 10_000_000, - isCmdLine: false, ) setTargetFromSystem(result.target) # enable colors by default on terminals From 1056f9ecff60f624522f24e508d0976994bcef80 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 26 Feb 2020 12:49:44 -0800 Subject: [PATCH 10/77] properly handle note override logic/verbosity/config/cmdline using modifiedyNotes, cmdlineNotes --- compiler/commands.nim | 41 ++++++++++++++++++++--------------------- compiler/msgs.nim | 5 +---- compiler/options.nim | 12 +++++++----- compiler/passaux.nim | 4 ++-- compiler/pragmas.nim | 1 + 5 files changed, 31 insertions(+), 32 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index d2f1f78b37a36..0545ee2b0043f 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -199,23 +199,21 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass, let x = findStr(lineinfos.WarningsToStr, id) if x >= 0: n = TNoteKind(x + ord(warnMin)) else: localError(conf, info, "unknown warning: " & id) - case substr(arg, i).normalize - of "on": - incl(conf.notes, n) - incl(conf.mainPackageNotes, n) - incl(conf.enableNotes, n) - if pass == passCmd1: - incl(conf.cmdLineNotes, n) - excl(conf.cmdLineDisabledNotes, n) - of "off": - excl(conf.notes, n) - excl(conf.mainPackageNotes, n) - incl(conf.disableNotes, n) - excl(conf.foreignPackageNotes, n) - if pass == passCmd1: - incl(conf.cmdLineDisabledNotes, n) - excl(conf.cmdLineNotes, n) - else: localError(conf, info, errOnOrOffExpectedButXFound % arg) + + let val = substr(arg, i).normalize + if val notin ["on", "off"]: + localError(conf, info, errOnOrOffExpectedButXFound % arg) + elif n notin conf.cmdlineNotes or pass == passCmd1: + if pass == passCmd1: incl(conf.cmdlineNotes, n) + incl(conf.modifiedyNotes, n) + case val + of "on": + incl(conf.notes, n) + incl(conf.mainPackageNotes, n) + of "off": + excl(conf.notes, n) + excl(conf.mainPackageNotes, n) + excl(conf.foreignPackageNotes, n) proc processCompile(conf: ConfigRef; filename: string) = var found = findFile(conf, filename) @@ -598,7 +596,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "deadcodeelim": discard # deprecated, dead code elim always on of "threads": processOnOffSwitchG(conf, {optThreads}, arg, pass, info) - #if optThreads in conf.globalOptions: incl(conf.notes, warnGcUnsafe) + #if optThreads in conf.globalOptions: conf.setNote(warnGcUnsafe) of "tlsemulation": processOnOffSwitchG(conf, {optTlsEmulation}, arg, pass, info) of "taintmode": processOnOffSwitchG(conf, {optTaintMode}, arg, pass, info) of "implicitstatic": @@ -710,9 +708,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; if verbosity notin {0..3}: localError(conf, info, "invalid verbosity level: '$1'" % arg) conf.verbosity = verbosity - conf.notes = NotesVerbosity[conf.verbosity] - incl(conf.notes, conf.enableNotes) - excl(conf.notes, conf.disableNotes) + var verb = NotesVerbosity[conf.verbosity] + ## We override the default `verb` by explicitly modified (set/unset) notes. + conf.notes = (conf.modifiedyNotes * conf.notes + verb) - + (conf.modifiedyNotes * verb - conf.notes) conf.mainPackageNotes = conf.notes of "parallelbuild": expectArg(conf, switch, arg, pass, info) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 1972952493da2..6d1ecb2b14e4e 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -418,10 +418,7 @@ proc rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) = inc(conf.warnCounter) of hintMin..hintMax: sev = Severity.Hint - if msg in conf.cmdLineDisabledNotes: return # eg: `--hints:conf:off` passed on cmdline - # handle `--hints:off` (regardless of cmdline/cfg file) - # handle `--hints:conf:on` on cmdline - if not conf.hasHint(msg) and not (optHints in conf.options and msg in conf.cmdLineNotes): return + if not conf.hasHint(msg): return title = HintTitle color = HintColor if msg != hintUserRaw: kind = HintsToStr[ord(msg) - ord(hintMin)] diff --git a/compiler/options.nim b/compiler/options.nim index 9d073ce4574eb..c93fc06484e44 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -226,13 +226,11 @@ type ideCmd*: IdeCmd oldNewlines*: bool cCompiler*: TSystemCC - enableNotes*: TNoteKinds - disableNotes*: TNoteKinds + modifiedyNotes*: TNoteKinds # notes that have been set/unset from either cmdline/configs + cmdlineNotes*: TNoteKinds # notes that have been set/unset from cmdline foreignPackageNotes*: TNoteKinds - notes*: TNoteKinds + notes*: TNoteKinds # notes after resolving all logic(defaults, verbosity)/cmdline/configs mainPackageNotes*: TNoteKinds - cmdLineNotes*: TNoteKinds - cmdLineDisabledNotes*: TNoteKinds mainPackageId*: int errorCounter*: int hintCounter*: int @@ -289,6 +287,10 @@ type severity: Severity) {.closure, gcsafe.} cppCustomNamespace*: string +proc setNote*(conf: ConfigRef, note: TNoteKind, enabled = true) = + if note notin conf.cmdlineNotes: + if enabled: incl(conf.notes, note) else: excl(conf.notes, note) + proc hasHint*(conf: ConfigRef, note: TNoteKind): bool = optHints in conf.options and note in conf.notes diff --git a/compiler/passaux.nim b/compiler/passaux.nim index 2ac89c24e00f6..155b956df0e1c 100644 --- a/compiler/passaux.nim +++ b/compiler/passaux.nim @@ -28,8 +28,8 @@ proc verboseProcess(context: PPassContext, n: PNode): PNode = let v = VerboseRef(context) if v.config.verbosity == 3: # system.nim deactivates all hints, for verbosity:3 we want the processing - # messages nonetheless, so we activate them again unconditionally: - incl(v.config.notes, hintProcessing) + # messages nonetheless, so we activate them again (but honor cmdlineNotes) + v.config.setNote(hintProcessing) message(v.config, n.info, hintProcessing, $idgen.gFrontEndId) const verbosePass* = makePass(open = verboseOpen, process = verboseProcess) diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 02912808673cb..559c1c0640878 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -328,6 +328,7 @@ proc processNote(c: PContext, n: PNode) = n[1] = x if x.kind == nkIntLit and x.intVal != 0: incl(c.config.notes, nk) else: excl(c.config.notes, nk) + # checkme: honor cmdlineNotes with: c.setNote(nk, x.kind == nkIntLit and x.intVal != 0) else: invalidPragma(c, n) From 25dc5f76e024de88478bc6ee3b559e731f227b69 Mon Sep 17 00:00:00 2001 From: cooldome Date: Fri, 28 Feb 2020 09:54:17 +0000 Subject: [PATCH 11/77] fixes #12627 (#13521) * fixes #12627 --- compiler/sem.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/sem.nim b/compiler/sem.nim index 7cd0aa6f2079e..831e160175122 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -395,7 +395,7 @@ proc semAfterMacroCall(c: PContext, call, macroResult: PNode, if c.config.evalTemplateCounter > evalTemplateLimit: globalError(c.config, s.info, "template instantiation too nested") c.friendModules.add(s.owner.getModule) - + idSynchronizationPoint(5000) result = macroResult excl(result.flags, nfSem) #resetSemFlag n From c79df2fb6abb5663505de13c0ae5995d72ae71a8 Mon Sep 17 00:00:00 2001 From: cooldome Date: Fri, 28 Feb 2020 09:55:06 +0000 Subject: [PATCH 12/77] EndsInNoReturn in expressions extension, fixes #13490 (#13520) * fix #13490 --- compiler/ast.nim | 2 ++ compiler/sem.nim | 2 +- compiler/semstmts.nim | 7 ++----- compiler/vmgen.nim | 2 -- tests/casestmt/tcasestmt.nim | 37 ++++++++++++++++++++++++++++++++++++ 5 files changed, 42 insertions(+), 8 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 5938d4e53808f..70c14ccaf185e 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -336,6 +336,8 @@ const tagEffects* = 3 # user defined tag ('gc', 'time' etc.) pragmasEffects* = 4 # not an effect, but a slot for pragmas in proc type effectListLen* = 5 # list of effects list + nkLastBlockStmts* = {nkRaiseStmt, nkReturnStmt, nkBreakStmt, nkContinueStmt} + # these must be last statements in a block type TTypeKind* = enum # order is important! diff --git a/compiler/sem.nim b/compiler/sem.nim index 831e160175122..48f767af7c569 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -181,7 +181,7 @@ proc endsInNoReturn(n: PNode): bool = var it = n while it.kind in {nkStmtList, nkStmtListExpr} and it.len > 0: it = it.lastSon - result = it.kind == nkRaiseStmt or + result = it.kind in nkLastBlockStmts or it.kind in nkCallKinds and it[0].kind == nkSym and sfNoReturn in it[0].sym.flags proc commonType*(x: PType, y: PNode): PType = diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 7636705e0e6b5..deffe563c9bfe 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -120,7 +120,7 @@ const proc implicitlyDiscardable(n: PNode): bool = var n = n while n.kind in skipForDiscardable: n = n.lastSon - result = n.kind == nkRaiseStmt or + result = n.kind in nkLastBlockStmts or (isCallExpr(n) and n[0].kind == nkSym and sfDiscardable in n[0].sym.flags) @@ -2162,9 +2162,6 @@ proc inferConceptStaticParam(c: PContext, inferred, n: PNode) = typ.n = res proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode = - # these must be last statements in a block: - const - LastBlockStmts = {nkRaiseStmt, nkReturnStmt, nkBreakStmt, nkContinueStmt} result = n result.transitionSonsKind(nkStmtList) var voidContext = false @@ -2209,7 +2206,7 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode = else: n.typ = n[i].typ if not isEmptyType(n.typ): n.transitionSonsKind(nkStmtListExpr) - if n[i].kind in LastBlockStmts or + if n[i].kind in nkLastBlockStmts or n[i].kind in nkCallKinds and n[i][0].kind == nkSym and sfNoReturn in n[i][0].sym.flags: for j in i + 1.. Date: Sat, 29 Feb 2020 17:57:00 +0000 Subject: [PATCH 13/77] make it possible to pass linker options for vcc (#13535) [backport] --- compiler/extccomp.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index c6f062c781d2c..b9d0f0ff6afb3 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -149,7 +149,7 @@ compiler vcc: buildDll: " /LD", buildLib: "lib /OUT:$libfile $objfiles", linkerExe: "cl", - linkTmpl: "$options $builddll$vccplatform /Fe$exefile $objfiles $buildgui", + linkTmpl: "$builddll$vccplatform /Fe$exefile $objfiles $buildgui $options", includeCmd: " /I", linkDirCmd: " /LIBPATH:", linkLibCmd: " $1.lib", From 525ab5a497d1b998bc67c5b1225db3556ac77d27 Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Sun, 1 Mar 2020 23:52:29 +0300 Subject: [PATCH 14/77] Document import/include outside of top level semantics (#13548) --- doc/manual.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/manual.rst b/doc/manual.rst index ba13f2f02bb6c..71f6f73f9170c 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -5645,6 +5645,8 @@ It is not checked that the ``except`` list is really exported from the module. This feature allows to compile against an older version of the module that does not export these identifiers. +The ``import`` statement is only allowed at the top level. + Include statement ~~~~~~~~~~~~~~~~~ @@ -5655,6 +5657,18 @@ statement is useful to split up a large module into several files: .. code-block:: nim include fileA, fileB, fileC +The ``include`` statement can be used outside of the top level, as such: + +.. code-block:: nim + # Module A + echo "Hello World!" + +.. code-block:: nim + # Module B + proc main() = + include A + + main() # => Hello World! Module names in imports From 22d1ba4be76569d1df96cc8bfed47fccfc833019 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 1 Mar 2020 12:56:20 -0800 Subject: [PATCH 15/77] fixes #13543 and added times.isLeapDay (#13547) --- changelog.md | 1 + lib/pure/times.nim | 13 +++++++++++++ tests/stdlib/ttimes.nim | 10 +++++++--- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/changelog.md b/changelog.md index f106ac2c23c07..e0fd649428495 100644 --- a/changelog.md +++ b/changelog.md @@ -81,6 +81,7 @@ ``` +- Added `times.isLeapDay` ## Library changes diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 3fc4d69dfa65d..92b6cd3b7e1d3 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -637,6 +637,19 @@ proc isLeapYear*(year: int): bool = doAssert not isLeapYear(1900) year mod 4 == 0 and (year mod 100 != 0 or year mod 400 == 0) +proc isLeapDay*(t: DateTime): bool {.since: (1,1).} = + ## returns whether `t` is a leap day, ie, Feb 29 in a leap year. This matters + ## as it affects time offset calculations. + runnableExamples: + let t = initDateTime(29, mFeb, 2020, 00, 00, 00, utc()) + doAssert t.isLeapDay + doAssert t+1.years-1.years != t + let t2 = initDateTime(28, mFeb, 2020, 00, 00, 00, utc()) + doAssert not t2.isLeapDay + doAssert t2+1.years-1.years == t2 + doAssertRaises(Exception): discard initDateTime(29, mFeb, 2021, 00, 00, 00, utc()) + t.year.isLeapYear and t.month == mFeb and t.monthday == 29 + proc getDaysInMonth*(month: Month, year: int): int = ## Get the number of days in ``month`` of ``year``. # http://www.dispersiondesign.com/articles/time/number_of_days_in_a_month diff --git a/tests/stdlib/ttimes.nim b/tests/stdlib/ttimes.nim index 97945b21db045..0602aaa94c9a0 100644 --- a/tests/stdlib/ttimes.nim +++ b/tests/stdlib/ttimes.nim @@ -345,20 +345,24 @@ suite "ttimes": test "adding/subtracting TimeInterval": # add/subtract TimeIntervals and Time/TimeInfo let now = getTime().utc + let isSpecial = now.isLeapDay check now + convert(Seconds, Nanoseconds, 1).nanoseconds == now + 1.seconds check now + 1.weeks == now + 7.days check now - 1.seconds == now - 3.seconds + 2.seconds check now + 65.seconds == now + 1.minutes + 5.seconds check now + 60.minutes == now + 1.hours check now + 24.hours == now + 1.days - check now + 13.months == now + 1.years + 1.months + if not isSpecial: + check now + 13.months == now + 1.years + 1.months check toUnix(fromUnix(0) + 2.seconds) == 2 check toUnix(fromUnix(0) - 2.seconds) == -2 var ti1 = now + 1.years ti1 = ti1 - 1.years - check ti1 == now + if not isSpecial: + check ti1 == now ti1 = ti1 + 1.days - check ti1 == now + 1.days + if not isSpecial: + check ti1 == now + 1.days # Bug with adding a day to a Time let day = 24.hours From 653de5f27a89c781376be370e28f8f3ce7744040 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 2 Mar 2020 01:26:10 -0800 Subject: [PATCH 16/77] fix broken nim CI, disable blscurve (#13555) --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 3018a21ffd56d..344aff339b3c2 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -11,7 +11,7 @@ pkg "ast_pattern_matching", "nim c -r --oldgensym:on tests/test1.nim" pkg "asyncmysql", "", true pkg "bigints" pkg "binaryheap", "nim c -r binaryheap.nim" -pkg "blscurve", "", true +# pkg "blscurve", "", true # pending https://github.com/status-im/nim-blscurve/issues/39 pkg "bncurve", "", true pkg "c2nim", "nim c testsuite/tester.nim" pkg "cascade" From 05e9473f80ec477c2e53e072e330d03bc3b0d4a0 Mon Sep 17 00:00:00 2001 From: PMunch Date: Mon, 2 Mar 2020 17:52:42 +0100 Subject: [PATCH 17/77] Add signatures object to jsondoc for routine types (#13530) jsondoc is meant to be read by computers, but yet the signatures of procedures where simply a string of the whole thing. This adds a signature object that unpacks this information into an object so it's easier to analyse the documented signatures. --- compiler/docgen.nim | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 15ff2e167e741..daa3a708de9f7 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -794,6 +794,35 @@ proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode = result["description"] = %comm if r.buf.len > 0: result["code"] = %r.buf + if k in routineKinds: + result["signature"] = newJObject() + if n[paramsPos][0].kind != nkEmpty: + result["signature"]["return"] = %($n[paramsPos][0]) + if n[paramsPos].len > 1: + result["signature"]["arguments"] = newJArray() + for paramIdx in 1 ..< n[paramsPos].len: + for identIdx in 0 ..< n[paramsPos][paramIdx].len - 2: + let + paramName = $n[paramsPos][paramIdx][identIdx] + paramType = $n[paramsPos][paramIdx][^2] + if n[paramsPos][paramIdx][^1].kind != nkEmpty: + let paramDefault = $n[paramsPos][paramIdx][^1] + result["signature"]["arguments"].add %{"name": %paramName, "type": %paramType, "default": %paramDefault} + else: + result["signature"]["arguments"].add %{"name": %paramName, "type": %paramType} + if n[pragmasPos].kind != nkEmpty: + result["signature"]["pragmas"] = newJArray() + for pragma in n[pragmasPos]: + result["signature"]["pragmas"].add %($pragma) + if n[genericParamsPos].kind != nkEmpty: + result["signature"]["genericParams"] = newJArray() + for genericParam in n[genericParamsPos]: + var param = %{"name": %($genericParam)} + if genericParam.sym.typ.sons.len > 0: + param["types"] = newJArray() + for kind in genericParam.sym.typ.sons: + param["types"].add %($kind) + result["signature"]["genericParams"].add param proc checkForFalse(n: PNode): bool = result = n.kind == nkIdent and cmpIgnoreStyle(n.ident.s, "false") == 0 From 27241fda23905280c8b8a13d94efb46a7ebe9836 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 2 Mar 2020 08:55:17 -0800 Subject: [PATCH 18/77] fix #13528 nimgrep --word now works better with operators (#13537) --- tools/nimgrep.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/nimgrep.nim b/tools/nimgrep.nim index 51685e556263b..26e219330bd85 100644 --- a/tools/nimgrep.nim +++ b/tools/nimgrep.nim @@ -649,7 +649,8 @@ else: if optIgnoreStyle in options: pattern = styleInsensitive(pattern) if optWord in options: - pattern = r"\b(:?" & pattern & r")\b" + # see https://github.com/nim-lang/Nim/issues/13528#issuecomment-592786443 + pattern = r"(^|\W)(:?" & pattern & r")($|\W)" if {optIgnoreCase, optIgnoreStyle} * options != {}: reflags.incl reIgnoreCase let rep = if optRex in options: rex(pattern, reflags) From 451b724c40bc04e4a9c756b2fcd457cca3fd437c Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 2 Mar 2020 09:05:14 -0800 Subject: [PATCH 19/77] make genericParams support static[T] generic params (#13433) * make genericParams support static[T] generic params * WrapStatic => StaticParam --- lib/pure/typetraits.nim | 34 ++++++++++++++++++++----- tests/metatype/ttypetraits.nim | 45 +++++++++++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index 04231db093996..0335d54883681 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -86,13 +86,13 @@ since (1, 1): # Note: `[]` currently gives: `Error: no generic parameters allowed for ...` type(default(T)[i]) + type StaticParam*[value] = object + ## used to wrap a static value in `genericParams` + import std/macros -macro genericParams*(T: typedesc): untyped {.since: (1, 1).} = - ## return tuple of generic params for generic `T` - runnableExamples: - type Foo[T1, T2]=object - doAssert genericParams(Foo[float, string]) is (float, string) +macro genericParamsImpl(T: typedesc): untyped = + # auxiliary macro needed, can't do it directly in `genericParams` result = newNimNode(nnkTupleConstr) var impl = getTypeImpl(T) expectKind(impl, nnkBracketExpr) @@ -107,11 +107,33 @@ macro genericParams*(T: typedesc): untyped {.since: (1, 1).} = continue of nnkBracketExpr: for i in 1.. Date: Mon, 2 Mar 2020 19:30:41 -0800 Subject: [PATCH 20/77] workaround refs #13563 freebsd CI (#13564) --- .builds/freebsd.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.builds/freebsd.yml b/.builds/freebsd.yml index 7810e39afd02f..b53cc310454ed 100644 --- a/.builds/freebsd.yml +++ b/.builds/freebsd.yml @@ -1,4 +1,9 @@ -image: freebsd/latest +# see https://man.sr.ht/builds.sr.ht/compatibility.md#freebsd +# these are all broken, pending https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=244549 +# image: freebsd/latest +# image: freebsd/current +# image: freebsd/12.x +image: freebsd/11.x packages: - databases/sqlite3 - devel/boehm-gc-threaded From eb42f38088d278d396855e7e284ea798c8a884f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20D=C3=B6ring?= Date: Tue, 3 Mar 2020 12:42:43 +0100 Subject: [PATCH 21/77] Remove dead magics (#13551) --- compiler/ast.nim | 34 ++++++++++++++++------------------ compiler/ccgexprs.nim | 34 +++------------------------------- compiler/forloops.nim | 2 +- compiler/guards.nim | 11 ++++------- compiler/jsgen.nim | 14 ++------------ compiler/semfold.nim | 10 +--------- compiler/semmagic.nim | 4 +--- compiler/semtypes.nim | 6 ------ compiler/sigmatch.nim | 3 --- compiler/vm.nim | 5 ----- compiler/vmdef.nim | 2 +- compiler/vmgen.nim | 35 +++-------------------------------- lib/core/macros.nim | 2 +- lib/system/basic_types.nim | 22 +++++++++++----------- 14 files changed, 44 insertions(+), 140 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 70c14ccaf185e..3b10a89eaf039 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -336,7 +336,7 @@ const tagEffects* = 3 # user defined tag ('gc', 'time' etc.) pragmasEffects* = 4 # not an effect, but a slot for pragmas in proc type effectListLen* = 5 # list of effects list - nkLastBlockStmts* = {nkRaiseStmt, nkReturnStmt, nkBreakStmt, nkContinueStmt} + nkLastBlockStmts* = {nkRaiseStmt, nkReturnStmt, nkBreakStmt, nkContinueStmt} # these must be last statements in a block type @@ -599,12 +599,11 @@ type mDefined, mDefinedInScope, mCompiles, mArrGet, mArrPut, mAsgn, mLow, mHigh, mSizeOf, mAlignOf, mOffsetOf, mTypeTrait, mIs, mOf, mAddr, mType, mTypeOf, - mRoof, mPlugin, mEcho, mShallowCopy, mSlurp, mStaticExec, mStatic, + mPlugin, mEcho, mShallowCopy, mSlurp, mStaticExec, mStatic, mParseExprToAst, mParseStmtToAst, mExpandToAst, mQuoteAst, - mUnaryLt, mInc, mDec, mOrd, + mInc, mDec, mOrd, mNew, mNewFinalize, mNewSeq, mNewSeqOfCap, mLengthOpenArray, mLengthStr, mLengthArray, mLengthSeq, - mXLenStr, mXLenSeq, mIncl, mExcl, mCard, mChr, mGCref, mGCunref, mAddI, mSubI, mMulI, mDivI, mModI, @@ -620,7 +619,7 @@ type mEqEnum, mLeEnum, mLtEnum, mEqCh, mLeCh, mLtCh, mEqB, mLeB, mLtB, - mEqRef, mEqUntracedRef, mLePtr, mLtPtr, + mEqRef, mLePtr, mLtPtr, mXor, mEqCString, mEqProc, mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot, mUnaryPlusI, mBitnotI, @@ -629,18 +628,18 @@ type mStrToStr, mEnumToStr, mAnd, mOr, mEqStr, mLeStr, mLtStr, - mEqSet, mLeSet, mLtSet, mMulSet, mPlusSet, mMinusSet, mSymDiffSet, + mEqSet, mLeSet, mLtSet, mMulSet, mPlusSet, mMinusSet, mConStrStr, mSlice, mDotDot, # this one is only necessary to give nice compile time warnings mFields, mFieldPairs, mOmpParFor, mAppendStrCh, mAppendStrStr, mAppendSeqElem, - mInRange, mInSet, mRepr, mExit, + mInSet, mRepr, mExit, mSetLengthStr, mSetLengthSeq, mIsPartOf, mAstToStr, mParallel, - mSwap, mIsNil, mArrToSeq, mCopyStr, mCopyStrLast, + mSwap, mIsNil, mArrToSeq, mNewString, mNewStringOfCap, mParseBiggestFloat, mMove, mWasMoved, mDestroy, - mDefault, mUnown, mAccessEnv, mAccessTypeInfo, mReset, + mDefault, mUnown, mAccessEnv, mReset, mArray, mOpenArray, mRange, mSet, mSeq, mOpt, mVarargs, mRef, mPtr, mVar, mDistinct, mVoid, mTuple, mOrdinal, @@ -648,8 +647,8 @@ type mUInt, mUInt8, mUInt16, mUInt32, mUInt64, mFloat, mFloat32, mFloat64, mFloat128, mBool, mChar, mString, mCstring, - mPointer, mEmptySet, mIntSetBaseType, mNil, mExpr, mStmt, mTypeDesc, - mVoidType, mPNimrodNode, mShared, mGuarded, mLock, mSpawn, mDeepCopy, + mPointer, mNil, mExpr, mStmt, mTypeDesc, + mVoidType, mPNimrodNode, mSpawn, mDeepCopy, mIsMainModule, mCompileDate, mCompileTime, mProcCall, mCpuEndian, mHostOS, mHostCPU, mBuildOS, mBuildCPU, mAppType, mCompileOption, mCompileOptionArg, @@ -662,7 +661,7 @@ type mNIntVal, mNFloatVal, mNSymbol, mNIdent, mNGetType, mNStrVal, mNSetIntVal, mNSetFloatVal, mNSetSymbol, mNSetIdent, mNSetType, mNSetStrVal, mNLineInfo, mNNewNimNode, mNCopyNimNode, mNCopyNimTree, mStrToIdent, mNSigHash, mNSizeOf, - mNBindSym, mLocals, mNCallSite, + mNBindSym, mNCallSite, mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl, mNGenSym, mNHint, mNWarning, mNError, mInstantiationInfo, mGetTypeInfo, @@ -673,9 +672,9 @@ type # things that we can evaluate safely at compile time, even if not asked for it: const - ctfeWhitelist* = {mNone, mUnaryLt, mSucc, + ctfeWhitelist* = {mNone, mSucc, mPred, mInc, mDec, mOrd, mLengthOpenArray, - mLengthStr, mLengthArray, mLengthSeq, mXLenStr, mXLenSeq, + mLengthStr, mLengthArray, mLengthSeq, mArrGet, mArrPut, mAsgn, mDestroy, mIncl, mExcl, mCard, mChr, mAddI, mSubI, mMulI, mDivI, mModI, @@ -690,17 +689,16 @@ const mEqEnum, mLeEnum, mLtEnum, mEqCh, mLeCh, mLtCh, mEqB, mLeB, mLtB, - mEqRef, mEqProc, mEqUntracedRef, mLePtr, mLtPtr, mEqCString, mXor, + mEqRef, mEqProc, mLePtr, mLtPtr, mEqCString, mXor, mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot, mUnaryPlusI, mBitnotI, mUnaryPlusF64, mUnaryMinusF64, mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr, mStrToStr, mEnumToStr, mAnd, mOr, mEqStr, mLeStr, mLtStr, - mEqSet, mLeSet, mLtSet, mMulSet, mPlusSet, mMinusSet, mSymDiffSet, + mEqSet, mLeSet, mLtSet, mMulSet, mPlusSet, mMinusSet, mConStrStr, mAppendStrCh, mAppendStrStr, mAppendSeqElem, - mInRange, mInSet, mRepr, - mCopyStr, mCopyStrLast} + mInSet, mRepr} type PNode* = ref TNode diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 632e4bb31cc6d..111e1053ba5d3 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -617,7 +617,6 @@ proc binaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mLeB: applyFormat("($1 <= $2)") of mLtB: applyFormat("($1 < $2)") of mEqRef: applyFormat("($1 == $2)") - of mEqUntracedRef: applyFormat("($1 == $2)") of mLePtr: applyFormat("($1 <= $2)") of mLtPtr: applyFormat("($1 < $2)") of mXor: applyFormat("($1 != $2)") @@ -1797,7 +1796,7 @@ proc genInOp(p: BProc, e: PNode, d: var TLoc) = proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) = const - lookupOpr: array[mLeSet..mSymDiffSet, string] = [ + lookupOpr: array[mLeSet..mMinusSet, string] = [ "for ($1 = 0; $1 < $2; $1++) { $n" & " $3 = (($4[$1] & ~ $5[$1]) == 0);$n" & " if (!$3) break;}$n", @@ -1807,8 +1806,7 @@ proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) = "if ($3) $3 = (#nimCmpMem($4, $5, $2) != 0);$n", "&", "|", - "& ~", - "^"] + "& ~"] var a, b, i: TLoc var setType = skipTypes(e[1].typ, abstractVar) var size = int(getSize(p.config, setType)) @@ -1838,7 +1836,6 @@ proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mMulSet: binaryExpr(p, e, d, "($1 & $2)") of mPlusSet: binaryExpr(p, e, d, "($1 | $2)") of mMinusSet: binaryExpr(p, e, d, "($1 & ~ $2)") - of mSymDiffSet: binaryExpr(p, e, d, "($1 ^ $2)") of mInSet: genInOp(p, e, d) else: internalError(p.config, e.info, "genSetOp()") @@ -1868,7 +1865,7 @@ proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) = initLocExpr(p, e[1], a) initLocExpr(p, e[2], b) putIntoDest(p, d, e, ropecg(p.module, "(#nimCmpMem($1, $2, $3)==0)", [a.rdCharLoc, b.rdCharLoc, size])) - of mMulSet, mPlusSet, mMinusSet, mSymDiffSet: + of mMulSet, mPlusSet, mMinusSet: # we inline the simple for loop for better code generation: getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt), i) # our counter initLocExpr(p, e[1], a) @@ -2118,9 +2115,6 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mRepr: genRepr(p, e, d) of mGetTypeInfo: genGetTypeInfo(p, e, d) of mSwap: genSwap(p, e, d) - of mUnaryLt: - if optOverflowCheck notin p.options: unaryExpr(p, e, d, "($1 - 1)") - else: unaryExpr(p, e, d, "#subInt($1, 1)") of mInc, mDec: const opr: array[mInc..mDec, string] = ["+=", "-="] const fun64: array[mInc..mDec, string] = ["addInt64", @@ -2218,21 +2212,6 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mOrd: genOrd(p, e, d) of mLengthArray, mHigh, mLengthStr, mLengthSeq, mLengthOpenArray: genArrayLen(p, e, d, op) - of mXLenStr: - if not p.module.compileToCpp: - unaryExpr(p, e, d, "($1->Sup.len)") - else: - unaryExpr(p, e, d, "$1->len") - of mXLenSeq: - # see 'taddhigh.nim' for why we need to use a temporary here: - var a, tmp: TLoc - initLocExpr(p, e[1], a) - getIntTemp(p, tmp) - if not p.module.compileToCpp: - lineCg(p, cpsStmts, "$1 = $2->Sup.len;$n", [tmp.r, rdLoc(a)]) - else: - lineCg(p, cpsStmts, "$1 = $2->len;$n", [tmp.r, rdLoc(a)]) - putIntoDest(p, d, e, tmp.r) of mGCref: unaryStmt(p, e, d, "if ($1) { #nimGCref($1); }$n") of mGCunref: unaryStmt(p, e, d, "if ($1) { #nimGCunref($1); }$n") of mSetLengthStr: genSetLengthStr(p, e, d) @@ -2240,8 +2219,6 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mIncl, mExcl, mCard, mLtSet, mLeSet, mEqSet, mMulSet, mPlusSet, mMinusSet, mInSet: genSetOp(p, e, d, op) - of mCopyStr, mCopyStrLast: - genCall(p, e, d) of mNewString, mNewStringOfCap, mExit, mParseBiggestFloat: var opr = e[0].sym # Why would anyone want to set nodecl to one of these hardcoded magics? @@ -2296,11 +2273,6 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mMove: genMove(p, e, d) of mDestroy: genDestroy(p, e) of mAccessEnv: unaryExpr(p, e, d, "$1.ClE_0") - of mAccessTypeInfo: - var a: TLoc - var dummy: Rope - initLocExpr(p, e[1], a) - putIntoDest(p, d, e, rdMType(p, a, dummy)) of mSlice: localError(p.config, e.info, "invalid context for 'toOpenArray'; " & "'toOpenArray' is only valid within a call expression") diff --git a/compiler/forloops.nim b/compiler/forloops.nim index d857b3c51b0f7..729afb92054b8 100644 --- a/compiler/forloops.nim +++ b/compiler/forloops.nim @@ -13,7 +13,7 @@ import ast, astalgo const someCmp = {mEqI, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc, - mEqUntracedRef, mLeI, mLeF64, mLeU, mLeU64, mLeEnum, + mLeI, mLeF64, mLeU, mLeU64, mLeEnum, mLeCh, mLeB, mLePtr, mLtI, mLtF64, mLtU, mLtU64, mLtEnum, mLtCh, mLtB, mLtPtr} diff --git a/compiler/guards.nim b/compiler/guards.nim index a2cbdbda2b2a8..97dc4f418d0c8 100644 --- a/compiler/guards.nim +++ b/compiler/guards.nim @@ -14,7 +14,7 @@ import ast, astalgo, msgs, magicsys, nimsets, trees, types, renderer, idents, const someEq = {mEqI, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc, - mEqUntracedRef, mEqStr, mEqSet, mEqCString} + mEqStr, mEqSet, mEqCString} # set excluded here as the semantics are vastly different: someLe = {mLeI, mLeF64, mLeU, mLeU64, mLeEnum, @@ -22,10 +22,9 @@ const someLt = {mLtI, mLtF64, mLtU, mLtU64, mLtEnum, mLtCh, mLtB, mLtPtr, mLtStr} - someLen = {mLengthOpenArray, mLengthStr, mLengthArray, mLengthSeq, - mXLenStr, mXLenSeq} + someLen = {mLengthOpenArray, mLengthStr, mLengthArray, mLengthSeq} - someIn = {mInRange, mInSet} + someIn = {mInSet} someHigh = {mHigh} # we don't list unsigned here because wrap around semantics suck for @@ -258,7 +257,7 @@ proc canon*(n: PNode; o: Operators): PNode = result[i] = canon(n[i], o) elif n.kind == nkSym and n.sym.kind == skLet and n.sym.astdef.getMagic in (someEq + someAdd + someMul + someMin + - someMax + someHigh + {mUnaryLt} + someSub + someLen + someDiv): + someMax + someHigh + someSub + someLen + someDiv): result = n.sym.astdef.copyTree else: result = n @@ -271,8 +270,6 @@ proc canon*(n: PNode; o: Operators): PNode = of someHigh: # high == len+(-1) result = o.opAdd.buildCall(o.opLen.buildCall(result[1]), minusOne()) - of mUnaryLt: - result = buildCall(o.opAdd, result[1], minusOne()) of someSub: # x - 4 --> x + (-4) result = negate(result[1], result[2], result, o) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 51d23662cb972..1130e35f8191a 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -370,7 +370,7 @@ type const # magic checked op; magic unchecked op; jsMagics: TMagicOps = [ - ["addInt", ""], # AddI + mAddI: ["addInt", ""], ["subInt", ""], # SubI ["mulInt", ""], # MulI ["divInt", ""], # DivI @@ -414,7 +414,6 @@ const # magic checked op; magic unchecked op; ["", ""], # LeB ["", ""], # LtB ["", ""], # EqRef - ["", ""], # EqUntracedRef ["", ""], # LePtr ["", ""], # LtPtr ["", ""], # Xor @@ -588,7 +587,6 @@ proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) = of mLeB: applyFormat("($1 <= $2)", "($1 <= $2)") of mLtB: applyFormat("($1 < $2)", "($1 < $2)") of mEqRef: applyFormat("($1 == $2)", "($1 == $2)") - of mEqUntracedRef: applyFormat("($1 == $2)", "($1 == $2)") of mLePtr: applyFormat("($1 <= $2)", "($1 <= $2)") of mLtPtr: applyFormat("($1 < $2)", "($1 < $2)") of mXor: applyFormat("($1 != $2)", "($1 != $2)") @@ -631,7 +629,7 @@ proc arith(p: PProc, n: PNode, r: var TCompRes, op: TMagic) = of mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr, mStrToStr, mEnumToStr: arithAux(p, n, r, op) - of mEqRef, mEqUntracedRef: + of mEqRef: if mapType(n[1].typ) != etyBaseIndex: arithAux(p, n, r, op) else: @@ -1890,10 +1888,6 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) = of mAddI..mStrToStr: arith(p, n, r, op) of mRepr: genRepr(p, n, r) of mSwap: genSwap(p, n) - of mUnaryLt: - # XXX: range checking? - if not (optOverflowCheck in p.options): unaryExpr(p, n, r, "", "$1 - 1") - else: unaryExpr(p, n, r, "subInt", "subInt($1, 1)") of mAppendStrCh: binaryExpr(p, n, r, "addChar", "if ($1 != null) { addChar($3, $2); } else { $3 = [$2]; }") @@ -1959,8 +1953,6 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) = of mOrd: genOrd(p, n, r) of mLengthStr, mLengthSeq, mLengthOpenArray, mLengthArray: unaryExpr(p, n, r, "", "($1 != null ? $2.length : 0)") - of mXLenStr, mXLenSeq: - unaryExpr(p, n, r, "", "$1.length") of mHigh: unaryExpr(p, n, r, "", "($1 != null ? ($2.length-1) : -1)") of mInc: @@ -2007,8 +1999,6 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) = of mEcho: genEcho(p, n, r) of mNLen..mNError, mSlurp, mStaticExec: localError(p.config, n.info, errXMustBeCompileTime % n[0].sym.name.s) - of mCopyStr: - binaryExpr(p, n, r, "", "($1.slice($2))") of mNewString: unaryExpr(p, n, r, "mnewString", "mnewString($1)") of mNewStringOfCap: unaryExpr(p, n, r, "mnewString", "mnewString(0)") diff --git a/compiler/semfold.nim b/compiler/semfold.nim index e8ca42db3c959..b20616e1d366f 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -188,7 +188,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; g: ModuleGraph): PNode = else: result = newIntNodeT(bitnot(getInt(a)), n, g) of mLengthArray: result = newIntNodeT(lengthOrd(g.config, a.typ), n, g) - of mLengthSeq, mLengthOpenArray, mXLenSeq, mLengthStr, mXLenStr: + of mLengthSeq, mLengthOpenArray, mLengthStr: if a.kind == nkNilLit: result = newIntNodeT(Zero, n, g) elif a.kind in {nkStrLit..nkTripleStrLit}: @@ -198,7 +198,6 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; g: ModuleGraph): PNode = of mUnaryPlusI, mUnaryPlusF64: result = a # throw `+` away # XXX: Hides overflow/underflow of mAbsI: result = foldAbs(getInt(a), n, g) - of mUnaryLt: result = foldSub(getOrdValue(a), One, n, g) of mSucc: result = foldAdd(getOrdValue(a), getInt(b), n, g) of mPred: result = foldSub(getOrdValue(a), getInt(b), n, g) of mAddI: result = foldAdd(getInt(a), getInt(b), n, g) @@ -330,9 +329,6 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; g: ModuleGraph): PNode = of mMinusSet: result = nimsets.diffSets(g.config, a, b) result.info = n.info - of mSymDiffSet: - result = nimsets.symdiffSets(g.config, a, b) - result.info = n.info of mConStrStr: result = newStrNodeT(getStrOrChar(a) & getStrOrChar(b), n, g) of mInSet: result = newIntNodeT(toInt128(ord(inSet(a, b))), n, g) of mRepr: @@ -342,10 +338,6 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; g: ModuleGraph): PNode = of mBoolToStr: if getOrdValue(a) == 0: result = newStrNodeT("false", n, g) else: result = newStrNodeT("true", n, g) - of mCopyStr: result = newStrNodeT(substr(getStr(a), int(toInt64(getOrdValue(b)))), n, g) - of mCopyStrLast: - result = newStrNodeT(substr(getStr(a), toInt(getOrdValue(b)), - toInt(getOrdValue(c))), n, g) of mFloatToStr: result = newStrNodeT($getFloat(a), n, g) of mCStrToStr, mCharToStr: if a.kind == nkBracket: diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index 1671a8a26c903..ffdad5628a107 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -140,7 +140,7 @@ proc evalTypeTrait(c: PContext; traitCall: PNode, operand: PType, context: PSym) if operand.kind == tyGenericParam or (traitCall.len > 2 and operand2.kind == tyGenericParam): return traitCall ## too early to evaluate - + let s = trait.sym.name.s case s of "or", "|": @@ -479,8 +479,6 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode, result.typ = n[1].typ of mDotDot: result = n - of mRoof: - localError(c.config, n.info, "builtin roof operator is not supported anymore") of mPlugin: let plugin = getPlugin(c.cache, n[0].sym) if plugin.isNil: diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 61cbecec6806c..abc5de7e8b07d 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1898,10 +1898,6 @@ proc processMagicType(c: PContext, m: PSym) = setMagicIntegral(c.config, m, tyCString, c.config.target.ptrSize) rawAddSon(m.typ, getSysType(c.graph, m.info, tyChar)) of mPointer: setMagicIntegral(c.config, m, tyPointer, c.config.target.ptrSize) - of mEmptySet: - setMagicIntegral(c.config, m, tySet, 1) - rawAddSon(m.typ, newTypeS(tyEmpty, c)) - of mIntSetBaseType: setMagicIntegral(c.config, m, tyRange, c.config.target.intSize) of mNil: setMagicType(c.config, m, tyNil, c.config.target.ptrSize) of mExpr: if m.name.s == "auto": @@ -1937,8 +1933,6 @@ proc processMagicType(c: PContext, m: PSym) = incl m.typ.flags, tfHasAsgn assert c.graph.sysTypes[tySequence] == nil c.graph.sysTypes[tySequence] = m.typ - of mOpt: - setMagicType(c.config, m, tyOpt, szUncomputedSize) of mOrdinal: setMagicIntegral(c.config, m, tyOrdinal, szUncomputedSize) rawAddSon(m.typ, newTypeS(tyNone, c)) diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 4105d3780b170..4f05282a3c541 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -832,9 +832,6 @@ proc inferStaticParam*(c: var TCandidate, lhs: PNode, rhs: BiggestInt): bool = # if lhs.kind in nkCallKinds and lhs[0].kind == nkSym: case lhs[0].sym.magic - of mUnaryLt: - return inferStaticParam(c, lhs[1], rhs + 1) - of mAddI, mAddU, mInc, mSucc: if lhs[1].kind == nkIntLit: return inferStaticParam(c, lhs[2], rhs - lhs[1].intVal) diff --git a/compiler/vm.nim b/compiler/vm.nim index 2d4d06c419ac4..6a12b615e9030 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1089,11 +1089,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = createSet(regs[ra]) move(regs[ra].node.sons, nimsets.diffSets(c.config, regs[rb].node, regs[rc].node).sons) - of opcSymdiffSet: - decodeBC(rkNode) - createSet(regs[ra]) - move(regs[ra].node.sons, - nimsets.symdiffSets(c.config, regs[rb].node, regs[rc].node).sons) of opcConcatStr: decodeBC(rkNode) createStr regs[ra] diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index a9ff7f2d0a0ad..176558eaca1a3 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -95,7 +95,7 @@ type opcEqRef, opcEqNimNode, opcSameNodeType, opcXor, opcNot, opcUnaryMinusInt, opcUnaryMinusFloat, opcBitnotInt, opcEqStr, opcLeStr, opcLtStr, opcEqSet, opcLeSet, opcLtSet, - opcMulSet, opcPlusSet, opcMinusSet, opcSymdiffSet, opcConcatStr, + opcMulSet, opcPlusSet, opcMinusSet, opcConcatStr, opcContainsSet, opcRepr, opcSetLenStr, opcSetLenSeq, opcIsNil, opcOf, opcIs, opcSubStr, opcParseFloat, opcConv, opcCast, diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 0b8eafd17e0a7..75f673c0b9468 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -973,11 +973,6 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = case m of mAnd: c.genAndOr(n, opcFJmp, dest) of mOr: c.genAndOr(n, opcTJmp, dest) - of mUnaryLt: - let tmp = c.genx(n[1]) - if dest < 0: dest = c.getTemp(n.typ) - c.gABI(n, opcSubImmInt, dest, tmp, 1) - c.freeTemp(tmp) of mPred, mSubI: c.genAddSubInt(n, dest, opcSubInt) of mSucc, mAddI: @@ -1020,9 +1015,9 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = c.gABC(n, opcNewStr, dest, tmp) c.freeTemp(tmp) # XXX buggy - of mLengthOpenArray, mLengthArray, mLengthSeq, mXLenSeq: + of mLengthOpenArray, mLengthArray, mLengthSeq: genUnaryABI(c, n, dest, opcLenSeq) - of mLengthStr, mXLenStr: + of mLengthStr: genUnaryABI(c, n, dest, opcLenStr) of mIncl, mExcl: unused(c, n, dest) @@ -1078,7 +1073,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = of mLtF64: genBinaryABC(c, n, dest, opcLtFloat) of mLePtr, mLeU, mLeU64: genBinaryABC(c, n, dest, opcLeu) of mLtPtr, mLtU, mLtU64: genBinaryABC(c, n, dest, opcLtu) - of mEqProc, mEqRef, mEqUntracedRef: + of mEqProc, mEqRef: genBinaryABC(c, n, dest, opcEqRef) of mXor: genBinaryABC(c, n, dest, opcXor) of mNot: genUnaryABC(c, n, dest, opcNot) @@ -1105,7 +1100,6 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = of mMulSet: genBinarySet(c, n, dest, opcMulSet) of mPlusSet: genBinarySet(c, n, dest, opcPlusSet) of mMinusSet: genBinarySet(c, n, dest, opcMinusSet) - of mSymDiffSet: genBinarySet(c, n, dest, opcSymdiffSet) of mConStrStr: genVarargsABC(c, n, dest, opcConcatStr) of mInSet: genBinarySet(c, n, dest, opcContainsSet) of mRepr: genUnaryABC(c, n, dest, opcRepr) @@ -1126,29 +1120,6 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = unused(c, n, dest) c.gen(lowerSwap(c.graph, n, if c.prc == nil: c.module else: c.prc.sym)) of mIsNil: genUnaryABC(c, n, dest, opcIsNil) - of mCopyStr: - if dest < 0: dest = c.getTemp(n.typ) - var - tmp1 = c.genx(n[1]) - tmp2 = c.genx(n[2]) - tmp3 = c.getTemp(n[2].typ) - c.gABC(n, opcLenStr, tmp3, tmp1) - c.gABC(n, opcSubStr, dest, tmp1, tmp2) - c.gABC(n, opcSubStr, tmp3) - c.freeTemp(tmp1) - c.freeTemp(tmp2) - c.freeTemp(tmp3) - of mCopyStrLast: - if dest < 0: dest = c.getTemp(n.typ) - var - tmp1 = c.genx(n[1]) - tmp2 = c.genx(n[2]) - tmp3 = c.genx(n[3]) - c.gABC(n, opcSubStr, dest, tmp1, tmp2) - c.gABC(n, opcSubStr, tmp3) - c.freeTemp(tmp1) - c.freeTemp(tmp2) - c.freeTemp(tmp3) of mParseBiggestFloat: if dest < 0: dest = c.getTemp(n.typ) var d2: TRegister diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 31e133090204f..cc20f1dac655a 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1458,7 +1458,7 @@ proc boolVal*(n: NimNode): bool {.compileTime, noSideEffect.} = else: n == bindSym"true" # hacky solution for now when defined(nimMacrosGetNodeId): - proc nodeID*(n: NimNode): int {.magic: NodeId.} + proc nodeID*(n: NimNode): int {.magic: "NodeId".} ## Returns the id of ``n``, when the compiler has been compiled ## with the flag ``-d:useNodeids``, otherwise returns ``-1``. This ## proc is for the purpose to debug the compiler only. diff --git a/lib/system/basic_types.nim b/lib/system/basic_types.nim index a6bf69ebc2f5d..9db81d1c4487a 100644 --- a/lib/system/basic_types.nim +++ b/lib/system/basic_types.nim @@ -1,18 +1,18 @@ type - int* {.magic: Int.} ## Default integer type; bitwidth depends on + int* {.magic: "Int".} ## Default integer type; bitwidth depends on ## architecture, but is always the same as a pointer. - int8* {.magic: Int8.} ## Signed 8 bit integer type. - int16* {.magic: Int16.} ## Signed 16 bit integer type. - int32* {.magic: Int32.} ## Signed 32 bit integer type. - int64* {.magic: Int64.} ## Signed 64 bit integer type. - uint* {.magic: UInt.} ## Unsigned default integer type. - uint8* {.magic: UInt8.} ## Unsigned 8 bit integer type. - uint16* {.magic: UInt16.} ## Unsigned 16 bit integer type. - uint32* {.magic: UInt32.} ## Unsigned 32 bit integer type. - uint64* {.magic: UInt64.} ## Unsigned 64 bit integer type. + int8* {.magic: "Int8".} ## Signed 8 bit integer type. + int16* {.magic: "Int16".} ## Signed 16 bit integer type. + int32* {.magic: "Int32".} ## Signed 32 bit integer type. + int64* {.magic: "Int64".} ## Signed 64 bit integer type. + uint* {.magic: "UInt".} ## Unsigned default integer type. + uint8* {.magic: "UInt8".} ## Unsigned 8 bit integer type. + uint16* {.magic: "UInt16".} ## Unsigned 16 bit integer type. + uint32* {.magic: "UInt32".} ## Unsigned 32 bit integer type. + uint64* {.magic: "UInt64".} ## Unsigned 64 bit integer type. type # we need to start a new type section here, so that ``0`` can have a type - bool* {.magic: Bool.} = enum ## Built-in boolean type. + bool* {.magic: "Bool".} = enum ## Built-in boolean type. false = 0, true = 1 const From e8c057fb2729b2598b8cb62ae46f85139fb480fa Mon Sep 17 00:00:00 2001 From: Miran Date: Tue, 3 Mar 2020 16:25:59 +0100 Subject: [PATCH 22/77] important_packages: change the order of arguments in the template (#13577) Leads to a cleaner and less confusing code - an empty command doesn't mean 'nimble test' anymore. --- testament/categories.nim | 4 +- testament/important_packages.nim | 103 +++++++++++++++---------------- 2 files changed, 52 insertions(+), 55 deletions(-) diff --git a/testament/categories.nim b/testament/categories.nim index f8cc625c421cc..9f50a4705b42c 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -10,7 +10,7 @@ ## Include for the tester that contains test suites that test special features ## of the compiler. -# included from tester.nim +# included from testament.nim import important_packages @@ -456,10 +456,8 @@ let packageIndex = nimbleDir / "packages_official.json" iterator listPackages(): tuple[name, url, cmd: string, hasDeps: bool] = - let defaultCmd = "nimble test" let packageList = parseFile(packageIndex) for n, cmd, hasDeps, url in important_packages.packages.items: - let cmd = if cmd.len == 0: defaultCmd else: cmd if url.len != 0: yield (n, url, cmd, hasDeps) else: diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 344aff339b3c2..c90debfd6ffa3 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -1,96 +1,95 @@ -template pkg(name: string; cmd = "nimble test"; hasDeps = false; url = ""): untyped = +template pkg(name: string; hasDeps = false; cmd = "nimble test"; url = ""): untyped = packages.add((name, cmd, hasDeps, url)) var packages*: seq[tuple[name, cmd: string; hasDeps: bool; url: string]] = @[] pkg "argparse" -pkg "arraymancer", "nim c tests/tests_cpu.nim", true -pkg "ast_pattern_matching", "nim c -r --oldgensym:on tests/test1.nim" -pkg "asyncmysql", "", true +pkg "arraymancer", true, "nim c tests/tests_cpu.nim" +pkg "ast_pattern_matching", false, "nim c -r --oldgensym:on tests/test1.nim" +pkg "asyncmysql", true pkg "bigints" -pkg "binaryheap", "nim c -r binaryheap.nim" +pkg "binaryheap", false, "nim c -r binaryheap.nim" # pkg "blscurve", "", true # pending https://github.com/status-im/nim-blscurve/issues/39 -pkg "bncurve", "", true -pkg "c2nim", "nim c testsuite/tester.nim" +pkg "bncurve", true +pkg "c2nim", false, "nim c testsuite/tester.nim" pkg "cascade" pkg "chroma" -pkg "chronicles", "nim c -o:chr -r chronicles.nim", true -pkg "chronos", "", true -pkg "cligen", "nim c -o:cligenn -r cligen.nim" -pkg "coco", "", true +pkg "chronicles", true, "nim c -o:chr -r chronicles.nim" +pkg "chronos", true +pkg "cligen", false, "nim c -o:cligenn -r cligen.nim" +pkg "coco", true pkg "combparser" pkg "compactdict" -pkg "comprehension", "", false, "https://github.com/alehander42/comprehension" +pkg "comprehension", false, "nimble test", "https://github.com/alehander42/comprehension" pkg "criterion" -pkg "dashing", "nim c tests/functional.nim" +pkg "dashing", false, "nim c tests/functional.nim" pkg "docopt" -pkg "easygl", "nim c -o:egl -r src/easygl.nim", true, "https://github.com/jackmott/easygl" +pkg "easygl", true, "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl" pkg "elvis" -pkg "fragments", "nim c -r fragments/dsl.nim" +pkg "fragments", false, "nim c -r fragments/dsl.nim" pkg "gara" -pkg "ggplotnim", "nimble testCI", true +pkg "ggplotnim", true, "nimble testCI" pkg "glob" pkg "gnuplot" -# pkg "godot", "nim c -r godot/godot.nim" # not yet compatible with Nim 0.19 -pkg "hts", "nim c -o:htss src/hts.nim" -pkg "illwill", "nimble examples" +pkg "hts", false, "nim c -o:htss src/hts.nim" +pkg "illwill", false, "nimble examples" pkg "inim" -pkg "itertools", "nim doc src/itertools.nim" +pkg "itertools", false, "nim doc src/itertools.nim" pkg "iterutils" pkg "jstin" -pkg "karax", "nim c -r tests/tester.nim" +pkg "karax", false, "nim c -r tests/tester.nim" pkg "loopfusion" pkg "msgpack4nim" -pkg "nake", "nim c nakefile.nim" -pkg "neo", "nim c -d:blas=openblas tests/all.nim", true +pkg "nake", false, "nim c nakefile.nim" +pkg "neo", true, "nim c -d:blas=openblas tests/all.nim" # pkg "nico", "", true -pkg "nicy", "nim c src/nicy.nim" -pkg "nigui", "nim c -o:niguii -r src/nigui.nim" -pkg "nimcrypto", "nim c -r tests/testall.nim" -pkg "NimData", "nim c -o:nimdataa src/nimdata.nim", true -pkg "nimes", "nim c src/nimes.nim", true -pkg "nimfp", "nim c -o:nfp -r src/fp.nim", true -pkg "nimgame2", "nim c nimgame2/nimgame.nim", true -pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim", true +pkg "nicy", false, "nim c src/nicy.nim" +pkg "nigui", false, "nim c -o:niguii -r src/nigui.nim" +pkg "nimcrypto", false, "nim c -r tests/testall.nim" +pkg "NimData", true, "nim c -o:nimdataa src/nimdata.nim" +pkg "nimes", true, "nim c src/nimes.nim" +pkg "nimfp", true, "nim c -o:nfp -r src/fp.nim" +pkg "nimgame2", true, "nim c nimgame2/nimgame.nim" +pkg "nimgen", true, "nim c -o:nimgenn -r src/nimgen/runcfg.nim" # pkg "nimlsp", "", true -pkg "nimly", "", true +pkg "nimly", true # pkg "nimongo", "nimble test_ci", true -pkg "nimpy", "nim c -r tests/nimfrompy.nim" +pkg "nimpy", false, "nim c -r tests/nimfrompy.nim" pkg "nimquery" -pkg "nimsl", "", true +pkg "nimsl", true pkg "nimsvg" # pkg "nimterop", "", true -pkg "nimx", "nim c --threads:on test/main.nim", true -pkg "norm", "nim c -r tests/tsqlite.nim", true +pkg "nimx", true, "nim c --threads:on test/main.nim" +pkg "norm", true, "nim c -r tests/tsqlite.nim" pkg "npeg" -pkg "ormin", "nim c -o:orminn ormin.nim", true +pkg "ormin", true, "nim c -o:orminn ormin.nim" pkg "parsetoml" pkg "patty" -pkg "plotly", "nim c --oldgensym:on examples/all.nim", true +pkg "plotly", true, "nim c --oldgensym:on examples/all.nim" pkg "pnm" pkg "polypbren" -pkg "protobuf", "nim c -o:protobuff -r src/protobuf.nim", true +pkg "protobuf", true, "nim c -o:protobuff -r src/protobuf.nim" pkg "rbtree" -pkg "react", "nimble example" -pkg "regex", "nim c src/regex", true -pkg "result", "nim c -r result.nim" -pkg "rosencrantz", "nim c -o:rsncntz -r rosencrantz.nim" -pkg "sdl1", "nim c -r src/sdl.nim" -pkg "sdl2_nim", "nim c -r sdl2/sdl.nim" -pkg "snip", "", false, "https://github.com/genotrance/snip" -pkg "stint", "nim c -o:stintt -r stint.nim" -pkg "strunicode", "nim c -r src/strunicode.nim", true -pkg "telebot", "nim c -o:tbot --oldgensym:on -r telebot.nim", true +pkg "react", false, "nimble example" +pkg "regex", true, "nim c src/regex" +pkg "result", false, "nim c -r result.nim" +pkg "rosencrantz", false, "nim c -o:rsncntz -r rosencrantz.nim" +pkg "sdl1", false, "nim c -r src/sdl.nim" +pkg "sdl2_nim", false, "nim c -r sdl2/sdl.nim" +pkg "snip", false, "nimble test", "https://github.com/genotrance/snip" +pkg "stint", false, "nim c -o:stintt -r stint.nim" +pkg "strunicode", true, "nim c -r src/strunicode.nim" +pkg "telebot", true, "nim c -o:tbot --oldgensym:on -r telebot.nim" pkg "tempdir" -pkg "tensordsl", "nim c -r tests/tests.nim", false, "https://krux02@bitbucket.org/krux02/tensordslnim.git" +pkg "tensordsl", false, "nim c -r tests/tests.nim", "https://krux02@bitbucket.org/krux02/tensordslnim.git" pkg "tiny_sqlite" pkg "unicodedb" -pkg "unicodeplus", "", true +pkg "unicodeplus", true pkg "unpack" # pkg "winim", "", true pkg "with" pkg "ws" pkg "yaml" -pkg "zero_functional", "nim c -r test.nim" +pkg "zero_functional", false, "nim c -r test.nim" From f8175688a3b9c88b53685a35a454aeba4b2bcc5d Mon Sep 17 00:00:00 2001 From: genotrance Date: Tue, 3 Mar 2020 13:52:16 -0600 Subject: [PATCH 23/77] Fix docgen snippet numbering (#13507) --- compiler/docgen.nim | 2 +- lib/system/inclrtl.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index daa3a708de9f7..6c98266b5caef 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -191,10 +191,10 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, initStrTable result.types result.onTestSnippet = proc (gen: var RstGenerator; filename, cmd: string; status: int; content: string) = + inc(gen.id) var d = TDocumentor(gen) var outp: AbsoluteFile if filename.len == 0: - inc(d.id) let nameOnly = splitFile(d.filename).name outp = getNimcacheDir(conf) / RelativeDir(nameOnly) / RelativeFile(nameOnly & "_snippet_" & $d.id & ".nim") diff --git a/lib/system/inclrtl.nim b/lib/system/inclrtl.nim index adbda9fd65342..e84794c745062 100644 --- a/lib/system/inclrtl.nim +++ b/lib/system/inclrtl.nim @@ -49,7 +49,7 @@ when defined(nimlocks): else: {.pragma: benign, gcsafe.} -template since(version, body: untyped) {.dirty.} = +template since(version, body: untyped) {.dirty, used.} = ## limitation: can't be used to annotate a template (eg typetraits.get), would ## error: cannot attach a custom pragma. when (NimMajor, NimMinor) >= version: From 8705ee7015382ac2957733dfef5e02a0831e7fb4 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 3 Mar 2020 23:03:33 +0100 Subject: [PATCH 24/77] ARC hotfix; proper destruction of seqs and strings after a move --- compiler/ccgexprs.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 111e1053ba5d3..277913b177bde 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -2040,14 +2040,14 @@ proc genDestroy(p: BProc; n: PNode) = initLocExpr(p, arg, a) linefmt(p, cpsStmts, "if ($1.p && !($1.p->cap & NIM_STRLIT_FLAG)) {$n" & " #deallocShared($1.p);$n" & - " $1.p = NIM_NIL; }$n", + " $1.p = NIM_NIL; $1.len = 0; }$n", [rdLoc(a)]) of tySequence: var a: TLoc initLocExpr(p, arg, a) linefmt(p, cpsStmts, "if ($1.p && !($1.p->cap & NIM_STRLIT_FLAG)) {$n" & " #deallocShared($1.p);$n" & - " $1.p = NIM_NIL; }$n", + " $1.p = NIM_NIL; $1.len = 0; }$n", [rdLoc(a), getTypeDesc(p.module, t.lastSon)]) else: discard "nothing to do" else: From 3ca8f0bde69d060c0b2298113aa02268c35c1550 Mon Sep 17 00:00:00 2001 From: narimiran Date: Wed, 4 Mar 2020 07:15:34 +0100 Subject: [PATCH 25/77] [ci skip] important_packages: change the order in commented out packages too --- testament/important_packages.nim | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index c90debfd6ffa3..9dbb4a4f9f6e6 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -11,7 +11,7 @@ pkg "ast_pattern_matching", false, "nim c -r --oldgensym:on tests/test1.nim" pkg "asyncmysql", true pkg "bigints" pkg "binaryheap", false, "nim c -r binaryheap.nim" -# pkg "blscurve", "", true # pending https://github.com/status-im/nim-blscurve/issues/39 +# pkg "blscurve", true # pending https://github.com/status-im/nim-blscurve/issues/39 pkg "bncurve", true pkg "c2nim", false, "nim c testsuite/tester.nim" pkg "cascade" @@ -44,7 +44,7 @@ pkg "loopfusion" pkg "msgpack4nim" pkg "nake", false, "nim c nakefile.nim" pkg "neo", true, "nim c -d:blas=openblas tests/all.nim" -# pkg "nico", "", true +# pkg "nico", true pkg "nicy", false, "nim c src/nicy.nim" pkg "nigui", false, "nim c -o:niguii -r src/nigui.nim" pkg "nimcrypto", false, "nim c -r tests/testall.nim" @@ -53,14 +53,14 @@ pkg "nimes", true, "nim c src/nimes.nim" pkg "nimfp", true, "nim c -o:nfp -r src/fp.nim" pkg "nimgame2", true, "nim c nimgame2/nimgame.nim" pkg "nimgen", true, "nim c -o:nimgenn -r src/nimgen/runcfg.nim" -# pkg "nimlsp", "", true +# pkg "nimlsp", true pkg "nimly", true -# pkg "nimongo", "nimble test_ci", true +# pkg "nimongo", true, "nimble test_ci" pkg "nimpy", false, "nim c -r tests/nimfrompy.nim" pkg "nimquery" pkg "nimsl", true pkg "nimsvg" -# pkg "nimterop", "", true +# pkg "nimterop", true pkg "nimx", true, "nim c --threads:on test/main.nim" pkg "norm", true, "nim c -r tests/tsqlite.nim" pkg "npeg" @@ -88,7 +88,7 @@ pkg "tiny_sqlite" pkg "unicodedb" pkg "unicodeplus", true pkg "unpack" -# pkg "winim", "", true +# pkg "winim", true pkg "with" pkg "ws" pkg "yaml" From 34c16f5ecae765c014335525ef518233832832db Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Wed, 4 Mar 2020 09:50:36 +0300 Subject: [PATCH 26/77] fix #13409: Document as infix operator (#13570) --- doc/manual.rst | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 71f6f73f9170c..516be24eaaede 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -559,7 +559,7 @@ following characters:: defined here.) These keywords are also operators: -``and or not xor shl shr div mod in notin is isnot of``. +``and or not xor shl shr div mod in notin is isnot of as``. `.`:tok: `=`:tok:, `:`:tok:, `::`:tok: are not available as general operators; they are used for other notational purposes. @@ -632,21 +632,21 @@ has the second lowest precedence. Otherwise precedence is determined by the first character. -================ =============================================== ================== =============== -Precedence level Operators First character Terminal symbol -================ =============================================== ================== =============== - 10 (highest) ``$ ^`` OP10 - 9 ``* / div mod shl shr %`` ``* % \ /`` OP9 - 8 ``+ -`` ``+ - ~ |`` OP8 - 7 ``&`` ``&`` OP7 - 6 ``..`` ``.`` OP6 - 5 ``== <= < >= > != in notin is isnot not of`` ``= < > !`` OP5 - 4 ``and`` OP4 - 3 ``or xor`` OP3 - 2 ``@ : ?`` OP2 - 1 *assignment operator* (like ``+=``, ``*=``) OP1 - 0 (lowest) *arrow like operator* (like ``->``, ``=>``) OP0 -================ =============================================== ================== =============== +================ ================================================== ================== =============== +Precedence level Operators First character Terminal symbol +================ ================================================== ================== =============== + 10 (highest) ``$ ^`` OP10 + 9 ``* / div mod shl shr %`` ``* % \ /`` OP9 + 8 ``+ -`` ``+ - ~ |`` OP8 + 7 ``&`` ``&`` OP7 + 6 ``..`` ``.`` OP6 + 5 ``== <= < >= > != in notin is isnot not of as`` ``= < > !`` OP5 + 4 ``and`` OP4 + 3 ``or xor`` OP3 + 2 ``@ : ?`` OP2 + 1 *assignment operator* (like ``+=``, ``*=``) OP1 + 0 (lowest) *arrow like operator* (like ``->``, ``=>``) OP0 +================ ================================================== ================== =============== Whether an operator is used a prefix operator is also affected by preceding From 9961d1f67d623bc8a4f6bbab5714d945cbe4abc0 Mon Sep 17 00:00:00 2001 From: Miran Date: Wed, 4 Mar 2020 10:25:59 +0100 Subject: [PATCH 27/77] fix #13531 by adding a test (#13581) --- tests/stdlib/tjson_unmarshall.nim | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 tests/stdlib/tjson_unmarshall.nim diff --git a/tests/stdlib/tjson_unmarshall.nim b/tests/stdlib/tjson_unmarshall.nim new file mode 100644 index 0000000000000..69bed3ac9b8a0 --- /dev/null +++ b/tests/stdlib/tjson_unmarshall.nim @@ -0,0 +1,31 @@ +discard """ + output: ''' +Original: (kind: P, pChildren: @[(kind: Text, textStr: "mychild"), (kind: Br)]) +jsonNode: {"kind":"P","pChildren":[{"kind":"Text","textStr":"mychild"},{"kind":"Br"}]} +Reversed: (kind: P, pChildren: @[(kind: Text, textStr: "mychild"), (kind: Br)]) +''' +""" + +import json + +type + ContentNodeKind* = enum + P, + Br, + Text, + ContentNode* = object + case kind*: ContentNodeKind + of P: pChildren*: seq[ContentNode] + of Br: nil + of Text: textStr*: string + +let mynode = ContentNode(kind: P, pChildren: @[ + ContentNode(kind: Text, textStr: "mychild"), + ContentNode(kind: Br) +]) + +echo "Original: " & $mynode + +let jsonNode = %*mynode +echo "jsonNode: " & $jsonNode +echo "Reversed: " & $jsonNode.to(ContentNode) From 9fa070b1dc566c23b9f8c1684d33b63e9ed3b3c4 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 4 Mar 2020 01:26:56 -0800 Subject: [PATCH 28/77] fix hintSuccess: `out` was wrong for `nim doc` without `-o` flag (#13569) --- compiler/docgen.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 6c98266b5caef..064bdc93e9f96 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -1135,8 +1135,8 @@ proc generateIndex*(d: PDoc) = proc updateOutfile(d: PDoc, outfile: AbsoluteFile) = if d.module == nil or sfMainModule in d.module.flags: # nil for eg for commandRst2Html - if d.conf.outFile.isEmpty and not d.conf.outDir.isEmpty: - d.conf.outFile = outfile.relativeTo(d.conf.outDir) + if d.conf.outDir.isEmpty: d.conf.outDir = d.conf.projectPath + if d.conf.outFile.isEmpty: d.conf.outFile = outfile.relativeTo(d.conf.outDir) proc writeOutput*(d: PDoc, useWarning = false) = runAllExamples(d) From 0809098971080674f723cb219722487e42011fd1 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Wed, 4 Mar 2020 06:27:52 -0300 Subject: [PATCH 29/77] Nimpretty Fix negative indent breaks code (#13580) --- changelog.md | 1 + nimpretty/nimpretty.nim | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index e0fd649428495..87bf8bd08e25e 100644 --- a/changelog.md +++ b/changelog.md @@ -126,6 +126,7 @@ ### Tool changes +- Fix Nimpretty must not accept negative indentation argument because breaks file. ### Compiler changes diff --git a/nimpretty/nimpretty.nim b/nimpretty/nimpretty.nim index 95f4285984203..50d0d725f8dfa 100644 --- a/nimpretty/nimpretty.nim +++ b/nimpretty/nimpretty.nim @@ -44,8 +44,8 @@ proc writeVersion() = type PrettyOptions = object - indWidth: int - maxLineLen: int + indWidth: Natural + maxLineLen: Positive proc prettyPrint(infile, outfile: string, opt: PrettyOptions) = var conf = newConfigRef() From 614fb7567c80c3b071394714c3809c005aaad397 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 4 Mar 2020 13:46:42 +0100 Subject: [PATCH 30/77] std/compilesettings implementation (#13584) * Implement compileSetting() and compileSettingSeq() * Change from magic to vmop * better design for querySetting Co-authored-by: genotrance --- changelog.md | 12 ++++++++ compiler/vmops.nim | 33 ++++++++++++++++++++++ lib/std/compilesettings.nim | 54 ++++++++++++++++++++++++++++++++++++ tests/vm/tcompilesetting.nim | 19 +++++++++++++ 4 files changed, 118 insertions(+) create mode 100644 lib/std/compilesettings.nim create mode 100644 tests/vm/tcompilesetting.nim diff --git a/changelog.md b/changelog.md index 87bf8bd08e25e..ccc969ca641d1 100644 --- a/changelog.md +++ b/changelog.md @@ -78,10 +78,22 @@ ```nim +type + Foo = object + col, pos: string +proc setColor(f: var Foo; r, g, b: int) = f.col = $(r, g, b) +proc setPosition(f: var Foo; x, y: float) = f.pos = $(x, y) + +var f: Foo +with(f, setColor(2, 3, 4), setPosition(0.0, 1.0)) +echo f ``` + - Added `times.isLeapDay` +- Added a new module, `std / compilesettings` for querying the compiler about + diverse configuration settings. ## Library changes diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 1e3a258f3111b..8cb2a055234d3 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -102,6 +102,35 @@ proc staticWalkDirImpl(path: string, relative: bool): PNode = result.add newTree(nkTupleConstr, newIntNode(nkIntLit, k.ord), newStrNode(nkStrLit, f)) +from std / compilesettings import SingleValueSetting, MultipleValueSetting + +proc querySettingImpl(a: VmArgs, conf: ConfigRef, switch: BiggestInt): string = + case SingleValueSetting(switch) + of arguments: result = conf.arguments + of outFile: result = conf.outFile.string + of outDir: result = conf.outDir.string + of nimcacheDir: result = conf.nimcacheDir.string + of projectName: result = conf.projectName + of projectPath: result = conf.projectPath.string + of projectFull: result = conf.projectFull.string + of command: result = conf.command + of commandLine: result = conf.commandLine + of linkOptions: result = conf.linkOptions + of compileOptions: result = conf.compileOptions + of ccompilerPath: result = conf.cCompilerPath + +proc querySettingSeqImpl(a: VmArgs, conf: ConfigRef, switch: BiggestInt): seq[string] = + template copySeq(field: untyped): untyped = + for i in field: result.add i.string + + case MultipleValueSetting(switch) + of nimblePaths: copySeq(conf.nimblePaths) + of searchPaths: copySeq(conf.searchPaths) + of lazyPaths: copySeq(conf.lazyPaths) + of commandArgs: result = conf.commandArgs + of cincludes: copySeq(conf.cIncludes) + of clibs: copySeq(conf.cLibs) + proc registerAdditionalOps*(c: PCtx) = proc gorgeExWrapper(a: VmArgs) = let (s, e) = opGorge(getString(a, 0), getString(a, 1), getString(a, 2), @@ -152,6 +181,10 @@ proc registerAdditionalOps*(c: PCtx) = systemop getCurrentException registerCallback c, "stdlib.*.staticWalkDir", proc (a: VmArgs) {.nimcall.} = setResult(a, staticWalkDirImpl(getString(a, 0), getBool(a, 1))) + registerCallback c, "stdlib.compilesettings.querySetting", proc (a: VmArgs) {.nimcall.} = + setResult(a, querySettingImpl(a, c.config, getInt(a, 0))) + registerCallback c, "stdlib.compilesettings.querySettingSeq", proc (a: VmArgs) {.nimcall.} = + setResult(a, querySettingSeqImpl(a, c.config, getInt(a, 0))) if defined(nimsuggest) or c.config.cmd == cmdCheck: discard "don't run staticExec for 'nim suggest'" diff --git a/lib/std/compilesettings.nim b/lib/std/compilesettings.nim new file mode 100644 index 0000000000000..12204658d45cc --- /dev/null +++ b/lib/std/compilesettings.nim @@ -0,0 +1,54 @@ +# +# +# The Nim Compiler +# (c) Copyright 2020 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module allows querying the compiler about +## diverse configuration settings. + +# Note: Only add new enum values at the end to ensure binary compatibility with +# other Nim compiler versions! + +type + SingleValueSetting* {.pure.} = enum ## \ + ## settings resulting in a single string value + arguments, ## experimental: the arguments passed after '-r' + outFile, ## experimental: the output file + outDir, ## the output directory + nimcacheDir, ## the location of the 'nimcache' directory + projectName, ## the project's name that is being compiled + projectPath, ## experimental: some path to the project that is being compiled + projectFull, ## the full path to the project that is being compiled + command, ## experimental: the command (e.g. 'c', 'cpp', 'doc') passed to + ## the Nim compiler + commandLine, ## experimental: the command line passed to Nim + linkOptions, ## additional options passed to the linker + compileOptions, ## additional options passed to the C/C++ compiler + ccompilerPath ## the path to the C/C++ compiler + + MultipleValueSetting* {.pure.} = enum ## \ + ## settings resulting in a seq of string values + nimblePaths, ## the nimble path(s) + searchPaths, ## the search path for modules + lazyPaths, ## experimental: even more paths + commandArgs, ## the arguments passed to the Nim compiler + cincludes, ## the #include paths passed to the C/C++ compiler + clibs ## libraries passed to the C/C++ compiler + +proc querySetting*(setting: SingleValueSetting): string {. + compileTime, noSideEffect.} = discard + ## Can be used to get a string compile-time option. Example: + ## + ## .. code-block:: Nim + ## const nimcache = querySetting(SingleValueSetting.nimcacheDir) + +proc querySettingSeq*(setting: MultipleValueSetting): seq[string] {. + compileTime, noSideEffect.} = discard + ## Can be used to get a multi-string compile-time option. Example: + ## + ## .. code-block:: Nim + ## const nimblePaths = compileSettingSeq(MultipleValueSetting.nimblePaths) diff --git a/tests/vm/tcompilesetting.nim b/tests/vm/tcompilesetting.nim new file mode 100644 index 0000000000000..0e936d377e2cb --- /dev/null +++ b/tests/vm/tcompilesetting.nim @@ -0,0 +1,19 @@ +discard """ +cmd: "nim c --nimcache:myNimCache --nimblePath:myNimblePath $file" +joinable: false +""" + +import strutils + +import std / compilesettings + +const + nc = querySetting(nimcacheDir) + np = querySettingSeq(nimblePaths) + +static: + echo nc + echo np + +doAssert "myNimCache" in nc +doAssert "myNimblePath" in np[0] From a0eca7518223a18f3633150de2c8d3c1c9e71560 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 4 Mar 2020 14:28:53 +0100 Subject: [PATCH 31/77] sink parameter inference for types that have destructors (#13544) * ensure capitalize doesn't take an inferred sink parameter * sink parameter inference: first version, for now disabled. Changed that sink parameters can be consumed multiple times in order to adhere to our spec. * sink inference can now be disabled with .nosinks; sometimes for proc type interop this is required * fixes yet another critical DFA bug * better implementation that also understands if expressions etc * document sink parameter inference and allow for global disabling --- changelog.md | 4 +- compiler/ast.nim | 9 +++- compiler/commands.nim | 2 + compiler/condsyms.nim | 2 + compiler/dfa.nim | 2 + compiler/injectdestructors.nim | 12 ++--- compiler/options.nim | 7 +-- compiler/pragmas.nim | 8 +++- compiler/sempass2.nim | 22 +++++++-- compiler/semstmts.nim | 2 + compiler/sinkparameter_inference.nim | 70 ++++++++++++++++++++++++++++ compiler/wordrecg.nim | 8 ++-- doc/advopt.txt | 3 +- doc/destructors.rst | 17 +++++++ lib/pure/unicode.nim | 2 +- lib/system/assertions.nim | 4 +- tests/assert/tassert_c.nim | 4 +- tests/destructor/tarc2.nim | 4 ++ tests/destructor/tconsume_twice.nim | 4 +- 19 files changed, 158 insertions(+), 28 deletions(-) create mode 100644 compiler/sinkparameter_inference.nim diff --git a/changelog.md b/changelog.md index ccc969ca641d1..63296ce041fa5 100644 --- a/changelog.md +++ b/changelog.md @@ -127,7 +127,6 @@ echo f - `=sink` type bound operator is now optional. Compiler can now use combination of `=destroy` and `copyMem` to move objects efficiently. - ## Language changes - Unsigned integer operators have been fixed to allow promotion of the first operand. @@ -150,6 +149,9 @@ echo f - The Nim compiler now supports a new pragma called ``.localPassc`` to pass specific compiler options to the C(++) backend for the C(++) file that was produced from the current Nim module. +- The compiler now inferes "sink parameters". To disable this for a specific routine, + annotate it with `.nosinks`. To disable it for a section of code, use + `{.push sinkInference: off.}`...`{.pop.}`. ## Bugfixes diff --git a/compiler/ast.nim b/compiler/ast.nim index 3b10a89eaf039..43ea2b39c44fb 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -229,7 +229,7 @@ type TNodeKinds* = set[TNodeKind] type - TSymFlag* = enum # 40 flags! + TSymFlag* = enum # 41 flags! sfUsed, # read access of sym (for warnings) or simply used sfExported, # symbol is exported from module sfFromGeneric, # symbol is instantiation of a generic; this is needed @@ -238,6 +238,8 @@ type sfGlobal, # symbol is at global scope sfForward, # symbol is forward declared + sfWasForwarded, # symbol had a forward declaration + # (implies it's too dangerous to patch its type signature) sfImportc, # symbol is external; imported sfExportc, # symbol is exported (under a specified name) sfMangleCpp, # mangle as cpp (combines with `sfExportc`) @@ -847,7 +849,7 @@ type position*: int # used for many different things: # for enum fields its position; # for fields its offset - # for parameters its position + # for parameters its position (starting with 0) # for a conditional: # 1 iff the symbol is defined, else 0 # (or not in symbol table) @@ -1865,6 +1867,9 @@ proc isInlineIterator*(typ: PType): bool {.inline.} = proc isClosureIterator*(typ: PType): bool {.inline.} = typ.kind == tyProc and tfIterator in typ.flags and typ.callConv == ccClosure +proc isClosure*(typ: PType): bool {.inline.} = + typ.kind == tyProc and typ.callConv == ccClosure + proc isSinkParam*(s: PSym): bool {.inline.} = s.kind == skParam and (s.typ.kind == tySink or tfHasOwned in s.typ.flags) diff --git a/compiler/commands.nim b/compiler/commands.nim index 0545ee2b0043f..935d7e2be5f85 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -877,6 +877,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; localError(conf, info, "unknown Nim version; currently supported values are: {1.0}") of "benchmarkvm": processOnOffSwitchG(conf, {optBenchmarkVM}, arg, pass, info) + of "sinkinference": + processOnOffSwitch(conf, {optSinkInference}, arg, pass, info) of "": # comes from "-" in for example: `nim c -r -` (gets stripped from -) handleStdinInput(conf) else: diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index d82f4811dd04e..a84b0d8780baa 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -111,3 +111,5 @@ proc initDefines*(symbols: StringTableRef) = # will report the right thing regardless of whether user adds # `-d:nimHasLibFFI` in his user config. defineSymbol("nimHasLibFFIEnabled") + + defineSymbol("nimHasSinkInference") diff --git a/compiler/dfa.nim b/compiler/dfa.nim index b46db9aed8cd7..0f23d7f1a9c11 100644 --- a/compiler/dfa.nim +++ b/compiler/dfa.nim @@ -591,6 +591,8 @@ proc genUse(c: var Con; orig: PNode) = of PathKinds1: n = n[1] else: break + if n.kind in nkCallKinds: + gen(c, n) if n.kind == nkSym and n.sym.kind in InterestingSyms: c.code.add Instr(n: orig, kind: use) diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index bdd6242816be3..520ba46ceea88 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -321,8 +321,8 @@ proc destructiveMoveVar(n: PNode; c: var Con): PNode = proc sinkParamIsLastReadCheck(c: var Con, s: PNode) = assert s.kind == nkSym and s.sym.kind == skParam if not isLastRead(s, c): - localError(c.graph.config, c.otherRead.info, "sink parameter `" & $s.sym.name.s & - "` is already consumed at " & toFileLineCol(c. graph.config, s.info)) + localError(c.graph.config, c.otherRead.info, "sink parameter `" & $s.sym.name.s & + "` is already consumed at " & toFileLineCol(c. graph.config, s.info)) type ProcessMode = enum @@ -498,10 +498,10 @@ proc p(n: PNode; c: var Con; mode: ProcessMode): PNode = elif n.kind in {nkBracket, nkObjConstr, nkTupleConstr, nkClosure, nkNilLit} + nkCallKinds + nkLiterals: result = p(n, c, consumed) - elif n.kind == nkSym and isSinkParam(n.sym): + elif n.kind == nkSym and isSinkParam(n.sym) and isLastRead(n, c): # Sinked params can be consumed only once. We need to reset the memory # to disable the destructor which we have not elided - sinkParamIsLastReadCheck(c, n) + #sinkParamIsLastReadCheck(c, n) result = destructiveMoveVar(n, c) elif isAnalysableFieldAccess(n, c.owner) and isLastRead(n, c): # it is the last read, can be sinkArg. We need to reset the memory @@ -687,9 +687,9 @@ proc moveOrCopy(dest, ri: PNode; c: var Con): PNode = of nkObjConstr, nkTupleConstr, nkClosure, nkCharLit..nkNilLit: result = genSink(c, dest, p(ri, c, consumed)) of nkSym: - if isSinkParam(ri.sym): + if isSinkParam(ri.sym) and isLastRead(ri, c): # Rule 3: `=sink`(x, z); wasMoved(z) - sinkParamIsLastReadCheck(c, ri) + #sinkParamIsLastReadCheck(c, ri) let snk = genSink(c, dest, ri) result = newTree(nkStmtList, snk, genWasMoved(ri, c)) elif ri.sym.kind != skParam and ri.sym.owner == c.owner and diff --git a/compiler/options.nim b/compiler/options.nim index c93fc06484e44..42406272cc206 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -39,7 +39,8 @@ type # please make sure we have under 32 options optMemTracker, optLaxStrings, optNilSeqs, - optOldAst + optOldAst, + optSinkInference # 'sink T' inference TOptions* = set[TOption] TGlobalOption* = enum # **keep binary compatible** @@ -200,7 +201,7 @@ type ## the incremental compilation mechanisms ## (+) means "part of the dependency" target*: Target # (+) - linesCompiled*: int # all lines that have been compiled + linesCompiled*: int # all lines that have been compiled options*: TOptions # (+) globalOptions*: TGlobalOptions # (+) macrosToExpand*: StringTableRef @@ -315,7 +316,7 @@ const DefaultOptions* = {optObjCheck, optFieldCheck, optRangeCheck, optBoundsCheck, optOverflowCheck, optAssert, optWarns, optRefCheck, optHints, optStackTrace, optLineTrace, - optTrMacros, optNilCheck, optStyleCheck} + optTrMacros, optNilCheck, optStyleCheck, optSinkInference} DefaultGlobalOptions* = {optThreadAnalysis, optExcessiveStackTrace, optListFullPaths} diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 559c1c0640878..3e89917ed92bb 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -23,7 +23,7 @@ const wExportNims, wExtern, wDeprecated, wNodecl, wError, wUsed, wAlign} ## common pragmas for declarations, to a good approximation procPragmas* = declPragmas + {FirstCallConv..LastCallConv, - wMagic, wNoSideEffect, wSideEffect, wNoreturn, wDynlib, wHeader, + wMagic, wNoSideEffect, wSideEffect, wNoreturn, wNosinks, wDynlib, wHeader, wCompilerProc, wNonReloadable, wCore, wProcVar, wVarargs, wCompileTime, wMerge, wBorrow, wImportCompilerProc, wThread, wAsmNoStackFrame, wDiscardable, wNoInit, wCodegenDecl, @@ -53,7 +53,7 @@ const wLinearScanEnd, wPatterns, wTrMacros, wEffects, wNoForward, wReorder, wComputedGoto, wInjectStmt, wExperimental, wThis, wUsed} lambdaPragmas* = declPragmas + {FirstCallConv..LastCallConv, - wNoSideEffect, wSideEffect, wNoreturn, wDynlib, wHeader, + wNoSideEffect, wSideEffect, wNoreturn, wNosinks, wDynlib, wHeader, wThread, wAsmNoStackFrame, wRaises, wLocks, wTags, wGcSafe, wCodegenDecl} - {wExportNims, wError, wUsed} # why exclude these? typePragmas* = declPragmas + {wMagic, wAcyclic, @@ -357,6 +357,7 @@ proc pragmaToOptions(w: TSpecialWord): TOptions {.inline.} = of wByRef: {optByRef} of wImplicitStatic: {optImplicitStatic} of wPatterns, wTrMacros: {optTrMacros} + of wSinkInference: {optSinkInference} else: {} proc processExperimental(c: PContext; n: PNode) = @@ -889,6 +890,9 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, of wNoDestroy: noVal(c, it) incl(sym.flags, sfGeneratedOp) + of wNosinks: + noVal(c, it) + incl(sym.flags, sfWasForwarded) of wDynlib: processDynLib(c, it, sym) of wCompilerProc, wCore: diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 98ef8cf8582ac..af706bee12ec6 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -16,6 +16,7 @@ when defined(useDfa): import dfa import liftdestructors +include sinkparameter_inference #[ Second semantic checking pass over the AST. Necessary because the old way had some inherent problems. Performs: @@ -774,6 +775,10 @@ proc track(tracked: PEffects, n: PNode) = for i in 0.. p: sink T + passToSink(p) # p: sink + ObjConstr(fieldName: p) + [p, q] # array construction + + # Open question: + var local = p # sink parameter? + passToSink(local) + ]# + if optSinkInference notin config.options: return + case arg.kind + of nkSym: + if arg.sym.kind == skParam and + arg.sym.owner == owner and + owner.typ != nil and owner.typ.kind == tyProc and + arg.sym.typ.hasDestructor and + arg.sym.typ.kind notin {tyVar, tySink, tyOwned}: + # Watch out: cannot do this inference for procs with forward + # declarations. + if sfWasForwarded notin owner.flags: + let argType = arg.sym.typ + + let sinkType = newType(tySink, owner) + sinkType.size = argType.size + sinkType.align = argType.align + sinkType.paddingAtEnd = argType.paddingAtEnd + sinkType.add argType + + arg.sym.typ = sinkType + owner.typ[arg.sym.position+1] = sinkType + + #message(config, arg.info, warnUser, + # ("turned '$1' to a sink parameter") % [$arg]) + #echo config $ arg.info, " turned into a sink parameter ", arg.sym.name.s + elif sfWasForwarded notin arg.sym.flags: + # we only report every potential 'sink' parameter only once: + incl arg.sym.flags, sfWasForwarded + message(config, arg.info, hintPerformance, + ("could not turn '$1' to a sink parameter " & + "because '$2' was forward declared") % [arg.sym.name.s, owner.name.s]) + #echo config $ arg.info, " candidate for a sink parameter here" + of nkStmtList, nkStmtListExpr, nkBlockStmt, nkBlockExpr: + if not isEmptyType(arg.typ): + checkForSink(config, owner, arg.lastSon) + of nkIfStmt, nkIfExpr, nkWhen: + for branch in arg: + let value = branch.lastSon + if not isEmptyType(value.typ): + checkForSink(config, owner, value) + of nkCaseStmt: + for i in 1.. Date: Wed, 4 Mar 2020 14:56:40 +0000 Subject: [PATCH 32/77] nimsuggest: don't add CRLF to replies (#13545) This is already sent by replTcp after the message is sent. --- nimsuggest/nimsuggest.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nimsuggest/nimsuggest.nim b/nimsuggest/nimsuggest.nim index a48e9443d8846..31cedb72ec1a8 100644 --- a/nimsuggest/nimsuggest.nim +++ b/nimsuggest/nimsuggest.nim @@ -252,10 +252,10 @@ proc toSocket(stdoutSocket: Socket) {.gcsafe.} = let res = results.recv() case res.section of ideNone: break - of ideMsg: stdoutSocket.send(res.doc & "\c\L") - of ideKnown: stdoutSocket.send($(res.quality == 1) & "\c\L") - of ideProject: stdoutSocket.send(res.filePath & "\c\L") - else: stdoutSocket.send($res & "\c\L") + of ideMsg: stdoutSocket.send(res.doc) + of ideKnown: stdoutSocket.send($(res.quality == 1)) + of ideProject: stdoutSocket.send(res.filePath) + else: stdoutSocket.send($res) proc toEpc(client: Socket; uid: BiggestInt) {.gcsafe.} = var list = newSList() From 357edd86b429dfaa3c24dedabdad0eb68915616f Mon Sep 17 00:00:00 2001 From: Varriount Date: Wed, 4 Mar 2020 14:44:42 -0500 Subject: [PATCH 33/77] fix nightlies builds on Windows (#13587) Before the files that should have been ignored weren't ignored because of different dir separator ('\' vs '/') on Windows. --- tools/kochdocs.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 4a07b7fd9df19..a5166973aa96e 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -1,6 +1,6 @@ ## Part of 'koch' responsible for the documentation generation. -import os, strutils, osproc, sets +import os, strutils, osproc, sets, pathnorm const gaCode* = " --doc.googleAnalytics:UA-48159761-1" @@ -195,11 +195,11 @@ lib/system/widestrs.nim a.isRelativeTo("lib/pure/includes") or a.isRelativeTo("lib/genode") or a.isRelativeTo("lib/deprecated") or - (a.isRelativeTo("lib/system") and a notin goodSystem) or - a in docIgnore: + (a.isRelativeTo("lib/system") and a.replace('\\', '/') notin goodSystem) or + a.replace('\\', '/') in docIgnore: continue result.add a - result.add "nimsuggest/sexp.nim" + result.add normalizePath("nimsuggest/sexp.nim") let doc = getDocList() From 62c113ebc7ba6fa1594b24158a6cc3e1170f4030 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 5 Mar 2020 15:31:22 +0100 Subject: [PATCH 34/77] fix #13579 joinPath("/foo/", "../a") is now /a (#13586) --- compiler/pathutils.nim | 2 ++ lib/pure/os.nim | 1 + lib/pure/pathnorm.nim | 5 +++++ tests/stdlib/tos.nim | 13 +++++++++++++ 4 files changed, 21 insertions(+) diff --git a/compiler/pathutils.nim b/compiler/pathutils.nim index 1c35eb0b23b51..888e0a572d74f 100644 --- a/compiler/pathutils.nim +++ b/compiler/pathutils.nim @@ -102,6 +102,8 @@ when true: when isMainModule: doAssert AbsoluteDir"/Users/me///" / RelativeFile"z.nim" == AbsoluteFile"/Users/me/z.nim" + doAssert AbsoluteDir"/Users/me" / RelativeFile"../z.nim" == AbsoluteFile"/Users/z.nim" + doAssert AbsoluteDir"/Users/me/" / RelativeFile"../z.nim" == AbsoluteFile"/Users/z.nim" doAssert relativePath("/foo/bar.nim", "/foo/", '/') == "bar.nim" doAssert $RelativeDir"foo/bar" == "foo/bar" doAssert RelativeDir"foo/bar" == RelativeDir"foo/bar" diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 3db518273f5c5..ba092da05167a 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -127,6 +127,7 @@ template endsWith(a: string, b: set[char]): bool = proc joinPathImpl(result: var string, state: var int, tail: string) = let trailingSep = tail.endsWith({DirSep, AltSep}) or tail.len == 0 and result.endsWith({DirSep, AltSep}) + normalizePathEnd(result, trailingSep=false) addNormalizePath(tail, result, state, DirSep) normalizePathEnd(result, trailingSep=trailingSep) diff --git a/lib/pure/pathnorm.nim b/lib/pure/pathnorm.nim index 3659e85e623e5..8763ba8780eb0 100644 --- a/lib/pure/pathnorm.nim +++ b/lib/pure/pathnorm.nim @@ -76,6 +76,11 @@ proc addNormalizePath*(x: string; result: var string; state: var int; if (state shr 1) >= 1: var d = result.len # f/.. + # We could handle stripping trailing sep here: foo// => foo like this: + # while (d-1) > (state and 1) and result[d-1] in {DirSep, AltSep}: dec d + # but right now we instead handle it inside os.joinPath + + # strip path component: foo/bar => foo while (d-1) > (state and 1) and result[d-1] notin {DirSep, AltSep}: dec d if d > 0: diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim index c649749e35444..d713bfe0ceb03 100644 --- a/tests/stdlib/tos.nim +++ b/tests/stdlib/tos.nim @@ -389,6 +389,19 @@ block ospaths: doAssert joinPath("foo", "./") == unixToNativePath"foo/" doAssert joinPath("foo", "", "bar/") == unixToNativePath"foo/bar/" + # issue #13579 + doAssert joinPath("/foo", "../a") == unixToNativePath"/a" + doAssert joinPath("/foo/", "../a") == unixToNativePath"/a" + doAssert joinPath("/foo/.", "../a") == unixToNativePath"/a" + doAssert joinPath("/foo/.b", "../a") == unixToNativePath"/foo/a" + doAssert joinPath("/foo///", "..//a/") == unixToNativePath"/a/" + doAssert joinPath("foo/", "../a") == unixToNativePath"a" + + when doslikeFileSystem: + doAssert joinPath("C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\Common7\\Tools\\", "..\\..\\VC\\vcvarsall.bat") == r"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" + doAssert joinPath("C:\\foo", "..\\a") == r"C:\a" + doAssert joinPath("C:\\foo\\", "..\\a") == r"C:\a" + block getTempDir: block TMPDIR: # TMPDIR env var is not used if either of these are defined. From 83e715c5b6b2501c02459398a2f394c457758a55 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 5 Mar 2020 16:02:34 +0100 Subject: [PATCH 35/77] fixes #5170 (#13589) * fixes #5170 * make tests green --- compiler/semstmts.nim | 2 ++ compiler/sighashes.nim | 19 +++++++++---------- tests/errmsgs/tsigmatch.nim | 6 +++--- tests/errmsgs/tsigmatch2.nim | 4 ++-- tests/types/tissues_types.nim | 30 ++++++++++++++++++++++++++++++ 5 files changed, 46 insertions(+), 15 deletions(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 4d4f702451401..b91e93025c40f 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1298,6 +1298,8 @@ proc typeSectionFinalPass(c: PContext, n: PNode) = checkConstructedType(c.config, s.info, s.typ) if s.typ.kind in {tyObject, tyTuple} and not s.typ.n.isNil: checkForMetaFields(c, s.typ.n) + # fix bug #5170: ensure locally scoped object types get a unique name: + if s.typ.kind == tyObject and not isTopLevel(c): incl(s.flags, sfGenSym) #instAllTypeBoundOp(c, n.info) diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index f94cd5bce16b3..0902a839b8d23 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -35,6 +35,7 @@ type CoIgnoreRange CoConsiderOwned CoDistinct + CoHashTypeInsideNode proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) @@ -61,7 +62,7 @@ proc hashTypeSym(c: var MD5Context, s: PSym) = c &= "." it = it.owner -proc hashTree(c: var MD5Context, n: PNode) = +proc hashTree(c: var MD5Context, n: PNode; flags: set[ConsiderFlag]) = if n == nil: c &= "\255" return @@ -75,6 +76,8 @@ proc hashTree(c: var MD5Context, n: PNode) = c &= n.ident.s of nkSym: hashSym(c, n.sym) + if CoHashTypeInsideNode in flags and n.sym.typ != nil: + hashType(c, n.sym.typ, flags) of nkCharLit..nkUInt64Lit: let v = n.intVal lowlevel v @@ -84,7 +87,7 @@ proc hashTree(c: var MD5Context, n: PNode) = of nkStrLit..nkTripleStrLit: c &= n.strVal else: - for i in 0.. Date: Thu, 5 Mar 2020 21:03:39 +0100 Subject: [PATCH 36/77] Removed simpleGetOrDefault (#13590) --- lib/pure/json.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 82923ced01825..beb3f7049bbea 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -523,8 +523,10 @@ proc getOrDefault*(node: JsonNode, key: string): JsonNode = if not isNil(node) and node.kind == JObject: result = node.fields.getOrDefault(key) -template simpleGetOrDefault*{`{}`(node, [key])}(node: JsonNode, - key: string): JsonNode = node.getOrDefault(key) +proc `{}`*(node: JsonNode, key: string): JsonNode = + ## Gets a field from a `node`. If `node` is nil or not an object or + ## value at `key` does not exist, returns nil + node.getOrDefault(key) proc `{}=`*(node: JsonNode, keys: varargs[string], value: JsonNode) = ## Traverses the node and tries to set the value at the given location From b80d293a3f8c301a2d9188fb409162028a00edca Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 5 Mar 2020 23:55:53 -0800 Subject: [PATCH 37/77] close #12704 by adding a test (tuple codegen error) (#13592) * close #12704 by adding a test * move test to tests/metatype/ttypedesc2.nim --- tests/metatype/ttypedesc2.nim | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/metatype/ttypedesc2.nim b/tests/metatype/ttypedesc2.nim index 89c38f367808c..96dab90527684 100644 --- a/tests/metatype/ttypedesc2.nim +++ b/tests/metatype/ttypedesc2.nim @@ -35,9 +35,23 @@ type Point[T] = tuple[x, y: T] proc origin(T: typedesc): Point[T] = discard discard origin(int) +block: # issue #12704 + const a = $("a", "b") + proc fun() = + const str = $int + let b = $(str, "asdf") + fun() + # https://github.com/nim-lang/Nim/issues/7516 import typetraits +block: #issue #12704 + const a = $("a", "b") + proc fun() = + const str = name(int) + let b = $(str, "asdf") + fun() + proc hasDefault1(T: type = int): auto = return T.name doAssert hasDefault1(int) == "int" doAssert hasDefault1(string) == "string" From ec8a17cc861c8282ae33ae36030d1702e41b275a Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 22 Feb 2020 11:46:24 +0000 Subject: [PATCH 38/77] Revert broken asynchttpserver FutureStream additions. As discussed in #13394, these changes cannot work. Reverted via ``` git revert --no-commit 5bf571f061d53d35aab727f420afd9f415987723 git revert --no-commit abd660c407d00d0c4f2129ff11bfc69badda8ece git revert --no-commit 955465e5f42b1353f69f3bd884908a7ef91ce13b git commit ``` --- changelog.md | 2 - lib/pure/asynchttpserver.nim | 176 ++++++----------------------------- 2 files changed, 31 insertions(+), 147 deletions(-) diff --git a/changelog.md b/changelog.md index 63296ce041fa5..52406a97218ac 100644 --- a/changelog.md +++ b/changelog.md @@ -98,8 +98,6 @@ echo f ## Library changes -- `asynchttpserver` added an iterator that allows the request body to be read in - chunks of data when new server "stream" option is set to true. - `asyncdispatch.drain` now properly takes into account `selector.hasPendingOperations` and only returns once all pending async operations are guaranteed to have completed. - `asyncdispatch.drain` now consistently uses the passed timeout value for all diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 9aed0255bbf36..186f0da414650 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -29,129 +29,43 @@ ## await req.respond(Http200, "Hello World") ## ## waitFor server.serve(Port(8080), cb) -## -## Basic Post request handle -## ========================= -## -## This example will create an HTTP server on port 8080. The server will -## respond with a page with the actual and expected body length after -## submitting a file. -## -## .. code-block::nim -## import asynchttpserver, asyncdispatch -## import strutils, strformat -## -## const stream = true # for test purposes switch from true to false -## -## proc htmlpage(contentLength, bodyLength: int): string = -## return &""" -## -## -## -## -##
-## File:
-## -##

-## Expected Body Length: {contentLength} bytes
-## Actual Body Length: {bodyLength} bytes -## -## -## """ -## -## proc cb(req: Request) {.async.} = -## var -## contentLength = 0 -## bodyLength = 0 -## if req.reqMethod == HttpPost: -## contentLength = req.headers["Content-length"].parseInt -## if stream: -## # Read 8*1024 bytes at a time -## # optional chunkSize parameter. The default is 8*1024 -## for length, data in req.bodyStream(8*1024): -## let content = await data -## if length == content.len: -## bodyLength += content.len -## else: -## # Handle exception -## await req.respond(Http400, -## "Bad Request. Data read has a different length than the expected.") -## return -## else: -## bodyLength += req.body.len -## await req.respond(Http200, htmlpage(contentLength, bodyLength)) -## -## let server = newAsyncHttpServer(maxBody = 10485760, stream = stream) # 10 MB -## waitFor server.serve(Port(8080), cb) import tables, asyncnet, asyncdispatch, parseutils, uri, strutils import httpcore export httpcore except parseHeader +const + maxLine = 8*1024 + # TODO: If it turns out that the decisions that asynchttpserver makes # explicitly, about whether to close the client sockets or upgrade them are # wrong, then add a return value which determines what to do for the callback. # Also, maybe move `client` out of `Request` object and into the args for # the proc. - -const - maxLine = 8*1024 - -when (NimMajor, NimMinor) >= (1, 1): - type - Request* = object - client*: AsyncSocket # TODO: Separate this into a Response object? - reqMethod*: HttpMethod - headers*: HttpHeaders - protocol*: tuple[orig: string, major, minor: int] - url*: Uri - hostname*: string ## The hostname of the client that made the request. - body*: string - contentLength*: int - - type - AsyncHttpServer* = ref object - socket: AsyncSocket - reuseAddr: bool - reusePort: bool - maxBody: int ## The maximum content-length that will be read for the body. - stream: bool ## By default (stream = false), the body of the request is read immediately -else: - type - Request* = object - client*: AsyncSocket # TODO: Separate this into a Response object? - reqMethod*: HttpMethod - headers*: HttpHeaders - protocol*: tuple[orig: string, major, minor: int] - url*: Uri - hostname*: string ## The hostname of the client that made the request. - body*: string - - type - AsyncHttpServer* = ref object - socket: AsyncSocket - reuseAddr: bool - reusePort: bool - maxBody: int ## The maximum content-length that will be read for the body. - -when (NimMajor, NimMinor) >= (1, 1): - proc newAsyncHttpServer*(reuseAddr = true, reusePort = false, - maxBody = 8388608, stream = false): AsyncHttpServer = - ## Creates a new ``AsyncHttpServer`` instance. - new result - result.reuseAddr = reuseAddr - result.reusePort = reusePort - result.maxBody = maxBody - result.stream = stream -else: - proc newAsyncHttpServer*(reuseAddr = true, reusePort = false, +type + Request* = object + client*: AsyncSocket # TODO: Separate this into a Response object? + reqMethod*: HttpMethod + headers*: HttpHeaders + protocol*: tuple[orig: string, major, minor: int] + url*: Uri + hostname*: string ## The hostname of the client that made the request. + body*: string + + AsyncHttpServer* = ref object + socket: AsyncSocket + reuseAddr: bool + reusePort: bool + maxBody: int ## The maximum content-length that will be read for the body. + +proc newAsyncHttpServer*(reuseAddr = true, reusePort = false, maxBody = 8388608): AsyncHttpServer = - ## Creates a new ``AsyncHttpServer`` instance. - new result - result.reuseAddr = reuseAddr - result.reusePort = reusePort - result.maxBody = maxBody + ## Creates a new ``AsyncHttpServer`` instance. + new result + result.reuseAddr = reuseAddr + result.reusePort = reusePort + result.maxBody = maxBody proc addHeaders(msg: var string, headers: HttpHeaders) = for k, v in headers: @@ -214,30 +128,13 @@ proc parseProtocol(protocol: string): tuple[orig: string, major, minor: int] = proc sendStatus(client: AsyncSocket, status: string): Future[void] = client.send("HTTP/1.1 " & status & "\c\L\c\L") -when (NimMajor, NimMinor) >= (1, 1): - iterator bodyStream*( - request: Request, - chunkSize: int = 8*1024): (int, Future[string]) = - ## The chunkSize parameter is optional and default value is 8*1024 bytes. - ## - ## This iterator return a tuple with the length of the data that was read - ## and a future. - var remainder = request.contentLength - while remainder > 0: - let readSize = min(remainder, chunkSize) - let data = request.client.recv(readSize) - if data.failed: - raise newException(ValueError, "Error reading POST data from client.") - yield (readSize, data) - remainder -= readSize - proc processRequest( server: AsyncHttpServer, req: FutureVar[Request], client: AsyncSocket, address: string, lineFut: FutureVar[string], - callback: proc (request: Request): Future[void] {.closure, gcsafe.} + callback: proc (request: Request): Future[void] {.closure, gcsafe.}, ): Future[bool] {.async.} = # Alias `request` to `req.mget()` so we don't have to write `mget` everywhere. @@ -248,12 +145,10 @@ proc processRequest( # Header: val # \n request.headers.clear() + request.body = "" request.hostname.shallowCopy(address) assert client != nil request.client = client - request.body = "" - when (NimMajor, NimMinor) >= (1, 1): - request.contentLength = 0 # We should skip at least one empty line before the request # https://tools.ietf.org/html/rfc7230#section-3.5 @@ -348,19 +243,10 @@ proc processRequest( if contentLength > server.maxBody: await request.respondError(Http413) return false - - when (NimMajor, NimMinor) >= (1, 1): - request.contentLength = contentLength - if not server.stream: - request.body = await client.recv(contentLength) - if request.body.len != contentLength: - await request.respond(Http400, "Bad Request. Content-Length does not match actual.") - return true - else: - request.body = await client.recv(contentLength) - if request.body.len != contentLength: - await request.respond(Http400, "Bad Request. Content-Length does not match actual.") - return true + request.body = await client.recv(contentLength) + if request.body.len != contentLength: + await request.respond(Http400, "Bad Request. Content-Length does not match actual.") + return true elif request.reqMethod == HttpPost: await request.respond(Http411, "Content-Length required.") return true From 4bd388ad58ba246dce74aaa41b617b7cad0fff36 Mon Sep 17 00:00:00 2001 From: alaviss Date: Fri, 6 Mar 2020 18:29:28 +0000 Subject: [PATCH 39/77] Revert "nimsuggest: don't add CRLF to replies (#13545)" (#13597) This reverts commit a974684b149faa19645f1e18b7b11aa67f169997. The CRLF was to separate between each response, and removing them causes the responses to not be splitted correctly. I didn't notice that it was in a loop when I made the #13545. --- nimsuggest/nimsuggest.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nimsuggest/nimsuggest.nim b/nimsuggest/nimsuggest.nim index 31cedb72ec1a8..a48e9443d8846 100644 --- a/nimsuggest/nimsuggest.nim +++ b/nimsuggest/nimsuggest.nim @@ -252,10 +252,10 @@ proc toSocket(stdoutSocket: Socket) {.gcsafe.} = let res = results.recv() case res.section of ideNone: break - of ideMsg: stdoutSocket.send(res.doc) - of ideKnown: stdoutSocket.send($(res.quality == 1)) - of ideProject: stdoutSocket.send(res.filePath) - else: stdoutSocket.send($res) + of ideMsg: stdoutSocket.send(res.doc & "\c\L") + of ideKnown: stdoutSocket.send($(res.quality == 1) & "\c\L") + of ideProject: stdoutSocket.send(res.filePath & "\c\L") + else: stdoutSocket.send($res & "\c\L") proc toEpc(client: Socket; uid: BiggestInt) {.gcsafe.} = var list = newSList() From 7ae081181831dcfe8ab83811f2b209c552c61fc7 Mon Sep 17 00:00:00 2001 From: Andrea Ferretti Date: Fri, 6 Mar 2020 19:38:56 +0100 Subject: [PATCH 40/77] Fix #13573 and #13574 (#13575) * Fix https://github.com/nim-lang/Nim/issues/13573 and https://github.com/nim-lang/Nim/issues/13574 * Restored asynchttpserver --- lib/pure/httpcore.nim | 12 ++++++++---- tests/stdlib/thttpcore.nim | 7 ++++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim index 1e9a85a57e819..e6e0d35c47fb7 100644 --- a/lib/pure/httpcore.nim +++ b/lib/pure/httpcore.nim @@ -105,11 +105,15 @@ proc newHttpHeaders*(): HttpHeaders = proc newHttpHeaders*(keyValuePairs: openArray[tuple[key: string, val: string]]): HttpHeaders = - var pairs: seq[tuple[key: string, val: seq[string]]] = @[] - for pair in keyValuePairs: - pairs.add((pair.key.toLowerAscii(), @[pair.val])) new result - result.table = newTable[string, seq[string]](pairs) + result.table = newTable[string, seq[string]]() + for pair in keyValuePairs: + let key = pair.key.toLowerAscii() + if key in result.table: + result.table[key].add(pair.val) + else: + result.table[key] = @[pair.val] + proc `$`*(headers: HttpHeaders): string = return $headers.table diff --git a/tests/stdlib/thttpcore.nim b/tests/stdlib/thttpcore.nim index 889c734c58913..f96fcc3b852d5 100644 --- a/tests/stdlib/thttpcore.nim +++ b/tests/stdlib/thttpcore.nim @@ -4,7 +4,7 @@ output: "[Suite] httpcore" import unittest -import httpcore +import httpcore, strutils suite "httpcore": @@ -29,3 +29,8 @@ suite "httpcore": assert "baR" in h["cookiE"] h.del("coOKie") assert h.len == 0 + + # Test that header constructor works with repeated values + let h1 = newHttpHeaders({"a": "1", "a": "2", "A": "3"}) + + assert seq[string](h1["a"]).join(",") == "1,2,3" From e056298ceb0fbddf6190f1b97f415b61323d446c Mon Sep 17 00:00:00 2001 From: Zed Date: Fri, 6 Mar 2020 19:55:45 +0100 Subject: [PATCH 41/77] Implement file streaming for httpclient's MultipartData (#12982) * Add `uploadFile` to POST files by streaming them * Use constant for \c\L * Formatting * Remove uploadFile * Implement MultipartData file streaming * Remove unnecessary var annotations * Call string on TaintedStrings Fixes #12789 * Move cl constant to httpcore * Fix `request` inconsistencies * Update documentaion * Clean up * Skip multipart formatting when there's 0 entries * Remove extraneous `cl` from multipart formatting * Update MultipartData `$` to match old behaviour * Update comment * Address comments --- lib/pure/httpclient.nim | 358 +++++++++++++++++++++++----------------- lib/pure/httpcore.nim | 1 + 2 files changed, 212 insertions(+), 147 deletions(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 7c062c2e10a2c..b7bdc8d17f11e 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -52,6 +52,19 @@ ## ## echo client.postContent("http://validator.w3.org/check", multipart=data) ## +## To stream files from disk when performing the request, use ``addFiles``. +## +## **Note:** This will allocate a new ``Mimetypes`` database every time you call +## it, you can pass your own via the ``mimeDb`` parameter to avoid this. +## +## .. code-block:: Nim +## let mimes = newMimetypes() +## var client = newHttpClient() +## var data = newMultipartData() +## data.addFiles({"uploaded_file": "test.html"}, mimeDb = mimes) +## +## echo client.postContent("http://validator.w3.org/check", multipart=data) +## ## You can also make post requests with custom headers. ## This example sets ``Content-Type`` to ``application/json`` ## and uses a json object for the body @@ -177,7 +190,7 @@ include "system/inclrtl" -import net, strutils, uri, parseutils, base64, os, mimetypes, +import net, strutils, uri, parseutils, base64, os, mimetypes, streams, math, random, httpcore, times, tables, streams, std/monotimes import asyncnet, asyncdispatch, asyncfile import nativesockets @@ -252,9 +265,18 @@ type url*: Uri auth*: string + MultipartEntry = object + name, content: string + case isFile: bool + of true: + filename, contentType: string + fileSize: int64 + isStream: bool + else: discard + MultipartEntries* = openArray[tuple[name, content: string]] MultipartData* = ref object - content: seq[string] + content: seq[MultipartEntry] ProtocolError* = object of IOError ## exception that is raised when server ## does not conform to the implemented @@ -278,7 +300,6 @@ proc fileError(msg: string) = e.msg = msg raise e - when not defined(ssl): type SslContext = ref object var defaultSslContext {.threadvar.}: SslContext @@ -297,24 +318,28 @@ proc newProxy*(url: string, auth = ""): Proxy = proc newMultipartData*: MultipartData = ## Constructs a new ``MultipartData`` object. - MultipartData(content: @[]) - + MultipartData() proc `$`*(data: MultipartData): string {.since: (1, 1).} = ## convert MultipartData to string so it's human readable when echo ## see https://github.com/nim-lang/Nim/issues/11863 - const prefixLen = "Content-Disposition: form-data; ".len - for pos, item in data.content: - result &= "------------------------------ " - result.addInt pos - result &= " ------------------------------\n" - result &= item[prefixLen .. item.high] - -proc add*(p: var MultipartData, name, content: string, filename: string = "", - contentType: string = "") = - ## Add a value to the multipart data. Raises a `ValueError` exception if - ## `name`, `filename` or `contentType` contain newline characters. - + const sep = "-".repeat(30) + for pos, entry in data.content: + result.add(sep & center($pos, 3) & sep) + result.add("\nname=\"" & entry.name & "\"") + if entry.isFile: + result.add("; filename=\"" & entry.filename & "\"\n") + result.add("Content-Type: " & entry.contentType) + result.add("\n\n" & entry.content & "\n") + +proc add*(p: MultipartData, name, content: string, filename: string = "", + contentType: string = "", useStream = true) = + ## Add a value to the multipart data. + ## + ## When ``useStream`` is ``false``, the file will be read into memory. + ## + ## Raises a ``ValueError`` exception if + ## ``name``, ``filename`` or ``contentType`` contain newline characters. if {'\c', '\L'} in name: raise newException(ValueError, "name contains a newline character") if {'\c', '\L'} in filename: @@ -322,19 +347,22 @@ proc add*(p: var MultipartData, name, content: string, filename: string = "", if {'\c', '\L'} in contentType: raise newException(ValueError, "contentType contains a newline character") - var str = "Content-Disposition: form-data; name=\"" & name & "\"" - if filename.len > 0: - str.add("; filename=\"" & filename & "\"") - str.add("\c\L") - if contentType.len > 0: - str.add("Content-Type: " & contentType & "\c\L") - str.add("\c\L" & content & "\c\L") + var entry = MultipartEntry( + name: name, + content: content, + isFile: filename.len > 0 + ) + + if entry.isFile: + entry.isStream = useStream + entry.filename = filename + entry.contentType = contentType - p.content.add(str) + p.content.add(entry) -proc add*(p: var MultipartData, xs: MultipartEntries): MultipartData +proc add*(p: MultipartData, xs: MultipartEntries): MultipartData {.discardable.} = - ## Add a list of multipart entries to the multipart data `p`. All values are + ## Add a list of multipart entries to the multipart data ``p``. All values are ## added without a filename and without a content type. ## ## .. code-block:: Nim @@ -344,70 +372,77 @@ proc add*(p: var MultipartData, xs: MultipartEntries): MultipartData result = p proc newMultipartData*(xs: MultipartEntries): MultipartData = - ## Create a new multipart data object and fill it with the entries `xs` + ## Create a new multipart data object and fill it with the entries ``xs`` ## directly. ## ## .. code-block:: Nim ## var data = newMultipartData({"action": "login", "format": "json"}) - result = MultipartData(content: @[]) - result.add(xs) - -proc addFiles*(p: var MultipartData, xs: openArray[tuple[name, file: string]]): - MultipartData {.discardable.} = - ## Add files to a multipart data object. The file will be opened from your - ## disk, read and sent with the automatically determined MIME type. Raises an - ## `IOError` if the file cannot be opened or reading fails. To manually - ## specify file content, filename and MIME type, use `[]=` instead. + result = MultipartData() + for entry in xs: + result.add(entry.name, entry.content) + +proc addFiles*(p: MultipartData, xs: openArray[tuple[name, file: string]], + mimeDb = newMimetypes(), useStream = true): + MultipartData {.discardable.} = + ## Add files to a multipart data object. The files will be streamed from disk + ## when the request is being made. When ``stream`` is ``false``, the files are + ## instead read into memory, but beware this is very memory ineffecient even + ## for small files. The MIME types will automatically be determined. + ## Raises an ``IOError`` if the file cannot be opened or reading fails. To + ## manually specify file content, filename and MIME type, use ``[]=`` instead. ## ## .. code-block:: Nim ## data.addFiles({"uploaded_file": "public/test.html"}) - var m = newMimetypes() for name, file in xs.items: var contentType: string let (_, fName, ext) = splitFile(file) if ext.len > 0: - contentType = m.getMimetype(ext[1..ext.high], "") - p.add(name, readFile(file).string, fName & ext, contentType) + contentType = mimeDb.getMimetype(ext[1..ext.high], "") + let content = if useStream: file else: readFile(file).string + p.add(name, content, fName & ext, contentType, useStream = useStream) result = p -proc `[]=`*(p: var MultipartData, name, content: string) = - ## Add a multipart entry to the multipart data `p`. The value is added +proc `[]=`*(p: MultipartData, name, content: string) = + ## Add a multipart entry to the multipart data ``p``. The value is added ## without a filename and without a content type. ## ## .. code-block:: Nim ## data["username"] = "NimUser" p.add(name, content) -proc `[]=`*(p: var MultipartData, name: string, +proc `[]=`*(p: MultipartData, name: string, file: tuple[name, contentType, content: string]) = - ## Add a file to the multipart data `p`, specifying filename, contentType and - ## content manually. + ## Add a file to the multipart data ``p``, specifying filename, contentType + ## and content manually. ## ## .. code-block:: Nim ## data["uploaded_file"] = ("test.html", "text/html", ## "

test

") - p.add(name, file.content, file.name, file.contentType) + p.add(name, file.content, file.name, file.contentType, useStream = false) -proc format(p: MultipartData): tuple[contentType, body: string] = - if p == nil or p.content.len == 0: - return ("", "") - - # Create boundary that is not in the data to be formatted - var bound: string +proc getBoundary(p: MultipartData): string = + if p == nil or p.content.len == 0: return while true: - bound = $random(int.high) - var found = false - for s in p.content: - if bound in s: - found = true - if not found: - break - - result.contentType = "multipart/form-data; boundary=" & bound - result.body = "" - for s in p.content: - result.body.add("--" & bound & "\c\L" & s) - result.body.add("--" & bound & "--\c\L") + result = $random(int.high) + for i, entry in p.content: + if result in entry.content: break + elif i == p.content.high: return + +proc sendFile(socket: Socket | AsyncSocket, + entry: MultipartEntry) {.multisync.} = + const chunkSize = 2^18 + let file = + when socket is AsyncSocket: openAsync(entry.content) + else: newFileStream(entry.content, fmRead) + + var buffer: string + while true: + buffer = + when socket is AsyncSocket: (await read(file, chunkSize)).string + else: readStr(file, chunkSize).string + if buffer.len == 0: break + await socket.send(buffer) + file.close() proc redirection(status: string): bool = const redirectionNRs = ["301", "302", "303", "307"] @@ -427,8 +462,8 @@ proc getNewLocation(lastURL: string, headers: HttpHeaders): string = parsed.anchor = r.anchor result = $parsed -proc generateHeaders(requestUrl: Uri, httpMethod: string, - headers: HttpHeaders, body: string, proxy: Proxy): string = +proc generateHeaders(requestUrl: Uri, httpMethod: string, headers: HttpHeaders, + proxy: Proxy): string = # GET let upperMethod = httpMethod.toUpperAscii() result = upperMethod @@ -447,34 +482,28 @@ proc generateHeaders(requestUrl: Uri, httpMethod: string, result.add($modifiedUrl) # HTTP/1.1\c\l - result.add(" HTTP/1.1\c\L") + result.add(" HTTP/1.1" & httpNewLine) # Host header. if not headers.hasKey("Host"): if requestUrl.port == "": - add(result, "Host: " & requestUrl.hostname & "\c\L") + add(result, "Host: " & requestUrl.hostname & httpNewLine) else: - add(result, "Host: " & requestUrl.hostname & ":" & requestUrl.port & "\c\L") + add(result, "Host: " & requestUrl.hostname & ":" & requestUrl.port & httpNewLine) # Connection header. if not headers.hasKey("Connection"): - add(result, "Connection: Keep-Alive\c\L") - - # Content length header. - const requiresBody = ["POST", "PUT", "PATCH"] - let needsContentLength = body.len > 0 or upperMethod in requiresBody - if needsContentLength and not headers.hasKey("Content-Length"): - add(result, "Content-Length: " & $body.len & "\c\L") + add(result, "Connection: Keep-Alive" & httpNewLine) # Proxy auth header. if not proxy.isNil and proxy.auth != "": let auth = base64.encode(proxy.auth) - add(result, "Proxy-Authorization: basic " & auth & "\c\L") + add(result, "Proxy-Authorization: basic " & auth & httpNewLine) for key, val in headers: - add(result, key & ": " & val & "\c\L") + add(result, key & ": " & val & httpNewLine) - add(result, "\c\L") + add(result, httpNewLine) type ProgressChangedProc*[ReturnType] = @@ -511,9 +540,9 @@ type type HttpClient* = HttpClientBase[Socket] -proc newHttpClient*(userAgent = defUserAgent, - maxRedirects = 5, sslContext = getDefaultSSL(), proxy: Proxy = nil, - timeout = -1, headers = newHttpHeaders()): HttpClient = +proc newHttpClient*(userAgent = defUserAgent, maxRedirects = 5, + sslContext = getDefaultSSL(), proxy: Proxy = nil, + timeout = -1, headers = newHttpHeaders()): HttpClient = ## Creates a new HttpClient instance. ## ## ``userAgent`` specifies the user agent that will be used when making @@ -546,9 +575,9 @@ proc newHttpClient*(userAgent = defUserAgent, type AsyncHttpClient* = HttpClientBase[AsyncSocket] -proc newAsyncHttpClient*(userAgent = defUserAgent, - maxRedirects = 5, sslContext = getDefaultSSL(), - proxy: Proxy = nil, headers = newHttpHeaders()): AsyncHttpClient = +proc newAsyncHttpClient*(userAgent = defUserAgent, maxRedirects = 5, + sslContext = getDefaultSSL(), proxy: Proxy = nil, + headers = newHttpHeaders()): AsyncHttpClient = ## Creates a new AsyncHttpClient instance. ## ## ``userAgent`` specifies the user agent that will be used when making @@ -671,8 +700,7 @@ proc parseChunks(client: HttpClient | AsyncHttpClient): Future[void] # Trailer headers will only be sent if the request specifies that we want # them: http://tools.ietf.org/html/rfc2616#section-3.6.1 -proc parseBody(client: HttpClient | AsyncHttpClient, - headers: HttpHeaders, +proc parseBody(client: HttpClient | AsyncHttpClient, headers: HttpHeaders, httpVersion: string): Future[void] {.multisync.} = # Reset progress from previous requests. client.contentTotal = 0 @@ -745,7 +773,7 @@ proc parseResponse(client: HttpClient | AsyncHttpClient, # We've been disconnected. client.close() break - if line == "\c\L": + if line == httpNewLine: fullyRead = true break if not parsedStatus: @@ -846,7 +874,7 @@ proc newConnection(client: HttpClient | AsyncHttpClient, connectUrl.port = if url.port != "": url.port else: "443" let proxyHeaderString = generateHeaders(connectUrl, $HttpConnect, - newHttpHeaders(), "", client.proxy) + newHttpHeaders(), client.proxy) await client.socket.send(proxyHeaderString) let proxyResp = await parseResponse(client, false) @@ -864,6 +892,45 @@ proc newConnection(client: HttpClient | AsyncHttpClient, client.currentURL = url client.connected = true +proc readFileSizes(client: HttpClient | AsyncHttpClient, + multipart: MultipartData) {.multisync.} = + for entry in multipart.content.mitems(): + if not entry.isFile: continue + if not entry.isStream: + entry.fileSize = entry.content.len + continue + + # TODO: look into making getFileSize work with async + let fileSize = getFileSize(entry.content) + entry.fileSize = fileSize + +proc format(entry: MultipartEntry, boundary: string): string = + result = "--" & boundary & httpNewLine + result.add("Content-Disposition: form-data; name=\"" & entry.name & "\"") + if entry.isFile: + result.add("; filename=\"" & entry.filename & "\"" & httpNewLine) + result.add("Content-Type: " & entry.contentType & httpNewLine) + else: + result.add(httpNewLine & httpNewLine & entry.content) + +proc format(client: HttpClient | AsyncHttpClient, + multipart: MultipartData): Future[seq[string]] {.multisync.} = + let bound = getBoundary(multipart) + client.headers["Content-Type"] = "multipart/form-data; boundary=" & bound + + await client.readFileSizes(multipart) + + var length: int64 + for entry in multipart.content: + result.add(format(entry, bound) & httpNewLine) + if entry.isFile: + length += entry.fileSize + httpNewLine.len + + result.add "--" & bound & "--" + + for s in result: length += s.len + client.headers["Content-Length"] = $length + proc override(fallback, override: HttpHeaders): HttpHeaders = # Right-biased map union for `HttpHeaders` if override.isNil: @@ -875,9 +942,9 @@ proc override(fallback, override: HttpHeaders): HttpHeaders = for k, vs in override.table: result[k] = vs -proc requestAux(client: HttpClient | AsyncHttpClient, url: string, - httpMethod: string, body = "", - headers: HttpHeaders = nil): Future[Response | AsyncResponse] +proc requestAux(client: HttpClient | AsyncHttpClient, url, httpMethod: string, + body = "", headers: HttpHeaders = nil, + multipart: MultipartData = nil): Future[Response | AsyncResponse] {.multisync.} = # Helper that actually makes the request. Does not handle redirects. let requestUrl = parseUri(url) @@ -885,6 +952,12 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url: string, if requestUrl.scheme == "": raise newException(ValueError, "No uri scheme supplied.") + var data: seq[string] + if multipart != nil and multipart.content.len > 0: + data = await client.format(multipart) + else: + client.headers["Content-Length"] = $body.len + when client is AsyncHttpClient: if not client.parseBodyFut.isNil: # let the current operation finish before making another request @@ -893,16 +966,30 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url: string, await newConnection(client, requestUrl) - let effectiveHeaders = client.headers.override(headers) - - if not effectiveHeaders.hasKey("user-agent") and client.userAgent != "": - effectiveHeaders["User-Agent"] = client.userAgent - - var headersString = generateHeaders(requestUrl, httpMethod, - effectiveHeaders, body, client.proxy) - - await client.socket.send(headersString) - if body != "": + let newHeaders = client.headers.override(headers) + if not newHeaders.hasKey("user-agent") and client.userAgent.len > 0: + newHeaders["User-Agent"] = client.userAgent + + let headerString = generateHeaders(requestUrl, httpMethod, newHeaders, + client.proxy) + await client.socket.send(headerString) + + if data.len > 0: + var buffer: string + for i, entry in multipart.content: + buffer.add data[i] + if not entry.isFile: continue + if buffer.len > 0: + await client.socket.send(buffer) + buffer.setLen(0) + if entry.isStream: + await client.socket.sendFile(entry) + else: + await client.socket.send(entry.content) + buffer.add httpNewLine + # send the rest and the last boundary + await client.socket.send(buffer & data[^1]) + elif body.len > 0: await client.socket.send(body) let getBody = httpMethod.toLowerAscii() notin ["head", "connect"] and @@ -910,8 +997,8 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url: string, result = await parseResponse(client, getBody) proc request*(client: HttpClient | AsyncHttpClient, url: string, - httpMethod: string, body = "", - headers: HttpHeaders = nil): Future[Response | AsyncResponse] + httpMethod: string, body = "", headers: HttpHeaders = nil, + multipart: MultipartData = nil): Future[Response | AsyncResponse] {.multisync.} = ## Connects to the hostname specified by the URL and performs a request ## using the custom method string specified by ``httpMethod``. @@ -922,7 +1009,7 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, ## ## This procedure will follow redirects up to a maximum number of redirects ## specified in ``client.maxRedirects``. - result = await client.requestAux(url, httpMethod, body, headers) + result = await client.requestAux(url, httpMethod, body, headers, multipart) var lastURL = url for i in 1..client.maxRedirects: @@ -930,13 +1017,12 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, let redirectTo = getNewLocation(lastURL, result.headers) # Guarantee method for HTTP 307: see https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/307 var meth = if result.status == "307": httpMethod else: "GET" - result = await client.requestAux(redirectTo, meth, body, headers) + result = await client.requestAux(redirectTo, meth, body, headers, multipart) lastURL = redirectTo - proc request*(client: HttpClient | AsyncHttpClient, url: string, - httpMethod = HttpGet, body = "", - headers: HttpHeaders = nil): Future[Response | AsyncResponse] + httpMethod = HttpGet, body = "", headers: HttpHeaders = nil, + multipart: MultipartData = nil): Future[Response | AsyncResponse] {.multisync.} = ## Connects to the hostname specified by the URL and performs a request ## using the method specified. @@ -947,7 +1033,7 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, ## ## When a request is made to a different hostname, the current connection will ## be closed. - result = await request(client, url, $httpMethod, body, headers) + result = await request(client, url, $httpMethod, body, headers, multipart) proc responseContent(resp: Response | AsyncResponse): Future[string] {.multisync.} = ## Returns the content of a response as a string. @@ -980,42 +1066,25 @@ proc getContent*(client: HttpClient | AsyncHttpClient, return await responseContent(resp) proc delete*(client: HttpClient | AsyncHttpClient, - url: string): Future[Response | AsyncResponse] {.multisync.} = + url: string): Future[Response | AsyncResponse] {.multisync.} = ## Connects to the hostname specified by the URL and performs a DELETE request. ## This procedure uses httpClient values such as ``client.maxRedirects``. result = await client.request(url, HttpDelete) proc deleteContent*(client: HttpClient | AsyncHttpClient, - url: string): Future[string] {.multisync.} = + url: string): Future[string] {.multisync.} = ## Connects to the hostname specified by the URL and returns the content of a DELETE request. let resp = await delete(client, url) return await responseContent(resp) -proc makeRequestContent(body = "", multipart: MultipartData = nil): (string, HttpHeaders) = - let (mpContentType, mpBody) = format(multipart) - # TODO: Support FutureStream for `body` parameter. - template withNewLine(x): untyped = - if x.len > 0 and not x.endsWith("\c\L"): - x & "\c\L" - else: - x - var xb = mpBody.withNewLine() & body - var headers = newHttpHeaders() - if multipart != nil: - headers["Content-Type"] = mpContentType - headers["Content-Length"] = $len(xb) - return (xb, headers) - proc post*(client: HttpClient | AsyncHttpClient, url: string, body = "", multipart: MultipartData = nil): Future[Response | AsyncResponse] {.multisync.} = ## Connects to the hostname specified by the URL and performs a POST request. ## This procedure uses httpClient values such as ``client.maxRedirects``. - var (xb, headers) = makeRequestContent(body, multipart) - result = await client.request(url, $HttpPost, xb, headers) + result = await client.request(url, $HttpPost, body, multipart=multipart) -proc postContent*(client: HttpClient | AsyncHttpClient, url: string, - body = "", +proc postContent*(client: HttpClient | AsyncHttpClient, url: string, body = "", multipart: MultipartData = nil): Future[string] {.multisync.} = ## Connects to the hostname specified by the URL and returns the content of a POST request. @@ -1023,32 +1092,27 @@ proc postContent*(client: HttpClient | AsyncHttpClient, url: string, return await responseContent(resp) proc put*(client: HttpClient | AsyncHttpClient, url: string, body = "", - multipart: MultipartData = nil): Future[Response | AsyncResponse] - {.multisync.} = + multipart: MultipartData = nil): Future[Response | AsyncResponse] + {.multisync.} = ## Connects to the hostname specified by the URL and performs a PUT request. ## This procedure uses httpClient values such as ``client.maxRedirects``. - var (xb, headers) = makeRequestContent(body, multipart) - result = await client.request(url, $HttpPut, xb, headers) + result = await client.request(url, $HttpPut, body, multipart=multipart) -proc putContent*(client: HttpClient | AsyncHttpClient, url: string, - body = "", - multipart: MultipartData = nil): Future[string] - {.multisync.} = +proc putContent*(client: HttpClient | AsyncHttpClient, url: string, body = "", + multipart: MultipartData = nil): Future[string] {.multisync.} = ## Connects to the hostname specified by the URL andreturns the content of a PUT request. let resp = await put(client, url, body, multipart) return await responseContent(resp) proc patch*(client: HttpClient | AsyncHttpClient, url: string, body = "", - multipart: MultipartData = nil): Future[Response | AsyncResponse] - {.multisync.} = + multipart: MultipartData = nil): Future[Response | AsyncResponse] + {.multisync.} = ## Connects to the hostname specified by the URL and performs a PATCH request. ## This procedure uses httpClient values such as ``client.maxRedirects``. - var (xb, headers) = makeRequestContent(body, multipart) - result = await client.request(url, $HttpPatch, xb, headers) + result = await client.request(url, $HttpPatch, body, multipart=multipart) -proc patchContent*(client: HttpClient | AsyncHttpClient, url: string, - body = "", - multipart: MultipartData = nil): Future[string] +proc patchContent*(client: HttpClient | AsyncHttpClient, url: string, body = "", + multipart: MultipartData = nil): Future[string] {.multisync.} = ## Connects to the hostname specified by the URL and returns the content of a PATCH request. let resp = await patch(client, url, body, multipart) diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim index e6e0d35c47fb7..8c679c017d79f 100644 --- a/lib/pure/httpcore.nim +++ b/lib/pure/httpcore.nim @@ -97,6 +97,7 @@ const Http504* = HttpCode(504) Http505* = HttpCode(505) +const httpNewLine* = "\c\L" const headerLimit* = 10_000 proc newHttpHeaders*(): HttpHeaders = From 6a5b3811e7c8f0d20e23cf8b57ac28024f2620e3 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Sat, 7 Mar 2020 11:10:22 -0300 Subject: [PATCH 42/77] Add isValidFilename (#13561) * Add os.isValidFilename * Add os.isValidFilename * Peer Review Feedbacks https://github.com/nim-lang/Nim/pull/13561#discussion_r388013139 * Peer Review Feedbacks https://github.com/nim-lang/Nim/pull/13561#issuecomment-595259568 * Add since to const * Update the documentation comment * Update the changelog * Update lib/pure/os.nim Co-Authored-By: Dominik Picheta * Update lib/pure/os.nim Co-Authored-By: Dominik Picheta * Peer Review Feedbacks, Add more Tests Co-authored-by: Dominik Picheta --- changelog.md | 2 +- lib/pure/os.nim | 68 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 52406a97218ac..38f9589db4011 100644 --- a/changelog.md +++ b/changelog.md @@ -69,7 +69,7 @@ - Added `minIndex`, `maxIndex` and `unzip` to the `sequtils` module. - Added `os.isRelativeTo` to tell whether a path is relative to another - Added `resetOutputFormatters` to `unittest` - +- Added `os.isValidFilename` that returns `true` if `filename` argument is valid for crossplatform use. - Added a `with` macro for easy function chaining that's available everywhere, there is no need to concern your APIs with returning the first argument diff --git a/lib/pure/os.nim b/lib/pure/os.nim index ba092da05167a..1ee71c1fb04e2 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -48,6 +48,20 @@ import const weirdTarget = defined(nimscript) or defined(js) +since (1, 1): + const + invalidFilenameChars* = {'/', '\\', ':', '*', '?', '"', '<', '>', '|', '^', '\0'} ## \ + ## Characters that may produce invalid filenames across Linux, Windows, Mac, etc. + ## You can check if your filename contains these char and strip them for safety. + ## Mac bans ``':'``, Linux bans ``'/'``, Windows bans all others. + invalidFilenames* = [ + "CON", "PRN", "AUX", "NUL", + "COM0", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", + "LPT0", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9"] ## \ + ## Filenames that may be invalid across Linux, Windows, Mac, etc. + ## You can check if your filename match these and rename it for safety + ## (Currently all invalid filenames are from Windows only). + when weirdTarget: discard elif defined(windows): @@ -3194,6 +3208,32 @@ proc setLastModificationTime*(file: string, t: times.Time) {.noNimScript.} = discard h.closeHandle if res == 0'i32: raiseOSError(osLastError(), file) +func isValidFilename*(filename: string, maxLen = 259.Positive): bool {.since: (1, 1).} = + ## Returns true if ``filename`` is valid for crossplatform use. + ## + ## This is useful if you want to copy or save files across Windows, Linux, Mac, etc. + ## You can pass full paths as argument too, but func only checks filenames. + ## It uses ``invalidFilenameChars``, ``invalidFilenames`` and ``maxLen`` to verify the specified ``filename``. + ## + ## .. code-block:: nim + ## assert not isValidFilename(" foo") ## Leading white space + ## assert not isValidFilename("foo ") ## Trailing white space + ## assert not isValidFilename("foo.") ## Ends with Dot + ## assert not isValidFilename("con.txt") ## "CON" is invalid (Windows) + ## assert not isValidFilename("OwO:UwU") ## ":" is invalid (Mac) + ## assert not isValidFilename("aux.bat") ## "AUX" is invalid (Windows) + ## + # https://docs.microsoft.com/en-us/dotnet/api/system.io.pathtoolongexception + # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file + # https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247%28v=vs.85%29.aspx + result = true + let f = filename.splitFile() + if unlikely(f.name.len + f.ext.len > maxLen or + f.name[0] == ' ' or f.name[^1] == ' ' or f.name[^1] == '.' or + find(f.name, invalidFilenameChars) != -1): return false + for invalid in invalidFilenames: + if cmpIgnoreCase(f.name, invalid) == 0: return false + when isMainModule: assert quoteShellWindows("aaa") == "aaa" @@ -3228,3 +3268,31 @@ when isMainModule: doAssert r"D:\".normalizePathEnd == r"D:" doAssert r"E:/".normalizePathEnd(trailingSep = true) == r"E:\" doAssert "/".normalizePathEnd == r"\" + + + block isValidFilenameTest: + # Negative Tests. + doAssert not isValidFilename("abcd", maxLen = 2) + doAssert not isValidFilename("0123456789", maxLen = 8) + doAssert not isValidFilename("con") + doAssert not isValidFilename("aux") + doAssert not isValidFilename("prn") + doAssert not isValidFilename("OwO|UwU") + doAssert not isValidFilename(" foo") + doAssert not isValidFilename("foo ") + doAssert not isValidFilename("foo.") + doAssert not isValidFilename("con.txt") + doAssert not isValidFilename("aux.bat") + doAssert not isValidFilename("prn.exe") + doAssert not isValidFilename("nim>.nim") + doAssert not isValidFilename(" foo.log") + # Positive Tests. + doAssert isValidFilename("abcd", maxLen = 42.Positive) + doAssert isValidFilename("c0n") + doAssert isValidFilename("foo.aux") + doAssert isValidFilename("bar.prn") + doAssert isValidFilename("OwO_UwU") + doAssert isValidFilename("cron") + doAssert isValidFilename("ux.bat") + doAssert isValidFilename("nim.nim") + doAssert isValidFilename("foo.log") From eae31a7f8d5f26bbfc27c3f0746a583154d451c9 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Sat, 7 Mar 2020 23:53:43 +0100 Subject: [PATCH 43/77] Only print the link command when listCmd is active; fix docs (#13603) --- compiler/extccomp.nim | 2 +- compiler/nim.cfg | 1 - doc/advopt.txt | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index b9d0f0ff6afb3..b115721df1b7d 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -489,7 +489,7 @@ proc addExternalFileToLink*(conf: ConfigRef; filename: AbsoluteFile) = conf.externalToLink.insert(filename.string, 0) proc execWithEcho(conf: ConfigRef; cmd: string, msg = hintExecuting): int = - rawMessage(conf, msg, cmd) + rawMessage(conf, msg, if msg == hintLinking and not(optListCmd in conf.globalOptions or conf.verbosity > 1): "" else: cmd) result = execCmd(cmd) proc execExternalProgram*(conf: ConfigRef; cmd: string, msg = hintExecuting) = diff --git a/compiler/nim.cfg b/compiler/nim.cfg index f913e76a3b0e3..a77fa84d36ed0 100644 --- a/compiler/nim.cfg +++ b/compiler/nim.cfg @@ -1,7 +1,6 @@ # Special configuration file for the Nim project hint[XDeclaredButNotUsed]:off -hint[Link]:off define:booting define:nimcore diff --git a/doc/advopt.txt b/doc/advopt.txt index 5c0ed86ee034b..517edab5b1984 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -125,8 +125,8 @@ Advanced options: dynlib: "liblua.so.3" --dynlibOverrideAll disables the effects of the dynlib pragma - --listCmd list the compilation commands; can be combined with: - --hint:exec:on and --hint:link:on + --listCmd list the compilation commands; can be combined with + `--hint:exec:on` and `--hint:link:on` --asm produce assembler code --parallelBuild:0|1|... perform a parallel build value = number of processors (0 for auto-detect) From a693ce776557371a159c074d1d3f55683114a03a Mon Sep 17 00:00:00 2001 From: Andy Davidoff Date: Sun, 8 Mar 2020 05:55:19 -0400 Subject: [PATCH 44/77] surgical fix for #13319 (#13604) --- compiler/rodimpl.nim | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/compiler/rodimpl.nim b/compiler/rodimpl.nim index 7a0f6fcef8d99..8ee7ae56418c5 100644 --- a/compiler/rodimpl.nim +++ b/compiler/rodimpl.nim @@ -667,7 +667,7 @@ proc loadType(g; id: int; info: TLineInfo): PType = rawAddSon(result, nil) else: let d = decodeVInt(b.s, b.pos) - rawAddSon(result, loadType(g, d, info)) + result.sons.add loadType(g, d, info) proc decodeLib(g; b; info: TLineInfo): PLib = result = nil @@ -719,10 +719,8 @@ proc loadSymFromBlob(g; b; info: TLineInfo): PSym = else: internalError(g.config, info, "decodeSym: no ident") #echo "decoding: {", ident.s - new(result) - result.id = id - result.kind = k - result.name = ident # read the rest of the symbol description: + result = PSym(id: id, kind: k, name: ident) + # read the rest of the symbol description: g.incr.r.syms.add(result.id, result) if b.s[b.pos] == '^': inc(b.pos) From dc94c81cb0807dbb5cf60c782baa916bbd8458ed Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 9 Mar 2020 13:13:54 +0100 Subject: [PATCH 45/77] fixes #13605 (#13611) --- changelog.md | 3 +++ lib/pure/parseutils.nim | 9 +++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/changelog.md b/changelog.md index 38f9589db4011..49567e9298d82 100644 --- a/changelog.md +++ b/changelog.md @@ -115,6 +115,9 @@ echo f serve no purpose whatsoever. - `httpclient.newHttpClient` and `httpclient.newAsyncHttpClient` added `headers` argument to set initial HTTP Headers, instead of a hardcoded empty `newHttpHeader()`. +- `parseutils.parseUntil` has now a different behaviour if the `until` parameter is + empty. This was required for intuitive behaviour of the strscans module + (see bug #13605). ## Language additions diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index 2c21c495278c6..6116751ff2ddf 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -354,12 +354,13 @@ proc parseUntil*(s: string, token: var string, until: string, doAssert myToken == "Hello " doAssert parseUntil("Hello World", myToken, "Wor", 2) == 4 doAssert myToken == "llo " - if until.len == 0: - token.setLen(0) - return 0 + when (NimMajor, NimMinor) <= (1, 0): + if until.len == 0: + token.setLen(0) + return 0 var i = start while i < s.len: - if s[i] == until[0]: + if until.len > 0 and s[i] == until[0]: var u = 1 while i+u < s.len and u < until.len and s[i+u] == until[u]: inc u From 4aecc6b3465411d99dbd94e89c1bb3eb371d6fa6 Mon Sep 17 00:00:00 2001 From: Miran Date: Mon, 9 Mar 2020 14:08:50 +0100 Subject: [PATCH 46/77] fix #12508, unaligned access on sparc64 (#13594) --- lib/pure/hashes.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index 52a724d7bbd69..d799fb0620c10 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -162,7 +162,7 @@ proc murmurHash(x: openArray[byte]): Hash = # body while i < n * stepSize: var k1: uint32 - when defined(js): + when defined(js) or defined(sparc) or defined(sparc64): var j = stepSize while j > 0: dec j From 63af8ae53ca8e1ffd61a2c2d55d09c5fe30310e1 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 9 Mar 2020 15:32:38 +0100 Subject: [PATCH 47/77] fixes #13596 (#13612) --- compiler/transf.nim | 13 +++++ tests/arc/tamemfiles.nim | 110 +++++++++++++++++++++++++++++++++++++++ tests/arc/testfile.txt | 2 + 3 files changed, 125 insertions(+) create mode 100644 tests/arc/tamemfiles.nim create mode 100644 tests/arc/testfile.txt diff --git a/compiler/transf.nim b/compiler/transf.nim index 68ef464c4f23d..cd5eaa10059e4 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -128,6 +128,19 @@ proc transformSymAux(c: PTransf, n: PNode): PNode = b = s.getBody if b.kind != nkSym: internalError(c.graph.config, n.info, "wrong AST for borrowed symbol") b = newSymNode(b.sym, n.info) + elif c.inlining > 0: + # see bug #13596: we use ref-based equality in the DFA for destruction + # injections so we need to ensure unique nodes after iterator inlining + # which can lead to duplicated for loop bodies! Consider: + #[ + while remaining > 0: + if ending == nil: + yield ms + break + ... + yield ms + ]# + b = newSymNode(n.sym, n.info) else: b = n while tc != nil: diff --git a/tests/arc/tamemfiles.nim b/tests/arc/tamemfiles.nim new file mode 100644 index 0000000000000..97deb489f28f8 --- /dev/null +++ b/tests/arc/tamemfiles.nim @@ -0,0 +1,110 @@ +discard """ + output: ''' +loop 1a +loop 1b; cols: @[1, x] +loop 1c +loop 1d +loop 1a +loop 1b; cols: @[2, y] +loop 1c +loop 1d +''' + cmd: "nim c --gc:arc $file" +""" + +# bug #13596 + +import tables, memfiles, strutils, os + +type Splitr* = tuple[ repeat: bool, chrDlm: char, setDlm: set[char], n: int ] + +type csize = uint +proc cmemchr*(s: pointer, c: char, n: csize): pointer {. + importc: "memchr", header: "" .} +proc `-!`*(p, q: pointer): int {.inline.} = + (cast[int](p) -% cast[int](q)).int +proc `+!`*(p: pointer, i: int): pointer {.inline.} = + cast[pointer](cast[int](p) +% i) +proc `+!`*(p: pointer, i: uint64): pointer {.inline.} = + cast[pointer](cast[uint64](p) + i) + +proc charEq(x, c: char): bool {.inline.} = x == c + +proc initSplitr*(delim: string): Splitr = + if delim == "white": #User can use any other permutation if needed + result.repeat = true + result.chrDlm = ' ' + result.setDlm = { ' ', '\t', '\n' } + result.n = result.setDlm.card + return + for c in delim: + if c in result.setDlm: + result.repeat = true + continue + result.setDlm.incl(c) + inc(result.n) + if result.n == 1: #support n==1 test to allow memchr optimization + result.chrDlm = delim[0] + +proc hash(x: MemSlice): int = 55542 + +template defSplit[T](slc: T, fs: var seq[MemSlice], n: int, repeat: bool, + sep: untyped, nextSep: untyped, isSep: untyped) {.dirty.} = + fs.setLen(if n < 1: 16 else: n) + var b = slc.data + var eob = b +! slc.size + while repeat and eob -! b > 0 and isSep((cast[cstring](b))[0], sep): + b = b +! 1 + if b == eob: fs.setLen(0); return + var e = nextSep(b, sep, (eob -! b).csize) + while e != nil: + if n < 1: #Unbounded msplit + if result == fs.len - 1: #Expand capacity + fs.setLen(if fs.len < 512: 2*fs.len else: fs.len + 512) + elif result == n - 1: #Need 1 more slot for final field + break + fs[result].data = b + fs[result].size = e -! b + result += 1 + while repeat and eob -! e > 0 and isSep((cast[cstring](e))[1], sep): + e = e +! 1 + b = e +! 1 + if eob -! b <= 0: + b = eob + break + e = nextSep(b, sep, (eob -! b).csize) + if not repeat or eob -! b > 0: + fs[result].data = b + fs[result].size = eob -! b + result += 1 + fs.setLen(result) + +proc msplit*(s: MemSlice, fs: var seq[MemSlice], sep=' ', n=0, + repeat=false): int = + defSplit(s, fs, n, repeat, sep, cmemchr, charEq) + +proc split*(s: Splitr, line: MemSlice, cols: var seq[MemSlice], + n=0) {.inline.} = + discard msplit(line, cols, s.chrDlm, n, s.repeat) + +######################################################################## +# Using lines instead of memSlices & split instead of splitr.split seems +# to mask the arc problem, as does simplifying `Table` to `seq[char]`. + +proc load(path: string, delim=" "): Table[MemSlice, seq[char]] = + let f = memfiles.open(path) + let splitr = initSplitr(delim) + var cols: seq[MemSlice] = @[ ] # re-used seq buffer + var nwSq = newSeqOfCap[char](1) # re-used seq value + nwSq.setLen 1 + for line in memSlices(f, eat='\0'): + stderr.write "loop 1a\n" + splitr.split(line, cols, 2) + stderr.write "loop 1b; cols: ", cols, "\n" + let cs = cast[cstring](cols[0].data) + stderr.write "loop 1c\n" #..reports exception here, but + nwSq[0] = cs[0] #..actually doing out of bounds here + stderr.write "loop 1d\n" + result[cols[1]] = nwSq + +discard load(getAppDir() / "testfile.txt") diff --git a/tests/arc/testfile.txt b/tests/arc/testfile.txt new file mode 100644 index 0000000000000..5169490be7386 --- /dev/null +++ b/tests/arc/testfile.txt @@ -0,0 +1,2 @@ +1 x +2 y From ec5cef13cffdb43a7298736b65e1a76e83741dff Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 9 Mar 2020 18:12:52 +0100 Subject: [PATCH 48/77] fixes #13599 (#13614) --- compiler/ccgstmts.nim | 2 +- tests/destructor/tgotoexceptions5.nim | 39 +++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 tests/destructor/tgotoexceptions5.nim diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index c430dd0628b04..9c067a339e81e 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -1115,10 +1115,10 @@ proc genTryGoto(p: BProc; t: PNode; d: var TLoc) = # 3. finally is run for exception handling code without any 'except' # handler present or only handlers that did not match. linefmt(p, cpsStmts, "*nimErr_ += oldNimErr$1_ + (*nimErr_ - oldNimErrFin$1_); oldNimErr$1_ = 0;$n", [lab]) - raiseExit(p) endBlock(p) # restore the real error value: linefmt(p, cpsStmts, "*nimErr_ += oldNimErr$1_;$n", [lab]) + if p.prc != nil: raiseExit(p) proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) = # code to generate: diff --git a/tests/destructor/tgotoexceptions5.nim b/tests/destructor/tgotoexceptions5.nim new file mode 100644 index 0000000000000..41e62aba0ca58 --- /dev/null +++ b/tests/destructor/tgotoexceptions5.nim @@ -0,0 +1,39 @@ +discard """ + output: ''' +before +swallowed +before +swallowed B +''' + cmd: "nim c --gc:arc --exceptions:goto $file" +""" + +# bug #13599 +proc main() = + try: + echo "before" + raise newException(CatchableError, "foo") + except AssertionError: + echo "caught" + echo "after" + +try: + main() +except: + echo "swallowed" + +proc mainB() = + try: + echo "before" + raise newException(CatchableError, "foo") + # except CatchableError: # would work + except AssertionError: + echo "caught" + except: + raise + echo "after" + +try: + mainB() +except: + echo "swallowed B" From 7d07897a9941717432d5872c3e0f203704902738 Mon Sep 17 00:00:00 2001 From: narimiran Date: Mon, 9 Mar 2020 21:16:56 +0100 Subject: [PATCH 49/77] travis now only builds devel docs --- .travis.yml | 56 +++++++---------------------------------------------- 1 file changed, 7 insertions(+), 49 deletions(-) diff --git a/.travis.yml b/.travis.yml index d62fe20d24c28..4c4c35334b6d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,45 +4,12 @@ language: c dist: xenial matrix: - fast_finish: true - include: - os: linux env: - NIM_COMPILE_TO_CPP=false - CPU=amd64 - - os: linux - env: - - NIM_COMPILE_TO_CPP=false - - CPU=i386 - addons: - apt: - packages: - - gcc-multilib - - g++-multilib - - libcurl4-openssl-dev:i386 - - libgc-dev:i386 - - libglib2.0-dev:i386 - - libpulse-dev:i386 - - libsdl1.2-dev:i386 - - libsfml-dev:i386 - - - os: osx - env: - - NIM_COMPILE_TO_CPP=false - - CPU=amd64 - - - os: osx - env: - - NIM_COMPILE_TO_CPP=true - - CPU=amd64 - -# To allow failures for a failing configuration, use something like: -# allow_failures: -# - env: NIM_COMPILE_TO_CPP=true -# - os: osx - addons: apt: # update the list above if more deps are introduced @@ -53,18 +20,6 @@ addons: - libsfml-dev - libc6-dbg - valgrind - homebrew: - packages: - - boehmgc - - make - - sfml - update: true - -install: - - if [[ $CPU = i386 ]]; then echo -e "#!/bin/bash\n$(which gcc) -m32 \"\$@\"" > bin/gcc; fi - - if [[ $CPU = i386 ]]; then chmod 755 bin/gcc; fi - - if [[ $CPU = i386 ]]; then echo -e "#!/bin/bash\n$(which g++) -m32 \"\$@\"" > bin/g++; fi - - if [[ $CPU = i386 ]]; then chmod 755 bin/g++; fi before_script: - git clone --depth 1 https://github.com/nim-lang/csources.git @@ -75,11 +30,17 @@ script: - echo "travis_fold:start:nim_c_koch" - nim c koch - echo "travis_fold:end:nim_c_koch" - - ./koch runCI + - echo "travis_fold:start:koch_boot" + - ./koch boot + - echo "travis_fold:end:koch_boot" + - echo "travis_fold:start:koch_doc" + - ./koch doc + - echo "travis_fold:end:koch_doc" before_deploy: # Make https://nim-lang.github.io/Nim work the same as https://nim-lang.github.io/Nim/overview.html - cp -f ./doc/html/overview.html ./doc/html/index.html + deploy: # https://nim-lang.github.io/Nim provider: pages # local-dir *has* to be a relative path from the repo root. @@ -89,6 +50,3 @@ deploy: # https://nim-lang.github.io/Nim keep-history: false on: branch: devel - -# Extract failed tests -after_failure: nim c -r tools/ci_testresults.nim From 090ba1e3a390e9997f45001e18dc93ba7aa091c3 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 10 Mar 2020 00:52:46 +0100 Subject: [PATCH 50/77] fixes #13436 (#13615) --- compiler/cgen.nim | 12 +++++++++--- compiler/injectdestructors.nim | 3 ++- tests/destructor/tgotoexceptions6.nim | 10 ++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 tests/destructor/tgotoexceptions6.nim diff --git a/compiler/cgen.nim b/compiler/cgen.nim index fd14817e0dc7f..57424ec9548c8 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1663,6 +1663,11 @@ proc genInitCode(m: BModule) = if beforeRetNeeded in m.initProc.flags: prc.add(~"\tBeforeRet_: ;$n") + + if sfMainModule in m.module.flags and m.config.exc == excGoto: + if getCompilerProc(m.g.graph, "nimTestErrorFlag") != nil: + m.appcg(prc, "\t#nimTestErrorFlag();$n", []) + if optStackTrace in m.initProc.options and preventStackTrace notin m.flags: prc.add(deinitFrame(m.initProc)) @@ -1990,9 +1995,10 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = if b == nil: return var m = BModule(b) if sfMainModule in m.module.flags: - let testForError = getCompilerProc(graph, "nimTestErrorFlag") - if testForError != nil and graph.config.exc == excGoto: - n.add newTree(nkCall, testForError.newSymNode) + # phase ordering problem here: We need to announce this + # dependency to 'nimTestErrorFlag' before system.c has been written to disk. + if m.config.exc == excGoto and getCompilerProc(graph, "nimTestErrorFlag") != nil: + discard cgsym(m, "nimTestErrorFlag") for i in countdown(high(graph.globalDestructors), 0): n.add graph.globalDestructors[i] diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index 520ba46ceea88..161cb76524494 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -591,7 +591,8 @@ proc p(n: PNode; c: var Con; mode: ProcessMode): PNode = # move the variable declaration to the top of the frame: c.addTopVar v # make sure it's destroyed at the end of the proc: - if not isUnpackedTuple(v): + if not isUnpackedTuple(v) and sfThread notin v.sym.flags: + # do not destroy thread vars for now at all for consistency. c.destroys.add genDestroy(c, v) elif c.inLoop > 0: # unpacked tuple needs reset at every loop iteration diff --git a/tests/destructor/tgotoexceptions6.nim b/tests/destructor/tgotoexceptions6.nim new file mode 100644 index 0000000000000..7c01f6a52bb57 --- /dev/null +++ b/tests/destructor/tgotoexceptions6.nim @@ -0,0 +1,10 @@ +discard """ + cmd: "nim c --gc:arc --exceptions:goto $file" + outputsub: "Error: unhandled exception: virus detected [ValueError]" + exitcode: "1" +""" + +# bug #13436 +proc foo = + raise newException(ValueError, "virus detected") +foo() From ab5d962b508d6578232d8924d7c59f388ae6d9a3 Mon Sep 17 00:00:00 2001 From: genotrance Date: Tue, 10 Mar 2020 09:58:27 -0500 Subject: [PATCH 51/77] Revert "Support cross compiling from host to host (#12859)" (#13591) This reverts commit e4f7656772657069cd3e27704d687b292d8d24ab. --- compiler/extccomp.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index b115721df1b7d..4cc5043f8618b 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -411,7 +411,8 @@ proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string = else: suffix - if optCompileOnly notin conf.globalOptions: + if (conf.target.hostOS != conf.target.targetOS or conf.target.hostCPU != conf.target.targetCPU) and + optCompileOnly notin conf.globalOptions: let fullCCname = platform.CPU[conf.target.targetCPU].name & '.' & platform.OS[conf.target.targetOS].name & '.' & CC[c].name & fullSuffix From 9e2bce2a8cd6f7f8edb0558ce7b553553eb94d02 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 10 Mar 2020 15:59:14 +0100 Subject: [PATCH 52/77] hotfix: make --useVersion:1.0 work --- lib/pure/typetraits.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index 0335d54883681..750c6233dedf9 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -111,7 +111,8 @@ macro genericParamsImpl(T: typedesc): untyped = var ret: NimNode case ai.typeKind of ntyStatic: - ret = newTree(nnkBracketExpr, @[bindSym"StaticParam", ai]) + since (1, 1): + ret = newTree(nnkBracketExpr, @[bindSym"StaticParam", ai]) of ntyTypeDesc: ret = ai else: From 3b7b01779856f50de94467fef9409ce228aa1ff1 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 10 Mar 2020 16:00:58 +0100 Subject: [PATCH 53/77] fixes #13607 --- lib/windows/winlean.nim | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 649358cb6f9ac..9793429b2d33b 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -128,12 +128,10 @@ const CREATE_NO_WINDOW* = 0x08000000'i32 -when useWinUnicode: - proc getVersionExW*(lpVersionInfo: ptr OSVERSIONINFO): WINBOOL {. - stdcall, dynlib: "kernel32", importc: "GetVersionExW", sideEffect.} -else: - proc getVersionExA*(lpVersionInfo: ptr OSVERSIONINFO): WINBOOL {. - stdcall, dynlib: "kernel32", importc: "GetVersionExA", sideEffect.} +proc getVersionExW*(lpVersionInfo: ptr OSVERSIONINFO): WINBOOL {. + stdcall, dynlib: "kernel32", importc: "GetVersionExW", sideEffect.} +proc getVersionExA*(lpVersionInfo: ptr OSVERSIONINFO): WINBOOL {. + stdcall, dynlib: "kernel32", importc: "GetVersionExA", sideEffect.} proc getVersion*(): DWORD {.stdcall, dynlib: "kernel32", importc: "GetVersion", sideEffect.} From e64f1c7ee4a25a1df837617f4a79ce7e515b9e0d Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 10 Mar 2020 16:31:34 -0700 Subject: [PATCH 54/77] `koch --nim:pathto/nim boot` and `koch boot --hint:cc:off` now work (#13516) * `koch boot --hint:cc:off` now works * `koch --nim:pathto/nim boot` now works; code cleanup --- compiler/extccomp.nim | 33 +++++++++++++++++++-------------- compiler/lineinfos.nim | 2 +- koch.nim | 9 +++------ tools/kochdocs.nim | 18 +++++++++++------- 4 files changed, 34 insertions(+), 28 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 4cc5043f8618b..6b4223433870f 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -705,6 +705,11 @@ proc addExternalFileToCompile*(conf: ConfigRef; filename: AbsoluteFile) = flags: {CfileFlag.External}) addExternalFileToCompile(conf, c) +proc displayProgressCC(conf: ConfigRef, path: string): string = + if conf.hasHint(hintCC): + let (_, name, _) = splitFile(path) + result = MsgKindToStr[hintCC] % demanglePackageName(name) + proc compileCFiles(conf: ConfigRef; list: CfileList, script: var Rope, cmds: var TStringSeq, prettyCmds: var TStringSeq) = var currIdx = 0 @@ -715,8 +720,7 @@ proc compileCFiles(conf: ConfigRef; list: CfileList, script: var Rope, cmds: var inc currIdx if optCompileOnly notin conf.globalOptions: cmds.add(compileCmd) - let (_, name, _) = splitFile(it.cname) - prettyCmds.add(if conf.hasHint(hintCC): "CC: " & demanglePackageName(name) else: "") + prettyCmds.add displayProgressCC(conf, $it.cname) if optGenScript in conf.globalOptions: script.add(compileCmd) script.add("\n") @@ -908,6 +912,11 @@ proc hcrLinkTargetName(conf: ConfigRef, objFile: string, isMain = false): Absolu else: platform.OS[conf.target.targetOS].dllFrmt % basename result = conf.getNimcacheDir / RelativeFile(targetName) +template callbackPrettyCmd(cmd) = + when declared(echo): + let cmd2 = cmd + if cmd2.len > 0: echo cmd2 + proc callCCompiler*(conf: ConfigRef) = var linkCmd: string @@ -918,10 +927,7 @@ proc callCCompiler*(conf: ConfigRef) = var script: Rope = nil var cmds: TStringSeq = @[] var prettyCmds: TStringSeq = @[] - let prettyCb = proc (idx: int) = - when declared(echo): - let cmd = prettyCmds[idx] - if cmd != "": echo cmd + let prettyCb = proc (idx: int) = callbackPrettyCmd(prettyCmds[idx]) compileCFiles(conf, conf.toCompile, script, cmds, prettyCmds) if optCompileOnly notin conf.globalOptions: execCmdsInParallel(conf, cmds, prettyCb) @@ -1126,12 +1132,9 @@ proc runJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) = doAssert c.len >= 2 cmds.add(c[1].getStr) - let (_, name, _) = splitFile(c[0].getStr) - prettyCmds.add("CC: " & demanglePackageName(name)) + prettyCmds.add displayProgressCC(conf, c[0].getStr) - let prettyCb = proc (idx: int) = - when declared(echo): - echo prettyCmds[idx] + let prettyCb = proc (idx: int) = callbackPrettyCmd(prettyCmds[idx]) execCmdsInParallel(conf, cmds, prettyCb) let linkCmd = data["linkcmd"] @@ -1146,9 +1149,11 @@ proc runJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) = execExternalProgram(conf, cmd2, hintExecuting) except: - when declared(echo): - echo getCurrentException().getStackTrace() - quit "error evaluating JSON file: " & jsonFile.string + let e = getCurrentException() + var msg = "\ncaught exception:n" & e.msg & "\nstacktrace:\n" & + getCurrentException().getStackTrace() & + "error evaluating JSON file: " & jsonFile.string + quit msg proc genMappingFiles(conf: ConfigRef; list: CfileList): Rope = for it in list: diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 9095dd13d21f3..5ac65201ebb06 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -99,7 +99,7 @@ const hintSuccess: "operation successful: $#", # keep in sync with `pegSuccess` see testament.nim hintSuccessX: "$loc LOC; $sec sec; $mem; $build build; proj: $project; out: $output", - hintCC: "CC: \'$1\'", # unused + hintCC: "CC: $1", hintLineTooLong: "line too long", hintXDeclaredButNotUsed: "'$1' is declared but not used", hintConvToBaseNotNeeded: "conversion to base object is not needed", diff --git a/koch.nim b/koch.nim index 75d5047586a22..982517656990c 100644 --- a/koch.nim +++ b/koch.nim @@ -265,15 +265,12 @@ when false: proc findStartNim: string = # we try several things before giving up: + # * nimExe # * bin/nim # * $PATH/nim # If these fail, we try to build nim with the "build.(sh|bat)" script. - var nim = "nim".exe - result = "bin" / nim - if existsFile(result): return - for dir in split(getEnv("PATH"), PathSep): - if existsFile(dir / nim): return dir / nim - + let (nim, ok) = findNimImpl() + if ok: return nim when defined(Posix): const buildScript = "build.sh" if existsFile(buildScript): diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index a5166973aa96e..75df629e3306d 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -18,15 +18,19 @@ proc exe*(f: string): string = when defined(windows): result = result.replace('/','\\') -proc findNim*(): string = - if nimExe.len > 0: return nimExe - var nim = "nim".exe - result = "bin" / nim - if existsFile(result): return +proc findNimImpl*(): tuple[path: string, ok: bool] = + if nimExe.len > 0: return (nimExe, true) + let nim = "nim".exe + result.path = "bin" / nim + result.ok = true + if existsFile(result.path): return for dir in split(getEnv("PATH"), PathSep): - if existsFile(dir / nim): return dir / nim + result.path = dir / nim + if existsFile(result.path): return # assume there is a symlink to the exe or something: - return nim + return (nim, false) + +proc findNim*(): string = findNimImpl().path proc exec*(cmd: string, errorcode: int = QuitFailure, additionalPath = "") = let prevPath = getEnv("PATH") From 2f557652d41dea81e43d36c3e89258ece620609e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20D=C3=B6ring?= Date: Wed, 11 Mar 2020 01:01:25 +0100 Subject: [PATCH 55/77] fix operators containing percent for VM usage (#13536) * fixes #13513 * merge tarithmetics in tarithm --- compiler/ast.nim | 2 - compiler/ccgexprs.nim | 2 - compiler/forloops.nim | 4 +- compiler/guards.nim | 4 +- compiler/jsgen.nim | 126 ++++++++++++++++++------------------- compiler/semfold.nim | 4 +- compiler/vmgen.nim | 4 +- lib/system/arithmetics.nim | 91 +++++++++++++++------------ lib/system/comparisons.nim | 45 +++++++------ tests/arithm/tarithm.nim | 18 ++++++ tests/pragmas/t6448.nim | 2 +- 11 files changed, 165 insertions(+), 137 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 43ea2b39c44fb..b82002b06e535 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -617,7 +617,6 @@ type mEqI, mLeI, mLtI, mEqF64, mLeF64, mLtF64, mLeU, mLtU, - mLeU64, mLtU64, mEqEnum, mLeEnum, mLtEnum, mEqCh, mLeCh, mLtCh, mEqB, mLeB, mLtB, @@ -687,7 +686,6 @@ const mEqI, mLeI, mLtI, mEqF64, mLeF64, mLtF64, mLeU, mLtU, - mLeU64, mLtU64, mEqEnum, mLeEnum, mLtEnum, mEqCh, mLeCh, mLtCh, mEqB, mLeB, mLtB, diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 277913b177bde..f133b97a2d456 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -605,8 +605,6 @@ proc binaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mLtF64: applyFormat("($1 < $2)") of mLeU: applyFormat("((NU$3)($1) <= (NU$3)($2))") of mLtU: applyFormat("((NU$3)($1) < (NU$3)($2))") - of mLeU64: applyFormat("((NU64)($1) <= (NU64)($2))") - of mLtU64: applyFormat("((NU64)($1) < (NU64)($2))") of mEqEnum: applyFormat("($1 == $2)") of mLeEnum: applyFormat("($1 <= $2)") of mLtEnum: applyFormat("($1 < $2)") diff --git a/compiler/forloops.nim b/compiler/forloops.nim index 729afb92054b8..ee91000d4b858 100644 --- a/compiler/forloops.nim +++ b/compiler/forloops.nim @@ -13,8 +13,8 @@ import ast, astalgo const someCmp = {mEqI, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc, - mLeI, mLeF64, mLeU, mLeU64, mLeEnum, - mLeCh, mLeB, mLePtr, mLtI, mLtF64, mLtU, mLtU64, mLtEnum, + mLeI, mLeF64, mLeU, mLeEnum, + mLeCh, mLeB, mLePtr, mLtI, mLtF64, mLtU, mLtEnum, mLtCh, mLtB, mLtPtr} proc isCounter(s: PSym): bool {.inline.} = diff --git a/compiler/guards.nim b/compiler/guards.nim index 97dc4f418d0c8..5d0fa64ebdc4d 100644 --- a/compiler/guards.nim +++ b/compiler/guards.nim @@ -17,9 +17,9 @@ const mEqStr, mEqSet, mEqCString} # set excluded here as the semantics are vastly different: - someLe = {mLeI, mLeF64, mLeU, mLeU64, mLeEnum, + someLe = {mLeI, mLeF64, mLeU, mLeEnum, mLeCh, mLeB, mLePtr, mLeStr} - someLt = {mLtI, mLtF64, mLtU, mLtU64, mLtEnum, + someLt = {mLtI, mLtF64, mLtU, mLtEnum, mLtCh, mLtB, mLtPtr, mLtStr} someLen = {mLengthOpenArray, mLengthStr, mLengthArray, mLengthSeq} diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 1130e35f8191a..e63ce74895de8 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -371,69 +371,67 @@ type const # magic checked op; magic unchecked op; jsMagics: TMagicOps = [ mAddI: ["addInt", ""], - ["subInt", ""], # SubI - ["mulInt", ""], # MulI - ["divInt", ""], # DivI - ["modInt", ""], # ModI - ["addInt", ""], # Succ - ["subInt", ""], # Pred - ["", ""], # AddF64 - ["", ""], # SubF64 - ["", ""], # MulF64 - ["", ""], # DivF64 - ["", ""], # ShrI - ["", ""], # ShlI - ["", ""], # AshrI - ["", ""], # BitandI - ["", ""], # BitorI - ["", ""], # BitxorI - ["nimMin", "nimMin"], # MinI - ["nimMax", "nimMax"], # MaxI - ["", ""], # addU - ["", ""], # subU - ["", ""], # mulU - ["", ""], # divU - ["", ""], # modU - ["", ""], # EqI - ["", ""], # LeI - ["", ""], # LtI - ["", ""], # EqF64 - ["", ""], # LeF64 - ["", ""], # LtF64 - ["", ""], # leU - ["", ""], # ltU - ["", ""], # leU64 - ["", ""], # ltU64 - ["", ""], # EqEnum - ["", ""], # LeEnum - ["", ""], # LtEnum - ["", ""], # EqCh - ["", ""], # LeCh - ["", ""], # LtCh - ["", ""], # EqB - ["", ""], # LeB - ["", ""], # LtB - ["", ""], # EqRef - ["", ""], # LePtr - ["", ""], # LtPtr - ["", ""], # Xor - ["", ""], # EqCString - ["", ""], # EqProc - ["negInt", ""], # UnaryMinusI - ["negInt64", ""], # UnaryMinusI64 - ["absInt", ""], # AbsI - ["", ""], # Not - ["", ""], # UnaryPlusI - ["", ""], # BitnotI - ["", ""], # UnaryPlusF64 - ["", ""], # UnaryMinusF64 - ["nimCharToStr", "nimCharToStr"], - ["nimBoolToStr", "nimBoolToStr"], - ["cstrToNimstr", "cstrToNimstr"], - ["cstrToNimstr", "cstrToNimstr"], - ["cstrToNimstr", "cstrToNimstr"], - ["cstrToNimstr", "cstrToNimstr"], - ["", ""]] + mSubI: ["subInt", ""], + mMulI: ["mulInt", ""], + mDivI: ["divInt", ""], + mModI: ["modInt", ""], + mSucc: ["addInt", ""], + mPred: ["subInt", ""], + mAddF64: ["", ""], + mSubF64: ["", ""], + mMulF64: ["", ""], + mDivF64: ["", ""], + mShrI: ["", ""], + mShlI: ["", ""], + mAshrI: ["", ""], + mBitandI: ["", ""], + mBitorI: ["", ""], + mBitxorI: ["", ""], + mMinI: ["nimMin", "nimMin"], + mMaxI: ["nimMax", "nimMax"], + mAddU: ["", ""], + mSubU: ["", ""], + mMulU: ["", ""], + mDivU: ["", ""], + mModU: ["", ""], + mEqI: ["", ""], + mLeI: ["", ""], + mLtI: ["", ""], + mEqF64: ["", ""], + mLeF64: ["", ""], + mLtF64: ["", ""], + mLeU: ["", ""], + mLtU: ["", ""], + mEqEnum: ["", ""], + mLeEnum: ["", ""], + mLtEnum: ["", ""], + mEqCh: ["", ""], + mLeCh: ["", ""], + mLtCh: ["", ""], + mEqB: ["", ""], + mLeB: ["", ""], + mLtB: ["", ""], + mEqRef: ["", ""], + mLePtr: ["", ""], + mLtPtr: ["", ""], + mXor: ["", ""], + mEqCString: ["", ""], + mEqProc: ["", ""], + mUnaryMinusI: ["negInt", ""], + mUnaryMinusI64: ["negInt64", ""], + mAbsI: ["absInt", ""], + mNot: ["", ""], + mUnaryPlusI: ["", ""], + mBitnotI: ["", ""], + mUnaryPlusF64: ["", ""], + mUnaryMinusF64: ["", ""], + mCharToStr: ["nimCharToStr", "nimCharToStr"], + mBoolToStr: ["nimBoolToStr", "nimBoolToStr"], + mIntToStr: ["cstrToNimstr", "cstrToNimstr"], + mInt64ToStr: ["cstrToNimstr", "cstrToNimstr"], + mFloatToStr: ["cstrToNimstr", "cstrToNimstr"], + mCStrToStr: ["cstrToNimstr", "cstrToNimstr"], + mStrToStr: ["", ""]] proc needsTemp(p: PProc; n: PNode): bool = # check if n contains a call to determine @@ -575,8 +573,6 @@ proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) = of mLtF64: applyFormat("($1 < $2)", "($1 < $2)") of mLeU: applyFormat("($1 <= $2)", "($1 <= $2)") of mLtU: applyFormat("($1 < $2)", "($1 < $2)") - of mLeU64: applyFormat("($1 <= $2)", "($1 <= $2)") - of mLtU64: applyFormat("($1 < $2)", "($1 < $2)") of mEqEnum: applyFormat("($1 == $2)", "($1 == $2)") of mLeEnum: applyFormat("($1 <= $2)", "($1 <= $2)") of mLtEnum: applyFormat("($1 < $2)", "($1 < $2)") diff --git a/compiler/semfold.nim b/compiler/semfold.nim index b20616e1d366f..54850fcfed93d 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -288,9 +288,9 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; g: ModuleGraph): PNode = of mLtStr: result = newIntNodeT(toInt128(ord(getStr(a) < getStr(b))), n, g) of mLeStr: result = newIntNodeT(toInt128(ord(getStr(a) <= getStr(b))), n, g) of mEqStr: result = newIntNodeT(toInt128(ord(getStr(a) == getStr(b))), n, g) - of mLtU, mLtU64: + of mLtU: result = newIntNodeT(toInt128(ord(`<%`(toInt64(getOrdValue(a)), toInt64(getOrdValue(b))))), n, g) - of mLeU, mLeU64: + of mLeU: result = newIntNodeT(toInt128(ord(`<=%`(toInt64(getOrdValue(a)), toInt64(getOrdValue(b))))), n, g) of mBitandI, mAnd: result = newIntNodeT(bitand(a.getInt, b.getInt), n, g) of mBitorI, mOr: result = newIntNodeT(bitor(getInt(a), getInt(b)), n, g) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 75f673c0b9468..5b355bcab8100 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1071,8 +1071,8 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = of mEqF64: genBinaryABC(c, n, dest, opcEqFloat) of mLeF64: genBinaryABC(c, n, dest, opcLeFloat) of mLtF64: genBinaryABC(c, n, dest, opcLtFloat) - of mLePtr, mLeU, mLeU64: genBinaryABC(c, n, dest, opcLeu) - of mLtPtr, mLtU, mLtU64: genBinaryABC(c, n, dest, opcLtu) + of mLePtr, mLeU: genBinaryABC(c, n, dest, opcLeu) + of mLtPtr, mLtU: genBinaryABC(c, n, dest, opcLtu) of mEqProc, mEqRef: genBinaryABC(c, n, dest, opcEqRef) of mXor: genBinaryABC(c, n, dest, opcXor) diff --git a/lib/system/arithmetics.nim b/lib/system/arithmetics.nim index 757a813e8619e..710566a6ddb54 100644 --- a/lib/system/arithmetics.nim +++ b/lib/system/arithmetics.nim @@ -342,45 +342,6 @@ proc `xor`*(x, y: int16): int16 {.magic: "BitxorI", noSideEffect.} proc `xor`*(x, y: int32): int32 {.magic: "BitxorI", noSideEffect.} proc `xor`*(x, y: int64): int64 {.magic: "BitxorI", noSideEffect.} -type - IntMax32 = int|int8|int16|int32 - -proc `+%`*(x, y: IntMax32): IntMax32 {.magic: "AddU", noSideEffect.} -proc `+%`*(x, y: int64): int64 {.magic: "AddU", noSideEffect.} - ## Treats `x` and `y` as unsigned and adds them. - ## - ## The result is truncated to fit into the result. - ## This implements modulo arithmetic. No overflow errors are possible. - -proc `-%`*(x, y: IntMax32): IntMax32 {.magic: "SubU", noSideEffect.} -proc `-%`*(x, y: int64): int64 {.magic: "SubU", noSideEffect.} - ## Treats `x` and `y` as unsigned and subtracts them. - ## - ## The result is truncated to fit into the result. - ## This implements modulo arithmetic. No overflow errors are possible. - -proc `*%`*(x, y: IntMax32): IntMax32 {.magic: "MulU", noSideEffect.} -proc `*%`*(x, y: int64): int64 {.magic: "MulU", noSideEffect.} - ## Treats `x` and `y` as unsigned and multiplies them. - ## - ## The result is truncated to fit into the result. - ## This implements modulo arithmetic. No overflow errors are possible. - -proc `/%`*(x, y: IntMax32): IntMax32 {.magic: "DivU", noSideEffect.} -proc `/%`*(x, y: int64): int64 {.magic: "DivU", noSideEffect.} - ## Treats `x` and `y` as unsigned and divides them. - ## - ## The result is truncated to fit into the result. - ## This implements modulo arithmetic. No overflow errors are possible. - -proc `%%`*(x, y: IntMax32): IntMax32 {.magic: "ModU", noSideEffect.} -proc `%%`*(x, y: int64): int64 {.magic: "ModU", noSideEffect.} - ## Treats `x` and `y` as unsigned and compute the modulo of `x` and `y`. - ## - ## The result is truncated to fit into the result. - ## This implements modulo arithmetic. No overflow errors are possible. - - # unsigned integer operations: proc `not`*(x: uint): uint {.magic: "BitnotI", noSideEffect.} ## Computes the `bitwise complement` of the integer `x`. @@ -461,8 +422,60 @@ proc `mod`*(x, y: uint16): uint16 {.magic: "ModU", noSideEffect.} proc `mod`*(x, y: uint32): uint32 {.magic: "ModU", noSideEffect.} proc `mod`*(x, y: uint64): uint64 {.magic: "ModU", noSideEffect.} +proc `+%`*(x, y: int): int {.inline.} = + ## Treats `x` and `y` as unsigned and adds them. + ## + ## The result is truncated to fit into the result. + ## This implements modulo arithmetic. No overflow errors are possible. + cast[int](cast[uint](x) + cast[uint](y)) +proc `+%`*(x, y: int8): int8 {.inline.} = cast[int8](cast[uint8](x) + cast[uint8](y)) +proc `+%`*(x, y: int16): int16 {.inline.} = cast[int16](cast[uint16](x) + cast[uint16](y)) +proc `+%`*(x, y: int32): int32 {.inline.} = cast[int32](cast[uint32](x) + cast[uint32](y)) +proc `+%`*(x, y: int64): int64 {.inline.} = cast[int64](cast[uint64](x) + cast[uint64](y)) + +proc `-%`*(x, y: int): int {.inline.} = + ## Treats `x` and `y` as unsigned and subtracts them. + ## + ## The result is truncated to fit into the result. + ## This implements modulo arithmetic. No overflow errors are possible. + cast[int](cast[uint](x) - cast[uint](y)) +proc `-%`*(x, y: int8): int8 {.inline.} = cast[int8](cast[uint8](x) - cast[uint8](y)) +proc `-%`*(x, y: int16): int16 {.inline.} = cast[int16](cast[uint16](x) - cast[uint16](y)) +proc `-%`*(x, y: int32): int32 {.inline.} = cast[int32](cast[uint32](x) - cast[uint32](y)) +proc `-%`*(x, y: int64): int64 {.inline.} = cast[int64](cast[uint64](x) - cast[uint64](y)) + +proc `*%`*(x, y: int): int {.inline.} = + ## Treats `x` and `y` as unsigned and multiplies them. + ## + ## The result is truncated to fit into the result. + ## This implements modulo arithmetic. No overflow errors are possible. + cast[int](cast[uint](x) * cast[uint](y)) +proc `*%`*(x, y: int8): int8 {.inline.} = cast[int8](cast[uint8](x) * cast[uint8](y)) +proc `*%`*(x, y: int16): int16 {.inline.} = cast[int16](cast[uint16](x) * cast[uint16](y)) +proc `*%`*(x, y: int32): int32 {.inline.} = cast[int32](cast[uint32](x) * cast[uint32](y)) +proc `*%`*(x, y: int64): int64 {.inline.} = cast[int64](cast[uint64](x) * cast[uint64](y)) +proc `/%`*(x, y: int): int {.inline.} = + ## Treats `x` and `y` as unsigned and divides them. + ## + ## The result is truncated to fit into the result. + ## This implements modulo arithmetic. No overflow errors are possible. + cast[int](cast[uint](x) div cast[uint](y)) +proc `/%`*(x, y: int8): int8 {.inline.} = cast[int8](cast[uint8](x) div cast[uint8](y)) +proc `/%`*(x, y: int16): int16 {.inline.} = cast[int16](cast[uint16](x) div cast[uint16](y)) +proc `/%`*(x, y: int32): int32 {.inline.} = cast[int32](cast[uint32](x) div cast[uint32](y)) +proc `/%`*(x, y: int64): int64 {.inline.} = cast[int64](cast[uint64](x) div cast[uint64](y)) +proc `%%`*(x, y: int): int {.inline.} = + ## Treats `x` and `y` as unsigned and compute the modulo of `x` and `y`. + ## + ## The result is truncated to fit into the result. + ## This implements modulo arithmetic. No overflow errors are possible. + cast[int](cast[uint](x) mod cast[uint](y)) +proc `%%`*(x, y: int8): int8 {.inline.} = cast[int8](cast[uint8](x) mod cast[uint8](y)) +proc `%%`*(x, y: int16): int16 {.inline.} = cast[int16](cast[uint16](x) mod cast[uint16](y)) +proc `%%`*(x, y: int32): int32 {.inline.} = cast[int32](cast[uint32](x) mod cast[uint32](y)) +proc `%%`*(x, y: int64): int64 {.inline.} = cast[int64](cast[uint64](x) mod cast[uint64](y)) proc `+=`*[T: SomeInteger](x: var T, y: T) {. magic: "Inc", noSideEffect.} diff --git a/lib/system/comparisons.nim b/lib/system/comparisons.nim index 787820989e93e..60ac554a624f4 100644 --- a/lib/system/comparisons.nim +++ b/lib/system/comparisons.nim @@ -161,16 +161,37 @@ proc `<`*(x, y: int16): bool {.magic: "LtI", noSideEffect.} proc `<`*(x, y: int32): bool {.magic: "LtI", noSideEffect.} proc `<`*(x, y: int64): bool {.magic: "LtI", noSideEffect.} +proc `<=`*(x, y: uint): bool {.magic: "LeU", noSideEffect.} + ## Returns true if ``x <= y``. +proc `<=`*(x, y: uint8): bool {.magic: "LeU", noSideEffect.} +proc `<=`*(x, y: uint16): bool {.magic: "LeU", noSideEffect.} +proc `<=`*(x, y: uint32): bool {.magic: "LeU", noSideEffect.} +proc `<=`*(x, y: uint64): bool {.magic: "LeU", noSideEffect.} + +proc `<`*(x, y: uint): bool {.magic: "LtU", noSideEffect.} + ## Returns true if ``x < y``. +proc `<`*(x, y: uint8): bool {.magic: "LtU", noSideEffect.} +proc `<`*(x, y: uint16): bool {.magic: "LtU", noSideEffect.} +proc `<`*(x, y: uint32): bool {.magic: "LtU", noSideEffect.} +proc `<`*(x, y: uint64): bool {.magic: "LtU", noSideEffect.} -proc `<=%`*(x, y: IntMax32): bool {.magic: "LeU", noSideEffect.} -proc `<=%`*(x, y: int64): bool {.magic: "LeU64", noSideEffect.} +proc `<=%`*(x, y: int): bool {.inline.} = ## Treats `x` and `y` as unsigned and compares them. ## Returns true if ``unsigned(x) <= unsigned(y)``. + cast[uint](x) <= cast[uint](y) +proc `<=%`*(x, y: int8): bool {.inline.} = cast[uint8](x) <= cast[uint8](y) +proc `<=%`*(x, y: int16): bool {.inline.} = cast[uint16](x) <= cast[uint16](y) +proc `<=%`*(x, y: int32): bool {.inline.} = cast[uint32](x) <= cast[uint32](y) +proc `<=%`*(x, y: int64): bool {.inline.} = cast[uint64](x) <= cast[uint64](y) -proc `<%`*(x, y: IntMax32): bool {.magic: "LtU", noSideEffect.} -proc `<%`*(x, y: int64): bool {.magic: "LtU64", noSideEffect.} +proc `<%`*(x, y: int): bool {.inline.} = ## Treats `x` and `y` as unsigned and compares them. ## Returns true if ``unsigned(x) < unsigned(y)``. + cast[uint](x) < cast[uint](y) +proc `<%`*(x, y: int8): bool {.inline.} = cast[uint8](x) < cast[uint8](y) +proc `<%`*(x, y: int16): bool {.inline.} = cast[uint16](x) < cast[uint16](y) +proc `<%`*(x, y: int32): bool {.inline.} = cast[uint32](x) < cast[uint32](y) +proc `<%`*(x, y: int64): bool {.inline.} = cast[uint64](x) < cast[uint64](y) template `>=%`*(x, y: untyped): untyped = y <=% x ## Treats `x` and `y` as unsigned and compares them. @@ -180,7 +201,6 @@ template `>%`*(x, y: untyped): untyped = y <% x ## Treats `x` and `y` as unsigned and compares them. ## Returns true if ``unsigned(x) > unsigned(y)``. - proc `==`*(x, y: uint): bool {.magic: "EqI", noSideEffect.} ## Compares two unsigned integers for equality. proc `==`*(x, y: uint8): bool {.magic: "EqI", noSideEffect.} @@ -189,21 +209,6 @@ proc `==`*(x, y: uint32): bool {.magic: "EqI", noSideEffect.} proc `==`*(x, y: uint64): bool {.magic: "EqI", noSideEffect.} -proc `<=`*(x, y: uint): bool {.magic: "LeU", noSideEffect.} - ## Returns true if ``x <= y``. -proc `<=`*(x, y: uint8): bool {.magic: "LeU", noSideEffect.} -proc `<=`*(x, y: uint16): bool {.magic: "LeU", noSideEffect.} -proc `<=`*(x, y: uint32): bool {.magic: "LeU", noSideEffect.} -proc `<=`*(x, y: uint64): bool {.magic: "LeU", noSideEffect.} - -proc `<`*(x, y: uint): bool {.magic: "LtU", noSideEffect.} - ## Returns true if ``unsigned(x) < unsigned(y)``. -proc `<`*(x, y: uint8): bool {.magic: "LtU", noSideEffect.} -proc `<`*(x, y: uint16): bool {.magic: "LtU", noSideEffect.} -proc `<`*(x, y: uint32): bool {.magic: "LtU", noSideEffect.} -proc `<`*(x, y: uint64): bool {.magic: "LtU", noSideEffect.} - - {.push stackTrace: off.} proc min*(x, y: int): int {.magic: "MinI", noSideEffect.} = diff --git a/tests/arithm/tarithm.nim b/tests/arithm/tarithm.nim index fcb78bd7a4b2c..99306b3e84dc8 100644 --- a/tests/arithm/tarithm.nim +++ b/tests/arithm/tarithm.nim @@ -12,7 +12,9 @@ int32 4294967295 2 0 +tUnsignedOps OK ''' +nimout: "tUnsignedOps OK" """ import typetraits @@ -167,3 +169,19 @@ block tissue12177: echo(a - b) echo(a * b) echo(a div b) + +block tUnsignedOps: + proc testUnsignedOps() = + let a: int8 = -128 + let b: int8 = 127 + + doAssert b +% 1 == -128 + doAssert b -% -1 == -128 + doAssert b *% 2 == -2 + doAssert a /% 4 == 32 + doAssert a %% 7 == 2 + echo "tUnsignedOps OK" + + testUnsignedOps() + static: + testUnsignedOps() diff --git a/tests/pragmas/t6448.nim b/tests/pragmas/t6448.nim index ab2e3811f9bba..6efb56e08ae38 100644 --- a/tests/pragmas/t6448.nim +++ b/tests/pragmas/t6448.nim @@ -1,5 +1,5 @@ discard """ - errormsg: '''ambiguous call; both foobar.async''' + errormsg: '''ambiguous call''' line: 10 disabled: "32bit" """ From 8e3a349561fb0af7f5bc77a4d436abb3ced75d9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=82=81=E3=81=90=E3=81=BF=E7=99=BA=E5=8B=95=E6=A9=9F=20?= =?UTF-8?q?=28isVowel=20/=20GreenWing=29?= <60494954+pancakevirus@users.noreply.github.com> Date: Wed, 11 Mar 2020 16:26:10 +0900 Subject: [PATCH 56/77] fixed to jsonArrayEnd comment. (#13624) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit typo: start → end --- lib/pure/parsejson.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/parsejson.nim b/lib/pure/parsejson.nim index 0d7d7093ecd04..18e6037f3cf64 100644 --- a/lib/pure/parsejson.nim +++ b/lib/pure/parsejson.nim @@ -27,7 +27,7 @@ type jsonObjectStart, ## start of an object: the ``{`` token jsonObjectEnd, ## end of an object: the ``}`` token jsonArrayStart, ## start of an array: the ``[`` token - jsonArrayEnd ## start of an array: the ``]`` token + jsonArrayEnd ## end of an array: the ``]`` token TokKind* = enum # must be synchronized with TJsonEventKind! tkError, From f95eef99a97dc813cf2f819a6bbaa3b3ead67267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20D=C3=B6ring?= Date: Wed, 11 Mar 2020 08:27:31 +0100 Subject: [PATCH 57/77] add expectIdent to macros (#12778) * add expectIdent to macros * apply feedback * Update lib/core/macros.nim Co-Authored-By: Clyybber * Update texpectIdent2.nim * Update texpectIdent1.nim Co-authored-by: Clyybber Co-authored-by: Andreas Rumpf --- changelog.md | 2 +- lib/core/macros.nim | 7 +++++++ tests/macros/texpectIdent1.nim | 18 ++++++++++++++++++ tests/macros/texpectIdent2.nim | 24 ++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 tests/macros/texpectIdent1.nim create mode 100644 tests/macros/texpectIdent2.nim diff --git a/changelog.md b/changelog.md index 49567e9298d82..44c654fd3e1e2 100644 --- a/changelog.md +++ b/changelog.md @@ -69,6 +69,7 @@ - Added `minIndex`, `maxIndex` and `unzip` to the `sequtils` module. - Added `os.isRelativeTo` to tell whether a path is relative to another - Added `resetOutputFormatters` to `unittest` +- Added `expectIdent` to the `macros` module. - Added `os.isValidFilename` that returns `true` if `filename` argument is valid for crossplatform use. - Added a `with` macro for easy function chaining that's available @@ -95,7 +96,6 @@ echo f - Added a new module, `std / compilesettings` for querying the compiler about diverse configuration settings. - ## Library changes - `asyncdispatch.drain` now properly takes into account `selector.hasPendingOperations` diff --git a/lib/core/macros.nim b/lib/core/macros.nim index cc20f1dac655a..57ce89c02ed27 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1433,6 +1433,13 @@ else: else: result = false +proc expectIdent*(n: NimNode, name: string) {.compileTime, since: (1,1).} = + ## Check that ``eqIdent(n,name)`` holds true. If this is not the + ## case, compilation aborts with an error message. This is useful + ## for writing macros that check the AST that is passed to them. + if not eqIdent(n, name): + error("Expected identifier to be `" & name & "` here", n) + proc hasArgOfName*(params: NimNode; name: string): bool {.compileTime.}= ## Search ``nnkFormalParams`` for an argument. expectKind(params, nnkFormalParams) diff --git a/tests/macros/texpectIdent1.nim b/tests/macros/texpectIdent1.nim new file mode 100644 index 0000000000000..26e52afb556ef --- /dev/null +++ b/tests/macros/texpectIdent1.nim @@ -0,0 +1,18 @@ +discard """ +errormsg: "Expected identifier to be `foo` here" +line: 18 +""" + +import macros + +macro testUntyped(arg: untyped): void = + arg.expectKind nnkStmtList + arg.expectLen 2 + arg[0].expectKind nnkCall + arg[0][0].expectIdent "foo" # must pass + arg[1].expectKind nnkCall + arg[1][0].expectIdent "foo" # must fail + +testUntyped: + foo(123) + bar(321) diff --git a/tests/macros/texpectIdent2.nim b/tests/macros/texpectIdent2.nim new file mode 100644 index 0000000000000..887a6ddc369be --- /dev/null +++ b/tests/macros/texpectIdent2.nim @@ -0,0 +1,24 @@ +discard """ +errormsg: "Expected identifier to be `foo` here" +line: 24 +""" + +import macros + +macro testTyped(arg: typed): void = + arg.expectKind nnkStmtList + arg.expectLen 2 + arg[0].expectKind nnkCall + arg[0][0].expectIdent "foo" # must pass + arg[1].expectKind nnkCall + arg[1][0].expectIdent "foo" # must fail + +proc foo(arg: int) = + discard + +proc bar(arg: int) = + discard + +testTyped: + foo(123) + bar(321) From b0684ec425dca5e76eb6b27eb09a84fb523af49c Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 10 Mar 2020 09:42:23 +0100 Subject: [PATCH 58/77] fixes #12757 --- compiler/semtypes.nim | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index abc5de7e8b07d..f05affc6b56b7 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -893,7 +893,7 @@ proc semAnyRef(c: PContext; n: PNode; kind: TTypeKind; prev: PType): PType = localError(c.config, n.info, "type '$1 void' is not allowed" % kindToStr[kind]) result = newOrPrevType(kind, prev, c) var isNilable = false - var isOwned = false + var wrapperKind = tyNone # check every except the last is an object: for i in isCall.. Date: Tue, 10 Mar 2020 10:39:43 +0100 Subject: [PATCH 59/77] fixes #13519 --- changelog.md | 4 ++++ compiler/ast.nim | 7 +++++-- compiler/commands.nim | 6 ++++-- lib/pure/asyncdispatch.nim | 2 +- lib/pure/asyncmacro.nim | 2 +- lib/system/assertions.nim | 2 +- tests/generics/tarc_misc.nim | 21 +++++++++++++++++++++ 7 files changed, 37 insertions(+), 7 deletions(-) create mode 100644 tests/generics/tarc_misc.nim diff --git a/changelog.md b/changelog.md index 44c654fd3e1e2..b769464640cda 100644 --- a/changelog.md +++ b/changelog.md @@ -40,6 +40,10 @@ - The `{.dynlib.}` pragma is now required for exporting symbols when making shared objects on POSIX and macOS, which make it consistent with the behavior on Windows. +- The compiler is now more strict about type conversions concerning proc + types: Type conversions cannot be used to hide `.raise` effects or side + effects, instead a `cast` must be used. With the flag `--useVersion:1.0` the + old behaviour is emulated. ## Library additions diff --git a/compiler/ast.nim b/compiler/ast.nim index b82002b06e535..62c301a43adb9 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -592,8 +592,11 @@ const tfReturnsNew* = tfInheritable skError* = skUnknown - # type flags that are essential for type equality: - eqTypeFlags* = {tfIterator, tfNotNil, tfVarIsPtr} +var + eqTypeFlags* = {tfIterator, tfNotNil, tfVarIsPtr, tfGcSafe, tfNoSideEffect} + ## type flags that are essential for type equality. + ## This is now a variable because for emulation of version:1.0 we + ## might exclude {tfGcSafe, tfNoSideEffect}. type TMagic* = enum # symbols that require compiler magic: diff --git a/compiler/commands.nim b/compiler/commands.nim index 935d7e2be5f85..0c93654da2874 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -32,6 +32,7 @@ import pathutils, strtabs from incremental import nimIncremental +from ast import eqTypeFlags, tfGcSafe, tfNoSideEffect # but some have deps to imported modules. Yay. bootSwitch(usedTinyC, hasTinyCBackend, "-d:tinyc") @@ -869,10 +870,11 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "1.0": defineSymbol(conf.symbols, "NimMajor", "1") defineSymbol(conf.symbols, "NimMinor", "0") - # always be compatible with 1.0.2 for now: - defineSymbol(conf.symbols, "NimPatch", "2") + # always be compatible with 1.0.100: + defineSymbol(conf.symbols, "NimPatch", "100") # old behaviors go here: defineSymbol(conf.symbols, "nimOldRelativePathBehavior") + ast.eqTypeFlags.excl {tfGcSafe, tfNoSideEffect} else: localError(conf, info, "unknown Nim version; currently supported values are: {1.0}") of "benchmarkvm": diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index d724242d7170d..732381ccbfc7b 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -278,7 +278,7 @@ when defined(windows) or defined(nimdoc): result.ioPort = createIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1) result.handles = initSet[AsyncFD]() result.timers.newHeapQueue() - result.callbacks = initDeque[proc ()](64) + result.callbacks = initDeque[proc () {.closure, gcsafe.}](64) var gDisp{.threadvar.}: owned PDispatcher ## Global dispatcher diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index 11eba427bd346..ce84491eb4f8d 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -46,7 +46,7 @@ template createCb(retFutureSym, iteratorNameSym, else: {.gcsafe.}: {.push hint[ConvFromXtoItselfNotNeeded]: off.} - next.callback = (proc() {.closure, gcsafe.})(identName) + next.callback = cast[proc() {.closure, gcsafe.}](identName) {.pop.} except: futureVarCompletions diff --git a/lib/system/assertions.nim b/lib/system/assertions.nim index b918729dc3393..5eb700f4c30ae 100644 --- a/lib/system/assertions.nim +++ b/lib/system/assertions.nim @@ -26,7 +26,7 @@ proc failedAssertImpl*(msg: string) {.raises: [], tags: [].} = # by ``assert``. type Hide = proc (msg: string) {.noinline, raises: [], noSideEffect, tags: [].} - Hide(raiseAssert)(msg) + cast[Hide](raiseAssert)(msg) template assertImpl(cond: bool, msg: string, expr: string, enabled: static[bool]) = when enabled: diff --git a/tests/generics/tarc_misc.nim b/tests/generics/tarc_misc.nim new file mode 100644 index 0000000000000..3e77625565925 --- /dev/null +++ b/tests/generics/tarc_misc.nim @@ -0,0 +1,21 @@ +discard """ + output: '''''' + cmd: "nim c --gc:arc $file" +""" + +# bug #13519 + +var unrelated: seq[proc() {.closure, gcsafe.}] + +unrelated.add proc () = + echo "gcsafe" + +import tables, sequtils +let t = newTable[int, proc()]() + +type + MyProc = proc() {.closure.} + +var result: seq[MyProc] = @[] +for x in t.values: + result.add(x) From 60f8fdcdabf99ae60f7a743cfd682bd7c320408b Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 10 Mar 2020 11:10:52 +0100 Subject: [PATCH 60/77] fixes #13240 --- compiler/ccgexprs.nim | 9 ++++++++- tests/arc/tconst_to_sink.nim | 25 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 tests/arc/tconst_to_sink.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index f133b97a2d456..1bed4bc6cc5eb 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1340,7 +1340,14 @@ proc genObjConstr(p: BProc, e: PNode, d: var TLoc) = #echo rendertree e, " ", e.isDeepConstExpr # inheritance in C++ does not allow struct initialization so # we skip this step here: - if not p.module.compileToCpp: + if not p.module.compileToCpp and optSeqDestructors notin p.config.globalOptions: + # disabled optimization: it is wrong for C++ and now also + # causes trouble for --gc:arc, see bug #13240 + #[ + var box: seq[Thing] + for i in 0..3: + box.add Thing(s1: "121") # pass by sink can mutate Thing. + ]# if handleConstExpr(p, e, d): return var t = e.typ.skipTypes(abstractInstOwned) let isRef = t.kind == tyRef diff --git a/tests/arc/tconst_to_sink.nim b/tests/arc/tconst_to_sink.nim new file mode 100644 index 0000000000000..ddcc46e67eeb1 --- /dev/null +++ b/tests/arc/tconst_to_sink.nim @@ -0,0 +1,25 @@ +discard """ + output: '''@[(s1: "333", s2: ""), (s1: "abc", s2: "def"), (s1: "3x", s2: ""), (s1: "3x", s2: ""), (s1: "3x", s2: ""), (s1: "3x", s2: ""), (s1: "lastone", s2: "")]''' + cmd: "nim c --gc:arc $file" +""" + +# bug #13240 + +type + Thing = object + s1: string + s2: string + +var box: seq[Thing] + +const c = [Thing(s1: "333"), Thing(s1: "abc", s2: "def")] + +for i in 0..high(c): + box.add c[i] + +for i in 0..3: + box.add Thing(s1: "3x") + +box.add Thing(s1: "lastone") + +echo box From 3aed8ff4326cbfabef074b28366d487e2bdc8ae2 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 10 Mar 2020 12:46:19 +0100 Subject: [PATCH 61/77] fixes async regression --- lib/pure/asyncdispatch.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 732381ccbfc7b..5e4e3f6998a97 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1122,7 +1122,7 @@ else: new result result.selector = newSelector[AsyncData]() result.timers.newHeapQueue() - result.callbacks = initDeque[proc ()](InitDelayedCallbackListSize) + result.callbacks = initDeque[proc () {.closure, gcsafe.}](InitDelayedCallbackListSize) var gDisp{.threadvar.}: owned PDispatcher ## Global dispatcher From f5f9243cc05cae998e4380a347da8e93ac547dd4 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 11 Mar 2020 09:28:10 +0100 Subject: [PATCH 62/77] disable chronos testing for now --- testament/important_packages.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 9dbb4a4f9f6e6..3631710828cf3 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -17,7 +17,8 @@ pkg "c2nim", false, "nim c testsuite/tester.nim" pkg "cascade" pkg "chroma" pkg "chronicles", true, "nim c -o:chr -r chronicles.nim" -pkg "chronos", true +# disable until my chronos fix was merged +#pkg "chronos", true pkg "cligen", false, "nim c -o:cligenn -r cligen.nim" pkg "coco", true pkg "combparser" From 281e02fc797ad98013ab29525256ba2ffdaf96a7 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 11 Mar 2020 01:47:34 -0700 Subject: [PATCH 63/77] fixes #13558: toDateTime buggy on 29th, 30th and 31th of each month; breaking change: do not use `now` to compute result, was undocumented and non-sensical (#13565) --- changelog.md | 4 ++++ lib/pure/times.nim | 16 +++------------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/changelog.md b/changelog.md index b769464640cda..3cb9547f34bfb 100644 --- a/changelog.md +++ b/changelog.md @@ -31,6 +31,10 @@ and shouldn't be conflated with `"."`; use -d:nimOldRelativePathBehavior to restore the old behavior - `joinPath(a,b)` now honors trailing slashes in `b` (or `a` if `b` = "") +- `times.parse` now only uses input to compute its result, and not `now`: + `parse("2020", "YYYY", utc())` is now `2020-01-01T00:00:00Z` instead of + `2020-03-02T00:00:00Z` if run on 03-02; it also doesn't crash anymore when + used on 29th, 30th, 31st of each month. ### Breaking changes in the compiler diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 92b6cd3b7e1d3..45ed6dd674a35 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -2248,19 +2248,9 @@ proc parsePattern(input: string, pattern: FormatPattern, i: var int, proc toDateTime(p: ParsedTime, zone: Timezone, f: TimeFormat, input: string): DateTime = - var month = mJan - var year: int - var monthday: int - # `now()` is an expensive call, so we avoid it when possible - (year, month, monthday) = - if p.year.isNone or p.month.isNone or p.monthday.isNone: - let n = now() - (p.year.get(n.year), - p.month.get(n.month.int).Month, - p.monthday.get(n.monthday)) - else: - (p.year.get(), p.month.get().Month, p.monthday.get()) - + var year = p.year.get(0) + var month = p.month.get(1).Month + var monthday = p.monthday.get(1) year = case p.era of eraUnknown: From 70bd41dae08a7ba9a9e8f19a935bee26b689d00d Mon Sep 17 00:00:00 2001 From: Miran Date: Wed, 11 Mar 2020 17:30:36 +0100 Subject: [PATCH 64/77] fix #13310, Deque misbehaves on VM (#13625) * fix #13310, Deque misbehaves on VM * use 'when nimVM' --- lib/pure/collections/deques.nim | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim index 1aa525bd6b559..77579a52bec17 100644 --- a/lib/pure/collections/deques.nim +++ b/lib/pure/collections/deques.nim @@ -274,8 +274,9 @@ proc expandIfNeeded[T](deq: var Deque[T]) = if unlikely(deq.count >= cap): var n = newSeq[T](cap * 2) var i = 0 - for x in mitems(deq): # don't use copyMem because of the GC and because it's slower. - n[i] = move(x) + for x in mitems(deq): + when nimVM: n[i] = x # workaround for VM bug + else: n[i] = move(x) inc i deq.data = move(n) deq.mask = cap * 2 - 1 @@ -569,3 +570,15 @@ when isMainModule: foo(2, 1) foo(1, 5) foo(3, 2) + + import sets + block t13310: + proc main() = + var q = initDeque[HashSet[int16]](2) + q.addFirst([1'i16].toHashSet) + q.addFirst([2'i16].toHashSet) + q.addFirst([3'i16].toHashSet) + assert $q == "[{3}, {2}, {1}]" + + static: + main() From 6b3098c378430e3387854ffc413ce96bd94a3d2c Mon Sep 17 00:00:00 2001 From: Clyybber Date: Wed, 11 Mar 2020 23:04:33 +0100 Subject: [PATCH 65/77] Make listCmd honor hint:cc:off (#13606) * Make listCmd honor hint:cc:off * Tiny cleanup * Tiny tiny cleanup * VERY IMPORTANT: --hint:cc:on will overwrite --verbosity:0 :p * Tiny cleanup * Stupid * Move displayProgressCC to where its required * Tiny cleanup --- compiler/extccomp.nim | 85 ++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 50 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 6b4223433870f..34552e6f3bbf1 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -623,7 +623,7 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, var objfile = if cfile.obj.isEmpty: - if not cfile.flags.contains(CfileFlag.External) or noAbsolutePaths(conf): + if CfileFlag.External notin cfile.flags or noAbsolutePaths(conf): toObjFile(conf, cf).string else: completeCfilePath(conf, toObjFile(conf, cf)).string @@ -705,26 +705,6 @@ proc addExternalFileToCompile*(conf: ConfigRef; filename: AbsoluteFile) = flags: {CfileFlag.External}) addExternalFileToCompile(conf, c) -proc displayProgressCC(conf: ConfigRef, path: string): string = - if conf.hasHint(hintCC): - let (_, name, _) = splitFile(path) - result = MsgKindToStr[hintCC] % demanglePackageName(name) - -proc compileCFiles(conf: ConfigRef; list: CfileList, script: var Rope, cmds: var TStringSeq, - prettyCmds: var TStringSeq) = - var currIdx = 0 - for it in list: - # call the C compiler for the .c file: - if it.flags.contains(CfileFlag.Cached): continue - var compileCmd = getCompileCFileCmd(conf, it, currIdx == list.len - 1, produceOutput=true) - inc currIdx - if optCompileOnly notin conf.globalOptions: - cmds.add(compileCmd) - prettyCmds.add displayProgressCC(conf, $it.cname) - if optGenScript in conf.globalOptions: - script.add(compileCmd) - script.add("\n") - proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile, objfiles: string, isDllBuild: bool): string = if optGenStaticLib in conf.globalOptions: @@ -862,15 +842,8 @@ proc execCmdsInParallel(conf: ConfigRef; cmds: seq[string]; prettyCb: proc (idx: cmds[i]) else: tryExceptOSErrorMessage(conf, "invocation of external compiler program failed."): - if optListCmd in conf.globalOptions or conf.verbosity > 1: - res = execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams}, - conf.numberOfProcessors, afterRunEvent=runCb) - elif conf.verbosity == 1: - res = execProcesses(cmds, {poStdErrToStdOut, poUsePath, poParentStreams}, + res = execProcesses(cmds, {poStdErrToStdOut, poUsePath, poParentStreams}, conf.numberOfProcessors, prettyCb, afterRunEvent=runCb) - else: - res = execProcesses(cmds, {poStdErrToStdOut, poUsePath, poParentStreams}, - conf.numberOfProcessors, afterRunEvent=runCb) if res != 0: if conf.numberOfProcessors <= 1: rawMessage(conf, errGenerated, "execution of an external program failed: '$1'" % @@ -912,10 +885,12 @@ proc hcrLinkTargetName(conf: ConfigRef, objFile: string, isMain = false): Absolu else: platform.OS[conf.target.targetOS].dllFrmt % basename result = conf.getNimcacheDir / RelativeFile(targetName) -template callbackPrettyCmd(cmd) = - when declared(echo): - let cmd2 = cmd - if cmd2.len > 0: echo cmd2 +proc displayProgressCC(conf: ConfigRef, path, compileCmd: string): string = + if conf.hasHint(hintCC): + if optListCmd in conf.globalOptions or conf.verbosity > 1: + result = MsgKindToStr[hintCC] % (demanglePackageName(path.splitFile.name) & ": " & compileCmd) + else: + result = MsgKindToStr[hintCC] % demanglePackageName(path.splitFile.name) proc callCCompiler*(conf: ConfigRef) = var @@ -925,10 +900,22 @@ proc callCCompiler*(conf: ConfigRef) = # generated #var c = cCompiler var script: Rope = nil - var cmds: TStringSeq = @[] - var prettyCmds: TStringSeq = @[] - let prettyCb = proc (idx: int) = callbackPrettyCmd(prettyCmds[idx]) - compileCFiles(conf, conf.toCompile, script, cmds, prettyCmds) + var cmds: TStringSeq + var prettyCmds: TStringSeq + let prettyCb = proc (idx: int) = + if prettyCmds[idx].len > 0: echo prettyCmds[idx] + + for idx, it in conf.toCompile: + # call the C compiler for the .c file: + if CfileFlag.Cached in it.flags: continue + let compileCmd = getCompileCFileCmd(conf, it, idx == conf.toCompile.len - 1, produceOutput=true) + if optCompileOnly notin conf.globalOptions: + cmds.add(compileCmd) + prettyCmds.add displayProgressCC(conf, $it.cname, compileCmd) + if optGenScript in conf.globalOptions: + script.add(compileCmd) + script.add("\n") + if optCompileOnly notin conf.globalOptions: execCmdsInParallel(conf, cmds, prettyCb) if optNoLinking notin conf.globalOptions: @@ -947,7 +934,7 @@ proc callCCompiler*(conf: ConfigRef) = # don't relink each of the many binaries (one for each source file) if the nim code is # cached because that would take too much time for small changes - the only downside to # this is that if an external-to-link file changes the final target wouldn't be relinked - if x.flags.contains(CfileFlag.Cached): continue + if CfileFlag.Cached in x.flags: continue # we pass each object file as if it is the project file - a .dll will be created for each such # object file in the nimcache directory, and only in the case of the main project file will # there be probably an executable (if the project is such) which will be copied out of the nimcache @@ -965,7 +952,7 @@ proc callCCompiler*(conf: ConfigRef) = prettyCmds = map(prettyCmds, proc (curr: string): string = return curr.replace("CC", "Link")) execCmdsInParallel(conf, cmds, prettyCb) # only if not cached - copy the resulting main file from the nimcache folder to its originally intended destination - if not conf.toCompile[mainFileIdx].flags.contains(CfileFlag.Cached): + if CfileFlag.Cached notin conf.toCompile[mainFileIdx].flags: let mainObjFile = getObjFilePath(conf, conf.toCompile[mainFileIdx]) var src = conf.hcrLinkTargetName(mainObjFile, true) var dst = conf.prepareToWriteOutput @@ -1011,8 +998,7 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) = f.write escapeJson(x) proc cfiles(conf: ConfigRef; f: File; buf: var string; clist: CfileList, isExternal: bool) = - var i = 0 - for it in clist: + for i, it in clist: if CfileFlag.Cached in it.flags: continue let compileCmd = getCompileCFileCmd(conf, it) if i > 0: lit ",\L" @@ -1021,7 +1007,6 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) = lit ", " str compileCmd lit "]" - inc i proc linkfiles(conf: ConfigRef; f: File; buf, objfiles: var string; clist: CfileList; llist: seq[string]) = @@ -1125,16 +1110,18 @@ proc runJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) = let data = json.parseFile(jsonFile.string) let toCompile = data["compile"] doAssert toCompile.kind == JArray - var cmds: TStringSeq = @[] - var prettyCmds: TStringSeq = @[] + var cmds: TStringSeq + var prettyCmds: TStringSeq + let prettyCb = proc (idx: int) = + if prettyCmds[idx].len > 0: echo prettyCmds[idx] + for c in toCompile: doAssert c.kind == JArray doAssert c.len >= 2 cmds.add(c[1].getStr) - prettyCmds.add displayProgressCC(conf, c[0].getStr) + prettyCmds.add displayProgressCC(conf, c[0].getStr, c[1].getStr) - let prettyCb = proc (idx: int) = callbackPrettyCmd(prettyCmds[idx]) execCmdsInParallel(conf, cmds, prettyCb) let linkCmd = data["linkcmd"] @@ -1150,10 +1137,8 @@ proc runJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) = except: let e = getCurrentException() - var msg = "\ncaught exception:n" & e.msg & "\nstacktrace:\n" & - getCurrentException().getStackTrace() & - "error evaluating JSON file: " & jsonFile.string - quit msg + quit "\ncaught exception:n" & e.msg & "\nstacktrace:\n" & e.getStackTrace() & + "error evaluating JSON file: " & jsonFile.string proc genMappingFiles(conf: ConfigRef; list: CfileList): Rope = for it in list: From f2c71527709377ddf6825f852a70eef32c2952a6 Mon Sep 17 00:00:00 2001 From: treeform Date: Wed, 11 Mar 2020 16:24:56 -0700 Subject: [PATCH 66/77] Add more JS stuff to dom.nim (#13483) * Add more JS stuff to dom.nim * Make all links to docs doc comments. * Fix minor textContent * space. * Remove Selection object. * More work on docs. * Fixing links. * Made the links be "see ". --- lib/js/dom.nim | 71 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 45 insertions(+), 26 deletions(-) diff --git a/lib/js/dom.nim b/lib/js/dom.nim index cd78f556055d1..92cc038c9b9ba 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -40,8 +40,8 @@ type onsubmit*: proc (event: Event) {.nimcall.} onunload*: proc (event: Event) {.nimcall.} - # https://developer.mozilla.org/en-US/docs/Web/Events DomEvent* {.pure.} = enum + ## see `docs`_ Abort = "abort", BeforeInput = "beforeinput", Blur = "blur", @@ -103,6 +103,8 @@ type memory*: PerformanceMemory timing*: PerformanceTiming + LocalStorage* {.importc.} = ref object + Window* = ref WindowObj WindowObj {.importc.} = object of EventTargetObj document*: Document @@ -129,6 +131,7 @@ type screen*: Screen performance*: Performance onpopstate*: proc (event: Event) + localStorage*: LocalStorage Frame* = ref FrameObj FrameObj {.importc.} = object of WindowObj @@ -165,10 +168,13 @@ type parentNode*: Node previousSibling*: Node innerHTML*: cstring + innerText*: cstring + textContent*: cstring style*: Style Document* = ref DocumentObj DocumentObj {.importc.} = object of NodeObj + activeElement*: Element alinkColor*: cstring bgColor*: cstring body*: Element @@ -211,8 +217,7 @@ type offsetLeft*: int offsetTop*: int - # https://developer.mozilla.org/en-US/docs/Web/API/ValidityState - ValidityState* = ref ValidityStateObj + ValidityState* = ref ValidityStateObj ## see `docs`_ ValidityStateObj {.importc.} = object badInput*: bool customError*: bool @@ -226,20 +231,24 @@ type valid*: bool valueMissing*: bool - # https://developer.mozilla.org/en-US/docs/Web/API/Blob - Blob* = ref BlobObj + Blob* = ref BlobObj ## see `docs`_ BlobObj {.importc.} = object of RootObj size*: int `type`*: cstring - # https://developer.mozilla.org/en-US/docs/Web/API/File - File* = ref FileObj + File* = ref FileObj ## see `docs`_ FileObj {.importc.} = object of Blob lastModified*: int name*: cstring - # https://developer.mozilla.org/en-US/docs/Web/API/HTMLInputElement - InputElement* = ref InputElementObj + TextAreaElement* = ref TextAreaElementObj ## see `docs`_ + TextAreaElementObj {.importc.} = object of Element + value*: cstring + selectionStart*, selectionEnd*: int + selectionDirection*: cstring + rows*, cols*: int + + InputElement* = ref InputElementObj ## see `docs`_ InputElementObj {.importc.} = object of Element # Properties related to the parent form formAction*: cstring @@ -321,8 +330,7 @@ type text*: cstring value*: cstring - # https://developer.mozilla.org/en-US/docs/Web/API/HTMLFormElement - FormElement* = ref FormObj + FormElement* = ref FormObj ## see `docs`_ FormObj {.importc.} = object of ElementObj acceptCharset*: cstring action*: cstring @@ -354,6 +362,7 @@ type backgroundImage*: cstring backgroundPosition*: cstring backgroundRepeat*: cstring + backgroundSize*: cstring border*: cstring borderBottom*: cstring borderBottomColor*: cstring @@ -364,6 +373,7 @@ type borderLeftColor*: cstring borderLeftStyle*: cstring borderLeftWidth*: cstring + borderRadius*: cstring borderRight*: cstring borderRightColor*: cstring borderRightStyle*: cstring @@ -375,6 +385,8 @@ type borderTopWidth*: cstring borderWidth*: cstring bottom*: cstring + boxSizing*: cstring + boxShadow*: cstring captionSide*: cstring clear*: cstring clip*: cstring @@ -409,7 +421,10 @@ type minHeight*: cstring minWidth*: cstring opacity*: cstring + outline*: cstring overflow*: cstring + overflowX*: cstring + overflowY*: cstring padding*: cstring paddingBottom*: cstring paddingLeft*: cstring @@ -419,6 +434,7 @@ type pageBreakBefore*: cstring pointerEvents*: cstring position*: cstring + resize*: cstring right*: cstring scrollbar3dLightColor*: cstring scrollbarArrowColor*: cstring @@ -447,8 +463,7 @@ type AtTarget, BubblingPhase - # https://developer.mozilla.org/en-US/docs/Web/API/Event - Event* = ref EventObj + Event* = ref EventObj ## see `docs`_ EventObj {.importc.} = object of RootObj bubbles*: bool cancelBubble*: bool @@ -461,14 +476,12 @@ type `type`*: cstring isTrusted*: bool - # https://developer.mozilla.org/en-US/docs/Web/API/UIEvent - UIEvent* = ref UIEventObj + UIEvent* = ref UIEventObj ## see `docs`_ UIEventObj {.importc.} = object of Event detail*: int64 view*: Window - # https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent - KeyboardEvent* = ref KeyboardEventObj + KeyboardEvent* = ref KeyboardEventObj ## see `docs`_ KeyboardEventObj {.importc.} = object of UIEvent altKey*, ctrlKey*, metaKey*, shiftKey*: bool code*: cstring @@ -477,8 +490,7 @@ type keyCode*: int location*: int - # https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent/key/Key_Values - KeyboardEventKey* {.pure.} = enum + KeyboardEventKey* {.pure.} = enum ## see `docs`_ # Modifier keys Alt, AltGraph, @@ -832,8 +844,7 @@ type FourthButton = 8, FifthButton = 16 - # https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent - MouseEvent* = ref MouseEventObj + MouseEvent* = ref MouseEventObj ## see `docs`_ MouseEventObj {.importc.} = object of UIEvent altKey*, ctrlKey*, metaKey*, shiftKey*: bool button*: int @@ -851,14 +862,12 @@ type File = "file", String = "string" - # https://developer.mozilla.org/en-US/docs/Web/API/DataTransferItem - DataTransferItem* = ref DataTransferItemObj + DataTransferItem* = ref DataTransferItemObj ## see `docs`_ DataTransferItemObj {.importc.} = object of RootObj kind*: cstring `type`*: cstring - # https://developer.mozilla.org/en-US/docs/Web/API/DataTransfer - DataTransfer* = ref DataTransferObj + DataTransfer* = ref DataTransferObj ## see `docs`_ DataTransferObj {.importc.} = object of RootObj dropEffect*: cstring effectAllowed*: cstring @@ -893,8 +902,8 @@ type DragStart = "dragstart", Drop = "drop" - # https://developer.mozilla.org/en-US/docs/Web/API/DragEvent DragEvent* {.importc.} = object of MouseEvent + ## see `docs`_ dataTransfer*: DataTransfer TouchList* {.importc.} = ref object of RootObj @@ -1223,6 +1232,16 @@ proc checkValidity*(e: InputElement): bool # Blob "methods" proc slice*(e: Blob, startindex: int = 0, endindex: int = e.size, contentType: cstring = "") +# Performance "methods" +proc now*(p: Performance): float + +# LocalStorage "methods" +proc getItem*(ls: LocalStorage, key: cstring): cstring +proc setItem*(ls: LocalStorage, key, value: cstring) +proc hasItem*(ls: LocalStorage, key: cstring): bool +proc clear*(ls: LocalStorage) +proc removeItem*(ls: LocalStorage, key: cstring) + {.pop.} proc setAttr*(n: Node; key, val: cstring) {.importcpp: "#.setAttribute(@)".} From 64995db4fdc8d39a9d9c7a1bfb5d8e1dd8c0f902 Mon Sep 17 00:00:00 2001 From: Kaushal Modi Date: Wed, 11 Mar 2020 19:41:45 -0400 Subject: [PATCH 67/77] unicode.split: Fix the splitting when a Rune separator is used [backport] (#13629) * unicode.split: Fix the splitting when a Rune separator is used [backport] - Fixes https://github.com/nim-lang/Nim/issues/13628 - Ref https://irclogs.nim-lang.org/11-03-2020.html#20:01:34 * unicode.split: Remove the sepLen based logic.. resulted in wrong jumps --- lib/pure/unicode.nim | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index e1d6f9f9a6a98..d166721d17473 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -933,27 +933,23 @@ proc stringHasSep(s: string, index: int, sep: Rune): bool = fastRuneAt(s, index, rune, false) return sep == rune -template splitCommon(s, sep, maxsplit: untyped, sepLen: int = -1) = +template splitCommon(s, sep, maxsplit: untyped) = ## Common code for split procedures. + let + sLen = len(s) var last = 0 splits = maxsplit - if len(s) > 0: - while last <= len(s): + if sLen > 0: + while last <= sLen: var first = last - while last < len(s) and not stringHasSep(s, last, sep): - when sep is Rune: - inc(last, sepLen) - else: - inc(last, runeLenAt(s, last)) - if splits == 0: last = len(s) + while last < sLen and not stringHasSep(s, last, sep): + inc(last, runeLenAt(s, last)) + if splits == 0: last = sLen yield s[first .. (last - 1)] if splits == 0: break dec(splits) - when sep is Rune: - inc(last, sepLen) - else: - inc(last, if last < len(s): runeLenAt(s, last) else: 1) + inc(last, if last < sLen: runeLenAt(s, last) else: 1) iterator split*(s: string, seps: openArray[Rune] = unicodeSpaces, maxsplit: int = -1): string = @@ -1037,7 +1033,7 @@ iterator split*(s: string, sep: Rune, maxsplit: int = -1): string = ## "" ## "" ## - splitCommon(s, sep, maxsplit, sep.size) + splitCommon(s, sep, maxsplit) proc split*(s: string, seps: openArray[Rune] = unicodeSpaces, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nucSplitRunes".} = @@ -1424,6 +1420,7 @@ when isMainModule: "an", "example", "", ""] doAssert s.split(maxsplit = 4) == @["", "this", "is", "an", "example "] doAssert s.split(' '.Rune, maxsplit = 1) == @["", "this is an example "] + doAssert s3.split("×".runeAt(0)) == @[":this", "is", "an:example", "", ""] block stripTests: doAssert(strip("") == "") From bbc231f8e06c18fe640fa56a927df476c2426ccb Mon Sep 17 00:00:00 2001 From: genotrance Date: Thu, 12 Mar 2020 02:53:11 -0500 Subject: [PATCH 68/77] Fix #12676 (#13634) --- compiler/scriptconfig.nim | 4 ++-- lib/system/nimscript.nim | 22 +++++++++++++++------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim index 6aad1b25ff975..220486aed1724 100644 --- a/compiler/scriptconfig.nim +++ b/compiler/scriptconfig.nim @@ -57,9 +57,9 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string; # Idea: Treat link to file as a file, but ignore link to directory to prevent # endless recursions out of the box. - cbos listFiles: + cbos listFilesImpl: listDirs(a, {pcFile, pcLinkToFile}) - cbos listDirs: + cbos listDirsImpl: listDirs(a, {pcDir}) cbos removeDir: if defined(nimsuggest) or graph.config.cmd == cmdCheck: diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index 719e92c4a505f..b8abdaa384bc7 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -26,13 +26,10 @@ template builtin = discard # We know the effects better than the compiler: {.push hint[XDeclaredButNotUsed]: off.} -proc listDirs*(dir: string): seq[string] = - ## Lists all the subdirectories (non-recursively) in the directory `dir`. - builtin -proc listFiles*(dir: string): seq[string] = - ## Lists all the files (non-recursively) in the directory `dir`. - builtin - +proc listDirsImpl(dir: string): seq[string] {. + tags: [ReadIOEffect], raises: [OSError].} = builtin +proc listFilesImpl(dir: string): seq[string] {. + tags: [ReadIOEffect], raises: [OSError].} = builtin proc removeDir(dir: string) {. tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin proc removeFile(dir: string) {. @@ -47,6 +44,7 @@ proc copyDir(src, dest: string) {. tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin proc createDir(dir: string) {.tags: [WriteIOEffect], raises: [OSError].} = builtin + proc getError: string = builtin proc setCurrentDir(dir: string) = builtin proc getCurrentDir*(): string = @@ -196,6 +194,16 @@ template log(msg: string, body: untyped) = if mode != ScriptMode.WhatIf: body +proc listDirs*(dir: string): seq[string] = + ## Lists all the subdirectories (non-recursively) in the directory `dir`. + result = listDirsImpl(dir) + checkOsError() + +proc listFiles*(dir: string): seq[string] = + ## Lists all the files (non-recursively) in the directory `dir`. + result = listFilesImpl(dir) + checkOsError() + proc rmDir*(dir: string) {.raises: [OSError].} = ## Removes the directory `dir`. log "rmDir: " & dir: From d84c4bba9b5b20386fbf88b1695de6071a48da7d Mon Sep 17 00:00:00 2001 From: Clyybber Date: Thu, 12 Mar 2020 11:01:03 +0100 Subject: [PATCH 69/77] Fix #13633 --- compiler/extccomp.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 34552e6f3bbf1..dfc13edf56024 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -998,10 +998,11 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) = f.write escapeJson(x) proc cfiles(conf: ConfigRef; f: File; buf: var string; clist: CfileList, isExternal: bool) = + var comma = false for i, it in clist: if CfileFlag.Cached in it.flags: continue let compileCmd = getCompileCFileCmd(conf, it) - if i > 0: lit ",\L" + if comma: lit ",\L"; comma = false lit "[" str it.cname.string lit ", " From 2bb0ada79740b0961fd06ffd477c41dfef531544 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Thu, 12 Mar 2020 11:20:08 +0100 Subject: [PATCH 70/77] Amend fix for #13633 (#13636) --- compiler/extccomp.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index dfc13edf56024..2e1846424e898 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -1002,7 +1002,7 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) = for i, it in clist: if CfileFlag.Cached in it.flags: continue let compileCmd = getCompileCFileCmd(conf, it) - if comma: lit ",\L"; comma = false + if comma: lit ",\L" else: comma = true lit "[" str it.cname.string lit ", " From 60a3e036f6aaecabdedfab1103bb859053359f8d Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 12 Mar 2020 03:39:10 -0700 Subject: [PATCH 71/77] fix #13633 fix koch boot crashing regression (#13635) --- compiler/extccomp.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 2e1846424e898..ee921f4e1604c 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -1138,7 +1138,7 @@ proc runJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) = except: let e = getCurrentException() - quit "\ncaught exception:n" & e.msg & "\nstacktrace:\n" & e.getStackTrace() & + quit "\ncaught exception:\n" & e.msg & "\nstacktrace:\n" & e.getStackTrace() & "error evaluating JSON file: " & jsonFile.string proc genMappingFiles(conf: ConfigRef; list: CfileList): Rope = From f0cd50f9c48ee6bb585cd85615e5844a8082a8c1 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Thu, 12 Mar 2020 16:15:46 +0100 Subject: [PATCH 72/77] Change order of forwarded koch boot command line options, so as to be able to overwrite the nimcache location (#13637) --- koch.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/koch.nim b/koch.nim index 982517656990c..c2f28f2e034fa 100644 --- a/koch.nim +++ b/koch.nim @@ -316,10 +316,10 @@ proc boot(args: string) = # in order to use less memory, we split the build into two steps: # --compileOnly produces a $project.json file and does not run GCC/Clang. # jsonbuild then uses the $project.json file to build the Nim binary. - exec "$# $# $# $# --nimcache:$# --compileOnly compiler" / "nim.nim" % - [nimi, bootOptions, extraOption, args, smartNimcache] - exec "$# jsonscript $# --nimcache:$# compiler" / "nim.nim" % - [nimi, args, smartNimcache] + exec "$# $# $# --nimcache:$# $# --compileOnly compiler" / "nim.nim" % + [nimi, bootOptions, extraOption, smartNimcache, args] + exec "$# jsonscript --nimcache:$# $# compiler" / "nim.nim" % + [nimi, smartNimcache, args] if sameFileContent(output, i.thVersion): copyExe(output, finalDest) From 14b2354b7da36041ca046e60e833b5be9a04f1e4 Mon Sep 17 00:00:00 2001 From: Miran Date: Thu, 12 Mar 2020 20:07:02 +0100 Subject: [PATCH 73/77] rename `lenTuple` and `lenVarargs` (#13639) * rename 'lenTuple' to 'tupleLen' Rationale: `lenTuple` is a tuple consisting of lengths (e.g. `(1, 5, 0)`), `tupleLen` is a length of a tuple (e.g. `tupleLen((1, 5, 0) == 3`) * rename 'lenVarargs' to 'varargsLen' The same rationale as a previous commit. Consistency. --- changelog.md | 2 +- compiler/semmagic.nim | 2 +- lib/pure/typetraits.nim | 6 ++-- lib/system.nim | 6 ++-- tests/metatype/ttypetraits.nim | 30 +++++++++---------- .../{tlenvarargs.nim => tvarargslen.nim} | 19 ++++++------ 6 files changed, 33 insertions(+), 32 deletions(-) rename tests/system/{tlenvarargs.nim => tvarargslen.nim} (84%) diff --git a/changelog.md b/changelog.md index 3cb9547f34bfb..0fb99c6f4e1c7 100644 --- a/changelog.md +++ b/changelog.md @@ -66,7 +66,7 @@ - Added `sugar.collect` that does comprehension for seq/set/table collections. - Added `sugar.capture` for capturing some local loop variables when creating a closure. This is an enhanced version of `closureScope`. -- Added `typetraits.lenTuple` to get number of elements of a tuple/type tuple, +- Added `typetraits.tupleLen` to get number of elements of a tuple/type tuple, and `typetraits.get` to get the ith element of a type tuple. - Added `typetraits.genericParams` to return a tuple of generic params from a generic instantiation - Added `os.normalizePathEnd` for additional path sanitization. diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index ffdad5628a107..af80b3ca34b43 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -187,7 +187,7 @@ proc evalTypeTrait(c: PContext; traitCall: PNode, operand: PType, context: PSym) var operand = operand.skipTypes({tyGenericInst}) let cond = operand.kind == tyTuple and operand.n != nil result = newIntNodeT(toInt128(ord(cond)), traitCall, c.graph) - of "lenTuple": + of "tupleLen": var operand = operand.skipTypes({tyGenericInst}) assert operand.kind == tyTuple, $operand.kind result = newIntNodeT(toInt128(operand.len), traitCall, c.graph) diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index 750c6233dedf9..66b94e23846e7 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -72,13 +72,13 @@ proc distinctBase*(T: typedesc): typedesc {.magic: "TypeTrait".} ## compile time error otherwise -proc lenTuple*(T: typedesc[tuple]): int {.magic: "TypeTrait", since: (1, 1).} +proc tupleLen*(T: typedesc[tuple]): int {.magic: "TypeTrait", since: (1, 1).} ## Return number of elements of `T` since (1, 1): - template lenTuple*(t: tuple): int = + template tupleLen*(t: tuple): int = ## Return number of elements of `t` - lenTuple(type(t)) + tupleLen(type(t)) since (1, 1): template get*(T: typedesc[tuple], i: static int): untyped = diff --git a/lib/system.nim b/lib/system.nim index 51ff3af400341..5dff62806ee78 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2682,10 +2682,10 @@ when defined(nimV2): import system/repr_v2 export repr_v2 -macro lenVarargs*(x: varargs[untyped]): int {.since: (1, 1).} = +macro varargsLen*(x: varargs[untyped]): int {.since: (1, 1).} = ## returns number of variadic arguments in `x` - proc lenVarargsImpl(x: NimNode): NimNode {.magic: "LengthOpenArray", noSideEffect.} - lenVarargsImpl(x) + proc varargsLenImpl(x: NimNode): NimNode {.magic: "LengthOpenArray", noSideEffect.} + varargsLenImpl(x) when false: template eval*(blk: typed): typed = diff --git a/tests/metatype/ttypetraits.nim b/tests/metatype/ttypetraits.nim index 7badf8317185d..db8c8e5de58bd 100644 --- a/tests/metatype/ttypetraits.nim +++ b/tests/metatype/ttypetraits.nim @@ -92,40 +92,40 @@ block distinctBase: doAssert($distinctBase(typeof(b2)) == "string") doAssert($distinctBase(typeof(c2)) == "int") -block: # lenTuple - doAssert not compiles(lenTuple(int)) +block: # tupleLen + doAssert not compiles(tupleLen(int)) type MyTupleType = (int,float,string) - static: doAssert MyTupleType.lenTuple == 3 + static: doAssert MyTupleType.tupleLen == 3 type MyGenericTuple[T] = (T,int,float) MyGenericAlias = MyGenericTuple[string] - static: doAssert MyGenericAlias.lenTuple == 3 + static: doAssert MyGenericAlias.tupleLen == 3 type MyGenericTuple2[T,U] = (T,U,string) MyGenericTuple2Alias[T] = MyGenericTuple2[T,int] MyGenericTuple2Alias2 = MyGenericTuple2Alias[float] - static: doAssert MyGenericTuple2Alias2.lenTuple == 3 + static: doAssert MyGenericTuple2Alias2.tupleLen == 3 - static: doAssert (int, float).lenTuple == 2 - static: doAssert (1, ).lenTuple == 1 - static: doAssert ().lenTuple == 0 + static: doAssert (int, float).tupleLen == 2 + static: doAssert (1, ).tupleLen == 1 + static: doAssert ().tupleLen == 0 let x = (1,2,) - doAssert x.lenTuple == 2 - doAssert ().lenTuple == 0 - doAssert (1,).lenTuple == 1 - doAssert (int,).lenTuple == 1 - doAssert type(x).lenTuple == 2 - doAssert type(x).default.lenTuple == 2 + doAssert x.tupleLen == 2 + doAssert ().tupleLen == 0 + doAssert (1,).tupleLen == 1 + doAssert (int,).tupleLen == 1 + doAssert type(x).tupleLen == 2 + doAssert type(x).default.tupleLen == 2 type T1 = (int,float) type T2 = T1 - doAssert T2.lenTuple == 2 + doAssert T2.tupleLen == 2 block genericParams: type Foo[T1, T2]=object diff --git a/tests/system/tlenvarargs.nim b/tests/system/tvarargslen.nim similarity index 84% rename from tests/system/tlenvarargs.nim rename to tests/system/tvarargslen.nim index ca12c8171b652..a129aa5c2f239 100644 --- a/tests/system/tlenvarargs.nim +++ b/tests/system/tvarargslen.nim @@ -1,9 +1,9 @@ discard """ output: ''' -tlenvarargs.nim:35:9 (1, 2) -tlenvarargs.nim:36:9 12 -tlenvarargs.nim:37:9 1 -tlenvarargs.nim:38:8 +tvarargslen.nim:35:9 (1, 2) +tvarargslen.nim:36:9 12 +tvarargslen.nim:37:9 1 +tvarargslen.nim:38:8 done ''' """ @@ -14,19 +14,19 @@ template myecho*(a: varargs[untyped]) = ## on macros.nim) so can be used in more contexts const info = instantiationInfo(-1, false) const loc = info.filename & ":" & $info.line & ":" & $info.column & " " - when lenVarargs(a) > 0: + when varargsLen(a) > 0: echo(loc, a) else: echo(loc) template fun*(a: varargs[untyped]): untyped = - lenVarargs(a) + varargsLen(a) template fun2*(a: varargs[typed]): untyped = - a.lenVarargs + a.varargsLen template fun3*(a: varargs[int]): untyped = - a.lenVarargs + a.varargsLen template fun4*(a: varargs[untyped]): untyped = len(a) @@ -49,11 +49,12 @@ proc main()= doAssert fun3(10) == 1 doAssert fun3(10, 11) == 2 - ## shows why `lenVarargs` can't be named `len` + ## shows why `varargsLen` can't be named `len` doAssert fun4("abcdef") == len("abcdef") ## workaround for BUG:D20191218T171447 whereby if testament expected output ends ## in space, testament strips it from expected output but not actual output, ## which leads to a mismatch when running test via megatest echo "done" + main() From a6682de0045468ae1d15afbd2fd5378960e15eb7 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 12 Mar 2020 23:44:33 +0100 Subject: [PATCH 74/77] catchable defects (#13626) * allow defects to be caught even for --exceptions:goto (WIP) * implemented the new --panics:on|off switch; refs https://github.com/nim-lang/RFCs/issues/180 * new implementation for integer overflow checking * produce a warning if a user-defined exception type inherits from Exception directly * applied Timothee's suggestions; improved the documentation and replace the term 'checked runtime check' by 'panic' * fixes #13627 * don't inherit from Exception directly --- changelog.md | 8 ++ compiler/ast.nim | 4 +- compiler/ccgcalls.nim | 16 +++- compiler/ccgexprs.nim | 126 +++++++++++++----------- compiler/ccgstmts.nim | 63 ++++++------ compiler/cgen.nim | 1 + compiler/commands.nim | 4 + compiler/condsyms.nim | 1 + compiler/enumtostr.nim | 2 + compiler/lineinfos.nim | 7 +- compiler/modulegraphs.nim | 1 + compiler/options.nim | 11 ++- compiler/semstmts.nim | 2 +- compiler/semtypes.nim | 3 + compiler/sizealignoffsetimpl.nim | 2 +- doc/advopt.txt | 1 + doc/manual.rst | 34 ++++--- koch.nim | 2 +- lib/nimbase.h | 22 +++++ lib/system.nim | 5 +- lib/system/arithm.nim | 4 +- lib/system/chcks.nim | 18 +++- lib/system/fatal.nim | 19 +--- lib/system/gc.nim | 8 +- lib/system/gc_common.nim | 2 +- lib/system/gc_ms.nim | 6 +- lib/system/gc_regions.nim | 7 +- lib/system/integerops.nim | 132 ++++++++++++++++++++++++++ lib/system/mm/boehm.nim | 4 +- lib/system/mm/none.nim | 4 +- lib/system/mmdisp.nim | 6 +- tests/assert/tassert_c.nim | 2 +- tests/destructor/tgotoexceptions7.nim | 52 ++++++++++ tests/range/tsubrange2.nim | 2 +- tests/range/tsubrange3.nim | 2 +- 35 files changed, 441 insertions(+), 142 deletions(-) create mode 100644 lib/system/integerops.nim create mode 100644 tests/destructor/tgotoexceptions7.nim diff --git a/changelog.md b/changelog.md index 0fb99c6f4e1c7..fa753e7c472c2 100644 --- a/changelog.md +++ b/changelog.md @@ -161,6 +161,14 @@ echo f - The compiler now inferes "sink parameters". To disable this for a specific routine, annotate it with `.nosinks`. To disable it for a section of code, use `{.push sinkInference: off.}`...`{.pop.}`. +- The compiler now supports a new switch `--panics:on` that turns runtime + errors like `IndexError` or `OverflowError` into fatal errors that **cannot** + be caught via Nim's `try` statement. `--panics:on` can improve the + runtime efficiency and code size of your program significantly. +- The compiler now warns about inheriting directly from `system.Exception` as + this is **very bad** style. You should inherit from `ValueError`, `IOError`, + `OSError` or from a different specific exception type that inherits from + `CatchableError` and cannot be confused with a `Defect`. ## Bugfixes diff --git a/compiler/ast.nim b/compiler/ast.nim index 62c301a43adb9..55ad2ba4bccc9 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -229,7 +229,7 @@ type TNodeKinds* = set[TNodeKind] type - TSymFlag* = enum # 41 flags! + TSymFlag* = enum # 42 flags! sfUsed, # read access of sym (for warnings) or simply used sfExported, # symbol is exported from module sfFromGeneric, # symbol is instantiation of a generic; this is needed @@ -289,6 +289,8 @@ type sfTemplateParam # symbol is a template parameter sfCursor # variable/field is a cursor, see RFC 177 for details sfInjectDestructors # whether the proc needs the 'injectdestructors' transformation + sfAlwaysReturn # proc can never raise an exception, not even OverflowError + # or out-of-memory TSymFlags* = set[TSymFlag] diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index da5dd9b766d40..3e2cf22718d7f 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -557,6 +557,18 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) = pl.add(~"];$n") line(p, cpsStmts, pl) +proc canRaiseDisp(p: BProc; n: PNode): bool = + # we assume things like sysFatal cannot raise themselves + if n.kind == nkSym and sfAlwaysReturn in n.sym.flags: + result = false + elif optPanics in p.config.globalOptions or + (n.kind == nkSym and sfSystemModule in getModule(n.sym).flags): + # we know we can be strict: + result = canRaise(n) + else: + # we have to be *very* conservative: + result = canRaiseConservative(n) + proc genCall(p: BProc, e: PNode, d: var TLoc) = if e[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure: genClosureCall(p, nil, e, d) @@ -567,7 +579,7 @@ proc genCall(p: BProc, e: PNode, d: var TLoc) = else: genPrefixCall(p, nil, e, d) postStmtActions(p) - if p.config.exc == excGoto and canRaise(e[0]): + if p.config.exc == excGoto and canRaiseDisp(p, e[0]): raiseExit(p) proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) = @@ -580,5 +592,5 @@ proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) = else: genPrefixCall(p, le, ri, d) postStmtActions(p) - if p.config.exc == excGoto and canRaise(ri[0]): + if p.config.exc == excGoto and canRaiseDisp(p, ri[0]): raiseExit(p) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 1bed4bc6cc5eb..4aad1a4f8a55a 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -506,23 +506,25 @@ template binaryArithOverflowRaw(p: BProc, t: PType, a, b: TLoc; else: getTypeDesc(p.module, t) var result = getTempName(p.module) linefmt(p, cpsLocals, "$1 $2;$n", [storage, result]) - lineCg(p, cpsStmts, "$1 = #$2($3, $4);$n", [result, cpname, rdCharLoc(a), rdCharLoc(b)]) + lineCg(p, cpsStmts, "if (#$2($3, $4, &$1)) { #raiseOverflow(); $5};$n", + [result, cpname, rdCharLoc(a), rdCharLoc(b), raiseInstr(p)]) if size < p.config.target.intSize or t.kind in {tyRange, tyEnum}: - linefmt(p, cpsStmts, "if ($1 < $2 || $1 > $3) #raiseOverflow();$n", - [result, intLiteral(firstOrd(p.config, t)), intLiteral(lastOrd(p.config, t))]) + linefmt(p, cpsStmts, "if ($1 < $2 || $1 > $3){ #raiseOverflow(); $4}$n", + [result, intLiteral(firstOrd(p.config, t)), intLiteral(lastOrd(p.config, t)), + raiseInstr(p)]) result proc binaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) = const prc: array[mAddI..mPred, string] = [ - "addInt", "subInt", - "mulInt", "divInt", "modInt", - "addInt", "subInt" + "nimAddInt", "nimSubInt", + "nimMulInt", "nimDivInt", "nimModInt", + "nimAddInt", "nimSubInt" ] prc64: array[mAddI..mPred, string] = [ - "addInt64", "subInt64", - "mulInt64", "divInt64", "modInt64", - "addInt64", "subInt64" + "nimAddInt64", "nimSubInt64", + "nimMulInt64", "nimDivInt64", "nimModInt64", + "nimAddInt64", "nimSubInt64" ] opr: array[mAddI..mPred, string] = ["+", "-", "*", "/", "%", "+", "-"] var a, b: TLoc @@ -537,6 +539,12 @@ proc binaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) = let res = "($1)($2 $3 $4)" % [getTypeDesc(p.module, e.typ), rdLoc(a), rope(opr[m]), rdLoc(b)] putIntoDest(p, d, e, res) else: + # we handle div by zero here so that we know that the compilerproc's + # result is only for overflows. + if m in {mDivI, mModI}: + linefmt(p, cpsStmts, "if ($1 == 0){ #raiseDivByZero(); $2}$n", + [rdLoc(b), raiseInstr(p)]) + let res = binaryArithOverflowRaw(p, t, a, b, if t.kind == tyInt64: prc64[m] else: prc[m]) putIntoDest(p, d, e, "($#)($#)" % [getTypeDesc(p.module, e.typ), res]) @@ -549,8 +557,8 @@ proc unaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) = initLocExpr(p, e[1], a) t = skipTypes(e.typ, abstractRange) if optOverflowCheck in p.options: - linefmt(p, cpsStmts, "if ($1 == $2) #raiseOverflow();$n", - [rdLoc(a), intLiteral(firstOrd(p.config, t))]) + linefmt(p, cpsStmts, "if ($1 == $2){ #raiseOverflow(); $3}$n", + [rdLoc(a), intLiteral(firstOrd(p.config, t)), raiseInstr(p)]) case m of mUnaryMinusI: putIntoDest(p, d, e, "((NI$2)-($1))" % [rdLoc(a), rope(getSize(p.config, t) * 8)]) @@ -817,12 +825,12 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) = let strLit = genStringLiteral(p.module, newStrNode(nkStrLit, msg)) if op.magic == mNot: linefmt(p, cpsStmts, - "if ($1) #raiseFieldError($2);$n", - [rdLoc(test), strLit]) + "if ($1){ #raiseFieldError($2); $3}$n", + [rdLoc(test), strLit, raiseInstr(p)]) else: linefmt(p, cpsStmts, - "if (!($1)) #raiseFieldError($2);$n", - [rdLoc(test), strLit]) + "if (!($1)){ #raiseFieldError($2); $3}$n", + [rdLoc(test), strLit, raiseInstr(p)]) proc genCheckedRecordField(p: BProc, e: PNode, d: var TLoc) = if optFieldCheck in p.options: @@ -861,11 +869,11 @@ proc genArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) = # semantic pass has already checked for const index expressions if firstOrd(p.config, ty) == 0: if (firstOrd(p.config, b.t) < firstOrd(p.config, ty)) or (lastOrd(p.config, b.t) > lastOrd(p.config, ty)): - linefmt(p, cpsStmts, "if ((NU)($1) > (NU)($2)) #raiseIndexError2($1, $2);$n", - [rdCharLoc(b), intLiteral(lastOrd(p.config, ty))]) + linefmt(p, cpsStmts, "if ((NU)($1) > (NU)($2)){ #raiseIndexError2($1, $2); $3}$n", + [rdCharLoc(b), intLiteral(lastOrd(p.config, ty)), raiseInstr(p)]) else: - linefmt(p, cpsStmts, "if ($1 < $2 || $1 > $3) #raiseIndexError3($1, $2, $3);$n", - [rdCharLoc(b), first, intLiteral(lastOrd(p.config, ty))]) + linefmt(p, cpsStmts, "if ($1 < $2 || $1 > $3){ #raiseIndexError3($1, $2, $3); $4}$n", + [rdCharLoc(b), first, intLiteral(lastOrd(p.config, ty)), raiseInstr(p)]) else: let idx = getOrdValue(y) if idx < firstOrd(p.config, ty) or idx > lastOrd(p.config, ty): @@ -888,19 +896,19 @@ proc genBoundsCheck(p: BProc; arr, a, b: TLoc) = of tyOpenArray, tyVarargs: linefmt(p, cpsStmts, "if ($2-$1 != -1 && " & - "((NU)($1) >= (NU)($3Len_0) || (NU)($2) >= (NU)($3Len_0))) #raiseIndexError();$n", - [rdLoc(a), rdLoc(b), rdLoc(arr)]) + "((NU)($1) >= (NU)($3Len_0) || (NU)($2) >= (NU)($3Len_0))){ #raiseIndexError(); $4}$n", + [rdLoc(a), rdLoc(b), rdLoc(arr), raiseInstr(p)]) of tyArray: let first = intLiteral(firstOrd(p.config, ty)) linefmt(p, cpsStmts, "if ($2-$1 != -1 && " & - "($2-$1 < -1 || $1 < $3 || $1 > $4 || $2 < $3 || $2 > $4)) #raiseIndexError();$n", - [rdCharLoc(a), rdCharLoc(b), first, intLiteral(lastOrd(p.config, ty))]) + "($2-$1 < -1 || $1 < $3 || $1 > $4 || $2 < $3 || $2 > $4)){ #raiseIndexError(); $5}$n", + [rdCharLoc(a), rdCharLoc(b), first, intLiteral(lastOrd(p.config, ty)), raiseInstr(p)]) of tySequence, tyString: linefmt(p, cpsStmts, "if ($2-$1 != -1 && " & - "((NU)($1) >= (NU)$3 || (NU)($2) >= (NU)$3)) #raiseIndexError();$n", - [rdLoc(a), rdLoc(b), lenExpr(p, arr)]) + "((NU)($1) >= (NU)$3 || (NU)($2) >= (NU)$3)){ #raiseIndexError(); $4}$n", + [rdLoc(a), rdLoc(b), lenExpr(p, arr), raiseInstr(p)]) else: discard proc genOpenArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) = @@ -908,8 +916,8 @@ proc genOpenArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) = initLocExpr(p, x, a) initLocExpr(p, y, b) # emit range check: if optBoundsCheck in p.options: - linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)($2Len_0)) #raiseIndexError2($1,$2Len_0-1);$n", - [rdLoc(b), rdLoc(a)]) # BUGFIX: ``>=`` and not ``>``! + linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)($2Len_0)){ #raiseIndexError2($1,$2Len_0-1); $3}$n", + [rdLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``! inheritLocation(d, a) putIntoDest(p, d, n, ropecg(p.module, "$1[$2]", [rdLoc(a), rdCharLoc(b)]), a.storage) @@ -924,12 +932,12 @@ proc genSeqElem(p: BProc, n, x, y: PNode, d: var TLoc) = if optBoundsCheck in p.options: if ty.kind == tyString and (not defined(nimNoZeroTerminator) or optLaxStrings in p.options): linefmt(p, cpsStmts, - "if ((NU)($1) > (NU)$2) #raiseIndexError2($1,$2);$n", - [rdLoc(b), lenExpr(p, a)]) + "if ((NU)($1) > (NU)$2){ #raiseIndexError2($1,$2); $3}$n", + [rdLoc(b), lenExpr(p, a), raiseInstr(p)]) else: linefmt(p, cpsStmts, - "if ((NU)($1) >= (NU)$2) #raiseIndexError2($1,$2-1);$n", - [rdLoc(b), lenExpr(p, a)]) + "if ((NU)($1) >= (NU)$2){ #raiseIndexError2($1,$2-1); $3}$n", + [rdLoc(b), lenExpr(p, a), raiseInstr(p)]) if d.k == locNone: d.storage = OnHeap if skipTypes(a.t, abstractVar).kind in {tyRef, tyPtr}: a.r = ropecg(p.module, "(*$1)", [a.r]) @@ -1938,21 +1946,33 @@ proc genCast(p: BProc, e: PNode, d: var TLoc) = # C code; plus it's the right thing to do for closures: genSomeCast(p, e, d) -proc genRangeChck(p: BProc, n: PNode, d: var TLoc, magic: string) = +proc genRangeChck(p: BProc, n: PNode, d: var TLoc) = var a: TLoc var dest = skipTypes(n.typ, abstractVar) + initLocExpr(p, n[0], a) if optRangeCheck notin p.options or (dest.kind in {tyUInt..tyUInt64} and checkUnsignedConversions notin p.config.legacyFeatures): - initLocExpr(p, n[0], a) - putIntoDest(p, d, n, "(($1) ($2))" % - [getTypeDesc(p.module, dest), rdCharLoc(a)], a.storage) + discard "no need to generate a check because it was disabled" else: - let mm = if dest.kind in {tyUInt32, tyUInt64, tyUInt}: "chckRangeU" else: magic - initLocExpr(p, n[0], a) - putIntoDest(p, d, lodeTyp dest, ropecg(p.module, "(($1)#$5($2, $3, $4))", [ - getTypeDesc(p.module, dest), rdCharLoc(a), - genLiteral(p, n[1], dest), genLiteral(p, n[2], dest), - mm]), a.storage) + let raiser = + case skipTypes(n.typ, abstractVarRange).kind + of tyUInt..tyUInt64, tyChar: "raiseRangeErrorU" + of tyFloat..tyFloat128: "raiseRangeErrorF" + else: "raiseRangeErrorI" + discard cgsym(p.module, raiser) + # This seems to be bug-compatible with Nim version 1 but what we + # should really do here is to check if uint64Value < high(int) + let boundaryCast = + if n[0].typ.skipTypes(abstractVarRange).kind in {tyUInt, tyUInt32, tyUInt64}: + "(NI64)" + else: + "" + # emit range check: + linefmt(p, cpsStmts, "if ($6($1) < $2 || $6($1) > $3){ $4($1, $2, $3); $5}$n", + [rdCharLoc(a), genLiteral(p, n[1], dest), genLiteral(p, n[2], dest), + raiser, raiseInstr(p), boundaryCast]) + putIntoDest(p, d, n, "(($1) ($2))" % + [getTypeDesc(p.module, dest), rdCharLoc(a)], a.storage) proc genConv(p: BProc, e: PNode, d: var TLoc) = let destType = e.typ.skipTypes({tyVar, tyLent, tyGenericInst, tyAlias, tySink}) @@ -2004,9 +2024,9 @@ proc binaryFloatArith(p: BProc, e: PNode, d: var TLoc, m: TMagic) = [opr[m], rdLoc(a), rdLoc(b), getSimpleTypeDesc(p.module, e[1].typ)])) if optNaNCheck in p.options: - linefmt(p, cpsStmts, "#nanCheck($1);$n", [rdLoc(d)]) + linefmt(p, cpsStmts, "if ($1 != $1){ #raiseFloatInvalidOp(); $2}$n", [rdLoc(d), raiseInstr(p)]) if optInfCheck in p.options: - linefmt(p, cpsStmts, "#infCheck($1);$n", [rdLoc(d)]) + linefmt(p, cpsStmts, "if ($1 != 0.0 && $1*0.5 == $1) { #raiseFloatOverflow($1); $2}$n", [rdLoc(d), raiseInstr(p)]) else: binaryArith(p, e, d, m) @@ -2122,10 +2142,8 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mSwap: genSwap(p, e, d) of mInc, mDec: const opr: array[mInc..mDec, string] = ["+=", "-="] - const fun64: array[mInc..mDec, string] = ["addInt64", - "subInt64"] - const fun: array[mInc..mDec, string] = ["addInt", - "subInt"] + const fun64: array[mInc..mDec, string] = ["nimAddInt64", "nimSubInt64"] + const fun: array[mInc..mDec, string] = ["nimAddInt","nimSubInt"] let underlying = skipTypes(e[1].typ, {tyGenericInst, tyAlias, tySink, tyVar, tyLent, tyRange}) if optOverflowCheck notin p.options or underlying.kind in {tyUInt..tyUInt64}: binaryStmt(p, e, d, opr[op]) @@ -2432,11 +2450,11 @@ proc upConv(p: BProc, n: PNode, d: var TLoc) = else: genTypeInfo(p.module, dest, n.info) if nilCheck != nil: - linefmt(p, cpsStmts, "if ($1) #chckObj($2, $3);$n", - [nilCheck, r, checkFor]) + linefmt(p, cpsStmts, "if ($1 && !#isObj($2, $3)){ #raiseObjectConversionError(); $4}$n", + [nilCheck, r, checkFor, raiseInstr(p)]) else: - linefmt(p, cpsStmts, "#chckObj($1, $2);$n", - [r, checkFor]) + linefmt(p, cpsStmts, "if (!#isObj($1, $2)){ #raiseObjectConversionError(); $3}$n", + [r, checkFor, raiseInstr(p)]) if n[0].typ.kind != tyObject: putIntoDest(p, d, n, "(($1) ($2))" % [getTypeDesc(p.module, n.typ), rdLoc(a)], a.storage) @@ -2629,9 +2647,9 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = expr(p, n[1][0], d) of nkObjDownConv: downConv(p, n, d) of nkObjUpConv: upConv(p, n, d) - of nkChckRangeF: genRangeChck(p, n, d, "chckRangeF") - of nkChckRange64: genRangeChck(p, n, d, "chckRange64") - of nkChckRange: genRangeChck(p, n, d, "chckRange") + of nkChckRangeF: genRangeChck(p, n, d) + of nkChckRange64: genRangeChck(p, n, d) + of nkChckRange: genRangeChck(p, n, d) of nkStringToCString: convStrToCStr(p, n, d) of nkCStringToString: convCStrToStr(p, n, d) of nkLambdaKinds: diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index 9c067a339e81e..1c63ebaeac83e 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -706,6 +706,22 @@ proc finallyActions(p: BProc) = if finallyBlock != nil: genSimpleBlock(p, finallyBlock[0]) +proc raiseInstr(p: BProc): Rope = + if p.config.exc == excGoto: + let L = p.nestedTryStmts.len + if L == 0: + p.flags.incl beforeRetNeeded + # easy case, simply goto 'ret': + result = ropecg(p.module, "goto BeforeRet_;$n", []) + else: + # raise inside an 'except' must go to the finally block, + # raise outside an 'except' block must go to the 'except' list. + result = ropecg(p.module, "goto LA$1_;$n", + [p.nestedTryStmts[L-1].label]) + # + ord(p.nestedTryStmts[L-1].inExcept)]) + else: + result = nil + proc genRaiseStmt(p: BProc, t: PNode) = if p.config.exc == excCpp: discard cgsym(p.module, "popCurrentExceptionEx") @@ -733,18 +749,9 @@ proc genRaiseStmt(p: BProc, t: PNode) = line(p, cpsStmts, ~"throw;$n") else: linefmt(p, cpsStmts, "#reraiseException();$n", []) - if p.config.exc == excGoto: - let L = p.nestedTryStmts.len - if L == 0: - p.flags.incl beforeRetNeeded - # easy case, simply goto 'ret': - lineCg(p, cpsStmts, "goto BeforeRet_;$n", []) - else: - # raise inside an 'except' must go to the finally block, - # raise outside an 'except' block must go to the 'except' list. - lineCg(p, cpsStmts, "goto LA$1_;$n", - [p.nestedTryStmts[L-1].label]) - # + ord(p.nestedTryStmts[L-1].inExcept)]) + let gotoInstr = raiseInstr(p) + if gotoInstr != nil: + line(p, cpsStmts, gotoInstr) template genCaseGenericBranch(p: BProc, b: PNode, e: TLoc, rangeFormat, eqFormat: FormatStr, labl: TLabel) = @@ -1009,14 +1016,14 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) = genSimpleBlock(p, t[^1][0]) -proc bodyCanRaise(n: PNode): bool = +proc bodyCanRaise(p: BProc; n: PNode): bool = case n.kind of nkCallKinds: - result = canRaise(n[0]) + result = canRaiseDisp(p, n[0]) if not result: # also check the arguments: for i in 1 ..< n.len: - if bodyCanRaise(n[i]): return true + if bodyCanRaise(p, n[i]): return true of nkRaiseStmt: result = true of nkTypeSection, nkProcDef, nkConverterDef, nkMethodDef, nkIteratorDef, @@ -1024,22 +1031,24 @@ proc bodyCanRaise(n: PNode): bool = result = false else: for i in 0 ..< safeLen(n): - if bodyCanRaise(n[i]): return true + if bodyCanRaise(p, n[i]): return true result = false proc genTryGoto(p: BProc; t: PNode; d: var TLoc) = - if not bodyCanRaise(t): + let fin = if t[^1].kind == nkFinally: t[^1] else: nil + inc p.labels + let lab = p.labels + p.nestedTryStmts.add((fin, false, Natural lab)) + + if not bodyCanRaise(p, t): # optimize away the 'try' block: expr(p, t[0], d) - if t.len > 1 and t[^1].kind == nkFinally: - genStmts(p, t[^1][0]) + linefmt(p, cpsStmts, "LA$1_: ;$n", [lab]) + if fin != nil: + genStmts(p, fin[0]) + discard pop(p.nestedTryStmts) return - let fin = if t[^1].kind == nkFinally: t[^1] else: nil - inc p.labels, 2 - let lab = p.labels-1 - p.nestedTryStmts.add((fin, false, Natural lab)) - p.flags.incl nimErrorFlagAccessed p.procSec(cpsLocals).add(ropecg(p.module, "NI oldNimErr$1_;$n", [lab])) linefmt(p, cpsStmts, "oldNimErr$1_ = *nimErr_; *nimErr_ = 0;;$n", [lab]) @@ -1099,7 +1108,7 @@ proc genTryGoto(p: BProc; t: PNode; d: var TLoc) = #linefmt(p, cpsStmts, "LA$1_:;$n", [lab+1]) if i < t.len and t[i].kind == nkFinally: startBlock(p) - if not bodyCanRaise(t[i][0]): + if not bodyCanRaise(p, t[i][0]): # this is an important optimization; most destroy blocks are detected not to raise an # exception and so we help the C optimizer by not mutating nimErr_ pointlessly: genStmts(p, t[i][0]) @@ -1380,8 +1389,8 @@ proc asgnFieldDiscriminant(p: BProc, e: PNode) = genCaseObjDiscMapping(p, e[0], t, field, oldVal) genCaseObjDiscMapping(p, e[1], t, field, newVal) lineCg(p, cpsStmts, - "#nimFieldDiscriminantCheckV2($1, $2);$n", - [rdLoc(oldVal), rdLoc(newVal)]) + "if ($1 != $2) { #raiseObjectCaseTransition(); $3}$n", + [rdLoc(oldVal), rdLoc(newVal), raiseInstr(p)]) else: genDiscriminantCheck(p, a, tmp, dotExpr[0].typ, field) genAssignment(p, a, tmp, {}) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 57424ec9548c8..06507b6e6ea05 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -276,6 +276,7 @@ proc postStmtActions(p: BProc) {.inline.} = proc accessThreadLocalVar(p: BProc, s: PSym) proc emulatedThreadVars(conf: ConfigRef): bool {.inline.} proc genProc(m: BModule, prc: PSym) +proc raiseInstr(p: BProc): Rope template compileToCpp(m: BModule): untyped = m.config.cmd == cmdCompileToCpp or sfCompileToCpp in m.module.flags diff --git a/compiler/commands.nim b/compiler/commands.nim index 0c93654da2874..62d4d75dd1c7f 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -881,6 +881,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; processOnOffSwitchG(conf, {optBenchmarkVM}, arg, pass, info) of "sinkinference": processOnOffSwitch(conf, {optSinkInference}, arg, pass, info) + of "panics": + processOnOffSwitchG(conf, {optPanics}, arg, pass, info) + if optPanics in conf.globalOptions: + defineSymbol(conf.symbols, "nimPanics") of "": # comes from "-" in for example: `nim c -r -` (gets stripped from -) handleStdinInput(conf) else: diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index a84b0d8780baa..da40dd2b7d795 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -113,3 +113,4 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasLibFFIEnabled") defineSymbol("nimHasSinkInference") + defineSymbol("nimNewIntegerOps") diff --git a/compiler/enumtostr.nim b/compiler/enumtostr.nim index 42d844609bb4c..614190ac31648 100644 --- a/compiler/enumtostr.nim +++ b/compiler/enumtostr.nim @@ -40,6 +40,7 @@ proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph): PSym = n[resultPos] = newSymNode(res) result.ast = n incl result.flags, sfFromGeneric + incl result.flags, sfAlwaysReturn proc searchObjCaseImpl(obj: PNode; field: PSym): PNode = case obj.kind @@ -101,3 +102,4 @@ proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGra n[resultPos] = newSymNode(res) result.ast = n incl result.flags, sfFromGeneric + incl result.flags, sfAlwaysReturn diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 5ac65201ebb06..6eac5bf1be13a 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -34,6 +34,7 @@ type warnTypelessParam, warnUseBase, warnWriteToForeignHeap, warnUnsafeCode, warnUnusedImportX, + warnInheritFromException, warnEachIdentIsTuple, warnProveInit, warnProveField, warnProveIndex, warnGcUnsafe, warnGcUnsafe2, warnUninit, warnGcMem, warnDestructor, warnLockLevel, warnResultShadowed, @@ -81,6 +82,7 @@ const warnWriteToForeignHeap: "write to foreign heap", warnUnsafeCode: "unsafe code: '$1'", warnUnusedImportX: "imported and not used: '$1'", + warnInheritFromException: "inherit from a more precise exception type like ValueError, IOError or OSError", warnEachIdentIsTuple: "each identifier is a tuple", warnProveInit: "Cannot prove that '$1' is initialized. This will become a compile time error in the future.", warnProveField: "cannot prove that field '$1' is accessible", @@ -138,7 +140,8 @@ const "LanguageXNotSupported", "FieldXNotSupported", "CommentXIgnored", "TypelessParam", "UseBase", "WriteToForeignHeap", - "UnsafeCode", "UnusedImport", "EachIdentIsTuple", + "UnsafeCode", "UnusedImport", "InheritFromException", + "EachIdentIsTuple", "ProveInit", "ProveField", "ProveIndex", "GcUnsafe", "GcUnsafe2", "Uninit", "GcMem", "Destructor", "LockLevel", "ResultShadowed", "Spacing", "CaseTransition", "CycleCreated", "User"] @@ -227,7 +230,7 @@ type TErrorOutputs* = set[TErrorOutput] ERecoverableError* = object of ValueError - ESuggestDone* = object of Exception + ESuggestDone* = object of ValueError proc `==`*(a, b: FileIndex): bool {.borrow.} diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 3a6d287083b65..d3be4b3b60464 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -165,6 +165,7 @@ proc stopCompile*(g: ModuleGraph): bool {.inline.} = proc createMagic*(g: ModuleGraph; name: string, m: TMagic): PSym = result = newSym(skProc, getIdent(g.cache, name), nil, unknownLineInfo, {}) result.magic = m + result.flags = {sfAlwaysReturn} proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph = result = ModuleGraph() diff --git a/compiler/options.nim b/compiler/options.nim index 42406272cc206..88251a42e724e 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -90,6 +90,7 @@ type # please make sure we have under 32 options optNimV019 optBenchmarkVM # Enables cpuTime() in the VM optProduceAsm # produce assembler code + optPanics # turn panics (sysFatal) into a process termination TGlobalOptions* = set[TGlobalOption] @@ -509,15 +510,17 @@ proc getOutFile*(conf: ConfigRef; filename: RelativeFile, ext: string): Absolute proc absOutFile*(conf: ConfigRef): AbsoluteFile = result = conf.outDir / conf.outFile - if dirExists(result.string): - result.string.add ".out" + when defined(posix): + if dirExists(result.string): + result.string.add ".out" proc prepareToWriteOutput*(conf: ConfigRef): AbsoluteFile = ## Create the output directory and returns a full path to the output file createDir conf.outDir result = conf.outDir / conf.outFile - if dirExists(result.string): - result.string.add ".out" + when defined(posix): + if dirExists(result.string): + result.string.add ".out" proc getPrefixDir*(conf: ConfigRef): AbsoluteDir = ## Gets the prefix dir, usually the parent directory where the binary resides. diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index b91e93025c40f..cdb47d2a45ca9 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1283,7 +1283,7 @@ proc typeSectionFinalPass(c: PContext, n: PNode) = while x.kind in {nkStmtList, nkStmtListExpr} and x.len > 0: x = x.lastSon if x.kind notin {nkObjectTy, nkDistinctTy, nkEnumTy, nkEmpty} and - s.typ.kind notin {tyObject, tyEnum}: + s.typ.skipTypes(abstractPtrs-{tyAlias}).kind notin {tyObject, tyEnum}: # type aliases are hard: var t = semTypeNode(c, x, nil) assert t != nil diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index f05affc6b56b7..e0be620c3fe5f 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -847,6 +847,9 @@ proc semObjectNode(c: PContext, n: PNode, prev: PType; isInheritable: bool): PTy # specialized object, there will be second check after instantiation # located in semGeneric. if concreteBase.kind == tyObject: + if concreteBase.sym != nil and concreteBase.sym.magic == mException and + sfSystemModule notin c.module.flags: + message(c.config, n.info, warnInheritFromException, "") addInheritedFields(c, check, pos, concreteBase) else: if concreteBase.kind != tyError: diff --git a/compiler/sizealignoffsetimpl.nim b/compiler/sizealignoffsetimpl.nim index ee88e276f2ef7..81d2f5fafffc6 100644 --- a/compiler/sizealignoffsetimpl.nim +++ b/compiler/sizealignoffsetimpl.nim @@ -23,7 +23,7 @@ const szUncomputedSize* = -1 szTooBigSize* = -4 -type IllegalTypeRecursionError = object of Exception +type IllegalTypeRecursionError = object of ValueError proc raiseIllegalTypeRecursion() = raise newException(IllegalTypeRecursionError, "illegal type recursion") diff --git a/doc/advopt.txt b/doc/advopt.txt index 517edab5b1984..1ebec0f492b6f 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -146,3 +146,4 @@ Advanced options: see also https://nim-lang.github.io/Nim/estp.html --benchmarkVM:on|off enable benchmarking of VM code with cpuTime() --sinkInference:on|off en-/disable sink parameter inference (default: on) + --panics:on|off turn panics into process terminations (default: off) diff --git a/doc/manual.rst b/doc/manual.rst index 516be24eaaede..3a0faed2e7ec6 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -105,13 +105,13 @@ identifier meanings, and in some cases expression values. An error detected during semantic analysis is called a `static error`:idx:. Errors described in this manual are static errors when not otherwise specified. -A `checked runtime error`:idx: is an error that the implementation detects +A `panic`:idx: is an error that the implementation detects and reports at runtime. The method for reporting such errors is via *raising exceptions* or *dying with a fatal error*. However, the implementation provides a means to disable these `runtime checks`:idx:. See the section pragmas_ for details. -Whether a checked runtime error results in an exception or in a fatal error is +Whether a panic results in an exception or in a fatal error is implementation specific. Thus the following program is invalid; even though the code purports to catch the `IndexError` from an out-of-bounds array access, the compiler may instead choose to allow the program to die with a fatal error. @@ -124,6 +124,12 @@ compiler may instead choose to allow the program to die with a fatal error. except IndexError: echo "invalid index" +The current implementation allows to switch between these different behaviors +via ``--panics:on|off``. When panics are turned on, the program dies on a +panic, if they are turned off the runtime errors are turned into +exceptions. The benefit of ``--panics:on`` is that it produces smaller binary +code and the compiler has more freedom to optimize the code. + An `unchecked runtime error`:idx: is an error that is not guaranteed to be detected, and can cause the subsequent behavior of the computation to be arbitrary. Unchecked runtime errors cannot occur if only `safe`:idx: @@ -869,9 +875,9 @@ Ordinal types have the following characteristics: the operation of functions as ``inc``, ``ord``, ``dec`` on ordinal types to be defined. - Ordinal values have a smallest possible value. Trying to count further - down than the smallest value gives a checked runtime or static error. + down than the smallest value produces a panic or a static error. - Ordinal values have a largest possible value. Trying to count further - than the largest value gives a checked runtime or static error. + than the largest value produces a panic or a static error. Integers, bool, characters and enumeration types (and subranges of these types) belong to ordinal types. For reasons of simplicity of implementation @@ -982,7 +988,7 @@ lowest and highest value of the type. For example: to 5. ``PositiveFloat`` defines a subrange of all positive floating point values. NaN does not belong to any subrange of floating point types. Assigning any other value to a variable of type ``Subrange`` is a -checked runtime error (or static error if it can be determined during +panic (or a static error if it can be determined during semantic analysis). Assignments from the base type to one of its subrange types (and vice versa) are allowed. @@ -1763,9 +1769,9 @@ Nil If a reference points to *nothing*, it has the value ``nil``. ``nil`` is also the default value for all ``ref`` and ``ptr`` types. Dereferencing ``nil`` -is an unrecoverable fatal runtime error. A dereferencing operation ``p[]`` -implies that ``p`` is not nil. This can be exploited by the implementation to -optimize code like: +is an unrecoverable fatal runtime error (and not a panic). +A dereferencing operation ``p[]`` implies that ``p`` is not nil. This can be +exploited by the implementation to optimize code like: .. code-block:: nim @@ -4107,7 +4113,7 @@ branch always has to be ``void``: .. code-block:: nim from strutils import parseInt - + let x = try: parseInt("133a") except: -1 finally: echo "hi" @@ -4242,9 +4248,11 @@ The exception tree is defined in the `system `_ module. Every exception inherits from ``system.Exception``. Exceptions that indicate programming bugs inherit from ``system.Defect`` (which is a subtype of ``Exception``) and are stricly speaking not catchable as they can also be mapped to an operation -that terminates the whole process. Exceptions that indicate any other runtime -error that can be caught inherit from ``system.CatchableError`` -(which is a subtype of ``Exception``). +that terminates the whole process. If panics are turned into exceptions, these +exceptions inherit from `Defect`. + +Exceptions that indicate any other runtime error that can be caught inherit from +``system.CatchableError`` (which is a subtype of ``Exception``). Imported exceptions @@ -5667,7 +5675,7 @@ The ``include`` statement can be used outside of the top level, as such: # Module B proc main() = include A - + main() # => Hello World! diff --git a/koch.nim b/koch.nim index c2f28f2e034fa..60d0e3544ecb8 100644 --- a/koch.nim +++ b/koch.nim @@ -475,7 +475,7 @@ proc hostInfo(): string = proc runCI(cmd: string) = doAssert cmd.len == 0, cmd # avoid silently ignoring - echo "runCI:", cmd + echo "runCI: ", cmd echo hostInfo() # note(@araq): Do not replace these commands with direct calls (eg boot()) # as that would weaken our testing efforts. diff --git a/lib/nimbase.h b/lib/nimbase.h index 004ba170b1b59..d25c4dad070e8 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -545,4 +545,26 @@ typedef int Nim_and_C_compiler_disagree_on_target_architecture[sizeof(NI) == siz #define NIM_CHECK_SIZE(typ, sz) \ _Static_assert(sizeof(typ) == sz, "Nim & C disagree on type size") +/* these exist to make the codegen logic simpler */ +#define nimModInt(a, b, res) (((*res) = (a) % (b)), 0) +#define nimModInt64(a, b, res) (((*res) = (a) % (b)), 0) + +/* these exist because we cannot have .compilerProcs that are importc'ed + by a different name */ + +#define nimAddInt64(a, b, res) __builtin_saddll_overflow(a, b, (long long int*)res) +#define nimSubInt64(a, b, res) __builtin_ssubll_overflow(a, b, (long long int*)res) +#define nimMulInt64(a, b, res) __builtin_smulll_overflow(a, b, (long long int*)res) + +#if NIM_INTBITS == 32 + #define nimAddInt(a, b, res) __builtin_sadd_overflow(a, b, res) + #define nimSubInt(a, b, res) __builtin_ssub_overflow(a, b, res) + #define nimMulInt(a, b, res) __builtin_smul_overflow(a, b, res) +#else + /* map it to the 'long long' variant */ + #define nimAddInt(a, b, res) __builtin_saddll_overflow(a, b, (long long int*)res) + #define nimSubInt(a, b, res) __builtin_ssubll_overflow(a, b, (long long int*)res) + #define nimMulInt(a, b, res) __builtin_smulll_overflow(a, b, (long long int*)res) +#endif + #endif /* NIMBASE_H */ diff --git a/lib/system.nim b/lib/system.nim index 5dff62806ee78..602fbc1f11cf1 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2221,7 +2221,10 @@ when notJSnotNims: # we cannot compile this with stack tracing on # as it would recurse endlessly! - include "system/arithm" + when defined(nimNewIntegerOps): + include "system/integerops" + else: + include "system/arithm" {.pop.} diff --git a/lib/system/arithm.nim b/lib/system/arithm.nim index 16ac8affea5ea..9e3ec79c4ab5b 100644 --- a/lib/system/arithm.nim +++ b/lib/system/arithm.nim @@ -409,13 +409,13 @@ when not declared(mulInt): # We avoid setting the FPU control word here for compatibility with libraries # written in other languages. -proc raiseFloatInvalidOp {.noinline.} = +proc raiseFloatInvalidOp {.compilerproc, noinline.} = sysFatal(FloatInvalidOpError, "FPU operation caused a NaN result") proc nanCheck(x: float64) {.compilerproc, inline.} = if x != x: raiseFloatInvalidOp() -proc raiseFloatOverflow(x: float64) {.noinline.} = +proc raiseFloatOverflow(x: float64) {.compilerproc, noinline.} = if x > 0.0: sysFatal(FloatOverflowError, "FPU operation caused an overflow") else: diff --git a/lib/system/chcks.nim b/lib/system/chcks.nim index 52642bbf91780..117543fcda987 100644 --- a/lib/system/chcks.nim +++ b/lib/system/chcks.nim @@ -28,6 +28,19 @@ proc raiseIndexError() {.compilerproc, noinline.} = proc raiseFieldError(f: string) {.compilerproc, noinline.} = sysFatal(FieldError, f) +proc raiseRangeErrorI(i, a, b: BiggestInt) {.compilerproc, noinline.} = + sysFatal(RangeError, "value out of range: " & $i & " notin " & $a & " .. " & $b) + +proc raiseRangeErrorF(i, a, b: float) {.compilerproc, noinline.} = + sysFatal(RangeError, "value out of range: " & $i & " notin " & $a & " .. " & $b) + +proc raiseRangeErrorU(i, a, b: uint64) {.compilerproc, noinline.} = + # todo: better error reporting + sysFatal(RangeError, "value out of range") + +proc raiseObjectConversionError() {.compilerproc, noinline.} = + sysFatal(ObjectConversionError, "invalid object conversion") + proc chckIndx(i, a, b: int): int = if i >= a and i <= b: return i @@ -116,6 +129,5 @@ when not defined(nimV2): return true when defined(nimV2): - proc nimFieldDiscriminantCheckV2(oldDiscVal, newDiscVal: uint8) {.compilerproc.} = - if oldDiscVal != newDiscVal: - sysFatal(FieldError, "assignment to discriminant changes object branch") + proc raiseObjectCaseTransition() {.compilerproc.} = + sysFatal(FieldError, "assignment to discriminant changes object branch") diff --git a/lib/system/fatal.nim b/lib/system/fatal.nim index d68d067125b56..761e0dd69b2c0 100644 --- a/lib/system/fatal.nim +++ b/lib/system/fatal.nim @@ -24,7 +24,7 @@ when hostOS == "standalone": rawoutput(message) panic(arg) -elif (defined(nimQuirky) or gotoBasedExceptions) and not defined(nimscript): +elif (defined(nimQuirky) or defined(nimPanics)) and not defined(nimscript): import ansi_c proc name(t: typedesc): string {.magic: "TypeTrait".} @@ -46,20 +46,9 @@ elif (defined(nimQuirky) or gotoBasedExceptions) and not defined(nimscript): else: proc sysFatal(exceptn: typedesc, message: string) {.inline, noreturn.} = - when declared(owned): - var e: owned(ref exceptn) - else: - var e: ref exceptn - new(e) - e.msg = message - raise e + raise (ref exceptn)(msg: message) proc sysFatal(exceptn: typedesc, message, arg: string) {.inline, noreturn.} = - when declared(owned): - var e: owned(ref exceptn) - else: - var e: ref exceptn - new(e) - e.msg = message & arg - raise e + raise (ref exceptn)(msg: message & arg) + {.pop.} diff --git a/lib/system/gc.nim b/lib/system/gc.nim index 7be9f4b1fc4b5..15f316cccbd1d 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -441,13 +441,15 @@ proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} = zeroMem(result, size) when defined(memProfiler): nimProfile(size) +{.push overflowChecks: on.} proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = # `newObj` already uses locks, so no need for them here. - let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) + let size = len * typ.base.size + GenericSeqSize result = newObj(typ, size) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len when defined(memProfiler): nimProfile(size) +{.pop.} proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = # generates a new object and sets its reference counter to 1 @@ -476,12 +478,14 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = sysAssert(allocInv(gch.region), "newObjRC1 end") when defined(memProfiler): nimProfile(size) +{.push overflowChecks: on.} proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = - let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) + let size = len * typ.base.size + GenericSeqSize result = newObjRC1(typ, size) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len when defined(memProfiler): nimProfile(size) +{.pop.} proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = collectCT(gch) diff --git a/lib/system/gc_common.nim b/lib/system/gc_common.nim index fe07766d92885..ff2b6ad6a6c02 100644 --- a/lib/system/gc_common.nim +++ b/lib/system/gc_common.nim @@ -79,7 +79,7 @@ template decTypeSize(cell, t) = if t.kind in {tyString, tySequence}: let cap = cast[PGenericSeq](cellToUsr(cell)).space let size = if t.kind == tyString: cap+1+GenericSeqSize - else: addInt(mulInt(cap, t.base.size), GenericSeqSize) + else: cap * t.base.size + GenericSeqSize atomicDec t.sizes, size+sizeof(Cell) else: atomicDec t.sizes, t.base.size+sizeof(Cell) diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index 271543445f066..a026b404baefd 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -305,20 +305,22 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = when defined(memProfiler): nimProfile(size) when not defined(nimSeqsV2): + {.push overflowChecks: on.} proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = # `newObj` already uses locks, so no need for them here. - let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) + let size = len * typ.base.size + GenericSeqSize result = newObj(typ, size) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len when defined(memProfiler): nimProfile(size) proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = - let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) + let size = len * typ.base.size + GenericSeqSize result = newObj(typ, size) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len when defined(memProfiler): nimProfile(size) + {.pop.} proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = collectCT(gch, newsize + sizeof(Cell)) diff --git a/lib/system/gc_regions.nim b/lib/system/gc_regions.nim index 6b1640ab14484..38365469edf50 100644 --- a/lib/system/gc_regions.nim +++ b/lib/system/gc_regions.nim @@ -334,20 +334,21 @@ proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} = result = rawNewObj(tlRegion, typ, size) when defined(memProfiler): nimProfile(size) +{.push overflowChecks: on.} proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = - let size = roundup(addInt(mulInt(len, typ.base.size), GenericSeqSize), - MemAlign) + let size = roundup(len * typ.base.size + GenericSeqSize, MemAlign) result = rawNewSeq(tlRegion, typ, size) zeroMem(result, size) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len proc newStr(typ: PNimType, len: int; init: bool): pointer {.compilerRtl.} = - let size = roundup(addInt(len, GenericSeqSize), MemAlign) + let size = roundup(len + GenericSeqSize, MemAlign) result = rawNewSeq(tlRegion, typ, size) if init: zeroMem(result, size) cast[PGenericSeq](result).len = 0 cast[PGenericSeq](result).reserved = len +{.pop.} proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = result = rawNewObj(tlRegion, typ, size) diff --git a/lib/system/integerops.nim b/lib/system/integerops.nim new file mode 100644 index 0000000000000..dc0197f14c467 --- /dev/null +++ b/lib/system/integerops.nim @@ -0,0 +1,132 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Integer arithmetic with overflow checking. Uses +# intrinsics or inline assembler. + +proc raiseOverflow {.compilerproc, noinline.} = + # a single proc to reduce code size to a minimum + sysFatal(OverflowError, "over- or underflow") + +proc raiseDivByZero {.compilerproc, noinline.} = + sysFatal(DivByZeroError, "division by zero") + +{.pragma: nimbaseH, importc, nodecl, noSideEffect, compilerproc.} + +when defined(gcc) or defined(clang): + # take the #define from nimbase.h + + proc nimAddInt(a, b: int, res: ptr int): bool {.nimbaseH.} + proc nimSubInt(a, b: int, res: ptr int): bool {.nimbaseH.} + proc nimMulInt(a, b: int, res: ptr int): bool {.nimbaseH.} + + proc nimAddInt64(a, b: int64; res: ptr int64): bool {.nimbaseH.} + proc nimSubInt64(a, b: int64; res: ptr int64): bool {.nimbaseH.} + proc nimMulInt64(a, b: int64; res: ptr int64): bool {.nimbaseH.} + +# unary minus and 'abs' not required here anymore and are directly handled +# in the code generator. +# 'nimModInt' does exist in nimbase.h without check as we moved the +# check for 0 to the codgen. +proc nimModInt(a, b: int; res: ptr int): bool {.nimbaseH.} + +proc nimModInt64(a, b: int64; res: ptr int64): bool {.nimbaseH.} + +# Platform independent versions. + +template addImplFallback(name, T, U) {.dirty.} = + when not declared(name): + proc name(a, b: T; res: ptr T): bool {.compilerproc, inline.} = + let r = cast[T](cast[U](a) + cast[U](b)) + if (r xor a) >= T(0) or (r xor b) >= T(0): + res[] = r + else: + result = true + +addImplFallback(nimAddInt, int, uint) +addImplFallback(nimAddInt64, int64, uint64) + +template subImplFallback(name, T, U) {.dirty.} = + when not declared(name): + proc name(a, b: T; res: ptr T): bool {.compilerproc, inline.} = + let r = cast[T](cast[U](a) - cast[U](b)) + if (r xor a) >= 0 or (r xor not b) >= 0: + res[] = r + else: + result = true + +subImplFallback(nimSubInt, int, uint) +subImplFallback(nimSubInt64, int64, uint64) + +template mulImplFallback(name, T, U, conv) {.dirty.} = + # + # This code has been inspired by Python's source code. + # The native int product x*y is either exactly right or *way* off, being + # just the last n bits of the true product, where n is the number of bits + # in an int (the delivered product is the true product plus i*2**n for + # some integer i). + # + # The native float64 product x*y is subject to three + # rounding errors: on a sizeof(int)==8 box, each cast to double can lose + # info, and even on a sizeof(int)==4 box, the multiplication can lose info. + # But, unlike the native int product, it's not in *range* trouble: even + # if sizeof(int)==32 (256-bit ints), the product easily fits in the + # dynamic range of a float64. So the leading 50 (or so) bits of the float64 + # product are correct. + # + # We check these two ways against each other, and declare victory if + # they're approximately the same. Else, because the native int product is + # the only one that can lose catastrophic amounts of information, it's the + # native int product that must have overflowed. + # + when not declared(name): + proc name(a, b: T; res: ptr T): bool {.compilerproc, inline.} = + let r = cast[T](cast[U](a) * cast[U](b)) + let floatProd = conv(a) * conv(b) + let resAsFloat = conv(r) + # Fast path for normal case: small multiplicands, and no info + # is lost in either method. + if resAsFloat == floatProd: + res[] = r + else: + # Somebody somewhere lost info. Close enough, or way off? Note + # that a != 0 and b != 0 (else resAsFloat == floatProd == 0). + # The difference either is or isn't significant compared to the + # true value (of which floatProd is a good approximation). + + # abs(diff)/abs(prod) <= 1/32 iff + # 32 * abs(diff) <= abs(prod) -- 5 good bits is "close enough" + if 32.0 * abs(resAsFloat - floatProd) <= abs(floatProd): + res[] = r + else: + result = true + +mulImplFallback(nimMulInt, int, uint, toFloat) +mulImplFallback(nimMulInt64, int64, uint64, toBiggestFloat) + + +template divImplFallback(name, T) {.dirty.} = + proc name(a, b: T; res: ptr T): bool {.compilerproc, inline.} = + # we moved the b == 0 case out into the codegen. + if a == low(T) and b == T(-1): + result = true + else: + res[] = a div b + +divImplFallback(nimDivInt, int) +divImplFallback(nimDivInt64, int64) + +proc raiseFloatInvalidOp {.compilerproc, noinline.} = + sysFatal(FloatInvalidOpError, "FPU operation caused a NaN result") + +proc raiseFloatOverflow(x: float64) {.compilerproc, noinline.} = + if x > 0.0: + sysFatal(FloatOverflowError, "FPU operation caused an overflow") + else: + sysFatal(FloatUnderflowError, "FPU operations caused an underflow") diff --git a/lib/system/mm/boehm.nim b/lib/system/mm/boehm.nim index d02d52b525f5a..a8b0e17b409ed 100644 --- a/lib/system/mm/boehm.nim +++ b/lib/system/mm/boehm.nim @@ -101,10 +101,12 @@ proc newObj(typ: PNimType, size: int): pointer {.compilerproc.} = else: result = alloc(size) if typ.finalizer != nil: boehmRegisterFinalizer(result, boehmgc_finalizer, typ.finalizer, nil, nil) +{.push overflowChecks: on.} proc newSeq(typ: PNimType, len: int): pointer {.compilerproc.} = - result = newObj(typ, addInt(mulInt(len, typ.base.size), GenericSeqSize)) + result = newObj(typ, len * typ.base.size + GenericSeqSize) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len +{.pop.} proc growObj(old: pointer, newsize: int): pointer = result = realloc(old, newsize) diff --git a/lib/system/mm/none.nim b/lib/system/mm/none.nim index 3572b062c8253..0079daac39836 100644 --- a/lib/system/mm/none.nim +++ b/lib/system/mm/none.nim @@ -19,10 +19,12 @@ proc newObj(typ: PNimType, size: int): pointer {.compilerproc.} = proc newObjNoInit(typ: PNimType, size: int): pointer = result = alloc(size) +{.push overflowChecks: on.} proc newSeq(typ: PNimType, len: int): pointer {.compilerproc.} = - result = newObj(typ, addInt(mulInt(len, typ.base.size), GenericSeqSize)) + result = newObj(typ, len * typ.base.size + GenericSeqSize) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len +{.pop.} proc growObj(old: pointer, newsize: int): pointer = result = realloc(old, newsize) diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index d2d0d576fbf3b..2c3394df945b2 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -102,18 +102,20 @@ else: include "system/gc" when not declared(nimNewSeqOfCap) and not defined(nimSeqsV2): + {.push overflowChecks: on.} proc nimNewSeqOfCap(typ: PNimType, cap: int): pointer {.compilerproc.} = when defined(gcRegions): - let s = mulInt(cap, typ.base.size) # newStr already adds GenericSeqSize + let s = cap * typ.base.size # newStr already adds GenericSeqSize result = newStr(typ, s, ntfNoRefs notin typ.base.flags) else: - let s = addInt(mulInt(cap, typ.base.size), GenericSeqSize) + let s = cap * typ.base.size + GenericSeqSize when declared(newObjNoInit): result = if ntfNoRefs in typ.base.flags: newObjNoInit(typ, s) else: newObj(typ, s) else: result = newObj(typ, s) cast[PGenericSeq](result).len = 0 cast[PGenericSeq](result).reserved = cap + {.pop.} {.pop.} diff --git a/tests/assert/tassert_c.nim b/tests/assert/tassert_c.nim index 98b859c1cf08c..e9e6b519256bd 100644 --- a/tests/assert/tassert_c.nim +++ b/tests/assert/tassert_c.nim @@ -8,7 +8,7 @@ tassert_c.nim(35) tassert_c tassert_c.nim(34) foo assertions.nim(29) failedAssertImpl assertions.nim(22) raiseAssert -fatal.nim(55) sysFatal""" +fatal.nim(49) sysFatal""" proc tmatch(x, p: string): bool = var i = 0 diff --git a/tests/destructor/tgotoexceptions7.nim b/tests/destructor/tgotoexceptions7.nim new file mode 100644 index 0000000000000..30fe28e7c4151 --- /dev/null +++ b/tests/destructor/tgotoexceptions7.nim @@ -0,0 +1,52 @@ +discard """ + cmd: "nim c --gc:arc --exceptions:goto --panics:off $file" + output: '''field error prevented +prevented! +caught +AssertionError +900''' +""" + +type + E = enum + kindA, kindB + Obj = object + case kind: E + of kindA: s: string + of kindB: i: int + + ObjA = ref object of RootObj + ObjB = ref object of ObjA + +proc takeRange(x: range[0..4]) = discard + +proc bplease(x: ObjB) = discard + +proc helper = doAssert(false) + +proc main(i: int) = + var obj = Obj(kind: kindA, s: "abc") + try: + obj.kind = kindB + except FieldError: + echo "field error prevented" + + try: + var objA = ObjA() + bplease(ObjB(objA)) + except ObjectConversionError: + echo "prevented!" + + try: + takeRange(i) + except RangeError: + echo "caught" + + try: + helper() + except AssertionError: + echo "AssertionError" + + echo i * i + +main(30) diff --git a/tests/range/tsubrange2.nim b/tests/range/tsubrange2.nim index e0fb71c5f4252..1b90b52c791d4 100644 --- a/tests/range/tsubrange2.nim +++ b/tests/range/tsubrange2.nim @@ -1,5 +1,5 @@ discard """ - outputsub: "value out of range: 50 [RangeError]" + outputsub: "value out of range: 50 notin 0 .. 40 [RangeError]" exitcode: "1" """ diff --git a/tests/range/tsubrange3.nim b/tests/range/tsubrange3.nim index d3aae87dfda0b..ba9e6a143ec0f 100644 --- a/tests/range/tsubrange3.nim +++ b/tests/range/tsubrange3.nim @@ -1,5 +1,5 @@ discard """ - outputsub: "value out of range: 50 [RangeError]" + outputsub: "value out of range: 50 notin 0 .. 40 [RangeError]" exitcode: "1" """ From 9eeb514dda08f1caadb0d8e01a8595d991530b52 Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 13 Mar 2020 11:50:39 +0100 Subject: [PATCH 75/77] disable nimgame2 for now --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 3631710828cf3..d9c49cb7b56b4 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -52,7 +52,7 @@ pkg "nimcrypto", false, "nim c -r tests/testall.nim" pkg "NimData", true, "nim c -o:nimdataa src/nimdata.nim" pkg "nimes", true, "nim c src/nimes.nim" pkg "nimfp", true, "nim c -o:nfp -r src/fp.nim" -pkg "nimgame2", true, "nim c nimgame2/nimgame.nim" +#pkg "nimgame2", true, "nim c nimgame2/nimgame.nim" pkg "nimgen", true, "nim c -o:nimgenn -r src/nimgen/runcfg.nim" # pkg "nimlsp", true pkg "nimly", true From 380a505507a6df9cbb1fe77b5b6251dc14894921 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 13 Mar 2020 06:02:47 -0700 Subject: [PATCH 76/77] azure-pipelines: use OSX 10.15 (was just enabled upstream) (#13546) --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index dccd0d05e75fc..e04974f5bd514 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -12,10 +12,10 @@ jobs: vmImage: 'ubuntu-16.04' CPU: i386 OSX_amd64: - vmImage: 'macOS-10.14' + vmImage: 'macOS-10.15' CPU: amd64 OSX_amd64_cpp: - vmImage: 'macOS-10.14' + vmImage: 'macOS-10.15' CPU: amd64 NIM_COMPILE_TO_CPP: true Windows_amd64: From 6e0c06f50e0450522b32a1bad23a3d2a4e36e9f9 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 13 Mar 2020 12:42:41 -0700 Subject: [PATCH 77/77] fix #13218: avoid some irrelevant warnings for nim doc,rst2html,--app:lib, + other fixes (#13550) * fix #13218: avoid some irrelevant warnings for nim doc,rst2html * suppress warnRedefinitionOfLabel for nim doc * lots of fixes for UnusedImport warnings --- compiler/main.nim | 7 +++++++ compiler/options.nim | 8 ++++++++ doc/docgen_sample.nim | 4 ++-- lib/impure/rdstdin.nim | 2 +- lib/pure/asyncdispatch.nim | 2 +- lib/pure/asynchttpserver.nim | 2 +- lib/pure/encodings.nim | 3 ++- lib/pure/lenientops.nim | 2 -- lib/pure/selectors.nim | 3 ++- lib/pure/strformat.nim | 2 -- tools/kochdocs.nim | 4 +++- 11 files changed, 27 insertions(+), 12 deletions(-) diff --git a/compiler/main.nim b/compiler/main.nim index c8bdc94475660..0b538cb5aaad5 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -225,6 +225,12 @@ proc mainCommand*(graph: ModuleGraph) = loadConfigs(DocConfig, cache, conf) commandDoc(cache, conf) of "doc2", "doc": + conf.setNoteDefaults(warnLockLevel, false) # issue #13218 + conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218 + # because currently generates lots of false positives due to conflation + # of labels links in doc comments, eg for random.rand: + # ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer + # ## * `rand proc<#rand,Rand,range[]>`_ that returns a float when defined(leanCompiler): quit "compiler wasn't built with documentation generator" else: @@ -233,6 +239,7 @@ proc mainCommand*(graph: ModuleGraph) = defineSymbol(conf.symbols, "nimdoc") commandDoc2(graph, false) of "rst2html": + conf.setNoteDefaults(warnRedefinitionOfLabel, false) # similar to issue #13218 when defined(leanCompiler): quit "compiler wasn't built with documentation generator" else: diff --git a/compiler/options.nim b/compiler/options.nim index 88251a42e724e..26260a77014ef 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -289,7 +289,15 @@ type severity: Severity) {.closure, gcsafe.} cppCustomNamespace*: string +proc setNoteDefaults*(conf: ConfigRef, note: TNoteKind, enabled = true) = + template fun(op) = + conf.notes.op note + conf.mainPackageNotes.op note + conf.foreignPackageNotes.op note + if enabled: fun(incl) else: fun(excl) + proc setNote*(conf: ConfigRef, note: TNoteKind, enabled = true) = + # see also `prepareConfigNotes` which sets notes if note notin conf.cmdlineNotes: if enabled: incl(conf.notes, note) else: excl(conf.notes, note) diff --git a/doc/docgen_sample.nim b/doc/docgen_sample.nim index 8759931873b8c..7a167cb4572fc 100644 --- a/doc/docgen_sample.nim +++ b/doc/docgen_sample.nim @@ -4,9 +4,9 @@ import strutils proc helloWorld*(times: int) = ## Takes an integer and outputs - ## as many "hello world!"s + ## as many indented "hello world!"s for i in 0 .. times-1: - echo "hello world!" + echo "hello world!".indent(2) # using indent to avoid `UnusedImport` helloWorld(5) diff --git a/lib/impure/rdstdin.nim b/lib/impure/rdstdin.nim index b78c0d8cf7a46..4c36afe6e0c82 100644 --- a/lib/impure/rdstdin.nim +++ b/lib/impure/rdstdin.nim @@ -43,7 +43,7 @@ elif defined(genode): stdin.readLine(line) else: - import linenoise, termios + import linenoise proc readLineFromStdin*(prompt: string): TaintedString {. tags: [ReadIOEffect, WriteIOEffect].} = diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 5e4e3f6998a97..35eb646c65670 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -168,7 +168,7 @@ include "system/inclrtl" -import os, tables, strutils, times, heapqueue, lists, options, asyncstreams +import os, tables, strutils, times, heapqueue, options, asyncstreams import options, math, std/monotimes import asyncfutures except callSoon diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 186f0da414650..c806d972abecf 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -30,7 +30,7 @@ ## ## waitFor server.serve(Port(8080), cb) -import tables, asyncnet, asyncdispatch, parseutils, uri, strutils +import asyncnet, asyncdispatch, parseutils, uri, strutils import httpcore export httpcore except parseHeader diff --git a/lib/pure/encodings.nim b/lib/pure/encodings.nim index e3be1764bf604..ff9c3889af50c 100644 --- a/lib/pure/encodings.nim +++ b/lib/pure/encodings.nim @@ -10,7 +10,7 @@ ## Converts between different character encodings. On UNIX, this uses ## the `iconv`:idx: library, on Windows the Windows API. -import os, parseutils, strutils +import os when not defined(windows): type @@ -28,6 +28,7 @@ type ## for encoding errors when defined(windows): + import parseutils, strutils proc eqEncodingNames(a, b: string): bool = var i = 0 var j = 0 diff --git a/lib/pure/lenientops.nim b/lib/pure/lenientops.nim index cdea780a2d92a..f73df5e5f6fba 100644 --- a/lib/pure/lenientops.nim +++ b/lib/pure/lenientops.nim @@ -24,8 +24,6 @@ ## either casting to float or rounding to int might be preferred, and users ## should make an explicit choice. -import typetraits - proc `+`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.noSideEffect, inline.} = F(i) + f proc `+`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.noSideEffect, inline.} = diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index 0ab1e77548295..743a92dd0d641 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -27,7 +27,7 @@ ## ## TODO: ``/dev/poll``, ``event ports`` and filesystem events. -import os, strutils, nativesockets +import os, nativesockets const hasThreadSupport = compileOption("threads") and defined(threadsafe) @@ -230,6 +230,7 @@ when defined(nimdoc): ## For *poll* and *select* selectors ``-1`` is returned. else: + import strutils when hasThreadSupport: import locks diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index bc0d4b069283d..838baaf0267f8 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -744,8 +744,6 @@ when isMainModule: check &"{high(int64)}", "9223372036854775807" check &"{low(int64)}", "-9223372036854775808" - import json - doAssert fmt"{'a'} {'b'}" == "a b" echo("All tests ok") diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 75df629e3306d..87cdf53aab6de 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -149,8 +149,10 @@ lib/posix/linux.nim lib/posix/termios.nim lib/js/jscore.nim """.splitWhitespace() - + + # some of these are include files so shouldn't be docgen'd ignoredModules = """ +lib/prelude.nim lib/pure/future.nim lib/impure/osinfo_posix.nim lib/impure/osinfo_win.nim