Implement lexical id generator for macro system

The id generator ensure that all the id automatically generated by
macros does not conflict with user-defined variables name.

lang/Makefile

@@ -1,6 +1,6 @@
 
 LANG_LUA_FILES = ast-boolean-const-eval.lua ast-const-eval.lua bcread.lua bcsave.lua \
-	bytecode.lua compile.lua generator.lua lexer.lua lua-ast.lua operator.lua \
+	bytecode.lua compile.lua generator.lua lexer.lua id-generator.lua lua-ast.lua operator.lua \
 	parser.lua reader.lua
 
 LANG_H_FILES = $(LANG_LUA_FILES:%.lua=%.h)

lang/compile.lua

@@ -1,6 +1,7 @@
 local lex_setup = require('lang.lexer')
 local parse = require('lang.parser')
-local ast = require('lang.lua-ast').New()
+local lua_ast = require('lang.lua-ast')
+local id_generator = require('lang.id-generator')
 local reader = require('lang.reader')
 
 -- Two kind of backend can be used to generate the code from the AST:
@@ -21,6 +22,10 @@ local function lang_toolkit_error(msg)
     end
 end
 
+local function create_ident(name)
+    return { kind = "Identifier", name = name }
+end
+
 local function compile(reader, filename, options)
     local generator
     if options and options.code then
@@ -29,11 +34,14 @@ local function compile(reader, filename, options)
         generator = require('lang.generator')
     end
     local ls = lex_setup(reader, filename)
-    local parse_success, tree = pcall(parse, ast, ls)
+    local genid = id_generator.lexical(create_ident)
+    local ast_builder = lua_ast.New(genid)
+    local parse_success, ast_tree = pcall(parse, ast_builder, ls)
+    ast_builder:close()
     if not parse_success then
-        return lang_toolkit_error(tree)
+        return lang_toolkit_error(ast_tree)
     end
-    local success, luacode = pcall(generator, tree, filename)
+    local success, luacode = pcall(generator, ast_tree, filename)
     if not success then
         return lang_toolkit_error(luacode)
     end

lang/id-generator.lua

@@ -0,0 +1,82 @@
+-- Create a two pass identifier generator. In the first pass the identifier are in the
+-- form "@<number>". Then all the lexical variables should be declared using "var_declare".
+-- In the final stage the function "normalize" is used to transform the temporary
+-- identifier, like "@2" into something like "__12". All this is to ensure that the
+-- * the identifier is a valid identifier string
+-- * there are no conflict with other local variables declared in the program
+local function create_genid_lexical(create_ident)
+    local intervals = { {1, 2^32 - 1} }
+    local longest = 1
+    local current = 0
+
+    local pending_idents = {}
+
+    local function find_longest()
+        local ilong, isize = 1, -1
+        for i = 1, #intervals do
+            local size = intervals[i][2] - intervals[i][1]
+            if size > isize then
+                ilong, isize = i, size
+            end
+        end
+        longest = ilong
+    end
+
+    local function remove_id(n)
+        for i = 1, #intervals do
+            local a, b = intervals[i][1], intervals[i][2]
+            if a <= n and n <= b then
+                table.remove(intervals, i)
+                if n > a then table.insert(intervals, i, {a, n - 1}) end
+                if n < b then table.insert(intervals, i, {n + 1, b}) end
+                if longest >= i then find_longest() end
+                break
+            end
+        end
+    end
+
+    local function var_declare(name)
+        local idn = string.match(name, "^__(%d+)$")
+        if idn then
+            remove_id(tonumber(idn))
+        end
+    end
+
+    local function normal_name(n)
+        local name = intervals[longest][1] + (n - 1)
+        assert(name <= intervals[longest][2], "cannot generate new identifier")
+        return "__" .. name
+    end
+
+    local function normalize(raw_name)
+        local n = tonumber(string.match(raw_name, "^@(%d+)$"))
+        return normal_name(n)
+    end
+
+    local function new_ident()
+        current = current + 1
+        local id
+        if pending_idents then -- Create a temporary name.
+            id = create_ident("@" .. current)
+            pending_idents[#pending_idents+1] = id
+        else -- Generate the final name.
+            local name = normal_name(current)
+            id = create_ident(name)
+        end
+        return id
+    end
+
+    -- When called this means that all the lexical variables have been
+    -- declared with "var_declare".
+    local function close_lexical()
+        for i = 1, #pending_idents do
+            local id = pending_idents[i]
+            id.name = normalize(id.name)
+        end
+        pending_idents = nil
+    end
+
+    return { new_ident = new_ident, var_declare = var_declare, close_lexical = close_lexical }
+end
+
+return { create = create_genid_simple, lexical = create_genid_lexical }

lang/lua-ast.lua

@@ -193,9 +193,14 @@ end
 function AST.var_declare(ast, name)
     local id = ident(name)
     ast.current.vars[name] = true
+    ast.id_generator.var_declare(name)
     return id
 end
 
+function AST.genid(ast)
+    return ast.id_generator.new_ident()
+end
+
 function AST.fscope_begin(ast)
     ast.current = new_scope(ast.current)
 end
@@ -204,10 +209,14 @@ function AST.fscope_end(ast)
     ast.current = ast.current.parent
 end
 
+function AST.close(ast)
+    ast.id_generator.close_lexical()
+end
+
 local ASTClass = { __index = AST }
 
-local function new_ast()
-    return setmetatable({ }, ASTClass)
+local function new_ast(genid)
+    return setmetatable({ id_generator = genid }, ASTClass)
 end
 
 return { New = new_ast }

src/language_bcloader.c

@@ -12,6 +12,7 @@
 #include "lang/compile.h"
 #include "lang/generator.h"
 #include "lang/lexer.h"
+#include "lang/id-generator.h"
 #include "lang/lua-ast.h"
 #include "lang/operator.h"
 #include "lang/parser.h"
@@ -32,6 +33,7 @@ static struct bcpair bcmodule[] = {
     {"compile", luaJIT_BC_compile, luaJIT_BC_compile_SIZE},
     {"generator", luaJIT_BC_generator, luaJIT_BC_generator_SIZE},
     {"lexer", luaJIT_BC_lexer, luaJIT_BC_lexer_SIZE},
+    {"id-generator", luaJIT_BC_id_generator, luaJIT_BC_id_generator_SIZE},
     {"lua-ast", luaJIT_BC_lua_ast, luaJIT_BC_lua_ast_SIZE},
     {"operator", luaJIT_BC_operator, luaJIT_BC_operator_SIZE},
     {"parser", luaJIT_BC_parser, luaJIT_BC_parser_SIZE},