Implement scoped lexical id generator

The new id generator creates identifier names based on a given name.

The name is initially prefixed with "@" to avoid conflicts with ordinary
variable names. Then, each time a scope is closed the name of generated
variable is changed removing the "@" and ensuring that there is no
conflicts with the variables in the scope.

lang/compile.lua

@@ -1,7 +1,6 @@
 local lex_setup = require('lang.lexer')
 local parse = require('lang.parser')
 local lua_ast = require('lang.lua-ast')
-local id_generator = require('lang.id-generator')
 local reader = require('lang.reader')
 
 -- Two kind of backend can be used to generate the code from the AST:
@@ -22,10 +21,6 @@ local function lang_toolkit_error(msg)
     end
 end
 
-local function create_ident(name)
-    return { kind = "Identifier", name = name }
-end
-
 local function compile(reader, filename, options)
     local generator
     if options and options.code then
@@ -34,10 +29,8 @@ local function compile(reader, filename, options)
         generator = require('lang.generator')
     end
     local ls = lex_setup(reader, filename)
-    local genid = id_generator.lexical(create_ident)
-    local ast_builder = lua_ast.New(genid)
+    local ast_builder = lua_ast.New()
     local parse_success, ast_tree = pcall(parse, ast_builder, ls)
-    ast_builder:close()
     if not parse_success then
         return lang_toolkit_error(ast_tree)
     end

lang/id-generator.lua

@@ -1,82 +1,50 @@
--- Create a two pass identifier generator. In the first pass the identifier are in the
--- form "@<number>". Then all the lexical variables should be declared using "var_declare".
--- In the final stage the function "normalize" is used to transform the temporary
--- identifier, like "@2" into something like "__12". All this is to ensure that the
--- * the identifier is a valid identifier string
--- * there are no conflict with other local variables declared in the program
-local function create_genid_lexical(create_ident)
-    local intervals = { {1, 2^32 - 1} }
-    local longest = 1
-    local current = 0
-
-    local pending_idents = {}
-
-    local function find_longest()
-        local ilong, isize = 1, -1
-        for i = 1, #intervals do
-            local size = intervals[i][2] - intervals[i][1]
-            if size > isize then
-                ilong, isize = i, size
-            end
+local function unique_name(variables, name)
+    if variables:lookup(name) ~= nil then
+        local prefix, index = string.match(name, "^(.+)(%d+)$")
+        if not prefix then
+            prefix, index = name, 1
+        else
+            index = tonumber(index) + 1
         end
-        longest = ilong
-    end
-
-    local function remove_id(n)
-        for i = 1, #intervals do
-            local a, b = intervals[i][1], intervals[i][2]
-            if a <= n and n <= b then
-                table.remove(intervals, i)
-                if n > a then table.insert(intervals, i, {a, n - 1}) end
-                if n < b then table.insert(intervals, i, {n + 1, b}) end
-                if longest >= i then find_longest() end
-                break
-            end
+        local test_name = prefix .. tostring(index)
+        while variables:lookup(test_name) ~= nil do
+            index = index + 1
+            test_name = prefix .. tostring(index)
         end
+        return test_name
+    else
+        return name
     end
+end
 
-    local function var_declare(name)
-        local idn = string.match(name, "^__(%d+)$")
-        if idn then
-            remove_id(tonumber(idn))
-        end
-    end
+local function pseudo(name)
+    return '@' .. name
+end
 
-    local function normal_name(n)
-        local name = intervals[longest][1] + (n - 1)
-        assert(name <= intervals[longest][2], "cannot generate new identifier")
-        return "__" .. name
-    end
+local function pseudo_match(pseudo_name)
+    return string.match(pseudo_name, "^@(.+)$")
+end
 
-    local function normalize(raw_name)
-        local n = tonumber(string.match(raw_name, "^@(%d+)$"))
-        return normal_name(n)
-    end
+local function genid(variables, name)
+    local pname = pseudo(name or "_")
+    local uname = unique_name(variables, pname)
+    return variables:declare(uname)
+end
 
-    local function new_ident()
-        current = current + 1
-        local id
-        if pending_idents then -- Create a temporary name.
-            id = create_ident("@" .. current)
-            pending_idents[#pending_idents+1] = id
-        else -- Generate the final name.
-            local name = normal_name(current)
-            id = create_ident(name)
-        end
-        return id
-    end
+local function normalize(variables, raw_name)
+    local name = pseudo_match(raw_name)
+    local uname = unique_name(variables, name)
+    return uname
+end
 
-    -- When called this means that all the lexical variables have been
-    -- declared with "var_declare".
-    local function close_lexical()
-        for i = 1, #pending_idents do
-            local id = pending_idents[i]
-            id.name = normalize(id.name)
+local function close_gen_variables(variables)
+    local vars = variables.current.vars
+    for i = 1, #vars do
+        local id = vars[i]
+        if pseudo_match(id.name) then
+            id.name = normalize(variables, id.name)
         end
-        pending_idents = nil
     end
-
-    return { new_ident = new_ident, var_declare = var_declare, close_lexical = close_lexical }
 end
 
-return { create = create_genid_simple, lexical = create_genid_lexical }
+return { genid = genid, close_gen_variables = close_gen_variables }

lang/lua-ast.lua

@@ -1,3 +1,5 @@
+local id_generator = require("lang.id-generator")
+
 local function build(kind, node)
     node.kind = kind
     return node
@@ -183,40 +185,71 @@ function AST.goto_stmt(ast, name, line)
     return build("GotoStatement", { label = name, line = line })
 end
 
-local function new_scope(parent_scope)
-    return {
-        vars = { },
-        parent = parent_scope,
-    }
-end
-
 function AST.var_declare(ast, name)
     local id = ident(name)
-    ast.current.vars[name] = true
-    ast.id_generator.var_declare(name)
+    ast.variables:declare(name)
     return id
 end
 
-function AST.genid(ast)
-    return ast.id_generator.new_ident()
+function AST.genid(ast, name)
+    return id_generator.genid(ast.variables, name)
 end
 
 function AST.fscope_begin(ast)
-    ast.current = new_scope(ast.current)
+    ast.variables:scope_enter()
 end
 
 function AST.fscope_end(ast)
-    ast.current = ast.current.parent
+    -- It is important to call id_generator.close_gen_variables before
+    -- leaving the "variables" scope.
+    id_generator.close_gen_variables(ast.variables)
+    ast.variables:scope_exit()
 end
 
-function AST.close(ast)
-    ast.id_generator.close_lexical()
+local ASTClass = { __index = AST }
+
+local function new_scope(parent_scope)
+    return {
+        vars = { },
+        parent = parent_scope,
+    }
 end
 
-local ASTClass = { __index = AST }
+local function new_variables_registry(create, match)
+    local declare = function(self, name)
+        local vars = self.current.vars
+        local entry = create(name)
+        vars[#vars+1] = entry
+        return entry
+    end
+
+    local scope_enter = function(self)
+        self.current = new_scope(self.current)
+    end
+
+    local scope_exit = function(self)
+        self.current = self.current.parent
+    end
+
+    local lookup = function(self, name)
+        local scope = self.current
+        while scope do
+            for i = 1, #scope.vars do
+                if match(scope.vars[i], name) then
+                    return scope
+                end
+            end
+            scope = scope.parent
+        end
+    end
+
+    return { declare = declare, scope_enter = scope_enter, scope_exit = scope_exit, lookup = lookup }
+end
 
-local function new_ast(genid)
-    return setmetatable({ id_generator = genid }, ASTClass)
+local function new_ast()
+    local match_id_name = function(id, name) return id.name == name end
+    local vars = new_variables_registry(ident, match_id_name)
+    return setmetatable({ variables = vars }, ASTClass)
 end
 
 return { New = new_ast }