Merge branch 'bc-standalone-exec'

Conflicts:
	README.md
	src/Makefile

Makefile

@@ -18,10 +18,16 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 #
 
-all:
+all: lang src
+
+lang:
+	$(MAKE) -C lang
+
+src: lang
 	$(MAKE) -C src
 
 clean:
 	$(MAKE) -C src clean
+	$(MAKE) -C lang clean
 
-.PHONY: clean all
+.PHONY: clean all lang src

README.md

@@ -179,6 +179,10 @@ extern int language_loadfile(lua_State *L, const char *filename);
    last one, loader, can be used as a customized "loader" function for
    the "require" function. */
 extern int luaopen_langloaders(lua_State *L);
+
+/* OPTIONAL:
+   Load into package.preload lang.* modules using embedded bytecode. */
+extern void language_bc_preload(lua_State *L)
 ```
 
 The functions above can be used to create a custom LuaJIT executable that use the language toolkit implementation.
@@ -187,7 +191,10 @@ When the function `language_*` is used, an independent `lua_State` is created be
 Once the bytecode is generated it is loaded into the user's `lua_State` ready to be executed.
 The approach of using a separate Lua's state ensure that the process of compiling does not interfere with the user's application.
 
-It should be noted that even when an executable is created with the C API, the lang/* Lua files need to be available at run time because they are used by the language toolkit's Lua state.
+The function `language_bc_preload` is useful to create a standalone executable that does not depend on the presence of the Lua files at runtime.
+The `lang.*` are compiled into bytecode and stored as static C data into the executable.
+By calling the function `language_bc_preload` all the modules are *preloaded* using the embedded bytecode.
+This feature can be disabled by changing the `BC_PRELOAD` variable in `src/Makefile`.
 
 Running the Application
 ---
@@ -202,7 +209,13 @@ The "run.lua" script will just invoke the complete pipeline of the lexer, parser
 
 The language toolkit also provides a customized executable named `luajit-x` that uses the language toolkit's pipeline instead of the native one.
 Otherwise, the program `luajit-x` works exactly the same as `luajit` itself, and accepts the same options.
-This means that you can experiment with the language by modifying the toolkit's implementation, and test the changes immediately without recompiling anything by using `luajit-x` as a REPL.
+
+In the standard build `luajit-x` will contain the `lang.*` modules as embedded bytecode data so that it does not rely on the Lua files at runtime.
+
+This means that you can experiment with the language by modifying the Lua implementation of the language and test the changes immediately.
+If the option `BC_PRELOAD` in `src/Makefile` is activated you just need to recompile `luajit-x`.
+
+If you works with the Lua files of the language toolkit you may choose to disable the `BC_PRELOAD` variable to avoid recompiling the executable for each change in the Lua code.
 
 ### Generated Bytecode ###
 
@@ -222,7 +235,6 @@ or alternatively:
 ```
 
 where we suppose that you are running `luajit-x` from the language toolkit's root directory.
-This is somewhat *required* since the `luajit-x` program needs to be able to find the lang/* Lua modules when is executed.
 
 Either way, when you use one of the two commands above to generate the bytecode you will the see following on the screen:
 

lang/.gitignore

@@ -0,0 +1,2 @@
+*.h
+

lang/Makefile

@@ -0,0 +1,16 @@
+
+LANG_LUA_FILES = ast-boolean-const-eval.lua ast-const-eval.lua bcread.lua bcsave.lua \
+	bytecode.lua compile.lua generator.lua lexer.lua lua-ast.lua operator.lua \
+	parser.lua reader.lua
+
+LANG_H_FILES = $(LANG_LUA_FILES:%.lua=%.h)
+
+all: $(LANG_H_FILES)
+
+clean:
+	rm -fr $(LANG_H_FILES)
+
+%.h: %.lua
+	luajit -b $< $@
+
+.PHONY: all clean

src/Makefile

@@ -18,6 +18,10 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 #
 
+# If the following variable is set to "yes" the lang.* modules will be
+# embedded in the executable and preloaded into the Lua's state.
+BC_PRELOAD = yes
+
 ifneq (,$(findstring Windows,$(OS)))
   HOST_SYS= Windows
 else
@@ -40,14 +44,22 @@ else
   LUAJIT_LIBS := $(shell pkg-config luajit --libs)
 endif
 
+CC = gcc
 CFLAGS = -g -Wall
 
 CFLAGS += $(LUAJIT_CFLAGS)
 LIBS += $(LUAJIT_LIBS)
 
+INCLUDES += -I..
+
 COMPILE = $(CC) $(CFLAGS) $(DEFS) $(INCLUDES)
 
-LANG_SRC_FILES = language.c language_loaders.c
+ifeq ($(strip $(BC_PRELOAD)),yes)
+  LANG_SRC_FILES = language_bcloader.c
+  DEFS += -DBC_PRELOAD
+endif
+
+LANG_SRC_FILES += language.c language_loaders.c
 LANG_OBJ_FILES := $(LANG_SRC_FILES:%.c=%.o)
 DEP_FILES := $(LANG_SRC_FILES:%.c=.deps/%.P)
 
@@ -58,7 +70,7 @@ TARGETS = liblang.a
 all: $(TARGETS) luajit-x
 
 luajit-x: $(LANG_OBJ_FILES) luajit-x.o
-	$(COMPILE) -o $@ $(LANG_OBJ_FILES) luajit-x.o $(LIBS) $(LFLAGS)
+	$(CC) -o $@ $(LANG_OBJ_FILES) luajit-x.o $(LIBS)
 
 liblang.a: $(LANG_OBJ_FILES)
 	@echo Archive $@

src/language.c

@@ -9,6 +9,10 @@
 #include "lualib.h"
 #include "language.h"
 
+#ifdef BC_PRELOAD
+#include "language_bcloader.h"
+#endif
+
 #define likely(x)   __builtin_expect(!!(x), 1)
 #define unlikely(x) __builtin_expect(!!(x), 0)
 
@@ -43,6 +47,9 @@ language_init(lua_State *L) {
         return LUA_ERRRUN;
     }
     luaL_openlibs(parser_L);
+#ifdef BC_PRELOAD
+    language_bc_preload(parser_L);
+#endif
 
     lua_getglobal(parser_L, "require");
     lua_pushstring(parser_L, "lang.compile");

src/language_bcloader.c

@@ -0,0 +1,56 @@
+#include <string.h>
+#include <lua.h>
+#include <lauxlib.h>
+
+#include "language_bcloader.h"
+
+#include "lang/ast-boolean-const-eval.h"
+#include "lang/ast-const-eval.h"
+#include "lang/bcread.h"
+#include "lang/bcsave.h"
+#include "lang/bytecode.h"
+#include "lang/compile.h"
+#include "lang/generator.h"
+#include "lang/lexer.h"
+#include "lang/lua-ast.h"
+#include "lang/operator.h"
+#include "lang/parser.h"
+#include "lang/reader.h"
+
+struct bcpair {
+    const char *name;
+    const char *bc;
+    size_t size;
+};
+
+static struct bcpair bcmodule[] = {
+    {"ast-boolean-const-eval", luaJIT_BC_ast_boolean_const_eval, luaJIT_BC_ast_boolean_const_eval_SIZE},
+    {"ast-const-eval", luaJIT_BC_ast_const_eval, luaJIT_BC_ast_const_eval_SIZE},
+    {"bcread", luaJIT_BC_bcread, luaJIT_BC_bcread_SIZE},
+    {"bcsave", luaJIT_BC_bcsave, luaJIT_BC_bcsave_SIZE},
+    {"bytecode", luaJIT_BC_bytecode, luaJIT_BC_bytecode_SIZE},
+    {"compile", luaJIT_BC_compile, luaJIT_BC_compile_SIZE},
+    {"generator", luaJIT_BC_generator, luaJIT_BC_generator_SIZE},
+    {"lexer", luaJIT_BC_lexer, luaJIT_BC_lexer_SIZE},
+    {"lua-ast", luaJIT_BC_lua_ast, luaJIT_BC_lua_ast_SIZE},
+    {"operator", luaJIT_BC_operator, luaJIT_BC_operator_SIZE},
+    {"parser", luaJIT_BC_parser, luaJIT_BC_parser_SIZE},
+    {"reader", luaJIT_BC_reader, luaJIT_BC_reader_SIZE},
+    {0, 0, 0}
+};
+
+/* Load into package.preload lang.* modules using embedded bytecode. */
+void language_bc_preload(lua_State *L)
+{
+    struct bcpair *i;
+    lua_getfield(L, LUA_GLOBALSINDEX, "package");
+    lua_getfield(L, -1, "preload");
+    for (i = bcmodule; i->name; i++) {
+        lua_pushstring(L, "lang.");
+        lua_pushstring(L, i->name);
+        lua_concat(L, 2);
+        luaL_loadbuffer(L, i->bc, i->size, lua_tostring(L, -1));
+        lua_rawset(L, -3);
+    }
+    lua_pop(L, 2);
+}

src/language_bcloader.h

@@ -0,0 +1,8 @@
+#ifndef LANG_LANGBC_H
+#define LANG_LANGBC_H
+
+#include <lua.h>
+
+extern void language_bc_preload(lua_State *L);
+
+#endif

src/luajit-x.c

@@ -24,6 +24,10 @@
 #include "language.h"
 #include "language_loaders.h"
 
+#ifdef BC_PRELOAD
+#include "language_bcloader.h"
+#endif
+
 #if defined(__linux__)
 #include <unistd.h>
 #define lua_stdin_is_tty()  isatty(0)
@@ -310,6 +314,9 @@ static int loadlangmodule(lua_State *L)
 static int dobytecode(lua_State *L, char **argv)
 {
   int narg = 0;
+#ifdef BC_PRELOAD
+  language_bc_preload(L);
+#endif
   lua_pushliteral(L, "bcsave");
   if (loadlangmodule(L))
     return 1;