Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
src: support UTF-8 in compiled-in JS source files
Detect it when source files in lib/ are not ASCII. Decode them as UTF-8 and store them as UTF-16 in the binary so they can be used as external string resources without non-ASCII characters getting mangled. Fixes: #10673 PR-URL: #11129 Reviewed-By: Anna Henningsen <[email protected]> Reviewed-By: James M Snell <[email protected]>
- Loading branch information
1 parent
f2023d7
commit c30fc8d
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -37,13 +37,16 @@ | |
| import string | ||
|
|
||
|
|
||
| def ToCArray(elements, step=10): | ||
| slices = (elements[i:i+step] for i in xrange(0, len(elements), step)) | ||
| slices = map(lambda s: ','.join(str(x) for x in s), slices) | ||
| return ',\n'.join(slices) | ||
|
|
||
|
|
||
| def ToCString(contents): | ||
| return ToCArray(map(ord, contents), step=20) | ||
|
|
||
|
|
||
| def ReadFile(filename): | ||
| file = open(filename, "rt") | ||
| try: | ||
|
|
@@ -161,34 +164,72 @@ def ReadMacros(lines): | |
| return (constants, macros) | ||
|
|
||
|
|
||
| TEMPLATE = """ | ||
| #include "node.h" | ||
| #include "node_javascript.h" | ||
| #include "v8.h" | ||
| #include "env.h" | ||
| #include "env-inl.h" | ||
| namespace node {{ | ||
| {definitions} | ||
| v8::Local<v8::String> MainSource(Environment* env) {{ | ||
| return internal_bootstrap_node_value.ToStringChecked(env->isolate()); | ||
This comment has been minimized.
Sorry, something went wrong.
This comment has been minimized.
Sorry, something went wrong. 
addaleax
Member
|
||
| }} | ||
| void DefineJavaScript(Environment* env, v8::Local<v8::Object> target) {{ | ||
| {initializers} | ||
| }} | ||
| }} // namespace node | ||
| """ | ||
|
|
||
| ONE_BYTE_STRING = """ | ||
| static const uint8_t raw_{var}[] = {{ {data} }}; | ||
| static struct : public v8::String::ExternalOneByteStringResource {{ | ||
| const char* data() const override {{ | ||
| return reinterpret_cast<const char*>(raw_{var}); | ||
| }} | ||
| size_t length() const override {{ return arraysize(raw_{var}); }} | ||
| void Dispose() override {{ /* Default calls `delete this`. */ }} | ||
| v8::Local<v8::String> ToStringChecked(v8::Isolate* isolate) {{ | ||
| return v8::String::NewExternalOneByte(isolate, this).ToLocalChecked(); | ||
| }} | ||
| }} {var}; | ||
| """ | ||
|
|
||
| TWO_BYTE_STRING = """ | ||
| static const uint16_t raw_{var}[] = {{ {data} }}; | ||
| static struct : public v8::String::ExternalStringResource {{ | ||
| const uint16_t* data() const override {{ return raw_{var}; }} | ||
| size_t length() const override {{ return arraysize(raw_{var}); }} | ||
| void Dispose() override {{ /* Default calls `delete this`. */ }} | ||
| v8::Local<v8::String> ToStringChecked(v8::Isolate* isolate) {{ | ||
| return v8::String::NewExternalTwoByte(isolate, this).ToLocalChecked(); | ||
| }} | ||
| }} {var}; | ||
| """ | ||
|
|
||
| INITIALIZER = """\ | ||
| CHECK(target->Set(env->context(), | ||
| {key}.ToStringChecked(env->isolate()), | ||
| {value}.ToStringChecked(env->isolate())).FromJust()); | ||
| """ | ||
|
|
||
|
|
||
| def Render(var, data): | ||
| # Treat non-ASCII as UTF-8 and convert it to UTF-16. | ||
| if any(ord(c) > 127 for c in data): | ||
| template = TWO_BYTE_STRING | ||
| data = map(ord, data.decode('utf-8').encode('utf-16be')) | ||
| data = [data[i] * 256 + data[i+1] for i in xrange(0, len(data), 2)] | ||
| data = ToCArray(data) | ||
| else: | ||
| template = ONE_BYTE_STRING | ||
| data = ToCString(data) | ||
| return template.format(var=var, data=data) | ||
|
|
||
|
|
||
| def JS2C(source, target): | ||
|
|
@@ -207,36 +248,32 @@ def JS2C(source, target): | |
| (consts, macros) = ReadMacros(macro_lines) | ||
|
|
||
| # Build source code lines | ||
| definitions = [] | ||
| initializers = [] | ||
|
|
||
| for name in modules: | ||
| lines = ReadFile(str(name)) | ||
| lines = ExpandConstants(lines, consts) | ||
| lines = ExpandMacros(lines, macros) | ||
|
|
||
| # On Windows, "./foo.bar" in the .gyp file is passed as "foo.bar" | ||
| # so don't assume there is always a slash in the file path. | ||
| if '/' in name or '\\' in name: | ||
| name = '/'.join(re.split('/|\\\\', name)[1:]) | ||
|
|
||
| name = name.split('.', 1)[0] | ||
| var = name.replace('-', '_').replace('/', '_') | ||
| key = '%s_key' % var | ||
| value = '%s_value' % var | ||
|
|
||
| definitions.append(Render(key, name)) | ||
| definitions.append(Render(value, lines)) | ||
| initializers.append(INITIALIZER.format(key=key, value=value)) | ||
|
|
||
| # Emit result | ||
| output = open(str(target[0]), "w") | ||
| output.write(TEMPLATE.format(definitions=''.join(definitions), | ||
| initializers=''.join(initializers))) | ||
| output.close() | ||
|
|
||
| def main(): | ||
|
|
||
this causes
error: use of undeclared identifier 'internal_bootstrap_node_value'for anything else that uses js2c.py