Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

src: support UTF-8 in compiled-in JS source files #11129

Merged
merged 2 commits into from
Feb 13, 2017
Merged

src: support UTF-8 in compiled-in JS source files #11129

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c30fc8d
src: support UTF-8 in compiled-in JS source files
bnoordhuis Feb 2, 2017
fd8587e
lib: replace \u2019 with regular ascii quote
bnoordhuis Feb 6, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion lib/buffer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ function fromString(string, encoding) {
var b = new FastBuffer(allocPool, poolOffset, length);
var actual = b.write(string, encoding);
if (actual !== length) {
// byteLength() may overestimate. That’s a rare case, though.
// byteLength() may overestimate. That's a rare case, though.
b = new FastBuffer(allocPool, poolOffset, actual);
}
poolOffset += actual;
Expand Down
2 changes: 1 addition & 1 deletion lib/repl.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,7 +318,7 @@ function REPLServer(prompt,
let previouslyInRawMode;
if (self.breakEvalOnSigint) {
// Start the SIGINT watchdog before entering raw mode so that a very
// quick Ctrl+C doesn’t lead to aborting the process completely.
// quick Ctrl+C doesn't lead to aborting the process completely.
utilBinding.startSigintWatchdog();
previouslyInRawMode = self._setRawMode(false);
}
Expand Down
8 changes: 4 additions & 4 deletions node.gyp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@
'src',
'tools/msvs/genfiles',
'deps/uv/src/ares',
'<(SHARED_INTERMEDIATE_DIR)', # for node_natives.h
'<(SHARED_INTERMEDIATE_DIR)',
],

'sources': [
Expand All @@ -167,7 +167,6 @@
'src/node_debug_options.cc',
'src/node_file.cc',
'src/node_http_parser.cc',
'src/node_javascript.cc',
'src/node_main.cc',
'src/node_os.cc',
'src/node_revert.cc',
Expand Down Expand Up @@ -243,11 +242,11 @@
'deps/http_parser/http_parser.h',
'deps/v8/include/v8.h',
'deps/v8/include/v8-debug.h',
'<(SHARED_INTERMEDIATE_DIR)/node_natives.h',
# javascript files to make for an even more pleasant IDE experience
'<@(library_files)',
# node.gyp is added to the project by default.
'common.gypi',
'<(SHARED_INTERMEDIATE_DIR)/node_javascript.cc',
],

'defines': [
Expand Down Expand Up @@ -720,12 +719,13 @@
'actions': [
{
'action_name': 'node_js2c',
'process_outputs_as_sources': 1,
'inputs': [
'<@(library_files)',
'./config.gypi',
],
'outputs': [
'<(SHARED_INTERMEDIATE_DIR)/node_natives.h',
'<(SHARED_INTERMEDIATE_DIR)/node_javascript.cc',
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You'd expect INTERMEDIATE_DIR to work here but it doesn't: js2c generates the file but the build doesn't pick it up (and I updated the 'sources' list in case you're wondering.) I decided to leave well enough alone for now.

],
'conditions': [
[ 'node_use_dtrace=="false" and node_use_etw=="false"', {
Expand Down
51 changes: 0 additions & 51 deletions src/node_javascript.cc
View file
Open in desktop

This file was deleted.

117 changes: 77 additions & 40 deletions tools/js2c.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,16 @@
import string


def ToCString(contents):
step = 20
slices = (contents[i:i+step] for i in xrange(0, len(contents), step))
slices = map(lambda s: ','.join(str(ord(c)) for c in s), slices)
def ToCArray(elements, step=10):
slices = (elements[i:i+step] for i in xrange(0, len(elements), step))
slices = map(lambda s: ','.join(str(x) for x in s), slices)
return ',\n'.join(slices)


def ToCString(contents):
return ToCArray(map(ord, contents), step=20)


def ReadFile(filename):
file = open(filename, "rt")
try:
Expand Down Expand Up @@ -161,34 +164,72 @@ def ReadMacros(lines):
return (constants, macros)


HEADER_TEMPLATE = """\
#ifndef NODE_NATIVES_H_
#define NODE_NATIVES_H_
TEMPLATE = """
#include "node.h"
#include "node_javascript.h"
#include "v8.h"
#include "env.h"
#include "env-inl.h"

#include <stdint.h>
namespace node {{

#define NODE_NATIVES_MAP(V) \\
{node_natives_map}
{definitions}

v8::Local<v8::String> MainSource(Environment* env) {{
return internal_bootstrap_node_value.ToStringChecked(env->isolate());
}}

void DefineJavaScript(Environment* env, v8::Local<v8::Object> target) {{
{initializers}
}}

namespace node {{
{sources}
}} // namespace node
"""

#endif // NODE_NATIVES_H_
ONE_BYTE_STRING = """
static const uint8_t raw_{var}[] = {{ {data} }};
static struct : public v8::String::ExternalOneByteStringResource {{
const char* data() const override {{
return reinterpret_cast<const char*>(raw_{var});
}}
size_t length() const override {{ return arraysize(raw_{var}); }}
void Dispose() override {{ /* Default calls `delete this`. */ }}
v8::Local<v8::String> ToStringChecked(v8::Isolate* isolate) {{
return v8::String::NewExternalOneByte(isolate, this).ToLocalChecked();
}}
}} {var};
"""

TWO_BYTE_STRING = """
static const uint16_t raw_{var}[] = {{ {data} }};
static struct : public v8::String::ExternalStringResource {{
const uint16_t* data() const override {{ return raw_{var}; }}
size_t length() const override {{ return arraysize(raw_{var}); }}
void Dispose() override {{ /* Default calls `delete this`. */ }}
v8::Local<v8::String> ToStringChecked(v8::Isolate* isolate) {{
return v8::String::NewExternalTwoByte(isolate, this).ToLocalChecked();
}}
}} {var};
"""

NODE_NATIVES_MAP = """\
V({escaped_id}) \\
INITIALIZER = """\
CHECK(target->Set(env->context(),
{key}.ToStringChecked(env->isolate()),
{value}.ToStringChecked(env->isolate())).FromJust());
"""


SOURCES = """\
static const uint8_t {escaped_id}_name[] = {{
{name}}};
static const uint8_t {escaped_id}_data[] = {{
{data}}};
"""
def Render(var, data):
# Treat non-ASCII as UTF-8 and convert it to UTF-16.
if any(ord(c) > 127 for c in data):
template = TWO_BYTE_STRING
data = map(ord, data.decode('utf-8').encode('utf-16be'))
data = [data[i] * 256 + data[i+1] for i in xrange(0, len(data), 2)]
data = ToCArray(data)
else:
template = ONE_BYTE_STRING
data = ToCString(data)
return template.format(var=var, data=data)


def JS2C(source, target):
Expand All @@ -207,36 +248,32 @@ def JS2C(source, target):
(consts, macros) = ReadMacros(macro_lines)

# Build source code lines
node_natives_map = []
sources = []
definitions = []
initializers = []

for s in modules:
lines = ReadFile(str(s))
for name in modules:
lines = ReadFile(str(name))
lines = ExpandConstants(lines, consts)
lines = ExpandMacros(lines, macros)
data = ToCString(lines)

# On Windows, "./foo.bar" in the .gyp file is passed as "foo.bar"
# so don't assume there is always a slash in the file path.
if '/' in s or '\\' in s:
id = '/'.join(re.split('/|\\\\', s)[1:])
else:
id = s

if '.' in id:
id = id.split('.', 1)[0]
if '/' in name or '\\' in name:
name = '/'.join(re.split('/|\\\\', name)[1:])

name = ToCString(id)
escaped_id = id.replace('-', '_').replace('/', '_')
node_natives_map.append(NODE_NATIVES_MAP.format(**locals()))
sources.append(SOURCES.format(**locals()))
name = name.split('.', 1)[0]
var = name.replace('-', '_').replace('/', '_')
key = '%s_key' % var
value = '%s_value' % var

node_natives_map = ''.join(node_natives_map)
sources = ''.join(sources)
definitions.append(Render(key, name))
definitions.append(Render(value, lines))
initializers.append(INITIALIZER.format(key=key, value=value))

# Emit result
output = open(str(target[0]), "w")
output.write(HEADER_TEMPLATE.format(**locals()))
output.write(TEMPLATE.format(definitions=''.join(definitions),
initializers=''.join(initializers)))
output.close()

def main():
Expand Down