Skip to content

Commit

Permalink
Merge pull request #8917 from ifreund/fix-float-tokenize2
Browse files Browse the repository at this point in the history 
stage1, stage2: disallow 1.e9 and 0x1.p9 as float literals
  • Loading branch information
@andrewrk
andrewrk authored Jun 1, 2021
2 parents d496400 + abd1c75 commit f693247
Show file tree
Hide file tree
Showing 10 changed files with 32 additions and 36 deletions.
15 changes: 9 additions & 6 deletions lib/std/fmt.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1121,9 +1121,12 @@ pub fn formatFloatHexadecimal(

try writer.writeAll("0x");
try writer.writeByte(buf[0]);
if (options.precision != @as(usize, 0))
try writer.writeAll(".");
const trimmed = mem.trimRight(u8, buf[1..], "0");
if (options.precision) |precision| {
if (precision > 0) try writer.writeAll(".");
} else if (trimmed.len > 0) {
try writer.writeAll(".");
}
try writer.writeAll(trimmed);
// Add trailing zeros if explicitly requested.
if (options.precision) |precision| if (precision > 0) {
Expand Down Expand Up @@ -2048,10 +2051,10 @@ test "float.hexadecimal" {
try expectFmt("f64: 0x1.5555555555555p-2", "f64: {x}", .{@as(f64, 1.0 / 3.0)});
try expectFmt("f128: 0x1.5555555555555555555555555555p-2", "f128: {x}", .{@as(f128, 1.0 / 3.0)});

try expectFmt("f16: 0x1.p-14", "f16: {x}", .{@as(f16, math.f16_min)});
try expectFmt("f32: 0x1.p-126", "f32: {x}", .{@as(f32, math.f32_min)});
try expectFmt("f64: 0x1.p-1022", "f64: {x}", .{@as(f64, math.f64_min)});
try expectFmt("f128: 0x1.p-16382", "f128: {x}", .{@as(f128, math.f128_min)});
try expectFmt("f16: 0x1p-14", "f16: {x}", .{@as(f16, math.f16_min)});
try expectFmt("f32: 0x1p-126", "f32: {x}", .{@as(f32, math.f32_min)});
try expectFmt("f64: 0x1p-1022", "f64: {x}", .{@as(f64, math.f64_min)});
try expectFmt("f128: 0x1p-16382", "f128: {x}", .{@as(f128, math.f128_min)});

try expectFmt("f16: 0x0.004p-14", "f16: {x}", .{@as(f16, math.f16_true_min)});
try expectFmt("f32: 0x0.000002p-126", "f32: {x}", .{@as(f32, math.f32_true_min)});
Expand Down
2 changes: 1 addition & 1 deletion lib/std/special/compiler_rt/fixunstfdi_test.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ test "fixunstfdi" {
try test__fixunstfdi(0x1.0000000000000000p+63, 0x8000000000000000);
try test__fixunstfdi(0x1.FFFFFFFFFFFFFFFCp+62, 0x7FFFFFFFFFFFFFFF);
try test__fixunstfdi(0x1.FFFFFFFFFFFFFFF8p+62, 0x7FFFFFFFFFFFFFFE);
try test__fixunstfdi(0x1.p+64, 0xFFFFFFFFFFFFFFFF);
try test__fixunstfdi(0x1p+64, 0xFFFFFFFFFFFFFFFF);

try test__fixunstfdi(-0x1.0000000000000000p+63, 0);
try test__fixunstfdi(-0x1.FFFFFFFFFFFFFFFCp+62, 0);
Expand Down
2 changes: 1 addition & 1 deletion lib/std/special/compiler_rt/fixunstfsi_test.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,5 +23,5 @@ test "fixunstfsi" {
try test__fixunstfsi(0x1.23456789abcdefp+256, 0xffffffff);
try test__fixunstfsi(-0x1.23456789abcdefp+3, 0x0);

try test__fixunstfsi(0x1.p+32, 0xFFFFFFFF);
try test__fixunstfsi(0x1p+32, 0xFFFFFFFF);
}
2 changes: 1 addition & 1 deletion lib/std/special/compiler_rt/fixunstfti_test.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ test "fixunstfti" {
try test__fixunstfti(-0.01, 0);
try test__fixunstfti(-0.99, 0);

try test__fixunstfti(0x1.p+128, 0xffffffffffffffffffffffffffffffff);
try test__fixunstfti(0x1p+128, 0xffffffffffffffffffffffffffffffff);

try test__fixunstfti(0x1.FFFFFEp+126, 0x7fffff80000000000000000000000000);
try test__fixunstfti(0x1.FFFFFEp+127, 0xffffff00000000000000000000000000);
Expand Down
4 changes: 2 additions & 2 deletions lib/std/special/compiler_rt/mulXf3_test.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,12 +92,12 @@ test "multf3" {
// Denormal operands.
try test__multf3(
0x0.0000000000000000000000000001p-16382,
0x1.p16383,
0x1p16383,
0x3f90000000000000,
0x0,
);
try test__multf3(
0x1.p16383,
0x1p16383,
0x0.0000000000000000000000000001p-16382,
0x3f90000000000000,
0x0,
Expand Down
8 changes: 4 additions & 4 deletions lib/std/zig/parser_test.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4152,13 +4152,13 @@ test "zig fmt: hex literals with underscore separators" {
test "zig fmt: decimal float literals with underscore separators" {
try testTransform(
\\pub fn main() void {
\\ const a:f64=(10.0e-0+(10.e+0))+10_00.00_00e-2+00_00.00_10e+4;
\\ const a:f64=(10.0e-0+(10.0e+0))+10_00.00_00e-2+00_00.00_10e+4;
\\ const b:f64=010.0--0_10.0+0_1_0.0_0+1e2;
\\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });
\\}
,
\\pub fn main() void {
\\ const a: f64 = (10.0e-0 + (10.e+0)) + 10_00.00_00e-2 + 00_00.00_10e+4;
\\ const a: f64 = (10.0e-0 + (10.0e+0)) + 10_00.00_00e-2 + 00_00.00_10e+4;
\\ const b: f64 = 010.0 - -0_10.0 + 0_1_0.0_0 + 1e2;
\\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });
\\}
Expand All @@ -4169,13 +4169,13 @@ test "zig fmt: decimal float literals with underscore separators" {
test "zig fmt: hexadeciaml float literals with underscore separators" {
try testTransform(
\\pub fn main() void {
\\ const a: f64 = (0x10.0p-0+(0x10.p+0))+0x10_00.00_00p-8+0x00_00.00_10p+16;
\\ const a: f64 = (0x10.0p-0+(0x10.0p+0))+0x10_00.00_00p-8+0x00_00.00_10p+16;
\\ const b: f64 = 0x0010.0--0x00_10.0+0x10.00+0x1p4;
\\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });
\\}
,
\\pub fn main() void {
\\ const a: f64 = (0x10.0p-0 + (0x10.p+0)) + 0x10_00.00_00p-8 + 0x00_00.00_10p+16;
\\ const a: f64 = (0x10.0p-0 + (0x10.0p+0)) + 0x10_00.00_00p-8 + 0x00_00.00_10p+16;
\\ const b: f64 = 0x0010.0 - -0x00_10.0 + 0x10.00 + 0x1p4;
\\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });
\\}
Expand Down
14 changes: 3 additions & 11 deletions lib/std/zig/tokenizer.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1210,10 +1210,6 @@ pub const Tokenizer = struct {
state = .start;
break;
},
'e', 'E' => {
result.tag = .float_literal;
state = .float_exponent_unsigned;
},
'0'...'9' => {
result.tag = .float_literal;
state = .float_fraction_dec;
Expand All @@ -1232,10 +1228,6 @@ pub const Tokenizer = struct {
state = .start;
break;
},
'p', 'P' => {
result.tag = .float_literal;
state = .float_exponent_unsigned;
},
'0'...'9', 'a'...'f', 'A'...'F' => {
result.tag = .float_literal;
state = .float_fraction_hex;
Expand Down Expand Up @@ -1861,14 +1853,14 @@ test "tokenizer - number literals decimal" {
try testTokenize("0e0", &.{.float_literal});
try testTokenize("1e0", &.{.float_literal});
try testTokenize("1e100", &.{.float_literal});
try testTokenize("1.e100", &.{.float_literal});
try testTokenize("1.0e100", &.{.float_literal});
try testTokenize("1.0e+100", &.{.float_literal});
try testTokenize("1.0e-100", &.{.float_literal});
try testTokenize("1_0_0_0.0_0_0_0_0_1e1_0_0_0", &.{.float_literal});

try testTokenize("1.", &.{.invalid});
try testTokenize("1e", &.{.invalid});
try testTokenize("1.e100", &.{ .invalid, .identifier });
try testTokenize("1.0e1f0", &.{ .invalid, .identifier });
try testTokenize("1.0p100", &.{ .invalid, .identifier });
try testTokenize("1.0p-100", &.{ .invalid, .identifier, .minus, .integer_literal });
Expand Down Expand Up @@ -2019,6 +2011,7 @@ test "tokenizer - number literals hexadecimal" {
try testTokenize("0x1.", &.{.invalid});
try testTokenize("0xF.", &.{.invalid});
try testTokenize("0x1.+0xF.", &.{ .invalid, .plus, .invalid });
try testTokenize("0xff.p10", &.{ .invalid, .identifier });

try testTokenize("0x0123456.789ABCDEF", &.{.float_literal});
try testTokenize("0x0_123_456.789_ABC_DEF", &.{.float_literal});
Expand All @@ -2027,7 +2020,6 @@ test "tokenizer - number literals hexadecimal" {
try testTokenize("0x0.0p0", &.{.float_literal});
try testTokenize("0xff.ffp10", &.{.float_literal});
try testTokenize("0xff.ffP10", &.{.float_literal});
try testTokenize("0xff.p10", &.{.float_literal});
try testTokenize("0xffp10", &.{.float_literal});
try testTokenize("0xff_ff.ff_ffp1_0_0_0", &.{.float_literal});
try testTokenize("0xf_f_f_f.f_f_f_fp+1_000", &.{.float_literal});
Expand All @@ -2038,7 +2030,7 @@ test "tokenizer - number literals hexadecimal" {
try testTokenize("0x1p", &.{.invalid});
try testTokenize("0xfp0z1", &.{ .invalid, .identifier });
try testTokenize("0xff.ffpff", &.{ .invalid, .identifier });
try testTokenize("0x0.p", &.{.invalid});
try testTokenize("0x0.p", &.{ .invalid, .identifier });
try testTokenize("0x0.z", &.{ .invalid, .identifier });
try testTokenize("0x0._", &.{ .invalid, .identifier });
try testTokenize("0x0_.0", &.{ .invalid, .period, .integer_literal });
Expand Down
8 changes: 0 additions & 8 deletions src/stage1/tokenizer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1286,10 +1286,6 @@ void tokenize(const char *source, Tokenization *out) {
t.column -= 1;
t.state = TokenizeState_start;
continue;
case 'e':
case 'E':
t.state = TokenizeState_float_exponent_unsigned;
break;
case DIGIT:
t.state = TokenizeState_float_fraction_dec;
break;
Expand All @@ -1308,10 +1304,6 @@ void tokenize(const char *source, Tokenization *out) {
t.column -= 1;
t.state = TokenizeState_start;
continue;
case 'p':
case 'P':
t.state = TokenizeState_float_exponent_unsigned;
break;
case HEXDIGIT:
t.out->ids.last() = TokenIdFloatLiteral;
t.state = TokenizeState_float_fraction_hex;
Expand Down
11 changes: 10 additions & 1 deletion src/translate_c.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4932,8 +4932,17 @@ fn parseCNumLit(c: *Context, m: *MacroCtx) ParseError!Node {
}
},
.FloatLiteral => |suffix| {
if (lit_bytes[0] == '.')
const dot_index = mem.indexOfScalar(u8, lit_bytes, '.').?;
if (dot_index == 0) {
lit_bytes = try std.fmt.allocPrint(c.arena, "0{s}", .{lit_bytes});
} else if (dot_index + 1 == lit_bytes.len or !std.ascii.isDigit(lit_bytes[dot_index + 1])) {
// If the literal lacks a digit after the `.`, we need to
// add one since `1.` or `1.e10` would be invalid syntax in Zig.
lit_bytes = try std.fmt.allocPrint(c.arena, "{s}0{s}", .{
lit_bytes[0 .. dot_index + 1],
lit_bytes[dot_index + 1 ..],
});
}
if (suffix == .none) {
return transCreateNodeNumber(c, lit_bytes, .float);
}
Expand Down
2 changes: 1 addition & 1 deletion test/translate_c.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1110,7 +1110,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void {
\\#define BAR .12345
, &[_][]const u8{
"pub const foo = @as(f32, 3.14);",
"pub const bar = @as(c_longdouble, 16.e-2);",
"pub const bar = @as(c_longdouble, 16.0e-2);",
"pub const FOO = 0.12345;",
"pub const BAR = 0.12345;",
});
Expand Down

0 comments on commit f693247

Please sign in to comment.