From 119ac13eda7fe95b6c1139b4bd20e05928abe427 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Fri, 7 Feb 2020 14:17:40 +1100 Subject: [PATCH 1/9] std: add .startsWith and .endsWith to std.ArrayList --- lib/std/array_list.zig | 11 +++++++++++ lib/std/buffer.zig | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/std/array_list.zig b/lib/std/array_list.zig index d1946be02aef..33a1220f9dca 100644 --- a/lib/std/array_list.zig +++ b/lib/std/array_list.zig @@ -248,6 +248,17 @@ pub fn AlignedArrayList(comptime T: type, comptime alignment: ?u29) type { if (self.len == 0) return null; return self.pop(); } + + pub fn startsWith(self: Self, m: []const T) bool { + if (self.len < m.len) return false; + return mem.eql(T, self.items[0..m.len], m); + } + + pub fn endsWith(self: Self, m: []const T) bool { + if (self.len < m.len) return false; + const start = self.len - m.len; + return mem.eql(T, self.items[start..self.len], m); + } }; } diff --git a/lib/std/buffer.zig b/lib/std/buffer.zig index f808af14852d..bcefe5bdfb9f 100644 --- a/lib/std/buffer.zig +++ b/lib/std/buffer.zig @@ -133,7 +133,7 @@ pub const Buffer = struct { pub fn startsWith(self: Buffer, m: []const u8) bool { if (self.len() < m.len) return false; - return mem.eql(u8, self.list.items[0..m.len], m); + return self.list.startsWith(m); } pub fn endsWith(self: Buffer, m: []const u8) bool { From fd23decbd9dc39b044867993b67607e33ec9de45 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Fri, 7 Feb 2020 14:20:21 +1100 Subject: [PATCH 2/9] std: add ArrayList.eql for parity with std.Buffer --- lib/std/array_list.zig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/std/array_list.zig b/lib/std/array_list.zig index 33a1220f9dca..ca006d79e35b 100644 --- a/lib/std/array_list.zig +++ b/lib/std/array_list.zig @@ -249,6 +249,10 @@ pub fn AlignedArrayList(comptime T: type, comptime alignment: ?u29) type { return self.pop(); } + pub fn eql(self: Self, m: []const T) bool { + return mem.eql(T, self.toSliceConst(), m); + } + pub fn startsWith(self: Self, m: []const T) bool { if (self.len < m.len) return false; return mem.eql(T, self.items[0..m.len], m); From bcf56c32eb11306613c92128c1b95ff280ba4f68 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Fri, 7 Feb 2020 14:43:09 +1100 Subject: [PATCH 3/9] std: use ArrayList rather than Buffer for in_stream helper functions Buffer's behaviour of retaining a trailing 0 isn't helpful here --- lib/std/io/in_stream.zig | 51 ++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/lib/std/io/in_stream.zig b/lib/std/io/in_stream.zig index 4f3bac3c21a7..809e388ebd8a 100644 --- a/lib/std/io/in_stream.zig +++ b/lib/std/io/in_stream.zig @@ -65,22 +65,31 @@ pub fn InStream(comptime ReadError: type) type { /// If `buffer.len()` would exceed `max_size`, `error.StreamTooLong` is returned and /// the contents read from the stream are lost. pub fn readAllBuffer(self: *Self, buffer: *Buffer, max_size: usize) !void { - try buffer.resize(0); + try buffer.list.ensureCapacity(1); + buffer.list.len = 0; + errdefer buffer.resize(0) catch unreachable; // make sure we leave buffer in a valid state on error + try self.readAllArrayList(&buffer.list, max_size); + try buffer.list.append(0); + } + /// Appends to the ArrayList contents by reading from the stream until end of stream is found. + /// If the ArrayList length would exceed `max_size`, `error.StreamTooLong` is returned and the contents + /// read from the stream so far are lost. + pub fn readAllArrayList(self: *Self, array_list: *std.ArrayList(u8), max_size: usize) !void { var actual_buf_len: usize = 0; while (true) { - const dest_slice = buffer.toSlice()[actual_buf_len..]; + const dest_slice = array_list.toSlice()[actual_buf_len..]; const bytes_read = try self.readFull(dest_slice); actual_buf_len += bytes_read; if (bytes_read != dest_slice.len) { - buffer.shrink(actual_buf_len); + array_list.shrink(actual_buf_len); return; } const new_buf_size = math.min(max_size, actual_buf_len + mem.page_size); if (new_buf_size == actual_buf_len) return error.StreamTooLong; - try buffer.resize(new_buf_size); + try array_list.resize(new_buf_size); } } @@ -89,11 +98,10 @@ pub fn InStream(comptime ReadError: type) type { /// Caller owns returned memory. /// If this function returns an error, the contents from the stream read so far are lost. pub fn readAllAlloc(self: *Self, allocator: *mem.Allocator, max_size: usize) ![]u8 { - var buf = Buffer.initNull(allocator); - defer buf.deinit(); - - try self.readAllBuffer(&buf, max_size); - return buf.toOwnedSlice(); + var array_list = std.ArrayList(u8).init(allocator); + defer array_list.deinit(); + try self.readAllArrayList(&array_list, max_size); + return array_list.toOwnedSlice(); } /// Replaces `buffer` contents by reading from the stream until `delimiter` is found. @@ -101,8 +109,18 @@ pub fn InStream(comptime ReadError: type) type { /// If `buffer.len()` would exceed `max_size`, `error.StreamTooLong` is returned and the contents /// read from the stream so far are lost. pub fn readUntilDelimiterBuffer(self: *Self, buffer: *Buffer, delimiter: u8, max_size: usize) !void { - try buffer.resize(0); + try buffer.list.ensureCapacity(1); + buffer.list.len = 0; + errdefer buffer.resize(0) catch unreachable; // make sure we leave buffer in a valid state on error + try self.readUntilDelimiterArrayList(&buffer.list, delimiter, max_size); + try buffer.list.append(0); + } + /// Appends to the ArrayList contents by reading from the stream until `delimiter` is found. + /// Does not include the delimiter in the result. + /// If the ArrayList length would exceed `max_size`, `error.StreamTooLong` is returned and the contents + /// read from the stream so far are lost. + pub fn readUntilDelimiterArrayList(self: *Self, array_list: *std.ArrayList(u8), delimiter: u8, max_size: usize) !void { while (true) { var byte: u8 = try self.readByte(); @@ -110,11 +128,11 @@ pub fn InStream(comptime ReadError: type) type { return; } - if (buffer.len() == max_size) { + if (array_list.len == max_size) { return error.StreamTooLong; } - try buffer.appendByte(byte); + try array_list.append(byte); } } @@ -123,11 +141,10 @@ pub fn InStream(comptime ReadError: type) type { /// Caller owns returned memory. /// If this function returns an error, the contents from the stream read so far are lost. pub fn readUntilDelimiterAlloc(self: *Self, allocator: *mem.Allocator, delimiter: u8, max_size: usize) ![]u8 { - var buf = Buffer.initNull(allocator); - defer buf.deinit(); - - try self.readUntilDelimiterBuffer(&buf, delimiter, max_size); - return buf.toOwnedSlice(); + var array_list = std.ArrayList(u8).init(allocator); + defer array_list.deinit(); + try self.readUntilDelimiterArrayList(&array_list, delimiter, max_size); + return array_list.toOwnedSlice(); } /// Reads from the stream until specified byte is found. If the buffer is not From 8f627593ebb7ce073e43f06235fe1d9b9c1a1579 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Fri, 7 Feb 2020 16:11:57 +1100 Subject: [PATCH 4/9] Use in_stream.readAllAlloc where sensible --- lib/std/build.zig | 8 +++----- lib/std/build/run.zig | 35 +++++++++++++++++++++-------------- lib/std/child_process.zig | 16 +++++++--------- test/tests.zig | 16 ++++++---------- 4 files changed, 37 insertions(+), 38 deletions(-) diff --git a/lib/std/build.zig b/lib/std/build.zig index 59e6c4e87c1a..bc8b8acba089 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -926,11 +926,9 @@ pub const Builder = struct { try child.spawn(); - var stdout = std.Buffer.initNull(self.allocator); - defer std.Buffer.deinit(&stdout); - var stdout_file_in_stream = child.stdout.?.inStream(); - try stdout_file_in_stream.stream.readAllBuffer(&stdout, max_output_size); + const stdout = try stdout_file_in_stream.stream.readAllAlloc(self.allocator, max_output_size); + errdefer self.allocator.free(stdout); const term = try child.wait(); switch (term) { @@ -939,7 +937,7 @@ pub const Builder = struct { out_code.* = @truncate(u8, code); return error.ExitCodeFailure; } - return stdout.toOwnedSlice(); + return stdout; }, .Signal, .Stopped, .Unknown => |code| { out_code.* = @truncate(u8, code); diff --git a/lib/std/build/run.zig b/lib/std/build/run.zig index eacf408ba90f..dc4e0df898a3 100644 --- a/lib/std/build/run.zig +++ b/lib/std/build/run.zig @@ -169,26 +169,33 @@ pub const RunStep = struct { return err; }; - var stdout = Buffer.initNull(self.builder.allocator); - var stderr = Buffer.initNull(self.builder.allocator); - // TODO need to poll to read these streams to prevent a deadlock (or rely on evented I/O). + var stdout: []const u8 = undefined; switch (self.stdout_action) { .expect_exact, .expect_matches => { var stdout_file_in_stream = child.stdout.?.inStream(); - stdout_file_in_stream.stream.readAllBuffer(&stdout, max_stdout_size) catch unreachable; + stdout = stdout_file_in_stream.stream.readAllAlloc(self.builder.allocator, max_stdout_size) catch unreachable; }, .inherit, .ignore => {}, } + defer switch (self.stdout_action) { + .expect_exact, .expect_matches => self.builder.allocator.free(stdout), + .inherit, .ignore => {}, + }; - switch (self.stdout_action) { + var stderr: []const u8 = undefined; + switch (self.stdout_behavior) { .expect_exact, .expect_matches => { var stderr_file_in_stream = child.stderr.?.inStream(); - stderr_file_in_stream.stream.readAllBuffer(&stderr, max_stdout_size) catch unreachable; + stderr = stderr_file_in_stream.stream.readAllAlloc(self.builder.allocator, max_stdout_size) catch unreachable; }, .inherit, .ignore => {}, } + defer switch (self.stderr_action) { + .expect_exact, .expect_matches => self.builder.allocator.free(stderr), + .inherit, .ignore => {}, + }; const term = child.wait() catch |err| { warn("Unable to spawn {}: {}\n", .{ argv[0], @errorName(err) }); @@ -216,7 +223,7 @@ pub const RunStep = struct { switch (self.stderr_action) { .inherit, .ignore => {}, .expect_exact => |expected_bytes| { - if (!mem.eql(u8, expected_bytes, stderr.toSliceConst())) { + if (!mem.eql(u8, expected_bytes, stderr)) { warn( \\ \\========= Expected this stderr: ========= @@ -224,13 +231,13 @@ pub const RunStep = struct { \\========= But found: ==================== \\{} \\ - , .{ expected_bytes, stderr.toSliceConst() }); + , .{ expected_bytes, stderr }); printCmd(cwd, argv); return error.TestFailed; } }, .expect_matches => |matches| for (matches) |match| { - if (mem.indexOf(u8, stderr.toSliceConst(), match) == null) { + if (mem.indexOf(u8, stderr, match) == null) { warn( \\ \\========= Expected to find in stderr: ========= @@ -238,7 +245,7 @@ pub const RunStep = struct { \\========= But stderr does not contain it: ===== \\{} \\ - , .{ match, stderr.toSliceConst() }); + , .{ match, stderr }); printCmd(cwd, argv); return error.TestFailed; } @@ -248,7 +255,7 @@ pub const RunStep = struct { switch (self.stdout_action) { .inherit, .ignore => {}, .expect_exact => |expected_bytes| { - if (!mem.eql(u8, expected_bytes, stdout.toSliceConst())) { + if (!mem.eql(u8, expected_bytes, stdout)) { warn( \\ \\========= Expected this stdout: ========= @@ -256,13 +263,13 @@ pub const RunStep = struct { \\========= But found: ==================== \\{} \\ - , .{ expected_bytes, stdout.toSliceConst() }); + , .{ expected_bytes, stdout }); printCmd(cwd, argv); return error.TestFailed; } }, .expect_matches => |matches| for (matches) |match| { - if (mem.indexOf(u8, stdout.toSliceConst(), match) == null) { + if (mem.indexOf(u8, stdout, match) == null) { warn( \\ \\========= Expected to find in stdout: ========= @@ -270,7 +277,7 @@ pub const RunStep = struct { \\========= But stdout does not contain it: ===== \\{} \\ - , .{ match, stdout.toSliceConst() }); + , .{ match, stdout }); printCmd(cwd, argv); return error.TestFailed; } diff --git a/lib/std/child_process.zig b/lib/std/child_process.zig index d5e914b28632..4f5fc2b49694 100644 --- a/lib/std/child_process.zig +++ b/lib/std/child_process.zig @@ -217,21 +217,19 @@ pub const ChildProcess = struct { try child.spawn(); - var stdout = Buffer.initNull(args.allocator); - var stderr = Buffer.initNull(args.allocator); - defer Buffer.deinit(&stdout); - defer Buffer.deinit(&stderr); - var stdout_file_in_stream = child.stdout.?.inStream(); var stderr_file_in_stream = child.stderr.?.inStream(); - try stdout_file_in_stream.stream.readAllBuffer(&stdout, args.max_output_bytes); - try stderr_file_in_stream.stream.readAllBuffer(&stderr, args.max_output_bytes); + // TODO need to poll to read these streams to prevent a deadlock (or rely on evented I/O). + const stdout = try stdout_file_in_stream.stream.readAllAlloc(args.allocator, args.max_output_bytes); + errdefer args.allocator.free(stdout); + const stderr = try stderr_file_in_stream.stream.readAllAlloc(args.allocator, args.max_output_bytes); + errdefer args.allocator.free(stderr); return ExecResult{ .term = try child.wait(), - .stdout = stdout.toOwnedSlice(), - .stderr = stderr.toOwnedSlice(), + .stdout = stdout, + .stderr = stderr, }; } diff --git a/test/tests.zig b/test/tests.zig index 9cf4e7bd9879..e3249025792c 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -566,14 +566,13 @@ pub const StackTracesContext = struct { } child.spawn() catch |err| debug.panic("Unable to spawn {}: {}\n", .{ full_exe_path, @errorName(err) }); - var stdout = Buffer.initNull(b.allocator); - var stderr = Buffer.initNull(b.allocator); - var stdout_file_in_stream = child.stdout.?.inStream(); var stderr_file_in_stream = child.stderr.?.inStream(); - stdout_file_in_stream.stream.readAllBuffer(&stdout, max_stdout_size) catch unreachable; - stderr_file_in_stream.stream.readAllBuffer(&stderr, max_stdout_size) catch unreachable; + const stdout = stdout_file_in_stream.stream.readAllAlloc(b.allocator, max_stdout_size) catch unreachable; + defer b.allocator.free(stdout); + const stderr = stderr_file_in_stream.stream.readAllAlloc(b.allocator, max_stdout_size) catch unreachable; + defer b.allocator.free(stderr); const term = child.wait() catch |err| { debug.panic("Unable to spawn {}: {}\n", .{ full_exe_path, @errorName(err) }); @@ -616,11 +615,8 @@ pub const StackTracesContext = struct { const got: []const u8 = got_result: { var buf = try Buffer.initSize(b.allocator, 0); defer buf.deinit(); - const bytes = if (stderr.endsWith("\n")) - stderr.toSliceConst()[0 .. stderr.len() - 1] - else - stderr.toSliceConst()[0..stderr.len()]; - var it = mem.separate(bytes, "\n"); + if (stderr.len != 0 and stderr[stderr.len - 1] == '\n') stderr = stderr[0 .. stderr.len - 1]; + var it = mem.separate(stderr, "\n"); process_lines: while (it.next()) |line| { if (line.len == 0) continue; const delims = [_][]const u8{ ":", ":", ":", " in " }; From d136c795affe9d3a7522eefe3ccc098a52434de0 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Fri, 7 Feb 2020 16:12:16 +1100 Subject: [PATCH 5/9] Fix bug where stdout was checked instead of stderr --- lib/std/build/run.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/build/run.zig b/lib/std/build/run.zig index dc4e0df898a3..b10476fcf03d 100644 --- a/lib/std/build/run.zig +++ b/lib/std/build/run.zig @@ -185,7 +185,7 @@ pub const RunStep = struct { }; var stderr: []const u8 = undefined; - switch (self.stdout_behavior) { + switch (self.stderr_action) { .expect_exact, .expect_matches => { var stderr_file_in_stream = child.stderr.?.inStream(); stderr = stderr_file_in_stream.stream.readAllAlloc(self.builder.allocator, max_stdout_size) catch unreachable; From 5c0d6ef5ec0fa31da15be1fd579c9f70e4cef7dd Mon Sep 17 00:00:00 2001 From: daurnimator Date: Tue, 11 Feb 2020 23:01:30 +1100 Subject: [PATCH 6/9] std: use ArrayList instead of Buffer from std/process.zig --- lib/std/process.zig | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/lib/std/process.zig b/lib/std/process.zig index 01b9947518d5..084aec4c49c5 100644 --- a/lib/std/process.zig +++ b/lib/std/process.zig @@ -3,7 +3,6 @@ const builtin = std.builtin; const os = std.os; const fs = std.fs; const BufMap = std.BufMap; -const Buffer = std.Buffer; const mem = std.mem; const math = std.math; const Allocator = mem.Allocator; @@ -266,7 +265,7 @@ pub const ArgIteratorWindows = struct { } fn internalNext(self: *ArgIteratorWindows, allocator: *Allocator) NextError![]u8 { - var buf = try Buffer.initSize(allocator, 0); + var buf = std.ArrayList(u8).init(allocator); defer buf.deinit(); var backslash_count: usize = 0; @@ -282,10 +281,10 @@ pub const ArgIteratorWindows = struct { if (quote_is_real) { self.seen_quote_count += 1; if (self.seen_quote_count == self.quote_count and self.seen_quote_count % 2 == 1) { - try buf.appendByte('"'); + try buf.append('"'); } } else { - try buf.appendByte('"'); + try buf.append('"'); } }, '\\' => { @@ -295,7 +294,7 @@ pub const ArgIteratorWindows = struct { try self.emitBackslashes(&buf, backslash_count); backslash_count = 0; if (self.seen_quote_count % 2 == 1 and self.seen_quote_count != self.quote_count) { - try buf.appendByte(byte); + try buf.append(byte); } else { return buf.toOwnedSlice(); } @@ -303,16 +302,16 @@ pub const ArgIteratorWindows = struct { else => { try self.emitBackslashes(&buf, backslash_count); backslash_count = 0; - try buf.appendByte(byte); + try buf.append(byte); }, } } } - fn emitBackslashes(self: *ArgIteratorWindows, buf: *Buffer, emit_count: usize) !void { + fn emitBackslashes(self: *ArgIteratorWindows, buf: *std.ArrayList(u8), emit_count: usize) !void { var i: usize = 0; while (i < emit_count) : (i += 1) { - try buf.appendByte('\\'); + try buf.append('\\'); } } @@ -410,7 +409,7 @@ pub fn argsAlloc(allocator: *mem.Allocator) ![][]u8 { // TODO refactor to only make 1 allocation. var it = args(); - var contents = try Buffer.initSize(allocator, 0); + var contents = std.ArrayList(u8).init(allocator); defer contents.deinit(); var slice_list = std.ArrayList(usize).init(allocator); @@ -419,7 +418,7 @@ pub fn argsAlloc(allocator: *mem.Allocator) ![][]u8 { while (it.next(allocator)) |arg_or_err| { const arg = try arg_or_err; defer allocator.free(arg); - try contents.append(arg); + try contents.appendSlice(arg); try slice_list.append(arg.len); } From 1cbf352cfbe55bd81d05543dc95370b496dccc6f Mon Sep 17 00:00:00 2001 From: daurnimator Date: Tue, 11 Feb 2020 23:04:19 +1100 Subject: [PATCH 7/9] Remove unused std.Buffer imports --- build.zig | 1 - lib/std/build/run.zig | 1 - src-self-hosted/main.zig | 1 - 3 files changed, 3 deletions(-) diff --git a/build.zig b/build.zig index 32694afb0ace..7de4f50702c0 100644 --- a/build.zig +++ b/build.zig @@ -6,7 +6,6 @@ const BufMap = std.BufMap; const warn = std.debug.warn; const mem = std.mem; const ArrayList = std.ArrayList; -const Buffer = std.Buffer; const io = std.io; const fs = std.fs; const InstallDirectoryOptions = std.build.InstallDirectoryOptions; diff --git a/lib/std/build/run.zig b/lib/std/build/run.zig index b10476fcf03d..da86b15d78a7 100644 --- a/lib/std/build/run.zig +++ b/lib/std/build/run.zig @@ -9,7 +9,6 @@ const mem = std.mem; const process = std.process; const ArrayList = std.ArrayList; const BufMap = std.BufMap; -const Buffer = std.Buffer; const warn = std.debug.warn; const max_stdout_size = 1 * 1024 * 1024; // 1 MiB diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 4446a974d4f3..20e39bd2a111 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -9,7 +9,6 @@ const mem = std.mem; const process = std.process; const Allocator = mem.Allocator; const ArrayList = std.ArrayList; -const Buffer = std.Buffer; const c = @import("c.zig"); const introspect = @import("introspect.zig"); From 4114b63d751cd00ea759f4c0de8568efff58e945 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Tue, 11 Feb 2020 23:07:33 +1100 Subject: [PATCH 8/9] std: use std.ArrayList instead of std.Buffer in std/fmt.zig tests --- lib/std/fmt.zig | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/std/fmt.zig b/lib/std/fmt.zig index 9b35e56e5ab6..deae0a717bb7 100644 --- a/lib/std/fmt.zig +++ b/lib/std/fmt.zig @@ -1643,10 +1643,10 @@ test "hexToBytes" { test "formatIntValue with comptime_int" { const value: comptime_int = 123456789123456789; - var buf = try std.Buffer.init(std.testing.allocator, ""); + var buf = std.ArrayList(u8).init(std.testing.allocator); defer buf.deinit(); - try formatIntValue(value, "", FormatOptions{}, &buf, @TypeOf(std.Buffer.append).ReturnType.ErrorSet, std.Buffer.append); - std.testing.expect(mem.eql(u8, buf.toSlice(), "123456789123456789")); + try formatIntValue(value, "", FormatOptions{}, &buf, @TypeOf(std.ArrayList(u8).appendSlice).ReturnType.ErrorSet, std.ArrayList(u8).appendSlice); + std.testing.expect(mem.eql(u8, buf.toSliceConst(), "123456789123456789")); } test "formatType max_depth" { @@ -1698,24 +1698,24 @@ test "formatType max_depth" { inst.a = &inst; inst.tu.ptr = &inst.tu; - var buf0 = try std.Buffer.init(std.testing.allocator, ""); + var buf0 = std.ArrayList(u8).init(std.testing.allocator); defer buf0.deinit(); - try formatType(inst, "", FormatOptions{}, &buf0, @TypeOf(std.Buffer.append).ReturnType.ErrorSet, std.Buffer.append, 0); + try formatType(inst, "", FormatOptions{}, &buf0, @TypeOf(std.ArrayList(u8).appendSlice).ReturnType.ErrorSet, std.ArrayList(u8).appendSlice, 0); std.testing.expect(mem.eql(u8, buf0.toSlice(), "S{ ... }")); - var buf1 = try std.Buffer.init(std.testing.allocator, ""); + var buf1 = std.ArrayList(u8).init(std.testing.allocator); defer buf1.deinit(); - try formatType(inst, "", FormatOptions{}, &buf1, @TypeOf(std.Buffer.append).ReturnType.ErrorSet, std.Buffer.append, 1); + try formatType(inst, "", FormatOptions{}, &buf1, @TypeOf(std.ArrayList(u8).appendSlice).ReturnType.ErrorSet, std.ArrayList(u8).appendSlice, 1); std.testing.expect(mem.eql(u8, buf1.toSlice(), "S{ .a = S{ ... }, .tu = TU{ ... }, .e = E.Two, .vec = (10.200,2.220) }")); - var buf2 = try std.Buffer.init(std.testing.allocator, ""); + var buf2 = std.ArrayList(u8).init(std.testing.allocator); defer buf2.deinit(); - try formatType(inst, "", FormatOptions{}, &buf2, @TypeOf(std.Buffer.append).ReturnType.ErrorSet, std.Buffer.append, 2); + try formatType(inst, "", FormatOptions{}, &buf2, @TypeOf(std.ArrayList(u8).appendSlice).ReturnType.ErrorSet, std.ArrayList(u8).appendSlice, 2); std.testing.expect(mem.eql(u8, buf2.toSlice(), "S{ .a = S{ .a = S{ ... }, .tu = TU{ ... }, .e = E.Two, .vec = (10.200,2.220) }, .tu = TU{ .ptr = TU{ ... } }, .e = E.Two, .vec = (10.200,2.220) }")); - var buf3 = try std.Buffer.init(std.testing.allocator, ""); + var buf3 = std.ArrayList(u8).init(std.testing.allocator); defer buf3.deinit(); - try formatType(inst, "", FormatOptions{}, &buf3, @TypeOf(std.Buffer.append).ReturnType.ErrorSet, std.Buffer.append, 3); + try formatType(inst, "", FormatOptions{}, &buf3, @TypeOf(std.ArrayList(u8).appendSlice).ReturnType.ErrorSet, std.ArrayList(u8).appendSlice, 3); std.testing.expect(mem.eql(u8, buf3.toSlice(), "S{ .a = S{ .a = S{ .a = S{ ... }, .tu = TU{ ... }, .e = E.Two, .vec = (10.200,2.220) }, .tu = TU{ .ptr = TU{ ... } }, .e = E.Two, .vec = (10.200,2.220) }, .tu = TU{ .ptr = TU{ .ptr = TU{ ... } } }, .e = E.Two, .vec = (10.200,2.220) }")); } From 231a4b8fde6ff061198c76d02990a471ec48c977 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 6 Mar 2020 18:01:20 -0500 Subject: [PATCH 9/9] fixups & make some API decisions Removed: std.io.InStream.readUntilDelimiterBuffer Deprecated: std.ArrayList.toSlice std.ArrayList.toSliceConst std.ArrayList.at std.ArrayList.ptrAt std.ArrayList.setOrError std.ArrayList.set std.ArrayList.swapRemoveOrError std.Buffer.toSlice std.Buffer.toSliceConst std.io.InStream.readFull => std.io.InStream.readAll std.io.InStream.readAllBuffer New: std.ArrayList.span std.ArrayList.expandToCapacity std.Buffer.span std.io.InStream.readUntilDelimiterArrayList --- lib/std/array_list.zig | 97 ++++++++++++++++++++-------------------- lib/std/buffer.zig | 12 +++-- lib/std/build/run.zig | 32 ++++++------- lib/std/io/in_stream.zig | 80 ++++++++++++++++++--------------- 4 files changed, 114 insertions(+), 107 deletions(-) diff --git a/lib/std/array_list.zig b/lib/std/array_list.zig index ca006d79e35b..cbbec0b4f373 100644 --- a/lib/std/array_list.zig +++ b/lib/std/array_list.zig @@ -5,10 +5,8 @@ const testing = std.testing; const mem = std.mem; const Allocator = mem.Allocator; -/// List of items. -/// -/// This is a wrapper around an array of T values. Initialize with -/// `init`. +/// A contiguous, growable list of items in memory. +/// This is a wrapper around an array of T values. Initialize with `init`. pub fn ArrayList(comptime T: type) type { return AlignedArrayList(T, null); } @@ -22,7 +20,7 @@ pub fn AlignedArrayList(comptime T: type, comptime alignment: ?u29) type { return struct { const Self = @This(); - /// Use toSlice instead of slicing this directly, because if you don't + /// Use `span` instead of slicing this directly, because if you don't /// specify the end position of the slice, this will potentially give /// you uninitialized memory. items: Slice, @@ -56,34 +54,37 @@ pub fn AlignedArrayList(comptime T: type, comptime alignment: ?u29) type { /// Return contents as a slice. Only valid while the list /// doesn't change size. - pub fn toSlice(self: Self) Slice { + pub fn span(self: var) @TypeOf(self.items[0..self.len]) { return self.items[0..self.len]; } - /// Return list as const slice. Only valid while the list - /// doesn't change size. + /// Deprecated: use `span`. + pub fn toSlice(self: Self) Slice { + return self.span(); + } + + /// Deprecated: use `span`. pub fn toSliceConst(self: Self) SliceConst { - return self.items[0..self.len]; + return self.span(); } - /// Safely access index i of the list. + /// Deprecated: use `span()[i]`. pub fn at(self: Self, i: usize) T { - return self.toSliceConst()[i]; + return self.span()[i]; } - /// Safely access ptr to index i of the list. + /// Deprecated: use `&span()[i]`. pub fn ptrAt(self: Self, i: usize) *T { - return &self.toSlice()[i]; + return &self.span()[i]; } - /// Sets the value at index `i`, or returns `error.OutOfBounds` if - /// the index is not in range. + /// Deprecated: use `if (i >= list.len) return error.OutOfBounds else span()[i] = item`. pub fn setOrError(self: Self, i: usize, item: T) !void { if (i >= self.len) return error.OutOfBounds; self.items[i] = item; } - /// Sets the value at index `i`, asserting that the value is in range. + /// Deprecated: use `list.span()[i] = item`. pub fn set(self: *Self, i: usize, item: T) void { assert(i < self.len); self.items[i] = item; @@ -124,18 +125,18 @@ pub fn AlignedArrayList(comptime T: type, comptime alignment: ?u29) type { self.items[n] = item; } - /// Insert slice `items` at index `n`. Moves - /// `list[n .. list.len]` to make room. - pub fn insertSlice(self: *Self, n: usize, items: SliceConst) !void { + /// Insert slice `items` at index `i`. Moves + /// `list[i .. list.len]` to make room. + /// This operation is O(N). + pub fn insertSlice(self: *Self, i: usize, items: SliceConst) !void { try self.ensureCapacity(self.len + items.len); self.len += items.len; - mem.copyBackwards(T, self.items[n + items.len .. self.len], self.items[n .. self.len - items.len]); - mem.copy(T, self.items[n .. n + items.len], items); + mem.copyBackwards(T, self.items[i + items.len .. self.len], self.items[i .. self.len - items.len]); + mem.copy(T, self.items[i .. i + items.len], items); } - /// Extend the list by 1 element. Allocates more memory as - /// necessary. + /// Extend the list by 1 element. Allocates more memory as necessary. pub fn append(self: *Self, item: T) !void { const new_item_ptr = try self.addOne(); new_item_ptr.* = item; @@ -148,8 +149,9 @@ pub fn AlignedArrayList(comptime T: type, comptime alignment: ?u29) type { new_item_ptr.* = item; } - /// Remove the element at index `i` from the list and return - /// its value. Asserts the array has at least one item. + /// Remove the element at index `i` from the list and return its value. + /// Asserts the array has at least one item. + /// This operation is O(N). pub fn orderedRemove(self: *Self, i: usize) T { const newlen = self.len - 1; if (newlen == i) return self.pop(); @@ -163,18 +165,17 @@ pub fn AlignedArrayList(comptime T: type, comptime alignment: ?u29) type { /// Removes the element at the specified index and returns it. /// The empty slot is filled from the end of the list. + /// This operation is O(1). pub fn swapRemove(self: *Self, i: usize) T { if (self.len - 1 == i) return self.pop(); - const slice = self.toSlice(); + const slice = self.span(); const old_item = slice[i]; slice[i] = self.pop(); return old_item; } - /// Removes the element at the specified index and returns it - /// or an error.OutOfBounds is returned. If no error then - /// the empty slot is filled from the end of the list. + /// Deprecated: use `if (i >= list.len) return error.OutOfBounds else list.swapRemove(i)`. pub fn swapRemoveOrError(self: *Self, i: usize) !T { if (i >= self.len) return error.OutOfBounds; return self.swapRemove(i); @@ -204,6 +205,7 @@ pub fn AlignedArrayList(comptime T: type, comptime alignment: ?u29) type { } /// Reduce allocated capacity to `new_len`. + /// Invalidates element pointers. pub fn shrink(self: *Self, new_len: usize) void { assert(new_len <= self.len); self.len = new_len; @@ -222,13 +224,24 @@ pub fn AlignedArrayList(comptime T: type, comptime alignment: ?u29) type { self.items = try self.allocator.realloc(self.items, better_capacity); } + /// Increases the array's length to match the full capacity that is already allocated. + /// The new elements have `undefined` values. This operation does not invalidate any + /// element pointers. + pub fn expandToCapacity(self: *Self) void { + self.len = self.items.len; + } + /// Increase length by 1, returning pointer to the new item. + /// The returned pointer becomes invalid when the list is resized. pub fn addOne(self: *Self) !*T { const new_length = self.len + 1; try self.ensureCapacity(new_length); return self.addOneAssumeCapacity(); } + /// Increase length by 1, returning pointer to the new item. + /// Asserts that there is already space for the new item without allocating more. + /// The returned pointer becomes invalid when the list is resized. pub fn addOneAssumeCapacity(self: *Self) *T { assert(self.len < self.capacity()); const result = &self.items[self.len]; @@ -236,33 +249,19 @@ pub fn AlignedArrayList(comptime T: type, comptime alignment: ?u29) type { return result; } - /// Remove and return the last element from the list. Asserts - /// the list has at least one item. + /// Remove and return the last element from the list. + /// Asserts the list has at least one item. pub fn pop(self: *Self) T { self.len -= 1; return self.items[self.len]; } - /// Like `pop` but returns `null` if empty. + /// Remove and return the last element from the list. + /// If the list is empty, returns `null`. pub fn popOrNull(self: *Self) ?T { if (self.len == 0) return null; return self.pop(); } - - pub fn eql(self: Self, m: []const T) bool { - return mem.eql(T, self.toSliceConst(), m); - } - - pub fn startsWith(self: Self, m: []const T) bool { - if (self.len < m.len) return false; - return mem.eql(T, self.items[0..m.len], m); - } - - pub fn endsWith(self: Self, m: []const T) bool { - if (self.len < m.len) return false; - const start = self.len - m.len; - return mem.eql(T, self.items[start..self.len], m); - } }; } @@ -302,7 +301,7 @@ test "std.ArrayList.basic" { } } - for (list.toSlice()) |v, i| { + for (list.span()) |v, i| { testing.expect(v == @intCast(i32, i + 1)); } @@ -340,7 +339,7 @@ test "std.ArrayList.appendNTimes" { try list.appendNTimes(2, 10); testing.expectEqual(@as(usize, 10), list.len); - for (list.toSlice()) |element| { + for (list.span()) |element| { testing.expectEqual(@as(i32, 2), element); } } diff --git a/lib/std/buffer.zig b/lib/std/buffer.zig index bcefe5bdfb9f..9bf024191e42 100644 --- a/lib/std/buffer.zig +++ b/lib/std/buffer.zig @@ -81,12 +81,18 @@ pub const Buffer = struct { self.list.deinit(); } + pub fn span(self: var) @TypeOf(self.list.items[0 .. self.list.len - 1 :0]) { + return self.list.span()[0..self.len() :0]; + } + + /// Deprecated: use `span` pub fn toSlice(self: Buffer) [:0]u8 { - return self.list.toSlice()[0..self.len() :0]; + return self.span(); } + /// Deprecated: use `span` pub fn toSliceConst(self: Buffer) [:0]const u8 { - return self.list.toSliceConst()[0..self.len() :0]; + return self.span(); } pub fn shrink(self: *Buffer, new_len: usize) void { @@ -133,7 +139,7 @@ pub const Buffer = struct { pub fn startsWith(self: Buffer, m: []const u8) bool { if (self.len() < m.len) return false; - return self.list.startsWith(m); + return mem.eql(u8, self.list.items[0..m.len], m); } pub fn endsWith(self: Buffer, m: []const u8) bool { diff --git a/lib/std/build/run.zig b/lib/std/build/run.zig index da86b15d78a7..41417572ca4b 100644 --- a/lib/std/build/run.zig +++ b/lib/std/build/run.zig @@ -170,7 +170,9 @@ pub const RunStep = struct { // TODO need to poll to read these streams to prevent a deadlock (or rely on evented I/O). - var stdout: []const u8 = undefined; + var stdout: ?[]const u8 = null; + defer if (stdout) |s| self.builder.allocator.free(s); + switch (self.stdout_action) { .expect_exact, .expect_matches => { var stdout_file_in_stream = child.stdout.?.inStream(); @@ -178,12 +180,10 @@ pub const RunStep = struct { }, .inherit, .ignore => {}, } - defer switch (self.stdout_action) { - .expect_exact, .expect_matches => self.builder.allocator.free(stdout), - .inherit, .ignore => {}, - }; - var stderr: []const u8 = undefined; + var stderr: ?[]const u8 = null; + defer if (stderr) |s| self.builder.allocator.free(s); + switch (self.stderr_action) { .expect_exact, .expect_matches => { var stderr_file_in_stream = child.stderr.?.inStream(); @@ -191,10 +191,6 @@ pub const RunStep = struct { }, .inherit, .ignore => {}, } - defer switch (self.stderr_action) { - .expect_exact, .expect_matches => self.builder.allocator.free(stderr), - .inherit, .ignore => {}, - }; const term = child.wait() catch |err| { warn("Unable to spawn {}: {}\n", .{ argv[0], @errorName(err) }); @@ -222,7 +218,7 @@ pub const RunStep = struct { switch (self.stderr_action) { .inherit, .ignore => {}, .expect_exact => |expected_bytes| { - if (!mem.eql(u8, expected_bytes, stderr)) { + if (!mem.eql(u8, expected_bytes, stderr.?)) { warn( \\ \\========= Expected this stderr: ========= @@ -230,13 +226,13 @@ pub const RunStep = struct { \\========= But found: ==================== \\{} \\ - , .{ expected_bytes, stderr }); + , .{ expected_bytes, stderr.? }); printCmd(cwd, argv); return error.TestFailed; } }, .expect_matches => |matches| for (matches) |match| { - if (mem.indexOf(u8, stderr, match) == null) { + if (mem.indexOf(u8, stderr.?, match) == null) { warn( \\ \\========= Expected to find in stderr: ========= @@ -244,7 +240,7 @@ pub const RunStep = struct { \\========= But stderr does not contain it: ===== \\{} \\ - , .{ match, stderr }); + , .{ match, stderr.? }); printCmd(cwd, argv); return error.TestFailed; } @@ -254,7 +250,7 @@ pub const RunStep = struct { switch (self.stdout_action) { .inherit, .ignore => {}, .expect_exact => |expected_bytes| { - if (!mem.eql(u8, expected_bytes, stdout)) { + if (!mem.eql(u8, expected_bytes, stdout.?)) { warn( \\ \\========= Expected this stdout: ========= @@ -262,13 +258,13 @@ pub const RunStep = struct { \\========= But found: ==================== \\{} \\ - , .{ expected_bytes, stdout }); + , .{ expected_bytes, stdout.? }); printCmd(cwd, argv); return error.TestFailed; } }, .expect_matches => |matches| for (matches) |match| { - if (mem.indexOf(u8, stdout, match) == null) { + if (mem.indexOf(u8, stdout.?, match) == null) { warn( \\ \\========= Expected to find in stdout: ========= @@ -276,7 +272,7 @@ pub const RunStep = struct { \\========= But stdout does not contain it: ===== \\{} \\ - , .{ match, stdout }); + , .{ match, stdout.? }); printCmd(cwd, argv); return error.TestFailed; } diff --git a/lib/std/io/in_stream.zig b/lib/std/io/in_stream.zig index 809e388ebd8a..e23418277132 100644 --- a/lib/std/io/in_stream.zig +++ b/lib/std/io/in_stream.zig @@ -41,10 +41,13 @@ pub fn InStream(comptime ReadError: type) type { } } + /// Deprecated: use `readAll`. + pub const readFull = readAll; + /// Returns the number of bytes read. If the number read is smaller than buf.len, it /// means the stream reached the end. Reaching the end of a stream is not an error /// condition. - pub fn readFull(self: *Self, buffer: []u8) Error!usize { + pub fn readAll(self: *Self, buffer: []u8) Error!usize { var index: usize = 0; while (index != buffer.len) { const amt = try self.read(buffer[index..]); @@ -57,39 +60,43 @@ pub fn InStream(comptime ReadError: type) type { /// Returns the number of bytes read. If the number read would be smaller than buf.len, /// error.EndOfStream is returned instead. pub fn readNoEof(self: *Self, buf: []u8) !void { - const amt_read = try self.readFull(buf); + const amt_read = try self.readAll(buf); if (amt_read < buf.len) return error.EndOfStream; } - /// Replaces `buffer` contents by reading from the stream until it is finished. - /// If `buffer.len()` would exceed `max_size`, `error.StreamTooLong` is returned and - /// the contents read from the stream are lost. + /// Deprecated: use `readAllArrayList`. pub fn readAllBuffer(self: *Self, buffer: *Buffer, max_size: usize) !void { - try buffer.list.ensureCapacity(1); - buffer.list.len = 0; - errdefer buffer.resize(0) catch unreachable; // make sure we leave buffer in a valid state on error + buffer.list.shrink(0); try self.readAllArrayList(&buffer.list, max_size); + errdefer buffer.shrink(0); try buffer.list.append(0); } - /// Appends to the ArrayList contents by reading from the stream until end of stream is found. - /// If the ArrayList length would exceed `max_size`, `error.StreamTooLong` is returned and the contents - /// read from the stream so far are lost. - pub fn readAllArrayList(self: *Self, array_list: *std.ArrayList(u8), max_size: usize) !void { - var actual_buf_len: usize = 0; + /// Appends to the `std.ArrayList` contents by reading from the stream until end of stream is found. + /// If the number of bytes appended would exceed `max_append_size`, `error.StreamTooLong` is returned + /// and the `std.ArrayList` has exactly `max_append_size` bytes appended. + pub fn readAllArrayList(self: *Self, array_list: *std.ArrayList(u8), max_append_size: usize) !void { + try array_list.ensureCapacity(math.min(max_append_size, 4096)); + const original_len = array_list.len; + var start_index: usize = original_len; while (true) { - const dest_slice = array_list.toSlice()[actual_buf_len..]; - const bytes_read = try self.readFull(dest_slice); - actual_buf_len += bytes_read; + array_list.expandToCapacity(); + const dest_slice = array_list.span()[start_index..]; + const bytes_read = try self.readAll(dest_slice); + start_index += bytes_read; + + if (start_index - original_len > max_append_size) { + array_list.shrink(original_len + max_append_size); + return error.StreamTooLong; + } if (bytes_read != dest_slice.len) { - array_list.shrink(actual_buf_len); + array_list.shrink(start_index); return; } - const new_buf_size = math.min(max_size, actual_buf_len + mem.page_size); - if (new_buf_size == actual_buf_len) return error.StreamTooLong; - try array_list.resize(new_buf_size); + // This will trigger ArrayList to expand superlinearly at whatever its growth rate is. + try array_list.ensureCapacity(start_index + 1); } } @@ -104,23 +111,17 @@ pub fn InStream(comptime ReadError: type) type { return array_list.toOwnedSlice(); } - /// Replaces `buffer` contents by reading from the stream until `delimiter` is found. - /// Does not include the delimiter in the result. - /// If `buffer.len()` would exceed `max_size`, `error.StreamTooLong` is returned and the contents - /// read from the stream so far are lost. - pub fn readUntilDelimiterBuffer(self: *Self, buffer: *Buffer, delimiter: u8, max_size: usize) !void { - try buffer.list.ensureCapacity(1); - buffer.list.len = 0; - errdefer buffer.resize(0) catch unreachable; // make sure we leave buffer in a valid state on error - try self.readUntilDelimiterArrayList(&buffer.list, delimiter, max_size); - try buffer.list.append(0); - } - - /// Appends to the ArrayList contents by reading from the stream until `delimiter` is found. + /// Replaces the `std.ArrayList` contents by reading from the stream until `delimiter` is found. /// Does not include the delimiter in the result. - /// If the ArrayList length would exceed `max_size`, `error.StreamTooLong` is returned and the contents - /// read from the stream so far are lost. - pub fn readUntilDelimiterArrayList(self: *Self, array_list: *std.ArrayList(u8), delimiter: u8, max_size: usize) !void { + /// If the `std.ArrayList` length would exceed `max_size`, `error.StreamTooLong` is returned and the + /// `std.ArrayList` is populated with `max_size` bytes from the stream. + pub fn readUntilDelimiterArrayList( + self: *Self, + array_list: *std.ArrayList(u8), + delimiter: u8, + max_size: usize, + ) !void { + array_list.shrink(0); while (true) { var byte: u8 = try self.readByte(); @@ -140,7 +141,12 @@ pub fn InStream(comptime ReadError: type) type { /// memory would be greater than `max_size`, returns `error.StreamTooLong`. /// Caller owns returned memory. /// If this function returns an error, the contents from the stream read so far are lost. - pub fn readUntilDelimiterAlloc(self: *Self, allocator: *mem.Allocator, delimiter: u8, max_size: usize) ![]u8 { + pub fn readUntilDelimiterAlloc( + self: *Self, + allocator: *mem.Allocator, + delimiter: u8, + max_size: usize, + ) ![]u8 { var array_list = std.ArrayList(u8).init(allocator); defer array_list.deinit(); try self.readUntilDelimiterArrayList(&array_list, delimiter, max_size);