From d805ea2347f3163b02d3dd47acaddb590388e100 Mon Sep 17 00:00:00 2001 From: x19 <0x39015319@gmail.com> Date: Tue, 4 Jun 2024 15:21:29 -0400 Subject: [PATCH 01/16] init --- .github/workflows/test.yml | 3 +++ scripts/gossip_test.sh | 15 +++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 scripts/gossip_test.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 545ae5213..0fca2357f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -48,6 +48,9 @@ jobs: - name: test run: zig build test + - name: gossip + run: bash scripts/gossip_test.sh 120 # in seconds + # build-windows: # runs-on: windows-latest # defaults: diff --git a/scripts/gossip_test.sh b/scripts/gossip_test.sh new file mode 100644 index 000000000..ab111d1e7 --- /dev/null +++ b/scripts/gossip_test.sh @@ -0,0 +1,15 @@ +#!/bin/bash +echo "Running gossip test for $1 seconds" + +# build and run gossip +zig build +./zig-out/bin/sig gossip -e entrypoint.testnet.solana.com:8001 -e entrypoint2.testnet.solana.com:8001 & + +# Get the process ID of the last background command +PID=$! + +# Sleep for 30 seconds +sleep $1 + +# Kill the process +kill $PID From fd7bcabc30341d0c0ac34bbf46cddaf7a684061b Mon Sep 17 00:00:00 2001 From: x19 <0x39015319@gmail.com> Date: Tue, 4 Jun 2024 15:23:55 -0400 Subject: [PATCH 02/16] push test --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0fca2357f..345c42bb8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: test on: push: - branches: [main, pre-release] + branches: [main, pre-release, 19/cicd-gossip] pull_request: branches: [main, pre-release] From 4a8467090f90d2ccd2a3922fff773bacc1dd25a9 Mon Sep 17 00:00:00 2001 From: x19 <0x39015319@gmail.com> Date: Wed, 5 Jun 2024 11:30:45 -0400 Subject: [PATCH 03/16] fix benchmarks --- .github/workflows/test.yml | 76 +++++++++--------- scripts/gossip_test.sh | 6 +- src/accountsdb/db.zig | 157 ++++++++++++++++++++++--------------- src/cmd/cmd.zig | 4 +- src/gossip/service.zig | 16 ++-- src/net/socket_utils.zig | 25 +++--- 6 files changed, 160 insertions(+), 124 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 345c42bb8..b24e094c2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,7 +24,7 @@ jobs: zig fmt --check src/ zig fmt --check build.zig - build-unix: + test: strategy: matrix: os: [ubuntu-latest] @@ -42,41 +42,47 @@ jobs: with: version: 0.12.0 - - name: build - run: zig build - - name: test run: zig build test - - name: gossip - run: bash scripts/gossip_test.sh 120 # in seconds + benchmarks: + strategy: + matrix: + os: [ubuntu-latest] + + runs-on: ${{matrix.os}} + + steps: + - name: checkout + uses: actions/checkout@v2 + with: + submodules: recursive + + - name: setup-zig + uses: goto-bus-stop/setup-zig@v1 + with: + version: 0.12.0 + + - name: benchmarks + run: zig build benchmark - # build-windows: - # runs-on: windows-latest - # defaults: - # run: - # shell: bash - - # steps: - # - name: setup-git - # run: | - # git config --global core.autocrlf false - # git config --global core.eol lf - - # - name: checkout - # uses: actions/checkout@v2 - # with: - # submodules: recursive - - # - name: setup-zig - # uses: goto-bus-stop/setup-zig@v1 - # with: - # version: master - - # - name: build - # run: | - # zig build -Dtarget=x86_64-windows --prefix tmp - # zig build - - # - name: test - # run: zig build test + gossip: + strategy: + matrix: + os: [ubuntu-latest] + + runs-on: ${{matrix.os}} + + steps: + - name: checkout + uses: actions/checkout@v2 + with: + submodules: recursive + + - name: setup-zig + uses: goto-bus-stop/setup-zig@v1 + with: + version: 0.12.0 + + - name: gossip + run: bash scripts/gossip_test.sh 120 # in seconds diff --git a/scripts/gossip_test.sh b/scripts/gossip_test.sh index ab111d1e7..063606992 100644 --- a/scripts/gossip_test.sh +++ b/scripts/gossip_test.sh @@ -2,8 +2,10 @@ echo "Running gossip test for $1 seconds" # build and run gossip -zig build -./zig-out/bin/sig gossip -e entrypoint.testnet.solana.com:8001 -e entrypoint2.testnet.solana.com:8001 & +zig build -Doptimize=ReleaseSafe && \ +./zig-out/bin/sig gossip \ + -e entrypoint.testnet.solana.com:8001 \ + -e entrypoint2.testnet.solana.com:8001 & # Get the process ID of the last background command PID=$! diff --git a/src/accountsdb/db.zig b/src/accountsdb/db.zig index ce889dc54..9eea6ec3b 100644 --- a/src/accountsdb/db.zig +++ b/src/accountsdb/db.zig @@ -1249,7 +1249,6 @@ pub const BenchmarkAccountsDB = struct { }; pub const args = [_]BenchArgs{ - // test accounts in ram BenchArgs{ .n_accounts = 100_000, .slot_list_len = 1, @@ -1258,81 +1257,111 @@ pub const BenchmarkAccountsDB = struct { .name = "100k accounts (1_slot - ram index - ram accounts)", }, BenchArgs{ - .n_accounts = 10_000, - .slot_list_len = 10, - .accounts = .ram, - .index = .ram, - .name = "10k accounts (10_slots - ram index - ram accounts)", - }, - - // tests large number of accounts on disk - BenchArgs{ - .n_accounts = 10_000, - .slot_list_len = 10, - .accounts = .disk, - .index = .ram, - .name = "10k accounts (10_slots - ram index - disk accounts)", - }, - BenchArgs{ - .n_accounts = 500_000, - .slot_list_len = 1, - .accounts = .disk, - .index = .ram, - .name = "500k accounts (1_slot - ram index - disk accounts)", - }, - BenchArgs{ - .n_accounts = 500_000, - .slot_list_len = 3, - .accounts = .disk, - .index = .ram, - .name = "500k accounts (3_slot - ram index - disk accounts)", - }, - BenchArgs{ - .n_accounts = 3_000_000, - .slot_list_len = 1, - .accounts = .disk, - .index = .ram, - .name = "3M accounts (1_slot - ram index - disk accounts)", - }, - BenchArgs{ - .n_accounts = 3_000_000, - .slot_list_len = 3, - .accounts = .disk, - .index = .ram, - .name = "3M accounts (3_slot - ram index - disk accounts)", - }, - BenchArgs{ - .n_accounts = 500_000, - .slot_list_len = 1, - .accounts = .disk, - .n_accounts_multiple = 2, // 1 mill accounts init - .index = .ram, - .name = "3M accounts (3_slot - ram index - disk accounts)", - }, - - // testing disk indexes - BenchArgs{ - .n_accounts = 500_000, + .n_accounts = 100_000, .slot_list_len = 1, - .accounts = .disk, + .accounts = .ram, .index = .disk, - .name = "500k accounts (1_slot - disk index - disk accounts)", + .name = "100k accounts (1_slot - disk index - ram accounts)", }, BenchArgs{ - .n_accounts = 3_000_000, + .n_accounts = 100_000, .slot_list_len = 1, .accounts = .disk, - .index = .disk, - .name = "3m accounts (1_slot - disk index - disk accounts)", + .index = .ram, + .name = "100k accounts (1_slot - ram index - disk accounts)", }, BenchArgs{ - .n_accounts = 500_000, + .n_accounts = 100_000, .slot_list_len = 1, .accounts = .disk, .index = .disk, - .n_accounts_multiple = 2, - .name = "500k accounts (1_slot - disk index - disk accounts)", + .name = "100k accounts (1_slot - disk index - disk accounts)", }, + + // // test accounts in ram + // BenchArgs{ + // .n_accounts = 100_000, + // .slot_list_len = 1, + // .accounts = .ram, + // .index = .ram, + // .name = "100k accounts (1_slot - ram index - ram accounts)", + // }, + // BenchArgs{ + // .n_accounts = 10_000, + // .slot_list_len = 10, + // .accounts = .ram, + // .index = .ram, + // .name = "10k accounts (10_slots - ram index - ram accounts)", + // }, + + // // tests large number of accounts on disk + // BenchArgs{ + // .n_accounts = 10_000, + // .slot_list_len = 10, + // .accounts = .disk, + // .index = .ram, + // .name = "10k accounts (10_slots - ram index - disk accounts)", + // }, + // BenchArgs{ + // .n_accounts = 500_000, + // .slot_list_len = 1, + // .accounts = .disk, + // .index = .ram, + // .name = "500k accounts (1_slot - ram index - disk accounts)", + // }, + // BenchArgs{ + // .n_accounts = 500_000, + // .slot_list_len = 3, + // .accounts = .disk, + // .index = .ram, + // .name = "500k accounts (3_slot - ram index - disk accounts)", + // }, + // BenchArgs{ + // .n_accounts = 3_000_000, + // .slot_list_len = 1, + // .accounts = .disk, + // .index = .ram, + // .name = "3M accounts (1_slot - ram index - disk accounts)", + // }, + // BenchArgs{ + // .n_accounts = 3_000_000, + // .slot_list_len = 3, + // .accounts = .disk, + // .index = .ram, + // .name = "3M accounts (3_slot - ram index - disk accounts)", + // }, + // BenchArgs{ + // .n_accounts = 500_000, + // .slot_list_len = 1, + // .accounts = .disk, + // .n_accounts_multiple = 2, // 1 mill accounts init + // .index = .ram, + // .name = "3M accounts (3_slot - ram index - disk accounts)", + // }, + + // // testing disk indexes + // BenchArgs{ + // .n_accounts = 500_000, + // .slot_list_len = 1, + // .accounts = .disk, + // .index = .disk, + // .name = "500k accounts (1_slot - disk index - disk accounts)", + // }, + // BenchArgs{ + // .n_accounts = 3_000_000, + // .slot_list_len = 1, + // .accounts = .disk, + // .index = .disk, + // .name = "3m accounts (1_slot - disk index - disk accounts)", + // }, + // BenchArgs{ + // .n_accounts = 500_000, + // .slot_list_len = 1, + // .accounts = .disk, + // .index = .disk, + // .n_accounts_multiple = 2, + // .name = "500k accounts (1_slot - disk index - disk accounts)", + // }, }; pub fn readAccounts(bench_args: BenchArgs) !u64 { diff --git a/src/cmd/cmd.zig b/src/cmd/cmd.zig index 500cb2224..60834d79e 100644 --- a/src/cmd/cmd.zig +++ b/src/cmd/cmd.zig @@ -39,7 +39,7 @@ const requestIpEcho = @import("../net/echo.zig").requestIpEcho; const servePrometheus = @import("../prometheus/http.zig").servePrometheus; const parallelUnpackZstdTarBall = @import("../accountsdb/snapshots.zig").parallelUnpackZstdTarBall; const downloadSnapshotsFromGossip = @import("../accountsdb/download.zig").downloadSnapshotsFromGossip; -const SOCKET_TIMEOUT_US = @import("../net/socket_utils.zig").SOCKET_TIMEOUT_US; +const SOCKET_TIMEOUT_MS = @import("../net/socket_utils.zig").SOCKET_TIMEOUT_MS; const config = @import("config.zig"); // var validator_config = config.current; @@ -418,7 +418,7 @@ fn validator() !void { // repair var repair_socket = try Socket.create(network.AddressFamily.ipv4, network.Protocol.udp); try repair_socket.bindToPort(repair_port); - try repair_socket.setReadTimeout(SOCKET_TIMEOUT_US); + try repair_socket.setReadTimeout(SOCKET_TIMEOUT_MS); var repair_svc = try initRepair( logger, diff --git a/src/gossip/service.zig b/src/gossip/service.zig index 6b8186f41..b037f16e9 100644 --- a/src/gossip/service.zig +++ b/src/gossip/service.zig @@ -173,7 +173,7 @@ pub const GossipService = struct { const gossip_address = my_contact_info.getSocket(socket_tag.GOSSIP) orelse return error.GossipAddrUnspecified; var gossip_socket = UdpSocket.create(.ipv4, .udp) catch return error.SocketCreateFailed; gossip_socket.bindToPort(gossip_address.port()) catch return error.SocketBindFailed; - gossip_socket.setReadTimeout(socket_utils.SOCKET_TIMEOUT_US) catch return error.SocketSetTimeoutFailed; // 1 second + gossip_socket.setReadTimeout(socket_utils.SOCKET_TIMEOUT_MS) catch return error.SocketSetTimeoutFailed; // 1 second const failed_pull_hashes = HashTimeQueue.init(allocator); const push_msg_q = ArrayList(SignedGossipData).init(allocator); @@ -2933,27 +2933,27 @@ pub const BenchmarkGossipServiceGeneral = struct { pub const args = [_]BenchmarkArgs{ .{ - .name = "10k_ping_msgs", + .name = "5k_ping_msgs", .message_counts = .{ - .n_ping = 10_000, + .n_ping = 5_000, .n_push_message = 0, .n_pull_response = 0, }, }, .{ - .name = "10k_push_msgs", + .name = "5k_push_msgs", .message_counts = .{ .n_ping = 0, - .n_push_message = 10_000, + .n_push_message = 5_000, .n_pull_response = 0, }, }, .{ - .name = "10k_pull_resp_msgs", + .name = "1k_pull_resp_msgs", .message_counts = .{ .n_ping = 0, .n_push_message = 0, - .n_pull_response = 10_000, + .n_pull_response = 1_000, }, }, }; @@ -3050,10 +3050,10 @@ pub const BenchmarkGossipServiceGeneral = struct { var timer = try std.time.Timer.start(); while (true) { const v = gossip_service.stats.gossip_packets_processed.get(); + std.debug.print("{d} messages processed\r", .{v}); if (v >= msg_sent) { break; } - std.debug.print("{d} messages processed\r", .{v}); } const elapsed = timer.read(); std.debug.print("\r", .{}); diff --git a/src/net/socket_utils.zig b/src/net/socket_utils.zig index 22e94fde4..d577cdbb6 100644 --- a/src/net/socket_utils.zig +++ b/src/net/socket_utils.zig @@ -7,7 +7,7 @@ const Channel = @import("../sync/channel.zig").Channel; const std = @import("std"); const Logger = @import("../trace/log.zig").Logger; -pub const SOCKET_TIMEOUT_US: usize = 1 * std.time.us_per_s; +pub const SOCKET_TIMEOUT_MS: usize = 1 * std.time.ms_per_s; pub const PACKETS_PER_BATCH: usize = 64; pub fn readSocket( @@ -17,15 +17,10 @@ pub fn readSocket( exit: *const std.atomic.Value(bool), logger: Logger, ) !void { - // Performance out of the IO without poll - // * block on the socket until it's readable - // * set the socket to non blocking - // * read until it fails - // * set it back to blocking before returning + // NOTE: we set to non-blocking to periodically check if we should exit + try socket.setReadTimeout(SOCKET_TIMEOUT_MS); - try socket.setReadTimeout(SOCKET_TIMEOUT_US); - - while (!exit.load(.unordered)) { + inf_loop: while (!exit.load(.unordered)) { // init a new batch var packet_batch = try std.ArrayList(Packet).initCapacity( allocator, @@ -33,16 +28,16 @@ pub fn readSocket( ); errdefer packet_batch.deinit(); - // NOTE: usually this would be null (ie, blocking) - // but in order to exit cleanly in tests - we set to 1 second - try socket.setReadTimeout(std.time.ms_per_s); - // recv packets into batch while (packet_batch.items.len != packet_batch.capacity) { var packet: Packet = Packet.default(); const recv_meta = socket.receiveFrom(&packet.data) catch |err| switch (err) { error.WouldBlock => { if (packet_batch.items.len > 0) break; + if (exit.load(.unordered)) { + packet_batch.deinit(); + break :inf_loop; + } continue; }, else => |e| return e, @@ -57,6 +52,7 @@ pub fn readSocket( packet_batch.shrinkAndFree(packet_batch.items.len); try incoming_channel.send(packet_batch); } + logger.debugf("readSocket loop closed", .{}); } @@ -171,6 +167,7 @@ pub const BenchmarkPacketProcessing = struct { var packet_buf: [PACKET_DATA_SIZE]u8 = undefined; var timer = try std.time.Timer.start(); + // NOTE: send more packets than we need because UDP drops some for (1..(n_packets * 2 + 1)) |i| { rand.fill(&packet_buf); _ = try socket.sendTo(to_endpoint, &packet_buf); @@ -185,6 +182,7 @@ pub const BenchmarkPacketProcessing = struct { } } } + std.debug.print("sent all packets.. waiting on receiver\r", .{}); recv_handle.join(); const elapsed = timer.read(); @@ -208,6 +206,7 @@ pub fn benchmarkChannelRecv( for (values) |packet_batch| { count += packet_batch.items.len; } + std.debug.print("recv packet count: {d}\r", .{count}); if (count >= n_values_to_receive) { break; } From d52deb0dc3b3aa3b78cdef6c664dd8fe188e4168 Mon Sep 17 00:00:00 2001 From: x19 <0x39015319@gmail.com> Date: Wed, 5 Jun 2024 11:39:48 -0400 Subject: [PATCH 04/16] fix accountsdb benchmarks --- scripts/gossip_test.sh | 19 ++++++++++++++++++- src/accountsdb/db.zig | 6 ++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/scripts/gossip_test.sh b/scripts/gossip_test.sh index 063606992..bca413884 100644 --- a/scripts/gossip_test.sh +++ b/scripts/gossip_test.sh @@ -1,11 +1,28 @@ #!/bin/bash +usage() { + echo "Usage: $0 " + exit 1 +} + +# Check if an argument is provided +if [ -z "$1" ]; then + echo "Error: No duration provided." + usage +fi + +# Check if the argument is a valid number +if ! [[ "$1" =~ ^[0-9]+$ ]]; then + echo "Error: Duration must be a positive integer." + usage +fi + echo "Running gossip test for $1 seconds" # build and run gossip zig build -Doptimize=ReleaseSafe && \ ./zig-out/bin/sig gossip \ -e entrypoint.testnet.solana.com:8001 \ - -e entrypoint2.testnet.solana.com:8001 & + -e entrypoint2.testnet.solana.com:8001 2>&1 & # Get the process ID of the last background command PID=$! diff --git a/src/accountsdb/db.zig b/src/accountsdb/db.zig index 9eea6ec3b..7acfefcbe 100644 --- a/src/accountsdb/db.zig +++ b/src/accountsdb/db.zig @@ -1377,7 +1377,7 @@ pub const BenchmarkAccountsDB = struct { if (bench_args.index == .disk) { // std.debug.print("using disk index\n", .{}); accounts_db = try AccountsDB.init(allocator, logger, .{ - .disk_index_path = "test_data/tmp_benchmarks", + .disk_index_path = "test_data/tmp/tmp", }); } else { // std.debug.print("using ram index\n", .{}); @@ -1447,7 +1447,9 @@ pub const BenchmarkAccountsDB = struct { ); } const aligned_size = std.mem.alignForward(usize, size, std.mem.page_size); - const filepath = try std.fmt.allocPrint(allocator, "test_data/tmp/slot{d}.bin", .{s}); + const disk_path = "test_data/tmp/"; + try std.fs.cwd().makeDir(disk_path); + const filepath = try std.fmt.allocPrint(allocator, disk_path ++ "slot{d}.bin", .{s}); const length = blk: { var file = try std.fs.cwd().createFile(filepath, .{ .read = true }); From a88cbe180fb1100dfa55e1e4cb56a9d43856f91c Mon Sep 17 00:00:00 2001 From: x19 <0x39015319@gmail.com> Date: Wed, 5 Jun 2024 11:43:41 -0400 Subject: [PATCH 05/16] fix disk path benchmark v2 --- src/accountsdb/db.zig | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/accountsdb/db.zig b/src/accountsdb/db.zig index 7acfefcbe..25c83aec2 100644 --- a/src/accountsdb/db.zig +++ b/src/accountsdb/db.zig @@ -1372,12 +1372,15 @@ pub const BenchmarkAccountsDB = struct { var gpa = std.heap.GeneralPurposeAllocator(.{}){}; var allocator = gpa.allocator(); + const disk_path = "test_data/tmp/"; + std.fs.cwd().makeDir(disk_path) catch {}; + const logger = Logger{ .noop = {} }; var accounts_db: AccountsDB = undefined; if (bench_args.index == .disk) { // std.debug.print("using disk index\n", .{}); accounts_db = try AccountsDB.init(allocator, logger, .{ - .disk_index_path = "test_data/tmp/tmp", + .disk_index_path = disk_path ++ "tmp", }); } else { // std.debug.print("using ram index\n", .{}); @@ -1447,8 +1450,6 @@ pub const BenchmarkAccountsDB = struct { ); } const aligned_size = std.mem.alignForward(usize, size, std.mem.page_size); - const disk_path = "test_data/tmp/"; - try std.fs.cwd().makeDir(disk_path); const filepath = try std.fmt.allocPrint(allocator, disk_path ++ "slot{d}.bin", .{s}); const length = blk: { From 60450ccf5d13a60db5dd51ac187617380c906f99 Mon Sep 17 00:00:00 2001 From: x19 <0x39015319@gmail.com> Date: Wed, 5 Jun 2024 13:25:52 -0400 Subject: [PATCH 06/16] build benchmarks in release & remove print statements --- .github/workflows/test.yml | 2 +- src/gossip/service.zig | 8 ++++---- src/net/socket_utils.zig | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b24e094c2..9c0dbc5aa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,7 +64,7 @@ jobs: version: 0.12.0 - name: benchmarks - run: zig build benchmark + run: zig build -Doptimize=ReleaseSafe benchmark gossip: strategy: diff --git a/src/gossip/service.zig b/src/gossip/service.zig index b037f16e9..4addc2538 100644 --- a/src/gossip/service.zig +++ b/src/gossip/service.zig @@ -3050,13 +3050,13 @@ pub const BenchmarkGossipServiceGeneral = struct { var timer = try std.time.Timer.start(); while (true) { const v = gossip_service.stats.gossip_packets_processed.get(); - std.debug.print("{d} messages processed\r", .{v}); + // std.debug.print("{d} messages processed\r", .{v}); if (v >= msg_sent) { break; } } const elapsed = timer.read(); - std.debug.print("\r", .{}); + // std.debug.print("\r", .{}); exit.store(true, .unordered); packet_handle.join(); @@ -3191,10 +3191,10 @@ pub const BenchmarkGossipServicePullRequests = struct { if (v >= msg_sent) { break; } - std.debug.print("{d} messages processed\r", .{v}); + // std.debug.print("{d} messages processed\r", .{v}); } const elapsed = timer.read(); - std.debug.print("\r", .{}); + // std.debug.print("\r", .{}); exit.store(true, .unordered); packet_handle.join(); diff --git a/src/net/socket_utils.zig b/src/net/socket_utils.zig index d577cdbb6..8eeb8beab 100644 --- a/src/net/socket_utils.zig +++ b/src/net/socket_utils.zig @@ -182,7 +182,7 @@ pub const BenchmarkPacketProcessing = struct { } } } - std.debug.print("sent all packets.. waiting on receiver\r", .{}); + // std.debug.print("sent all packets.. waiting on receiver\r", .{}); recv_handle.join(); const elapsed = timer.read(); @@ -206,7 +206,7 @@ pub fn benchmarkChannelRecv( for (values) |packet_batch| { count += packet_batch.items.len; } - std.debug.print("recv packet count: {d}\r", .{count}); + // std.debug.print("recv packet count: {d}\r", .{count}); if (count >= n_values_to_receive) { break; } From 2e7e3ac0d1841b4274445826f17ef3ebbe946d48 Mon Sep 17 00:00:00 2001 From: x19 <0x39015319@gmail.com> Date: Wed, 5 Jun 2024 13:30:15 -0400 Subject: [PATCH 07/16] fix gossip ci --- .github/workflows/test.yml | 4 +++- scripts/gossip_test.sh | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9c0dbc5aa..d43ca2541 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -84,5 +84,7 @@ jobs: with: version: 0.12.0 - - name: gossip + - name: build release + run: zig build -Doptimize=ReleaseSafe + - name: run gossip run: bash scripts/gossip_test.sh 120 # in seconds diff --git a/scripts/gossip_test.sh b/scripts/gossip_test.sh index bca413884..b8ea48e11 100644 --- a/scripts/gossip_test.sh +++ b/scripts/gossip_test.sh @@ -19,7 +19,6 @@ fi echo "Running gossip test for $1 seconds" # build and run gossip -zig build -Doptimize=ReleaseSafe && \ ./zig-out/bin/sig gossip \ -e entrypoint.testnet.solana.com:8001 \ -e entrypoint2.testnet.solana.com:8001 2>&1 & From ae95aa8a2a3189a820ac114bec0648a649ba23a6 Mon Sep 17 00:00:00 2001 From: x19 <0x39015319@gmail.com> Date: Wed, 5 Jun 2024 13:40:36 -0400 Subject: [PATCH 08/16] modify when ci is run --- .github/workflows/{test.yml => check.yml} | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) rename .github/workflows/{test.yml => check.yml} (94%) diff --git a/.github/workflows/test.yml b/.github/workflows/check.yml similarity index 94% rename from .github/workflows/test.yml rename to .github/workflows/check.yml index d43ca2541..e50462f55 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/check.yml @@ -1,15 +1,16 @@ -name: test +name: check on: push: - branches: [main, pre-release, 19/cicd-gossip] - pull_request: branches: [main, pre-release] + pull_request_review: + types: [submitted] + pull_request: + types: [review_requested] jobs: lint: runs-on: ubuntu-latest - steps: - name: checkout uses: actions/checkout@v2 From ef879a0ce764a5d6582f91e78c91e7b8960223ff Mon Sep 17 00:00:00 2001 From: x19 <0x39015319@gmail.com> Date: Wed, 5 Jun 2024 13:48:43 -0400 Subject: [PATCH 09/16] fix comment --- scripts/gossip_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gossip_test.sh b/scripts/gossip_test.sh index b8ea48e11..083879341 100644 --- a/scripts/gossip_test.sh +++ b/scripts/gossip_test.sh @@ -26,7 +26,7 @@ echo "Running gossip test for $1 seconds" # Get the process ID of the last background command PID=$! -# Sleep for 30 seconds +# Sleep for N seconds sleep $1 # Kill the process From 3e7faea923262f4e6a45d9b6ed527fafcd969dae Mon Sep 17 00:00:00 2001 From: x19 <0x39015319@gmail.com> Date: Wed, 5 Jun 2024 15:46:52 -0400 Subject: [PATCH 10/16] fix timeout value --- src/cmd/cmd.zig | 4 ++-- src/gossip/service.zig | 2 +- src/net/socket_utils.zig | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/cmd/cmd.zig b/src/cmd/cmd.zig index 60834d79e..500cb2224 100644 --- a/src/cmd/cmd.zig +++ b/src/cmd/cmd.zig @@ -39,7 +39,7 @@ const requestIpEcho = @import("../net/echo.zig").requestIpEcho; const servePrometheus = @import("../prometheus/http.zig").servePrometheus; const parallelUnpackZstdTarBall = @import("../accountsdb/snapshots.zig").parallelUnpackZstdTarBall; const downloadSnapshotsFromGossip = @import("../accountsdb/download.zig").downloadSnapshotsFromGossip; -const SOCKET_TIMEOUT_MS = @import("../net/socket_utils.zig").SOCKET_TIMEOUT_MS; +const SOCKET_TIMEOUT_US = @import("../net/socket_utils.zig").SOCKET_TIMEOUT_US; const config = @import("config.zig"); // var validator_config = config.current; @@ -418,7 +418,7 @@ fn validator() !void { // repair var repair_socket = try Socket.create(network.AddressFamily.ipv4, network.Protocol.udp); try repair_socket.bindToPort(repair_port); - try repair_socket.setReadTimeout(SOCKET_TIMEOUT_MS); + try repair_socket.setReadTimeout(SOCKET_TIMEOUT_US); var repair_svc = try initRepair( logger, diff --git a/src/gossip/service.zig b/src/gossip/service.zig index 4addc2538..85235c0f9 100644 --- a/src/gossip/service.zig +++ b/src/gossip/service.zig @@ -173,7 +173,7 @@ pub const GossipService = struct { const gossip_address = my_contact_info.getSocket(socket_tag.GOSSIP) orelse return error.GossipAddrUnspecified; var gossip_socket = UdpSocket.create(.ipv4, .udp) catch return error.SocketCreateFailed; gossip_socket.bindToPort(gossip_address.port()) catch return error.SocketBindFailed; - gossip_socket.setReadTimeout(socket_utils.SOCKET_TIMEOUT_MS) catch return error.SocketSetTimeoutFailed; // 1 second + gossip_socket.setReadTimeout(socket_utils.SOCKET_TIMEOUT_US) catch return error.SocketSetTimeoutFailed; // 1 second const failed_pull_hashes = HashTimeQueue.init(allocator); const push_msg_q = ArrayList(SignedGossipData).init(allocator); diff --git a/src/net/socket_utils.zig b/src/net/socket_utils.zig index 8eeb8beab..50ec33553 100644 --- a/src/net/socket_utils.zig +++ b/src/net/socket_utils.zig @@ -7,7 +7,7 @@ const Channel = @import("../sync/channel.zig").Channel; const std = @import("std"); const Logger = @import("../trace/log.zig").Logger; -pub const SOCKET_TIMEOUT_MS: usize = 1 * std.time.ms_per_s; +pub const SOCKET_TIMEOUT_US: usize = 1 * std.time.us_per_s; pub const PACKETS_PER_BATCH: usize = 64; pub fn readSocket( @@ -18,7 +18,7 @@ pub fn readSocket( logger: Logger, ) !void { // NOTE: we set to non-blocking to periodically check if we should exit - try socket.setReadTimeout(SOCKET_TIMEOUT_MS); + try socket.setReadTimeout(SOCKET_TIMEOUT_US); inf_loop: while (!exit.load(.unordered)) { // init a new batch From 2477118f339b6c0af7d2034fb190154c6bc3628b Mon Sep 17 00:00:00 2001 From: x19 <0x39015319@gmail.com> Date: Wed, 5 Jun 2024 15:50:26 -0400 Subject: [PATCH 11/16] add unused imports check --- .github/workflows/check.yml | 15 +++++++++++++-- remove_unused.py | 4 +++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index e50462f55..1c18dceb2 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -2,7 +2,7 @@ name: check on: push: - branches: [main, pre-release] + branches: [main, pre-release, 19/cicd-gossip] pull_request_review: types: [submitted] pull_request: @@ -23,7 +23,18 @@ jobs: - name: lint run: | zig fmt --check src/ - zig fmt --check build.zig + + unused_imports: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" zig fmt --check build.zig + + - name: remove unused imports + run: python remove_unused.py src/ test: strategy: diff --git a/remove_unused.py b/remove_unused.py index b32f260e9..fb0bd1991 100644 --- a/remove_unused.py +++ b/remove_unused.py @@ -80,4 +80,6 @@ print("total iterations: ", n_remove_iter) print("total lines removed: ", total_removes) - \ No newline at end of file + +if (total_removes > 0): + exit(1) \ No newline at end of file From 9d91aaf22402bb0e356d96176a6a88aabfe4c634 Mon Sep 17 00:00:00 2001 From: x19 <0x39015319@gmail.com> Date: Wed, 5 Jun 2024 15:52:33 -0400 Subject: [PATCH 12/16] fix yml --- .github/workflows/check.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 1c18dceb2..161d53e95 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -23,18 +23,19 @@ jobs: - name: lint run: | zig fmt --check src/ + zig fmt --check build.zig unused_imports: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: "3.10" zig fmt --check build.zig - - - name: remove unused imports - run: python remove_unused.py src/ + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + + - name: remove unused imports + run: python remove_unused.py src/ test: strategy: From 818abb981d9140d02f2ec9f92dec1ae0556ce686 Mon Sep 17 00:00:00 2001 From: x19 <0x39015319@gmail.com> Date: Wed, 5 Jun 2024 16:00:13 -0400 Subject: [PATCH 13/16] fix yaml formatting --- .github/workflows/check.yml | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 161d53e95..76deea79b 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -14,7 +14,6 @@ jobs: steps: - name: checkout uses: actions/checkout@v2 - - name: setup-zig uses: goto-bus-stop/setup-zig@v1 with: @@ -25,15 +24,15 @@ jobs: zig fmt --check src/ zig fmt --check build.zig - unused_imports: + unused_imports: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - name: checkout + uses: actions/checkout@v3 - name: Set up Python 3.10 uses: actions/setup-python@v3 - with: + with: python-version: "3.10" - - name: remove unused imports run: python remove_unused.py src/ @@ -41,15 +40,12 @@ jobs: strategy: matrix: os: [ubuntu-latest] - runs-on: ${{matrix.os}} - steps: - name: checkout uses: actions/checkout@v2 with: submodules: recursive - - name: setup-zig uses: goto-bus-stop/setup-zig@v1 with: @@ -62,15 +58,12 @@ jobs: strategy: matrix: os: [ubuntu-latest] - runs-on: ${{matrix.os}} - steps: - name: checkout uses: actions/checkout@v2 with: submodules: recursive - - name: setup-zig uses: goto-bus-stop/setup-zig@v1 with: @@ -83,15 +76,12 @@ jobs: strategy: matrix: os: [ubuntu-latest] - runs-on: ${{matrix.os}} - steps: - name: checkout uses: actions/checkout@v2 with: submodules: recursive - - name: setup-zig uses: goto-bus-stop/setup-zig@v1 with: From 3e4aa6034328e6522290168fc5ca75d9d7bd17c2 Mon Sep 17 00:00:00 2001 From: x19 <0x39015319@gmail.com> Date: Wed, 5 Jun 2024 16:01:09 -0400 Subject: [PATCH 14/16] remove unused imports --- src/cmd/cmd.zig | 1 - src/gossip/service.zig | 1 - src/net/echo.zig | 1 - src/trace/log.zig | 1 - src/tvu/repair_message.zig | 1 - 5 files changed, 5 deletions(-) diff --git a/src/cmd/cmd.zig b/src/cmd/cmd.zig index 500cb2224..60d2b3eb2 100644 --- a/src/cmd/cmd.zig +++ b/src/cmd/cmd.zig @@ -44,7 +44,6 @@ const SOCKET_TIMEOUT_US = @import("../net/socket_utils.zig").SOCKET_TIMEOUT_US; const config = @import("config.zig"); // var validator_config = config.current; -const ACCOUNT_INDEX_BINS = @import("../accountsdb/db.zig").ACCOUNT_INDEX_BINS; const socket_tag = @import("../gossip/data.zig").socket_tag; // TODO: use better allocator, unless GPA becomes more performant. diff --git a/src/gossip/service.zig b/src/gossip/service.zig index 85235c0f9..654666a23 100644 --- a/src/gossip/service.zig +++ b/src/gossip/service.zig @@ -1,5 +1,4 @@ const std = @import("std"); -const builtin = @import("builtin"); const network = @import("zig-network"); const EndPoint = network.EndPoint; const Packet = @import("../net/packet.zig").Packet; diff --git a/src/net/echo.zig b/src/net/echo.zig index 72a0dea94..ec94af753 100644 --- a/src/net/echo.zig +++ b/src/net/echo.zig @@ -4,7 +4,6 @@ const net = @import("net.zig"); const ShredVersion = @import("../core/shred.zig").ShredVersion; const SocketAddr = @import("net.zig").SocketAddr; const logger = @import("../trace/log.zig").default_logger; -const Channel = @import("../sync/channel.zig").Channel; const Atomic = std.atomic.Value; const assert = std.debug.assert; const testing = std.testing; diff --git a/src/trace/log.zig b/src/trace/log.zig index 08d3a2873..3c07ab208 100644 --- a/src/trace/log.zig +++ b/src/trace/log.zig @@ -8,7 +8,6 @@ const testing = std.testing; const Mutex = std.Thread.Mutex; const AtomicBool = std.atomic.Value(bool); const Channel = @import("../sync/channel.zig").Channel; -const OnceCell = @import("../sync/once_cell.zig").OnceCell; var gpa = std.heap.GeneralPurposeAllocator(.{}){}; var gpa_allocator = gpa.allocator(); diff --git a/src/tvu/repair_message.zig b/src/tvu/repair_message.zig index a78beb120..0b91e6a35 100644 --- a/src/tvu/repair_message.zig +++ b/src/tvu/repair_message.zig @@ -5,7 +5,6 @@ const bincode = @import("../bincode/bincode.zig"); const Allocator = std.mem.Allocator; const KeyPair = std.crypto.sign.Ed25519.KeyPair; -const LegacyContactInfo = @import("../gossip/data.zig").LegacyContactInfo; const Nonce = @import("../core/shred.zig").Nonce; const Pong = @import("../gossip/ping_pong.zig").Pong; const Pubkey = @import("../core/pubkey.zig").Pubkey; From fe05e23d6945049e0219ed6f8f89053ab1d75562 Mon Sep 17 00:00:00 2001 From: x19 <0x39015319@gmail.com> Date: Wed, 5 Jun 2024 16:01:56 -0400 Subject: [PATCH 15/16] remove this brnach from push checks --- .github/workflows/check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 76deea79b..3fa1aa3d2 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -2,7 +2,7 @@ name: check on: push: - branches: [main, pre-release, 19/cicd-gossip] + branches: [main, pre-release] pull_request_review: types: [submitted] pull_request: From 4802d63556c5de0615be2a1971eacecc2f1c1277 Mon Sep 17 00:00:00 2001 From: x19 <0x39015319@gmail.com> Date: Thu, 6 Jun 2024 10:29:35 -0400 Subject: [PATCH 16/16] update to run on pull request pushes --- .github/workflows/check.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 3fa1aa3d2..6fe99fbed 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -3,10 +3,8 @@ name: check on: push: branches: [main, pre-release] - pull_request_review: - types: [submitted] pull_request: - types: [review_requested] + branches: [main, pre-release] jobs: lint: