Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Expand target bit set, shrink file size, and add loongarch64-linux-gnusf #12

Merged
merged 7 commits into from
Oct 6, 2024
Merged

Expand target bit set, shrink file size, and add loongarch64-linux-gnusf #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
89ab891
Fix consolidate.zig compilation with Zig master.
alexrp Sep 23, 2024
dbe4e76
Print average popcount for target bit sets in consolidate.zig.
alexrp Sep 23, 2024
8a5e313
Change target bit set in the format from u32 to uleb128 (u64).
alexrp Sep 23, 2024
faaa40b
Change the format to store the inclusion terminal bit in the library …
alexrp Sep 23, 2024
a9ae296
Change object size in the format from u16 to uleb128 (u16).
alexrp Sep 24, 2024
67cfbae
Add asserts to consolidate.zig to notify when a format change is needed.
alexrp Sep 23, 2024
eb98a0a
Add loongarch64-linux-gnusf to consolidate.zig.
alexrp Sep 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# glibc ABI Tool

This repository contains `.abilist` files from every version of glibc. These
files are consolidated to generate a single 269 KB symbol mapping file that is
files are consolidated to generate a single 240 KB symbol mapping file that is
shipped with Zig to target any version of glibc. This repository is for Zig
maintainers to use when a new glibc version is tagged upstream; Zig users have
no need for this repository.
Expand Down Expand Up @@ -175,14 +175,14 @@ All integers are stored little-endian.
- Set of Sized Inclusions

Set of Unsized Inclusions:
- u32 set of targets this inclusion applies to (1 << INDEX_IN_TARGET_LIST)
- last inclusion is indicated if 1 << 31 bit is set in target bitset
- uleb128 (u64) set of targets this inclusion applies to (1 << INDEX_IN_TARGET_LIST)
- u8 index of glibc library this inclusion applies to
- last inclusion is indicated if 1 << 7 bit is set in library index
- [N]u8 set of glibc versions this inclusion applies to. MSB set indicates last.

Set of Sized Inclusions:
- u32 set of targets this inclusion applies to (1 << INDEX_IN_TARGET_LIST)
- last inclusion is indicated if 1 << 31 bit is set in target bitset
- u16 object size
- uleb128 (u64) set of targets this inclusion applies to (1 << INDEX_IN_TARGET_LIST)
- uleb128 (u16) object size
- u8 index of glibc library this inclusion applies to
- last inclusion is indicated if 1 << 7 bit is set in library index
- [N]u8 set of glibc versions this inclusion applies to. MSB set indicates last.
76 changes: 48 additions & 28 deletions consolidate.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,10 +77,15 @@ const zig_targets = [_]ZigTarget{
.{ .arch = .riscv64 , .abi = .gnu },
.{ .arch = .sparc64 , .abi = .gnu },
.{ .arch = .loongarch64, .abi = .gnu },
.{ .arch = .loongarch64, .abi = .gnusf },
.{ .arch = .s390x , .abi = .gnu },
// zig fmt: on
};

comptime {
assert(zig_targets.len <= @bitSizeOf(std.meta.FieldType(Inclusion, .targets)));
}

const versions = [_]Version{
.{.major = 2, .minor = 0},
.{.major = 2, .minor = 1},
Expand Down Expand Up @@ -137,6 +142,10 @@ const versions = [_]Version{
.{.major = 2, .minor = 40},
};

comptime {
assert(versions.len <= @bitSizeOf(std.meta.FieldType(Inclusion, .versions)));
}

// fpu/nofpu are hardcoded elsewhere, based on .gnueabi/.gnueabihf with an exception for .arm
// n64/n32 are hardcoded elsewhere, based on .gnuabi64/.gnuabin32
const abi_lists = [_]AbiList{
Expand Down Expand Up @@ -255,6 +264,7 @@ const abi_lists = [_]AbiList{
AbiList{
.targets = &[_]ZigTarget{
ZigTarget{ .arch = .loongarch64, .abi = .gnu },
ZigTarget{ .arch = .loongarch64, .abi = .gnusf },
},
.path = "loongarch/lp64",
},
Expand Down Expand Up @@ -342,7 +352,7 @@ const Symbol = struct {
for (versions_row, 0..) |ty, versions_i| {
switch (ty) {
.absent => {
if ((inc.targets & (@as(u32, 1) << @intCast(targets_i)) ) != 0 and
if ((inc.targets & (@as(u64, 1) << @intCast(targets_i)) ) != 0 and
(inc.versions & (@as(u64, 1) << @intCast(versions_i)) ) != 0)
{
return false;
Expand All @@ -358,7 +368,7 @@ const Symbol = struct {

const Inclusion = struct {
versions: u64,
targets: u32,
targets: u64,
lib: u8,
size: u16,
};
Expand Down Expand Up @@ -475,7 +485,7 @@ pub fn main() !void {
prefix, abi_list.path, "nofpu", basename,
});
} else if ((abi_list.targets[0].arch == .armeb or
abi_list.targets[0].arch == .arm) and fs_ver.order(ver30) == .gt)
abi_list.targets[0].arch == .arm) and fs_ver.order(ver30) == .gt)
{
const endian_suffix = switch (abi_list.targets[0].arch) {
.armeb => "be",
Expand All @@ -485,7 +495,7 @@ pub fn main() !void {
prefix, abi_list.path, endian_suffix, basename,
});
} else if ((abi_list.targets[0].arch == .powerpc64le or
abi_list.targets[0].arch == .powerpc64))
abi_list.targets[0].arch == .powerpc64))
{
if (fs_ver.order(ver28) == .gt) {
const endian_suffix = switch (abi_list.targets[0].arch) {
Expand Down Expand Up @@ -569,7 +579,7 @@ pub fn main() !void {
var it = symbols.iterator();
while (it.next()) |entry| {
const name = entry.key_ptr.*;
var prev_ty: @typeInfo(Symbol.Type).Union.tag_type.? = .absent;
var prev_ty: @typeInfo(Symbol.Type).@"union".tag_type.? = .absent;
for (entry.value_ptr.type) |targets_row| {
for (targets_row) |versions_row| {
for (versions_row) |ty| {
Expand Down Expand Up @@ -605,6 +615,7 @@ pub fn main() !void {
// as many targets as possible, then to as many versions as possible.
var fn_inclusions = std.ArrayList(NamedInclusion).init(arena);
var fn_count: usize = 0;
var fn_target_popcount: usize = 0;
var fn_version_popcount: usize = 0;
const none_handled = blk: {
const empty_row = [1]bool{false} ** versions.len;
Expand All @@ -630,7 +641,7 @@ pub fn main() !void {
}
const targets_row = entry.value_ptr.type[lib_i];

var wanted_targets: u32 = 0;
var wanted_targets: u64 = 0;
var wanted_versions_multi = [1]u64{0} ** zig_targets.len;

for (targets_row, 0..) |versions_row, targets_i| {
Expand All @@ -640,7 +651,7 @@ pub fn main() !void {
switch (ty) {
.absent => continue,
.function => {
wanted_targets |= @as(u32, 1) << @intCast(targets_i);
wanted_targets |= @as(u64, 1) << @intCast(targets_i);
wanted_versions_multi[targets_i] |=
@as(u64, 1) << @intCast(versions_i);
},
Expand All @@ -660,11 +671,11 @@ pub fn main() !void {
const first_ver_index = @ctz(wanted_versions);
var inc: Inclusion = .{
.versions = @as(u64, 1) << @intCast(first_ver_index),
.targets = @as(u32, 1) << @intCast(first_targ_index),
.targets = @as(u64, 1) << @intCast(first_targ_index),
.lib = @intCast(lib_i),
.size = 0,
};
wanted_targets &= ~(@as(u32, 1) << @intCast(first_targ_index));
wanted_targets &= ~(@as(u64, 1) << @intCast(first_targ_index));
wanted_versions &= ~(@as(u64, 1) << @intCast(first_ver_index));
assert(entry.value_ptr.testInclusion(inc, lib_i));

Expand All @@ -690,16 +701,17 @@ pub fn main() !void {
const test_targ_index = @ctz(wanted_targets);
const new_inc = .{
.versions = inc.versions,
.targets = inc.targets | (@as(u32, 1) << @intCast(test_targ_index)),
.targets = inc.targets | (@as(u64, 1) << @intCast(test_targ_index)),
.lib = inc.lib,
.size = 0,
};
if (entry.value_ptr.testInclusion(new_inc, lib_i)) {
inc = new_inc;
}
wanted_targets &= ~(@as(u32, 1) << @intCast(test_targ_index));
wanted_targets &= ~(@as(u64, 1) << @intCast(test_targ_index));
}

fn_target_popcount += @popCount(inc.targets);
fn_version_popcount += @popCount(inc.versions);

try fn_inclusions.append(.{
Expand All @@ -711,7 +723,7 @@ pub fn main() !void {
for (targets_row, 0..) |versions_row, targets_i| {
for (versions_row, 0..) |_, versions_i| {
if (handled[lib_i][targets_i][versions_i]) continue;
if ((inc.targets & (@as(u32, 1) << @intCast(targets_i)) ) != 0 and
if ((inc.targets & (@as(u64, 1) << @intCast(targets_i)) ) != 0 and
(inc.versions & (@as(u64, 1) << @intCast(versions_i)) ) != 0)
{
handled[lib_i][targets_i][versions_i] = true;
Expand All @@ -726,12 +738,16 @@ pub fn main() !void {
log.info("average inclusions per function: {d}", .{
@as(f64, @floatFromInt(fn_inclusions.items.len)) / @as(f64, @floatFromInt(fn_count)),
});
log.info("average function targets bits set: {d}", .{
@as(f64, @floatFromInt(fn_target_popcount)) / @as(f64, @floatFromInt(fn_inclusions.items.len)),
});
log.info("average function versions bits set: {d}", .{
@as(f64, @floatFromInt(fn_version_popcount)) / @as(f64, @floatFromInt(fn_inclusions.items.len)),
});

var obj_inclusions = std.ArrayList(NamedInclusion).init(arena);
var obj_count: usize = 0;
var obj_target_popcount: usize = 0;
var obj_version_popcount: usize = 0;
{
var it = symbols.iterator();
Expand All @@ -751,7 +767,7 @@ pub fn main() !void {
}
const targets_row = entry.value_ptr.type[lib_i];

var wanted_targets: u32 = 0;
var wanted_targets: u64 = 0;
var wanted_versions_multi = [1]u64{0} ** zig_targets.len;
var wanted_sizes_multi = [1]u16{0} ** zig_targets.len;

Expand All @@ -762,7 +778,7 @@ pub fn main() !void {
switch (ty) {
.absent => continue,
.object => |size| {
wanted_targets |= @as(u32, 1) << @intCast(targets_i);
wanted_targets |= @as(u64, 1) << @intCast(targets_i);

var ok = false;
if (wanted_sizes_multi[targets_i] == 0) {
Expand Down Expand Up @@ -793,11 +809,11 @@ pub fn main() !void {
const first_ver_index = @ctz(wanted_versions);
var inc: Inclusion = .{
.versions = @as(u64, 1) << @intCast(first_ver_index),
.targets = @as(u32, 1) << @intCast(first_targ_index),
.targets = @as(u64, 1) << @intCast(first_targ_index),
.lib = @intCast(lib_i),
.size = wanted_size,
};
wanted_targets &= ~(@as(u32, 1) << @intCast(first_targ_index));
wanted_targets &= ~(@as(u64, 1) << @intCast(first_targ_index));
wanted_versions &= ~(@as(u64, 1) << @intCast(first_ver_index));
assert(entry.value_ptr.testInclusion(inc, lib_i));

Expand All @@ -824,17 +840,18 @@ pub fn main() !void {
if (wanted_sizes_multi[test_targ_index] == wanted_size) {
const new_inc = .{
.versions = inc.versions,
.targets = inc.targets | (@as(u32, 1) << @intCast(test_targ_index)),
.targets = inc.targets | (@as(u64, 1) << @intCast(test_targ_index)),
.lib = inc.lib,
.size = wanted_size,
};
if (entry.value_ptr.testInclusion(new_inc, lib_i)) {
inc = new_inc;
}
}
wanted_targets &= ~(@as(u32, 1) << @intCast(test_targ_index));
wanted_targets &= ~(@as(u64, 1) << @intCast(test_targ_index));
}

obj_target_popcount += @popCount(inc.targets);
obj_version_popcount += @popCount(inc.versions);

try obj_inclusions.append(.{
Expand All @@ -846,7 +863,7 @@ pub fn main() !void {
for (targets_row, 0..) |versions_row, targets_i| {
for (versions_row, 0..) |_, versions_i| {
if (handled[lib_i][targets_i][versions_i]) continue;
if ((inc.targets & (@as(u32, 1) << @intCast(targets_i)) ) != 0 and
if ((inc.targets & (@as(u64, 1) << @intCast(targets_i)) ) != 0 and
(inc.versions & (@as(u64, 1) << @intCast(versions_i)) ) != 0)
{
handled[lib_i][targets_i][versions_i] = true;
Expand All @@ -861,6 +878,9 @@ pub fn main() !void {
log.info("average inclusions per object: {d}", .{
@as(f32, @floatFromInt(obj_inclusions.items.len)) / @as(f32, @floatFromInt(obj_count)),
});
log.info("average objects targets bits set: {d}", .{
@as(f64, @floatFromInt(obj_target_popcount)) / @as(f64, @floatFromInt(obj_inclusions.items.len)),
});
log.info("average objects versions bits set: {d}", .{
@as(f64, @floatFromInt(obj_version_popcount)) / @as(f64, @floatFromInt(obj_inclusions.items.len)),
});
Expand Down Expand Up @@ -904,14 +924,14 @@ pub fn main() !void {
while (true) {
const inc = fn_inclusions.items[i].inc;
i += 1;
try std.leb.writeUleb128(w, inc.targets);
const set_terminal_bit = i >= fn_inclusions.items.len or
!mem.eql(u8, name, fn_inclusions.items[i].name);
var target_bitset = inc.targets;
var lib = inc.lib;
if (set_terminal_bit) {
target_bitset |= 1 << 31;
lib |= 1 << 7;
}
try w.writeInt(u32, target_bitset, .little);
try w.writeByte(inc.lib);
try w.writeByte(lib);

var buf: [versions.len]u8 = undefined;
var buf_index: usize = 0;
Expand Down Expand Up @@ -940,15 +960,15 @@ pub fn main() !void {
while (true) {
const inc = obj_inclusions.items[i].inc;
i += 1;
try std.leb.writeUleb128(w, inc.targets);
try std.leb.writeUleb128(w, inc.size);
const set_terminal_bit = i >= obj_inclusions.items.len or
!mem.eql(u8, name, obj_inclusions.items[i].name);
var target_bitset = inc.targets;
var lib = inc.lib;
if (set_terminal_bit) {
target_bitset |= 1 << 31;
lib |= 1 << 7;
}
try w.writeInt(u32, target_bitset, .little);
try w.writeInt(u16, inc.size, .little);
try w.writeByte(inc.lib);
try w.writeByte(lib);

var buf: [versions.len]u8 = undefined;
var buf_index: usize = 0;
Expand Down
32 changes: 21 additions & 11 deletions list_symbols.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,15 @@ pub fn main() !void {
break :n name;
};
try w.print(" {s}:\n", .{symbol_name});
const targets = try r.readInt(u32, .little);
const lib_index = try r.readByte();
const is_terminal = (targets & (1 << 31)) != 0;
if (is_terminal) opt_symbol_name = null;
const targets = try std.leb.readUleb128(u64, r);
var lib_index = try r.readByte();
const is_terminal = (lib_index & (1 << 7)) != 0;
if (is_terminal) {
std.debug.print("before: {}\n", .{lib_index});
lib_index &= ~@as(u8, 1 << 7);
std.debug.print("after: {}\n", .{lib_index});
opt_symbol_name = null;
}

var ver_buf: [50]u8 = undefined;
var ver_buf_index: usize = 0;
Expand All @@ -112,7 +117,7 @@ pub fn main() !void {

try w.writeAll(" targets:");
for (all_targets, 0..) |target, target_i| {
if ((targets & (@as(u32, 1) << @as(u5, @intCast(target_i)))) != 0) {
if ((targets & (@as(u64, 1) << @as(u6, @intCast(target_i)))) != 0) {
try w.print(" {s}", .{target});
}
}
Expand All @@ -132,11 +137,16 @@ pub fn main() !void {
break :n name;
};
try w.print(" {s}:\n", .{symbol_name});
const targets = try r.readInt(u32, .little);
const size = try r.readInt(u16, .little);
const lib_index = try r.readByte();
const is_terminal = (targets & (1 << 31)) != 0;
if (is_terminal) opt_symbol_name = null;
const targets = try std.leb.readUleb128(u64, r);
const size = try std.leb.readUleb128(u16, r);
var lib_index = try r.readByte();
const is_terminal = (lib_index & (1 << 7)) != 0;
if (is_terminal) {
std.debug.print("before: {}\n", .{lib_index});
lib_index &= ~@as(u8, 1 << 7);
std.debug.print("after: {}\n", .{lib_index});
opt_symbol_name = null;
}

var ver_buf: [50]u8 = undefined;
var ver_buf_index: usize = 0;
Expand Down Expand Up @@ -164,7 +174,7 @@ pub fn main() !void {

try w.writeAll(" targets:");
for (all_targets, 0..) |target, target_i| {
if ((targets & (@as(u32, 1) << @as(u5, @intCast(target_i)))) != 0) {
if ((targets & (@as(u64, 1) << @as(u6, @intCast(target_i)))) != 0) {
try w.print(" {s}", .{target});
}
}
Expand Down