From 82f432695fdaaf0d4932981f19ca7dec6ec2e56b Mon Sep 17 00:00:00 2001 From: Viktor Kuznietsov Date: Thu, 20 Oct 2022 11:23:37 -0400 Subject: [PATCH] Remove dependency on zinfo C library Signed-off-by: Viktor Kuznietsov --- Makefile | 7 ++--- soci/gzip_zinfo.go | 4 +-- soci/testutils.go | 53 --------------------------------- {c => soci}/zinfo.c | 0 {c => soci}/zinfo.h | 0 soci/ztoc_test.go | 72 ++++++++++++--------------------------------- 6 files changed, 23 insertions(+), 113 deletions(-) rename {c => soci}/zinfo.c (100%) rename {c => soci}/zinfo.h (100%) diff --git a/Makefile b/Makefile index 8a3e39d85..818d5de41 100644 --- a/Makefile +++ b/Makefile @@ -17,8 +17,8 @@ CMD_DESTDIR ?= /usr/local GO111MODULE_VALUE=auto OUTDIR ?= $(CURDIR)/out -UTIL_CFLAGS=-I${CURDIR}/c -L${OUTDIR} -lzinfo -lz -UTIL_LDFLAGS=-L${OUTDIR} -lzinfo -lz +UTIL_CFLAGS=-I${CURDIR}/c -L${OUTDIR} -lz +UTIL_LDFLAGS=-L${OUTDIR} -lz PKG=github.com/awslabs/soci-snapshotter VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always --tags) REVISION=$(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi) @@ -48,9 +48,6 @@ soci: FORCE pre-build: rm -rf ${OUTDIR} @mkdir -p ${OUTDIR} - @gcc -c c/zinfo.c -o ${OUTDIR}/zinfo.o -O3 -Wall -Werror - @ar rvs ${OUTDIR}/libzinfo.a ${OUTDIR}/zinfo.o - @rm -f ${OUTDIR}/zinfo.o install-cmake: @wget https://github.com/Kitware/CMake/releases/download/v3.24.1/cmake-3.24.1-Linux-x86_64.sh -O cmake.sh diff --git a/soci/gzip_zinfo.go b/soci/gzip_zinfo.go index 03a7a61d5..fea4ccd18 100644 --- a/soci/gzip_zinfo.go +++ b/soci/gzip_zinfo.go @@ -16,8 +16,8 @@ package soci -// #cgo CFLAGS: -I${SRCDIR}/../c/ -// #cgo LDFLAGS: -L${SRCDIR}/../out -lzinfo -lz +// #cgo CFLAGS: -I${SRCDIR}/ +// #cgo LDFLAGS: -L${SRCDIR}/../out -lz // #include "zinfo.h" // #include // #include diff --git a/soci/testutils.go b/soci/testutils.go index 5962f08bb..faa4ae060 100644 --- a/soci/testutils.go +++ b/soci/testutils.go @@ -16,11 +16,6 @@ package soci -// #include "zinfo.h" -// #include -// #include -import "C" - import ( "archive/tar" "bytes" @@ -31,7 +26,6 @@ import ( "math/rand" "os" "sort" - "unsafe" "github.com/awslabs/soci-snapshotter/util/testutil" "github.com/containerd/containerd/content" @@ -39,22 +33,6 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -const windowSize = 32768 - -type gzipCheckpoint struct { - out int64 /* corresponding offset in uncompressed data */ - in int64 /* offset in input file of first full byte */ - bits int8 /* number of bits (1-7) from byte at in - 1, or 0 */ - window []byte /* preceding 32K of uncompressed data */ -} - -type gzipZinfo struct { - have uint64 /* number of list entries filled in */ - size uint64 /* number of list entries allocated */ - list []gzipCheckpoint /* allocated list */ - spanSize uint64 -} - type TempDirMaker interface { TempDir() string } @@ -64,37 +42,6 @@ func parseDigest(digestString string) digest.Digest { return dgst } -func unmarshalGzipZinfo(blob byte) (*gzipZinfo, error) { - var index *C.struct_gzip_zinfo = C.blob_to_zinfo(unsafe.Pointer(&blob)) - - if index == nil { - return nil, fmt.Errorf("cannot convert blob to gzip_zinfo") - } - - defer C.free_zinfo(index) - - list := make([]gzipCheckpoint, 0) - lst := unsafe.Slice(index.list, int(index.have)) - for i := 0; i < int(index.have); i++ { - indexPoint := lst[i] - window := C.GoBytes(unsafe.Pointer(&indexPoint.window[0]), windowSize) - listEntry := gzipCheckpoint{ - out: int64(indexPoint.out), - in: int64(indexPoint.in), - bits: int8(indexPoint.bits), - window: window, - } - list = append(list, listEntry) - } - - return &gzipZinfo{ - have: uint64(index.have), - size: uint64(index.size), - spanSize: uint64(index.span_size), - list: list, - }, nil -} - type fileContent struct { fileName string content []byte diff --git a/c/zinfo.c b/soci/zinfo.c similarity index 100% rename from c/zinfo.c rename to soci/zinfo.c diff --git a/c/zinfo.h b/soci/zinfo.h similarity index 100% rename from c/zinfo.h rename to soci/zinfo.h diff --git a/soci/ztoc_test.go b/soci/ztoc_test.go index b53f8b61d..421b1b062 100644 --- a/soci/ztoc_test.go +++ b/soci/ztoc_test.go @@ -144,12 +144,9 @@ func TestDecompress(t *testing.T) { t.Fatalf("%s: could not read file %s", tc.name, fileName) } if !bytes.Equal(extracted, original) { - for i := 0; i < len(original); i++ { - if extracted[i] != original[i] { - t.Fatalf("%s: span_size=%d: file %s extracted bytes != original bytes; byte %d is different", - tc.name, tc.spanSize, fileName, i) - } - } + diffIdx := getPositionOfFirstDiffInByteSlice(extracted, original) + t.Fatalf("%s: span_size=%d: file %s extracted bytes != original bytes; byte %d is different", + tc.name, tc.spanSize, fileName, diffIdx) } } @@ -236,53 +233,8 @@ func TestZtocGenerationConsistency(t *testing.T) { // Compare raw Checkpoints if !bytes.Equal(ztoc1.CompressionInfo.Checkpoints, ztoc2.CompressionInfo.Checkpoints) { - - // compare Checkpoints within Go - index1, err := unmarshalGzipZinfo(ztoc1.CompressionInfo.Checkpoints[0]) - if err != nil { - t.Fatalf("index from ztoc1 should contain data") - } - index2, err := unmarshalGzipZinfo(ztoc2.CompressionInfo.Checkpoints[0]) - if err != nil { - t.Fatalf("index from ztoc2 should contain data") - } - - if index1.have != index2.have { - t.Fatalf("index1.have=%d must be equal to index2.have=%d", index1.have, index2.have) - } - - if index1.size != index2.size { - t.Fatalf("index1.size=%d must be equal to index2.size=%d", index1.size, index2.size) - } - - if index1.spanSize != index2.spanSize { - t.Fatalf("index1.span_size=%d must be equal to index2.span_size=%d", index1.spanSize, index2.spanSize) - } - - if len(index1.list) != len(index2.list) { - t.Fatalf("len(index1.list)=%d must be equal to len(index2.list)=%d", len(index1.list), len(index2.list)) - } - - for i := 0; i < len(index1.list); i++ { - indexPoint1 := index1.list[i] - indexPoint2 := index2.list[i] - - if indexPoint1.bits != indexPoint2.bits { - t.Fatalf("index1.list[%d].bits=%d must be equal to index2.list[%d].bits=%d", i, index1.list[i].bits, i, index2.list[i].bits) - } - - if indexPoint1.in != indexPoint2.in { - t.Fatalf("index1.list[%d].in=%d must be equal to index2.list[%d].in=%d", i, index1.list[i].in, i, index2.list[i].in) - } - - if indexPoint1.out != indexPoint2.out { - t.Fatalf("index1.list[%d].out=%d must be equal to index2.list[%d].out=%d", i, index1.list[i].out, i, index2.list[i].out) - } - - if !reflect.DeepEqual(indexPoint1.window, indexPoint2.window) { - t.Fatalf("index1.list[%d].window must be identical to index2.list[%d].window", i, i) - } - } + diffIdx := getPositionOfFirstDiffInByteSlice(ztoc1.CompressionInfo.Checkpoints, ztoc2.CompressionInfo.Checkpoints) + t.Fatalf("ztoc1.CompressionInfo.Checkpoints differ ztoc2.CompressionInfo.Checkpoints starting from position %d", diffIdx) } }) @@ -669,3 +621,17 @@ func genRandomByteData(size int) []byte { rand.Read(b) return b } + +func getPositionOfFirstDiffInByteSlice(a, b []byte) int { + sz := len(a) + if len(b) < len(a) { + sz = len(b) + } + for i := 0; i < sz; i++ { + if a[i] != b[i] { + return i + } + } + + return -1 +}