Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove dependency on zinfo C library #120

Merged
merged 1 commit into from
Nov 4, 2022
Merged

Remove dependency on zinfo C library #120

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 2 additions & 5 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@
CMD_DESTDIR ?= /usr/local
GO111MODULE_VALUE=auto
OUTDIR ?= $(CURDIR)/out
UTIL_CFLAGS=-I${CURDIR}/c -L${OUTDIR} -lzinfo -lz
UTIL_LDFLAGS=-L${OUTDIR} -lzinfo -lz
UTIL_CFLAGS=-I${CURDIR}/c -L${OUTDIR} -lz
UTIL_LDFLAGS=-L${OUTDIR} -lz
PKG=github.com/awslabs/soci-snapshotter
VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always --tags)
REVISION=$(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi)
Expand Down Expand Up @@ -48,9 +48,6 @@ soci: FORCE
pre-build:
rm -rf ${OUTDIR}
@mkdir -p ${OUTDIR}
@gcc -c c/zinfo.c -o ${OUTDIR}/zinfo.o -O3 -Wall -Werror
@ar rvs ${OUTDIR}/libzinfo.a ${OUTDIR}/zinfo.o
@rm -f ${OUTDIR}/zinfo.o

install-cmake:
@wget https://github.com/Kitware/CMake/releases/download/v3.24.1/cmake-3.24.1-Linux-x86_64.sh -O cmake.sh
Expand Down
4 changes: 2 additions & 2 deletions soci/gzip_zinfo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@

package soci

// #cgo CFLAGS: -I${SRCDIR}/../c/
// #cgo LDFLAGS: -L${SRCDIR}/../out -lzinfo -lz
// #cgo CFLAGS: -I${SRCDIR}/
// #cgo LDFLAGS: -L${SRCDIR}/../out -lz
// #include "zinfo.h"
// #include <stdlib.h>
// #include <stdint.h>
Expand Down
53 changes: 0 additions & 53 deletions soci/testutils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,6 @@

package soci

// #include "zinfo.h"
// #include <stdlib.h>
// #include <stdio.h>
import "C"

import (
"archive/tar"
"bytes"
Expand All @@ -31,30 +26,13 @@ import (
"math/rand"
"os"
"sort"
"unsafe"

"github.com/awslabs/soci-snapshotter/util/testutil"
"github.com/containerd/containerd/content"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)

const windowSize = 32768

type gzipCheckpoint struct {
out int64 /* corresponding offset in uncompressed data */
in int64 /* offset in input file of first full byte */
bits int8 /* number of bits (1-7) from byte at in - 1, or 0 */
window []byte /* preceding 32K of uncompressed data */
}

type gzipZinfo struct {
have uint64 /* number of list entries filled in */
size uint64 /* number of list entries allocated */
list []gzipCheckpoint /* allocated list */
spanSize uint64
}

type TempDirMaker interface {
TempDir() string
}
Expand All @@ -64,37 +42,6 @@ func parseDigest(digestString string) digest.Digest {
return dgst
}

func unmarshalGzipZinfo(blob byte) (*gzipZinfo, error) {
var index *C.struct_gzip_zinfo = C.blob_to_zinfo(unsafe.Pointer(&blob))

if index == nil {
return nil, fmt.Errorf("cannot convert blob to gzip_zinfo")
}

defer C.free_zinfo(index)

list := make([]gzipCheckpoint, 0)
lst := unsafe.Slice(index.list, int(index.have))
for i := 0; i < int(index.have); i++ {
indexPoint := lst[i]
window := C.GoBytes(unsafe.Pointer(&indexPoint.window[0]), windowSize)
listEntry := gzipCheckpoint{
out: int64(indexPoint.out),
in: int64(indexPoint.in),
bits: int8(indexPoint.bits),
window: window,
}
list = append(list, listEntry)
}

return &gzipZinfo{
have: uint64(index.have),
size: uint64(index.size),
spanSize: uint64(index.span_size),
list: list,
}, nil
}

type fileContent struct {
fileName string
content []byte
Expand Down
0 c/zinfo.c → soci/zinfo.c
View file
Open in desktop
File renamed without changes.
0 c/zinfo.h → soci/zinfo.h
View file
Open in desktop
File renamed without changes.
72 changes: 19 additions & 53 deletions soci/ztoc_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,12 +144,9 @@ func TestDecompress(t *testing.T) {
t.Fatalf("%s: could not read file %s", tc.name, fileName)
}
if !bytes.Equal(extracted, original) {
for i := 0; i < len(original); i++ {
if extracted[i] != original[i] {
t.Fatalf("%s: span_size=%d: file %s extracted bytes != original bytes; byte %d is different",
tc.name, tc.spanSize, fileName, i)
}
}
diffIdx := getPositionOfFirstDiffInByteSlice(extracted, original)
t.Fatalf("%s: span_size=%d: file %s extracted bytes != original bytes; byte %d is different",
tc.name, tc.spanSize, fileName, diffIdx)
}
}

Expand Down Expand Up @@ -236,53 +233,8 @@ func TestZtocGenerationConsistency(t *testing.T) {

// Compare raw Checkpoints
if !bytes.Equal(ztoc1.CompressionInfo.Checkpoints, ztoc2.CompressionInfo.Checkpoints) {

// compare Checkpoints within Go
index1, err := unmarshalGzipZinfo(ztoc1.CompressionInfo.Checkpoints[0])
if err != nil {
t.Fatalf("index from ztoc1 should contain data")
}
index2, err := unmarshalGzipZinfo(ztoc2.CompressionInfo.Checkpoints[0])
if err != nil {
t.Fatalf("index from ztoc2 should contain data")
}

if index1.have != index2.have {
t.Fatalf("index1.have=%d must be equal to index2.have=%d", index1.have, index2.have)
}

if index1.size != index2.size {
t.Fatalf("index1.size=%d must be equal to index2.size=%d", index1.size, index2.size)
}

if index1.spanSize != index2.spanSize {
t.Fatalf("index1.span_size=%d must be equal to index2.span_size=%d", index1.spanSize, index2.spanSize)
}

if len(index1.list) != len(index2.list) {
t.Fatalf("len(index1.list)=%d must be equal to len(index2.list)=%d", len(index1.list), len(index2.list))
}

for i := 0; i < len(index1.list); i++ {
indexPoint1 := index1.list[i]
indexPoint2 := index2.list[i]

if indexPoint1.bits != indexPoint2.bits {
t.Fatalf("index1.list[%d].bits=%d must be equal to index2.list[%d].bits=%d", i, index1.list[i].bits, i, index2.list[i].bits)
}

if indexPoint1.in != indexPoint2.in {
t.Fatalf("index1.list[%d].in=%d must be equal to index2.list[%d].in=%d", i, index1.list[i].in, i, index2.list[i].in)
}

if indexPoint1.out != indexPoint2.out {
t.Fatalf("index1.list[%d].out=%d must be equal to index2.list[%d].out=%d", i, index1.list[i].out, i, index2.list[i].out)
}

if !reflect.DeepEqual(indexPoint1.window, indexPoint2.window) {
t.Fatalf("index1.list[%d].window must be identical to index2.list[%d].window", i, i)
}
}
diffIdx := getPositionOfFirstDiffInByteSlice(ztoc1.CompressionInfo.Checkpoints, ztoc2.CompressionInfo.Checkpoints)
vkuzniet marked this conversation as resolved.
Show resolved Hide resolved
t.Fatalf("ztoc1.CompressionInfo.Checkpoints differ ztoc2.CompressionInfo.Checkpoints starting from position %d", diffIdx)
}

})
Expand Down Expand Up @@ -669,3 +621,17 @@ func genRandomByteData(size int) []byte {
rand.Read(b)
return b
}

func getPositionOfFirstDiffInByteSlice(a, b []byte) int {
sz := len(a)
if len(b) < len(a) {
sz = len(b)
}
for i := 0; i < sz; i++ {
if a[i] != b[i] {
return i
}
}

return -1
}