From 941365e9ec0f27f8e605ddb816096ffe1fe20301 Mon Sep 17 00:00:00 2001 From: schou Date: Wed, 6 Jan 2021 13:55:48 -0500 Subject: [PATCH 1/7] initial commit of byte bit shifter --- src/bytes/bytes.go | 66 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/src/bytes/bytes.go b/src/bytes/bytes.go index ce52649f132bb9..8b3f256ce71b45 100644 --- a/src/bytes/bytes.go +++ b/src/bytes/bytes.go @@ -543,6 +543,39 @@ func HasSuffix(s, suffix []byte) bool { return len(s) >= len(suffix) && Equal(s[len(s)-len(suffix):], suffix) } +// Left shift an arbitrary amount +func Lsh(dst []byte, shiftBits int) { + lenDst := len(dst) + if shiftBits == 0 || lenDst == 0 { + return + } else if shiftBits < 0 { + panic("bytes error: negative shift amount") + } + + // determine the pad bytes + pad := shiftBits / 8 + + if shiftBits < lenDst*8 { + + // preset shift registers + shift := shiftBits % 8 + trunc := 8 - shift + + // do the shift + for i := 0; i < lenDst-pad-1; i++ { + dst[i] = (dst[i+pad] << shift) | (dst[i+pad+1] >> trunc) + } + dst[lenDst-pad-1] = dst[lenDst-1] << shift + } else { + pad = lenDst + } + + // pad out the rest + for i := lenDst - pad; i < lenDst; i++ { + dst[i] = 0 + } +} + // Map returns a copy of the byte slice s with all its characters modified // according to the mapping function. If mapping returns a negative value, the character is // dropped from the byte slice with no replacement. The characters in s and the @@ -607,6 +640,39 @@ func Repeat(b []byte, count int) []byte { return nb } +// Right shift an arbitrary amount +func Rsh(dst []byte, shiftBits int) { + lenDst := len(dst) + if shiftBits == 0 || lenDst == 0 { + return + } else if shiftBits < 0 { + panic("bytes error: negative shift amount") + } + + // determine the pad bytes + pad := shiftBits / 8 + + if shiftBits < lenDst*8 { + + // preset shift registers + shift := shiftBits % 8 + trunc := 8 - shift + + // do the shift + for i := lenDst - 1; i > pad; i-- { + dst[i] = (dst[i-pad] >> shift) | (dst[i-pad-1] << trunc) + } + dst[pad] = dst[0] >> shift + } else { + pad = lenDst + } + + // pad out the rest + for i := 0; i < pad; i++ { + dst[i] = 0 + } +} + // ToUpper returns a copy of the byte slice s with all Unicode letters mapped to // their upper case. func ToUpper(s []byte) []byte { From 2b78e0872b4e02f088cc6229c7b10c52af29cc84 Mon Sep 17 00:00:00 2001 From: schou Date: Wed, 6 Jan 2021 20:05:03 -0500 Subject: [PATCH 2/7] encourage the compiler to optimize --- src/bytes/bytes.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/bytes/bytes.go b/src/bytes/bytes.go index 8b3f256ce71b45..d147ebd5753ae1 100644 --- a/src/bytes/bytes.go +++ b/src/bytes/bytes.go @@ -562,8 +562,12 @@ func Lsh(dst []byte, shiftBits int) { trunc := 8 - shift // do the shift + cur := byte(0) + next := byte(dst[pad]) for i := 0; i < lenDst-pad-1; i++ { - dst[i] = (dst[i+pad] << shift) | (dst[i+pad+1] >> trunc) + cur = next + next = dst[i+pad+1] + dst[i] = (cur << shift) | (next >> trunc) } dst[lenDst-pad-1] = dst[lenDst-1] << shift } else { @@ -659,8 +663,12 @@ func Rsh(dst []byte, shiftBits int) { trunc := 8 - shift // do the shift + cur := byte(0) + next := byte(dst[lenDst-1-pad]) for i := lenDst - 1; i > pad; i-- { - dst[i] = (dst[i-pad] >> shift) | (dst[i-pad-1] << trunc) + cur = next + next = dst[i-pad-1] + dst[i] = (cur >> shift) | (next << trunc) } dst[pad] = dst[0] >> shift } else { From 4dc25bc41884ff77beb4c0ae00e7c599e72a5c29 Mon Sep 17 00:00:00 2001 From: schou Date: Thu, 7 Jan 2021 18:59:37 -0500 Subject: [PATCH 3/7] moving Rsh and Lsh to encoding/binary and creating Shift --- src/bytes/bytes.go | 74 ------------------------- src/encoding/binary/binary.go | 86 +++++++++++++++++++++++++++++ src/encoding/binary/example_test.go | 12 ++++ 3 files changed, 98 insertions(+), 74 deletions(-) diff --git a/src/bytes/bytes.go b/src/bytes/bytes.go index d147ebd5753ae1..ce52649f132bb9 100644 --- a/src/bytes/bytes.go +++ b/src/bytes/bytes.go @@ -543,43 +543,6 @@ func HasSuffix(s, suffix []byte) bool { return len(s) >= len(suffix) && Equal(s[len(s)-len(suffix):], suffix) } -// Left shift an arbitrary amount -func Lsh(dst []byte, shiftBits int) { - lenDst := len(dst) - if shiftBits == 0 || lenDst == 0 { - return - } else if shiftBits < 0 { - panic("bytes error: negative shift amount") - } - - // determine the pad bytes - pad := shiftBits / 8 - - if shiftBits < lenDst*8 { - - // preset shift registers - shift := shiftBits % 8 - trunc := 8 - shift - - // do the shift - cur := byte(0) - next := byte(dst[pad]) - for i := 0; i < lenDst-pad-1; i++ { - cur = next - next = dst[i+pad+1] - dst[i] = (cur << shift) | (next >> trunc) - } - dst[lenDst-pad-1] = dst[lenDst-1] << shift - } else { - pad = lenDst - } - - // pad out the rest - for i := lenDst - pad; i < lenDst; i++ { - dst[i] = 0 - } -} - // Map returns a copy of the byte slice s with all its characters modified // according to the mapping function. If mapping returns a negative value, the character is // dropped from the byte slice with no replacement. The characters in s and the @@ -644,43 +607,6 @@ func Repeat(b []byte, count int) []byte { return nb } -// Right shift an arbitrary amount -func Rsh(dst []byte, shiftBits int) { - lenDst := len(dst) - if shiftBits == 0 || lenDst == 0 { - return - } else if shiftBits < 0 { - panic("bytes error: negative shift amount") - } - - // determine the pad bytes - pad := shiftBits / 8 - - if shiftBits < lenDst*8 { - - // preset shift registers - shift := shiftBits % 8 - trunc := 8 - shift - - // do the shift - cur := byte(0) - next := byte(dst[lenDst-1-pad]) - for i := lenDst - 1; i > pad; i-- { - cur = next - next = dst[i-pad-1] - dst[i] = (cur >> shift) | (next << trunc) - } - dst[pad] = dst[0] >> shift - } else { - pad = lenDst - } - - // pad out the rest - for i := 0; i < pad; i++ { - dst[i] = 0 - } -} - // ToUpper returns a copy of the byte slice s with all Unicode letters mapped to // their upper case. func ToUpper(s []byte) []byte { diff --git a/src/encoding/binary/binary.go b/src/encoding/binary/binary.go index a31149979da629..05ad7e90631222 100644 --- a/src/encoding/binary/binary.go +++ b/src/encoding/binary/binary.go @@ -735,3 +735,89 @@ func intDataSize(data interface{}) int { } return 0 } + +// Shift function does bit shifts along a slice. When shiftBits is positive, the bits +// shift to the left, and when shiftBits is negative, the bits shift to the right. As +// this operates directly on the slice, memory is conserved. For I/O conservation, each +// element is read once and written once. +func Shift(dst []byte, shiftBits int) { + if shiftBits < 0 { + rsh(dst, -shiftBits) + } else { + lsh(dst, shiftBits) + } +} + +// Right shift a number of bits +func rsh(dst []byte, shiftBits int) { + lenDst := len(dst) + if shiftBits == 0 || lenDst == 0 { + return + } else if shiftBits < 0 { + panic("bytes error: negative shift amount") + } + + // determine the pad bytes + pad := shiftBits / 8 + + if shiftBits < lenDst*8 { + + // preset shift registers + shift := shiftBits % 8 + trunc := 8 - shift + + // do the shift + cur := byte(0) + next := byte(dst[lenDst-1-pad]) + for i := lenDst - 1; i > pad; i-- { + cur = next // encourage compilers to optimize + next = dst[i-pad-1] + dst[i] = (cur >> shift) | (next << trunc) + } + dst[pad] = dst[0] >> shift + } else { + pad = lenDst + } + + // pad out the rest + for i := 0; i < pad; i++ { + dst[i] = 0 + } +} + +// Left shift a number of bits +func lsh(dst []byte, shiftBits int) { + lenDst := len(dst) + if shiftBits == 0 || lenDst == 0 { + return + } else if shiftBits < 0 { + panic("bytes error: negative shift amount") + } + + // determine the pad bytes + pad := shiftBits / 8 + + if shiftBits < lenDst*8 { + + // preset shift registers + shift := shiftBits % 8 + trunc := 8 - shift + + // do the shift + cur := byte(0) + next := byte(dst[pad]) + for i := 0; i < lenDst-pad-1; i++ { + cur = next // encourage compilers to optimize + next = dst[i+pad+1] + dst[i] = (cur << shift) | (next >> trunc) + } + dst[lenDst-pad-1] = dst[lenDst-1] << shift + } else { + pad = lenDst + } + + // pad out the rest + for i := lenDst - pad; i < lenDst; i++ { + dst[i] = 0 + } +} diff --git a/src/encoding/binary/example_test.go b/src/encoding/binary/example_test.go index b994b897ce1b56..e504632e0681e7 100644 --- a/src/encoding/binary/example_test.go +++ b/src/encoding/binary/example_test.go @@ -185,3 +185,15 @@ func ExampleVarint() { // 63 // 64 } + +func ExampleShift() { + // Decode << to a byte array + data := []byte("<<") + fmt.Printf("%08b\n", data) // [00111100 00111100] + + // Shift 3 bits to the left (use -3 for right) + Shift(data, 3) + + // The output is now shifted positive 3 bits + fmt.Printf("%08b\n", data) // [11100001 11100000] +} From b61fa9acdf099df9fa676f45dcaea38b4262602c Mon Sep 17 00:00:00 2001 From: schou Date: Thu, 7 Jan 2021 22:07:37 -0500 Subject: [PATCH 4/7] initial commit --- src/encoding/pem/pem.go | 101 ++++++++++++++++++++++++++++++++++------ 1 file changed, 87 insertions(+), 14 deletions(-) diff --git a/src/encoding/pem/pem.go b/src/encoding/pem/pem.go index a7272da5ad3a81..3feb4ea6ba1026 100644 --- a/src/encoding/pem/pem.go +++ b/src/encoding/pem/pem.go @@ -106,6 +106,7 @@ func Decode(data []byte) (p *Block, rest []byte) { Type: string(typeLine), } + var prev_key string for { // This loop terminates because getLine's second result is // always smaller than its argument. @@ -114,17 +115,23 @@ func Decode(data []byte) (p *Block, rest []byte) { } line, next := getLine(rest) - i := bytes.IndexByte(line, ':') - if i == -1 { - break - } + if len(line) > 2 && (line[0] == 0x20 || line[0] == 0x9) && prev_key != "" { + // Joins values that spread across lines. + p.Headers[prev_key] = p.Headers[prev_key] + string(bytes.TrimSpace(line)) + rest = next + } else { + i := bytes.IndexByte(line, ':') + if i == -1 { + break + } - // TODO(agl): need to cope with values that spread across lines. - key, val := line[:i], line[i+1:] - key = bytes.TrimSpace(key) - val = bytes.TrimSpace(val) - p.Headers[string(key)] = string(val) - rest = next + key, val := line[:i], line[i+1:] + key = bytes.TrimSpace(key) + val = bytes.TrimSpace(val) + p.Headers[string(key)] = string(val) + prev_key = string(key) + rest = next + } } var endIndex, endTrailerIndex int @@ -212,17 +219,27 @@ type lineBreaker struct { line [pemLineLength]byte used int out io.Writer + pad bool } var nl = []byte{'\n'} +var sp = []byte{' '} func (l *lineBreaker) Write(b []byte) (n int, err error) { + // Write headers and PEM block. RFC1421 if l.used+len(b) < pemLineLength { copy(l.line[l.used:], b) l.used += len(b) return len(b), nil } + if l.pad { + _, err = l.out.Write(sp) + if err != nil { + return + } + } + n, err = l.out.Write(l.line[0:l.used]) if err != nil { return @@ -245,19 +262,75 @@ func (l *lineBreaker) Write(b []byte) (n int, err error) { func (l *lineBreaker) Close() (err error) { if l.used > 0 { + if l.pad { + _, err = l.out.Write(sp) + if err != nil { + return + } + } + _, err = l.out.Write(l.line[0:l.used]) if err != nil { return } _, err = l.out.Write(nl) } - return } -func writeHeader(out io.Writer, k, v string) error { - _, err := out.Write([]byte(k + ": " + v + "\n")) - return err +func writeHeader(out io.Writer, k, v string) (err error) { + // This follows the output specified by RFC1421 + + if len(k)+len(v)+2 < pemLineLength { + // Write header with key and value on one line. + _, err = out.Write([]byte(k + ": " + v + "\n")) + + } else { + // Write values that spread across lines. + _, err = out.Write([]byte(k + ": ")) + if err != nil { + return + } + used := len(k) + 2 + + for { + i := bytes.IndexByte([]byte(v), ',') + if i >= 0 && used+i < pemLineLength { + _, err = out.Write([]byte(v[:i+1])) + if err != nil { + return + } + v = v[i+1:] + used += i + 1 + } else { + if used > 0 { + // Break to a new line if we have too much. + _, err = out.Write(nl) + if err != nil { + return + } + } + breaker := &lineBreaker{pad: true} + breaker.out = out + if i == -1 { + _, err = breaker.Write([]byte(v)) + if err != nil { + return + } + breaker.Close() + return + } + _, err = breaker.Write([]byte(v[:i+1])) + if err != nil { + return + } + breaker.Close() + v = v[i+1:] + used = 0 + } + } + } + return } // Encode writes the PEM encoding of b to out. From bab48b9bbfcb01fd727f1790689b5f8b7e3dfade Mon Sep 17 00:00:00 2001 From: schou Date: Thu, 7 Jan 2021 22:12:16 -0500 Subject: [PATCH 5/7] removing extra files --- src/encoding/binary/binary.go | 86 ----------------------------- src/encoding/binary/example_test.go | 12 ---- 2 files changed, 98 deletions(-) diff --git a/src/encoding/binary/binary.go b/src/encoding/binary/binary.go index 05ad7e90631222..a31149979da629 100644 --- a/src/encoding/binary/binary.go +++ b/src/encoding/binary/binary.go @@ -735,89 +735,3 @@ func intDataSize(data interface{}) int { } return 0 } - -// Shift function does bit shifts along a slice. When shiftBits is positive, the bits -// shift to the left, and when shiftBits is negative, the bits shift to the right. As -// this operates directly on the slice, memory is conserved. For I/O conservation, each -// element is read once and written once. -func Shift(dst []byte, shiftBits int) { - if shiftBits < 0 { - rsh(dst, -shiftBits) - } else { - lsh(dst, shiftBits) - } -} - -// Right shift a number of bits -func rsh(dst []byte, shiftBits int) { - lenDst := len(dst) - if shiftBits == 0 || lenDst == 0 { - return - } else if shiftBits < 0 { - panic("bytes error: negative shift amount") - } - - // determine the pad bytes - pad := shiftBits / 8 - - if shiftBits < lenDst*8 { - - // preset shift registers - shift := shiftBits % 8 - trunc := 8 - shift - - // do the shift - cur := byte(0) - next := byte(dst[lenDst-1-pad]) - for i := lenDst - 1; i > pad; i-- { - cur = next // encourage compilers to optimize - next = dst[i-pad-1] - dst[i] = (cur >> shift) | (next << trunc) - } - dst[pad] = dst[0] >> shift - } else { - pad = lenDst - } - - // pad out the rest - for i := 0; i < pad; i++ { - dst[i] = 0 - } -} - -// Left shift a number of bits -func lsh(dst []byte, shiftBits int) { - lenDst := len(dst) - if shiftBits == 0 || lenDst == 0 { - return - } else if shiftBits < 0 { - panic("bytes error: negative shift amount") - } - - // determine the pad bytes - pad := shiftBits / 8 - - if shiftBits < lenDst*8 { - - // preset shift registers - shift := shiftBits % 8 - trunc := 8 - shift - - // do the shift - cur := byte(0) - next := byte(dst[pad]) - for i := 0; i < lenDst-pad-1; i++ { - cur = next // encourage compilers to optimize - next = dst[i+pad+1] - dst[i] = (cur << shift) | (next >> trunc) - } - dst[lenDst-pad-1] = dst[lenDst-1] << shift - } else { - pad = lenDst - } - - // pad out the rest - for i := lenDst - pad; i < lenDst; i++ { - dst[i] = 0 - } -} diff --git a/src/encoding/binary/example_test.go b/src/encoding/binary/example_test.go index e504632e0681e7..b994b897ce1b56 100644 --- a/src/encoding/binary/example_test.go +++ b/src/encoding/binary/example_test.go @@ -185,15 +185,3 @@ func ExampleVarint() { // 63 // 64 } - -func ExampleShift() { - // Decode << to a byte array - data := []byte("<<") - fmt.Printf("%08b\n", data) // [00111100 00111100] - - // Shift 3 bits to the left (use -3 for right) - Shift(data, 3) - - // The output is now shifted positive 3 bits - fmt.Printf("%08b\n", data) // [11100001 11100000] -} From a958a4d2a96bdff2e2b802e826e345f57c92b07a Mon Sep 17 00:00:00 2001 From: schou Date: Thu, 7 Jan 2021 22:15:32 -0500 Subject: [PATCH 6/7] add comments --- src/encoding/pem/pem.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/encoding/pem/pem.go b/src/encoding/pem/pem.go index 3feb4ea6ba1026..05b4fe9792cce7 100644 --- a/src/encoding/pem/pem.go +++ b/src/encoding/pem/pem.go @@ -234,6 +234,7 @@ func (l *lineBreaker) Write(b []byte) (n int, err error) { } if l.pad { + // Make sure to add a pad if we need to. _, err = l.out.Write(sp) if err != nil { return @@ -263,6 +264,7 @@ func (l *lineBreaker) Write(b []byte) (n int, err error) { func (l *lineBreaker) Close() (err error) { if l.used > 0 { if l.pad { + // Make sure to add a pad if we need to. _, err = l.out.Write(sp) if err != nil { return @@ -296,6 +298,7 @@ func writeHeader(out io.Writer, k, v string) (err error) { for { i := bytes.IndexByte([]byte(v), ',') if i >= 0 && used+i < pemLineLength { + // If we can fit it on the rest of the line. _, err = out.Write([]byte(v[:i+1])) if err != nil { return @@ -310,9 +313,11 @@ func writeHeader(out io.Writer, k, v string) (err error) { return } } + // Create a new lineBreaker writer. breaker := &lineBreaker{pad: true} breaker.out = out if i == -1 { + // Block out the rest and call it done. _, err = breaker.Write([]byte(v)) if err != nil { return @@ -320,6 +325,7 @@ func writeHeader(out io.Writer, k, v string) (err error) { breaker.Close() return } + // Write up to and include the comma. _, err = breaker.Write([]byte(v[:i+1])) if err != nil { return From d1db37ad37cffe35edcd66c860f65b4fc0eb7bbf Mon Sep 17 00:00:00 2001 From: schou Date: Fri, 8 Jan 2021 20:27:27 -0500 Subject: [PATCH 7/7] case in which additional sections follow a base64 in pem header block --- src/encoding/pem/pem.go | 75 +++++++++++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 25 deletions(-) diff --git a/src/encoding/pem/pem.go b/src/encoding/pem/pem.go index 05b4fe9792cce7..fb94e1996c7863 100644 --- a/src/encoding/pem/pem.go +++ b/src/encoding/pem/pem.go @@ -115,7 +115,7 @@ func Decode(data []byte) (p *Block, rest []byte) { } line, next := getLine(rest) - if len(line) > 2 && (line[0] == 0x20 || line[0] == 0x9) && prev_key != "" { + if len(line) > 1 && (line[0] == 0x20 || line[0] == 0x9) && prev_key != "" { // Joins values that spread across lines. p.Headers[prev_key] = p.Headers[prev_key] + string(bytes.TrimSpace(line)) rest = next @@ -293,46 +293,71 @@ func writeHeader(out io.Writer, k, v string) (err error) { if err != nil { return } + used := len(k) + 2 + var blob string - for { + for len(v) > 0 { i := bytes.IndexByte([]byte(v), ',') - if i >= 0 && used+i < pemLineLength { - // If we can fit it on the rest of the line. - _, err = out.Write([]byte(v[:i+1])) + if i == -1 { + i = len(v) + blob = v + v = v[:0] + } else { + i++ + blob = v[:i] + v = v[i:] + } + if used == 0 { + _, err = out.Write(sp) if err != nil { return } - v = v[i+1:] - used += i + 1 - } else { - if used > 0 { - // Break to a new line if we have too much. - _, err = out.Write(nl) - if err != nil { - return - } + used++ + } + + if used+i <= pemLineLength { + // If we can fit it on the rest of the line write the blob. + _, err = out.Write([]byte(blob)) + if err != nil { + return + } + used += i + } else if i > pemLineLength { + // PEM / long blocks, should have their own section with the same length + // as the usual PEM, start on a new line, and have a pad space in front. + // The reason behind this is that they can be converted back into + // a standard PEM just by removing the padding and any trailing ','. + _, err = out.Write(nl) + if err != nil { + return } // Create a new lineBreaker writer. breaker := &lineBreaker{pad: true} breaker.out = out - if i == -1 { - // Block out the rest and call it done. - _, err = breaker.Write([]byte(v)) - if err != nil { - return - } - breaker.Close() + + // Block out the rest and call it done. + _, err = breaker.Write([]byte(blob)) + if err != nil { return } - // Write up to and include the comma. - _, err = breaker.Write([]byte(v[:i+1])) + err = breaker.Close() if err != nil { return } - breaker.Close() - v = v[i+1:] used = 0 + } else { + _, err = out.Write([]byte("\n " + blob)) + if err != nil { + return + } + used = i + } + } + if used > 0 { + _, err = out.Write(nl) + if err != nil { + return } } }