Add coding function (#1264)

be/src/util/CMakeLists.txt

@@ -26,6 +26,7 @@ set(UTIL_FILES
   bfd_parser.cpp
   bitmap.cpp
   codec.cpp
+  coding.cpp
   compress.cpp
   cpu_info.cpp
   dynamic_util.cpp

be/src/util/coding.cpp

@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "util/coding.h"
+
+namespace doris {
+
+uint8_t* encode_varint32(uint8_t* dst, uint32_t v) {
+    // Operate on characters as unsigneds
+    static const int B = 128;
+    if (v < (1 << 7)) {
+        *(dst++) = v;
+    } else if (v < (1 << 14)) {
+        *(dst++) = v | B;
+        *(dst++) = v >> 7;
+    } else if (v < (1 << 21)) {
+        *(dst++) = v | B;
+        *(dst++) = (v >> 7) | B;
+        *(dst++) = v >> 14;
+    } else if (v < (1 << 28)) {
+        *(dst++) = v | B;
+        *(dst++) = (v >> 7) | B;
+        *(dst++) = (v >> 14) | B;
+        *(dst++) = v >> 21;
+    } else {
+        *(dst++) = v | B;
+        *(dst++) = (v >> 7) | B;
+        *(dst++) = (v >> 14) | B;
+        *(dst++) = (v >> 21) | B;
+        *(dst++) = v >> 28;
+    }
+    return dst;
+}
+
+const uint8_t* decode_varint32_ptr_fallback(const uint8_t* p, const uint8_t* limit,
+                                         uint32_t* value) {
+    uint32_t result = 0;
+    for (uint32_t shift = 0; shift <= 28 && p < limit; shift += 7) {
+        uint32_t byte = *p;
+        p++;
+        if (byte & 128) {
+            // More bytes are present
+            result |= ((byte & 127) << shift);
+        } else {
+            result |= (byte << shift);
+            *value = result;
+            return p;
+        }
+    }
+    return nullptr;
+}
+
+const uint8_t* decode_varint64_ptr(const uint8_t* p, const uint8_t* limit, uint64_t* value) {
+    uint64_t result = 0;
+    for (uint32_t shift = 0; shift <= 63 && p < limit; shift += 7) {
+        uint64_t byte = *p;
+        p++;
+        if (byte & 128) {
+            // More bytes are present
+            result |= ((byte & 127) << shift);
+        } else {
+            result |= (byte << shift);
+            *value = result;
+            return p;
+        }
+    }
+    return nullptr;
+}
+
+}

be/src/util/coding.h

@@ -0,0 +1,156 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <string>
+
+#include "gutil/endian.h"
+
+namespace doris {
+
+// TODO(zc): add encode big endian later when we need it
+// use big endian when we have order requirement.
+// little endian is more efficient when we use X86 CPU, so
+// when we have no order needs, we prefer little endian encoding
+inline void encode_fixed8(uint8_t* buf, uint8_t val) {
+    *buf = val;
+}
+
+inline void encode_fixed16_le(uint8_t* buf, uint16_t val) {
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+    memcpy(buf, &val, sizeof(val));
+#else
+    uint16_t res = bswap_16(val);
+    memcpy(buf, &res, sizeof(res));
+#endif
+}
+
+inline void encode_fixed32_le(uint8_t* buf, uint32_t val) {
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+    memcpy(buf, &val, sizeof(val));
+#else
+    uint32_t res = bswap_32(val);
+    memcpy(buf, &res, sizeof(res));
+#endif
+}
+
+inline void encode_fixed64_le(uint8_t* buf, uint64_t val) {
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+    memcpy(buf, &val, sizeof(val));
+#else
+    uint64_t res = gbswap_64(val);
+    memcpy(buf, &res, sizeof(res));
+#endif
+}
+
+inline uint8_t decode_fixed8(const uint8_t* buf) {
+    return *buf;
+}
+
+inline uint16_t decode_fixed16_le(const uint8_t* buf) {
+    uint16_t res;
+    memcpy(&res, buf, sizeof(res));
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+    return res;
+#else
+    return bswap_16(res);
+#endif
+}
+
+inline uint32_t decode_fixed32_le(const uint8_t* buf) {
+    uint32_t res;
+    memcpy(&res, buf, sizeof(res));
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+    return res;
+#else
+    return bswap_32(res);
+#endif
+}
+
+inline uint64_t decode_fixed64_le(const uint8_t* buf) {
+    uint64_t res;
+    memcpy(&res, buf, sizeof(res));
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+    return res;
+#else
+    return gbswap_64(res);
+#endif
+}
+
+inline void put_fixed32_le(std::string* dst, uint32_t val) {
+    uint8_t buf[sizeof(val)];
+    encode_fixed32_le(buf, val);
+    dst->append((char*)buf, sizeof(buf));
+}
+
+inline void put_fixed64_le(std::string* dst, uint64_t val) {
+    uint8_t buf[sizeof(val)];
+    encode_fixed64_le(buf, val);
+    dst->append((char*)buf, sizeof(buf));
+}
+
+extern uint8_t* encode_varint32(uint8_t* dst, uint32_t value);
+extern uint8_t* encode_varint64(uint8_t* dst, uint64_t value);
+
+inline uint8_t* encode_varint64(uint8_t* dst, uint64_t v) {
+    static const unsigned int B = 128;
+    while (v >= B) {
+        *(dst++) = (v & (B - 1)) | B;
+        v >>= 7;
+    }
+    *(dst++) = static_cast<unsigned char>(v);
+    return dst;
+}
+
+extern const uint8_t* decode_varint32_ptr_fallback(
+    const uint8_t* p, const uint8_t* limit, uint32_t* value);
+
+inline const uint8_t* decode_varint32_ptr(
+        const uint8_t* ptr, const uint8_t* limit, uint32_t* value) {
+    if (ptr < limit) {
+        uint32_t result = *ptr;
+        if ((result & 128) == 0) {
+            *value = result;
+            return ptr + 1;
+        }
+    }
+    return decode_varint32_ptr_fallback(ptr, limit, value);
+}
+
+extern const uint8_t* decode_varint64_ptr(const uint8_t* p, const uint8_t* limit, uint64_t* value);
+
+inline void put_varint32(std::string* dst, uint32_t v) {
+    uint8_t buf[5];
+    uint8_t* ptr = encode_varint32(buf, v);
+    dst->append((char*)buf, static_cast<size_t>(ptr - buf));
+}
+
+inline void put_varint64(std::string* dst, uint64_t v) {
+    uint8_t buf[10];
+    uint8_t* ptr = encode_varint64(buf, v);
+    dst->append((char*)buf, static_cast<size_t>(ptr - buf));
+}
+
+inline void put_varint64_varint32(std::string* dst, uint64_t v1, uint32_t v2) {
+    uint8_t buf[15];
+    uint8_t* ptr = encode_varint64(buf, v1);
+    ptr = encode_varint32(ptr, v2);
+    dst->append((char*)buf, static_cast<size_t>(ptr - buf));
+}
+
+}

be/test/util/CMakeLists.txt

@@ -22,6 +22,7 @@ ADD_BE_TEST(bit_util_test)
 ADD_BE_TEST(brpc_stub_cache_test)
 ADD_BE_TEST(path_trie_test)
 ADD_BE_TEST(count_down_latch_test)
+ADD_BE_TEST(coding_test)
 ADD_BE_TEST(lru_cache_util_test)
 ADD_BE_TEST(filesystem_util_test)
 ADD_BE_TEST(internal_queue_test)