Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Automated rollback of commit adb2c4b415c35cda3e4394384ec579ea8dd8ec92. #13323

Merged
merged 1 commit into from
Jul 31, 2023
Merged

Automated rollback of commit adb2c4b415c35cda3e4394384ec579ea8dd8ec92. #13323

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
217 changes: 74 additions & 143 deletions src/google/protobuf/generated_message_tctable_lite.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,45 +393,6 @@ inline PROTOBUF_ALWAYS_INLINE void InvertPacked(TcFieldData& data) {
data.data ^= Wt ^ WireFormatLite::WIRETYPE_LENGTH_DELIMITED;
}

constexpr uint32_t kAccumulatorBytesOnStack = 256;

// Accumulates fields to buffer repeated fields on parsing path to avoid growing
// repeated field container type too frequently. It flushes to the backing
// repeated fields if it's full or out of the scope. A larger buffer (e.g. 2KiB)
// is actually harmful due to:
// - increased stack overflow risk
// - extra cache misses on accessing local variables
// - less competitive to the cost of growing large buffer
template <typename ElementType, typename ContainerType>
class ScopedFieldAccumulator {
public:
constexpr explicit ScopedFieldAccumulator(ContainerType& field)
: field_(field) {}

~ScopedFieldAccumulator() {
if (ABSL_PREDICT_TRUE(current_size_ > 0)) {
field_.MergeFromArray(buffer_, current_size_);
}
}

PROTOBUF_NODISCARD ElementType& Next() {
if (ABSL_PREDICT_FALSE(current_size_ == kSize)) {
field_.MergeFromArray(buffer_, kSize);
current_size_ = 0;
}
return buffer_[current_size_++];
}

private:
static constexpr uint32_t kSize =
kAccumulatorBytesOnStack / sizeof(ElementType);
static_assert(kSize > 0, "Size cannot be zero");

uint32_t current_size_ = 0;
ElementType buffer_[kSize];
ContainerType& field_;
};

} // namespace

//////////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -659,17 +620,14 @@ PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedFixed(
}
auto& field = RefAt<RepeatedField<LayoutType>>(msg, data.offset());
const auto tag = UnalignedLoad<TagType>(ptr);
{
ScopedFieldAccumulator<LayoutType, decltype(field)> accumulator(field);
do {
accumulator.Next() = UnalignedLoad<LayoutType>(ptr + sizeof(TagType));
ptr += sizeof(TagType) + sizeof(LayoutType);
if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) goto parse_loop;
} while (UnalignedLoad<TagType>(ptr) == tag);
}
do {
field.Add(UnalignedLoad<LayoutType>(ptr + sizeof(TagType)));
ptr += sizeof(TagType) + sizeof(LayoutType);
if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) {
PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS);
}
} while (UnalignedLoad<TagType>(ptr) == tag);
PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS);
parse_loop:
PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS);
}

PROTOBUF_NOINLINE const char* TcParser::FastF32R1(PROTOBUF_TC_PARAM_DECL) {
Expand Down Expand Up @@ -1003,22 +961,19 @@ PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedVarint(
}
auto& field = RefAt<RepeatedField<FieldType>>(msg, data.offset());
const auto expected_tag = UnalignedLoad<TagType>(ptr);
{
ScopedFieldAccumulator<FieldType, decltype(field)> accumulator(field);
do {
ptr += sizeof(TagType);
FieldType tmp;
ptr = ParseVarint(ptr, &tmp);
if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) goto error;
accumulator.Next() = ZigZagDecodeHelper<FieldType, zigzag>(tmp);
if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) goto parse_loop;
} while (UnalignedLoad<TagType>(ptr) == expected_tag);
}
do {
ptr += sizeof(TagType);
FieldType tmp;
ptr = ParseVarint(ptr, &tmp);
if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) {
PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS);
}
field.Add(ZigZagDecodeHelper<FieldType, zigzag>(tmp));
if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) {
PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS);
}
} while (UnalignedLoad<TagType>(ptr) == expected_tag);
PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS);
parse_loop:
PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS);
error:
PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS);
}

PROTOBUF_NOINLINE const char* TcParser::FastV8R1(PROTOBUF_TC_PARAM_DECL) {
Expand Down Expand Up @@ -1080,8 +1035,7 @@ const char* TcParser::PackedVarint(PROTOBUF_TC_PARAM_DECL) {
// pending hasbits now:
SyncHasbits(msg, hasbits, table);
auto* field = &RefAt<RepeatedField<FieldType>>(msg, data.offset());
ScopedFieldAccumulator<FieldType, decltype(*field)> accumulator(*field);
return ctx->ReadPackedVarint(ptr, [&](uint64_t varint) {
return ctx->ReadPackedVarint(ptr, [field](uint64_t varint) {
FieldType val;
if (zigzag) {
if (sizeof(FieldType) == 8) {
Expand All @@ -1092,7 +1046,7 @@ const char* TcParser::PackedVarint(PROTOBUF_TC_PARAM_DECL) {
} else {
val = varint;
}
accumulator.Next() = val;
field->Add(val);
});
}

Expand Down Expand Up @@ -1227,33 +1181,28 @@ const char* TcParser::RepeatedEnum(PROTOBUF_TC_PARAM_DECL) {
auto& field = RefAt<RepeatedField<int32_t>>(msg, data.offset());
const auto expected_tag = UnalignedLoad<TagType>(ptr);
const TcParseTableBase::FieldAux aux = *table->field_aux(data.aux_idx());
{
ScopedFieldAccumulator<int32_t, decltype(field)> accumulator(field);
do {
const char* ptr2 = ptr; // save for unknown enum case
ptr += sizeof(TagType);
uint64_t tmp;
ptr = ParseVarint(ptr, &tmp);
if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) goto error;
if (PROTOBUF_PREDICT_FALSE(
!EnumIsValidAux(static_cast<int32_t>(tmp), xform_val, aux))) {
// We can avoid duplicate work in MiniParse by directly calling
// table->fallback.
ptr = ptr2;
goto unknown_enum_fallback;
}
accumulator.Next() = static_cast<int32_t>(tmp);
if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) goto parse_loop;
} while (UnalignedLoad<TagType>(ptr) == expected_tag);
}
do {
const char* ptr2 = ptr; // save for unknown enum case
ptr += sizeof(TagType);
uint64_t tmp;
ptr = ParseVarint(ptr, &tmp);
if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) {
PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS);
}
if (PROTOBUF_PREDICT_FALSE(
!EnumIsValidAux(static_cast<int32_t>(tmp), xform_val, aux))) {
// We can avoid duplicate work in MiniParse by directly calling
// table->fallback.
ptr = ptr2;
PROTOBUF_MUSTTAIL return FastUnknownEnumFallback(PROTOBUF_TC_PARAM_PASS);
}
field.Add(static_cast<int32_t>(tmp));
if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) {
PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS);
}
} while (UnalignedLoad<TagType>(ptr) == expected_tag);

PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS);
parse_loop:
PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS);
error:
PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS);
unknown_enum_fallback:
PROTOBUF_MUSTTAIL return FastUnknownEnumFallback(PROTOBUF_TC_PARAM_PASS);
}

const TcParser::UnknownFieldOps& TcParser::GetUnknownFieldOps(
Expand Down Expand Up @@ -1387,22 +1336,19 @@ const char* TcParser::RepeatedEnumSmallRange(PROTOBUF_TC_PARAM_DECL) {
auto& field = RefAt<RepeatedField<int32_t>>(msg, data.offset());
auto expected_tag = UnalignedLoad<TagType>(ptr);
const uint8_t max = data.aux_idx();
{
ScopedFieldAccumulator<int32_t, decltype(field)> accumulator(field);
do {
uint8_t v = ptr[sizeof(TagType)];
if (PROTOBUF_PREDICT_FALSE(min > v || v > max)) goto mini_parse;
accumulator.Next() = static_cast<int32_t>(v);
ptr += sizeof(TagType) + 1;
if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) goto parse_loop;
} while (UnalignedLoad<TagType>(ptr) == expected_tag);
}
do {
uint8_t v = ptr[sizeof(TagType)];
if (PROTOBUF_PREDICT_FALSE(min > v || v > max)) {
PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS);
}
field.Add(static_cast<int32_t>(v));
ptr += sizeof(TagType) + 1;
if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) {
PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS);
}
} while (UnalignedLoad<TagType>(ptr) == expected_tag);

PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS);
parse_loop:
PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS);
mini_parse:
PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS);
}

PROTOBUF_NOINLINE const char* TcParser::FastEr0R1(PROTOBUF_TC_PARAM_DECL) {
Expand Down Expand Up @@ -1904,10 +1850,9 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedFixed(
constexpr auto size = sizeof(uint64_t);
const char* ptr2 = ptr;
uint32_t next_tag;
ScopedFieldAccumulator<uint64_t, decltype(field)> accumulator(field);
do {
ptr = ptr2;
accumulator.Next() = UnalignedLoad<uint64_t>(ptr);
*field.Add() = UnalignedLoad<uint64_t>(ptr);
ptr += size;
if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) goto parse_loop;
ptr2 = ReadTag(ptr, &next_tag);
Expand All @@ -1922,10 +1867,9 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedFixed(
constexpr auto size = sizeof(uint32_t);
const char* ptr2 = ptr;
uint32_t next_tag;
ScopedFieldAccumulator<uint32_t, decltype(field)> accumulator(field);
do {
ptr = ptr2;
accumulator.Next() = UnalignedLoad<uint32_t>(ptr);
*field.Add() = UnalignedLoad<uint32_t>(ptr);
ptr += size;
if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) goto parse_loop;
ptr2 = ReadTag(ptr, &next_tag);
Expand Down Expand Up @@ -2055,13 +1999,11 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedVarint(
auto& field = RefAt<RepeatedField<uint64_t>>(msg, entry.offset);
const char* ptr2 = ptr;
uint32_t next_tag;
ScopedFieldAccumulator<uint64_t, decltype(field)> accumulator(field);
do {
uint64_t tmp;
ptr = ParseVarint(ptr2, &tmp);
if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) goto error;
accumulator.Next() =
is_zigzag ? WireFormatLite::ZigZagDecode64(tmp) : tmp;
field.Add(is_zigzag ? WireFormatLite::ZigZagDecode64(tmp) : tmp);
if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) goto parse_loop;
ptr2 = ReadTag(ptr, &next_tag);
if (PROTOBUF_PREDICT_FALSE(ptr2 == nullptr)) goto error;
Expand All @@ -2070,20 +2012,20 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedVarint(
auto& field = RefAt<RepeatedField<uint32_t>>(msg, entry.offset);
const char* ptr2 = ptr;
uint32_t next_tag;
ScopedFieldAccumulator<uint32_t, decltype(field)> accumulator(field);
do {
uint64_t tmp;
ptr = ParseVarint(ptr2, &tmp);
if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) goto error;
if (is_validated_enum) {
if (!EnumIsValidAux(tmp, xform_val, *table->field_aux(&entry))) {
ptr = ptr2;
goto unknown_enum_fallback;
PROTOBUF_MUSTTAIL return MpUnknownEnumFallback(
PROTOBUF_TC_PARAM_PASS);
}
} else if (is_zigzag) {
tmp = WireFormatLite::ZigZagDecode32(tmp);
}
accumulator.Next() = tmp;
field.Add(tmp);
if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) goto parse_loop;
ptr2 = ReadTag(ptr, &next_tag);
if (PROTOBUF_PREDICT_FALSE(ptr2 == nullptr)) goto error;
Expand All @@ -2093,12 +2035,11 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedVarint(
auto& field = RefAt<RepeatedField<bool>>(msg, entry.offset);
const char* ptr2 = ptr;
uint32_t next_tag;
ScopedFieldAccumulator<bool, decltype(field)> accumulator(field);
do {
uint64_t tmp;
ptr = ParseVarint(ptr2, &tmp);
if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) goto error;
accumulator.Next() = static_cast<bool>(tmp);
field.Add(static_cast<bool>(tmp));
if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) goto parse_loop;
ptr2 = ReadTag(ptr, &next_tag);
if (PROTOBUF_PREDICT_FALSE(ptr2 == nullptr)) goto error;
Expand All @@ -2110,8 +2051,6 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedVarint(
PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS);
error:
PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS);
unknown_enum_fallback:
PROTOBUF_MUSTTAIL return MpUnknownEnumFallback(PROTOBUF_TC_PARAM_PASS);
}

PROTOBUF_NOINLINE const char* TcParser::MpPackedVarint(PROTOBUF_TC_PARAM_DECL) {
Expand All @@ -2133,41 +2072,33 @@ PROTOBUF_NOINLINE const char* TcParser::MpPackedVarint(PROTOBUF_TC_PARAM_DECL) {

uint16_t rep = type_card & field_layout::kRepMask;
if (rep == field_layout::kRep64Bits) {
auto& field = RefAt<RepeatedField<uint64_t>>(msg, entry.offset);
ScopedFieldAccumulator<uint64_t, decltype(field)> accumulator(field);
return ctx->ReadPackedVarint(
ptr, [&accumulator, is_zigzag](uint64_t value) {
accumulator.Next() =
is_zigzag ? WireFormatLite::ZigZagDecode64(value) : value;
});
auto* field = &RefAt<RepeatedField<uint64_t>>(msg, entry.offset);
return ctx->ReadPackedVarint(ptr, [field, is_zigzag](uint64_t value) {
field->Add(is_zigzag ? WireFormatLite::ZigZagDecode64(value) : value);
});
} else if (rep == field_layout::kRep32Bits) {
auto& field = RefAt<RepeatedField<uint32_t>>(msg, entry.offset);
auto* field = &RefAt<RepeatedField<uint32_t>>(msg, entry.offset);
if (is_validated_enum) {
const TcParseTableBase::FieldAux aux = *table->field_aux(entry.aux_idx);
ScopedFieldAccumulator<uint32_t, decltype(field)> accumulator(field);
return ctx->ReadPackedVarint(ptr, [=, &accumulator](int32_t value) {
return ctx->ReadPackedVarint(ptr, [=](int32_t value) {
if (!EnumIsValidAux(value, xform_val, aux)) {
AddUnknownEnum(msg, table, data.tag(), value);
} else {
accumulator.Next() = value;
field->Add(value);
}
});
} else {
ScopedFieldAccumulator<uint32_t, decltype(field)> accumulator(field);
return ctx->ReadPackedVarint(
ptr, [&accumulator, is_zigzag](uint64_t value) {
accumulator.Next() = is_zigzag ? WireFormatLite::ZigZagDecode32(
static_cast<uint32_t>(value))
: value;
});
return ctx->ReadPackedVarint(ptr, [field, is_zigzag](uint64_t value) {
field->Add(is_zigzag ? WireFormatLite::ZigZagDecode32(
static_cast<uint32_t>(value))
: value);
});
}
} else {
ABSL_DCHECK_EQ(rep, static_cast<uint16_t>(field_layout::kRep8Bits));
auto& field = RefAt<RepeatedField<bool>>(msg, entry.offset);
ScopedFieldAccumulator<bool, decltype(field)> accumulator(field);
return ctx->ReadPackedVarint(ptr, [&](uint64_t value) {
accumulator.Next() = static_cast<bool>(value);
});
auto* field = &RefAt<RepeatedField<bool>>(msg, entry.offset);
return ctx->ReadPackedVarint(
ptr, [field](uint64_t value) { field->Add(static_cast<bool>(value)); });
}

PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS);
Expand Down
26 changes: 0 additions & 26 deletions src/google/protobuf/repeated_field.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -358,8 +358,6 @@ class RepeatedField final
// This is public due to it being called by generated code.
inline void InternalSwap(RepeatedField* other);

void MergeFromArray(const Element* array, size_t length);

private:
RepeatedField(Arena* arena, const RepeatedField& rhs);
template <typename T> friend class Arena::InternalHelper;
Expand Down Expand Up @@ -624,30 +622,6 @@ inline int RepeatedField<Element>::Capacity() const {
return total_size_;
}

template <typename Element>
inline void RepeatedField<Element>::MergeFromArray(const Element* array,
size_t length) {
// Only supports trivially copyable types.
static_assert(std::is_trivially_copyable<Element>::value,
"only trivialy copyable types are supported");

ABSL_DCHECK_GT(length, 0u);
if (ABSL_PREDICT_TRUE(current_size_ + static_cast<int>(length) >
total_size_)) {
Grow(current_size_, current_size_ + length);
}
Element* elem = unsafe_elements();
ABSL_DCHECK_NE(elem, nullptr);
void* p = elem + ExchangeCurrentSize(current_size_ + length);
memcpy(p, array, sizeof(Element) * length);
}

template <>
inline void RepeatedField<absl::Cord>::MergeFromArray(const absl::Cord* array,
size_t length) {
ABSL_LOG(FATAL) << "not supported";
}

template <typename Element>
inline void RepeatedField<Element>::AddAlreadyReserved(Element value) {
ABSL_DCHECK_LT(current_size_, total_size_);
Expand Down
Loading