Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix cast to decimal overflow bug (#3922) #3942

Merged
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix conflict
Signed-off-by: guo-shaoge <[email protected]>
  • Loading branch information
guo-shaoge committed Jan 26, 2022
commit e2189ee4a7ffdd9025f0c42e2fb69925991de1a2
165 changes: 4 additions & 161 deletions dbms/src/Flash/Coprocessor/DAGContext.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -11,85 +11,11 @@ extern const int DIVIDED_BY_ZERO;
extern const int INVALID_TIME;
} // namespace ErrorCodes

<<<<<<< HEAD
namespace
{
enum Flag
{
IGNORE_TRUNCATE = 1,
TRUNCATE_AS_WARNING = 1u << 1u,
PAD_CHAR_TO_FULL_LENGTH = 1u << 2u,
IN_INSERT_STMT = 1u << 3u,
IN_UPDATE_OR_DELETE_STMT = 1u << 4u,
IN_SELECT_STMT = 1u << 5u,
OVERFLOW_AS_WARNING = 1u << 6u,
IGNORE_ZERO_IN_DATE = 1u << 7u,
DIVIDED_BY_ZERO_AS_WARNING = 1u << 8u,
IN_LOAD_DATA_STMT = 1u << 10u,
};

enum SqlMode
{
REAL_AS_FLOAT = 1ul,
PIPES_AS_CONCAT = 1ul << 1ul,
ANSI_QUOTES = 1ul << 2ul,
IGNORE_SPACE = 1ul << 3ul,
NOT_USED = 1ul << 4ul,
ONLY_FULL_GROUP_BY = 1ul << 5ul,
NO_UNSIGNED_SUBTRACTION = 1ul << 6ul,
NO_DIR_IN_CREATE = 1ul << 7ul,
POSTGRESQL = 1ul << 8ul,
ORACLE = 1ul << 9ul,
MSSQL = 1ul << 10ul,
DB2 = 1ul << 11ul,
MAXDB = 1ul << 12ul,
NO_KEY_OPTIONS = 1ul << 13ul,
NO_TABLE_OPTIONS = 1ul << 14ul,
NO_FIELD_OPTIONS = 1ul << 15ul,
MYSQL323 = 1ul << 16ul,
MYSQL40 = 1ul << 17ul,
ANSI = 1ul << 18ul,
NO_AUTO_VALUE_ON_ZERO = 1ul << 19ul,
NO_BACK_SLASH_ESCAPES = 1ul << 20ul,
STRICT_TRANS_TABLES = 1ul << 21ul,
STRICT_ALL_TABLES = 1ul << 22ul,
NO_ZERO_IN_DATE = 1ul << 23ul,
NO_ZERO_DATE = 1ul << 24ul,
INVALID_DATES = 1ul << 25ul,
ERROR_FOR_DIVISION_BY_ZERO = 1ul << 26ul,
TRADITIONAL = 1ul << 27ul,
NO_AUTO_CREATE_USER = 1ul << 28ul,
HIGH_NOT_PRECEDENCE = 1ul << 29ul,
NO_ENGINE_SUBSTITUTION = 1ul << 30ul,

// Duplicated with Flag::PAD_CHAR_TO_FULL_LENGTH
// PAD_CHAR_TO_FULL_LENGTH = 1ul << 31ul,

ALLOW_INVALID_DATES = 1ul << 32ul,
};
} // namespace
bool strictSqlMode(UInt64 sql_mode) { return sql_mode & TiDBSQLMode::STRICT_ALL_TABLES || sql_mode & TiDBSQLMode::STRICT_TRANS_TABLES; }

bool strictSqlMode(UInt64 sql_mode) { return sql_mode & SqlMode::STRICT_ALL_TABLES || sql_mode & SqlMode::STRICT_TRANS_TABLES; }
bool DAGContext::allowZeroInDate() const { return flags & TiDBSQLFlags::IGNORE_ZERO_IN_DATE; }

bool DAGContext::allowZeroInDate() const { return flags & Flag::IGNORE_ZERO_IN_DATE; }

bool DAGContext::allowInvalidDate() const { return sql_mode & SqlMode::ALLOW_INVALID_DATES; }
=======
bool strictSqlMode(UInt64 sql_mode)
{
return sql_mode & TiDBSQLMode::STRICT_ALL_TABLES || sql_mode & TiDBSQLMode::STRICT_TRANS_TABLES;
}

bool DAGContext::allowZeroInDate() const
{
return flags & TiDBSQLFlags::IGNORE_ZERO_IN_DATE;
}

bool DAGContext::allowInvalidDate() const
{
return sql_mode & TiDBSQLMode::ALLOW_INVALID_DATES;
}
>>>>>>> 6ea6c80198 (Fix cast to decimal overflow bug (#3922))
bool DAGContext::allowInvalidDate() const { return sql_mode & TiDBSQLMode::ALLOW_INVALID_DATES; }

std::map<String, ProfileStreamsInfo> & DAGContext::getProfileStreamsMap() { return profile_streams_map; }

Expand Down Expand Up @@ -154,89 +80,6 @@ void DAGContext::handleInvalidTime(const String & msg, const TiFlashError & erro
}
}

<<<<<<< HEAD
bool DAGContext::shouldClipToZero() { return flags & Flag::IN_INSERT_STMT || flags & Flag::IN_LOAD_DATA_STMT; }
=======
void DAGContext::appendWarning(const String & msg, int32_t code)
{
tipb::Error warning;
warning.set_code(code);
warning.set_msg(msg);
appendWarning(warning);
}

bool DAGContext::shouldClipToZero() const
{
return flags & TiDBSQLFlags::IN_INSERT_STMT || flags & TiDBSQLFlags::IN_LOAD_DATA_STMT;
}

std::pair<bool, double> DAGContext::getTableScanThroughput()
{
if (table_scan_executor_id.empty())
return std::make_pair(false, 0.0);

// collect table scan metrics
UInt64 time_processed_ns = 0;
UInt64 num_produced_bytes = 0;
for (auto & p : getProfileStreamsMap())
{
if (p.first == table_scan_executor_id)
{
for (auto & stream_ptr : p.second.input_streams)
{
if (auto * p_stream = dynamic_cast<IProfilingBlockInputStream *>(stream_ptr.get()))
{
time_processed_ns = std::max(time_processed_ns, p_stream->getProfileInfo().execution_time);
num_produced_bytes += p_stream->getProfileInfo().bytes;
}
}
break;
}
}

// convert to bytes per second
return std::make_pair(true, num_produced_bytes / (static_cast<double>(time_processed_ns) / 1000000000ULL));
}

void DAGContext::attachBlockIO(const BlockIO & io_)
{
io = io_;
}
void DAGContext::initExchangeReceiverIfMPP(Context & context, size_t max_streams)
{
if (isMPPTask())
{
if (mpp_exchange_receiver_map_inited)
throw TiFlashException("Repeatedly initialize mpp_exchange_receiver_map", Errors::Coprocessor::Internal);
traverseExecutors(dag_request, [&](const tipb::Executor & executor) {
if (executor.tp() == tipb::ExecType::TypeExchangeReceiver)
{
assert(executor.has_executor_id());
const auto & executor_id = executor.executor_id();
mpp_exchange_receiver_map[executor_id] = std::make_shared<ExchangeReceiver>(
std::make_shared<GRPCReceiverContext>(
executor.exchange_receiver(),
getMPPTaskMeta(),
context.getTMTContext().getKVCluster(),
context.getTMTContext().getMPPTaskManager(),
context.getSettingsRef().enable_local_tunnel),
executor.exchange_receiver().encoded_task_meta_size(),
max_streams,
log);
}
});
mpp_exchange_receiver_map_inited = true;
}
}

const std::unordered_map<String, std::shared_ptr<ExchangeReceiver>> & DAGContext::getMPPExchangeReceiverMap() const
{
if (!isMPPTask())
throw TiFlashException("mpp_exchange_receiver_map is used in mpp only", Errors::Coprocessor::Internal);
if (!mpp_exchange_receiver_map_inited)
throw TiFlashException("mpp_exchange_receiver_map has not been initialized", Errors::Coprocessor::Internal);
return mpp_exchange_receiver_map;
}
>>>>>>> 6ea6c80198 (Fix cast to decimal overflow bug (#3922))
bool DAGContext::shouldClipToZero() { return flags & TiDBSQLFlags::IN_INSERT_STMT || flags & TiDBSQLFlags::IN_LOAD_DATA_STMT; }

} // namespace DB
27 changes: 1 addition & 26 deletions dbms/src/Flash/Coprocessor/DAGContext.h
Original file line number Diff line number Diff line change
Expand Up @@ -22,25 +22,6 @@ struct ProfileStreamsInfo
BlockInputStreams input_streams;
};

<<<<<<< HEAD
=======
class Join;
using JoinPtr = std::shared_ptr<Join>;
struct JoinExecuteInfo
{
String build_side_root_executor_id;
JoinPtr join_ptr;
BlockInputStreams non_joined_streams;
};

using MPPTunnelSetPtr = std::shared_ptr<MPPTunnelSet>;

UInt64 inline getMaxErrorCount(const tipb::DAGRequest &)
{
/// todo max_error_count is a system variable in mysql, TiDB should put it into dag request, now use the default value instead
return 1024;
}

namespace TiDBSQLFlags
{
constexpr UInt64 IGNORE_TRUNCATE = 1;
Expand Down Expand Up @@ -95,7 +76,6 @@ constexpr UInt64 NO_ENGINE_SUBSTITUTION = 1ul << 30ul;
constexpr UInt64 ALLOW_INVALID_DATES = 1ul << 32ul;
} // namespace TiDBSQLMode

>>>>>>> 6ea6c80198 (Fix cast to decimal overflow bug (#3922))
/// A context used to track the information that needs to be passed around during DAG planning.
class DAGContext
{
Expand Down Expand Up @@ -162,9 +142,8 @@ class DAGContext
return 0;
}

<<<<<<< HEAD
BlockInputStreams & getRemoteInputStreams() { return remote_block_input_streams; }
=======

UInt64 getFlags() const
{
return flags;
Expand All @@ -186,10 +165,6 @@ class DAGContext
return (flags & f);
}

void initExchangeReceiverIfMPP(Context & context, size_t max_streams);
const std::unordered_map<String, std::shared_ptr<ExchangeReceiver>> & getMPPExchangeReceiverMap() const;
>>>>>>> 6ea6c80198 (Fix cast to decimal overflow bug (#3922))

size_t final_concurrency;
Int64 compile_time_ns;
String exchange_sender_executor_id = "";
Expand Down
13 changes: 1 addition & 12 deletions dbms/src/Functions/FunctionsTiDBConversion.h
Original file line number Diff line number Diff line change
Expand Up @@ -796,20 +796,10 @@ struct TiDBConvertToDecimal
// And val * scale_mul is the scaled_value, which is stored in ColumnDecimal internally.
static_assert(std::is_integral_v<T>);
using UType = typename U::NativeType;
<<<<<<< HEAD
auto maxValue = DecimalMaxValue::Get(prec);
if (value > maxValue || value < -maxValue)
{
context.getDAGContext()->handleOverflowError("cast to decimal", Errors::Types::Truncated);
if (value > 0)
return static_cast<UType>(maxValue);
else
return static_cast<UType>(-maxValue);
=======
UType scale_mul = getScaleMultiplier<U>(scale);

Int256 scaled_value = static_cast<Int256>(int_value) * static_cast<Int256>(scale_mul);
Int256 scaled_max_value = DecimalMaxValue::get(prec);
Int256 scaled_max_value = DecimalMaxValue::Get(prec);

if (scaled_value > scaled_max_value || scaled_value < -scaled_max_value)
{
Expand All @@ -818,7 +808,6 @@ struct TiDBConvertToDecimal
return static_cast<UType>(scaled_max_value);
else
return static_cast<UType>(-scaled_max_value);
>>>>>>> 6ea6c80198 (Fix cast to decimal overflow bug (#3922))
}

return static_cast<UType>(scaled_value);
Expand Down