fix lint and add intest check

pingcap · Apr 15, 2024 · 6ada327 · 6ada327
1 parent a7b3939
commit 6ada327
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 3 deletions.
diff --git a/pkg/statistics/handle/BUILD.bazel b/pkg/statistics/handle/BUILD.bazel
@@ -33,6 +33,7 @@ go_library(
         "//pkg/statistics/handle/util",
         "//pkg/types",
         "//pkg/util/chunk",
+        "//pkg/util/intest",
         "//pkg/util/logutil",
         "@com_github_pingcap_errors//:errors",
         "@org_uber_go_zap//:zap",

diff --git a/pkg/statistics/handle/bootstrap.go b/pkg/statistics/handle/bootstrap.go
@@ -34,6 +34,7 @@ import (
 	"github.com/pingcap/tidb/pkg/statistics/handle/util"
 	"github.com/pingcap/tidb/pkg/types"
 	"github.com/pingcap/tidb/pkg/util/chunk"
+	"github.com/pingcap/tidb/pkg/util/intest"
 	"github.com/pingcap/tidb/pkg/util/logutil"
 	"go.uber.org/zap"
 )
@@ -522,7 +523,10 @@ func (h *Handle) initStatsFMSketch(cache statstypes.StatsCache) error {
 	return nil
 }
 
-func (h *Handle) initStatsBuckets4Chunk(sctx sessionctx.Context, cache statstypes.StatsCache, iter *chunk.Iterator4Chunk) {
+func (*Handle) initStatsBuckets4Chunk(sctx sessionctx.Context, cache statstypes.StatsCache, iter *chunk.Iterator4Chunk) {
+	// Hist buckets may have invalid datetime.
+	intest.Assert(sctx.GetSessionVars().StmtCtx.TypeFlags().IgnoreInvalidDateErr())
+	intest.Assert(sctx.GetSessionVars().StmtCtx.TypeFlags().IgnoreZeroDateErr())
 	var table *statistics.Table
 	for row := iter.Begin(); row != iter.End(); row = iter.Next() {
 		tableID, isIndex, histID := row.GetInt64(0), row.GetInt64(1), row.GetInt64(2)
@@ -556,8 +560,6 @@ func (h *Handle) initStatsBuckets4Chunk(sctx sessionctx.Context, cache statstype
 			}
 			hist = &column.Histogram
 			d := types.NewBytesDatum(row.GetBytes(5))
-			// Setting TimeZone to time.UTC aligns with HistogramFromStorage and can fix #41938. However, #41985 still exist.
-			// TODO: do the correct time zone conversion for timestamp-type columns' upper/lower bounds.
 			var err error
 			lower, err = d.ConvertTo(sctx.GetSessionVars().StmtCtx.TypeCtx(), &column.Info.FieldType)
 			if err != nil {

diff --git a/pkg/statistics/handle/handle.go b/pkg/statistics/handle/handle.go
@@ -122,6 +122,10 @@ func NewHandle(
 	handle.StatsGC = storage.NewStatsGC(handle)
 	handle.StatsReadWriter = storage.NewStatsReadWriter(handle)
 
+	// Invalid date values may be inserted into table under some relaxed sql mode. Those values may exist in statistics.
+	// Hence, when reading statistics, we should skip invalid date check. See #39336.
+	sc := initStatsCtx.GetSessionVars().StmtCtx
+	sc.SetTypeFlags(sc.TypeFlags().WithIgnoreInvalidDateErr(true).WithIgnoreZeroInDate(true))
 	handle.initStatsCtx = initStatsCtx
 	statsCache, err := cache.NewStatsCacheImpl(handle)
 	if err != nil {

diff --git a/pkg/statistics/handle/storage/read.go b/pkg/statistics/handle/storage/read.go
@@ -32,6 +32,7 @@ import (
 	"github.com/pingcap/tidb/pkg/statistics/handle/util"
 	"github.com/pingcap/tidb/pkg/types"
 	"github.com/pingcap/tidb/pkg/util/chunk"
+	"github.com/pingcap/tidb/pkg/util/intest"
 	"github.com/pingcap/tidb/pkg/util/logutil"
 	"github.com/pingcap/tidb/pkg/util/memory"
 	"github.com/pingcap/tidb/pkg/util/sqlexec"
@@ -82,6 +83,9 @@ func HistMetaFromStorage(sctx sessionctx.Context, item *model.TableItemID, possi
 
 // HistogramFromStorage reads histogram from storage.
 func HistogramFromStorage(sctx sessionctx.Context, tableID int64, colID int64, tp *types.FieldType, distinct int64, isIndex int, ver uint64, nullCount int64, totColSize int64, corr float64) (_ *statistics.Histogram, err error) {
+	// Hist buckets may have invalid datetime.
+	intest.Assert(sctx.GetSessionVars().StmtCtx.TypeFlags().IgnoreInvalidDateErr())
+	intest.Assert(sctx.GetSessionVars().StmtCtx.TypeFlags().IgnoreZeroDateErr())
 	rows, fields, err := util.ExecRows(sctx, "select count, repeats, lower_bound, upper_bound, ndv from mysql.stats_buckets where table_id = %? and is_index = %? and hist_id = %? order by bucket_id", tableID, isIndex, colID)
 	if err != nil {
 		return nil, errors.Trace(err)