staistics: fix load stats from old version json | tidb-test=pr/2114 (#42967)

statistics/handle/BUILD.bazel

@@ -0,0 +1,96 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "handle",
+    srcs = [
+        "bootstrap.go",
+        "ddl.go",
+        "dump.go",
+        "gc.go",
+        "handle.go",
+        "handle_hist.go",
+        "historical_stats_handler.go",
+        "lru_cache.go",
+        "statscache.go",
+        "update.go",
+    ],
+    importpath = "github.com/pingcap/tidb/statistics/handle",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//config",
+        "//ddl/util",
+        "//infoschema",
+        "//kv",
+        "//metrics",
+        "//parser/ast",
+        "//parser/model",
+        "//parser/mysql",
+        "//parser/terror",
+        "//sessionctx",
+        "//sessionctx/stmtctx",
+        "//sessionctx/variable",
+        "//sessiontxn",
+        "//statistics",
+        "//statistics/handle/metrics",
+        "//table",
+        "//types",
+        "//util",
+        "//util/chunk",
+        "//util/codec",
+        "//util/collate",
+        "//util/logutil",
+        "//util/mathutil",
+        "//util/memory",
+        "//util/ranger",
+        "//util/sqlexec",
+        "//util/syncutil",
+        "//util/timeutil",
+        "@com_github_ngaut_pools//:pools",
+        "@com_github_pingcap_errors//:errors",
+        "@com_github_pingcap_failpoint//:failpoint",
+        "@com_github_pingcap_log//:log",
+        "@com_github_pingcap_tipb//go-tipb",
+        "@com_github_tikv_client_go_v2//oracle",
+        "@org_golang_x_exp//slices",
+        "@org_uber_go_atomic//:atomic",
+        "@org_uber_go_zap//:zap",
+    ],
+)
+
+go_test(
+    name = "handle_test",
+    timeout = "short",
+    srcs = [
+        "ddl_test.go",
+        "dump_test.go",
+        "gc_test.go",
+        "handle_hist_test.go",
+        "lru_cache_test.go",
+        "main_test.go",
+        "update_list_test.go",
+    ],
+    embed = [":handle"],
+    flaky = True,
+    race = "on",
+    shard_count = 34,
+    deps = [
+        "//config",
+        "//domain",
+        "//parser/model",
+        "//parser/mysql",
+        "//sessionctx/stmtctx",
+        "//sessionctx/variable",
+        "//statistics",
+        "//statistics/handle/internal",
+        "//testkit",
+        "//testkit/testsetup",
+        "//types",
+        "//util",
+        "//util/mathutil",
+        "//util/mock",
+        "//util/sqlexec",
+        "@com_github_pingcap_failpoint//:failpoint",
+        "@com_github_stretchr_testify//require",
+        "@org_uber_go_goleak//:goleak",
+    ],
+)

statistics/handle/dump.go

@@ -276,11 +276,13 @@ func TableStatsFromJSON(tableInfo *model.TableInfo, physicalID int64, jsonTbl *J
 			hist := statistics.HistogramFromProto(jsonIdx.Histogram)
 			hist.ID, hist.NullCount, hist.LastUpdateVersion, hist.Correlation = idxInfo.ID, jsonIdx.NullCount, jsonIdx.LastUpdateVersion, jsonIdx.Correlation
 			cm, topN := statistics.CMSketchAndTopNFromProto(jsonIdx.CMSketch)
-			// If the statistics is loaded from a JSON without stats version,
-			// we set it to 1.
-			statsVer := int64(statistics.Version1)
+			statsVer := int64(statistics.Version0)
 			if jsonIdx.StatsVer != nil {
 				statsVer = *jsonIdx.StatsVer
+			} else if jsonIdx.Histogram.Ndv > 0 || jsonIdx.NullCount > 0 {
+				// If the statistics are collected without setting stats version(which happens in v4.0 and earlier versions),
+				// we set it to 1.
+				statsVer = int64(statistics.Version1)
 			}
 			idx := &statistics.Index{
 				Histogram: *hist,
@@ -315,11 +317,13 @@ func TableStatsFromJSON(tableInfo *model.TableInfo, physicalID int64, jsonTbl *J
 			cm, topN := statistics.CMSketchAndTopNFromProto(jsonCol.CMSketch)
 			fms := statistics.FMSketchFromProto(jsonCol.FMSketch)
 			hist.ID, hist.NullCount, hist.LastUpdateVersion, hist.TotColSize, hist.Correlation = colInfo.ID, jsonCol.NullCount, jsonCol.LastUpdateVersion, jsonCol.TotColSize, jsonCol.Correlation
-			// If the statistics is loaded from a JSON without stats version,
-			// we set it to 1.
-			statsVer := int64(statistics.Version1)
+			statsVer := int64(statistics.Version0)
 			if jsonCol.StatsVer != nil {
 				statsVer = *jsonCol.StatsVer
+			} else if jsonCol.Histogram.Ndv > 0 || jsonCol.NullCount > 0 {
+				// If the statistics are collected without setting stats version(which happens in v4.0 and earlier versions),
+				// we set it to 1.
+				statsVer = int64(statistics.Version1)
 			}
 			col := &statistics.Column{
 				PhysicalID: physicalID,

statistics/handle/dump_test.go

@@ -479,3 +479,69 @@ func TestJSONTableToBlocks(t *testing.T) {
 	require.NoError(t, err)
 	require.JSONEq(t, string(jsOrigin), string(jsonStr))
 }
+
+func TestLoadStatsFromOldVersion(t *testing.T) {
+	store, dom := testkit.CreateMockStoreAndDomain(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a int, b int, index idx(b))")
+	h := dom.StatsHandle()
+	is := dom.InfoSchema()
+	require.NoError(t, h.HandleDDLEvent(<-h.DDLEventCh()))
+	require.NoError(t, h.Update(is))
+
+	statsJSONFromOldVersion := `{
+ "database_name": "test",
+ "table_name": "t",
+ "columns": {
+  "a": {
+   "histogram": {
+    "ndv": 0
+   },
+   "cm_sketch": null,
+   "null_count": 0,
+   "tot_col_size": 256,
+   "last_update_version": 440735055846047747,
+   "correlation": 0
+  },
+  "b": {
+   "histogram": {
+    "ndv": 0
+   },
+   "cm_sketch": null,
+   "null_count": 0,
+   "tot_col_size": 256,
+   "last_update_version": 440735055846047747,
+   "correlation": 0
+  }
+ },
+ "indices": {
+  "idx": {
+   "histogram": {
+    "ndv": 0
+   },
+   "cm_sketch": null,
+   "null_count": 0,
+   "tot_col_size": 0,
+   "last_update_version": 440735055846047747,
+   "correlation": 0
+  }
+ },
+ "count": 256,
+ "modify_count": 256,
+ "partitions": null
+}`
+	jsonTbl := &handle.JSONTable{}
+	require.NoError(t, json.Unmarshal([]byte(statsJSONFromOldVersion), jsonTbl))
+	require.NoError(t, h.LoadStatsFromJSON(is, jsonTbl))
+	tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+	require.NoError(t, err)
+	statsTbl := h.GetTableStats(tbl.Meta())
+	for _, col := range statsTbl.Columns {
+		require.False(t, col.IsStatsInitialized())
+	}
+	for _, idx := range statsTbl.Indices {
+		require.False(t, idx.IsStatsInitialized())
+	}
+}