diff --git a/DEPS.bzl b/DEPS.bzl
index e2830653aedab..1e8925c7d518c 100644
--- a/DEPS.bzl
+++ b/DEPS.bzl
@@ -5152,13 +5152,13 @@ def go_deps():
name = "com_github_mattn_go_runewidth",
build_file_proto_mode = "disable_global",
importpath = "github.com/mattn/go-runewidth",
- sha256 = "d97c4f0667a14957569c932a8e2488f1c43757b4dcce313897aa001f07d149b0",
- strip_prefix = "github.com/mattn/go-runewidth@v0.0.15",
+ sha256 = "179d2d900c76ee3560fbeda60d0237a3be6acb734d0cb7423b55e5ccb0cedbca",
+ strip_prefix = "github.com/mattn/go-runewidth@v0.0.16",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/mattn/go-runewidth/com_github_mattn_go_runewidth-v0.0.15.zip",
- "http://ats.apps.svc/gomod/github.com/mattn/go-runewidth/com_github_mattn_go_runewidth-v0.0.15.zip",
- "https://cache.hawkingrei.com/gomod/github.com/mattn/go-runewidth/com_github_mattn_go_runewidth-v0.0.15.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/mattn/go-runewidth/com_github_mattn_go_runewidth-v0.0.15.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/mattn/go-runewidth/com_github_mattn_go_runewidth-v0.0.16.zip",
+ "http://ats.apps.svc/gomod/github.com/mattn/go-runewidth/com_github_mattn_go_runewidth-v0.0.16.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/mattn/go-runewidth/com_github_mattn_go_runewidth-v0.0.16.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/mattn/go-runewidth/com_github_mattn_go_runewidth-v0.0.16.zip",
],
)
go_repository(
@@ -5204,13 +5204,13 @@ def go_deps():
name = "com_github_mgechev_revive",
build_file_proto_mode = "disable_global",
importpath = "github.com/mgechev/revive",
- sha256 = "bebbe64597e76c9d7219de964b05d3f0659e4a31344d9dff2b5ec3fad50f7e3a",
- strip_prefix = "github.com/mgechev/revive@v1.3.7",
+ sha256 = "ea4607d0ce69fb36f74a9a8c0e525c15ad85bc88ee65fb16a8ab356e788f9ff5",
+ strip_prefix = "github.com/mgechev/revive@v1.3.10-0.20240809190117-a638ed6e2499",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/mgechev/revive/com_github_mgechev_revive-v1.3.7.zip",
- "http://ats.apps.svc/gomod/github.com/mgechev/revive/com_github_mgechev_revive-v1.3.7.zip",
- "https://cache.hawkingrei.com/gomod/github.com/mgechev/revive/com_github_mgechev_revive-v1.3.7.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/mgechev/revive/com_github_mgechev_revive-v1.3.7.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/mgechev/revive/com_github_mgechev_revive-v1.3.10-0.20240809190117-a638ed6e2499.zip",
+ "http://ats.apps.svc/gomod/github.com/mgechev/revive/com_github_mgechev_revive-v1.3.10-0.20240809190117-a638ed6e2499.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/mgechev/revive/com_github_mgechev_revive-v1.3.10-0.20240809190117-a638ed6e2499.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/mgechev/revive/com_github_mgechev_revive-v1.3.10-0.20240809190117-a638ed6e2499.zip",
],
)
go_repository(
@@ -5867,13 +5867,13 @@ def go_deps():
name = "com_github_petermattis_goid",
build_file_proto_mode = "disable_global",
importpath = "github.com/petermattis/goid",
- sha256 = "c85422e507367742d767fb4102d312f959feec26c11122f91a0e5e73948740f7",
- strip_prefix = "github.com/petermattis/goid@v0.0.0-20231207134359-e60b3f734c67",
+ sha256 = "3f47ab8e5713c36ec5b4295956a5ef012a192bc19198ae1b6591408c061e97ab",
+ strip_prefix = "github.com/petermattis/goid@v0.0.0-20240813172612-4fcff4a6cae7",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/petermattis/goid/com_github_petermattis_goid-v0.0.0-20231207134359-e60b3f734c67.zip",
- "http://ats.apps.svc/gomod/github.com/petermattis/goid/com_github_petermattis_goid-v0.0.0-20231207134359-e60b3f734c67.zip",
- "https://cache.hawkingrei.com/gomod/github.com/petermattis/goid/com_github_petermattis_goid-v0.0.0-20231207134359-e60b3f734c67.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/petermattis/goid/com_github_petermattis_goid-v0.0.0-20231207134359-e60b3f734c67.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/petermattis/goid/com_github_petermattis_goid-v0.0.0-20240813172612-4fcff4a6cae7.zip",
+ "http://ats.apps.svc/gomod/github.com/petermattis/goid/com_github_petermattis_goid-v0.0.0-20240813172612-4fcff4a6cae7.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/petermattis/goid/com_github_petermattis_goid-v0.0.0-20240813172612-4fcff4a6cae7.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/petermattis/goid/com_github_petermattis_goid-v0.0.0-20240813172612-4fcff4a6cae7.zip",
],
)
go_repository(
@@ -6010,13 +6010,13 @@ def go_deps():
name = "com_github_pingcap_kvproto",
build_file_proto_mode = "disable_global",
importpath = "github.com/pingcap/kvproto",
- sha256 = "9c11f4c5354534ac08526d61d180bf873786fef9089b1f018fc7365350839207",
- strip_prefix = "github.com/pingcap/kvproto@v0.0.0-20240620063548-118a4cab53e4",
+ sha256 = "b1af34db24f2650cd2a687fa9c58bd746eb3ca76e08afe5dbe2ac569ce54b597",
+ strip_prefix = "github.com/pingcap/kvproto@v0.0.0-20240716095229-5f7ffec83ea7",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240620063548-118a4cab53e4.zip",
- "http://ats.apps.svc/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240620063548-118a4cab53e4.zip",
- "https://cache.hawkingrei.com/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240620063548-118a4cab53e4.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240620063548-118a4cab53e4.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240716095229-5f7ffec83ea7.zip",
+ "http://ats.apps.svc/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240716095229-5f7ffec83ea7.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240716095229-5f7ffec83ea7.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240716095229-5f7ffec83ea7.zip",
],
)
go_repository(
@@ -6049,13 +6049,13 @@ def go_deps():
name = "com_github_pingcap_tipb",
build_file_proto_mode = "disable_global",
importpath = "github.com/pingcap/tipb",
- sha256 = "42c365f3f99d2577fe29c89d433894b2d0d69b054248bb3bafbced33332933e3",
- strip_prefix = "github.com/pingcap/tipb@v0.0.0-20240318032315-55a7867ddd50",
+ sha256 = "6e910c9689f1a81bad2ae55be1746d456c317d696ff2687390d3fb30f7d05c6d",
+ strip_prefix = "github.com/pingcap/tipb@v0.0.0-20240703084358-e46e4632bd2b",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20240318032315-55a7867ddd50.zip",
- "http://ats.apps.svc/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20240318032315-55a7867ddd50.zip",
- "https://cache.hawkingrei.com/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20240318032315-55a7867ddd50.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20240318032315-55a7867ddd50.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20240703084358-e46e4632bd2b.zip",
+ "http://ats.apps.svc/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20240703084358-e46e4632bd2b.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20240703084358-e46e4632bd2b.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20240703084358-e46e4632bd2b.zip",
],
)
go_repository(
@@ -6413,13 +6413,13 @@ def go_deps():
name = "com_github_rivo_uniseg",
build_file_proto_mode = "disable_global",
importpath = "github.com/rivo/uniseg",
- sha256 = "eca600065be5a1ead37478e645ad07d70dadaf4f06f681827d81158316538b23",
- strip_prefix = "github.com/rivo/uniseg@v0.4.6",
+ sha256 = "b995e4aa0cc1e5779cc61138ac925cb8c1e963e40c80e4b93ee8553812ebb792",
+ strip_prefix = "github.com/rivo/uniseg@v0.4.7",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/rivo/uniseg/com_github_rivo_uniseg-v0.4.6.zip",
- "http://ats.apps.svc/gomod/github.com/rivo/uniseg/com_github_rivo_uniseg-v0.4.6.zip",
- "https://cache.hawkingrei.com/gomod/github.com/rivo/uniseg/com_github_rivo_uniseg-v0.4.6.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/rivo/uniseg/com_github_rivo_uniseg-v0.4.6.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/rivo/uniseg/com_github_rivo_uniseg-v0.4.7.zip",
+ "http://ats.apps.svc/gomod/github.com/rivo/uniseg/com_github_rivo_uniseg-v0.4.7.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/rivo/uniseg/com_github_rivo_uniseg-v0.4.7.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/rivo/uniseg/com_github_rivo_uniseg-v0.4.7.zip",
],
)
go_repository(
@@ -6569,13 +6569,13 @@ def go_deps():
name = "com_github_sasha_s_go_deadlock",
build_file_proto_mode = "disable_global",
importpath = "github.com/sasha-s/go-deadlock",
- sha256 = "82eaa020f254a21d5025b6cae9a908315ffa382f941ef228431c10177b9657d4",
- strip_prefix = "github.com/sasha-s/go-deadlock@v0.3.1",
+ sha256 = "b927f67dd9a6dc183bac7249c019775e689aee67dc52bfa53354137139d722a1",
+ strip_prefix = "github.com/sasha-s/go-deadlock@v0.3.5",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/sasha-s/go-deadlock/com_github_sasha_s_go_deadlock-v0.3.1.zip",
- "http://ats.apps.svc/gomod/github.com/sasha-s/go-deadlock/com_github_sasha_s_go_deadlock-v0.3.1.zip",
- "https://cache.hawkingrei.com/gomod/github.com/sasha-s/go-deadlock/com_github_sasha_s_go_deadlock-v0.3.1.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/sasha-s/go-deadlock/com_github_sasha_s_go_deadlock-v0.3.1.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/sasha-s/go-deadlock/com_github_sasha_s_go_deadlock-v0.3.5.zip",
+ "http://ats.apps.svc/gomod/github.com/sasha-s/go-deadlock/com_github_sasha_s_go_deadlock-v0.3.5.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/sasha-s/go-deadlock/com_github_sasha_s_go_deadlock-v0.3.5.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/sasha-s/go-deadlock/com_github_sasha_s_go_deadlock-v0.3.5.zip",
],
)
go_repository(
@@ -7206,26 +7206,26 @@ def go_deps():
name = "com_github_tikv_client_go_v2",
build_file_proto_mode = "disable_global",
importpath = "github.com/tikv/client-go/v2",
- sha256 = "0093081c01fd5119490fb4145f770eb8a90da6c4e0e02708dae7b1fe24668cb2",
- strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20240703095801-d73cc1ed6503",
+ sha256 = "cbaceaa4bbd945e24be9ccbb2af622baeb7542facedb7538f483d116d82f72ff",
+ strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20240815020919-c810ed88fb02",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240703095801-d73cc1ed6503.zip",
- "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240703095801-d73cc1ed6503.zip",
- "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240703095801-d73cc1ed6503.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240703095801-d73cc1ed6503.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240815020919-c810ed88fb02.zip",
+ "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240815020919-c810ed88fb02.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240815020919-c810ed88fb02.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240815020919-c810ed88fb02.zip",
],
)
go_repository(
name = "com_github_tikv_pd_client",
build_file_proto_mode = "disable_global",
importpath = "github.com/tikv/pd/client",
- sha256 = "166b1edc9991d64fb62b480c1d29b8430bfbd590c20c2290f0d3b68bce3059c0",
- strip_prefix = "github.com/tikv/pd/client@v0.0.0-20240717053728-5ec6af403019",
+ sha256 = "bb4aa99260c2d1b22054d539cb37c18ce276c96b1b7c4a8f14ac4cbcda829654",
+ strip_prefix = "github.com/tikv/pd/client@v0.0.0-20240805092608-838ee7983b78",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240717053728-5ec6af403019.zip",
- "http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240717053728-5ec6af403019.zip",
- "https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240717053728-5ec6af403019.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240717053728-5ec6af403019.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240805092608-838ee7983b78.zip",
+ "http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240805092608-838ee7983b78.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240805092608-838ee7983b78.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240805092608-838ee7983b78.zip",
],
)
go_repository(
@@ -10300,13 +10300,13 @@ def go_deps():
name = "org_golang_x_crypto",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/crypto",
- sha256 = "5cd21f5fcb21845389eed13ad39186f57c86db70f3cc6cf8a4904633f07d1bdb",
- strip_prefix = "golang.org/x/crypto@v0.25.0",
+ sha256 = "ec96acfe28be3ff2fb14201c5f51132f0e24c7d0d6f3201a8aa69c84f989d014",
+ strip_prefix = "golang.org/x/crypto@v0.26.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.25.0.zip",
- "http://ats.apps.svc/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.25.0.zip",
- "https://cache.hawkingrei.com/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.25.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.25.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.26.0.zip",
+ "http://ats.apps.svc/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.26.0.zip",
+ "https://cache.hawkingrei.com/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.26.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.26.0.zip",
],
)
go_repository(
@@ -10378,26 +10378,26 @@ def go_deps():
name = "org_golang_x_mod",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/mod",
- sha256 = "9c64a3efda43c92014675361b2620de1f2815d59875a379f0b3361018e5bdf59",
- strip_prefix = "golang.org/x/mod@v0.18.0",
+ sha256 = "3c3528c39639b7cd699c121c100ddb71ab49f94bff257a4a3935e3ae9e8571fc",
+ strip_prefix = "golang.org/x/mod@v0.20.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/mod/org_golang_x_mod-v0.18.0.zip",
- "http://ats.apps.svc/gomod/golang.org/x/mod/org_golang_x_mod-v0.18.0.zip",
- "https://cache.hawkingrei.com/gomod/golang.org/x/mod/org_golang_x_mod-v0.18.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/mod/org_golang_x_mod-v0.18.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/mod/org_golang_x_mod-v0.20.0.zip",
+ "http://ats.apps.svc/gomod/golang.org/x/mod/org_golang_x_mod-v0.20.0.zip",
+ "https://cache.hawkingrei.com/gomod/golang.org/x/mod/org_golang_x_mod-v0.20.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/mod/org_golang_x_mod-v0.20.0.zip",
],
)
go_repository(
name = "org_golang_x_net",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/net",
- sha256 = "0adf4068f2b9d2b2852288b77caf1f76dac61f960e691e7544191de9bfa36406",
- strip_prefix = "golang.org/x/net@v0.27.0",
+ sha256 = "c6f7bde4bb418d1f5ee5dc437d09ce9f10743ddba043cdca82eb57ddeb18d6da",
+ strip_prefix = "golang.org/x/net@v0.28.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/net/org_golang_x_net-v0.27.0.zip",
- "http://ats.apps.svc/gomod/golang.org/x/net/org_golang_x_net-v0.27.0.zip",
- "https://cache.hawkingrei.com/gomod/golang.org/x/net/org_golang_x_net-v0.27.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/net/org_golang_x_net-v0.27.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/net/org_golang_x_net-v0.28.0.zip",
+ "http://ats.apps.svc/gomod/golang.org/x/net/org_golang_x_net-v0.28.0.zip",
+ "https://cache.hawkingrei.com/gomod/golang.org/x/net/org_golang_x_net-v0.28.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/net/org_golang_x_net-v0.28.0.zip",
],
)
go_repository(
@@ -10430,26 +10430,26 @@ def go_deps():
name = "org_golang_x_sync",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/sync",
- sha256 = "20b01085240e661bffc7f59383f21b90f112d669784220c6e59c801243216d22",
- strip_prefix = "golang.org/x/sync@v0.7.0",
+ sha256 = "c79473c265ca571d389bf64fa1e7b2d8999b4ab3eb7af5e3bc185644783a1087",
+ strip_prefix = "golang.org/x/sync@v0.8.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/sync/org_golang_x_sync-v0.7.0.zip",
- "http://ats.apps.svc/gomod/golang.org/x/sync/org_golang_x_sync-v0.7.0.zip",
- "https://cache.hawkingrei.com/gomod/golang.org/x/sync/org_golang_x_sync-v0.7.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/sync/org_golang_x_sync-v0.7.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/sync/org_golang_x_sync-v0.8.0.zip",
+ "http://ats.apps.svc/gomod/golang.org/x/sync/org_golang_x_sync-v0.8.0.zip",
+ "https://cache.hawkingrei.com/gomod/golang.org/x/sync/org_golang_x_sync-v0.8.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/sync/org_golang_x_sync-v0.8.0.zip",
],
)
go_repository(
name = "org_golang_x_sys",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/sys",
- sha256 = "2434299f530b049a5c8121d6465751ce58bd62f939afde34c442f79c88e9033c",
- strip_prefix = "golang.org/x/sys@v0.22.0",
+ sha256 = "fbc83ddb0aee236ee85c0f8294e7c9b6f79d28794d22e7d0957fcd64ca252960",
+ strip_prefix = "golang.org/x/sys@v0.24.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/sys/org_golang_x_sys-v0.22.0.zip",
- "http://ats.apps.svc/gomod/golang.org/x/sys/org_golang_x_sys-v0.22.0.zip",
- "https://cache.hawkingrei.com/gomod/golang.org/x/sys/org_golang_x_sys-v0.22.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/sys/org_golang_x_sys-v0.22.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/sys/org_golang_x_sys-v0.24.0.zip",
+ "http://ats.apps.svc/gomod/golang.org/x/sys/org_golang_x_sys-v0.24.0.zip",
+ "https://cache.hawkingrei.com/gomod/golang.org/x/sys/org_golang_x_sys-v0.24.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/sys/org_golang_x_sys-v0.24.0.zip",
],
)
go_repository(
@@ -10469,26 +10469,26 @@ def go_deps():
name = "org_golang_x_term",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/term",
- sha256 = "0766e30db0cea597a7c87ad6a829df2a538939dc65b5ca65791dfd6e4f1244b4",
- strip_prefix = "golang.org/x/term@v0.22.0",
+ sha256 = "2597a62b487b952c11c89b2001551af1fe1d29c484388ec1c3f5e3be7ff58ba5",
+ strip_prefix = "golang.org/x/term@v0.23.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/term/org_golang_x_term-v0.22.0.zip",
- "http://ats.apps.svc/gomod/golang.org/x/term/org_golang_x_term-v0.22.0.zip",
- "https://cache.hawkingrei.com/gomod/golang.org/x/term/org_golang_x_term-v0.22.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/term/org_golang_x_term-v0.22.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/term/org_golang_x_term-v0.23.0.zip",
+ "http://ats.apps.svc/gomod/golang.org/x/term/org_golang_x_term-v0.23.0.zip",
+ "https://cache.hawkingrei.com/gomod/golang.org/x/term/org_golang_x_term-v0.23.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/term/org_golang_x_term-v0.23.0.zip",
],
)
go_repository(
name = "org_golang_x_text",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/text",
- sha256 = "9b7c0575c894224bc7f85dfa2efb0ef93d7d54ae962cd95c8de90cecb407de94",
- strip_prefix = "golang.org/x/text@v0.16.0",
+ sha256 = "48464f2ab2f988ca8b7b0a9d098e3664224c3b128629b5a9cc08025ee4a7e4ec",
+ strip_prefix = "golang.org/x/text@v0.17.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/text/org_golang_x_text-v0.16.0.zip",
- "http://ats.apps.svc/gomod/golang.org/x/text/org_golang_x_text-v0.16.0.zip",
- "https://cache.hawkingrei.com/gomod/golang.org/x/text/org_golang_x_text-v0.16.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/text/org_golang_x_text-v0.16.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/text/org_golang_x_text-v0.17.0.zip",
+ "http://ats.apps.svc/gomod/golang.org/x/text/org_golang_x_text-v0.17.0.zip",
+ "https://cache.hawkingrei.com/gomod/golang.org/x/text/org_golang_x_text-v0.17.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/text/org_golang_x_text-v0.17.0.zip",
],
)
go_repository(
@@ -10508,13 +10508,13 @@ def go_deps():
name = "org_golang_x_tools",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/tools",
- sha256 = "6c12cd419d997290febb441698d0e52cab5a71be959ac7c4dd023f86b2d01d1e",
- strip_prefix = "golang.org/x/tools@v0.22.0",
+ sha256 = "92607be1cacf4647fd31b19ee64b1a7c198178f1005c75371e38e7b08fb138e7",
+ strip_prefix = "golang.org/x/tools@v0.24.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/tools/org_golang_x_tools-v0.22.0.zip",
- "http://ats.apps.svc/gomod/golang.org/x/tools/org_golang_x_tools-v0.22.0.zip",
- "https://cache.hawkingrei.com/gomod/golang.org/x/tools/org_golang_x_tools-v0.22.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/tools/org_golang_x_tools-v0.22.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/tools/org_golang_x_tools-v0.24.0.zip",
+ "http://ats.apps.svc/gomod/golang.org/x/tools/org_golang_x_tools-v0.24.0.zip",
+ "https://cache.hawkingrei.com/gomod/golang.org/x/tools/org_golang_x_tools-v0.24.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/tools/org_golang_x_tools-v0.24.0.zip",
],
)
go_repository(
diff --git a/Makefile b/Makefile
index 511c605092d43..03446f5104da1 100644
--- a/Makefile
+++ b/Makefile
@@ -627,7 +627,7 @@ bazel_coverage_test: failpoint-enable bazel_ci_simple_prepare
bazel_build:
mkdir -p bin
bazel $(BAZEL_GLOBAL_CONFIG) build $(BAZEL_CMD_CONFIG) \
- //... --//build:with_nogo_flag=$(NOGO_FLAG)
+ //... --//build:with_nogo_flag=$(NOGO_FLAG) --subcommands
bazel $(BAZEL_GLOBAL_CONFIG) build $(BAZEL_CMD_CONFIG) \
//cmd/importer:importer //cmd/tidb-server:tidb-server //cmd/tidb-server:tidb-server-check --//build:with_nogo_flag=$(NOGO_FLAG)
cp bazel-out/k8-fastbuild/bin/cmd/tidb-server/tidb-server_/tidb-server ./bin
diff --git a/OWNERS b/OWNERS
index 609be7f7022b3..724a54404e7a0 100644
--- a/OWNERS
+++ b/OWNERS
@@ -43,7 +43,6 @@ approvers:
- guo-shaoge
- hanfei1991
- hawkingrei
- - hi-rustin
- hicqu
- holys
- hongyunyan
@@ -80,6 +79,7 @@ approvers:
- qw4990
- rebelice
- Reminiscent
+ - Rustin170506
- sdojjy
- shenli
- siddontang
diff --git a/WORKSPACE b/WORKSPACE
index f237626eda481..f5d1790d7c8c6 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -12,9 +12,9 @@ http_archive(
http_archive(
name = "bazel_features",
- sha256 = "d7787da289a7fb497352211ad200ec9f698822a9e0757a4976fd9f713ff372b3",
- strip_prefix = "bazel_features-1.9.1",
- url = "https://github.com/bazel-contrib/bazel_features/releases/download/v1.9.1/bazel_features-v1.9.1.tar.gz",
+ sha256 = "ba1282c1aa1d1fffdcf994ab32131d7c7551a9bc960fbf05f42d55a1b930cbfb",
+ strip_prefix = "bazel_features-1.15.0",
+ url = "https://github.com/bazel-contrib/bazel_features/releases/download/v1.15.0/bazel_features-v1.15.0.tar.gz",
)
load("@bazel_features//:deps.bzl", "bazel_features_deps")
@@ -48,12 +48,11 @@ http_archive(
http_archive(
name = "bazel_gazelle",
- sha256 = "d76bf7a60fd8b050444090dfa2837a4eaf9829e1165618ee35dceca5cbdf58d5",
+ sha256 = "8ad77552825b078a10ad960bec6ef77d2ff8ec70faef2fd038db713f410f5d87",
urls = [
- "http://bazel-cache.pingcap.net:8080/bazelbuild/bazel-gazelle/releases/download/v0.37.0/bazel-gazelle-v0.37.0.tar.gz",
- "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.37.0/bazel-gazelle-v0.37.0.tar.gz",
- "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.37.0/bazel-gazelle-v0.37.0.tar.gz",
- "http://ats.apps.svc/bazelbuild/bazel-gazelle/releases/download/v0.37.0/bazel-gazelle-v0.37.0.tar.gz",
+ "http://bazel-cache.pingcap.net:8080/bazelbuild/bazel-gazelle/releases/download/v0.38.0/bazel-gazelle-v0.38.0.tar.gz",
+ "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.38.0/bazel-gazelle-v0.38.0.tar.gz",
+ "http://ats.apps.svc/bazelbuild/bazel-gazelle/releases/download/v0.38.0/bazel-gazelle-v0.38.0.tar.gz",
],
)
@@ -86,7 +85,7 @@ go_download_sdk(
"https://mirrors.aliyun.com/golang/{}",
"https://dl.google.com/go/{}",
],
- version = "1.21.12",
+ version = "1.21.13",
)
go_register_toolchains(
diff --git a/br/cmd/br/BUILD.bazel b/br/cmd/br/BUILD.bazel
index c6b4e1d646e11..b82ecbd2bc8dc 100644
--- a/br/cmd/br/BUILD.bazel
+++ b/br/cmd/br/BUILD.bazel
@@ -37,9 +37,11 @@ go_library(
"//pkg/util",
"//pkg/util/gctuner",
"//pkg/util/logutil",
+ "//pkg/util/mathutil",
"//pkg/util/memory",
"//pkg/util/metricsutil",
"//pkg/util/redact",
+ "//pkg/util/size",
"@com_github_gogo_protobuf//proto",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_kvproto//pkg/brpb",
@@ -63,5 +65,8 @@ go_test(
srcs = ["main_test.go"],
embed = [":br_lib"],
flaky = True,
- deps = ["@org_uber_go_goleak//:goleak"],
+ deps = [
+ "@com_github_stretchr_testify//require",
+ "@org_uber_go_goleak//:goleak",
+ ],
)
diff --git a/br/cmd/br/cmd.go b/br/cmd/br/cmd.go
index df0395fa1d719..02939c78b4676 100644
--- a/br/cmd/br/cmd.go
+++ b/br/cmd/br/cmd.go
@@ -5,8 +5,10 @@ package main
import (
"context"
"fmt"
+ "math"
"os"
"path/filepath"
+ "runtime/debug"
"sync"
"sync/atomic"
"time"
@@ -21,9 +23,12 @@ import (
"github.com/pingcap/tidb/pkg/config"
tidbutils "github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/logutil"
+ "github.com/pingcap/tidb/pkg/util/mathutil"
"github.com/pingcap/tidb/pkg/util/memory"
"github.com/pingcap/tidb/pkg/util/redact"
+ "github.com/pingcap/tidb/pkg/util/size"
"github.com/spf13/cobra"
+ "go.uber.org/zap"
)
var (
@@ -107,6 +112,25 @@ func AddFlags(cmd *cobra.Command) {
_ = cmd.PersistentFlags().MarkHidden(FlagRedactLog)
}
+const quarterGiB uint64 = 256 * size.MB
+const halfGiB uint64 = 512 * size.MB
+const fourGiB uint64 = 4 * size.GB
+
+func calculateMemoryLimit(memleft uint64) uint64 {
+ // memreserved = f(memleft) = 512MB * memleft / (memleft + 4GB)
+ // * f(0) = 0
+ // * f(4GB) = 256MB
+ // * f(+inf) -> 512MB
+ memreserved := halfGiB / (1 + fourGiB/(memleft|1))
+ // 0 memused memtotal-memreserved memtotal
+ // +--------+--------------------+----------------+
+ // ^ br mem upper limit
+ // +--------------------^
+ // GOMEMLIMIT range
+ memlimit := memleft - memreserved
+ return memlimit
+}
+
// Init initializes BR cli.
func Init(cmd *cobra.Command) (err error) {
initOnce.Do(func() {
@@ -162,6 +186,34 @@ func Init(cmd *cobra.Command) (err error) {
}
log.ReplaceGlobals(lg, p)
memory.InitMemoryHook()
+ if debug.SetMemoryLimit(-1) == math.MaxInt64 {
+ memtotal, e := memory.MemTotal()
+ if e != nil {
+ err = e
+ return
+ }
+ memused, e := memory.MemUsed()
+ if e != nil {
+ err = e
+ return
+ }
+ if memused >= memtotal {
+ log.Warn("failed to obtain memory size, skip setting memory limit",
+ zap.Uint64("memused", memused), zap.Uint64("memtotal", memtotal))
+ } else {
+ memleft := memtotal - memused
+ memlimit := calculateMemoryLimit(memleft)
+ // BR command needs 256 MiB at least, if the left memory is less than 256 MiB,
+ // the memory limit cannot limit anyway and then finally OOM.
+ memlimit = mathutil.Max(memlimit, quarterGiB)
+ log.Info("calculate the rest memory",
+ zap.Uint64("memtotal", memtotal), zap.Uint64("memused", memused), zap.Uint64("memlimit", memlimit))
+ // No need to set memory limit because the left memory is sufficient.
+ if memlimit < uint64(math.MaxInt64) {
+ debug.SetMemoryLimit(int64(memlimit))
+ }
+ }
+ }
redactLog, e := cmd.Flags().GetBool(FlagRedactLog)
if e != nil {
diff --git a/br/cmd/br/main_test.go b/br/cmd/br/main_test.go
index 5fa5b1439a2ec..bdd6f7282ee62 100644
--- a/br/cmd/br/main_test.go
+++ b/br/cmd/br/main_test.go
@@ -21,6 +21,7 @@ import (
"strings"
"testing"
+ "github.com/stretchr/testify/require"
"go.uber.org/goleak"
)
@@ -75,3 +76,18 @@ func TestRunMain(*testing.T) {
<-waitCh
}
+
+func TestCalculateMemoryLimit(t *testing.T) {
+ // f(0 Byte) = 0 Byte
+ require.Equal(t, uint64(0), calculateMemoryLimit(0))
+ // f(100 KB) = 87.5 KB
+ require.Equal(t, uint64(89600), calculateMemoryLimit(100*1024))
+ // f(100 MB) = 87.5 MB
+ require.Equal(t, uint64(91763188), calculateMemoryLimit(100*1024*1024))
+ // f(3.99 GB) = 3.74 GB
+ require.Equal(t, uint64(4026531839), calculateMemoryLimit(4*1024*1024*1024-1))
+ // f(4 GB) = 3.5 GB
+ require.Equal(t, uint64(3758096384), calculateMemoryLimit(4*1024*1024*1024))
+ // f(32 GB) = 31.5 GB
+ require.Equal(t, uint64(33822867456), calculateMemoryLimit(32*1024*1024*1024))
+}
diff --git a/br/pkg/backup/BUILD.bazel b/br/pkg/backup/BUILD.bazel
index 29ee9a06a654a..7b49957ca7c6b 100644
--- a/br/pkg/backup/BUILD.bazel
+++ b/br/pkg/backup/BUILD.bazel
@@ -63,11 +63,12 @@ go_test(
"client_test.go",
"main_test.go",
"schema_test.go",
+ "store_test.go",
],
embed = [":backup"],
flaky = True,
race = "on",
- shard_count = 12,
+ shard_count = 14,
deps = [
"//br/pkg/conn",
"//br/pkg/gluetidb/mock",
@@ -92,6 +93,7 @@ go_test(
"@com_github_tikv_client_go_v2//testutils",
"@com_github_tikv_pd_client//:client",
"@io_opencensus_go//stats/view",
+ "@org_golang_google_grpc//:grpc",
"@org_uber_go_goleak//:goleak",
],
)
diff --git a/br/pkg/backup/client.go b/br/pkg/backup/client.go
index 2040edaf7ab25..636afdaf24700 100644
--- a/br/pkg/backup/client.go
+++ b/br/pkg/backup/client.go
@@ -1233,9 +1233,9 @@ func (bc *Client) OnBackupResponse(
return txnlock.NewLock(lockErr), nil
}
}
- res := errContext.HandleIgnorableError(errPb, storeID)
+ res := utils.HandleBackupError(errPb, storeID, errContext)
switch res.Strategy {
- case utils.GiveUpStrategy:
+ case utils.StrategyGiveUp:
errMsg := res.Reason
if len(errMsg) <= 0 {
errMsg = errPb.Msg
diff --git a/br/pkg/backup/store.go b/br/pkg/backup/store.go
index 02f7166193918..0015efa0681d0 100644
--- a/br/pkg/backup/store.go
+++ b/br/pkg/backup/store.go
@@ -6,6 +6,7 @@ import (
"context"
"io"
"os"
+ "sync"
"time"
"github.com/pingcap/errors"
@@ -57,12 +58,73 @@ func (r ResponseAndStore) GetStoreID() uint64 {
return r.StoreID
}
+// timeoutRecv cancel the context if `Refresh()` is not called within the specified time `timeout`.
+type timeoutRecv struct {
+ wg sync.WaitGroup
+ parentCtx context.Context
+ cancel context.CancelCauseFunc
+
+ refresh chan struct{}
+}
+
+// Refresh the timeout ticker
+func (trecv *timeoutRecv) Refresh() {
+ select {
+ case <-trecv.parentCtx.Done():
+ case trecv.refresh <- struct{}{}:
+ }
+}
+
+// Stop the timeout ticker
+func (trecv *timeoutRecv) Stop() {
+ close(trecv.refresh)
+ trecv.wg.Wait()
+}
+
+var TimeoutOneResponse = time.Hour
+
+func (trecv *timeoutRecv) loop(timeout time.Duration) {
+ defer trecv.wg.Done()
+ ticker := time.NewTicker(timeout)
+ defer ticker.Stop()
+ for {
+ ticker.Reset(timeout)
+ select {
+ case <-trecv.parentCtx.Done():
+ return
+ case _, ok := <-trecv.refresh:
+ if !ok {
+ return
+ }
+ case <-ticker.C:
+ log.Warn("receive a backup response timeout")
+ trecv.cancel(errors.Errorf("receive a backup response timeout"))
+ }
+ }
+}
+
+func StartTimeoutRecv(ctx context.Context, timeout time.Duration) (context.Context, *timeoutRecv) {
+ cctx, cancel := context.WithCancelCause(ctx)
+ trecv := &timeoutRecv{
+ parentCtx: ctx,
+ cancel: cancel,
+ refresh: make(chan struct{}),
+ }
+ trecv.wg.Add(1)
+ go trecv.loop(timeout)
+ return cctx, trecv
+}
+
func doSendBackup(
- ctx context.Context,
+ pctx context.Context,
client backuppb.BackupClient,
req backuppb.BackupRequest,
respFn func(*backuppb.BackupResponse) error,
) error {
+ // Backup might be stuck on GRPC `waitonHeader`, so start a timeout ticker to
+ // terminate the backup if it does not receive any new response for a long time.
+ ctx, timerecv := StartTimeoutRecv(pctx, TimeoutOneResponse)
+ defer timerecv.Stop()
failpoint.Inject("hint-backup-start", func(v failpoint.Value) {
logutil.CL(ctx).Info("failpoint hint-backup-start injected, " +
"process will notify the shell.")
@@ -107,6 +169,7 @@ func doSendBackup(
for {
resp, err := bCli.Recv()
+ timerecv.Refresh()
if err != nil {
if errors.Cause(err) == io.EOF { // nolint:errorlint
logutil.CL(ctx).Debug("backup streaming finish",
diff --git a/br/pkg/backup/store_test.go b/br/pkg/backup/store_test.go
new file mode 100644
index 0000000000000..9ad928f01a635
--- /dev/null
+++ b/br/pkg/backup/store_test.go
@@ -0,0 +1,98 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package backup
+
+import (
+ "context"
+ "io"
+ "testing"
+ "time"
+
+ backuppb "github.com/pingcap/kvproto/pkg/brpb"
+ "github.com/stretchr/testify/require"
+ "google.golang.org/grpc"
+)
+
+type MockBackupClient struct {
+ backuppb.BackupClient
+
+ recvFunc func(context.Context) (*backuppb.BackupResponse, error)
+}
+
+func (mbc *MockBackupClient) Backup(ctx context.Context, _ *backuppb.BackupRequest, _ ...grpc.CallOption) (backuppb.Backup_BackupClient, error) {
+ return &MockBackupBackupClient{ctx: ctx, recvFunc: mbc.recvFunc}, nil
+}
+
+type MockBackupBackupClient struct {
+ backuppb.Backup_BackupClient
+
+ ctx context.Context
+ recvFunc func(context.Context) (*backuppb.BackupResponse, error)
+}
+
+func (mbbc *MockBackupBackupClient) CloseSend() error {
+ return nil
+}
+
+func (mbbc *MockBackupBackupClient) Recv() (*backuppb.BackupResponse, error) {
+ if mbbc.recvFunc != nil {
+ return mbbc.recvFunc(mbbc.ctx)
+ }
+ return &backuppb.BackupResponse{}, nil
+}
+
+func TestTimeoutRecv(t *testing.T) {
+ ctx := context.Background()
+ TimeoutOneResponse = time.Millisecond * 800
+ // Just Timeout Once
+ {
+ err := doSendBackup(ctx, &MockBackupClient{
+ recvFunc: func(ctx context.Context) (*backuppb.BackupResponse, error) {
+ time.Sleep(time.Second)
+ require.Error(t, ctx.Err())
+ return nil, io.EOF
+ },
+ }, backuppb.BackupRequest{}, func(br *backuppb.BackupResponse) error { return nil })
+ require.NoError(t, err)
+ }
+
+ // Timeout Not At First
+ {
+ count := 0
+ err := doSendBackup(ctx, &MockBackupClient{
+ recvFunc: func(ctx context.Context) (*backuppb.BackupResponse, error) {
+ require.NoError(t, ctx.Err())
+ if count == 15 {
+ time.Sleep(time.Second)
+ require.Error(t, ctx.Err())
+ return nil, io.EOF
+ }
+ count += 1
+ time.Sleep(time.Millisecond * 80)
+ return &backuppb.BackupResponse{}, nil
+ },
+ }, backuppb.BackupRequest{}, func(br *backuppb.BackupResponse) error { return nil })
+ require.NoError(t, err)
+ }
+}
+
+func TestTimeoutRecvCancel(t *testing.T) {
+ ctx := context.Background()
+ cctx, cancel := context.WithCancel(ctx)
+
+ _, trecv := StartTimeoutRecv(cctx, time.Hour)
+ cancel()
+ trecv.wg.Wait()
+}
diff --git a/br/pkg/checkpoint/checkpoint.go b/br/pkg/checkpoint/checkpoint.go
index 1d9f309843b88..4b397a60e5eeb 100644
--- a/br/pkg/checkpoint/checkpoint.go
+++ b/br/pkg/checkpoint/checkpoint.go
@@ -43,7 +43,7 @@ import (
"golang.org/x/sync/errgroup"
)
-const CheckpointDir = "/checkpoints"
+const CheckpointDir = "checkpoints"
type flushPosition struct {
CheckpointDataDir string
diff --git a/br/pkg/errors/errors.go b/br/pkg/errors/errors.go
index 1b78a861fa668..3bd2ab776ccb3 100644
--- a/br/pkg/errors/errors.go
+++ b/br/pkg/errors/errors.go
@@ -70,6 +70,7 @@ var (
ErrRestoreIncompatibleSys = errors.Normalize("incompatible system table", errors.RFCCodeText("BR:Restore:ErrRestoreIncompatibleSys"))
ErrUnsupportedSystemTable = errors.Normalize("the system table isn't supported for restoring yet", errors.RFCCodeText("BR:Restore:ErrUnsupportedSysTable"))
ErrDatabasesAlreadyExisted = errors.Normalize("databases already existed in restored cluster", errors.RFCCodeText("BR:Restore:ErrDatabasesAlreadyExisted"))
+ ErrTablesAlreadyExisted = errors.Normalize("tables already existed in restored cluster", errors.RFCCodeText("BR:Restore:ErrTablesAlreadyExisted"))
// ErrStreamLogTaskExist is the error when stream log task already exists, because of supporting single task currently.
ErrStreamLogTaskExist = errors.Normalize("stream task already exists", errors.RFCCodeText("BR:Stream:ErrStreamLogTaskExist"))
diff --git a/br/pkg/glue/glue.go b/br/pkg/glue/glue.go
index 5d5f611fa39e6..1895ee092cc78 100644
--- a/br/pkg/glue/glue.go
+++ b/br/pkg/glue/glue.go
@@ -13,6 +13,13 @@ import (
pd "github.com/tikv/pd/client"
)
+type GlueClient int
+
+const (
+ ClientCLP GlueClient = iota
+ ClientSql
+)
+
// Glue is an abstraction of TiDB function calls used in BR.
type Glue interface {
GetDomain(store kv.Storage) (*domain.Domain, error)
@@ -36,6 +43,9 @@ type Glue interface {
// we can close domain as soon as possible.
// and we must reuse the exists session and don't close it in SQL backup job.
UseOneShotSession(store kv.Storage, closeDomain bool, fn func(se Session) error) error
+
+ // GetClient returns the client type of the glue
+ GetClient() GlueClient
}
// Session is an abstraction of the session.Session interface.
diff --git a/br/pkg/gluetidb/BUILD.bazel b/br/pkg/gluetidb/BUILD.bazel
index b639da451010d..baebdba642728 100644
--- a/br/pkg/gluetidb/BUILD.bazel
+++ b/br/pkg/gluetidb/BUILD.bazel
@@ -34,6 +34,7 @@ go_test(
deps = [
"//br/pkg/glue",
"//pkg/parser/model",
+ "//pkg/session",
"//pkg/testkit",
"//pkg/types",
"@com_github_stretchr_testify//require",
diff --git a/br/pkg/gluetidb/glue.go b/br/pkg/gluetidb/glue.go
index 69971a52bd0d8..7963c14b0f5bb 100644
--- a/br/pkg/gluetidb/glue.go
+++ b/br/pkg/gluetidb/glue.go
@@ -4,6 +4,7 @@ package gluetidb
import (
"context"
+ "sync"
"time"
"github.com/pingcap/errors"
@@ -40,14 +41,17 @@ func New() Glue {
conf.Log.EnableSlowLog.Store(false)
conf.TiKVClient.CoprReqTimeout = 1800 * time.Second
})
- return Glue{}
+ return Glue{
+ startDomainMu: &sync.Mutex{},
+ }
}
// Glue is an implementation of glue.Glue using a new TiDB session.
type Glue struct {
glue.StdIOGlue
- tikvGlue gluetikv.Glue
+ tikvGlue gluetikv.Glue
+ startDomainMu *sync.Mutex
}
type tidbSession struct {
@@ -55,13 +59,13 @@ type tidbSession struct {
}
// GetDomain implements glue.Glue.
-func (Glue) GetDomain(store kv.Storage) (*domain.Domain, error) {
+func (g Glue) GetDomain(store kv.Storage) (*domain.Domain, error) {
existDom, _ := session.GetDomain(nil)
- initStatsSe, err := session.CreateSession(store)
+ initStatsSe, err := g.createTypesSession(store)
if err != nil {
return nil, errors.Trace(err)
}
- se, err := session.CreateSession(store)
+ se, err := g.createTypesSession(store)
if err != nil {
return nil, errors.Trace(err)
}
@@ -84,8 +88,8 @@ func (Glue) GetDomain(store kv.Storage) (*domain.Domain, error) {
}
// CreateSession implements glue.Glue.
-func (Glue) CreateSession(store kv.Storage) (glue.Session, error) {
- se, err := session.CreateSession(store)
+func (g Glue) CreateSession(store kv.Storage) (glue.Session, error) {
+ se, err := g.createTypesSession(store)
if err != nil {
return nil, errors.Trace(err)
}
@@ -95,6 +99,27 @@ func (Glue) CreateSession(store kv.Storage) (glue.Session, error) {
return tiSession, nil
}
+func (g Glue) startDomainAsNeeded(store kv.Storage) error {
+ g.startDomainMu.Lock()
+ defer g.startDomainMu.Unlock()
+ existDom, _ := session.GetDomain(nil)
+ if existDom != nil {
+ return nil
+ }
+ dom, err := session.GetDomain(store)
+ if err != nil {
+ return err
+ }
+ return dom.Start()
+}
+
+func (g Glue) createTypesSession(store kv.Storage) (sessiontypes.Session, error) {
+ if err := g.startDomainAsNeeded(store); err != nil {
+ return nil, errors.Trace(err)
+ }
+ return session.CreateSession(store)
+}
+
// Open implements glue.Glue.
func (g Glue) Open(path string, option pd.SecurityOption) (kv.Storage, error) {
return g.tikvGlue.Open(path, option)
@@ -122,7 +147,7 @@ func (g Glue) GetVersion() string {
// UseOneShotSession implements glue.Glue.
func (g Glue) UseOneShotSession(store kv.Storage, closeDomain bool, fn func(glue.Session) error) error {
- se, err := session.CreateSession(store)
+ se, err := g.createTypesSession(store)
if err != nil {
return errors.Trace(err)
}
@@ -133,7 +158,7 @@ func (g Glue) UseOneShotSession(store kv.Storage, closeDomain bool, fn func(glue
se.Close()
log.Info("one shot session closed")
}()
- // dom will be created during session.CreateSession.
+ // dom will be created during create session.
dom, err := session.GetDomain(store)
if err != nil {
return errors.Trace(err)
@@ -158,6 +183,10 @@ func (g Glue) UseOneShotSession(store kv.Storage, closeDomain bool, fn func(glue
return nil
}
+func (Glue) GetClient() glue.GlueClient {
+ return glue.ClientCLP
+}
+
// GetSessionCtx implements glue.Glue
func (gs *tidbSession) GetSessionCtx() sessionctx.Context {
return gs.se
diff --git a/br/pkg/gluetidb/glue_test.go b/br/pkg/gluetidb/glue_test.go
index 1813a7b357b16..5c7eb6a0cb598 100644
--- a/br/pkg/gluetidb/glue_test.go
+++ b/br/pkg/gluetidb/glue_test.go
@@ -20,6 +20,7 @@ import (
"github.com/pingcap/tidb/br/pkg/glue"
"github.com/pingcap/tidb/pkg/parser/model"
+ "github.com/pingcap/tidb/pkg/session"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/types"
"github.com/stretchr/testify/require"
@@ -27,14 +28,29 @@ import (
func TestTheSessionIsoation(t *testing.T) {
req := require.New(t)
- store := testkit.CreateMockStore(t)
+ store, dom := session.CreateStoreAndBootstrap(t)
ctx := context.Background()
- g := Glue{}
- session, err := g.CreateSession(store)
+ // we want to test glue start domain explicitly, so close it first.
+ dom.Close()
+ g := New()
+ glueSe, err := g.CreateSession(store)
req.NoError(err)
+ t.Cleanup(func() {
+ existDom, _ := session.GetDomain(nil)
+ if existDom != nil {
+ existDom.Close()
+ }
+ })
+
+ require.NoError(t, glueSe.CreateDatabase(ctx, &model.DBInfo{
+ Name: model.NewCIStr("test_db"),
+ }))
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test_db")
+ tk.MustExec("create table t(id int)")
- req.NoError(session.ExecuteInternal(ctx, "use test;"))
+ req.NoError(glueSe.ExecuteInternal(ctx, "use test;"))
infos := []*model.TableInfo{}
infos = append(infos, &model.TableInfo{
Name: model.NewCIStr("tables_1"),
@@ -75,12 +91,12 @@ func TestTheSessionIsoation(t *testing.T) {
},
}
for _, pinfo := range polices {
- before := session.(*tidbSession).se.GetInfoSchema().SchemaMetaVersion()
- req.NoError(session.CreatePlacementPolicy(ctx, pinfo))
- after := session.(*tidbSession).se.GetInfoSchema().SchemaMetaVersion()
+ before := glueSe.(*tidbSession).se.GetInfoSchema().SchemaMetaVersion()
+ req.NoError(glueSe.CreatePlacementPolicy(ctx, pinfo))
+ after := glueSe.(*tidbSession).se.GetInfoSchema().SchemaMetaVersion()
req.Greater(after, before)
}
- req.NoError(session.(glue.BatchCreateTableSession).CreateTables(ctx, map[string][]*model.TableInfo{
+ req.NoError(glueSe.(glue.BatchCreateTableSession).CreateTables(ctx, map[string][]*model.TableInfo{
"test": infos,
}))
}
diff --git a/br/pkg/gluetidb/mock/mock.go b/br/pkg/gluetidb/mock/mock.go
index 42cbd4a814657..dc54f48ebda3a 100644
--- a/br/pkg/gluetidb/mock/mock.go
+++ b/br/pkg/gluetidb/mock/mock.go
@@ -159,3 +159,7 @@ func (m *MockGlue) UseOneShotSession(store kv.Storage, closeDomain bool, fn func
}
return fn(glueSession)
}
+
+func (*MockGlue) GetClient() glue.GlueClient {
+ return glue.ClientCLP
+}
diff --git a/br/pkg/gluetikv/glue.go b/br/pkg/gluetikv/glue.go
index ad3ae045f478f..2fd990e92d2a5 100644
--- a/br/pkg/gluetikv/glue.go
+++ b/br/pkg/gluetikv/glue.go
@@ -73,3 +73,7 @@ func (Glue) GetVersion() string {
func (g Glue) UseOneShotSession(store kv.Storage, closeDomain bool, fn func(glue.Session) error) error {
return nil
}
+
+func (Glue) GetClient() glue.GlueClient {
+ return glue.ClientCLP
+}
diff --git a/br/pkg/metautil/load_test.go b/br/pkg/metautil/load_test.go
index ef885f3bcbf26..9a1f19ebdd92a 100644
--- a/br/pkg/metautil/load_test.go
+++ b/br/pkg/metautil/load_test.go
@@ -55,9 +55,9 @@ func TestLoadBackupMeta(t *testing.T) {
mockDB := model.DBInfo{
ID: 1,
Name: dbName,
- Tables: []*model.TableInfo{
- mockTbl,
- },
+ }
+ mockDB.Deprecated.Tables = []*model.TableInfo{
+ mockTbl,
}
dbBytes, err := json.Marshal(mockDB)
require.NoError(t, err)
@@ -140,9 +140,9 @@ func TestLoadBackupMetaPartionTable(t *testing.T) {
mockDB := model.DBInfo{
ID: 1,
Name: dbName,
- Tables: []*model.TableInfo{
- mockTbl,
- },
+ }
+ mockDB.Deprecated.Tables = []*model.TableInfo{
+ mockTbl,
}
dbBytes, err := json.Marshal(mockDB)
require.NoError(t, err)
@@ -249,9 +249,9 @@ func buildBenchmarkBackupmeta(b *testing.B, dbName string, tableCount, fileCount
mockDB := model.DBInfo{
ID: 1,
Name: model.NewCIStr(dbName),
- Tables: []*model.TableInfo{
- mockTbl,
- },
+ }
+ mockDB.Deprecated.Tables = []*model.TableInfo{
+ mockTbl,
}
dbBytes, err := json.Marshal(mockDB)
require.NoError(b, err)
diff --git a/br/pkg/mock/mock_cluster.go b/br/pkg/mock/mock_cluster.go
index 6aa97498bee4f..d749b5db58fa0 100644
--- a/br/pkg/mock/mock_cluster.go
+++ b/br/pkg/mock/mock_cluster.go
@@ -72,7 +72,6 @@ func NewCluster() (*Cluster, error) {
}
cluster.Storage = storage
- session.SetSchemaLease(0)
session.DisableStats4Test()
dom, err := session.BootstrapSession(storage)
if err != nil {
diff --git a/br/pkg/restore/ingestrec/ingest_recorder_test.go b/br/pkg/restore/ingestrec/ingest_recorder_test.go
index 56d655a01416a..b6473f724cdbe 100644
--- a/br/pkg/restore/ingestrec/ingest_recorder_test.go
+++ b/br/pkg/restore/ingestrec/ingest_recorder_test.go
@@ -162,7 +162,7 @@ func TestAddIngestRecorder(t *testing.T) {
},
State: model.StatePublic,
}
- err = m.CreateTableOrView(1, dbInfo.Name.L, tblInfo)
+ err = m.CreateTableOrView(1, tblInfo)
require.NoError(t, err)
})
dom, err := session.GetDomain(store)
@@ -357,7 +357,7 @@ func TestIndexesKind(t *testing.T) {
},
State: model.StatePublic,
}
- err = m.CreateTableOrView(1, dbInfo.Name.L, tblInfo)
+ err = m.CreateTableOrView(1, tblInfo)
require.NoError(t, err)
})
dom, err := session.GetDomain(store)
@@ -454,7 +454,7 @@ func TestRewriteTableID(t *testing.T) {
},
State: model.StatePublic,
}
- err = m.CreateTableOrView(1, dbInfo.Name.L, tblInfo)
+ err = m.CreateTableOrView(1, tblInfo)
require.NoError(t, err)
})
dom, err := session.GetDomain(store)
diff --git a/br/pkg/restore/log_client/client.go b/br/pkg/restore/log_client/client.go
index 9e11baf74bec2..52f15354d8e83 100644
--- a/br/pkg/restore/log_client/client.go
+++ b/br/pkg/restore/log_client/client.go
@@ -20,6 +20,7 @@ import (
"crypto/tls"
"fmt"
"math"
+ "os"
"slices"
"strconv"
"strings"
@@ -637,15 +638,71 @@ type FullBackupStorageConfig struct {
type InitSchemaConfig struct {
// required
- IsNewTask bool
- HasFullRestore bool
- TableFilter filter.Filter
+ IsNewTask bool
+ TableFilter filter.Filter
// optional
TiFlashRecorder *tiflashrec.TiFlashRecorder
FullBackupStorage *FullBackupStorageConfig
}
+const UnsafePITRLogRestoreStartBeforeAnyUpstreamUserDDL = "UNSAFE_PITR_LOG_RESTORE_START_BEFORE_ANY_UPSTREAM_USER_DDL"
+
+func (rc *LogClient) generateDBReplacesFromFullBackupStorage(
+ ctx context.Context,
+ cfg *InitSchemaConfig,
+) (map[stream.UpstreamID]*stream.DBReplace, error) {
+ dbReplaces := make(map[stream.UpstreamID]*stream.DBReplace)
+ if cfg.FullBackupStorage == nil {
+ envVal, ok := os.LookupEnv(UnsafePITRLogRestoreStartBeforeAnyUpstreamUserDDL)
+ if ok && len(envVal) > 0 {
+ log.Info(fmt.Sprintf("the environment variable %s is active, skip loading the base schemas.", UnsafePITRLogRestoreStartBeforeAnyUpstreamUserDDL))
+ return dbReplaces, nil
+ }
+ return nil, errors.Errorf("miss upstream table information at `start-ts`(%d) but the full backup path is not specified", rc.startTS)
+ }
+ s, err := storage.New(ctx, cfg.FullBackupStorage.Backend, cfg.FullBackupStorage.Opts)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ fullBackupTables, err := initFullBackupTables(ctx, s, cfg.TableFilter)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ for _, t := range fullBackupTables {
+ dbName, _ := utils.GetSysDBCIStrName(t.DB.Name)
+ newDBInfo, exist := rc.dom.InfoSchema().SchemaByName(dbName)
+ if !exist {
+ log.Info("db not existed", zap.String("dbname", dbName.String()))
+ continue
+ }
+
+ dbReplace, exist := dbReplaces[t.DB.ID]
+ if !exist {
+ dbReplace = stream.NewDBReplace(t.DB.Name.O, newDBInfo.ID)
+ dbReplaces[t.DB.ID] = dbReplace
+ }
+
+ if t.Info == nil {
+ // If the db is empty, skip it.
+ continue
+ }
+ newTableInfo, err := restore.GetTableSchema(rc.GetDomain(), dbName, t.Info.Name)
+ if err != nil {
+ log.Info("table not existed", zap.String("tablename", dbName.String()+"."+t.Info.Name.String()))
+ continue
+ }
+
+ dbReplace.TableMap[t.Info.ID] = &stream.TableReplace{
+ Name: newTableInfo.Name.O,
+ TableID: newTableInfo.ID,
+ PartitionMap: restoreutils.GetPartitionIDMap(newTableInfo, t.Info),
+ IndexMap: restoreutils.GetIndexIDMap(newTableInfo, t.Info),
+ }
+ }
+ return dbReplaces, nil
+}
+
// InitSchemasReplaceForDDL gets schemas information Mapping from old schemas to new schemas.
// It is used to rewrite meta kv-event.
func (rc *LogClient) InitSchemasReplaceForDDL(
@@ -658,7 +715,7 @@ func (rc *LogClient) InitSchemasReplaceForDDL(
// the id map doesn't need to construct only when it is not the first execution
needConstructIdMap bool
- dbReplaces = make(map[stream.UpstreamID]*stream.DBReplace)
+ dbReplaces map[stream.UpstreamID]*stream.DBReplace
)
// not new task, load schemas map from external storage
@@ -673,7 +730,7 @@ func (rc *LogClient) InitSchemasReplaceForDDL(
// a new task, but without full snapshot restore, tries to load
// schemas map whose `restore-ts`` is the task's `start-ts`.
- if len(dbMaps) <= 0 && !cfg.HasFullRestore {
+ if len(dbMaps) <= 0 && cfg.FullBackupStorage == nil {
log.Info("try to load pitr id maps of the previous task", zap.Uint64("start-ts", rc.startTS))
needConstructIdMap = true
dbMaps, err = rc.initSchemasMap(ctx, rc.GetClusterID(ctx), rc.startTS)
@@ -695,45 +752,10 @@ func (rc *LogClient) InitSchemasReplaceForDDL(
if len(dbMaps) <= 0 {
log.Info("no id maps, build the table replaces from cluster and full backup schemas")
needConstructIdMap = true
- s, err := storage.New(ctx, cfg.FullBackupStorage.Backend, cfg.FullBackupStorage.Opts)
+ dbReplaces, err = rc.generateDBReplacesFromFullBackupStorage(ctx, cfg)
if err != nil {
return nil, errors.Trace(err)
}
- fullBackupTables, err := initFullBackupTables(ctx, s, cfg.TableFilter)
- if err != nil {
- return nil, errors.Trace(err)
- }
- for _, t := range fullBackupTables {
- dbName, _ := utils.GetSysDBCIStrName(t.DB.Name)
- newDBInfo, exist := rc.dom.InfoSchema().SchemaByName(dbName)
- if !exist {
- log.Info("db not existed", zap.String("dbname", dbName.String()))
- continue
- }
-
- dbReplace, exist := dbReplaces[t.DB.ID]
- if !exist {
- dbReplace = stream.NewDBReplace(t.DB.Name.O, newDBInfo.ID)
- dbReplaces[t.DB.ID] = dbReplace
- }
-
- if t.Info == nil {
- // If the db is empty, skip it.
- continue
- }
- newTableInfo, err := restore.GetTableSchema(rc.GetDomain(), dbName, t.Info.Name)
- if err != nil {
- log.Info("table not existed", zap.String("tablename", dbName.String()+"."+t.Info.Name.String()))
- continue
- }
-
- dbReplace.TableMap[t.Info.ID] = &stream.TableReplace{
- Name: newTableInfo.Name.O,
- TableID: newTableInfo.ID,
- PartitionMap: restoreutils.GetPartitionIDMap(newTableInfo, t.Info),
- IndexMap: restoreutils.GetIndexIDMap(newTableInfo, t.Info),
- }
- }
} else {
dbReplaces = stream.FromSchemaMaps(dbMaps)
}
@@ -1555,7 +1577,7 @@ func (rc *LogClient) FailpointDoChecksumForLogRestore(
reidRules[downstreamID] = upstreamID
}
for upstreamID, downstreamID := range idrules {
- newTable, ok := infoSchema.TableByID(downstreamID)
+ newTable, ok := infoSchema.TableByID(ctx, downstreamID)
if !ok {
// a dropped table
continue
diff --git a/br/pkg/restore/snap_client/client.go b/br/pkg/restore/snap_client/client.go
index 061fad6388016..783687da1e164 100644
--- a/br/pkg/restore/snap_client/client.go
+++ b/br/pkg/restore/snap_client/client.go
@@ -672,12 +672,43 @@ func (rc *SnapClient) getRebasedTables() map[restore.UniqueTableName]bool {
return rc.rebasedTablesMap
}
+// CreateTables create tables, and generate their information.
+// this function will use workers as the same number of sessionPool,
+// leave sessionPool nil to send DDLs sequential.
+func (rc *SnapClient) CreateTables(
+ ctx context.Context,
+ tables []*metautil.Table,
+ newTS uint64,
+) ([]*CreatedTable, error) {
+ log.Info("start create tables", zap.Int("total count", len(tables)))
+ rc.generateRebasedTables(tables)
+
+ // try to restore tables in batch
+ if rc.batchDdlSize > minBatchDdlSize && len(rc.dbPool) > 0 {
+ tables, err := rc.createTablesBatch(ctx, tables, newTS)
+ if err == nil {
+ return tables, nil
+ } else if !utils.FallBack2CreateTable(err) {
+ return nil, errors.Trace(err)
+ }
+ // fall back to old create table (sequential create table)
+ log.Info("fall back to the sequential create table")
+ }
+
+ // restore tables in db pool
+ if len(rc.dbPool) > 0 {
+ return rc.createTablesSingle(ctx, rc.dbPool, tables, newTS)
+ }
+ // restore tables in one db
+ return rc.createTablesSingle(ctx, []*tidallocdb.DB{rc.db}, tables, newTS)
+}
+
func (rc *SnapClient) createTables(
ctx context.Context,
db *tidallocdb.DB,
tables []*metautil.Table,
newTS uint64,
-) ([]CreatedTable, error) {
+) ([]*CreatedTable, error) {
log.Info("client to create tables")
if rc.IsSkipCreateSQL() {
log.Info("skip create table and alter autoIncID")
@@ -687,7 +718,7 @@ func (rc *SnapClient) createTables(
return nil, errors.Trace(err)
}
}
- cts := make([]CreatedTable, 0, len(tables))
+ cts := make([]*CreatedTable, 0, len(tables))
for _, table := range tables {
newTableInfo, err := restore.GetTableSchema(rc.dom, table.DB.Name, table.Info.Name)
if err != nil {
@@ -701,7 +732,7 @@ func (rc *SnapClient) createTables(
newTableInfo.IsCommonHandle)
}
rules := restoreutils.GetRewriteRules(newTableInfo, table.Info, newTS, true)
- ct := CreatedTable{
+ ct := &CreatedTable{
RewriteRule: rules,
Table: newTableInfo,
OldTable: table,
@@ -712,11 +743,17 @@ func (rc *SnapClient) createTables(
return cts, nil
}
-func (rc *SnapClient) createTablesInWorkerPool(ctx context.Context, tables []*metautil.Table, newTS uint64, outCh chan<- CreatedTable) error {
+func (rc *SnapClient) createTablesBatch(ctx context.Context, tables []*metautil.Table, newTS uint64) ([]*CreatedTable, error) {
eg, ectx := errgroup.WithContext(ctx)
rater := logutil.TraceRateOver(logutil.MetricTableCreatedCounter)
workers := tidbutil.NewWorkerPool(uint(len(rc.dbPool)), "Create Tables Worker")
numOfTables := len(tables)
+ createdTables := struct {
+ sync.Mutex
+ tables []*CreatedTable
+ }{
+ tables: make([]*CreatedTable, 0, len(tables)),
+ }
for lastSent := 0; lastSent < numOfTables; lastSent += int(rc.batchDdlSize) {
end := min(lastSent+int(rc.batchDdlSize), len(tables))
@@ -735,21 +772,19 @@ func (rc *SnapClient) createTablesInWorkerPool(ctx context.Context, tables []*me
log.Error("create tables fail", zap.Error(err))
return err
}
- for _, ct := range cts {
- log.Debug("table created and send to next",
- zap.Int("output chan size", len(outCh)),
- zap.Stringer("table", ct.OldTable.Info.Name),
- zap.Stringer("database", ct.OldTable.DB.Name))
- outCh <- ct
- rater.Inc()
- rater.L().Info("table created",
- zap.Stringer("table", ct.OldTable.Info.Name),
- zap.Stringer("database", ct.OldTable.DB.Name))
- }
+ rater.Add(float64(len(cts)))
+ rater.L().Info("tables created", zap.Int("num", len(cts)))
+ createdTables.Lock()
+ createdTables.tables = append(createdTables.tables, cts...)
+ createdTables.Unlock()
return err
})
}
- return eg.Wait()
+ if err := eg.Wait(); err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ return createdTables.tables, nil
}
func (rc *SnapClient) createTable(
@@ -757,28 +792,28 @@ func (rc *SnapClient) createTable(
db *tidallocdb.DB,
table *metautil.Table,
newTS uint64,
-) (CreatedTable, error) {
+) (*CreatedTable, error) {
if rc.IsSkipCreateSQL() {
log.Info("skip create table and alter autoIncID", zap.Stringer("table", table.Info.Name))
} else {
err := db.CreateTable(ctx, table, rc.getRebasedTables(), rc.supportPolicy, rc.policyMap)
if err != nil {
- return CreatedTable{}, errors.Trace(err)
+ return nil, errors.Trace(err)
}
}
newTableInfo, err := restore.GetTableSchema(rc.dom, table.DB.Name, table.Info.Name)
if err != nil {
- return CreatedTable{}, errors.Trace(err)
+ return nil, errors.Trace(err)
}
if newTableInfo.IsCommonHandle != table.Info.IsCommonHandle {
- return CreatedTable{}, errors.Annotatef(berrors.ErrRestoreModeMismatch,
+ return nil, errors.Annotatef(berrors.ErrRestoreModeMismatch,
"Clustered index option mismatch. Restored cluster's @@tidb_enable_clustered_index should be %v (backup table = %v, created table = %v).",
restore.TransferBoolToValue(table.Info.IsCommonHandle),
table.Info.IsCommonHandle,
newTableInfo.IsCommonHandle)
}
rules := restoreutils.GetRewriteRules(newTableInfo, table.Info, newTS, true)
- et := CreatedTable{
+ et := &CreatedTable{
RewriteRule: rules,
Table: newTableInfo,
OldTable: table,
@@ -786,30 +821,49 @@ func (rc *SnapClient) createTable(
return et, nil
}
-func (rc *SnapClient) createTablesWithSoleDB(ctx context.Context,
- createOneTable func(ctx context.Context, db *tidallocdb.DB, t *metautil.Table) error,
- tables []*metautil.Table) error {
- for _, t := range tables {
- if err := createOneTable(ctx, rc.db, t); err != nil {
- return errors.Trace(err)
- }
- }
- return nil
-}
-
-func (rc *SnapClient) createTablesWithDBPool(ctx context.Context,
- createOneTable func(ctx context.Context, db *tidallocdb.DB, t *metautil.Table) error,
- tables []*metautil.Table) error {
+func (rc *SnapClient) createTablesSingle(
+ ctx context.Context,
+ dbPool []*tidallocdb.DB,
+ tables []*metautil.Table,
+ newTS uint64,
+) ([]*CreatedTable, error) {
eg, ectx := errgroup.WithContext(ctx)
- workers := tidbutil.NewWorkerPool(uint(len(rc.dbPool)), "DDL workers")
- for _, t := range tables {
- table := t
+ workers := tidbutil.NewWorkerPool(uint(len(dbPool)), "DDL workers")
+ rater := logutil.TraceRateOver(logutil.MetricTableCreatedCounter)
+ createdTables := struct {
+ sync.Mutex
+ tables []*CreatedTable
+ }{
+ tables: make([]*CreatedTable, 0, len(tables)),
+ }
+ for _, tbl := range tables {
+ table := tbl
workers.ApplyWithIDInErrorGroup(eg, func(id uint64) error {
- db := rc.dbPool[id%uint64(len(rc.dbPool))]
- return createOneTable(ectx, db, table)
+ db := dbPool[id%uint64(len(dbPool))]
+ rt, err := rc.createTable(ectx, db, table, newTS)
+ if err != nil {
+ log.Error("create table failed",
+ zap.Error(err),
+ zap.Stringer("db", table.DB.Name),
+ zap.Stringer("table", table.Info.Name))
+ return errors.Trace(err)
+ }
+ rater.Inc()
+ rater.L().Info("table created",
+ zap.Stringer("table", table.Info.Name),
+ zap.Stringer("database", table.DB.Name))
+
+ createdTables.Lock()
+ createdTables.tables = append(createdTables.tables, rt)
+ createdTables.Unlock()
+ return nil
})
}
- return eg.Wait()
+ if err := eg.Wait(); err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ return createdTables.tables, nil
}
// InitFullClusterRestore init fullClusterRestore and set SkipGrantTable as needed
diff --git a/br/pkg/restore/snap_client/client_test.go b/br/pkg/restore/snap_client/client_test.go
index 12b91b4937556..b43324d9fca64 100644
--- a/br/pkg/restore/snap_client/client_test.go
+++ b/br/pkg/restore/snap_client/client_test.go
@@ -77,7 +77,7 @@ func TestCreateTables(t *testing.T) {
},
}
}
- rules, newTables, err := client.CreateTables(m.Domain, tables, 0)
+ rules, newTables, err := client.CreateTablesTest(m.Domain, tables, 0)
require.NoError(t, err)
// make sure tables and newTables have same order
for i, tbl := range tables {
@@ -191,7 +191,7 @@ func TestCheckTargetClusterFreshWithTable(t *testing.T) {
Collate: "utf8mb4_bin",
},
}
- _, _, err = client.CreateTables(cluster.Domain, []*metautil.Table{table}, 0)
+ _, _, err = client.CreateTablesTest(cluster.Domain, []*metautil.Table{table}, 0)
require.NoError(t, err)
require.True(t, berrors.ErrRestoreNotFreshCluster.Equal(client.CheckTargetClusterFresh(ctx)))
diff --git a/br/pkg/restore/snap_client/export_test.go b/br/pkg/restore/snap_client/export_test.go
index c519fd0421200..3240ee1f77662 100644
--- a/br/pkg/restore/snap_client/export_test.go
+++ b/br/pkg/restore/snap_client/export_test.go
@@ -55,7 +55,7 @@ func MockCallSetSpeedLimit(ctx context.Context, fakeImportClient importclient.Im
}
// CreateTables creates multiple tables, and returns their rewrite rules.
-func (rc *SnapClient) CreateTables(
+func (rc *SnapClient) CreateTablesTest(
dom *domain.Domain,
tables []*metautil.Table,
newTS uint64,
@@ -65,28 +65,22 @@ func (rc *SnapClient) CreateTables(
Data: make([]*import_sstpb.RewriteRule, 0),
}
newTables := make([]*model.TableInfo, 0, len(tables))
- errCh := make(chan error, 1)
tbMapping := map[string]int{}
for i, t := range tables {
tbMapping[t.Info.Name.String()] = i
}
- dataCh := rc.GoCreateTables(context.TODO(), tables, newTS, errCh)
- for et := range dataCh {
- rules := et.RewriteRule
+ createdTables, err := rc.CreateTables(context.TODO(), tables, newTS)
+ if err != nil {
+ return nil, nil, err
+ }
+ for _, table := range createdTables {
+ rules := table.RewriteRule
rewriteRules.Data = append(rewriteRules.Data, rules.Data...)
- newTables = append(newTables, et.Table)
+ newTables = append(newTables, table.Table)
}
// Let's ensure that it won't break the original order.
slices.SortFunc(newTables, func(i, j *model.TableInfo) int {
return cmp.Compare(tbMapping[i.Name.String()], tbMapping[j.Name.String()])
})
-
- select {
- case err, ok := <-errCh:
- if ok {
- return nil, nil, errors.Trace(err)
- }
- default:
- }
return rewriteRules, newTables, nil
}
diff --git a/br/pkg/restore/snap_client/pipeline_items.go b/br/pkg/restore/snap_client/pipeline_items.go
index 79f47d3a9c71c..55c7cb4d44e54 100644
--- a/br/pkg/restore/snap_client/pipeline_items.go
+++ b/br/pkg/restore/snap_client/pipeline_items.go
@@ -16,23 +16,19 @@ package snapclient
import (
"context"
- "sort"
"sync"
"time"
- "github.com/opentracing/opentracing-go"
"github.com/pingcap/errors"
backuppb "github.com/pingcap/kvproto/pkg/brpb"
"github.com/pingcap/log"
"github.com/pingcap/tidb/br/pkg/glue"
"github.com/pingcap/tidb/br/pkg/logutil"
"github.com/pingcap/tidb/br/pkg/metautil"
- tidallocdb "github.com/pingcap/tidb/br/pkg/restore/internal/prealloc_db"
restoreutils "github.com/pingcap/tidb/br/pkg/restore/utils"
"github.com/pingcap/tidb/br/pkg/rtree"
"github.com/pingcap/tidb/br/pkg/storage"
"github.com/pingcap/tidb/br/pkg/summary"
- "github.com/pingcap/tidb/br/pkg/utils"
"github.com/pingcap/tidb/pkg/domain/infosync"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/model"
@@ -390,112 +386,6 @@ func concurrentHandleTablesCh(
}
}
-// GoCreateTables create tables, and generate their information.
-// this function will use workers as the same number of sessionPool,
-// leave sessionPool nil to send DDLs sequential.
-func (rc *SnapClient) GoCreateTables(
- ctx context.Context,
- tables []*metautil.Table,
- newTS uint64,
- errCh chan<- error,
-) <-chan CreatedTable {
- // Could we have a smaller size of tables?
- log.Info("start create tables")
-
- rc.generateRebasedTables(tables)
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("Client.GoCreateTables", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
- outCh := make(chan CreatedTable, len(tables))
- rater := logutil.TraceRateOver(logutil.MetricTableCreatedCounter)
-
- var err error
-
- if rc.batchDdlSize > minBatchDdlSize && len(rc.dbPool) > 0 {
- err = rc.createTablesInWorkerPool(ctx, tables, newTS, outCh)
- if err == nil {
- defer log.Debug("all tables are created")
- close(outCh)
- return outCh
- } else if !utils.FallBack2CreateTable(err) {
- errCh <- err
- close(outCh)
- return outCh
- }
- // fall back to old create table (sequential create table)
- log.Info("fall back to the sequential create table")
- }
-
- createOneTable := func(c context.Context, db *tidallocdb.DB, t *metautil.Table) error {
- select {
- case <-c.Done():
- return c.Err()
- default:
- }
- rt, err := rc.createTable(c, db, t, newTS)
- if err != nil {
- log.Error("create table failed",
- zap.Error(err),
- zap.Stringer("db", t.DB.Name),
- zap.Stringer("table", t.Info.Name))
- return errors.Trace(err)
- }
- log.Debug("table created and send to next",
- zap.Int("output chan size", len(outCh)),
- zap.Stringer("table", t.Info.Name),
- zap.Stringer("database", t.DB.Name))
- outCh <- rt
- rater.Inc()
- rater.L().Info("table created",
- zap.Stringer("table", t.Info.Name),
- zap.Stringer("database", t.DB.Name))
- return nil
- }
- go func() {
- defer close(outCh)
- defer log.Debug("all tables are created")
- var err error
- if len(rc.dbPool) > 0 {
- err = rc.createTablesWithDBPool(ctx, createOneTable, tables)
- } else {
- err = rc.createTablesWithSoleDB(ctx, createOneTable, tables)
- }
- if err != nil {
- errCh <- err
- }
- }()
-
- return outCh
-}
-
-func (rc *SnapClient) GoBlockCreateTablesPipeline(ctx context.Context, sz int, inCh <-chan CreatedTable) <-chan CreatedTable {
- outCh := make(chan CreatedTable, sz)
-
- go func() {
- defer close(outCh)
- cachedTables := make([]CreatedTable, 0, sz)
- for tbl := range inCh {
- cachedTables = append(cachedTables, tbl)
- }
-
- sort.Slice(cachedTables, func(a, b int) bool {
- return cachedTables[a].Table.ID < cachedTables[b].Table.ID
- })
-
- for _, tbl := range cachedTables {
- select {
- case <-ctx.Done():
- return
- default:
- outCh <- tbl
- }
- }
- }()
- return outCh
-}
-
// GoValidateFileRanges validate files by a stream of tables and yields
// tables with range.
func (rc *SnapClient) GoValidateFileRanges(
diff --git a/br/pkg/restore/tiflashrec/tiflash_recorder.go b/br/pkg/restore/tiflashrec/tiflash_recorder.go
index 692e3b6a82511..8add1bb8f3849 100644
--- a/br/pkg/restore/tiflashrec/tiflash_recorder.go
+++ b/br/pkg/restore/tiflashrec/tiflash_recorder.go
@@ -16,6 +16,7 @@ package tiflashrec
import (
"bytes"
+ "context"
"fmt"
"github.com/pingcap/log"
@@ -90,7 +91,7 @@ func (r *TiFlashRecorder) Rewrite(oldID int64, newID int64) {
func (r *TiFlashRecorder) GenerateResetAlterTableDDLs(info infoschema.InfoSchema) []string {
items := make([]string, 0, len(r.items))
r.Iterate(func(id int64, replica model.TiFlashReplicaInfo) {
- table, ok := info.TableByID(id)
+ table, ok := info.TableByID(context.Background(), id)
if !ok {
log.Warn("Table do not exist, skipping", zap.Int64("id", id))
return
@@ -130,7 +131,7 @@ func (r *TiFlashRecorder) GenerateResetAlterTableDDLs(info infoschema.InfoSchema
func (r *TiFlashRecorder) GenerateAlterTableDDLs(info infoschema.InfoSchema) []string {
items := make([]string, 0, len(r.items))
r.Iterate(func(id int64, replica model.TiFlashReplicaInfo) {
- table, ok := info.TableByID(id)
+ table, ok := info.TableByID(context.Background(), id)
if !ok {
log.Warn("Table do not exist, skipping", zap.Int64("id", id))
return
diff --git a/br/pkg/task/config_test.go b/br/pkg/task/config_test.go
index 6a86b43dc24c2..3090570046644 100644
--- a/br/pkg/task/config_test.go
+++ b/br/pkg/task/config_test.go
@@ -232,10 +232,10 @@ func mockReadSchemasFromBackupMeta(t *testing.T, db2Tables map[string][]string)
}
mockDB := model.DBInfo{
- ID: dbID,
- Name: dbName,
- Tables: mockTblList,
+ ID: dbID,
+ Name: dbName,
}
+ mockDB.Deprecated.Tables = mockTblList
dbID++
dbBytes, err := json.Marshal(mockDB)
require.NoError(t, err)
diff --git a/br/pkg/task/restore.go b/br/pkg/task/restore.go
index 8bc6383be78b6..f1ff4391634af 100644
--- a/br/pkg/task/restore.go
+++ b/br/pkg/task/restore.go
@@ -7,6 +7,7 @@ import (
"context"
"fmt"
"slices"
+ "sort"
"strings"
"time"
@@ -34,10 +35,10 @@ import (
"github.com/pingcap/tidb/br/pkg/version"
"github.com/pingcap/tidb/pkg/config"
"github.com/pingcap/tidb/pkg/domain"
+ "github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/tablecodec"
- "github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/collate"
"github.com/pingcap/tidb/pkg/util/engine"
"github.com/pingcap/tidb/pkg/util/mathutil"
@@ -101,7 +102,6 @@ const (
defaultBatchFlushInterval = 16 * time.Second
defaultFlagDdlBatchSize = 128
resetSpeedLimitRetryTimes = 3
- maxRestoreBatchSizeLimit = 10240
)
const (
@@ -179,6 +179,7 @@ func DefineRestoreCommonFlags(flags *pflag.FlagSet) {
_ = flags.MarkHidden(FlagStatsConcurrency)
_ = flags.MarkHidden(FlagBatchFlushInterval)
_ = flags.MarkHidden(FlagDdlBatchSize)
+ _ = flags.MarkHidden(flagUseFSR)
}
// ParseFromFlags parses the config from the flag set.
@@ -910,6 +911,11 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
if cfg.WithSysTable {
client.InitFullClusterRestore(cfg.ExplicitFilter)
}
+ } else if checkpointFirstRun && cfg.CheckRequirements {
+ if err := checkTableExistence(ctx, mgr, tables, g); err != nil {
+ schedulersRemovable = true
+ return errors.Trace(err)
+ }
}
if client.IsFullClusterRestore() && client.HasBackedUpSysDB() {
@@ -1047,7 +1053,10 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
// We make bigger errCh so we won't block on multi-part failed.
errCh := make(chan error, 32)
- tableStream := client.GoCreateTables(ctx, tables, newTS, errCh)
+ createdTables, err := client.CreateTables(ctx, tables, newTS)
+ if err != nil {
+ return errors.Trace(err)
+ }
if len(files) == 0 {
log.Info("no files, empty databases and tables are restored")
@@ -1067,30 +1076,25 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
}
// Hijack the tableStream and rewrite the rewrite rules.
- tableStream = util.ChanMap(tableStream, func(t snapclient.CreatedTable) snapclient.CreatedTable {
+ for _, createdTable := range createdTables {
// Set the keyspace info for the checksum requests
- t.RewriteRule.OldKeyspace = oldKeyspace
- t.RewriteRule.NewKeyspace = newKeyspace
+ createdTable.RewriteRule.OldKeyspace = oldKeyspace
+ createdTable.RewriteRule.NewKeyspace = newKeyspace
- for _, rule := range t.RewriteRule.Data {
+ for _, rule := range createdTable.RewriteRule.Data {
rule.OldKeyPrefix = append(append([]byte{}, oldKeyspace...), rule.OldKeyPrefix...)
rule.NewKeyPrefix = codec.EncodeKey(rule.NewKeyPrefix)
}
- return t
- })
+ }
}
if cfg.tiflashRecorder != nil {
- tableStream = util.ChanMap(tableStream, func(t snapclient.CreatedTable) snapclient.CreatedTable {
- if cfg.tiflashRecorder != nil {
- cfg.tiflashRecorder.Rewrite(t.OldTable.Info.ID, t.Table.ID)
- }
- return t
- })
+ for _, createdTable := range createdTables {
+ cfg.tiflashRecorder.Rewrite(createdTable.OldTable.Info.ID, createdTable.Table.ID)
+ }
}
- // Block on creating tables before restore starts. since create table is no longer a heavy operation any more.
- tableStream = client.GoBlockCreateTablesPipeline(ctx, maxRestoreBatchSizeLimit, tableStream)
+ tableStream := afterTableCreatedCh(ctx, createdTables)
tableFileMap := MapTableToFiles(files)
log.Debug("mapped table to files", zap.Any("result map", tableFileMap))
@@ -1315,6 +1319,10 @@ func checkDiskSpace(ctx context.Context, mgr *conn.Mgr, files []*backuppb.File,
}
return base * uint64(ratio*10) / 10
}
+
+ // The preserve rate for tikv is quite accurate, while rate for tiflash is a
+ // number calculated from tpcc testing with variable data sizes. 1.4 is a
+ // relative conservative value.
tikvUsage := preserve(EstimateTikvUsage(files, maxReplica, tikvCnt), 1.1)
tiflashUsage := preserve(EstimateTiflashUsage(tables, tiflashCnt), 1.4)
log.Info("preserved disk space", zap.Uint64("tikv", tikvUsage), zap.Uint64("tiflash", tiflashUsage))
@@ -1358,6 +1366,28 @@ func Exhaust(ec <-chan error) []error {
}
}
+func checkTableExistence(ctx context.Context, mgr *conn.Mgr, tables []*metautil.Table, g glue.Glue) error {
+ // Tasks from br clp client use other checks to validate
+ if g.GetClient() != glue.ClientSql {
+ return nil
+ }
+ message := "table already exists: "
+ allUnique := true
+ for _, table := range tables {
+ _, err := mgr.GetDomain().InfoSchema().TableByName(ctx, table.DB.Name, table.Info.Name)
+ if err == nil {
+ message += fmt.Sprintf("%s.%s ", table.DB.Name, table.Info.Name)
+ allUnique = false
+ } else if !infoschema.ErrTableNotExists.Equal(err) {
+ return errors.Trace(err)
+ }
+ }
+ if !allUnique {
+ return errors.Annotate(berrors.ErrTablesAlreadyExisted, message)
+ }
+ return nil
+}
+
// EstimateRangeSize estimates the total range count by file.
func EstimateRangeSize(files []*backuppb.File) int {
result := 0
@@ -1711,3 +1741,25 @@ func checkIsInActions(action model.ActionType, actions map[model.ActionType]stru
_, ok := actions[action]
return ok
}
+
+func afterTableCreatedCh(ctx context.Context, createdTables []*snapclient.CreatedTable) <-chan snapclient.CreatedTable {
+ outCh := make(chan snapclient.CreatedTable)
+
+ go func() {
+ defer close(outCh)
+
+ sort.Slice(createdTables, func(a, b int) bool {
+ return createdTables[a].Table.ID < createdTables[b].Table.ID
+ })
+
+ for _, createdTable := range createdTables {
+ select {
+ case <-ctx.Done():
+ return
+ default:
+ outCh <- *createdTable
+ }
+ }
+ }()
+ return outCh
+}
diff --git a/br/pkg/task/stream.go b/br/pkg/task/stream.go
index 5ec178a4ceeff..29e3177df7e0c 100644
--- a/br/pkg/task/stream.go
+++ b/br/pkg/task/stream.go
@@ -435,18 +435,6 @@ func (s *streamMgr) backupFullSchemas(ctx context.Context) error {
m.ClusterVersion = clusterVersion
})
- schemas := backup.NewBackupSchemas(func(storage kv.Storage, fn func(*model.DBInfo, *model.TableInfo)) error {
- return backup.BuildFullSchema(storage, s.cfg.StartTS, func(dbInfo *model.DBInfo, tableInfo *model.TableInfo) {
- fn(dbInfo, tableInfo)
- })
- }, 0)
-
- err = schemas.BackupSchemas(ctx, metaWriter, nil, s.mgr.GetStorage(), nil,
- s.cfg.StartTS, backup.DefaultSchemaConcurrency, 0, true, nil)
- if err != nil {
- return errors.Trace(err)
- }
-
if err = metaWriter.FlushBackupMeta(ctx); err != nil {
return errors.Trace(err)
}
@@ -1364,7 +1352,6 @@ func restoreStream(
// get the schemas ID replace information.
schemasReplace, err := client.InitSchemasReplaceForDDL(ctx, &logclient.InitSchemaConfig{
IsNewTask: newTask,
- HasFullRestore: len(cfg.FullBackupStorage) > 0,
TableFilter: cfg.TableFilter,
TiFlashRecorder: cfg.tiflashRecorder,
FullBackupStorage: fullBackupStorage,
@@ -1686,13 +1673,11 @@ func getFullBackupTS(
func parseFullBackupTablesStorage(
cfg *RestoreConfig,
) (*logclient.FullBackupStorageConfig, error) {
- var storageName string
- if len(cfg.FullBackupStorage) > 0 {
- storageName = cfg.FullBackupStorage
- } else {
- storageName = cfg.Storage
+ if len(cfg.FullBackupStorage) == 0 {
+ log.Info("the full backup path is not specified, so BR will try to get id maps")
+ return nil, nil
}
- u, err := storage.ParseBackend(storageName, &cfg.BackendOptions)
+ u, err := storage.ParseBackend(cfg.FullBackupStorage, &cfg.BackendOptions)
if err != nil {
return nil, errors.Trace(err)
}
diff --git a/br/pkg/utils/BUILD.bazel b/br/pkg/utils/BUILD.bazel
index 1b0272f37d956..e00548ccabca0 100644
--- a/br/pkg/utils/BUILD.bazel
+++ b/br/pkg/utils/BUILD.bazel
@@ -7,10 +7,10 @@ go_library(
"db.go",
"dyn_pprof_other.go",
"dyn_pprof_unix.go",
+ "error_handling.go",
"json.go",
"key.go",
"misc.go",
- "permission.go",
"pointer.go",
"pprof.go",
"progress.go",
@@ -55,6 +55,7 @@ go_library(
"@org_golang_google_grpc//credentials/insecure",
"@org_golang_google_grpc//keepalive",
"@org_golang_google_grpc//status",
+ "@org_golang_x_term//:term",
"@org_uber_go_atomic//:atomic",
"@org_uber_go_multierr//:multierr",
"@org_uber_go_zap//:zap",
@@ -68,6 +69,7 @@ go_test(
srcs = [
"backoff_test.go",
"db_test.go",
+ "error_handling_test.go",
"json_test.go",
"key_test.go",
"main_test.go",
diff --git a/br/pkg/utils/backoff.go b/br/pkg/utils/backoff.go
index b9ce6dcb0b5d6..385ed4319a06a 100644
--- a/br/pkg/utils/backoff.go
+++ b/br/pkg/utils/backoff.go
@@ -172,8 +172,8 @@ func (bo *importerBackoffer) NextBackoff(err error) time.Duration {
// we don't care storeID here.
errs := multierr.Errors(err)
lastErr := errs[len(errs)-1]
- res := bo.errContext.HandleErrorMsg(lastErr.Error(), 0)
- if res.Strategy == RetryStrategy {
+ res := HandleUnknownBackupError(lastErr.Error(), 0, bo.errContext)
+ if res.Strategy == StrategyRetry {
bo.delayTime = 2 * bo.delayTime
bo.attempt--
} else {
diff --git a/br/pkg/utils/error_handling.go b/br/pkg/utils/error_handling.go
new file mode 100644
index 0000000000000..6c6c84a2a1884
--- /dev/null
+++ b/br/pkg/utils/error_handling.go
@@ -0,0 +1,196 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+ "fmt"
+ "strings"
+ "sync"
+
+ backuppb "github.com/pingcap/kvproto/pkg/brpb"
+ "github.com/pingcap/log"
+ "go.uber.org/zap"
+)
+
+// UNSAFE! TODO: remove and map them to error types
+var retryableErrorMsg = []string{
+ "server closed",
+ "connection refused",
+ "connection reset by peer",
+ "channel closed",
+ "error trying to connect",
+ "connection closed before message completed",
+ "body write aborted",
+ "error during dispatch",
+ "put object timeout",
+ "timeout after",
+ "internalerror",
+ "not read from or written to within the timeout period",
+ "requesttimeout",
+ "invalidpart",
+ "end of file before message length reached",
+}
+
+// non-retryable error messages
+// UNSAFE! TODO: remove and map them to error types
+const (
+ ioMsg = "io"
+ notFoundMsg = "notfound"
+ permissionDeniedMsg = "permissiondenied"
+)
+
+// error messages
+const (
+ unreachableRetryMsg = "unreachable retry"
+ retryOnKvErrorMsg = "retry on kv error"
+ retryOnRegionErrorMsg = "retry on region error"
+ clusterIdMismatchMsg = "cluster id mismatch"
+ unknownErrorMsg = "unknown error"
+ contextCancelledMsg = "context canceled"
+ retryOnUnknownErrorMsg = "unknown error, retry it for a few times"
+ noRetryOnUnknownErrorMsg = "unknown error, retried too many times, give up"
+ retryableStorageErrorMsg = "retryable storage error"
+)
+
+type ErrorHandlingResult struct {
+ Strategy ErrorHandlingStrategy
+ Reason string
+}
+
+type ErrorHandlingStrategy int
+
+const (
+ // StrategyRetry error can be retried but will consume the backoff attempt quota.
+ StrategyRetry ErrorHandlingStrategy = iota
+ // StrategyGiveUp means unrecoverable error happened and the BR should exit
+ // for example:
+ // 1. permission not valid.
+ // 2. data not found.
+ // 3. retry too many times
+ StrategyGiveUp
+ // StrategyUnknown for StrategyUnknown error
+ StrategyUnknown
+)
+
+type ErrorContext struct {
+ mu sync.Mutex
+ // encounter times for one context on a store
+ // we may use this value to determine the retry policy
+ encounterTimes map[uint64]int
+ // unknown error retry limitation.
+ // encounter many times error makes Retry to GiveUp.
+ encounterTimesLimitation int
+ description string
+}
+
+func NewErrorContext(scenario string, limitation int) *ErrorContext {
+ return &ErrorContext{
+ description: scenario,
+ encounterTimes: make(map[uint64]int),
+ encounterTimesLimitation: limitation,
+ }
+}
+
+func NewDefaultContext() *ErrorContext {
+ return &ErrorContext{
+ description: "default",
+ encounterTimes: make(map[uint64]int),
+ encounterTimesLimitation: 1,
+ }
+}
+
+func HandleBackupError(err *backuppb.Error, storeId uint64, ec *ErrorContext) ErrorHandlingResult {
+ if err == nil {
+ return ErrorHandlingResult{StrategyRetry, unreachableRetryMsg}
+ }
+ res := handleBackupProtoError(err)
+ // try the best effort handle unknown error based on their error message
+ if res.Strategy == StrategyUnknown && len(err.Msg) != 0 {
+ return HandleUnknownBackupError(err.Msg, storeId, ec)
+ }
+ return res
+}
+
+func handleBackupProtoError(e *backuppb.Error) ErrorHandlingResult {
+ switch e.Detail.(type) {
+ case *backuppb.Error_KvError:
+ return ErrorHandlingResult{StrategyRetry, retryOnKvErrorMsg}
+ case *backuppb.Error_RegionError:
+ return ErrorHandlingResult{StrategyRetry, retryOnRegionErrorMsg}
+ case *backuppb.Error_ClusterIdError:
+ return ErrorHandlingResult{StrategyGiveUp, clusterIdMismatchMsg}
+ }
+ return ErrorHandlingResult{StrategyUnknown, unknownErrorMsg}
+}
+
+// HandleUnknownBackupError UNSAFE! TODO: remove this method and map all the current unknown errors to error types
+func HandleUnknownBackupError(msg string, uuid uint64, ec *ErrorContext) ErrorHandlingResult {
+ // UNSAFE! TODO: use meaningful error code instead of unstructured message to find failed to write error.
+ logger := log.L().With(zap.String("description", ec.description))
+ if messageIsNotFoundStorageError(msg) {
+ reason := fmt.Sprintf("File or directory not found on TiKV Node (store id: %v). "+
+ "workaround: please ensure br and tikv nodes share a same storage and the user of br and tikv has same uid.",
+ uuid)
+ return ErrorHandlingResult{StrategyGiveUp, reason}
+ }
+ if messageIsPermissionDeniedStorageError(msg) {
+ reason := fmt.Sprintf("I/O permission denied error occurs on TiKV Node(store id: %v). "+
+ "workaround: please ensure tikv has permission to read from & write to the storage.",
+ uuid)
+ return ErrorHandlingResult{StrategyGiveUp, reason}
+ }
+ msgLower := strings.ToLower(msg)
+ if strings.Contains(msgLower, contextCancelledMsg) {
+ return ErrorHandlingResult{StrategyGiveUp, contextCancelledMsg}
+ }
+
+ if MessageIsRetryableStorageError(msg) {
+ logger.Warn(retryableStorageErrorMsg, zap.String("error", msg))
+ return ErrorHandlingResult{StrategyRetry, retryableStorageErrorMsg}
+ }
+
+ // retry enough on same store
+ ec.mu.Lock()
+ defer ec.mu.Unlock()
+ ec.encounterTimes[uuid]++
+ if ec.encounterTimes[uuid] <= ec.encounterTimesLimitation {
+ return ErrorHandlingResult{StrategyRetry, retryOnUnknownErrorMsg}
+ }
+ return ErrorHandlingResult{StrategyGiveUp, noRetryOnUnknownErrorMsg}
+}
+
+// messageIsNotFoundStorageError checks whether the message returning from TiKV is "NotFound" storage I/O error
+func messageIsNotFoundStorageError(msg string) bool {
+ msgLower := strings.ToLower(msg)
+ return strings.Contains(msgLower, ioMsg) && strings.Contains(msgLower, notFoundMsg)
+}
+
+// MessageIsPermissionDeniedStorageError checks whether the message returning from TiKV is "PermissionDenied" storage I/O error
+func messageIsPermissionDeniedStorageError(msg string) bool {
+ msgLower := strings.ToLower(msg)
+ return strings.Contains(msgLower, permissionDeniedMsg)
+}
+
+// MessageIsRetryableStorageError checks whether the message returning from TiKV is retryable ExternalStorageError.
+func MessageIsRetryableStorageError(msg string) bool {
+ msgLower := strings.ToLower(msg)
+ // UNSAFE! TODO: Add a error type for retryable connection error.
+ for _, errStr := range retryableErrorMsg {
+ if strings.Contains(msgLower, errStr) {
+ return true
+ }
+ }
+ return false
+}
diff --git a/br/pkg/utils/error_handling_test.go b/br/pkg/utils/error_handling_test.go
new file mode 100644
index 0000000000000..6b82983843560
--- /dev/null
+++ b/br/pkg/utils/error_handling_test.go
@@ -0,0 +1,70 @@
+package utils
+
+import (
+ "testing"
+
+ backuppb "github.com/pingcap/kvproto/pkg/brpb"
+ "github.com/pingcap/kvproto/pkg/errorpb"
+ "github.com/stretchr/testify/require"
+)
+
+func TestHandleError(t *testing.T) {
+ ec := NewErrorContext("test", 3)
+ // Test case 1: Error is nil
+ result := HandleBackupError(nil, 123, ec)
+ require.Equal(t, ErrorHandlingResult{Strategy: StrategyRetry, Reason: unreachableRetryMsg}, result)
+
+ // Test case 2: Error is KvError and can be ignored
+ kvError := &backuppb.Error_KvError{}
+ result = HandleBackupError(&backuppb.Error{Detail: kvError}, 123, ec)
+ require.Equal(t, ErrorHandlingResult{Strategy: StrategyRetry, Reason: retryOnKvErrorMsg}, result)
+
+ // Test case 3: Error is RegionError and can be ignored
+ regionError := &backuppb.Error_RegionError{
+ RegionError: &errorpb.Error{NotLeader: &errorpb.NotLeader{RegionId: 1}}}
+ result = HandleBackupError(&backuppb.Error{Detail: regionError}, 123, ec)
+ require.Equal(t, ErrorHandlingResult{Strategy: StrategyRetry, Reason: retryOnRegionErrorMsg}, result)
+
+ // Test case 4: Error is ClusterIdError
+ clusterIdError := &backuppb.Error_ClusterIdError{}
+ result = HandleBackupError(&backuppb.Error{Detail: clusterIdError}, 123, ec)
+ require.Equal(t, ErrorHandlingResult{Strategy: StrategyGiveUp, Reason: clusterIdMismatchMsg}, result)
+}
+
+func TestHandleErrorMsg(t *testing.T) {
+ ec := NewErrorContext("test", 3)
+
+ // Test messageIsNotFoundStorageError
+ msg := "IO: files Notfound error"
+ uuid := uint64(456)
+ expectedReason := "File or directory not found on TiKV Node (store id: 456). workaround: please ensure br and tikv nodes share a same storage and the user of br and tikv has same uid."
+ expectedResult := ErrorHandlingResult{Strategy: StrategyGiveUp, Reason: expectedReason}
+ actualResult := HandleUnknownBackupError(msg, uuid, ec)
+ require.Equal(t, expectedResult, actualResult)
+
+ // Test messageIsPermissionDeniedStorageError
+ msg = "I/O permissiondenied error occurs on TiKV Node(store id: 456)."
+ expectedReason = "I/O permission denied error occurs on TiKV Node(store id: 456). workaround: please ensure tikv has permission to read from & write to the storage."
+ expectedResult = ErrorHandlingResult{Strategy: StrategyGiveUp, Reason: expectedReason}
+ actualResult = HandleUnknownBackupError(msg, uuid, ec)
+ require.Equal(t, expectedResult, actualResult)
+
+ // Test MessageIsRetryableStorageError
+ msg = "server closed"
+ expectedResult = ErrorHandlingResult{Strategy: StrategyRetry, Reason: retryableStorageErrorMsg}
+ actualResult = HandleUnknownBackupError(msg, uuid, ec)
+ require.Equal(t, expectedResult, actualResult)
+
+ // Test unknown error
+ msg = "unknown error"
+ expectedResult = ErrorHandlingResult{Strategy: StrategyRetry, Reason: retryOnUnknownErrorMsg}
+ actualResult = HandleUnknownBackupError(msg, uuid, ec)
+ require.Equal(t, expectedResult, actualResult)
+
+ // Test retry too many times
+ _ = HandleUnknownBackupError(msg, uuid, ec)
+ _ = HandleUnknownBackupError(msg, uuid, ec)
+ expectedResult = ErrorHandlingResult{Strategy: StrategyGiveUp, Reason: noRetryOnUnknownErrorMsg}
+ actualResult = HandleUnknownBackupError(msg, uuid, ec)
+ require.Equal(t, expectedResult, actualResult)
+}
diff --git a/br/pkg/utils/permission.go b/br/pkg/utils/permission.go
deleted file mode 100644
index 3c0795db11c47..0000000000000
--- a/br/pkg/utils/permission.go
+++ /dev/null
@@ -1,20 +0,0 @@
-package utils
-
-import "strings"
-
-var (
- ioNotFoundMsg = "notfound"
- permissionDeniedMsg = "permissiondenied"
-)
-
-// messageIsNotFoundStorageError checks whether the message returning from TiKV is "NotFound" storage I/O error
-func messageIsNotFoundStorageError(msg string) bool {
- msgLower := strings.ToLower(msg)
- return strings.Contains(msgLower, "io") && strings.Contains(msgLower, ioNotFoundMsg)
-}
-
-// MessageIsPermissionDeniedStorageError checks whether the message returning from TiKV is "PermissionDenied" storage I/O error
-func messageIsPermissionDeniedStorageError(msg string) bool {
- msgLower := strings.ToLower(msg)
- return strings.Contains(msgLower, permissionDeniedMsg)
-}
diff --git a/br/pkg/utils/progress.go b/br/pkg/utils/progress.go
index 5792677d90633..ba8e86a0dbcfe 100644
--- a/br/pkg/utils/progress.go
+++ b/br/pkg/utils/progress.go
@@ -6,6 +6,7 @@ import (
"context"
"encoding/json"
"io"
+ "os"
"sync"
"sync/atomic"
"time"
@@ -14,6 +15,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/log"
"go.uber.org/zap"
+ "golang.org/x/term"
)
type logFunc func(msg string, fields ...zap.Field)
@@ -74,14 +76,27 @@ func (pp *ProgressPrinter) Close() {
}
}
+// getTerminalOutput try to use os.Stderr as terminal output
+func getTerminalOutput() io.Writer {
+ output := os.Stdout
+ if term.IsTerminal(int(output.Fd())) {
+ return output
+ }
+ return nil
+}
+
// goPrintProgress starts a gorouinte and prints progress.
func (pp *ProgressPrinter) goPrintProgress(
ctx context.Context,
logFuncImpl logFunc,
testWriter io.Writer, // Only for tests
) {
+ var terminalOutput io.Writer
+ if !pp.redirectLog && testWriter == nil {
+ terminalOutput = getTerminalOutput()
+ }
bar := pb.New64(pp.total)
- if pp.redirectLog || testWriter != nil {
+ if terminalOutput == nil {
tmpl := `{"P":"{{percent .}}","C":"{{counters . }}","E":"{{etime .}}","R":"{{rtime .}}","S":"{{speed .}}"}`
bar.SetTemplateString(tmpl)
bar.SetRefreshRate(2 * time.Minute)
@@ -98,6 +113,7 @@ func (pp *ProgressPrinter) goPrintProgress(
tmpl := `{{string . "barName" | green}} {{ bar . "<" "-" (cycle . "-" "\\" "|" "/" ) "." ">"}} {{percent .}}`
bar.SetTemplateString(tmpl)
bar.Set("barName", pp.name)
+ bar.SetWriter(terminalOutput)
}
if testWriter != nil {
bar.SetWriter(testWriter)
diff --git a/br/pkg/utils/retry.go b/br/pkg/utils/retry.go
index 2efe37a8e8dff..6671f16b7842e 100644
--- a/br/pkg/utils/retry.go
+++ b/br/pkg/utils/retry.go
@@ -5,14 +5,11 @@ package utils
import (
"context"
stderrs "errors"
- "fmt"
- "strings"
"sync"
"time"
"github.com/google/uuid"
"github.com/pingcap/errors"
- backuppb "github.com/pingcap/kvproto/pkg/brpb"
"github.com/pingcap/log"
tmysql "github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/parser/terror"
@@ -22,175 +19,6 @@ import (
"go.uber.org/zap"
)
-var retryableServerError = []string{
- "server closed",
- "connection refused",
- "connection reset by peer",
- "channel closed",
- "error trying to connect",
- "connection closed before message completed",
- "body write aborted",
- "error during dispatch",
- "put object timeout",
- "timeout after",
- "internalerror",
- "not read from or written to within the timeout period",
- "requesttimeout",
- "invalidpart",
- "end of file before message length reached",
-}
-
-type ErrorResult struct {
- Strategy ErrorStrategy
- Reason string
-}
-
-type ErrorStrategy int
-
-const (
- // This type can be retry but consume the backoffer attempts.
- RetryStrategy ErrorStrategy = iota
- // This type means unrecoverable error and the whole progress should exits
- // for example:
- // 1. permission not valid.
- // 2. data has not found.
- // 3. retry too many times
- GiveUpStrategy
- // This type represents Unknown error
- UnknownStrategy
-)
-
-type ErrorContext struct {
- mu sync.Mutex
- // encounter times for one context on a store
- // we may use this value to determine the retry policy
- encounterTimes map[uint64]int
- // unknown error retry limitation.
- // encouter many times error makes Retry to GiveUp.
- encounterTimesLimitation int
- // whether in backup or restore
- scenario string
-}
-
-func NewErrorContext(scenario string, limitation int) *ErrorContext {
- return &ErrorContext{
- scenario: scenario,
- encounterTimes: make(map[uint64]int),
- encounterTimesLimitation: limitation,
- }
-}
-
-func NewDefaultContext() *ErrorContext {
- return &ErrorContext{
- scenario: "default",
- encounterTimes: make(map[uint64]int),
- encounterTimesLimitation: 1,
- }
-}
-
-func (ec *ErrorContext) HandleError(err *backuppb.Error, uuid uint64) ErrorResult {
- if err == nil {
- return ErrorResult{RetryStrategy, "unreachable retry"}
- }
- res := ec.handleErrorPb(err, uuid)
- // try the best effort to save progress from error here
- if res.Strategy == UnknownStrategy && len(err.Msg) != 0 {
- return ec.HandleErrorMsg(err.Msg, uuid)
- }
- return res
-}
-
-func (ec *ErrorContext) HandleIgnorableError(err *backuppb.Error, uuid uint64) ErrorResult {
- if err == nil {
- return ErrorResult{RetryStrategy, "unreachable retry"}
- }
- res := ec.handleIgnorableErrorPb(err, uuid)
- // try the best effort to save progress from error here
- if res.Strategy == UnknownStrategy && len(err.Msg) != 0 {
- return ec.HandleErrorMsg(err.Msg, uuid)
- }
- return res
-}
-
-func (ec *ErrorContext) HandleErrorMsg(msg string, uuid uint64) ErrorResult {
- // UNSAFE! TODO: use meaningful error code instead of unstructured message to find failed to write error.
- logger := log.L().With(zap.String("scenario", ec.scenario))
- if messageIsNotFoundStorageError(msg) {
- reason := fmt.Sprintf("File or directory not found on TiKV Node (store id: %v). "+
- "work around:please ensure br and tikv nodes share a same storage and the user of br and tikv has same uid.",
- uuid)
- return ErrorResult{GiveUpStrategy, reason}
- }
- if messageIsPermissionDeniedStorageError(msg) {
- reason := fmt.Sprintf("I/O permission denied error occurs on TiKV Node(store id: %v). "+
- "work around:please ensure tikv has permission to read from & write to the storage.",
- uuid)
- return ErrorResult{GiveUpStrategy, reason}
- }
- msgLower := strings.ToLower(msg)
- if strings.Contains(msgLower, "context canceled") {
- return ErrorResult{GiveUpStrategy, "context canceled, give up"}
- }
-
- if MessageIsRetryableStorageError(msg) {
- logger.Warn("occur storage error", zap.String("error", msg))
- return ErrorResult{RetryStrategy, "retrable error"}
- }
- // retry enough on same store
- ec.mu.Lock()
- defer ec.mu.Unlock()
- ec.encounterTimes[uuid]++
- if ec.encounterTimes[uuid] <= ec.encounterTimesLimitation {
- return ErrorResult{RetryStrategy, "unknown error, retry it for few times"}
- }
- return ErrorResult{GiveUpStrategy, "unknown error and retry too many times, give up"}
-}
-
-func (ec *ErrorContext) handleIgnorableErrorPb(e *backuppb.Error, uuid uint64) ErrorResult {
- switch e.Detail.(type) {
- case *backuppb.Error_KvError:
- return ErrorResult{RetryStrategy, "retry outside because the error can be ignored"}
- case *backuppb.Error_RegionError:
- return ErrorResult{RetryStrategy, "retry outside because the error can be ignored"}
- case *backuppb.Error_ClusterIdError:
- return ErrorResult{GiveUpStrategy, "cluster ID mismatch"}
- }
- return ErrorResult{UnknownStrategy, "unreachable code"}
-}
-
-func (ec *ErrorContext) handleErrorPb(e *backuppb.Error, uuid uint64) ErrorResult {
- logger := log.L().With(zap.String("scenario", ec.scenario))
- switch v := e.Detail.(type) {
- case *backuppb.Error_KvError:
- // should not meet error other than KeyLocked.
- return ErrorResult{GiveUpStrategy, "unknown kv error"}
-
- case *backuppb.Error_RegionError:
- regionErr := v.RegionError
- // Ignore following errors.
- if !(regionErr.EpochNotMatch != nil ||
- regionErr.NotLeader != nil ||
- regionErr.RegionNotFound != nil ||
- regionErr.ServerIsBusy != nil ||
- regionErr.StaleCommand != nil ||
- regionErr.StoreNotMatch != nil ||
- regionErr.ReadIndexNotReady != nil ||
- regionErr.ProposalInMergingMode != nil) {
- logger.Error("unexpect region error", zap.Reflect("RegionError", regionErr))
- return ErrorResult{GiveUpStrategy, "unknown kv error"}
- }
- logger.Warn("occur region error",
- zap.Reflect("RegionError", regionErr),
- zap.Uint64("uuid", uuid))
- return ErrorResult{RetryStrategy, "retrable error"}
-
- case *backuppb.Error_ClusterIdError:
- logger.Error("occur cluster ID error", zap.Reflect("error", v), zap.Uint64("uuid", uuid))
- return ErrorResult{GiveUpStrategy, "cluster ID mismatch"}
- }
- return ErrorResult{UnknownStrategy, "unreachable code"}
-}
-
// RetryableFunc presents a retryable operation.
type RetryableFunc func() error
@@ -281,18 +109,6 @@ func WithRetryReturnLastErr(
return lastErr
}
-// MessageIsRetryableStorageError checks whether the message returning from TiKV is retryable ExternalStorageError.
-func MessageIsRetryableStorageError(msg string) bool {
- msgLower := strings.ToLower(msg)
- // UNSAFE! TODO: Add a error type for retryable connection error.
- for _, errStr := range retryableServerError {
- if strings.Contains(msgLower, errStr) {
- return true
- }
- }
- return false
-}
-
func FallBack2CreateTable(err error) bool {
switch nerr := errors.Cause(err).(type) {
case *terror.Error:
diff --git a/br/pkg/utils/retry_test.go b/br/pkg/utils/retry_test.go
index 605b59fadd8b8..c2afe35f47741 100644
--- a/br/pkg/utils/retry_test.go
+++ b/br/pkg/utils/retry_test.go
@@ -9,8 +9,6 @@ import (
"time"
"github.com/pingcap/errors"
- backuppb "github.com/pingcap/kvproto/pkg/brpb"
- "github.com/pingcap/kvproto/pkg/errorpb"
berrors "github.com/pingcap/tidb/br/pkg/errors"
"github.com/pingcap/tidb/br/pkg/utils"
"github.com/stretchr/testify/require"
@@ -73,74 +71,3 @@ func TestFailNowIf(t *testing.T) {
assert.Equal(time.Duration(0), bo.NextBackoff(annotatedErr))
assert.Equal(0, bo.Attempt())
}
-
-func TestHandleError(t *testing.T) {
- ec := utils.NewErrorContext("test", 3)
- // Test case 1: Error is nil
- result := ec.HandleError(nil, 123)
- require.Equal(t, utils.ErrorResult{utils.RetryStrategy, "unreachable retry"}, result)
-
- // Test case 2: Error is KvError and can be ignored
- kvError := &backuppb.Error_KvError{}
- result = ec.HandleIgnorableError(&backuppb.Error{Detail: kvError}, 123)
- require.Equal(t, utils.ErrorResult{utils.RetryStrategy, "retry outside because the error can be ignored"}, result)
-
- // Test case 3: Error is KvError and cannot be ignored
- result = ec.HandleError(&backuppb.Error{Detail: kvError}, 123)
- require.Equal(t, utils.ErrorResult{utils.GiveUpStrategy, "unknown kv error"}, result)
-
- // Test case 4: Error is RegionError and can be ignored
- regionError := &backuppb.Error_RegionError{
- RegionError: &errorpb.Error{NotLeader: &errorpb.NotLeader{RegionId: 1}}}
- result = ec.HandleIgnorableError(&backuppb.Error{Detail: regionError}, 123)
- require.Equal(t, utils.ErrorResult{utils.RetryStrategy, "retry outside because the error can be ignored"}, result)
-
- // Test case 5: Error is RegionError and cannot be ignored
- regionError = &backuppb.Error_RegionError{
- RegionError: &errorpb.Error{DiskFull: &errorpb.DiskFull{}}}
- result = ec.HandleError(&backuppb.Error{Detail: regionError}, 123)
- require.Equal(t, utils.ErrorResult{utils.GiveUpStrategy, "unknown kv error"}, result)
-
- // Test case 6: Error is ClusterIdError
- clusterIdError := &backuppb.Error_ClusterIdError{}
- result = ec.HandleError(&backuppb.Error{Detail: clusterIdError}, 123)
- require.Equal(t, utils.ErrorResult{utils.GiveUpStrategy, "cluster ID mismatch"}, result)
-}
-
-func TestHandleErrorMsg(t *testing.T) {
- ec := utils.NewErrorContext("test", 3)
-
- // Test messageIsNotFoundStorageError
- msg := "IO: files Notfound error"
- uuid := uint64(456)
- expectedReason := "File or directory not found on TiKV Node (store id: 456). work around:please ensure br and tikv nodes share a same storage and the user of br and tikv has same uid."
- expectedResult := utils.ErrorResult{utils.GiveUpStrategy, expectedReason}
- actualResult := ec.HandleErrorMsg(msg, uuid)
- require.Equal(t, expectedResult, actualResult)
-
- // Test messageIsPermissionDeniedStorageError
- msg = "I/O permissiondenied error occurs on TiKV Node(store id: 456)."
- expectedReason = "I/O permission denied error occurs on TiKV Node(store id: 456). work around:please ensure tikv has permission to read from & write to the storage."
- expectedResult = utils.ErrorResult{utils.GiveUpStrategy, expectedReason}
- actualResult = ec.HandleErrorMsg(msg, uuid)
- require.Equal(t, expectedResult, actualResult)
-
- // Test MessageIsRetryableStorageError
- msg = "server closed"
- expectedResult = utils.ErrorResult{utils.RetryStrategy, "retrable error"}
- actualResult = ec.HandleErrorMsg(msg, uuid)
- require.Equal(t, expectedResult, actualResult)
-
- // Test unknown error
- msg = "unknown error"
- expectedResult = utils.ErrorResult{utils.RetryStrategy, "unknown error, retry it for few times"}
- actualResult = ec.HandleErrorMsg(msg, uuid)
- require.Equal(t, expectedResult, actualResult)
-
- // Test retry too many times
- _ = ec.HandleErrorMsg(msg, uuid)
- _ = ec.HandleErrorMsg(msg, uuid)
- expectedResult = utils.ErrorResult{utils.GiveUpStrategy, "unknown error and retry too many times, give up"}
- actualResult = ec.HandleErrorMsg(msg, uuid)
- require.Equal(t, expectedResult, actualResult)
-}
diff --git a/br/tests/config/pd.toml b/br/tests/config/pd.toml
index a0ff1422f2577..3adbcc89b320a 100644
--- a/br/tests/config/pd.toml
+++ b/br/tests/config/pd.toml
@@ -1,6 +1,3 @@
-lease = 360
-tso-save-interval = "360s"
-
[replication]
enable-placement-rules = true
diff --git a/build/image/base b/build/image/base
index b6a333a5c0f7c..de477e68b6095 100644
--- a/build/image/base
+++ b/build/image/base
@@ -17,7 +17,7 @@ FROM hub.pingcap.net/jenkins/centos7_jenkins
USER root
WORKDIR /root
-ENV GOLANG_VERSION 1.21.12
+ENV GOLANG_VERSION 1.21.13
ENV GOLANG_DOWNLOAD_URL https://dl.google.com/go/go$GOLANG_VERSION.linux-amd64.tar.gz
ENV GOLANG_DOWNLOAD_SHA256 b3075ae1ce5dab85f89bc7905d1632de23ca196bd8336afd93fa97434cfa55ae
ENV GOPATH /go
diff --git a/build/image/parser_test b/build/image/parser_test
index e3590b044333c..6acd458b58efc 100644
--- a/build/image/parser_test
+++ b/build/image/parser_test
@@ -14,7 +14,7 @@
FROM rockylinux:9
-ENV GOLANG_VERSION 1.21.12
+ENV GOLANG_VERSION 1.21.13
ENV ARCH amd64
ENV GOLANG_DOWNLOAD_URL https://dl.google.com/go/go$GOLANG_VERSION.linux-$ARCH.tar.gz
ENV GOPATH /home/prow/go
diff --git a/build/linter/allrevive/BUILD.bazel b/build/linter/allrevive/BUILD.bazel
index b13604af6eef3..6e5ab459f3c5f 100644
--- a/build/linter/allrevive/BUILD.bazel
+++ b/build/linter/allrevive/BUILD.bazel
@@ -7,6 +7,7 @@ go_library(
visibility = ["//visibility:public"],
deps = [
"//build/linter/util",
+ "@com_github_hashicorp_go_version//:go-version",
"@com_github_mgechev_revive//config",
"@com_github_mgechev_revive//lint",
"@com_github_mgechev_revive//rule",
diff --git a/build/linter/allrevive/analyzer.go b/build/linter/allrevive/analyzer.go
index 704ff7bfa184b..6edb07c4acdc9 100644
--- a/build/linter/allrevive/analyzer.go
+++ b/build/linter/allrevive/analyzer.go
@@ -16,10 +16,10 @@ package allrevive
import (
"encoding/json"
- "fmt"
"go/token"
"os"
+ goversion "github.com/hashicorp/go-version"
"github.com/mgechev/revive/config"
"github.com/mgechev/revive/lint"
"github.com/mgechev/revive/rule"
@@ -127,9 +127,13 @@ func run(pass *analysis.Pass) (any, error) {
files = append(files, pass.Fset.PositionFor(file.Pos(), false).Filename)
}
packages := [][]string{files}
-
+ gv, err := goversion.NewVersion("1.21")
+ if err != nil {
+ panic(err)
+ }
revive := lint.New(os.ReadFile, 1024)
conf := lint.Config{
+ GoVersion: gv,
IgnoreGeneratedHeader: false,
Confidence: 0.8,
Severity: "error",
@@ -188,12 +192,12 @@ func run(pass *analysis.Pass) (any, error) {
}
for i := range results {
res := &results[i]
- text := fmt.Sprintf("%s: %s", res.RuleName, res.Failure.Failure)
fileContent, tf, err := util.ReadFile(pass.Fset, res.Position.Start.Filename)
if err != nil {
panic(err)
}
- pass.Reportf(token.Pos(tf.Base()+util.FindOffset(string(fileContent), res.Position.Start.Line, res.Position.Start.Column)), text)
+ pass.Reportf(token.Pos(tf.Base()+util.FindOffset(string(fileContent), res.Position.Start.Line, res.Position.Start.Column)),
+ "%s: %s", res.RuleName, res.Failure.Failure)
}
return nil, nil
}
diff --git a/build/linter/gosec/analysis.go b/build/linter/gosec/analysis.go
index df6fd7e054b4b..f49b185c923c2 100644
--- a/build/linter/gosec/analysis.go
+++ b/build/linter/gosec/analysis.go
@@ -83,7 +83,6 @@ func run(pass *analysis.Pass) (any, error) {
if err != nil {
panic(err)
}
- text := fmt.Sprintf("[%s] %s: %s", Name, i.RuleID, i.What) // TODO: use severity and confidence
var r *result.Range
line, err := strconv.Atoi(i.Line)
if err != nil {
@@ -93,8 +92,8 @@ func run(pass *analysis.Pass) (any, error) {
}
line = r.From
}
-
- pass.Reportf(token.Pos(tf.Base()+util.FindOffset(string(fileContent), line, 1)), text)
+ pass.Reportf(token.Pos(tf.Base()+util.FindOffset(string(fileContent), line, 1)),
+ "[%s] %s: %s", Name, i.RuleID, i.What) // TODO: use severity and confidence
}
return nil, nil
diff --git a/build/linter/misspell/analyzer.go b/build/linter/misspell/analyzer.go
index 5246d82d527ff..ad50fd437d0a0 100644
--- a/build/linter/misspell/analyzer.go
+++ b/build/linter/misspell/analyzer.go
@@ -84,8 +84,8 @@ func runOnFile(fileName string, r *misspell.Replacer, pass *analysis.Pass) error
// tool uses r.Replace by default.
_, diffs := r.Replace(string(fileContent))
for _, diff := range diffs {
- text := fmt.Sprintf("[%s] `%s` is a misspelling of `%s`", Name, diff.Original, diff.Corrected)
- pass.Reportf(token.Pos(tf.Base()+util.FindOffset(string(fileContent), diff.Line, diff.Column)), text)
+ pass.Reportf(token.Pos(tf.Base()+util.FindOffset(string(fileContent), diff.Line, diff.Column)),
+ "[%s] `%s` is a misspelling of `%s`", Name, diff.Original, diff.Corrected)
}
return nil
}
diff --git a/build/linter/revive/BUILD.bazel b/build/linter/revive/BUILD.bazel
index 922f493b7160c..6493822598212 100644
--- a/build/linter/revive/BUILD.bazel
+++ b/build/linter/revive/BUILD.bazel
@@ -7,6 +7,7 @@ go_library(
visibility = ["//visibility:public"],
deps = [
"//build/linter/util",
+ "@com_github_hashicorp_go_version//:go-version",
"@com_github_mgechev_revive//config",
"@com_github_mgechev_revive//lint",
"@com_github_mgechev_revive//rule",
diff --git a/build/linter/revive/analyzer.go b/build/linter/revive/analyzer.go
index 5a853872da8e1..d81a2cf6a4462 100644
--- a/build/linter/revive/analyzer.go
+++ b/build/linter/revive/analyzer.go
@@ -16,10 +16,10 @@ package revive
import (
"encoding/json"
- "fmt"
"go/token"
"os"
+ goversion "github.com/hashicorp/go-version"
"github.com/mgechev/revive/config"
"github.com/mgechev/revive/lint"
"github.com/mgechev/revive/rule"
@@ -96,9 +96,13 @@ func run(pass *analysis.Pass) (any, error) {
files = append(files, pass.Fset.PositionFor(file.Pos(), false).Filename)
}
packages := [][]string{files}
-
+ gv, err := goversion.NewVersion("1.21")
+ if err != nil {
+ panic(err)
+ }
revive := lint.New(os.ReadFile, 1024)
conf := lint.Config{
+ GoVersion: gv,
IgnoreGeneratedHeader: false,
Confidence: 0.8,
Severity: "error",
@@ -156,12 +160,12 @@ func run(pass *analysis.Pass) (any, error) {
}
for i := range results {
res := &results[i]
- text := fmt.Sprintf("%s: %s", res.RuleName, res.Failure.Failure)
fileContent, tf, err := util.ReadFile(pass.Fset, res.Position.Start.Filename)
if err != nil {
panic(err)
}
- pass.Reportf(token.Pos(tf.Base()+util.FindOffset(string(fileContent), res.Position.Start.Line, res.Position.Start.Column)), text)
+ pass.Reportf(token.Pos(tf.Base()+util.FindOffset(string(fileContent), res.Position.Start.Line, res.Position.Start.Column)),
+ "%s: %s", res.RuleName, res.Failure.Failure)
}
return nil, nil
}
diff --git a/build/nogo_config.json b/build/nogo_config.json
index 898b4ba746849..1722910174b33 100644
--- a/build/nogo_config.json
+++ b/build/nogo_config.json
@@ -81,7 +81,9 @@
"exclude_files": {
"pkg/parser/parser.go": "parser/parser.go code",
"external/": "no need to vet third party code",
- ".*_generated\\.go$": "ignore generated code"
+ ".*_generated\\.go$": "ignore generated code",
+ "pkg/planner/core/plan_clone_generator.go": "ignore plan_clone_generator code",
+ "pkg/util/dbterror/ddl_terror.go": "ignore ddl_terror code"
}
},
"unreachable": {
diff --git a/cmd/tidb-server/main.go b/cmd/tidb-server/main.go
index e2fec3206ce12..61f0854e3d056 100644
--- a/cmd/tidb-server/main.go
+++ b/cmd/tidb-server/main.go
@@ -749,8 +749,15 @@ func setGlobalVars() {
util.SetGOGC(cfg.Performance.GOGC)
- ddlLeaseDuration := parseDuration(cfg.Lease)
- session.SetSchemaLease(ddlLeaseDuration)
+ schemaLeaseDuration := parseDuration(cfg.Lease)
+ if schemaLeaseDuration <= 0 {
+ // previous version allow set schema lease to 0, and mainly used on
+ // uni-store and for test, to be compatible we set it to default value here.
+ log.Warn("schema lease is invalid, use default value",
+ zap.String("lease", schemaLeaseDuration.String()))
+ schemaLeaseDuration = config.DefSchemaLease
+ }
+ session.SetSchemaLease(schemaLeaseDuration)
statsLeaseDuration := parseDuration(cfg.Performance.StatsLease)
session.SetStatsLease(statsLeaseDuration)
planReplayerGCLease := parseDuration(cfg.Performance.PlanReplayerGCLease)
diff --git a/docs/design/2022-01-04-integer-shard-index.md b/docs/design/2022-01-04-integer-shard-index.md
index 5ee835d8e8c93..f56280f42d6e5 100644
--- a/docs/design/2022-01-04-integer-shard-index.md
+++ b/docs/design/2022-01-04-integer-shard-index.md
@@ -115,7 +115,7 @@ The entry point to add the `tidb_shard` expression is the function as bellow. We
func (ds *DataSource) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) {
predicates = expression.PropagateConstant(ds.ctx, predicates)
- predicates = DeleteTrueExprs(ds, predicates)
+ predicates = constraint.DeleteTrueExprs(ds, predicates)
// Add tidb_shard() prefix to the condtion for shard index in some scenarios
// TODO: remove it to the place building logical plan
predicates = ds.AddPrefix4ShardIndexes(ds.ctx, predicates)
diff --git a/docs/design/2023-08-24-background-tasks-control.md b/docs/design/2023-08-24-background-tasks-control.md
index 2bee2437c8703..c163c68d46968 100644
--- a/docs/design/2023-08-24-background-tasks-control.md
+++ b/docs/design/2023-08-24-background-tasks-control.md
@@ -8,7 +8,7 @@
Resource control is used to solve some problems of resource usage under data consolidation. We can currently control some normal query tasks by means of RU limiter and scheduling. But it's not an adaptation for some background or bulk import/export tasks very well.
-Due to the implementation restriction, resource control can't be applied for some tasks such as BR and TiDB Lightning. And for some long-running tasks such as DDL or background auto-analyze, it's also hard to control the resource usage becase it's not easy to select a proper RU settrings for these kind of jobs.
+Due to the implementation restriction, resource control can't be applied for some tasks such as BR and TiDB Lightning. And for some long-running tasks such as DDL or background auto-analyze, it's also hard to control the resource usage because it's not easy to select a proper RU settings for these kind of jobs.
## Design Goals
@@ -35,7 +35,7 @@ CREATE/ALTER RESOURCE GROUP rg1
[ BACKGROUND = ( TASK_TYPES = "br,analyze" ) ];
```
-Currently, we only support set the task types that should be controlled in the background manner. We may extend this interface to include more setttings such as task priority in the future.
+Currently, we only support set the task types that should be controlled in the background manner. We may extend this interface to include more settings such as task priority in the future.
If a resource group's background setting is not set, we automatically apply the `default` resource group's settings to this group.
@@ -55,7 +55,7 @@ In order to control the background tasks' resource usage, we plan to add an extr
-- Control the resource usage of all background tasks by the Resource Limiter: The rate limit is dynamically adjusted to the value via the formula TiKVTotalRUCapcity - sum(RUCostRateOfForgroundTasks), with a fine-grained adjusting duration, we can ensure the foreground tasks' RU is always enough(or near the system's maximum if the foreground requirement reaches the maximum quota), so the background tasks' impact on foreground tasks should be very low; on the other hand, when the foreground resource consumption is low, the controller should increase the limit threshold, so background jobs can take advantage of the remaining resources.
+- Control the resource usage of all background tasks by the Resource Limiter: The rate limit is dynamically adjusted to the value via the formula TiKVTotalRUCapacity - sum(RUCostRateOfForegroundTasks), with a fine-grained adjusting duration, we can ensure the foreground tasks' RU is always enough(or near the system's maximum if the foreground requirement reaches the maximum quota), so the background tasks' impact on foreground tasks should be very low; on the other hand, when the foreground resource consumption is low, the controller should increase the limit threshold, so background jobs can take advantage of the remaining resources.
- The local resource manager will statics RU consumption of background jobs via the Resource Limiter: We will do statistics and report the resource consumption to the global resource manager. In the first stage, we only do statistics globally but control it locally.
- Feedback mechanism: It's better to give feedback on how fast the limiter layer executes tasks on tikv to the upper layer like tidb, so that the upper layer task framework can adjust the number of tasks.
@@ -134,7 +134,7 @@ impl Future for LimitedFuture {
In our implementation, we integrate this rate limiter in the following components so it can cover most use cases:
-- Coprocessor. All SQL read requests are handled via the coprocessor component, this can ensure all read reuqests are covered.
+- Coprocessor. All SQL read requests are handled via the coprocessor component, this can ensure all read requests are covered.
- Txn Scheduler. The write requests in tikv are handled via multiple threadpools via a pipeline manner, to make things simple, we only apply the rate limiter in the first phase, that is, the txn scheduler worker pool. Though this is not ideal, the result is acceptable in our benchmark. We may enhance this mechanism in the future.
- Backup. We apply the rate limiter in backup kv scan and sst upload procedure.
- SST Service. Most sst relate operations are handled via the sst service. This ensure BR, TiDB Lightning and DDL(fast mode) can be controlled.
diff --git a/docs/tidb_http_api.md b/docs/tidb_http_api.md
index 0ad3eb1c77169..4de2901c3925d 100644
--- a/docs/tidb_http_api.md
+++ b/docs/tidb_http_api.md
@@ -13,7 +13,10 @@
{
"connections": 0,
"git_hash": "f572e33854e1c0f942f031e9656d0004f99995c6",
- "version": "5.7.25-TiDB-v2.1.0-rc.3-355-gf572e3385-dirty"
+ "version": "5.7.25-TiDB-v2.1.0-rc.3-355-gf572e3385-dirty",
+ "status":{
+ "init_stats_percentage":100
+ }
}
```
diff --git a/dumpling/export/BUILD.bazel b/dumpling/export/BUILD.bazel
index e817a54df5ddc..3bbbff08eeba5 100644
--- a/dumpling/export/BUILD.bazel
+++ b/dumpling/export/BUILD.bazel
@@ -35,6 +35,7 @@ go_library(
"//dumpling/log",
"//pkg/config",
"//pkg/errno",
+ "//pkg/infoschema/context",
"//pkg/parser",
"//pkg/parser/ast",
"//pkg/parser/format",
diff --git a/dumpling/export/dump.go b/dumpling/export/dump.go
index 36964ea5e1cc8..0d94c814eaa77 100644
--- a/dumpling/export/dump.go
+++ b/dumpling/export/dump.go
@@ -30,6 +30,7 @@ import (
"github.com/pingcap/tidb/dumpling/cli"
tcontext "github.com/pingcap/tidb/dumpling/context"
"github.com/pingcap/tidb/dumpling/log"
+ infoschema "github.com/pingcap/tidb/pkg/infoschema/context"
"github.com/pingcap/tidb/pkg/parser"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/format"
@@ -1635,7 +1636,7 @@ func (d *Dumper) renewSelectTableRegionFuncForLowerTiDB(tctx *tcontext.Context)
return errors.Trace(err)
}
tikvHelper := &helper.Helper{}
- tableInfos := tikvHelper.GetRegionsTableInfo(regionsInfo, dbInfos)
+ tableInfos := tikvHelper.GetRegionsTableInfo(regionsInfo, infoschema.DBInfoAsInfoSchema(dbInfos), nil)
tableInfoMap := make(map[string]map[string][]int64, len(conf.Tables))
for _, region := range regionsInfo.Regions {
diff --git a/dumpling/export/sql.go b/dumpling/export/sql.go
index 0918be1640629..690ef65fe054f 100644
--- a/dumpling/export/sql.go
+++ b/dumpling/export/sql.go
@@ -1548,10 +1548,9 @@ func GetDBInfo(db *sql.Conn, tables map[string]map[string]struct{}) ([]*model.DB
}
last := len(schemas) - 1
if last < 0 || schemas[last].Name.O != tableSchema {
- schemas = append(schemas, &model.DBInfo{
- Name: model.CIStr{O: tableSchema},
- Tables: make([]*model.TableInfo, 0, len(tables[tableSchema])),
- })
+ dbInfo := &model.DBInfo{Name: model.CIStr{O: tableSchema}}
+ dbInfo.Deprecated.Tables = make([]*model.TableInfo, 0, len(tables[tableSchema]))
+ schemas = append(schemas, dbInfo)
last++
}
var partition *model.PartitionInfo
@@ -1566,7 +1565,7 @@ func GetDBInfo(db *sql.Conn, tables map[string]map[string]struct{}) ([]*model.DB
}
}
}
- schemas[last].Tables = append(schemas[last].Tables, &model.TableInfo{
+ schemas[last].Deprecated.Tables = append(schemas[last].Deprecated.Tables, &model.TableInfo{
ID: tidbTableID,
Name: model.CIStr{O: tableName},
Partition: partition,
diff --git a/errors.toml b/errors.toml
index b97bde0ae5c53..9c02f10d6ac04 100644
--- a/errors.toml
+++ b/errors.toml
@@ -291,6 +291,11 @@ error = '''
failed to write and ingest
'''
+["BR:Restore:ErrTablesAlreadyExisted"]
+error = '''
+tables already existed in restored cluster
+'''
+
["BR:Restore:ErrUnsupportedSysTable"]
error = '''
the system table isn't supported for restoring yet
@@ -1551,6 +1556,11 @@ error = '''
The operation is not allowed while the bdr role of this cluster is set to %s.
'''
+["ddl:8264"]
+error = '''
+Global Index is needed for index '%-.192s', since the unique index is not including all partitioning columns, and GLOBAL is not given as IndexOption
+'''
+
["domain:8027"]
error = '''
Information schema is out of date: schema failed to update in 1 lease, please make sure TiDB can connect to TiKV
diff --git a/go.mod b/go.mod
index fd3d5e21a7347..bad2430076094 100644
--- a/go.mod
+++ b/go.mod
@@ -73,7 +73,7 @@ require (
github.com/ks3sdklib/aws-sdk-go v1.2.9
github.com/kyoh86/exportloopref v0.1.11
github.com/lestrrat-go/jwx/v2 v2.0.21
- github.com/mgechev/revive v1.3.7
+ github.com/mgechev/revive v1.3.10-0.20240809190117-a638ed6e2499
github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7
github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef
github.com/nishanths/predeclared v0.2.2
@@ -85,18 +85,18 @@ require (
github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f
github.com/pingcap/failpoint v0.0.0-20240527053858-9b3b6e34194a
github.com/pingcap/fn v1.0.0
- github.com/pingcap/kvproto v0.0.0-20240620063548-118a4cab53e4
+ github.com/pingcap/kvproto v0.0.0-20240716095229-5f7ffec83ea7
github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d
github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5
github.com/pingcap/tidb/pkg/parser v0.0.0-20211011031125-9b13dc409c5e
- github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50
+ github.com/pingcap/tipb v0.0.0-20240703084358-e46e4632bd2b
github.com/prometheus/client_golang v1.19.1
github.com/prometheus/client_model v0.6.1
github.com/prometheus/common v0.55.0
github.com/prometheus/prometheus v0.50.1
github.com/qri-io/jsonschema v0.2.1
github.com/robfig/cron/v3 v3.0.1
- github.com/sasha-s/go-deadlock v0.3.1
+ github.com/sasha-s/go-deadlock v0.3.5
github.com/shirou/gopsutil/v3 v3.24.4
github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0
github.com/soheilhy/cmux v0.1.5
@@ -108,8 +108,8 @@ require (
github.com/tdakkota/asciicheck v0.2.0
github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2
github.com/tidwall/btree v1.7.0
- github.com/tikv/client-go/v2 v2.0.8-0.20240703095801-d73cc1ed6503
- github.com/tikv/pd/client v0.0.0-20240717053728-5ec6af403019
+ github.com/tikv/client-go/v2 v2.0.8-0.20240815020919-c810ed88fb02
+ github.com/tikv/pd/client v0.0.0-20240805092608-838ee7983b78
github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a
github.com/twmb/murmur3 v1.1.6
github.com/uber/jaeger-client-go v2.22.1+incompatible
@@ -130,14 +130,14 @@ require (
go.uber.org/multierr v1.11.0
go.uber.org/zap v1.27.0
golang.org/x/exp v0.0.0-20240404231335-c0f41cb1a7a0
- golang.org/x/net v0.27.0
+ golang.org/x/net v0.28.0
golang.org/x/oauth2 v0.21.0
- golang.org/x/sync v0.7.0
- golang.org/x/sys v0.22.0
- golang.org/x/term v0.22.0
- golang.org/x/text v0.16.0
+ golang.org/x/sync v0.8.0
+ golang.org/x/sys v0.24.0
+ golang.org/x/term v0.23.0
+ golang.org/x/text v0.17.0
golang.org/x/time v0.5.0
- golang.org/x/tools v0.22.0
+ golang.org/x/tools v0.24.0
google.golang.org/api v0.169.0
google.golang.org/grpc v1.63.2
gopkg.in/yaml.v2 v2.4.0
@@ -256,7 +256,7 @@ require (
github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
- github.com/mattn/go-runewidth v0.0.15 // indirect
+ github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mitchellh/copystructure v1.0.0 // indirect
github.com/mitchellh/reflectwalk v1.0.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -264,7 +264,7 @@ require (
github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 // indirect
github.com/ncw/directio v1.0.5 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
- github.com/petermattis/goid v0.0.0-20231207134359-e60b3f734c67 // indirect
+ github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
@@ -275,7 +275,7 @@ require (
github.com/prometheus/procfs v0.15.1 // indirect
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
- github.com/rivo/uniseg v0.4.6 // indirect
+ github.com/rivo/uniseg v0.4.7 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect
github.com/segmentio/asm v1.2.0 // indirect
@@ -307,9 +307,9 @@ require (
go.opentelemetry.io/otel/sdk v1.24.0 // indirect
go.opentelemetry.io/otel/trace v1.24.0 // indirect
go.opentelemetry.io/proto/otlp v1.1.0 // indirect
- golang.org/x/crypto v0.25.0 // indirect
+ golang.org/x/crypto v0.26.0 // indirect
golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f // indirect
- golang.org/x/mod v0.18.0 // indirect
+ golang.org/x/mod v0.20.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect
diff --git a/go.sum b/go.sum
index 80be4c4c85157..66783129fda56 100644
--- a/go.sum
+++ b/go.sum
@@ -612,12 +612,12 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
-github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
+github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/mgechev/revive v1.3.7 h1:502QY0vQGe9KtYJ9FpxMz9rL+Fc/P13CI5POL4uHCcE=
-github.com/mgechev/revive v1.3.7/go.mod h1:RJ16jUbF0OWC3co/+XTxmFNgEpUPwnnA0BRllX2aDNA=
+github.com/mgechev/revive v1.3.10-0.20240809190117-a638ed6e2499 h1:Z5grv/SOpby7XE79XWha4cxWPD+JDRNlHPmCDv8Tnyc=
+github.com/mgechev/revive v1.3.10-0.20240809190117-a638ed6e2499/go.mod h1:h3Q6QRohivg9tHOnLeyGcIO2jca7suy65iOdlFLvisw=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
@@ -674,9 +674,8 @@ github.com/otiai10/mint v1.3.1 h1:BCmzIS3n71sGfHB5NMNDB3lHYPz8fWSkCAErHed//qc=
github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
-github.com/petermattis/goid v0.0.0-20231207134359-e60b3f734c67 h1:jik8PHtAIsPlCRJjJzl4udgEf7hawInF9texMeO2jrU=
-github.com/petermattis/goid v0.0.0-20231207134359-e60b3f734c67/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
+github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 h1:Dx7Ovyv/SFnMFw3fD4oEoeorXc6saIiQ23LrGLth0Gw=
+github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY=
@@ -701,16 +700,16 @@ github.com/pingcap/fn v1.0.0/go.mod h1:u9WZ1ZiOD1RpNhcI42RucFh/lBuzTu6rw88a+oF2Z
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E=
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw=
github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
-github.com/pingcap/kvproto v0.0.0-20240620063548-118a4cab53e4 h1:6aIKNB2YGAec4IUDLw6G2eDECiGiufZcgEbZSCELBx0=
-github.com/pingcap/kvproto v0.0.0-20240620063548-118a4cab53e4/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8=
+github.com/pingcap/kvproto v0.0.0-20240716095229-5f7ffec83ea7 h1:V9XS3FQ/P6u+kFaoSyY5DBswIA774BMpIOLDBMrpxKc=
+github.com/pingcap/kvproto v0.0.0-20240716095229-5f7ffec83ea7/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8=
github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM=
github.com/pingcap/log v1.1.0/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d h1:y3EueKVfVykdpTyfUnQGqft0ud+xVFuCdp1XkVL0X1E=
github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d/go.mod h1:ORfBOFp1eteu2odzsyaxI+b8TzJwgjwyQcGhI+9SfEA=
github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 h1:T4pXRhBflzDeAhmOQHNPRRogMYxP13V7BkYw3ZsoSfE=
github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5/go.mod h1:rlimy0GcTvjiJqvD5mXTRr8O2eNZPBrcUgiWVYp9530=
-github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 h1:fVNBE06Rjec+EIHaYAKAHa/bIt5lnu3Zh9O6kV7ZAdg=
-github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs=
+github.com/pingcap/tipb v0.0.0-20240703084358-e46e4632bd2b h1:tySAGYw21A3Xa8CcA9jBTfrgAB3+KQWyqyW7fUyokzk=
+github.com/pingcap/tipb v0.0.0-20240703084358-e46e4632bd2b/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
@@ -756,8 +755,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.6 h1:Sovz9sDSwbOz9tgUy8JpT+KgCkPYJEN/oYzlJiYTNLg=
-github.com/rivo/uniseg v0.4.6/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
@@ -772,8 +771,8 @@ github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfF
github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk=
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8=
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
-github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71efZx0=
-github.com/sasha-s/go-deadlock v0.3.1/go.mod h1:F73l+cr82YSh10GxyRI6qZiCgK64VaZjwesgfQ1/iLM=
+github.com/sasha-s/go-deadlock v0.3.5 h1:tNCOEEDG6tBqrNDOX35j/7hL5FcFViG6awUGROb2NsU=
+github.com/sasha-s/go-deadlock v0.3.5/go.mod h1:bugP6EGbdGYObIlx7pUZtWqlvo8k9H6vCBBsiChJQ5U=
github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
@@ -857,10 +856,10 @@ github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM=
github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
-github.com/tikv/client-go/v2 v2.0.8-0.20240703095801-d73cc1ed6503 h1:0mUlg3+dA5LvwKs1U6i/ID/8RsYgLVLGyM8fSBMb630=
-github.com/tikv/client-go/v2 v2.0.8-0.20240703095801-d73cc1ed6503/go.mod h1:4HDOAx8OXAJPtqhCZ03IhChXgaFs4B3+vSrPWmiPxjg=
-github.com/tikv/pd/client v0.0.0-20240717053728-5ec6af403019 h1:7VoatJKzIrsjepOaXQjpAcgxQrx2QBAI4HuZ0wFdinA=
-github.com/tikv/pd/client v0.0.0-20240717053728-5ec6af403019/go.mod h1:QeMzXKDOW+GbbE+ckcVPBVS6vX3//QB99dXU+niYRq0=
+github.com/tikv/client-go/v2 v2.0.8-0.20240815020919-c810ed88fb02 h1:XKZTb6ZyosZSkvOlmROlhGVHlGHEa3FmIip86cRI1TY=
+github.com/tikv/client-go/v2 v2.0.8-0.20240815020919-c810ed88fb02/go.mod h1:4HDOAx8OXAJPtqhCZ03IhChXgaFs4B3+vSrPWmiPxjg=
+github.com/tikv/pd/client v0.0.0-20240805092608-838ee7983b78 h1:PtW+yTvs9eGTMblulaCHmJ5OtifuE4SJXCACCtkd6ko=
+github.com/tikv/pd/client v0.0.0-20240805092608-838ee7983b78/go.mod h1:TxrJRY949Vl14Lmarx6hTNP/HEDYzn4dP0KmjdzQ59w=
github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a h1:A6uKudFIfAEpoPdaal3aSqGxBzLyU8TqyXImLwo6dIo=
github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a/go.mod h1:mkjARE7Yr8qU23YcGMSALbIxTQ9r9QBVahQOBRfU460=
github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
@@ -1001,8 +1000,8 @@ golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898/go.mod h1:IxCIyHEi3zRg3s0
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
-golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1062,8 +1061,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=
-golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1112,8 +1111,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
-golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
+golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
+golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1136,8 +1135,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
-golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1203,8 +1202,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
-golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -1214,8 +1213,8 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
-golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
-golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
+golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
+golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1229,8 +1228,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
-golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
+golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1302,8 +1301,8 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
-golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
-golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/lightning/pkg/importer/BUILD.bazel b/lightning/pkg/importer/BUILD.bazel
index 83797e95530cb..af898abb2ca62 100644
--- a/lightning/pkg/importer/BUILD.bazel
+++ b/lightning/pkg/importer/BUILD.bazel
@@ -95,7 +95,6 @@ go_library(
"@org_golang_google_grpc//:grpc",
"@org_golang_google_grpc//codes",
"@org_golang_google_grpc//status",
- "@org_golang_x_exp//maps",
"@org_golang_x_sync//errgroup",
"@org_uber_go_atomic//:atomic",
"@org_uber_go_multierr//:multierr",
diff --git a/lightning/pkg/importer/get_pre_info.go b/lightning/pkg/importer/get_pre_info.go
index 5e34c6bf36186..9d5dbf6979d73 100644
--- a/lightning/pkg/importer/get_pre_info.go
+++ b/lightning/pkg/importer/get_pre_info.go
@@ -20,6 +20,7 @@ import (
"database/sql"
"fmt"
"io"
+ "maps"
"strings"
mysql_sql_driver "github.com/go-sql-driver/mysql"
@@ -52,7 +53,6 @@ import (
"github.com/pingcap/tidb/pkg/util/mock"
pdhttp "github.com/tikv/pd/client/http"
"go.uber.org/zap"
- "golang.org/x/exp/maps"
)
// compressionRatio is the tikv/tiflash's compression ratio
diff --git a/lightning/pkg/importer/import.go b/lightning/pkg/importer/import.go
index 04218f2f1102e..dfb441afdc8dc 100644
--- a/lightning/pkg/importer/import.go
+++ b/lightning/pkg/importer/import.go
@@ -640,8 +640,10 @@ func (rc *Controller) initCheckpoint(ctx context.Context) error {
log.FromContext(ctx).Warn("exit triggered", zap.String("failpoint", "InitializeCheckpointExit"))
os.Exit(0)
})
- if err := rc.loadDesiredTableInfos(ctx); err != nil {
- return err
+ if rc.cfg.TikvImporter.AddIndexBySQL {
+ if err := rc.loadDesiredTableInfos(ctx); err != nil {
+ return err
+ }
}
rc.checkpointsWg.Add(1) // checkpointsWg will be done in `rc.listenCheckpointUpdates`
diff --git a/lightning/pkg/importer/meta_manager.go b/lightning/pkg/importer/meta_manager.go
index 188ecb9987507..d4fe9523b08e4 100644
--- a/lightning/pkg/importer/meta_manager.go
+++ b/lightning/pkg/importer/meta_manager.go
@@ -185,7 +185,7 @@ func (m *dbTableMetaMgr) AllocTableRowIDs(ctx context.Context, rawRowIDMax int64
//nolint: errcheck
defer conn.Close()
exec := &common.SQLWithRetry{
- DB: m.session,
+ DB: conn,
Logger: m.tr.logger,
}
var newRowIDBase, newRowIDMax int64
@@ -391,7 +391,7 @@ func (m *dbTableMetaMgr) CheckAndUpdateLocalChecksum(ctx context.Context, checks
//nolint: errcheck
defer conn.Close()
exec := &common.SQLWithRetry{
- DB: m.session,
+ DB: conn,
Logger: m.tr.logger,
}
err = exec.Exec(ctx, "enable pessimistic transaction", "SET SESSION tidb_txn_mode = 'pessimistic';")
@@ -664,7 +664,7 @@ func (m *dbTaskMetaMgr) CheckTasksExclusively(ctx context.Context, action func(t
//nolint: errcheck
defer conn.Close()
exec := &common.SQLWithRetry{
- DB: m.session,
+ DB: conn,
Logger: log.FromContext(ctx),
}
err = exec.Exec(ctx, "enable pessimistic transaction", "SET SESSION tidb_txn_mode = 'pessimistic';")
@@ -735,7 +735,7 @@ func (m *dbTaskMetaMgr) CheckAndPausePdSchedulers(ctx context.Context) (pdutil.U
//nolint: errcheck
defer conn.Close()
exec := &common.SQLWithRetry{
- DB: m.session,
+ DB: conn,
Logger: log.FromContext(ctx),
}
err = exec.Exec(ctx, "enable pessimistic transaction", "SET SESSION tidb_txn_mode = 'pessimistic';")
@@ -875,7 +875,7 @@ func (m *dbTaskMetaMgr) CheckAndFinishRestore(ctx context.Context, finished bool
//nolint: errcheck
defer conn.Close()
exec := &common.SQLWithRetry{
- DB: m.session,
+ DB: conn,
Logger: log.FromContext(ctx),
}
err = exec.Exec(ctx, "enable pessimistic transaction", "SET SESSION tidb_txn_mode = 'pessimistic';")
diff --git a/lightning/pkg/importer/meta_manager_test.go b/lightning/pkg/importer/meta_manager_test.go
index e0a571eb537e4..8146d64c8c83a 100644
--- a/lightning/pkg/importer/meta_manager_test.go
+++ b/lightning/pkg/importer/meta_manager_test.go
@@ -81,7 +81,7 @@ func newTableRestore(t *testing.T,
if err := m.CreateDatabase(&model.DBInfo{ID: dbInfo.ID}); err != nil && !errors.ErrorEqual(err, meta.ErrDBExists) {
return err
}
- return m.CreateTableOrView(dbInfo.ID, db, ti.Core)
+ return m.CreateTableOrView(dbInfo.ID, ti.Core)
})
require.NoError(t, err)
diff --git a/lightning/pkg/importer/precheck_impl.go b/lightning/pkg/importer/precheck_impl.go
index 3327813e701fa..d36afc0254978 100644
--- a/lightning/pkg/importer/precheck_impl.go
+++ b/lightning/pkg/importer/precheck_impl.go
@@ -1296,7 +1296,7 @@ func checkFieldCompatibility(
values []types.Datum,
logger log.Logger,
) bool {
- se := kv.NewSessionCtx(&encode.SessionOptions{
+ se := kv.NewSession(&encode.SessionOptions{
SQLMode: mysql.ModeStrictTransTables,
}, logger)
for i, col := range tbl.Columns {
@@ -1307,7 +1307,7 @@ func checkFieldCompatibility(
if i >= len(values) {
break
}
- _, err := table.CastValue(se, values[i], col, true, false)
+ _, err := table.CastColumnValue(se.GetExprCtx(), values[i], col, true, false)
if err != nil {
logger.Error("field value is not consistent with column type", zap.String("value", values[i].GetString()),
zap.Any("column_info", col), zap.Error(err))
diff --git a/lightning/tests/config/pd.toml b/lightning/tests/config/pd.toml
index bc9f064451512..2557813034e9b 100644
--- a/lightning/tests/config/pd.toml
+++ b/lightning/tests/config/pd.toml
@@ -1,6 +1,3 @@
-lease = 360
-tso-save-interval = "360s"
-
[replication]
enable-placement-rules = true
diff --git a/lightning/tests/lightning_add_index/config2-file.toml b/lightning/tests/lightning_add_index/config2-file.toml
new file mode 100644
index 0000000000000..6c470b3cc531b
--- /dev/null
+++ b/lightning/tests/lightning_add_index/config2-file.toml
@@ -0,0 +1,8 @@
+[checkpoint]
+enable = true
+driver = "file"
+dsn = "/tmp/add-index-by-sql-checkpoint.pb"
+
+[tikv-importer]
+backend = 'local'
+add-index-by-sql = true
diff --git a/lightning/tests/lightning_add_index/config2.toml b/lightning/tests/lightning_add_index/config2-mysql.toml
similarity index 100%
rename from lightning/tests/lightning_add_index/config2.toml
rename to lightning/tests/lightning_add_index/config2-mysql.toml
diff --git a/lightning/tests/lightning_add_index/run.sh b/lightning/tests/lightning_add_index/run.sh
index d82b74b2a52b1..6f3fba046abeb 100644
--- a/lightning/tests/lightning_add_index/run.sh
+++ b/lightning/tests/lightning_add_index/run.sh
@@ -29,7 +29,7 @@ non_pk_auto_inc_kvs=$(run_sql "ADMIN CHECKSUM TABLE add_index.non_pk_auto_inc;"
non_pk_auto_inc_cksum=$(run_sql "ADMIN CHECKSUM TABLE add_index.non_pk_auto_inc;" | grep "Checksum_crc64_xor" | awk '{print $2}')
run_sql "DROP DATABASE add_index;"
-run_lightning --config "$CUR/config2.toml" --log-file "$LOG_FILE2"
+run_lightning --config "$CUR/config2-mysql.toml" --log-file "$LOG_FILE2"
actual_multi_indexes_kvs=$(run_sql "ADMIN CHECKSUM TABLE add_index.multi_indexes;" | grep "Total_kvs" | awk '{print $2}')
actual_multi_indexes_cksum=$(run_sql "ADMIN CHECKSUM TABLE add_index.multi_indexes;" | grep "Checksum_crc64_xor" | awk '{print $2}')
actual_non_pk_auto_inc_kvs=$(run_sql "ADMIN CHECKSUM TABLE add_index.non_pk_auto_inc;" | grep "Total_kvs" | awk '{print $2}')
@@ -82,18 +82,24 @@ grep -Fq "ALTER TABLE \`add_index\`.\`non_pk_auto_inc\` DROP PRIMARY KEY" "$LOG_
grep -Fq "ALTER TABLE \`add_index\`.\`non_pk_auto_inc\` ADD PRIMARY KEY (\`pk\`)" "$LOG_FILE2"
# 3. Check for recovering from checkpoint
-export GO_FAILPOINTS="github.com/pingcap/tidb/lightning/pkg/importer/AddIndexCrash=return()"
-run_sql "DROP DATABASE add_index;"
-run_lightning --enable-checkpoint=1 --config "$CUR/config2.toml" --log-file "$LOG_FILE2"
-grep -Fq "task canceled" "$LOG_FILE2"
-
-unset GO_FAILPOINTS
-run_lightning --enable-checkpoint=1 --config "$CUR/config2.toml" --log-file "$LOG_FILE2"
-actual_multi_indexes_kvs=$(run_sql "ADMIN CHECKSUM TABLE add_index.multi_indexes;" | grep "Total_kvs" | awk '{print $2}')
-actual_multi_indexes_cksum=$(run_sql "ADMIN CHECKSUM TABLE add_index.multi_indexes;" | grep "Checksum_crc64_xor" | awk '{print $2}')
-actual_non_pk_auto_inc_kvs=$(run_sql "ADMIN CHECKSUM TABLE add_index.non_pk_auto_inc;" | grep "Total_kvs" | awk '{print $2}')
-actual_non_pk_auto_inc_cksum=$(run_sql "ADMIN CHECKSUM TABLE add_index.non_pk_auto_inc;" | grep "Checksum_crc64_xor" | awk '{print $2}')
-
-set -x
-[ "$multi_indexes_kvs" == "$actual_multi_indexes_kvs" ] && [ "$multi_indexes_cksum" == "$actual_multi_indexes_cksum" ]
-[ "$non_pk_auto_inc_kvs" == "$actual_non_pk_auto_inc_kvs" ] && [ "$non_pk_auto_inc_cksum" == "$actual_non_pk_auto_inc_cksum" ]
+function recover_from_checkpoint() {
+ tp=$1
+ export GO_FAILPOINTS="github.com/pingcap/tidb/lightning/pkg/importer/AddIndexCrash=return()"
+ run_sql "DROP DATABASE add_index;"
+ rm -rf /tmp/add-index-by-sql-checkpoint.pb
+ run_lightning --enable-checkpoint=1 --config "$CUR/config2-$tp.toml" --log-file "$LOG_FILE2"
+ grep -Fq "task canceled" "$LOG_FILE2"
+
+ unset GO_FAILPOINTS
+ run_lightning --enable-checkpoint=1 --config "$CUR/config2-$tp.toml" --log-file "$LOG_FILE2"
+ actual_multi_indexes_kvs=$(run_sql "ADMIN CHECKSUM TABLE add_index.multi_indexes;" | grep "Total_kvs" | awk '{print $2}')
+ actual_multi_indexes_cksum=$(run_sql "ADMIN CHECKSUM TABLE add_index.multi_indexes;" | grep "Checksum_crc64_xor" | awk '{print $2}')
+ actual_non_pk_auto_inc_kvs=$(run_sql "ADMIN CHECKSUM TABLE add_index.non_pk_auto_inc;" | grep "Total_kvs" | awk '{print $2}')
+ actual_non_pk_auto_inc_cksum=$(run_sql "ADMIN CHECKSUM TABLE add_index.non_pk_auto_inc;" | grep "Checksum_crc64_xor" | awk '{print $2}')
+
+ set -x
+ [ "$multi_indexes_kvs" == "$actual_multi_indexes_kvs" ] && [ "$multi_indexes_cksum" == "$actual_multi_indexes_cksum" ]
+ [ "$non_pk_auto_inc_kvs" == "$actual_non_pk_auto_inc_kvs" ] && [ "$non_pk_auto_inc_cksum" == "$actual_non_pk_auto_inc_cksum" ]
+}
+recover_from_checkpoint file
+recover_from_checkpoint mysql
diff --git a/lightning/tests/lightning_distributed_import/run.sh b/lightning/tests/lightning_distributed_import/run.sh
index d376d009c9f30..d331f756f1353 100644
--- a/lightning/tests/lightning_distributed_import/run.sh
+++ b/lightning/tests/lightning_distributed_import/run.sh
@@ -22,12 +22,15 @@ LOG_FILE1="$TEST_DIR/lightning-distributed-import1.log"
LOG_FILE2="$TEST_DIR/lightning-distributed-import2.log"
# let lightning run a bit slow to avoid some table in the first lightning finish too fast.
-export GO_FAILPOINTS="github.com/pingcap/tidb/lightning/pkg/importer/SlowDownImport=sleep(250)"
+export GO_FAILPOINTS="github.com/pingcap/tidb/lightning/pkg/importer/SlowDownImport=sleep(2500)"
run_lightning --backend local --sorted-kv-dir "$TEST_DIR/lightning_distributed_import.sorted1" \
-d "$CUR/data1" --log-file "$LOG_FILE1" --config "$CUR/config.toml" &
pid1="$!"
+# sleep 1 second to avoid both lightning starting at the same time and have same ID.
+sleep 1
+
run_lightning --backend local --sorted-kv-dir "$TEST_DIR/lightning_distributed_import.sorted2" \
-d "$CUR/data2" --log-file "$LOG_FILE2" --config "$CUR/config.toml" &
pid2="$!"
diff --git a/lightning/tests/lightning_pd_leader_switch/run.sh b/lightning/tests/lightning_pd_leader_switch/run.sh
index 67035c6056d70..d2c8f38165bc8 100644
--- a/lightning/tests/lightning_pd_leader_switch/run.sh
+++ b/lightning/tests/lightning_pd_leader_switch/run.sh
@@ -20,6 +20,7 @@ set -eu
cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
. $UTILS_DIR/run_services
+echo "will start pd2"
pd-server --join "https://$PD_ADDR" \
--client-urls "https://${PD_ADDR}2" \
--peer-urls "https://${PD_PEER_ADDR}2" \
@@ -29,8 +30,9 @@ pd-server --join "https://$PD_ADDR" \
--config $PD_CONFIG &
# strange that new PD can't join too quickly
-sleep 10
+sleep 20
+echo "will start pd3"
pd-server --join "https://$PD_ADDR" \
--client-urls "https://${PD_ADDR}3" \
--peer-urls "https://${PD_PEER_ADDR}3" \
diff --git a/lightning/tests/run_group_lightning_tests.sh b/lightning/tests/run_group_lightning_tests.sh
index 639828d55931e..2c7d8b563be8c 100755
--- a/lightning/tests/run_group_lightning_tests.sh
+++ b/lightning/tests/run_group_lightning_tests.sh
@@ -42,7 +42,7 @@ groups=(
["G05"]='lightning_fail_fast lightning_fail_fast_on_nonretry_err lightning_file_routing lightning_foreign_key lightning_gcs lightning_generated_columns lightning_ignore_columns lightning_import_compress lightning_incremental lightning_issue_282 lightning_issue_40657 lightning_issue_410 lightning_issue_519 lightning_local_backend lightning_max_incr'
["G06"]='lightning_max_random lightning_multi_valued_index lightning_new_collation lightning_no_schema lightning_parquet lightning_partition_incremental lightning_partitioned-table lightning_record_network lightning_reload_cert lightning_restore lightning_routes lightning_routes_panic lightning_row-format-v2 lightning_s3'
["G07"]='lightning_shard_rowid lightning_source_linkfile lightning_sqlmode lightning_tidb_duplicate_data lightning_tidb_rowid lightning_tiflash lightning_tikv_multi_rocksdb lightning_too_many_columns lightning_tool_135'
- ["G08"]='lightning_tool_1420 lightning_tool_1472 lightning_tool_241 lightning_ttl lightning_unused_config_keys lightning_various_types lightning_view lightning_write_batch lightning_write_limit lightning_pd_leader_switch lightning_add_index lightning_alter_random lightning_auto_columns'
+ ["G08"]='lightning_pd_leader_switch lightning_tool_1420 lightning_tool_1472 lightning_tool_241 lightning_ttl lightning_unused_config_keys lightning_various_types lightning_view lightning_write_batch lightning_write_limit lightning_add_index lightning_alter_random lightning_auto_columns'
)
# Get other lightning cases not in groups, to avoid missing any case
diff --git a/pkg/bindinfo/BUILD.bazel b/pkg/bindinfo/BUILD.bazel
index 0f56829622c43..5732b1a6f82d4 100644
--- a/pkg/bindinfo/BUILD.bazel
+++ b/pkg/bindinfo/BUILD.bazel
@@ -29,6 +29,7 @@ go_library(
"//pkg/sessionctx/variable",
"//pkg/types",
"//pkg/types/parser_driver",
+ "//pkg/util",
"//pkg/util/chunk",
"//pkg/util/hack",
"//pkg/util/hint",
@@ -41,7 +42,6 @@ go_library(
"//pkg/util/stmtsummary/v2:stmtsummary",
"//pkg/util/stringutil",
"//pkg/util/table-filter",
- "@com_github_ngaut_pools//:pools",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
"@org_golang_x_sync//singleflight",
diff --git a/pkg/bindinfo/capture.go b/pkg/bindinfo/capture.go
index e1f7fc8dc3681..d98e5c563b470 100644
--- a/pkg/bindinfo/capture.go
+++ b/pkg/bindinfo/capture.go
@@ -194,7 +194,7 @@ func (h *globalBindingHandle) CaptureBaselines() {
SQLDigest: digest.String(),
}
// We don't need to pass the `sctx` because the BindSQL has been validated already.
- err = h.CreateGlobalBinding(nil, binding)
+ err = h.CreateGlobalBinding(nil, []*Binding{&binding})
if err != nil {
logutil.BindLogger().Debug("create bind record failed in baseline capture", zap.String("SQL", bindableStmt.Query), zap.Error(err))
}
diff --git a/pkg/bindinfo/capture_test.go b/pkg/bindinfo/capture_test.go
index d9ea79524a01d..dfda50c3d12ed 100644
--- a/pkg/bindinfo/capture_test.go
+++ b/pkg/bindinfo/capture_test.go
@@ -418,7 +418,7 @@ func TestUpdateSubqueryCapture(t *testing.T) {
tk.MustExec("admin capture bindings")
rows := tk.MustQuery("show global bindings").Rows()
require.Len(t, rows, 1)
- bindSQL := "UPDATE /*+ hash_join(@`upd_1` `test`.`t1`), use_index(@`upd_1` `test`.`t1` `idx_b`), no_order_index(@`upd_1` `test`.`t1` `idx_b`), use_index(@`sel_1` `test`.`t2` ), use_index(@`sel_2` `test`.`t2` )*/ `test`.`t1` SET `b`=1 WHERE `b` = 2 AND (`a` IN (SELECT `a` FROM `test`.`t2` WHERE `b` = 1) OR `c` IN (SELECT `a` FROM `test`.`t2` WHERE `b` = 1))"
+ bindSQL := "UPDATE /*+ hash_join(`test`.`t2`@`sel_2`), hash_join(`test`.`t1`), use_index(@`upd_1` `test`.`t1` `idx_b`), no_order_index(@`upd_1` `test`.`t1` `idx_b`), use_index(@`sel_1` `test`.`t2` ), use_index(@`sel_2` `test`.`t2` )*/ `test`.`t1` SET `b`=1 WHERE `b` = 2 AND (`a` IN (SELECT `a` FROM `test`.`t2` WHERE `b` = 1) OR `c` IN (SELECT `a` FROM `test`.`t2` WHERE `b` = 1))"
originSQL := "UPDATE `test`.`t1` SET `b`=1 WHERE `b` = 2 AND (`a` IN (SELECT `a` FROM `test`.`t2` WHERE `b` = 1) OR `c` IN (SELECT `a` FROM `test`.`t2` WHERE `b` = 1))"
require.Equal(t, bindSQL, rows[0][1])
tk.MustExec(originSQL)
diff --git a/pkg/bindinfo/global_handle.go b/pkg/bindinfo/global_handle.go
index ec3b39c2def3e..a9dff9b5701ca 100644
--- a/pkg/bindinfo/global_handle.go
+++ b/pkg/bindinfo/global_handle.go
@@ -35,6 +35,7 @@ import (
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/types"
driver "github.com/pingcap/tidb/pkg/types/parser_driver"
+ "github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/chunk"
"github.com/pingcap/tidb/pkg/util/hint"
utilparser "github.com/pingcap/tidb/pkg/util/parser"
@@ -54,10 +55,10 @@ type GlobalBindingHandle interface {
// CreateGlobalBinding creates a Bindings to the storage and the cache.
// It replaces all the exists bindings for the same normalized SQL.
- CreateGlobalBinding(sctx sessionctx.Context, binding Binding) (err error)
+ CreateGlobalBinding(sctx sessionctx.Context, bindings []*Binding) (err error)
// DropGlobalBinding drop Bindings to the storage and Bindings int the cache.
- DropGlobalBinding(sqlDigest string) (deletedRows uint64, err error)
+ DropGlobalBinding(sqlDigests []string) (deletedRows uint64, err error)
// SetGlobalBindingStatus set a Bindings's status to the storage and bind cache.
SetGlobalBindingStatus(newStatus, sqlDigest string) (ok bool, err error)
@@ -109,7 +110,7 @@ type GlobalBindingHandle interface {
// globalBindingHandle is used to handle all global sql bind operations.
type globalBindingHandle struct {
- sPool SessionPool
+ sPool util.SessionPool
fuzzyBindingCache atomic.Value
@@ -149,7 +150,7 @@ const (
)
// NewGlobalBindingHandle creates a new GlobalBindingHandle.
-func NewGlobalBindingHandle(sPool SessionPool) GlobalBindingHandle {
+func NewGlobalBindingHandle(sPool util.SessionPool) GlobalBindingHandle {
handle := &globalBindingHandle{sPool: sPool}
handle.Reset()
return handle
@@ -255,9 +256,11 @@ func (h *globalBindingHandle) LoadFromStorageToCache(fullLoad bool) (err error)
// CreateGlobalBinding creates a Bindings to the storage and the cache.
// It replaces all the exists bindings for the same normalized SQL.
-func (h *globalBindingHandle) CreateGlobalBinding(sctx sessionctx.Context, binding Binding) (err error) {
- if err := prepareHints(sctx, &binding); err != nil {
- return err
+func (h *globalBindingHandle) CreateGlobalBinding(sctx sessionctx.Context, bindings []*Binding) (err error) {
+ for _, binding := range bindings {
+ if err := prepareHints(sctx, binding); err != nil {
+ return err
+ }
}
defer func() {
if err == nil {
@@ -271,63 +274,90 @@ func (h *globalBindingHandle) CreateGlobalBinding(sctx sessionctx.Context, bindi
return err
}
- now := types.NewTime(types.FromGoTime(time.Now()), mysql.TypeTimestamp, 3)
-
- updateTs := now.String()
- _, err = exec(sctx, `UPDATE mysql.bind_info SET status = %?, update_time = %? WHERE original_sql = %? AND update_time < %?`,
- deleted, updateTs, binding.OriginalSQL, updateTs)
- if err != nil {
- return err
- }
+ for i, binding := range bindings {
+ now := types.NewTime(types.FromGoTime(time.Now()), mysql.TypeTimestamp, 3)
+
+ updateTs := now.String()
+ _, err = exec(
+ sctx,
+ `UPDATE mysql.bind_info SET status = %?, update_time = %? WHERE original_sql = %? AND update_time < %?`,
+ deleted,
+ updateTs,
+ binding.OriginalSQL,
+ updateTs,
+ )
+ if err != nil {
+ return err
+ }
- binding.CreateTime = now
- binding.UpdateTime = now
-
- // Insert the Bindings to the storage.
- _, err = exec(sctx, `INSERT INTO mysql.bind_info VALUES (%?,%?, %?, %?, %?, %?, %?, %?, %?, %?, %?)`,
- binding.OriginalSQL,
- binding.BindSQL,
- strings.ToLower(binding.Db),
- binding.Status,
- binding.CreateTime.String(),
- binding.UpdateTime.String(),
- binding.Charset,
- binding.Collation,
- binding.Source,
- binding.SQLDigest,
- binding.PlanDigest,
- )
- if err != nil {
- return err
+ binding.CreateTime = now
+ binding.UpdateTime = now
+
+ // Insert the Bindings to the storage.
+ _, err = exec(
+ sctx,
+ `INSERT INTO mysql.bind_info VALUES (%?,%?, %?, %?, %?, %?, %?, %?, %?, %?, %?)`,
+ binding.OriginalSQL,
+ binding.BindSQL,
+ strings.ToLower(binding.Db),
+ binding.Status,
+ binding.CreateTime.String(),
+ binding.UpdateTime.String(),
+ binding.Charset,
+ binding.Collation,
+ binding.Source,
+ binding.SQLDigest,
+ binding.PlanDigest,
+ )
+ failpoint.Inject("CreateGlobalBindingNthFail", func(val failpoint.Value) {
+ n := val.(int)
+ if n == i {
+ err = errors.NewNoStackError("An injected error")
+ }
+ })
+ if err != nil {
+ return err
+ }
}
return nil
})
}
// dropGlobalBinding drops a Bindings to the storage and Bindings int the cache.
-func (h *globalBindingHandle) dropGlobalBinding(sqlDigest string) (deletedRows uint64, err error) {
- err = h.callWithSCtx(false, func(sctx sessionctx.Context) error {
+func (h *globalBindingHandle) dropGlobalBinding(sqlDigests []string) (deletedRows uint64, err error) {
+ err = h.callWithSCtx(true, func(sctx sessionctx.Context) error {
// Lock mysql.bind_info to synchronize with CreateBinding / AddBinding / DropBinding on other tidb instances.
if err = lockBindInfoTable(sctx); err != nil {
return err
}
- updateTs := types.NewTime(types.FromGoTime(time.Now()), mysql.TypeTimestamp, 3).String()
-
- _, err = exec(sctx, `UPDATE mysql.bind_info SET status = %?, update_time = %? WHERE sql_digest = %? AND update_time < %? AND status != %?`,
- deleted, updateTs, sqlDigest, updateTs, deleted)
- if err != nil {
- return err
+ for _, sqlDigest := range sqlDigests {
+ updateTs := types.NewTime(types.FromGoTime(time.Now()), mysql.TypeTimestamp, 3).String()
+ _, err = exec(
+ sctx,
+ `UPDATE mysql.bind_info SET status = %?, update_time = %? WHERE sql_digest = %? AND update_time < %? AND status != %?`,
+ deleted,
+ updateTs,
+ sqlDigest,
+ updateTs,
+ deleted,
+ )
+ if err != nil {
+ return err
+ }
+ deletedRows += sctx.GetSessionVars().StmtCtx.AffectedRows()
}
- deletedRows = sctx.GetSessionVars().StmtCtx.AffectedRows()
return nil
})
+ if err != nil {
+ deletedRows = 0
+ }
return
}
// DropGlobalBinding drop Bindings to the storage and Bindings int the cache.
-func (h *globalBindingHandle) DropGlobalBinding(sqlDigest string) (deletedRows uint64, err error) {
- if sqlDigest == "" {
+func (h *globalBindingHandle) DropGlobalBinding(sqlDigests []string) (deletedRows uint64, err error) {
+ if len(sqlDigests) == 0 {
return 0, errors.New("sql digest is empty")
}
defer func() {
@@ -335,7 +365,7 @@ func (h *globalBindingHandle) DropGlobalBinding(sqlDigest string) (deletedRows u
err = h.LoadFromStorageToCache(false)
}
}()
- return h.dropGlobalBinding(sqlDigest)
+ return h.dropGlobalBinding(sqlDigests)
}
// SetGlobalBindingStatus set a Bindings's status to the storage and bind cache.
@@ -451,7 +481,7 @@ func (h *globalBindingHandle) DropInvalidGlobalBinding() {
invalidBindings := h.invalidBindings.getAll()
h.invalidBindings.reset()
for _, invalidBinding := range invalidBindings {
- if _, err := h.dropGlobalBinding(invalidBinding.SQLDigest); err != nil {
+ if _, err := h.dropGlobalBinding([]string{invalidBinding.SQLDigest}); err != nil {
logutil.BindLogger().Debug("flush bind record failed", zap.Error(err))
}
}
diff --git a/pkg/bindinfo/global_handle_test.go b/pkg/bindinfo/global_handle_test.go
index 3325e9be37898..2d11516cb84f8 100644
--- a/pkg/bindinfo/global_handle_test.go
+++ b/pkg/bindinfo/global_handle_test.go
@@ -609,5 +609,6 @@ func (p *mockSessionPool) Get() (pools.Resource, error) {
return p.se, nil
}
-func (p *mockSessionPool) Put(pools.Resource) {
-}
+func (p *mockSessionPool) Put(pools.Resource) {}
+
+func (p *mockSessionPool) Close() {}
diff --git a/pkg/bindinfo/session_handle.go b/pkg/bindinfo/session_handle.go
index 45e19adc64173..770b372fb4ef1 100644
--- a/pkg/bindinfo/session_handle.go
+++ b/pkg/bindinfo/session_handle.go
@@ -17,6 +17,7 @@ package bindinfo
import (
"context"
"encoding/json"
+ "slices"
"strings"
"time"
@@ -35,10 +36,10 @@ import (
// SessionBindingHandle is used to handle all session sql bind operations.
type SessionBindingHandle interface {
// CreateSessionBinding creates a binding to the cache.
- CreateSessionBinding(sctx sessionctx.Context, binding Binding) (err error)
+ CreateSessionBinding(sctx sessionctx.Context, bindings []*Binding) (err error)
// DropSessionBinding drops a binding by the sql digest.
- DropSessionBinding(sqlDigest string) error
+ DropSessionBinding(sqlDigests []string) error
// MatchSessionBinding returns the matched binding for this statement.
MatchSessionBinding(sctx sessionctx.Context, fuzzyDigest string, tableNames []*ast.TableName) (matchedBinding Binding, isMatched bool)
@@ -75,26 +76,36 @@ func (h *sessionBindingHandle) appendSessionBinding(sqlDigest string, meta Bindi
// CreateSessionBinding creates a Bindings to the cache.
// It replaces all the exists bindings for the same normalized SQL.
-func (h *sessionBindingHandle) CreateSessionBinding(sctx sessionctx.Context, binding Binding) (err error) {
- if err := prepareHints(sctx, &binding); err != nil {
- return err
+func (h *sessionBindingHandle) CreateSessionBinding(sctx sessionctx.Context, bindings []*Binding) (err error) {
+ for _, binding := range bindings {
+ if err := prepareHints(sctx, binding); err != nil {
+ return err
+ }
+ }
+ for _, binding := range bindings {
+ binding.Db = strings.ToLower(binding.Db)
+ now := types.NewTime(
+ types.FromGoTime(time.Now().In(sctx.GetSessionVars().StmtCtx.TimeZone())),
+ mysql.TypeTimestamp,
+ 3,
+ )
+ binding.CreateTime = now
+ binding.UpdateTime = now
+
+ // update the BindMeta to the cache.
+ h.appendSessionBinding(parser.DigestNormalized(binding.OriginalSQL).String(), []Binding{*binding})
}
- binding.Db = strings.ToLower(binding.Db)
- now := types.NewTime(types.FromGoTime(time.Now().In(sctx.GetSessionVars().StmtCtx.TimeZone())), mysql.TypeTimestamp, 3)
- binding.CreateTime = now
- binding.UpdateTime = now
-
- // update the BindMeta to the cache.
- h.appendSessionBinding(parser.DigestNormalized(binding.OriginalSQL).String(), []Binding{binding})
return nil
}
// DropSessionBinding drop Bindings in the cache.
-func (h *sessionBindingHandle) DropSessionBinding(sqlDigest string) error {
- if sqlDigest == "" {
+func (h *sessionBindingHandle) DropSessionBinding(sqlDigests []string) error {
+ if slices.Contains(sqlDigests, "") {
return errors.New("sql digest is empty")
}
- h.ch.RemoveBinding(sqlDigest)
+ for _, sqlDigest := range sqlDigests {
+ h.ch.RemoveBinding(sqlDigest)
+ }
return nil
}
diff --git a/pkg/bindinfo/tests/BUILD.bazel b/pkg/bindinfo/tests/BUILD.bazel
index e28e02aceb44d..0629e045c6169 100644
--- a/pkg/bindinfo/tests/BUILD.bazel
+++ b/pkg/bindinfo/tests/BUILD.bazel
@@ -9,7 +9,7 @@ go_test(
],
flaky = True,
race = "on",
- shard_count = 18,
+ shard_count = 19,
deps = [
"//pkg/bindinfo",
"//pkg/bindinfo/internal",
diff --git a/pkg/bindinfo/tests/bind_test.go b/pkg/bindinfo/tests/bind_test.go
index 53f6346db9367..9dd948861bb3c 100644
--- a/pkg/bindinfo/tests/bind_test.go
+++ b/pkg/bindinfo/tests/bind_test.go
@@ -17,7 +17,9 @@ package tests
import (
"context"
"fmt"
+ "math/rand"
"strconv"
+ "strings"
"testing"
"github.com/pingcap/tidb/pkg/bindinfo"
@@ -893,12 +895,42 @@ func removeAllBindings(tk *testkit.TestKit, global bool) {
scope = "global"
}
res := showBinding(tk, fmt.Sprintf("show %v bindings", scope))
+ digests := make([]string, 0, len(res))
for _, r := range res {
if r[4] == "builtin" {
continue
}
- tk.MustExec(fmt.Sprintf("drop %v binding for sql digest '%v'", scope, r[5]))
+ digests = append(digests, r[5].(string))
}
+ if len(digests) == 0 {
+ return
+ }
+ // test DROP BINDING FOR SQL DIGEST can handle empty strings correctly
+ digests = append(digests, "", "", "")
+ // randomly split digests into 4 groups using random number
+ // shuffle the slice
+ rand.Shuffle(len(digests), func(i, j int) {
+ digests[i], digests[j] = digests[j], digests[i]
+ })
+ split := make([][]string, 4)
+ for i, d := range digests {
+ split[i%4] = append(split[i%4], d)
+ }
+ // group 0: wrap with ' then connect by ,
+ var g0 string
+ for _, d := range split[0] {
+ g0 += "'" + d + "',"
+ }
+ // group 1: connect by , and set into a user variable
+ tk.MustExec(fmt.Sprintf("set @a = '%v'", strings.Join(split[1], ",")))
+ g1 := "@a,"
+ var g2 string
+ for _, d := range split[2] {
+ g2 += "'" + d + "',"
+ }
+ // group 2: connect by , and put into a normal string
+ g3 := "'" + strings.Join(split[3], ",") + "'"
+ tk.MustExec(fmt.Sprintf("drop %v binding for sql digest %s %s %s %s", scope, g0, g1, g2, g3))
tk.MustQuery(fmt.Sprintf("show %v bindings", scope)).Check(testkit.Rows()) // empty
}
@@ -1107,3 +1139,31 @@ func TestFuzzyBindingHintsWithSourceReturning(t *testing.T) {
}
}
}
+
+func TestBatchDropBindings(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec(`use test`)
+ tk.MustExec(`create table t1 (a int, b int, c int, d int, key(a), key(b), key(c), key(d))`)
+ tk.MustExec(`create table t2 (a int, b int, c int, d int, key(a), key(b), key(c), key(d))`)
+ tk.MustExec(`create table t3 (a int, b int, c int, d int, key(a), key(b), key(c), key(d))`)
+ tk.MustExec(`create global binding for select * from t1 using select /*+ use_index(t1, a) */ * from t1`)
+ tk.MustExec(`create global binding for select * from t1 where b < 1 using select /*+ use_index(t1,b) */ * from t1 where b < 1`)
+ tk.MustExec(`create global binding for select * from t1 where c < 1 using select /*+ use_index(t1,c) */ * from t1 where c < 1`)
+ tk.MustExec(`create global binding for select * from t1 join t2 on t1.a = t2.a using select /*+ hash_join(t1) */ * from t1 join t2 on t1.a = t2.a`)
+ tk.MustExec(`create global binding for select * from t1 join t2 on t1.a = t2.a join t3 on t2.b = t3.b where t1.a = 1 using select /*+ leading(t3,t2,t1) */ * from t1 join t2 on t1.a = t2.a join t3 on t2.b = t3.b where t1.a = 1`)
+ tk.MustExec(`create global binding for select * from t1 where a in (select sum(b) from t2) using select /*+ agg_to_cop(@sel_2) */ * from t1 where a in (select sum(b) from t2)`)
+ tk.MustExec(`create global binding for select * from t2 where a = 1 and b = 2 and c = 3 using select * from t2 ignore index (b) where a = 1 and b = 2 and c = 3`)
+ tk.MustExec(`create global binding for select * from t2 where a = 1 and b = 2 and c = 3 using select * from t2 use index (b) where a = 1 and b = 2 and c = 3`)
+
+ tk.MustExec(`create session binding for select * from t1 using select /*+ use_index(t1, a) */ * from t1`)
+ tk.MustExec(`create session binding for select * from t1 where b < 1 using select /*+ use_index(t1, b) */ * from t1 where b < 1`)
+ tk.MustExec(`create session binding for select * from t1 where c < 1 using select /*+ use_index(t1, c) */ * from t1 where c < 1`)
+ tk.MustExec(`create session binding for select * from t1 join t2 on t1.a = t2.a using select /*+ hash_join( t1) */ * from t1 join t2 on t1.a = t2.a`)
+ tk.MustExec(`create session binding for select * from t1 join t2 on t1.a = t2.a join t3 on t2.b = t3.b where t1. a = 1 using select /*+ leading(t3,t2,t1) */ * from t1 join t2 on t1.a = t2.a join t3 on t2.b = t3.b where t1.a = 1`)
+ tk.MustExec(`create session binding for select * from t1 where a in (select sum( b) from t2) using select /*+ agg_to_cop(@sel_2) */ * from t1 where a in (select sum(b) from t2)`)
+ tk.MustExec(`create session binding for select * from t2 where a = 1 and b = 2 and c = 3 using select * from t2 ignore index (b) where a = 1 and b = 2 and c = 3`)
+ tk.MustExec(`create session binding for select * from t2 where a = 1 and b = 2 and c = 3 using select * from t2 use index (b) where a = 1 and b = 2 and c = 3`)
+ removeAllBindings(tk, true)
+ removeAllBindings(tk, false)
+}
diff --git a/pkg/bindinfo/util.go b/pkg/bindinfo/util.go
index f3278f95a46df..b48d73911126d 100644
--- a/pkg/bindinfo/util.go
+++ b/pkg/bindinfo/util.go
@@ -17,7 +17,6 @@ package bindinfo
import (
"context"
- "github.com/ngaut/pools"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/sessionctx"
@@ -37,9 +36,3 @@ func execRows(sctx sessionctx.Context, sql string, args ...any) (rows []chunk.Ro
return sqlExec.ExecRestrictedSQL(kv.WithInternalSourceType(context.Background(), kv.InternalTxnBindInfo),
[]sqlexec.OptionFuncAlias{sqlexec.ExecOptionUseCurSession}, sql, args...)
}
-
-// SessionPool is used to recycle sessionctx.
-type SessionPool interface {
- Get() (pools.Resource, error)
- Put(pools.Resource)
-}
diff --git a/pkg/config/config.go b/pkg/config/config.go
index 3534f06b0a2e6..1fbec8af7b27d 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -98,7 +98,8 @@ const (
// EnvVarKeyspaceName is the system env name for keyspace name.
EnvVarKeyspaceName = "KEYSPACE_NAME"
// MaxTokenLimit is the max token limit value.
- MaxTokenLimit = 1024 * 1024
+ MaxTokenLimit = 1024 * 1024
+ DefSchemaLease = 45 * time.Second
)
// Valid config maps
@@ -716,15 +717,16 @@ type Performance struct {
MemProfileInterval string `toml:"-" json:"-"`
// Deprecated: this config will not have any effect
- IndexUsageSyncLease string `toml:"index-usage-sync-lease" json:"index-usage-sync-lease"`
- PlanReplayerGCLease string `toml:"plan-replayer-gc-lease" json:"plan-replayer-gc-lease"`
- GOGC int `toml:"gogc" json:"gogc"`
- EnforceMPP bool `toml:"enforce-mpp" json:"enforce-mpp"`
- StatsLoadConcurrency int `toml:"stats-load-concurrency" json:"stats-load-concurrency"`
- StatsLoadQueueSize uint `toml:"stats-load-queue-size" json:"stats-load-queue-size"`
- AnalyzePartitionConcurrencyQuota uint `toml:"analyze-partition-concurrency-quota" json:"analyze-partition-concurrency-quota"`
- PlanReplayerDumpWorkerConcurrency uint `toml:"plan-replayer-dump-worker-concurrency" json:"plan-replayer-dump-worker-concurrency"`
- EnableStatsCacheMemQuota bool `toml:"enable-stats-cache-mem-quota" json:"enable-stats-cache-mem-quota"`
+ IndexUsageSyncLease string `toml:"index-usage-sync-lease" json:"index-usage-sync-lease"`
+ PlanReplayerGCLease string `toml:"plan-replayer-gc-lease" json:"plan-replayer-gc-lease"`
+ GOGC int `toml:"gogc" json:"gogc"`
+ EnforceMPP bool `toml:"enforce-mpp" json:"enforce-mpp"`
+ StatsLoadConcurrency int `toml:"stats-load-concurrency" json:"stats-load-concurrency"`
+ StatsLoadQueueSize uint `toml:"stats-load-queue-size" json:"stats-load-queue-size"`
+ // Deprecated: this config has been deprecated. It has no effect.
+ AnalyzePartitionConcurrencyQuota uint `toml:"analyze-partition-concurrency-quota" json:"analyze-partition-concurrency-quota"`
+ PlanReplayerDumpWorkerConcurrency uint `toml:"plan-replayer-dump-worker-concurrency" json:"plan-replayer-dump-worker-concurrency"`
+ EnableStatsCacheMemQuota bool `toml:"enable-stats-cache-mem-quota" json:"enable-stats-cache-mem-quota"`
// The following items are deprecated. We need to keep them here temporarily
// to support the upgrade process. They can be removed in future.
@@ -912,7 +914,7 @@ var defaultConf = Config{
Path: "/tmp/tidb",
RunDDL: true,
SplitTable: true,
- Lease: "45s",
+ Lease: DefSchemaLease.String(),
TokenLimit: 1000,
OOMUseTmpStorage: true,
TempDir: DefTempDir,
diff --git a/pkg/ddl/BUILD.bazel b/pkg/ddl/BUILD.bazel
index 4f2e674ea5e68..39650b0c76155 100644
--- a/pkg/ddl/BUILD.bazel
+++ b/pkg/ddl/BUILD.bazel
@@ -11,6 +11,7 @@ package_group(
go_library(
name = "ddl",
srcs = [
+ "add_column.go",
"backfilling.go",
"backfilling_clean_s3.go",
"backfilling_dist_executor.go",
@@ -21,17 +22,16 @@ go_library(
"backfilling_read_index.go",
"backfilling_scheduler.go",
"bdr.go",
- "callback.go",
"cluster.go",
"column.go",
"constant.go",
"constraint.go",
+ "create_table.go",
"ddl.go",
"ddl_algorithm.go",
"ddl_history.go",
"ddl_running_jobs.go",
"ddl_tiflash_api.go",
- "ddl_worker.go",
"ddl_workerpool.go",
"delete_range.go",
"delete_range_util.go",
@@ -43,8 +43,11 @@ go_library(
"index.go",
"index_cop.go",
"index_merge_tmp.go",
- "job_table.go",
+ "job_scheduler.go",
+ "job_submitter.go",
+ "job_worker.go",
"mock.go",
+ "modify_column.go",
"multi_schema_change.go",
"options.go",
"partition.go",
@@ -75,8 +78,9 @@ go_library(
"//pkg/ddl/logutil",
"//pkg/ddl/placement",
"//pkg/ddl/resourcegroup",
+ "//pkg/ddl/schemaver",
+ "//pkg/ddl/serverstate",
"//pkg/ddl/session",
- "//pkg/ddl/syncer",
"//pkg/ddl/systable",
"//pkg/ddl/util",
"//pkg/distsql",
@@ -176,7 +180,6 @@ go_library(
"@com_github_pingcap_kvproto//pkg/errorpb",
"@com_github_pingcap_kvproto//pkg/kvrpcpb",
"@com_github_pingcap_kvproto//pkg/metapb",
- "@com_github_pingcap_log//:log",
"@com_github_pingcap_tipb//go-tipb",
"@com_github_prometheus_client_golang//prometheus",
"@com_github_tikv_client_go_v2//error",
@@ -189,7 +192,6 @@ go_library(
"@com_github_tikv_pd_client//:client",
"@com_github_tikv_pd_client//http",
"@io_etcd_go_etcd_client_v3//:client",
- "@io_etcd_go_etcd_client_v3//concurrency",
"@org_golang_x_sync//errgroup",
"@org_uber_go_atomic//:atomic",
"@org_uber_go_zap//:zap",
@@ -224,8 +226,8 @@ go_test(
"ddl_history_test.go",
"ddl_running_jobs_test.go",
"ddl_test.go",
- "ddl_worker_test.go",
"ddl_workerpool_test.go",
+ "executor_nokit_test.go",
"executor_test.go",
"export_test.go",
"fail_test.go",
@@ -236,7 +238,8 @@ go_test(
"index_test.go",
"integration_test.go",
"job_scheduler_test.go",
- "job_table_test.go",
+ "job_scheduler_testkit_test.go",
+ "job_worker_test.go",
"main_test.go",
"modify_column_test.go",
"multi_schema_change_test.go",
@@ -272,14 +275,14 @@ go_test(
"//pkg/ddl/mock",
"//pkg/ddl/placement",
"//pkg/ddl/schematracker",
+ "//pkg/ddl/serverstate",
"//pkg/ddl/session",
- "//pkg/ddl/syncer",
"//pkg/ddl/testutil",
"//pkg/ddl/util",
- "//pkg/ddl/util/callback",
"//pkg/disttask/framework/proto",
"//pkg/disttask/framework/scheduler",
"//pkg/disttask/framework/storage",
+ "//pkg/disttask/operator",
"//pkg/domain",
"//pkg/domain/infosync",
"//pkg/errctx",
diff --git a/pkg/ddl/add_column.go b/pkg/ddl/add_column.go
new file mode 100644
index 0000000000000..7b55caf5160ca
--- /dev/null
+++ b/pkg/ddl/add_column.go
@@ -0,0 +1,1294 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ddl
+
+import (
+ "fmt"
+ "strconv"
+ "strings"
+ "time"
+ "unicode/utf8"
+
+ "github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
+ "github.com/pingcap/tidb/pkg/config"
+ "github.com/pingcap/tidb/pkg/ddl/logutil"
+ "github.com/pingcap/tidb/pkg/errctx"
+ "github.com/pingcap/tidb/pkg/expression"
+ exprctx "github.com/pingcap/tidb/pkg/expression/context"
+ "github.com/pingcap/tidb/pkg/infoschema"
+ "github.com/pingcap/tidb/pkg/meta"
+ "github.com/pingcap/tidb/pkg/meta/autoid"
+ "github.com/pingcap/tidb/pkg/parser/ast"
+ "github.com/pingcap/tidb/pkg/parser/charset"
+ "github.com/pingcap/tidb/pkg/parser/format"
+ "github.com/pingcap/tidb/pkg/parser/model"
+ "github.com/pingcap/tidb/pkg/parser/mysql"
+ "github.com/pingcap/tidb/pkg/parser/terror"
+ field_types "github.com/pingcap/tidb/pkg/parser/types"
+ "github.com/pingcap/tidb/pkg/sessionctx"
+ "github.com/pingcap/tidb/pkg/sessionctx/variable"
+ statsutil "github.com/pingcap/tidb/pkg/statistics/handle/util"
+ "github.com/pingcap/tidb/pkg/table"
+ "github.com/pingcap/tidb/pkg/types"
+ driver "github.com/pingcap/tidb/pkg/types/parser_driver"
+ "github.com/pingcap/tidb/pkg/util/collate"
+ "github.com/pingcap/tidb/pkg/util/dbterror"
+ "github.com/pingcap/tidb/pkg/util/hack"
+ "go.uber.org/zap"
+)
+
+func onAddColumn(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
+ // Handle the rolling back job.
+ if job.IsRollingback() {
+ ver, err = onDropColumn(jobCtx, t, job)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ return ver, nil
+ }
+
+ failpoint.Inject("errorBeforeDecodeArgs", func(val failpoint.Value) {
+ //nolint:forcetypeassert
+ if val.(bool) {
+ failpoint.Return(ver, errors.New("occur an error before decode args"))
+ }
+ })
+
+ tblInfo, columnInfo, colFromArgs, pos, ifNotExists, err := checkAddColumn(t, job)
+ if err != nil {
+ if ifNotExists && infoschema.ErrColumnExists.Equal(err) {
+ job.Warning = toTError(err)
+ job.State = model.JobStateDone
+ return ver, nil
+ }
+ return ver, errors.Trace(err)
+ }
+ if columnInfo == nil {
+ columnInfo = InitAndAddColumnToTable(tblInfo, colFromArgs)
+ logutil.DDLLogger().Info("run add column job", zap.Stringer("job", job), zap.Reflect("columnInfo", *columnInfo))
+ if err = checkAddColumnTooManyColumns(len(tblInfo.Columns)); err != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+ }
+
+ originalState := columnInfo.State
+ switch columnInfo.State {
+ case model.StateNone:
+ // none -> delete only
+ columnInfo.State = model.StateDeleteOnly
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, originalState != columnInfo.State)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ job.SchemaState = model.StateDeleteOnly
+ case model.StateDeleteOnly:
+ // delete only -> write only
+ columnInfo.State = model.StateWriteOnly
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != columnInfo.State)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ // Update the job state when all affairs done.
+ job.SchemaState = model.StateWriteOnly
+ case model.StateWriteOnly:
+ // write only -> reorganization
+ columnInfo.State = model.StateWriteReorganization
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != columnInfo.State)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ // Update the job state when all affairs done.
+ job.SchemaState = model.StateWriteReorganization
+ job.MarkNonRevertible()
+ case model.StateWriteReorganization:
+ // reorganization -> public
+ // Adjust table column offset.
+ failpoint.InjectCall("onAddColumnStateWriteReorg")
+ offset, err := LocateOffsetToMove(columnInfo.Offset, pos, tblInfo)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ tblInfo.MoveColumnInfo(columnInfo.Offset, offset)
+ columnInfo.State = model.StatePublic
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != columnInfo.State)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+
+ // Finish this job.
+ job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tblInfo)
+ addColumnEvent := statsutil.NewAddColumnEvent(
+ job.SchemaID,
+ tblInfo,
+ []*model.ColumnInfo{columnInfo},
+ )
+ asyncNotifyEvent(jobCtx, addColumnEvent)
+ default:
+ err = dbterror.ErrInvalidDDLState.GenWithStackByArgs("column", columnInfo.State)
+ }
+
+ return ver, errors.Trace(err)
+}
+
+func checkAndCreateNewColumn(ctx sessionctx.Context, ti ast.Ident, schema *model.DBInfo, spec *ast.AlterTableSpec, t table.Table, specNewColumn *ast.ColumnDef) (*table.Column, error) {
+ err := checkUnsupportedColumnConstraint(specNewColumn, ti)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ colName := specNewColumn.Name.Name.O
+ // Check whether added column has existed.
+ col := table.FindCol(t.Cols(), colName)
+ if col != nil {
+ err = infoschema.ErrColumnExists.GenWithStackByArgs(colName)
+ if spec.IfNotExists {
+ ctx.GetSessionVars().StmtCtx.AppendNote(err)
+ return nil, nil
+ }
+ return nil, err
+ }
+ if err = checkColumnAttributes(colName, specNewColumn.Tp); err != nil {
+ return nil, errors.Trace(err)
+ }
+ if utf8.RuneCountInString(colName) > mysql.MaxColumnNameLength {
+ return nil, dbterror.ErrTooLongIdent.GenWithStackByArgs(colName)
+ }
+
+ return CreateNewColumn(ctx, schema, spec, t, specNewColumn)
+}
+
+func checkUnsupportedColumnConstraint(col *ast.ColumnDef, ti ast.Ident) error {
+ for _, constraint := range col.Options {
+ switch constraint.Tp {
+ case ast.ColumnOptionAutoIncrement:
+ return dbterror.ErrUnsupportedAddColumn.GenWithStack("unsupported add column '%s' constraint AUTO_INCREMENT when altering '%s.%s'", col.Name, ti.Schema, ti.Name)
+ case ast.ColumnOptionPrimaryKey:
+ return dbterror.ErrUnsupportedAddColumn.GenWithStack("unsupported add column '%s' constraint PRIMARY KEY when altering '%s.%s'", col.Name, ti.Schema, ti.Name)
+ case ast.ColumnOptionUniqKey:
+ return dbterror.ErrUnsupportedAddColumn.GenWithStack("unsupported add column '%s' constraint UNIQUE KEY when altering '%s.%s'", col.Name, ti.Schema, ti.Name)
+ case ast.ColumnOptionAutoRandom:
+ errMsg := fmt.Sprintf(autoid.AutoRandomAlterAddColumn, col.Name, ti.Schema, ti.Name)
+ return dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(errMsg)
+ }
+ }
+
+ return nil
+}
+
+// CreateNewColumn creates a new column according to the column information.
+func CreateNewColumn(ctx sessionctx.Context, schema *model.DBInfo, spec *ast.AlterTableSpec, t table.Table, specNewColumn *ast.ColumnDef) (*table.Column, error) {
+ // If new column is a generated column, do validation.
+ // NOTE: we do check whether the column refers other generated
+ // columns occurring later in a table, but we don't handle the col offset.
+ for _, option := range specNewColumn.Options {
+ if option.Tp == ast.ColumnOptionGenerated {
+ if err := checkIllegalFn4Generated(specNewColumn.Name.Name.L, typeColumn, option.Expr); err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ if option.Stored {
+ return nil, dbterror.ErrUnsupportedOnGeneratedColumn.GenWithStackByArgs("Adding generated stored column through ALTER TABLE")
+ }
+
+ _, dependColNames, err := findDependedColumnNames(schema.Name, t.Meta().Name, specNewColumn)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ if !ctx.GetSessionVars().EnableAutoIncrementInGenerated {
+ if err := checkAutoIncrementRef(specNewColumn.Name.Name.L, dependColNames, t.Meta()); err != nil {
+ return nil, errors.Trace(err)
+ }
+ }
+ duplicateColNames := make(map[string]struct{}, len(dependColNames))
+ for k := range dependColNames {
+ duplicateColNames[k] = struct{}{}
+ }
+ cols := t.Cols()
+
+ if err := checkDependedColExist(dependColNames, cols); err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ if err := verifyColumnGenerationSingle(duplicateColNames, cols, spec.Position); err != nil {
+ return nil, errors.Trace(err)
+ }
+ }
+ // Specially, since sequence has been supported, if a newly added column has a
+ // sequence nextval function as it's default value option, it won't fill the
+ // known rows with specific sequence next value under current add column logic.
+ // More explanation can refer: TestSequenceDefaultLogic's comment in sequence_test.go
+ if option.Tp == ast.ColumnOptionDefaultValue {
+ if f, ok := option.Expr.(*ast.FuncCallExpr); ok {
+ switch f.FnName.L {
+ case ast.NextVal:
+ if _, err := getSequenceDefaultValue(option); err != nil {
+ return nil, errors.Trace(err)
+ }
+ return nil, errors.Trace(dbterror.ErrAddColumnWithSequenceAsDefault.GenWithStackByArgs(specNewColumn.Name.Name.O))
+ case ast.Rand, ast.UUID, ast.UUIDToBin, ast.Replace, ast.Upper:
+ return nil, errors.Trace(dbterror.ErrBinlogUnsafeSystemFunction.GenWithStackByArgs())
+ }
+ }
+ }
+ }
+
+ tableCharset, tableCollate, err := ResolveCharsetCollation(ctx.GetSessionVars(),
+ ast.CharsetOpt{Chs: t.Meta().Charset, Col: t.Meta().Collate},
+ ast.CharsetOpt{Chs: schema.Charset, Col: schema.Collate},
+ )
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ // Ignore table constraints now, they will be checked later.
+ // We use length(t.Cols()) as the default offset firstly, we will change the column's offset later.
+ col, _, err := buildColumnAndConstraint(
+ ctx,
+ len(t.Cols()),
+ specNewColumn,
+ nil,
+ tableCharset,
+ tableCollate,
+ )
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ originDefVal, err := generateOriginDefaultValue(col.ToInfo(), ctx)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ err = col.SetOriginDefaultValue(originDefVal)
+ return col, err
+}
+
+// buildColumnAndConstraint builds table.Column and ast.Constraint from the parameters.
+// outPriKeyConstraint is the primary key constraint out of column definition. For example:
+// `create table t1 (id int , age int, primary key(id));`
+func buildColumnAndConstraint(
+ ctx sessionctx.Context,
+ offset int,
+ colDef *ast.ColumnDef,
+ outPriKeyConstraint *ast.Constraint,
+ tblCharset string,
+ tblCollate string,
+) (*table.Column, []*ast.Constraint, error) {
+ if colName := colDef.Name.Name.L; colName == model.ExtraHandleName.L {
+ return nil, nil, dbterror.ErrWrongColumnName.GenWithStackByArgs(colName)
+ }
+
+ // specifiedCollate refers to the last collate specified in colDef.Options.
+ chs, coll, err := getCharsetAndCollateInColumnDef(ctx.GetSessionVars(), colDef)
+ if err != nil {
+ return nil, nil, errors.Trace(err)
+ }
+ chs, coll, err = ResolveCharsetCollation(ctx.GetSessionVars(),
+ ast.CharsetOpt{Chs: chs, Col: coll},
+ ast.CharsetOpt{Chs: tblCharset, Col: tblCollate},
+ )
+ chs, coll = OverwriteCollationWithBinaryFlag(ctx.GetSessionVars(), colDef, chs, coll)
+ if err != nil {
+ return nil, nil, errors.Trace(err)
+ }
+
+ if err := setCharsetCollationFlenDecimal(colDef.Tp, colDef.Name.Name.O, chs, coll, ctx.GetSessionVars()); err != nil {
+ return nil, nil, errors.Trace(err)
+ }
+ decodeEnumSetBinaryLiteralToUTF8(colDef.Tp, chs)
+ col, cts, err := columnDefToCol(ctx, offset, colDef, outPriKeyConstraint)
+ if err != nil {
+ return nil, nil, errors.Trace(err)
+ }
+ return col, cts, nil
+}
+
+// getCharsetAndCollateInColumnDef will iterate collate in the options, validate it by checking the charset
+// of column definition. If there's no collate in the option, the default collate of column's charset will be used.
+func getCharsetAndCollateInColumnDef(sessVars *variable.SessionVars, def *ast.ColumnDef) (chs, coll string, err error) {
+ chs = def.Tp.GetCharset()
+ coll = def.Tp.GetCollate()
+ if chs != "" && coll == "" {
+ if coll, err = GetDefaultCollation(sessVars, chs); err != nil {
+ return "", "", errors.Trace(err)
+ }
+ }
+ for _, opt := range def.Options {
+ if opt.Tp == ast.ColumnOptionCollate {
+ info, err := collate.GetCollationByName(opt.StrValue)
+ if err != nil {
+ return "", "", errors.Trace(err)
+ }
+ if chs == "" {
+ chs = info.CharsetName
+ } else if chs != info.CharsetName {
+ return "", "", dbterror.ErrCollationCharsetMismatch.GenWithStackByArgs(info.Name, chs)
+ }
+ coll = info.Name
+ }
+ }
+ return
+}
+
+// OverwriteCollationWithBinaryFlag is used to handle the case like
+//
+// CREATE TABLE t (a VARCHAR(255) BINARY) CHARSET utf8 COLLATE utf8_general_ci;
+//
+// The 'BINARY' sets the column collation to *_bin according to the table charset.
+func OverwriteCollationWithBinaryFlag(sessVars *variable.SessionVars, colDef *ast.ColumnDef, chs, coll string) (newChs string, newColl string) {
+ ignoreBinFlag := colDef.Tp.GetCharset() != "" && (colDef.Tp.GetCollate() != "" || containsColumnOption(colDef, ast.ColumnOptionCollate))
+ if ignoreBinFlag {
+ return chs, coll
+ }
+ needOverwriteBinColl := types.IsString(colDef.Tp.GetType()) && mysql.HasBinaryFlag(colDef.Tp.GetFlag())
+ if needOverwriteBinColl {
+ newColl, err := GetDefaultCollation(sessVars, chs)
+ if err != nil {
+ return chs, coll
+ }
+ return chs, newColl
+ }
+ return chs, coll
+}
+
+func setCharsetCollationFlenDecimal(tp *types.FieldType, colName, colCharset, colCollate string, sessVars *variable.SessionVars) error {
+ var err error
+ if typesNeedCharset(tp.GetType()) {
+ tp.SetCharset(colCharset)
+ tp.SetCollate(colCollate)
+ } else {
+ tp.SetCharset(charset.CharsetBin)
+ tp.SetCollate(charset.CharsetBin)
+ }
+
+ // Use default value for flen or decimal when they are unspecified.
+ defaultFlen, defaultDecimal := mysql.GetDefaultFieldLengthAndDecimal(tp.GetType())
+ if tp.GetDecimal() == types.UnspecifiedLength {
+ tp.SetDecimal(defaultDecimal)
+ }
+ if tp.GetFlen() == types.UnspecifiedLength {
+ tp.SetFlen(defaultFlen)
+ if mysql.HasUnsignedFlag(tp.GetFlag()) && tp.GetType() != mysql.TypeLonglong && mysql.IsIntegerType(tp.GetType()) {
+ // Issue #4684: the flen of unsigned integer(except bigint) is 1 digit shorter than signed integer
+ // because it has no prefix "+" or "-" character.
+ tp.SetFlen(tp.GetFlen() - 1)
+ }
+ } else {
+ // Adjust the field type for blob/text types if the flen is set.
+ if err = adjustBlobTypesFlen(tp, colCharset); err != nil {
+ return err
+ }
+ }
+ return checkTooBigFieldLengthAndTryAutoConvert(tp, colName, sessVars)
+}
+
+func decodeEnumSetBinaryLiteralToUTF8(tp *types.FieldType, chs string) {
+ if tp.GetType() != mysql.TypeEnum && tp.GetType() != mysql.TypeSet {
+ return
+ }
+ enc := charset.FindEncoding(chs)
+ for i, elem := range tp.GetElems() {
+ if !tp.GetElemIsBinaryLit(i) {
+ continue
+ }
+ s, err := enc.Transform(nil, hack.Slice(elem), charset.OpDecodeReplace)
+ if err != nil {
+ logutil.DDLLogger().Warn("decode enum binary literal to utf-8 failed", zap.Error(err))
+ }
+ tp.SetElem(i, string(hack.String(s)))
+ }
+ tp.CleanElemIsBinaryLit()
+}
+
+func typesNeedCharset(tp byte) bool {
+ switch tp {
+ case mysql.TypeString, mysql.TypeVarchar, mysql.TypeVarString,
+ mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob,
+ mysql.TypeEnum, mysql.TypeSet:
+ return true
+ }
+ return false
+}
+
+// checkTooBigFieldLengthAndTryAutoConvert will check whether the field length is too big
+// in non-strict mode and varchar column. If it is, will try to adjust to blob or text, see issue #30328
+func checkTooBigFieldLengthAndTryAutoConvert(tp *types.FieldType, colName string, sessVars *variable.SessionVars) error {
+ if sessVars != nil && !sessVars.SQLMode.HasStrictMode() && tp.GetType() == mysql.TypeVarchar {
+ err := types.IsVarcharTooBigFieldLength(tp.GetFlen(), colName, tp.GetCharset())
+ if err != nil && terror.ErrorEqual(types.ErrTooBigFieldLength, err) {
+ tp.SetType(mysql.TypeBlob)
+ if err = adjustBlobTypesFlen(tp, tp.GetCharset()); err != nil {
+ return err
+ }
+ if tp.GetCharset() == charset.CharsetBin {
+ sessVars.StmtCtx.AppendWarning(dbterror.ErrAutoConvert.FastGenByArgs(colName, "VARBINARY", "BLOB"))
+ } else {
+ sessVars.StmtCtx.AppendWarning(dbterror.ErrAutoConvert.FastGenByArgs(colName, "VARCHAR", "TEXT"))
+ }
+ }
+ }
+ return nil
+}
+
+// columnDefToCol converts ColumnDef to Col and TableConstraints.
+// outPriKeyConstraint is the primary key constraint out of column definition. such as: create table t1 (id int , age int, primary key(id));
+func columnDefToCol(ctx sessionctx.Context, offset int, colDef *ast.ColumnDef, outPriKeyConstraint *ast.Constraint) (*table.Column, []*ast.Constraint, error) {
+ var constraints = make([]*ast.Constraint, 0)
+ col := table.ToColumn(&model.ColumnInfo{
+ Offset: offset,
+ Name: colDef.Name.Name,
+ FieldType: *colDef.Tp,
+ // TODO: remove this version field after there is no old version.
+ Version: model.CurrLatestColumnInfoVersion,
+ })
+
+ if !isExplicitTimeStamp() {
+ // Check and set TimestampFlag, OnUpdateNowFlag and NotNullFlag.
+ if col.GetType() == mysql.TypeTimestamp {
+ col.AddFlag(mysql.TimestampFlag | mysql.OnUpdateNowFlag | mysql.NotNullFlag)
+ }
+ }
+ var err error
+ setOnUpdateNow := false
+ hasDefaultValue := false
+ hasNullFlag := false
+ if colDef.Options != nil {
+ length := types.UnspecifiedLength
+
+ keys := []*ast.IndexPartSpecification{
+ {
+ Column: colDef.Name,
+ Length: length,
+ },
+ }
+
+ var sb strings.Builder
+ restoreFlags := format.RestoreStringSingleQuotes | format.RestoreKeyWordLowercase | format.RestoreNameBackQuotes |
+ format.RestoreSpacesAroundBinaryOperation | format.RestoreWithoutSchemaName | format.RestoreWithoutTableName
+ restoreCtx := format.NewRestoreCtx(restoreFlags, &sb)
+
+ for _, v := range colDef.Options {
+ switch v.Tp {
+ case ast.ColumnOptionNotNull:
+ col.AddFlag(mysql.NotNullFlag)
+ case ast.ColumnOptionNull:
+ col.DelFlag(mysql.NotNullFlag)
+ removeOnUpdateNowFlag(col)
+ hasNullFlag = true
+ case ast.ColumnOptionAutoIncrement:
+ col.AddFlag(mysql.AutoIncrementFlag | mysql.NotNullFlag)
+ case ast.ColumnOptionPrimaryKey:
+ // Check PriKeyFlag first to avoid extra duplicate constraints.
+ if col.GetFlag()&mysql.PriKeyFlag == 0 {
+ constraint := &ast.Constraint{Tp: ast.ConstraintPrimaryKey, Keys: keys,
+ Option: &ast.IndexOption{PrimaryKeyTp: v.PrimaryKeyTp}}
+ if v.StrValue == "Global" {
+ constraint.Option.Global = true
+ }
+ constraints = append(constraints, constraint)
+ col.AddFlag(mysql.PriKeyFlag)
+ // Add NotNullFlag early so that processColumnFlags() can see it.
+ col.AddFlag(mysql.NotNullFlag)
+ }
+ case ast.ColumnOptionUniqKey:
+ // Check UniqueFlag first to avoid extra duplicate constraints.
+ if col.GetFlag()&mysql.UniqueFlag == 0 {
+ constraint := &ast.Constraint{Tp: ast.ConstraintUniqKey, Keys: keys}
+ if v.StrValue == "Global" {
+ constraint.Option = &ast.IndexOption{Global: true}
+ }
+ constraints = append(constraints, constraint)
+ col.AddFlag(mysql.UniqueKeyFlag)
+ }
+ case ast.ColumnOptionDefaultValue:
+ hasDefaultValue, err = SetDefaultValue(ctx, col, v)
+ if err != nil {
+ return nil, nil, errors.Trace(err)
+ }
+ removeOnUpdateNowFlag(col)
+ case ast.ColumnOptionOnUpdate:
+ // TODO: Support other time functions.
+ if !(col.GetType() == mysql.TypeTimestamp || col.GetType() == mysql.TypeDatetime) {
+ return nil, nil, dbterror.ErrInvalidOnUpdate.GenWithStackByArgs(col.Name)
+ }
+ if !expression.IsValidCurrentTimestampExpr(v.Expr, colDef.Tp) {
+ return nil, nil, dbterror.ErrInvalidOnUpdate.GenWithStackByArgs(col.Name)
+ }
+ col.AddFlag(mysql.OnUpdateNowFlag)
+ setOnUpdateNow = true
+ case ast.ColumnOptionComment:
+ err := setColumnComment(ctx, col, v)
+ if err != nil {
+ return nil, nil, errors.Trace(err)
+ }
+ case ast.ColumnOptionGenerated:
+ sb.Reset()
+ err = v.Expr.Restore(restoreCtx)
+ if err != nil {
+ return nil, nil, errors.Trace(err)
+ }
+ col.GeneratedExprString = sb.String()
+ col.GeneratedStored = v.Stored
+ _, dependColNames, err := findDependedColumnNames(model.NewCIStr(""), model.NewCIStr(""), colDef)
+ if err != nil {
+ return nil, nil, errors.Trace(err)
+ }
+ col.Dependences = dependColNames
+ case ast.ColumnOptionCollate:
+ if field_types.HasCharset(colDef.Tp) {
+ col.FieldType.SetCollate(v.StrValue)
+ }
+ case ast.ColumnOptionFulltext:
+ ctx.GetSessionVars().StmtCtx.AppendWarning(dbterror.ErrTableCantHandleFt.FastGenByArgs())
+ case ast.ColumnOptionCheck:
+ if !variable.EnableCheckConstraint.Load() {
+ ctx.GetSessionVars().StmtCtx.AppendWarning(errCheckConstraintIsOff)
+ } else {
+ // Check the column CHECK constraint dependency lazily, after fill all the name.
+ // Extract column constraint from column option.
+ constraint := &ast.Constraint{
+ Tp: ast.ConstraintCheck,
+ Expr: v.Expr,
+ Enforced: v.Enforced,
+ Name: v.ConstraintName,
+ InColumn: true,
+ InColumnName: colDef.Name.Name.O,
+ }
+ constraints = append(constraints, constraint)
+ }
+ }
+ }
+ }
+
+ if err = processAndCheckDefaultValueAndColumn(ctx, col, outPriKeyConstraint, hasDefaultValue, setOnUpdateNow, hasNullFlag); err != nil {
+ return nil, nil, errors.Trace(err)
+ }
+ return col, constraints, nil
+}
+
+// isExplicitTimeStamp is used to check if explicit_defaults_for_timestamp is on or off.
+// Check out this link for more details.
+// https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_explicit_defaults_for_timestamp
+func isExplicitTimeStamp() bool {
+ // TODO: implement the behavior as MySQL when explicit_defaults_for_timestamp = off, then this function could return false.
+ return true
+}
+
+// SetDefaultValue sets the default value of the column.
+func SetDefaultValue(ctx sessionctx.Context, col *table.Column, option *ast.ColumnOption) (hasDefaultValue bool, err error) {
+ var value any
+ var isSeqExpr bool
+ value, isSeqExpr, err = getDefaultValue(
+ exprctx.CtxWithHandleTruncateErrLevel(ctx.GetExprCtx(), errctx.LevelError),
+ col, option,
+ )
+ if err != nil {
+ return false, errors.Trace(err)
+ }
+ if isSeqExpr {
+ if err := checkSequenceDefaultValue(col); err != nil {
+ return false, errors.Trace(err)
+ }
+ col.DefaultIsExpr = isSeqExpr
+ }
+
+ // When the default value is expression, we skip check and convert.
+ if !col.DefaultIsExpr {
+ if hasDefaultValue, value, err = checkColumnDefaultValue(ctx.GetExprCtx(), col, value); err != nil {
+ return hasDefaultValue, errors.Trace(err)
+ }
+ value, err = convertTimestampDefaultValToUTC(ctx, value, col)
+ if err != nil {
+ return hasDefaultValue, errors.Trace(err)
+ }
+ } else {
+ hasDefaultValue = true
+ }
+ err = setDefaultValueWithBinaryPadding(col, value)
+ if err != nil {
+ return hasDefaultValue, errors.Trace(err)
+ }
+ return hasDefaultValue, nil
+}
+
+// getFuncCallDefaultValue gets the default column value of function-call expression.
+func getFuncCallDefaultValue(col *table.Column, option *ast.ColumnOption, expr *ast.FuncCallExpr) (any, bool, error) {
+ switch expr.FnName.L {
+ case ast.CurrentTimestamp, ast.CurrentDate: // CURRENT_TIMESTAMP() and CURRENT_DATE()
+ tp, fsp := col.FieldType.GetType(), col.FieldType.GetDecimal()
+ if tp == mysql.TypeTimestamp || tp == mysql.TypeDatetime {
+ defaultFsp := 0
+ if len(expr.Args) == 1 {
+ if val := expr.Args[0].(*driver.ValueExpr); val != nil {
+ defaultFsp = int(val.GetInt64())
+ }
+ }
+ if defaultFsp != fsp {
+ return nil, false, dbterror.ErrInvalidDefaultValue.GenWithStackByArgs(col.Name.O)
+ }
+ }
+ return nil, false, nil
+ case ast.NextVal:
+ // handle default next value of sequence. (keep the expr string)
+ str, err := getSequenceDefaultValue(option)
+ if err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ return str, true, nil
+ case ast.Rand, ast.UUID, ast.UUIDToBin: // RAND(), UUID() and UUID_TO_BIN()
+ if err := expression.VerifyArgsWrapper(expr.FnName.L, len(expr.Args)); err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ str, err := restoreFuncCall(expr)
+ if err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ col.DefaultIsExpr = true
+ return str, false, nil
+ case ast.DateFormat: // DATE_FORMAT()
+ if err := expression.VerifyArgsWrapper(expr.FnName.L, len(expr.Args)); err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ // Support DATE_FORMAT(NOW(),'%Y-%m'), DATE_FORMAT(NOW(),'%Y-%m-%d'),
+ // DATE_FORMAT(NOW(),'%Y-%m-%d %H.%i.%s'), DATE_FORMAT(NOW(),'%Y-%m-%d %H:%i:%s').
+ nowFunc, ok := expr.Args[0].(*ast.FuncCallExpr)
+ if ok && nowFunc.FnName.L == ast.Now {
+ if err := expression.VerifyArgsWrapper(nowFunc.FnName.L, len(nowFunc.Args)); err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ valExpr, isValue := expr.Args[1].(ast.ValueExpr)
+ if !isValue || (valExpr.GetString() != "%Y-%m" && valExpr.GetString() != "%Y-%m-%d" &&
+ valExpr.GetString() != "%Y-%m-%d %H.%i.%s" && valExpr.GetString() != "%Y-%m-%d %H:%i:%s") {
+ return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(), valExpr)
+ }
+ str, err := restoreFuncCall(expr)
+ if err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ col.DefaultIsExpr = true
+ return str, false, nil
+ }
+ return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(),
+ fmt.Sprintf("%s with disallowed args", expr.FnName.String()))
+ case ast.Replace:
+ if err := expression.VerifyArgsWrapper(expr.FnName.L, len(expr.Args)); err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ funcCall := expr.Args[0]
+ // Support REPLACE(CONVERT(UPPER(UUID()) USING UTF8MB4), '-', ''))
+ if convertFunc, ok := funcCall.(*ast.FuncCallExpr); ok && convertFunc.FnName.L == ast.Convert {
+ if err := expression.VerifyArgsWrapper(convertFunc.FnName.L, len(convertFunc.Args)); err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ funcCall = convertFunc.Args[0]
+ }
+ // Support REPLACE(UPPER(UUID()), '-', '').
+ if upperFunc, ok := funcCall.(*ast.FuncCallExpr); ok && upperFunc.FnName.L == ast.Upper {
+ if err := expression.VerifyArgsWrapper(upperFunc.FnName.L, len(upperFunc.Args)); err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ if uuidFunc, ok := upperFunc.Args[0].(*ast.FuncCallExpr); ok && uuidFunc.FnName.L == ast.UUID {
+ if err := expression.VerifyArgsWrapper(uuidFunc.FnName.L, len(uuidFunc.Args)); err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ str, err := restoreFuncCall(expr)
+ if err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ col.DefaultIsExpr = true
+ return str, false, nil
+ }
+ }
+ return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(),
+ fmt.Sprintf("%s with disallowed args", expr.FnName.String()))
+ case ast.Upper:
+ if err := expression.VerifyArgsWrapper(expr.FnName.L, len(expr.Args)); err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ // Support UPPER(SUBSTRING_INDEX(USER(), '@', 1)).
+ if substringIndexFunc, ok := expr.Args[0].(*ast.FuncCallExpr); ok && substringIndexFunc.FnName.L == ast.SubstringIndex {
+ if err := expression.VerifyArgsWrapper(substringIndexFunc.FnName.L, len(substringIndexFunc.Args)); err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ if userFunc, ok := substringIndexFunc.Args[0].(*ast.FuncCallExpr); ok && userFunc.FnName.L == ast.User {
+ if err := expression.VerifyArgsWrapper(userFunc.FnName.L, len(userFunc.Args)); err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ valExpr, isValue := substringIndexFunc.Args[1].(ast.ValueExpr)
+ if !isValue || valExpr.GetString() != "@" {
+ return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(), valExpr)
+ }
+ str, err := restoreFuncCall(expr)
+ if err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ col.DefaultIsExpr = true
+ return str, false, nil
+ }
+ }
+ return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(),
+ fmt.Sprintf("%s with disallowed args", expr.FnName.String()))
+ case ast.StrToDate: // STR_TO_DATE()
+ if err := expression.VerifyArgsWrapper(expr.FnName.L, len(expr.Args)); err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ // Support STR_TO_DATE('1980-01-01', '%Y-%m-%d').
+ if _, ok1 := expr.Args[0].(ast.ValueExpr); ok1 {
+ if _, ok2 := expr.Args[1].(ast.ValueExpr); ok2 {
+ str, err := restoreFuncCall(expr)
+ if err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ col.DefaultIsExpr = true
+ return str, false, nil
+ }
+ }
+ return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(),
+ fmt.Sprintf("%s with disallowed args", expr.FnName.String()))
+ case ast.JSONObject, ast.JSONArray, ast.JSONQuote: // JSON_OBJECT(), JSON_ARRAY(), JSON_QUOTE()
+ if err := expression.VerifyArgsWrapper(expr.FnName.L, len(expr.Args)); err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ str, err := restoreFuncCall(expr)
+ if err != nil {
+ return nil, false, errors.Trace(err)
+ }
+ col.DefaultIsExpr = true
+ return str, false, nil
+
+ default:
+ return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(), expr.FnName.String())
+ }
+}
+
+// getDefaultValue will get the default value for column.
+// 1: get the expr restored string for the column which uses sequence next value as default value.
+// 2: get specific default value for the other column.
+func getDefaultValue(ctx exprctx.BuildContext, col *table.Column, option *ast.ColumnOption) (any, bool, error) {
+ // handle default value with function call
+ tp, fsp := col.FieldType.GetType(), col.FieldType.GetDecimal()
+ if x, ok := option.Expr.(*ast.FuncCallExpr); ok {
+ val, isSeqExpr, err := getFuncCallDefaultValue(col, option, x)
+ if val != nil || isSeqExpr || err != nil {
+ return val, isSeqExpr, err
+ }
+ // If the function call is ast.CurrentTimestamp, it needs to be continuously processed.
+ }
+
+ if tp == mysql.TypeTimestamp || tp == mysql.TypeDatetime || tp == mysql.TypeDate {
+ vd, err := expression.GetTimeValue(ctx, option.Expr, tp, fsp, nil)
+ value := vd.GetValue()
+ if err != nil {
+ return nil, false, dbterror.ErrInvalidDefaultValue.GenWithStackByArgs(col.Name.O)
+ }
+
+ // Value is nil means `default null`.
+ if value == nil {
+ return nil, false, nil
+ }
+
+ // If value is types.Time, convert it to string.
+ if vv, ok := value.(types.Time); ok {
+ return vv.String(), false, nil
+ }
+
+ return value, false, nil
+ }
+
+ // evaluate the non-function-call expr to a certain value.
+ v, err := expression.EvalSimpleAst(ctx, option.Expr)
+ if err != nil {
+ return nil, false, errors.Trace(err)
+ }
+
+ if v.IsNull() {
+ return nil, false, nil
+ }
+
+ if v.Kind() == types.KindBinaryLiteral || v.Kind() == types.KindMysqlBit {
+ if types.IsTypeBlob(tp) || tp == mysql.TypeJSON || tp == mysql.TypeTiDBVectorFloat32 {
+ // BLOB/TEXT/JSON column cannot have a default value.
+ // Skip the unnecessary decode procedure.
+ return v.GetString(), false, err
+ }
+ if tp == mysql.TypeBit || tp == mysql.TypeString || tp == mysql.TypeVarchar ||
+ tp == mysql.TypeVarString || tp == mysql.TypeEnum || tp == mysql.TypeSet {
+ // For BinaryLiteral or bit fields, we decode the default value to utf8 string.
+ str, err := v.GetBinaryStringDecoded(types.StrictFlags, col.GetCharset())
+ if err != nil {
+ // Overwrite the decoding error with invalid default value error.
+ err = dbterror.ErrInvalidDefaultValue.GenWithStackByArgs(col.Name.O)
+ }
+ return str, false, err
+ }
+ // For other kind of fields (e.g. INT), we supply its integer as string value.
+ value, err := v.GetBinaryLiteral().ToInt(ctx.GetEvalCtx().TypeCtx())
+ if err != nil {
+ return nil, false, err
+ }
+ return strconv.FormatUint(value, 10), false, nil
+ }
+
+ switch tp {
+ case mysql.TypeSet:
+ val, err := getSetDefaultValue(v, col)
+ return val, false, err
+ case mysql.TypeEnum:
+ val, err := getEnumDefaultValue(v, col)
+ return val, false, err
+ case mysql.TypeDuration, mysql.TypeDate:
+ if v, err = v.ConvertTo(ctx.GetEvalCtx().TypeCtx(), &col.FieldType); err != nil {
+ return "", false, errors.Trace(err)
+ }
+ case mysql.TypeBit:
+ if v.Kind() == types.KindInt64 || v.Kind() == types.KindUint64 {
+ // For BIT fields, convert int into BinaryLiteral.
+ return types.NewBinaryLiteralFromUint(v.GetUint64(), -1).ToString(), false, nil
+ }
+ case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong, mysql.TypeFloat, mysql.TypeDouble:
+ // For these types, convert it to standard format firstly.
+ // like integer fields, convert it into integer string literals. like convert "1.25" into "1" and "2.8" into "3".
+ // if raise a error, we will use original expression. We will handle it in check phase
+ if temp, err := v.ConvertTo(ctx.GetEvalCtx().TypeCtx(), &col.FieldType); err == nil {
+ v = temp
+ }
+ }
+
+ val, err := v.ToString()
+ return val, false, err
+}
+
+func getSequenceDefaultValue(c *ast.ColumnOption) (expr string, err error) {
+ var sb strings.Builder
+ restoreFlags := format.RestoreStringSingleQuotes | format.RestoreKeyWordLowercase | format.RestoreNameBackQuotes |
+ format.RestoreSpacesAroundBinaryOperation
+ restoreCtx := format.NewRestoreCtx(restoreFlags, &sb)
+ if err := c.Expr.Restore(restoreCtx); err != nil {
+ return "", err
+ }
+ return sb.String(), nil
+}
+
+func setDefaultValueWithBinaryPadding(col *table.Column, value any) error {
+ err := col.SetDefaultValue(value)
+ if err != nil {
+ return err
+ }
+ // https://dev.mysql.com/doc/refman/8.0/en/binary-varbinary.html
+ // Set the default value for binary type should append the paddings.
+ if value != nil {
+ if col.GetType() == mysql.TypeString && types.IsBinaryStr(&col.FieldType) && len(value.(string)) < col.GetFlen() {
+ padding := make([]byte, col.GetFlen()-len(value.(string)))
+ col.DefaultValue = string(append([]byte(col.DefaultValue.(string)), padding...))
+ }
+ }
+ return nil
+}
+
+func setColumnComment(ctx sessionctx.Context, col *table.Column, option *ast.ColumnOption) error {
+ value, err := expression.EvalSimpleAst(ctx.GetExprCtx(), option.Expr)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if col.Comment, err = value.ToString(); err != nil {
+ return errors.Trace(err)
+ }
+
+ sessionVars := ctx.GetSessionVars()
+ col.Comment, err = validateCommentLength(sessionVars.StmtCtx.ErrCtx(), sessionVars.SQLMode, col.Name.L, &col.Comment, dbterror.ErrTooLongFieldComment)
+ return errors.Trace(err)
+}
+
+func processAndCheckDefaultValueAndColumn(ctx sessionctx.Context, col *table.Column,
+ outPriKeyConstraint *ast.Constraint, hasDefaultValue, setOnUpdateNow, hasNullFlag bool) error {
+ processDefaultValue(col, hasDefaultValue, setOnUpdateNow)
+ processColumnFlags(col)
+
+ err := checkPriKeyConstraint(col, hasDefaultValue, hasNullFlag, outPriKeyConstraint)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if err = checkColumnValueConstraint(col, col.GetCollate()); err != nil {
+ return errors.Trace(err)
+ }
+ if err = checkDefaultValue(ctx.GetExprCtx(), col, hasDefaultValue); err != nil {
+ return errors.Trace(err)
+ }
+ if err = checkColumnFieldLength(col); err != nil {
+ return errors.Trace(err)
+ }
+ return nil
+}
+
+func restoreFuncCall(expr *ast.FuncCallExpr) (string, error) {
+ var sb strings.Builder
+ restoreFlags := format.RestoreStringSingleQuotes | format.RestoreKeyWordLowercase | format.RestoreNameBackQuotes |
+ format.RestoreSpacesAroundBinaryOperation
+ restoreCtx := format.NewRestoreCtx(restoreFlags, &sb)
+ if err := expr.Restore(restoreCtx); err != nil {
+ return "", err
+ }
+ return sb.String(), nil
+}
+
+// getSetDefaultValue gets the default value for the set type. See https://dev.mysql.com/doc/refman/5.7/en/set.html.
+func getSetDefaultValue(v types.Datum, col *table.Column) (string, error) {
+ if v.Kind() == types.KindInt64 {
+ setCnt := len(col.GetElems())
+ maxLimit := int64(1< maxLimit {
+ return "", dbterror.ErrInvalidDefaultValue.GenWithStackByArgs(col.Name.O)
+ }
+ setVal, err := types.ParseSetValue(col.GetElems(), uint64(val))
+ if err != nil {
+ return "", errors.Trace(err)
+ }
+ v.SetMysqlSet(setVal, col.GetCollate())
+ return v.ToString()
+ }
+
+ str, err := v.ToString()
+ if err != nil {
+ return "", errors.Trace(err)
+ }
+ if str == "" {
+ return str, nil
+ }
+ setVal, err := types.ParseSetName(col.GetElems(), str, col.GetCollate())
+ if err != nil {
+ return "", dbterror.ErrInvalidDefaultValue.GenWithStackByArgs(col.Name.O)
+ }
+ v.SetMysqlSet(setVal, col.GetCollate())
+
+ return v.ToString()
+}
+
+// getEnumDefaultValue gets the default value for the enum type. See https://dev.mysql.com/doc/refman/5.7/en/enum.html.
+func getEnumDefaultValue(v types.Datum, col *table.Column) (string, error) {
+ if v.Kind() == types.KindInt64 {
+ val := v.GetInt64()
+ if val < 1 || val > int64(len(col.GetElems())) {
+ return "", dbterror.ErrInvalidDefaultValue.GenWithStackByArgs(col.Name.O)
+ }
+ enumVal, err := types.ParseEnumValue(col.GetElems(), uint64(val))
+ if err != nil {
+ return "", errors.Trace(err)
+ }
+ v.SetMysqlEnum(enumVal, col.GetCollate())
+ return v.ToString()
+ }
+ str, err := v.ToString()
+ if err != nil {
+ return "", errors.Trace(err)
+ }
+ // Ref: https://dev.mysql.com/doc/refman/8.0/en/enum.html
+ // Trailing spaces are automatically deleted from ENUM member values in the table definition when a table is created.
+ str = strings.TrimRight(str, " ")
+ enumVal, err := types.ParseEnumName(col.GetElems(), str, col.GetCollate())
+ if err != nil {
+ return "", dbterror.ErrInvalidDefaultValue.GenWithStackByArgs(col.Name.O)
+ }
+ v.SetMysqlEnum(enumVal, col.GetCollate())
+
+ return v.ToString()
+}
+
+func removeOnUpdateNowFlag(c *table.Column) {
+ // For timestamp Col, if it is set null or default value,
+ // OnUpdateNowFlag should be removed.
+ if mysql.HasTimestampFlag(c.GetFlag()) {
+ c.DelFlag(mysql.OnUpdateNowFlag)
+ }
+}
+
+func processDefaultValue(c *table.Column, hasDefaultValue bool, setOnUpdateNow bool) {
+ setTimestampDefaultValue(c, hasDefaultValue, setOnUpdateNow)
+
+ setYearDefaultValue(c, hasDefaultValue)
+
+ // Set `NoDefaultValueFlag` if this field doesn't have a default value and
+ // it is `not null` and not an `AUTO_INCREMENT` field or `TIMESTAMP` field.
+ setNoDefaultValueFlag(c, hasDefaultValue)
+}
+
+func setYearDefaultValue(c *table.Column, hasDefaultValue bool) {
+ if hasDefaultValue {
+ return
+ }
+
+ if c.GetType() == mysql.TypeYear && mysql.HasNotNullFlag(c.GetFlag()) {
+ if err := c.SetDefaultValue("0000"); err != nil {
+ logutil.DDLLogger().Error("set default value failed", zap.Error(err))
+ }
+ }
+}
+
+func setTimestampDefaultValue(c *table.Column, hasDefaultValue bool, setOnUpdateNow bool) {
+ if hasDefaultValue {
+ return
+ }
+
+ // For timestamp Col, if is not set default value or not set null, use current timestamp.
+ if mysql.HasTimestampFlag(c.GetFlag()) && mysql.HasNotNullFlag(c.GetFlag()) {
+ if setOnUpdateNow {
+ if err := c.SetDefaultValue(types.ZeroDatetimeStr); err != nil {
+ logutil.DDLLogger().Error("set default value failed", zap.Error(err))
+ }
+ } else {
+ if err := c.SetDefaultValue(strings.ToUpper(ast.CurrentTimestamp)); err != nil {
+ logutil.DDLLogger().Error("set default value failed", zap.Error(err))
+ }
+ }
+ }
+}
+
+func setNoDefaultValueFlag(c *table.Column, hasDefaultValue bool) {
+ if hasDefaultValue {
+ return
+ }
+
+ if !mysql.HasNotNullFlag(c.GetFlag()) {
+ return
+ }
+
+ // Check if it is an `AUTO_INCREMENT` field or `TIMESTAMP` field.
+ if !mysql.HasAutoIncrementFlag(c.GetFlag()) && !mysql.HasTimestampFlag(c.GetFlag()) {
+ c.AddFlag(mysql.NoDefaultValueFlag)
+ }
+}
+
+func checkDefaultValue(ctx exprctx.BuildContext, c *table.Column, hasDefaultValue bool) (err error) {
+ if !hasDefaultValue {
+ return nil
+ }
+
+ if c.GetDefaultValue() != nil {
+ if c.DefaultIsExpr {
+ if mysql.HasAutoIncrementFlag(c.GetFlag()) {
+ return types.ErrInvalidDefault.GenWithStackByArgs(c.Name)
+ }
+ return nil
+ }
+ _, err = table.GetColDefaultValue(
+ exprctx.CtxWithHandleTruncateErrLevel(ctx, errctx.LevelError),
+ c.ToInfo(),
+ )
+ if err != nil {
+ return types.ErrInvalidDefault.GenWithStackByArgs(c.Name)
+ }
+ return nil
+ }
+ // Primary key default null is invalid.
+ if mysql.HasPriKeyFlag(c.GetFlag()) {
+ return dbterror.ErrPrimaryCantHaveNull
+ }
+
+ // Set not null but default null is invalid.
+ if mysql.HasNotNullFlag(c.GetFlag()) {
+ return types.ErrInvalidDefault.GenWithStackByArgs(c.Name)
+ }
+
+ return nil
+}
+
+func checkColumnFieldLength(col *table.Column) error {
+ if col.GetType() == mysql.TypeVarchar {
+ if err := types.IsVarcharTooBigFieldLength(col.GetFlen(), col.Name.O, col.GetCharset()); err != nil {
+ return errors.Trace(err)
+ }
+ }
+
+ return nil
+}
+
+// checkPriKeyConstraint check all parts of a PRIMARY KEY must be NOT NULL
+func checkPriKeyConstraint(col *table.Column, hasDefaultValue, hasNullFlag bool, outPriKeyConstraint *ast.Constraint) error {
+ // Primary key should not be null.
+ if mysql.HasPriKeyFlag(col.GetFlag()) && hasDefaultValue && col.GetDefaultValue() == nil {
+ return types.ErrInvalidDefault.GenWithStackByArgs(col.Name)
+ }
+ // Set primary key flag for outer primary key constraint.
+ // Such as: create table t1 (id int , age int, primary key(id))
+ if !mysql.HasPriKeyFlag(col.GetFlag()) && outPriKeyConstraint != nil {
+ for _, key := range outPriKeyConstraint.Keys {
+ if key.Expr == nil && key.Column.Name.L != col.Name.L {
+ continue
+ }
+ col.AddFlag(mysql.PriKeyFlag)
+ break
+ }
+ }
+ // Primary key should not be null.
+ if mysql.HasPriKeyFlag(col.GetFlag()) && hasNullFlag {
+ return dbterror.ErrPrimaryCantHaveNull
+ }
+ return nil
+}
+
+func checkColumnValueConstraint(col *table.Column, collation string) error {
+ if col.GetType() != mysql.TypeEnum && col.GetType() != mysql.TypeSet {
+ return nil
+ }
+ valueMap := make(map[string]bool, len(col.GetElems()))
+ ctor := collate.GetCollator(collation)
+ enumLengthLimit := config.GetGlobalConfig().EnableEnumLengthLimit
+ desc, err := charset.GetCharsetInfo(col.GetCharset())
+ if err != nil {
+ return errors.Trace(err)
+ }
+ for i := range col.GetElems() {
+ val := string(ctor.Key(col.GetElems()[i]))
+ // According to MySQL 8.0 Refman:
+ // The maximum supported length of an individual ENUM element is M <= 255 and (M x w) <= 1020,
+ // where M is the element literal length and w is the number of bytes required for the maximum-length character in the character set.
+ // See https://dev.mysql.com/doc/refman/8.0/en/string-type-syntax.html for more details.
+ if enumLengthLimit && (len(val) > 255 || len(val)*desc.Maxlen > 1020) {
+ return dbterror.ErrTooLongValueForType.GenWithStackByArgs(col.Name)
+ }
+ if _, ok := valueMap[val]; ok {
+ tpStr := "ENUM"
+ if col.GetType() == mysql.TypeSet {
+ tpStr = "SET"
+ }
+ return types.ErrDuplicatedValueInType.GenWithStackByArgs(col.Name, col.GetElems()[i], tpStr)
+ }
+ valueMap[val] = true
+ }
+ return nil
+}
+
+// checkColumnDefaultValue checks the default value of the column.
+// In non-strict SQL mode, if the default value of the column is an empty string, the default value can be ignored.
+// In strict SQL mode, TEXT/BLOB/JSON can't have not null default values.
+// In NO_ZERO_DATE SQL mode, TIMESTAMP/DATE/DATETIME type can't have zero date like '0000-00-00' or '0000-00-00 00:00:00'.
+func checkColumnDefaultValue(ctx exprctx.BuildContext, col *table.Column, value any) (bool, any, error) {
+ hasDefaultValue := true
+ if value != nil && (col.GetType() == mysql.TypeJSON ||
+ col.GetType() == mysql.TypeTinyBlob || col.GetType() == mysql.TypeMediumBlob ||
+ col.GetType() == mysql.TypeLongBlob || col.GetType() == mysql.TypeBlob) {
+ // In non-strict SQL mode.
+ if !ctx.GetEvalCtx().SQLMode().HasStrictMode() && value == "" {
+ if col.GetType() == mysql.TypeBlob || col.GetType() == mysql.TypeLongBlob {
+ // The TEXT/BLOB default value can be ignored.
+ hasDefaultValue = false
+ }
+ // In non-strict SQL mode, if the column type is json and the default value is null, it is initialized to an empty array.
+ if col.GetType() == mysql.TypeJSON {
+ value = `null`
+ }
+ ctx.GetEvalCtx().AppendWarning(dbterror.ErrBlobCantHaveDefault.FastGenByArgs(col.Name.O))
+ return hasDefaultValue, value, nil
+ }
+ // In strict SQL mode or default value is not an empty string.
+ return hasDefaultValue, value, dbterror.ErrBlobCantHaveDefault.GenWithStackByArgs(col.Name.O)
+ }
+ if value != nil && ctx.GetEvalCtx().SQLMode().HasNoZeroDateMode() &&
+ ctx.GetEvalCtx().SQLMode().HasStrictMode() && types.IsTypeTime(col.GetType()) {
+ if vv, ok := value.(string); ok {
+ timeValue, err := expression.GetTimeValue(ctx, vv, col.GetType(), col.GetDecimal(), nil)
+ if err != nil {
+ return hasDefaultValue, value, errors.Trace(err)
+ }
+ if timeValue.GetMysqlTime().CoreTime() == types.ZeroCoreTime {
+ return hasDefaultValue, value, types.ErrInvalidDefault.GenWithStackByArgs(col.Name.O)
+ }
+ }
+ }
+ return hasDefaultValue, value, nil
+}
+
+func checkSequenceDefaultValue(col *table.Column) error {
+ if mysql.IsIntegerType(col.GetType()) {
+ return nil
+ }
+ return dbterror.ErrColumnTypeUnsupportedNextValue.GenWithStackByArgs(col.ColumnInfo.Name.O)
+}
+
+func convertTimestampDefaultValToUTC(ctx sessionctx.Context, defaultVal any, col *table.Column) (any, error) {
+ if defaultVal == nil || col.GetType() != mysql.TypeTimestamp {
+ return defaultVal, nil
+ }
+ if vv, ok := defaultVal.(string); ok {
+ if vv != types.ZeroDatetimeStr && !strings.EqualFold(vv, ast.CurrentTimestamp) {
+ t, err := types.ParseTime(ctx.GetSessionVars().StmtCtx.TypeCtx(), vv, col.GetType(), col.GetDecimal())
+ if err != nil {
+ return defaultVal, errors.Trace(err)
+ }
+ err = t.ConvertTimeZone(ctx.GetSessionVars().Location(), time.UTC)
+ if err != nil {
+ return defaultVal, errors.Trace(err)
+ }
+ defaultVal = t.String()
+ }
+ }
+ return defaultVal, nil
+}
+
+// processColumnFlags is used by columnDefToCol and processColumnOptions. It is intended to unify behaviors on `create/add` and `modify/change` statements. Check tidb#issue#19342.
+func processColumnFlags(col *table.Column) {
+ if col.FieldType.EvalType().IsStringKind() {
+ if col.GetCharset() == charset.CharsetBin {
+ col.AddFlag(mysql.BinaryFlag)
+ } else {
+ col.DelFlag(mysql.BinaryFlag)
+ }
+ }
+ if col.GetType() == mysql.TypeBit {
+ // For BIT field, it's charset is binary but does not have binary flag.
+ col.DelFlag(mysql.BinaryFlag)
+ col.AddFlag(mysql.UnsignedFlag)
+ }
+ if col.GetType() == mysql.TypeYear {
+ // For Year field, it's charset is binary but does not have binary flag.
+ col.DelFlag(mysql.BinaryFlag)
+ col.AddFlag(mysql.ZerofillFlag)
+ }
+
+ // If you specify ZEROFILL for a numeric column, MySQL automatically adds the UNSIGNED attribute to the column.
+ // See https://dev.mysql.com/doc/refman/5.7/en/numeric-type-overview.html for more details.
+ // But some types like bit and year, won't show its unsigned flag in `show create table`.
+ if mysql.HasZerofillFlag(col.GetFlag()) {
+ col.AddFlag(mysql.UnsignedFlag)
+ }
+}
+
+func adjustBlobTypesFlen(tp *types.FieldType, colCharset string) error {
+ cs, err := charset.GetCharsetInfo(colCharset)
+ // when we meet the unsupported charset, we do not adjust.
+ if err != nil {
+ return err
+ }
+ l := tp.GetFlen() * cs.Maxlen
+ if tp.GetType() == mysql.TypeBlob {
+ if l <= tinyBlobMaxLength {
+ logutil.DDLLogger().Info(fmt.Sprintf("Automatically convert BLOB(%d) to TINYBLOB", tp.GetFlen()))
+ tp.SetFlen(tinyBlobMaxLength)
+ tp.SetType(mysql.TypeTinyBlob)
+ } else if l <= blobMaxLength {
+ tp.SetFlen(blobMaxLength)
+ } else if l <= mediumBlobMaxLength {
+ logutil.DDLLogger().Info(fmt.Sprintf("Automatically convert BLOB(%d) to MEDIUMBLOB", tp.GetFlen()))
+ tp.SetFlen(mediumBlobMaxLength)
+ tp.SetType(mysql.TypeMediumBlob)
+ } else if l <= longBlobMaxLength {
+ logutil.DDLLogger().Info(fmt.Sprintf("Automatically convert BLOB(%d) to LONGBLOB", tp.GetFlen()))
+ tp.SetFlen(longBlobMaxLength)
+ tp.SetType(mysql.TypeLongBlob)
+ }
+ }
+ return nil
+}
diff --git a/pkg/ddl/backfilling.go b/pkg/ddl/backfilling.go
index 6e11fd7e5eb59..49fa029d7e59a 100644
--- a/pkg/ddl/backfilling.go
+++ b/pkg/ddl/backfilling.go
@@ -162,13 +162,13 @@ type backfillCtx struct {
schemaName string
table table.Table
batchCnt int
- jobContext *JobContext
+ jobContext *ReorgContext
metricCounter prometheus.Counter
}
func newBackfillCtx(id int, rInfo *reorgInfo,
- schemaName string, tbl table.Table, jobCtx *JobContext, label string, isDistributed bool) (*backfillCtx, error) {
- sessCtx, err := newSessCtx(rInfo.d.store, rInfo.ReorgMeta)
+ schemaName string, tbl table.Table, jobCtx *ReorgContext, label string, isDistributed bool) (*backfillCtx, error) {
+ sessCtx, err := newSessCtx(rInfo.jobCtx.store, rInfo.ReorgMeta)
if err != nil {
return nil, err
}
@@ -178,9 +178,10 @@ func newBackfillCtx(id int, rInfo *reorgInfo,
}
exprCtx := sessCtx.GetExprCtx()
+ batchCnt := rInfo.ReorgMeta.GetBatchSizeOrDefault(int(variable.GetDDLReorgBatchSize()))
return &backfillCtx{
id: id,
- ddlCtx: rInfo.d,
+ ddlCtx: rInfo.jobCtx.oldDDLCtx,
sessCtx: sessCtx,
warnings: sessCtx.GetSessionVars().StmtCtx.WarnHandler,
exprCtx: exprCtx,
@@ -188,7 +189,7 @@ func newBackfillCtx(id int, rInfo *reorgInfo,
loc: exprCtx.GetEvalCtx().Location(),
schemaName: schemaName,
table: tbl,
- batchCnt: int(variable.GetDDLReorgBatchSize()),
+ batchCnt: batchCnt,
jobContext: jobCtx,
metricCounter: metrics.BackfillTotalCounter.WithLabelValues(
metrics.GenerateReorgLabel(label, schemaName, tbl.Meta().Name.String())),
@@ -415,7 +416,8 @@ func (w *backfillWorker) run(d *ddlCtx, bf backfiller, job *model.Job) {
})
// Change the batch size dynamically.
- w.GetCtx().batchCnt = int(variable.GetDDLReorgBatchSize())
+ newBatchCnt := job.ReorgMeta.GetBatchSizeOrDefault(int(variable.GetDDLReorgBatchSize()))
+ w.GetCtx().batchCnt = newBatchCnt
result := w.handleBackfillTask(d, task, bf)
w.sendResult(result)
@@ -450,6 +452,11 @@ func loadTableRanges(
zap.Int64("physicalTableID", t.GetPhysicalID()))
return []kv.KeyRange{{StartKey: startKey, EndKey: endKey}}, nil
}
+ failpoint.Inject("setLimitForLoadTableRanges", func(val failpoint.Value) {
+ if v, ok := val.(int); ok {
+ limit = v
+ }
+ })
rc := s.GetRegionCache()
maxSleep := 10000 // ms
@@ -464,6 +471,12 @@ func loadTableRanges(
if err != nil {
return false, errors.Trace(err)
}
+ var mockErr bool
+ failpoint.InjectCall("beforeLoadRangeFromPD", &mockErr)
+ if mockErr {
+ return false, kv.ErrTxnRetryable
+ }
+
ranges = make([]kv.KeyRange, 0, len(rs))
for _, r := range rs {
ranges = append(ranges, kv.KeyRange{StartKey: r.StartKey(), EndKey: r.EndKey()})
@@ -568,7 +581,7 @@ func getActualEndKey(
// backfill worker can't catch up, we shrink the end key to the actual written key for now.
jobCtx := reorgInfo.NewJobContext()
- actualEndKey, err := GetRangeEndKey(jobCtx, reorgInfo.d.store, job.Priority, t.RecordPrefix(), rangeStart, rangeEnd)
+ actualEndKey, err := GetRangeEndKey(jobCtx, reorgInfo.jobCtx.store, job.Priority, t.RecordPrefix(), rangeStart, rangeEnd)
if err != nil {
logutil.DDLLogger().Info("get backfill range task, get reverse key failed", zap.Error(err))
return rangeEnd
@@ -634,12 +647,7 @@ func makeupDecodeColMap(dbName model.CIStr, t table.Table) (map[int64]decoder.Co
return decodeColMap, nil
}
-var backfillTaskChanSize = 128
-
-// SetBackfillTaskChanSizeForTest is only used for test.
-func SetBackfillTaskChanSizeForTest(n int) {
- backfillTaskChanSize = n
-}
+const backfillTaskChanSize = 128
func (dc *ddlCtx) runAddIndexInLocalIngestMode(
ctx context.Context,
@@ -675,8 +683,9 @@ func (dc *ddlCtx) runAddIndexInLocalIngestMode(
//nolint: forcetypeassert
discovery := dc.store.(tikv.Storage).GetRegionCache().PDClient().GetServiceDiscovery()
+ importConc := job.ReorgMeta.GetConcurrencyOrDefault(int(variable.GetDDLReorgWorkerCounter()))
bcCtx, err := ingest.LitBackCtxMgr.Register(
- ctx, job.ID, hasUnique, dc.etcdCli, discovery, job.ReorgMeta.ResourceGroupName)
+ ctx, job.ID, hasUnique, dc.etcdCli, discovery, job.ReorgMeta.ResourceGroupName, importConc)
if err != nil {
return errors.Trace(err)
}
@@ -705,16 +714,15 @@ func (dc *ddlCtx) runAddIndexInLocalIngestMode(
bcCtx.AttachCheckpointManager(cpMgr)
}
- reorgCtx := dc.getReorgCtx(reorgInfo.Job.ID)
+ reorgCtx := dc.getReorgCtx(job.ID)
rowCntListener := &localRowCntListener{
prevPhysicalRowCnt: reorgCtx.getRowCount(),
- reorgCtx: dc.getReorgCtx(reorgInfo.Job.ID),
+ reorgCtx: reorgCtx,
counter: metrics.BackfillTotalCounter.WithLabelValues(
metrics.GenerateReorgLabel("add_idx_rate", job.SchemaName, job.TableName)),
}
avgRowSize := estimateTableRowSize(ctx, dc.store, sctx.GetRestrictedSQLExecutor(), t)
- concurrency := int(variable.GetDDLReorgWorkerCounter())
engines, err := bcCtx.Register(indexIDs, uniques, t)
if err != nil {
@@ -724,7 +732,6 @@ func (dc *ddlCtx) runAddIndexInLocalIngestMode(
zap.Int64s("index IDs", indexIDs))
return errors.Trace(err)
}
-
pipe, err := NewAddIndexIngestPipeline(
opCtx,
dc.store,
@@ -738,7 +745,7 @@ func (dc *ddlCtx) runAddIndexInLocalIngestMode(
reorgInfo.EndKey,
job.ReorgMeta,
avgRowSize,
- concurrency,
+ importConc,
cpMgr,
rowCntListener,
)
@@ -782,12 +789,17 @@ type localRowCntListener struct {
// prevPhysicalRowCnt records the row count from previous physical tables (partitions).
prevPhysicalRowCnt int64
// curPhysicalRowCnt records the row count of current physical table.
- curPhysicalRowCnt int64
+ curPhysicalRowCnt struct {
+ cnt int64
+ mu sync.Mutex
+ }
}
func (s *localRowCntListener) Written(rowCnt int) {
- s.curPhysicalRowCnt += int64(rowCnt)
- s.reorgCtx.setRowCount(s.prevPhysicalRowCnt + s.curPhysicalRowCnt)
+ s.curPhysicalRowCnt.mu.Lock()
+ s.curPhysicalRowCnt.cnt += int64(rowCnt)
+ s.reorgCtx.setRowCount(s.prevPhysicalRowCnt + s.curPhysicalRowCnt.cnt)
+ s.curPhysicalRowCnt.mu.Unlock()
s.counter.Add(float64(rowCnt))
}
@@ -970,7 +982,7 @@ func injectCheckBackfillWorkerNum(curWorkerSize int, isMergeWorker bool) error {
// recordIterFunc is used for low-level record iteration.
type recordIterFunc func(h kv.Handle, rowKey kv.Key, rawRecord []byte) (more bool, err error)
-func iterateSnapshotKeys(ctx *JobContext, store kv.Storage, priority int, keyPrefix kv.Key, version uint64,
+func iterateSnapshotKeys(ctx *ReorgContext, store kv.Storage, priority int, keyPrefix kv.Key, version uint64,
startKey kv.Key, endKey kv.Key, fn recordIterFunc) error {
isRecord := tablecodec.IsRecordKey(keyPrefix.Next())
var firstKey kv.Key
@@ -1035,7 +1047,7 @@ func iterateSnapshotKeys(ctx *JobContext, store kv.Storage, priority int, keyPre
}
// GetRangeEndKey gets the actual end key for the range of [startKey, endKey).
-func GetRangeEndKey(ctx *JobContext, store kv.Storage, priority int, keyPrefix kv.Key, startKey, endKey kv.Key) (kv.Key, error) {
+func GetRangeEndKey(ctx *ReorgContext, store kv.Storage, priority int, keyPrefix kv.Key, startKey, endKey kv.Key) (kv.Key, error) {
snap := store.GetSnapshot(kv.MaxVersion)
snap.SetOption(kv.Priority, priority)
if tagger := ctx.getResourceGroupTaggerForTopSQL(); tagger != nil {
diff --git a/pkg/ddl/backfilling_dist_executor.go b/pkg/ddl/backfilling_dist_executor.go
index c099c99750410..b635ddd83608c 100644
--- a/pkg/ddl/backfilling_dist_executor.go
+++ b/pkg/ddl/backfilling_dist_executor.go
@@ -28,6 +28,7 @@ import (
"github.com/pingcap/tidb/pkg/lightning/common"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/terror"
+ "github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/util/dbterror"
"github.com/tikv/client-go/v2/tikv"
@@ -147,7 +148,14 @@ func (s *backfillDistExecutor) getBackendCtx() (ingest.BackendCtx, error) {
ddlObj := s.d
discovery := ddlObj.store.(tikv.Storage).GetRegionCache().PDClient().GetServiceDiscovery()
- return ingest.LitBackCtxMgr.Register(s.BaseTaskExecutor.Ctx(), job.ID, hasUnique, ddlObj.etcdCli, discovery, job.ReorgMeta.ResourceGroupName)
+ return ingest.LitBackCtxMgr.Register(
+ s.BaseTaskExecutor.Ctx(),
+ job.ID, hasUnique,
+ ddlObj.etcdCli,
+ discovery,
+ job.ReorgMeta.ResourceGroupName,
+ job.ReorgMeta.GetConcurrencyOrDefault(int(variable.GetDDLReorgWorkerCounter())),
+ )
}
func hasUniqueIndex(job *model.Job) (bool, error) {
diff --git a/pkg/ddl/backfilling_dist_scheduler.go b/pkg/ddl/backfilling_dist_scheduler.go
index 55abe4225da12..e73a7bb1122f3 100644
--- a/pkg/ddl/backfilling_dist_scheduler.go
+++ b/pkg/ddl/backfilling_dist_scheduler.go
@@ -257,7 +257,7 @@ func generateNonPartitionPlan(
return nil, errors.Trace(err)
}
- startKey, endKey, err := getTableRange(d.jobContext(job.ID, job.ReorgMeta), d.ddlCtx, tbl.(table.PhysicalTable), ver.Ver, job.Priority)
+ startKey, endKey, err := getTableRange(d.jobContext(job.ID, job.ReorgMeta), d.store, tbl.(table.PhysicalTable), ver.Ver, job.Priority)
if startKey == nil && endKey == nil {
// Empty table.
return nil, nil
@@ -294,7 +294,7 @@ func generateNonPartitionPlan(
return true, nil
}
- regionBatch := calculateRegionBatch(len(recordRegionMetas), instanceCnt, !useCloud)
+ regionBatch := CalculateRegionBatch(len(recordRegionMetas), instanceCnt, !useCloud)
for i := 0; i < len(recordRegionMetas); i += regionBatch {
end := i + regionBatch
@@ -329,7 +329,8 @@ func generateNonPartitionPlan(
return subTaskMetas, nil
}
-func calculateRegionBatch(totalRegionCnt int, instanceCnt int, useLocalDisk bool) int {
+// CalculateRegionBatch is exported for test.
+func CalculateRegionBatch(totalRegionCnt int, instanceCnt int, useLocalDisk bool) int {
failpoint.Inject("mockRegionBatch", func(val failpoint.Value) {
failpoint.Return(val.(int))
})
diff --git a/pkg/ddl/backfilling_dist_scheduler_test.go b/pkg/ddl/backfilling_dist_scheduler_test.go
index 3d08208317b2a..6e4f1f32c9264 100644
--- a/pkg/ddl/backfilling_dist_scheduler_test.go
+++ b/pkg/ddl/backfilling_dist_scheduler_test.go
@@ -113,19 +113,19 @@ func TestBackfillingSchedulerLocalMode(t *testing.T) {
func TestCalculateRegionBatch(t *testing.T) {
// Test calculate in cloud storage.
- batchCnt := ddl.CalculateRegionBatchForTest(100, 8, false)
+ batchCnt := ddl.CalculateRegionBatch(100, 8, false)
require.Equal(t, 13, batchCnt)
- batchCnt = ddl.CalculateRegionBatchForTest(2, 8, false)
+ batchCnt = ddl.CalculateRegionBatch(2, 8, false)
require.Equal(t, 1, batchCnt)
- batchCnt = ddl.CalculateRegionBatchForTest(8, 8, false)
+ batchCnt = ddl.CalculateRegionBatch(8, 8, false)
require.Equal(t, 1, batchCnt)
// Test calculate in local storage.
- batchCnt = ddl.CalculateRegionBatchForTest(100, 8, true)
+ batchCnt = ddl.CalculateRegionBatch(100, 8, true)
require.Equal(t, 13, batchCnt)
- batchCnt = ddl.CalculateRegionBatchForTest(2, 8, true)
+ batchCnt = ddl.CalculateRegionBatch(2, 8, true)
require.Equal(t, 1, batchCnt)
- batchCnt = ddl.CalculateRegionBatchForTest(24, 8, true)
+ batchCnt = ddl.CalculateRegionBatch(24, 8, true)
require.Equal(t, 3, batchCnt)
}
diff --git a/pkg/ddl/backfilling_operators.go b/pkg/ddl/backfilling_operators.go
index 5ae5554ef290b..201c09702af80 100644
--- a/pkg/ddl/backfilling_operators.go
+++ b/pkg/ddl/backfilling_operators.go
@@ -171,15 +171,11 @@ func NewAddIndexIngestPipeline(
if err != nil {
return nil, err
}
- poolSize := copReadChunkPoolSize()
- srcChkPool := make(chan *chunk.Chunk, poolSize)
- for i := 0; i < poolSize; i++ {
- srcChkPool <- chunk.NewChunkWithCapacity(copCtx.GetBase().FieldTypes, copReadBatchSize())
- }
+ srcChkPool := createChunkPool(copCtx, concurrency, reorgMeta.BatchSize)
readerCnt, writerCnt := expectedIngestWorkerCnt(concurrency, avgRowSize)
srcOp := NewTableScanTaskSource(ctx, store, tbl, startKey, endKey, cpMgr)
- scanOp := NewTableScanOperator(ctx, sessPool, copCtx, srcChkPool, readerCnt, cpMgr)
+ scanOp := NewTableScanOperator(ctx, sessPool, copCtx, srcChkPool, readerCnt, cpMgr, reorgMeta.BatchSize)
ingestOp := NewIndexIngestOperator(ctx, copCtx, backendCtx, sessPool,
tbl, indexes, engines, srcChkPool, writerCnt, reorgMeta, cpMgr, rowCntListener)
sinkOp := newIndexWriteResultSink(ctx, backendCtx, tbl, indexes, cpMgr, rowCntListener)
@@ -226,11 +222,7 @@ func NewWriteIndexToExternalStoragePipeline(
if err != nil {
return nil, err
}
- poolSize := copReadChunkPoolSize()
- srcChkPool := make(chan *chunk.Chunk, poolSize)
- for i := 0; i < poolSize; i++ {
- srcChkPool <- chunk.NewChunkWithCapacity(copCtx.GetBase().FieldTypes, copReadBatchSize())
- }
+ srcChkPool := createChunkPool(copCtx, concurrency, reorgMeta.BatchSize)
readerCnt, writerCnt := expectedIngestWorkerCnt(concurrency, avgRowSize)
backend, err := storage.ParseBackend(extStoreURI, nil)
@@ -248,7 +240,7 @@ func NewWriteIndexToExternalStoragePipeline(
})
srcOp := NewTableScanTaskSource(ctx, store, tbl, startKey, endKey, nil)
- scanOp := NewTableScanOperator(ctx, sessPool, copCtx, srcChkPool, readerCnt, nil)
+ scanOp := NewTableScanOperator(ctx, sessPool, copCtx, srcChkPool, readerCnt, nil, reorgMeta.BatchSize)
writeOp := NewWriteExternalStoreOperator(
ctx, copCtx, sessPool, jobID, subtaskID, tbl, indexes, extStore, srcChkPool, writerCnt, onClose, memSizePerIndex, reorgMeta)
sinkOp := newIndexWriteResultSink(ctx, nil, tbl, indexes, nil, rowCntListener)
@@ -270,6 +262,16 @@ func NewWriteIndexToExternalStoragePipeline(
), nil
}
+func createChunkPool(copCtx copr.CopContext, hintConc, hintBatchSize int) chan *chunk.Chunk {
+ poolSize := ingest.CopReadChunkPoolSize(hintConc)
+ batchSize := ingest.CopReadBatchSize(hintBatchSize)
+ srcChkPool := make(chan *chunk.Chunk, poolSize)
+ for i := 0; i < poolSize; i++ {
+ srcChkPool <- chunk.NewChunkWithCapacity(copCtx.GetBase().FieldTypes, batchSize)
+ }
+ return srcChkPool
+}
+
// TableScanTask contains the start key and the end key of a region.
type TableScanTask struct {
ID int
@@ -344,7 +346,7 @@ func (src *TableScanTaskSource) adjustStartKey(start, end kv.Key) (adjusted kv.K
if src.cpMgr == nil {
return start, false
}
- cpKey := src.cpMgr.LastProcessedKey()
+ cpKey := src.cpMgr.NextKeyToProcess()
if len(cpKey) == 0 {
return start, false
}
@@ -362,7 +364,7 @@ func (src *TableScanTaskSource) adjustStartKey(start, end kv.Key) (adjusted kv.K
if cpKey.Cmp(end) == 0 {
return cpKey, true
}
- return cpKey.Next(), false
+ return cpKey, false
}
func (src *TableScanTaskSource) generateTasks() error {
@@ -457,6 +459,7 @@ func NewTableScanOperator(
srcChkPool chan *chunk.Chunk,
concurrency int,
cpMgr *ingest.CheckpointManager,
+ hintBatchSize int,
) *TableScanOperator {
pool := workerpool.NewWorkerPool(
"TableScanOperator",
@@ -464,12 +467,13 @@ func NewTableScanOperator(
concurrency,
func() workerpool.Worker[TableScanTask, IndexRecordChunk] {
return &tableScanWorker{
- ctx: ctx,
- copCtx: copCtx,
- sessPool: sessPool,
- se: nil,
- srcChkPool: srcChkPool,
- cpMgr: cpMgr,
+ ctx: ctx,
+ copCtx: copCtx,
+ sessPool: sessPool,
+ se: nil,
+ srcChkPool: srcChkPool,
+ cpMgr: cpMgr,
+ hintBatchSize: hintBatchSize,
}
})
return &TableScanOperator{
@@ -484,7 +488,8 @@ type tableScanWorker struct {
se *session.Session
srcChkPool chan *chunk.Chunk
- cpMgr *ingest.CheckpointManager
+ cpMgr *ingest.CheckpointManager
+ hintBatchSize int
}
func (w *tableScanWorker) HandleTask(task TableScanTask, sender func(IndexRecordChunk)) {
@@ -554,7 +559,7 @@ func (w *tableScanWorker) scanRecords(task TableScanTask, sender func(IndexRecor
func (w *tableScanWorker) getChunk() *chunk.Chunk {
chk := <-w.srcChkPool
- newCap := copReadBatchSize()
+ newCap := ingest.CopReadBatchSize(w.hintBatchSize)
if chk.Capacity() != newCap {
chk = chunk.NewChunkWithCapacity(w.copCtx.GetBase().FieldTypes, newCap)
}
diff --git a/pkg/ddl/backfilling_read_index.go b/pkg/ddl/backfilling_read_index.go
index 0c5ce7a62a538..a902a81a8a040 100644
--- a/pkg/ddl/backfilling_read_index.go
+++ b/pkg/ddl/backfilling_read_index.go
@@ -44,7 +44,7 @@ type readIndexExecutor struct {
job *model.Job
indexes []*model.IndexInfo
ptbl table.PhysicalTable
- jc *JobContext
+ jc *ReorgContext
avgRowSize int
cloudStorageURI string
@@ -65,7 +65,7 @@ func newReadIndexExecutor(
job *model.Job,
indexes []*model.IndexInfo,
ptbl table.PhysicalTable,
- jc *JobContext,
+ jc *ReorgContext,
bcGetter func() (ingest.BackendCtx, error),
cloudStorageURI string,
avgRowSize int,
@@ -191,7 +191,7 @@ func (r *readIndexExecutor) getTableStartEndKey(sm *BackfillSubTaskMeta) (
}
if parTbl, ok := r.ptbl.(table.PartitionedTable); ok {
pid := sm.PhysicalTableID
- start, end, err = getTableRange(r.jc, r.d.ddlCtx, parTbl.GetPartition(pid), currentVer.Ver, r.job.Priority)
+ start, end, err = getTableRange(r.jc, r.d.store, parTbl.GetPartition(pid), currentVer.Ver, r.job.Priority)
if err != nil {
logutil.DDLLogger().Error("get table range error",
zap.Error(err))
diff --git a/pkg/ddl/backfilling_scheduler.go b/pkg/ddl/backfilling_scheduler.go
index dc0201729787c..d37f6ccb7c15a 100644
--- a/pkg/ddl/backfilling_scheduler.go
+++ b/pkg/ddl/backfilling_scheduler.go
@@ -68,7 +68,7 @@ type txnBackfillScheduler struct {
tp backfillerType
tbl table.PhysicalTable
decodeColMap map[int64]decoder.Column
- jobCtx *JobContext
+ jobCtx *ReorgContext
workers []*backfillWorker
wg sync.WaitGroup
@@ -80,11 +80,12 @@ type txnBackfillScheduler struct {
func newTxnBackfillScheduler(ctx context.Context, info *reorgInfo, sessPool *sess.Pool,
tp backfillerType, tbl table.PhysicalTable,
- jobCtx *JobContext) (backfillScheduler, error) {
+ jobCtx *ReorgContext) (backfillScheduler, error) {
decColMap, err := makeupDecodeColMap(info.dbInfo.Name, tbl)
if err != nil {
return nil, err
}
+ workerCnt := info.ReorgMeta.GetConcurrencyOrDefault(int(variable.GetDDLReorgWorkerCounter()))
return &txnBackfillScheduler{
ctx: ctx,
reorgInfo: info,
@@ -93,7 +94,7 @@ func newTxnBackfillScheduler(ctx context.Context, info *reorgInfo, sessPool *ses
tbl: tbl,
decodeColMap: decColMap,
jobCtx: jobCtx,
- workers: make([]*backfillWorker, 0, variable.GetDDLReorgWorkerCounter()),
+ workers: make([]*backfillWorker, 0, workerCnt),
taskCh: make(chan *reorgBackfillTask, backfillTaskChanSize),
resultCh: make(chan *backfillResult, backfillTaskChanSize),
}, nil
@@ -230,8 +231,8 @@ func restoreSessCtx(sessCtx sessionctx.Context) func(sessCtx sessionctx.Context)
}
}
-func (*txnBackfillScheduler) expectedWorkerSize() (size int) {
- workerCnt := int(variable.GetDDLReorgWorkerCounter())
+func (b *txnBackfillScheduler) expectedWorkerSize() (size int) {
+ workerCnt := b.reorgInfo.ReorgMeta.GetConcurrencyOrDefault(int(variable.GetDDLReorgWorkerCounter()))
return min(workerCnt, maxBackfillWorkerSize)
}
@@ -304,7 +305,7 @@ func (b *txnBackfillScheduler) adjustWorkerSize() error {
runner.wg = &b.wg
b.workers = append(b.workers, runner)
b.wg.Add(1)
- go runner.run(reorgInfo.d, worker, job)
+ go runner.run(reorgInfo.jobCtx.oldDDLCtx, worker, job)
}
// Decrease the worker.
if len(b.workers) > workerCnt {
diff --git a/pkg/ddl/bench_test.go b/pkg/ddl/bench_test.go
index e0702e1ed367d..f778e4a5c6ec6 100644
--- a/pkg/ddl/bench_test.go
+++ b/pkg/ddl/bench_test.go
@@ -53,7 +53,7 @@ func BenchmarkExtractDatumByOffsets(b *testing.B) {
endKey := startKey.PrefixNext()
txn, err := store.Begin()
require.NoError(b, err)
- copChunk := ddl.FetchChunk4Test(copCtx, tbl.(table.PhysicalTable), startKey, endKey, store, 10)
+ copChunk, err := FetchChunk4Test(copCtx, tbl.(table.PhysicalTable), startKey, endKey, store, 10)
require.NoError(b, err)
require.NoError(b, txn.Rollback())
@@ -66,7 +66,7 @@ func BenchmarkExtractDatumByOffsets(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
- ddl.ExtractDatumByOffsetsForTest(tk.Session().GetExprCtx().GetEvalCtx(), row, offsets, c.ExprColumnInfos, handleDataBuf)
+ ddl.ExtractDatumByOffsets(tk.Session().GetExprCtx().GetEvalCtx(), row, offsets, c.ExprColumnInfos, handleDataBuf)
}
}
diff --git a/pkg/ddl/callback.go b/pkg/ddl/callback.go
deleted file mode 100644
index 7f51a61cc29f8..0000000000000
--- a/pkg/ddl/callback.go
+++ /dev/null
@@ -1,126 +0,0 @@
-// Copyright 2015 PingCAP, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package ddl
-
-import (
- "fmt"
- "strings"
- "time"
-
- "github.com/pingcap/errors"
- "github.com/pingcap/log"
- "github.com/pingcap/tidb/pkg/ddl/logutil"
- "github.com/pingcap/tidb/pkg/parser/model"
- "go.uber.org/zap"
-)
-
-// Callback is used for DDL.
-type Callback interface {
- // OnJobRunBefore is called before running job.
- OnJobRunBefore(job *model.Job)
- // OnJobRunAfter is called after running job.
- OnJobRunAfter(job *model.Job)
- // OnJobUpdated is called after the running job is updated.
- OnJobUpdated(job *model.Job)
-}
-
-// BaseCallback implements Callback.OnChanged interface.
-type BaseCallback struct {
-}
-
-// OnJobRunBefore implements Callback.OnJobRunBefore interface.
-func (*BaseCallback) OnJobRunBefore(_ *model.Job) {
- // Nothing to do.
-}
-
-// OnJobRunAfter implements Callback.OnJobRunAfter interface.
-func (*BaseCallback) OnJobRunAfter(_ *model.Job) {
- // Nothing to do.
-}
-
-// OnJobUpdated implements Callback.OnJobUpdated interface.
-func (*BaseCallback) OnJobUpdated(_ *model.Job) {
- // Nothing to do.
-}
-
-// SchemaLoader is used to avoid import loop, the only impl is domain currently.
-type SchemaLoader interface {
- Reload() error
-}
-
-// ****************************** Start of Customized DDL Callback Instance ****************************************
-
-// DefaultCallback is the default callback that TiDB will use.
-type DefaultCallback struct {
- *BaseCallback
- do SchemaLoader
-}
-
-func newDefaultCallBack(do SchemaLoader) Callback {
- return &DefaultCallback{BaseCallback: &BaseCallback{}, do: do}
-}
-
-// ****************************** End of Default DDL Callback Instance *********************************************
-
-// ****************************** Start of CTC DDL Callback Instance ***********************************************
-
-// ctcCallback is the customized callback that TiDB will use.
-// ctc is named from column type change, here after we call them ctc for short.
-type ctcCallback struct {
- *BaseCallback
- do SchemaLoader
-}
-
-// OnJobRunBefore is used to run the user customized logic of `onJobRunBefore` first.
-func (*ctcCallback) OnJobRunBefore(job *model.Job) {
- log.Info("on job run before", zap.String("job", job.String()))
- // Only block the ctc type ddl here.
- if job.Type != model.ActionModifyColumn {
- return
- }
- switch job.SchemaState {
- case model.StateDeleteOnly, model.StateWriteOnly, model.StateWriteReorganization:
- logutil.DDLLogger().Warn(fmt.Sprintf("[DDL_HOOK] Hang for 0.5 seconds on %s state triggered", job.SchemaState.String()))
- time.Sleep(500 * time.Millisecond)
- }
-}
-
-func newCTCCallBack(do SchemaLoader) Callback {
- return &ctcCallback{do: do}
-}
-
-// ****************************** End of CTC DDL Callback Instance ***************************************************
-
-var (
- customizedCallBackRegisterMap = map[string]func(do SchemaLoader) Callback{}
-)
-
-func init() {
- // init the callback register map.
- customizedCallBackRegisterMap["default_hook"] = newDefaultCallBack
- customizedCallBackRegisterMap["ctc_hook"] = newCTCCallBack
-}
-
-// GetCustomizedHook get the hook registered in the hookMap.
-func GetCustomizedHook(s string) (func(do SchemaLoader) Callback, error) {
- s = strings.ToLower(s)
- s = strings.TrimSpace(s)
- fact, ok := customizedCallBackRegisterMap[s]
- if !ok {
- logutil.DDLLogger().Error("bad ddl hook " + s)
- return nil, errors.Errorf("ddl hook `%s` is not found in hook registered map", s)
- }
- return fact, nil
-}
diff --git a/pkg/ddl/cancel_test.go b/pkg/ddl/cancel_test.go
index f84563f91b5e2..97272226ecd91 100644
--- a/pkg/ddl/cancel_test.go
+++ b/pkg/ddl/cancel_test.go
@@ -24,7 +24,6 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/ddl"
"github.com/pingcap/tidb/pkg/ddl/testutil"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/testkit"
@@ -214,7 +213,7 @@ func TestCancel(t *testing.T) {
return enterCnt.Load() == exitCnt.Load()
}, 10*time.Second, 10*time.Millisecond)
}
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, 100*time.Millisecond)
+ store := testkit.CreateMockStoreWithSchemaLease(t, 100*time.Millisecond)
tk := testkit.NewTestKit(t, store)
tkCancel := testkit.NewTestKit(t, store)
@@ -243,8 +242,8 @@ func TestCancel(t *testing.T) {
// Change some configurations.
ddl.ReorgWaitTimeout = 10 * time.Millisecond
- tk.MustExec("set @@global.tidb_ddl_reorg_batch_size = 8")
- tk.MustExec("set @@global.tidb_ddl_reorg_worker_cnt = 1")
+ tk.MustExec("set @@tidb_ddl_reorg_batch_size = 8")
+ tk.MustExec("set @@tidb_ddl_reorg_worker_cnt = 1")
tk = testkit.NewTestKit(t, store)
tk.MustExec("use test")
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/mockBackfillSlow", "return"))
@@ -252,7 +251,6 @@ func TestCancel(t *testing.T) {
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/mockBackfillSlow"))
}()
- hook := &callback.TestDDLCallback{Do: dom}
i := atomicutil.NewInt64(0)
canceled := atomicutil.NewBool(false)
cancelResult := atomicutil.NewBool(false)
@@ -268,20 +266,17 @@ func TestCancel(t *testing.T) {
canceled.Store(true)
}
}
- dom.DDL().SetHook(hook.Clone())
- resetHook := func(h *callback.TestDDLCallback) {
- h.OnJobRunBeforeExported = nil
- h.OnJobUpdatedExported.Store(nil)
- dom.DDL().SetHook(h.Clone())
+ resetHook := func() {
+ _ = failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
+ _ = failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/onJobRunBefore")
}
- registerHook := func(h *callback.TestDDLCallback, onJobRunBefore bool) {
+ registerHook := func(onJobRunBefore bool) {
if onJobRunBefore {
- h.OnJobRunBeforeExported = hookFunc
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", hookFunc)
} else {
- h.OnJobUpdatedExported.Store(&hookFunc)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", hookFunc)
}
- dom.DDL().SetHook(h.Clone())
}
waitDDLWorkerExited()
@@ -290,14 +285,14 @@ func TestCancel(t *testing.T) {
i.Store(int64(j))
msg := fmt.Sprintf("sql: %s, state: %s", tc.sql, tc.cancelState)
if tc.onJobBefore {
- resetHook(hook)
+ resetHook()
for _, prepareSQL := range tc.prepareSQL {
tk.MustExec(prepareSQL)
}
waitDDLWorkerExited()
canceled.Store(false)
cancelWhenReorgNotStart.Store(true)
- registerHook(hook, true)
+ registerHook(true)
if tc.expectCancelled {
tk.MustGetErrCode(tc.sql, errno.ErrCancelledDDLJob)
} else {
@@ -309,14 +304,14 @@ func TestCancel(t *testing.T) {
}
}
if tc.onJobUpdate {
- resetHook(hook)
+ resetHook()
for _, prepareSQL := range tc.prepareSQL {
tk.MustExec(prepareSQL)
}
waitDDLWorkerExited()
canceled.Store(false)
cancelWhenReorgNotStart.Store(false)
- registerHook(hook, false)
+ registerHook(false)
if tc.expectCancelled {
tk.MustGetErrCode(tc.sql, errno.ErrCancelledDDLJob)
} else {
@@ -331,7 +326,7 @@ func TestCancel(t *testing.T) {
}
func TestCancelForAddUniqueIndex(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tkCancel := testkit.NewTestKit(t, store)
@@ -342,14 +337,12 @@ func TestCancelForAddUniqueIndex(t *testing.T) {
tk.MustExec("insert into t values(2, 2, 2)")
tk.MustExec("insert into t values(1, 1, 1)")
- hook := &callback.TestDDLCallback{Do: dom}
var testCancelState model.SchemaState
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.SchemaState == testCancelState && job.State == model.JobStateRollingback {
tkCancel.MustExec(fmt.Sprintf("admin cancel ddl jobs %d", job.ID))
}
- }
- dom.DDL().SetHook(hook.Clone())
+ })
testCancelState = model.StateWriteOnly
tk.MustGetErrCode("alter table t add unique index idx1(c1)", errno.ErrDupEntry)
diff --git a/pkg/ddl/cluster.go b/pkg/ddl/cluster.go
index e3b449317f7c7..6dd816aebc939 100644
--- a/pkg/ddl/cluster.go
+++ b/pkg/ddl/cluster.go
@@ -220,7 +220,7 @@ func checkSystemSchemaID(t *meta.Meta, schemaID int64, flashbackTSString string)
return nil
}
-func checkAndSetFlashbackClusterInfo(ctx context.Context, se sessionctx.Context, d *ddlCtx, t *meta.Meta, job *model.Job, flashbackTS uint64) (err error) {
+func checkAndSetFlashbackClusterInfo(ctx context.Context, se sessionctx.Context, store kv.Storage, t *meta.Meta, job *model.Job, flashbackTS uint64) (err error) {
if err = ValidateFlashbackTS(ctx, se, flashbackTS); err != nil {
return err
}
@@ -246,7 +246,7 @@ func checkAndSetFlashbackClusterInfo(ctx context.Context, se sessionctx.Context,
return errors.Trace(err)
}
- flashbackSnapshotMeta := meta.NewSnapshotMeta(d.store.GetSnapshot(kv.NewVersion(flashbackTS)))
+ flashbackSnapshotMeta := meta.NewSnapshotMeta(store.GetSnapshot(kv.NewVersion(flashbackTS)))
flashbackSchemaVersion, err := flashbackSnapshotMeta.GetSchemaVersion()
if err != nil {
return errors.Trace(err)
@@ -322,9 +322,9 @@ func addToSlice(schema string, tableName string, tableID int64, flashbackIDs []i
return flashbackIDs
}
-// GetTableDataKeyRanges get keyRanges by `flashbackIDs`.
+// getTableDataKeyRanges get keyRanges by `flashbackIDs`.
// This func will return all flashback table data key ranges.
-func GetTableDataKeyRanges(nonFlashbackTableIDs []int64) []kv.KeyRange {
+func getTableDataKeyRanges(nonFlashbackTableIDs []int64) []kv.KeyRange {
var keyRanges []kv.KeyRange
nonFlashbackTableIDs = append(nonFlashbackTableIDs, -1)
@@ -349,11 +349,54 @@ func GetTableDataKeyRanges(nonFlashbackTableIDs []int64) []kv.KeyRange {
return keyRanges
}
-// GetFlashbackKeyRanges get keyRanges for flashback cluster.
+type keyRangeMayExclude struct {
+ r kv.KeyRange
+ exclude bool
+}
+
+// appendContinuousKeyRanges merges not exclude continuous key ranges and appends
+// to given []kv.KeyRange, assuming the gap between key ranges has no data.
+//
+// Precondition: schemaKeyRanges is sorted by start key. schemaKeyRanges are
+// non-overlapping.
+func appendContinuousKeyRanges(result []kv.KeyRange, schemaKeyRanges []keyRangeMayExclude) []kv.KeyRange {
+ var (
+ continuousStart, continuousEnd kv.Key
+ )
+
+ for _, r := range schemaKeyRanges {
+ if r.exclude {
+ if continuousStart != nil {
+ result = append(result, kv.KeyRange{
+ StartKey: continuousStart,
+ EndKey: continuousEnd,
+ })
+ continuousStart = nil
+ }
+ continue
+ }
+
+ if continuousStart == nil {
+ continuousStart = r.r.StartKey
+ }
+ continuousEnd = r.r.EndKey
+ }
+
+ if continuousStart != nil {
+ result = append(result, kv.KeyRange{
+ StartKey: continuousStart,
+ EndKey: continuousEnd,
+ })
+ }
+ return result
+}
+
+// getFlashbackKeyRanges get keyRanges for flashback cluster.
// It contains all non system table key ranges and meta data key ranges.
// The time complexity is O(nlogn).
-func GetFlashbackKeyRanges(sess sessionctx.Context, flashbackTS uint64) ([]kv.KeyRange, error) {
- schemas := sess.GetDomainInfoSchema().(infoschema.InfoSchema).AllSchemas()
+func getFlashbackKeyRanges(ctx context.Context, sess sessionctx.Context, flashbackTS uint64) ([]kv.KeyRange, error) {
+ is := sess.GetDomainInfoSchema().(infoschema.InfoSchema)
+ schemas := is.AllSchemas()
// The semantic of keyRanges(output).
keyRanges := make([]kv.KeyRange, 0)
@@ -366,27 +409,52 @@ func GetFlashbackKeyRanges(sess sessionctx.Context, flashbackTS uint64) ([]kv.Ke
}
schemaIDs := make(map[int64]struct{})
+ excludeSchemaIDs := make(map[int64]struct{})
for _, schema := range schemas {
- if !filter.IsSystemSchema(schema.Name.L) {
+ if filter.IsSystemSchema(schema.Name.L) {
+ excludeSchemaIDs[schema.ID] = struct{}{}
+ } else {
schemaIDs[schema.ID] = struct{}{}
}
}
for _, schema := range snapshotSchemas {
- if !filter.IsSystemSchema(schema.Name.L) {
+ if filter.IsSystemSchema(schema.Name.L) {
+ excludeSchemaIDs[schema.ID] = struct{}{}
+ } else {
schemaIDs[schema.ID] = struct{}{}
}
}
- // The meta data key ranges.
+ schemaKeyRanges := make([]keyRangeMayExclude, 0, len(schemaIDs)+len(excludeSchemaIDs))
for schemaID := range schemaIDs {
metaStartKey := tablecodec.EncodeMetaKeyPrefix(meta.DBkey(schemaID))
metaEndKey := tablecodec.EncodeMetaKeyPrefix(meta.DBkey(schemaID + 1))
- keyRanges = append(keyRanges, kv.KeyRange{
- StartKey: metaStartKey,
- EndKey: metaEndKey,
+ schemaKeyRanges = append(schemaKeyRanges, keyRangeMayExclude{
+ r: kv.KeyRange{
+ StartKey: metaStartKey,
+ EndKey: metaEndKey,
+ },
+ exclude: false,
+ })
+ }
+ for schemaID := range excludeSchemaIDs {
+ metaStartKey := tablecodec.EncodeMetaKeyPrefix(meta.DBkey(schemaID))
+ metaEndKey := tablecodec.EncodeMetaKeyPrefix(meta.DBkey(schemaID + 1))
+ schemaKeyRanges = append(schemaKeyRanges, keyRangeMayExclude{
+ r: kv.KeyRange{
+ StartKey: metaStartKey,
+ EndKey: metaEndKey,
+ },
+ exclude: true,
})
}
+ slices.SortFunc(schemaKeyRanges, func(a, b keyRangeMayExclude) int {
+ return bytes.Compare(a.r.StartKey, b.r.StartKey)
+ })
+
+ keyRanges = appendContinuousKeyRanges(keyRanges, schemaKeyRanges)
+
startKey := tablecodec.EncodeMetaKeyPrefix([]byte("DBs"))
keyRanges = append(keyRanges, kv.KeyRange{
StartKey: startKey,
@@ -395,7 +463,11 @@ func GetFlashbackKeyRanges(sess sessionctx.Context, flashbackTS uint64) ([]kv.Ke
var nonFlashbackTableIDs []int64
for _, db := range schemas {
- for _, table := range db.Tables {
+ tbls, err2 := is.SchemaTableInfos(ctx, db.Name)
+ if err2 != nil {
+ return nil, errors.Trace(err2)
+ }
+ for _, table := range tbls {
if !table.IsBaseTable() || table.ID > meta.MaxGlobalID {
continue
}
@@ -408,7 +480,7 @@ func GetFlashbackKeyRanges(sess sessionctx.Context, flashbackTS uint64) ([]kv.Ke
}
}
- return append(keyRanges, GetTableDataKeyRanges(nonFlashbackTableIDs)...), nil
+ return append(keyRanges, getTableDataKeyRanges(nonFlashbackTableIDs)...), nil
}
// SendPrepareFlashbackToVersionRPC prepares regions for flashback, the purpose is to put region into flashback state which region stop write
@@ -597,20 +669,20 @@ func SendFlashbackToVersionRPC(
func flashbackToVersion(
ctx context.Context,
- d *ddlCtx,
+ store kv.Storage,
handler rangetask.TaskHandler,
startKey []byte, endKey []byte,
) (err error) {
return rangetask.NewRangeTaskRunner(
"flashback-to-version-runner",
- d.store.(tikv.Storage),
+ store.(tikv.Storage),
int(variable.GetDDLFlashbackConcurrency()),
handler,
).RunOnRange(ctx, startKey, endKey)
}
-func splitRegionsByKeyRanges(ctx context.Context, d *ddlCtx, keyRanges []kv.KeyRange) {
- if s, ok := d.store.(kv.SplittableStore); ok {
+func splitRegionsByKeyRanges(ctx context.Context, store kv.Storage, keyRanges []kv.KeyRange) {
+ if s, ok := store.(kv.SplittableStore); ok {
for _, keys := range keyRanges {
for {
// tableID is useless when scatter == false
@@ -628,7 +700,7 @@ func splitRegionsByKeyRanges(ctx context.Context, d *ddlCtx, keyRanges []kv.KeyR
// 2. before flashback start, check timestamp, disable GC and close PD schedule, get flashback key ranges.
// 3. phase 1, lock flashback key ranges.
// 4. phase 2, send flashback RPC, do flashback jobs.
-func (w *worker) onFlashbackCluster(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func (w *worker) onFlashbackCluster(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
inFlashbackTest := false
failpoint.Inject("mockFlashbackTest", func(val failpoint.Value) {
if val.(bool) {
@@ -636,7 +708,7 @@ func (w *worker) onFlashbackCluster(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
}
})
// TODO: Support flashback in unistore.
- if d.store.Name() != "TiKV" && !inFlashbackTest {
+ if jobCtx.store.Name() != "TiKV" && !inFlashbackTest {
job.State = model.JobStateCancelled
return ver, errors.Errorf("Not support flashback cluster in non-TiKV env")
}
@@ -696,38 +768,38 @@ func (w *worker) onFlashbackCluster(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
return ver, nil
// Stage 2, check flashbackTS, close GC and PD schedule, get flashback key ranges.
case model.StateDeleteOnly:
- if err = checkAndSetFlashbackClusterInfo(w.ctx, sess, d, t, job, flashbackTS); err != nil {
+ if err = checkAndSetFlashbackClusterInfo(w.ctx, sess, jobCtx.store, t, job, flashbackTS); err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
}
// We should get startTS here to avoid lost startTS when TiDB crashed during send prepare flashback RPC.
- startTS, err = d.store.GetOracle().GetTimestamp(w.ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope})
+ startTS, err = jobCtx.store.GetOracle().GetTimestamp(w.ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope})
if err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
}
job.Args[startTSOffset] = startTS
- keyRanges, err = GetFlashbackKeyRanges(sess, flashbackTS)
+ keyRanges, err = getFlashbackKeyRanges(w.ctx, sess, flashbackTS)
if err != nil {
return ver, errors.Trace(err)
}
job.Args[keyRangesOffset] = keyRanges
job.SchemaState = model.StateWriteOnly
- return updateSchemaVersion(d, t, job)
+ return updateSchemaVersion(jobCtx, t, job)
// Stage 3, lock related key ranges.
case model.StateWriteOnly:
// TODO: Support flashback in unistore.
if inFlashbackTest {
job.SchemaState = model.StateWriteReorganization
- return updateSchemaVersion(d, t, job)
+ return updateSchemaVersion(jobCtx, t, job)
}
// Split region by keyRanges, make sure no unrelated key ranges be locked.
- splitRegionsByKeyRanges(w.ctx, d, keyRanges)
+ splitRegionsByKeyRanges(w.ctx, jobCtx.store, keyRanges)
totalRegions.Store(0)
for _, r := range keyRanges {
- if err = flashbackToVersion(w.ctx, d,
+ if err = flashbackToVersion(w.ctx, jobCtx.store,
func(ctx context.Context, r tikvstore.KeyRange) (rangetask.TaskStat, error) {
- stats, err := SendPrepareFlashbackToVersionRPC(ctx, d.store.(tikv.Storage), flashbackTS, startTS, r)
+ stats, err := SendPrepareFlashbackToVersionRPC(ctx, jobCtx.store.(tikv.Storage), flashbackTS, startTS, r)
totalRegions.Add(uint64(stats.CompletedRegions))
return stats, err
}, r.StartKey, r.EndKey); err != nil {
@@ -738,7 +810,7 @@ func (w *worker) onFlashbackCluster(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
job.Args[totalLockedRegionsOffset] = totalRegions.Load()
// We should get commitTS here to avoid lost commitTS when TiDB crashed during send flashback RPC.
- commitTS, err = d.store.GetOracle().GetTimestamp(w.ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope})
+ commitTS, err = jobCtx.store.GetOracle().GetTimestamp(w.ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope})
if err != nil {
return ver, errors.Trace(err)
}
@@ -749,17 +821,17 @@ func (w *worker) onFlashbackCluster(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
case model.StateWriteReorganization:
// TODO: Support flashback in unistore.
if inFlashbackTest {
- asyncNotifyEvent(d, statsutil.NewFlashbackClusterEvent())
+ asyncNotifyEvent(jobCtx, statsutil.NewFlashbackClusterEvent())
job.State = model.JobStateDone
job.SchemaState = model.StatePublic
return ver, nil
}
for _, r := range keyRanges {
- if err = flashbackToVersion(w.ctx, d,
+ if err = flashbackToVersion(w.ctx, jobCtx.store,
func(ctx context.Context, r tikvstore.KeyRange) (rangetask.TaskStat, error) {
// Use same startTS as prepare phase to simulate 1PC txn.
- stats, err := SendFlashbackToVersionRPC(ctx, d.store.(tikv.Storage), flashbackTS, startTS, commitTS, r)
+ stats, err := SendFlashbackToVersionRPC(ctx, jobCtx.store.(tikv.Storage), flashbackTS, startTS, commitTS, r)
completedRegions.Add(uint64(stats.CompletedRegions))
logutil.DDLLogger().Info("flashback cluster stats",
zap.Uint64("complete regions", completedRegions.Load()),
@@ -772,10 +844,10 @@ func (w *worker) onFlashbackCluster(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
}
}
- asyncNotifyEvent(d, statsutil.NewFlashbackClusterEvent())
+ asyncNotifyEvent(jobCtx, statsutil.NewFlashbackClusterEvent())
job.State = model.JobStateDone
job.SchemaState = model.StatePublic
- return updateSchemaVersion(d, t, job)
+ return updateSchemaVersion(jobCtx, t, job)
}
return ver, nil
}
diff --git a/pkg/ddl/cluster_test.go b/pkg/ddl/cluster_test.go
index 8844c11e98411..9003d25646f3b 100644
--- a/pkg/ddl/cluster_test.go
+++ b/pkg/ddl/cluster_test.go
@@ -21,14 +21,10 @@ import (
"time"
"github.com/pingcap/failpoint"
- "github.com/pingcap/tidb/pkg/ddl"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/domain/infosync"
"github.com/pingcap/tidb/pkg/errno"
- "github.com/pingcap/tidb/pkg/meta"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
- "github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/types"
@@ -38,37 +34,8 @@ import (
"github.com/tikv/client-go/v2/oracle"
)
-func TestGetTableDataKeyRanges(t *testing.T) {
- // case 1, empty flashbackIDs
- keyRanges := ddl.GetTableDataKeyRanges([]int64{})
- require.Len(t, keyRanges, 1)
- require.Equal(t, keyRanges[0].StartKey, tablecodec.EncodeTablePrefix(0))
- require.Equal(t, keyRanges[0].EndKey, tablecodec.EncodeTablePrefix(meta.MaxGlobalID))
-
- // case 2, insert a execluded table ID
- keyRanges = ddl.GetTableDataKeyRanges([]int64{3})
- require.Len(t, keyRanges, 2)
- require.Equal(t, keyRanges[0].StartKey, tablecodec.EncodeTablePrefix(0))
- require.Equal(t, keyRanges[0].EndKey, tablecodec.EncodeTablePrefix(3))
- require.Equal(t, keyRanges[1].StartKey, tablecodec.EncodeTablePrefix(4))
- require.Equal(t, keyRanges[1].EndKey, tablecodec.EncodeTablePrefix(meta.MaxGlobalID))
-
- // case 3, insert some execluded table ID
- keyRanges = ddl.GetTableDataKeyRanges([]int64{3, 5, 9})
- require.Len(t, keyRanges, 4)
- require.Equal(t, keyRanges[0].StartKey, tablecodec.EncodeTablePrefix(0))
- require.Equal(t, keyRanges[0].EndKey, tablecodec.EncodeTablePrefix(3))
- require.Equal(t, keyRanges[1].StartKey, tablecodec.EncodeTablePrefix(4))
- require.Equal(t, keyRanges[1].EndKey, tablecodec.EncodeTablePrefix(5))
- require.Equal(t, keyRanges[2].StartKey, tablecodec.EncodeTablePrefix(6))
- require.Equal(t, keyRanges[2].EndKey, tablecodec.EncodeTablePrefix(9))
- require.Equal(t, keyRanges[3].StartKey, tablecodec.EncodeTablePrefix(10))
- require.Equal(t, keyRanges[3].EndKey, tablecodec.EncodeTablePrefix(meta.MaxGlobalID))
-}
-
func TestFlashbackCloseAndResetPDSchedule(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
- originHook := dom.DDL().GetHook()
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
injectSafeTS := oracle.GoTimeToTS(time.Now().Add(10 * time.Second))
@@ -85,31 +52,30 @@ func TestFlashbackCloseAndResetPDSchedule(t *testing.T) {
defer resetGC()
tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop))
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
assert.Equal(t, model.ActionFlashbackCluster, job.Type)
if job.SchemaState == model.StateWriteReorganization {
closeValue, err := infosync.GetPDScheduleConfig(context.Background())
assert.NoError(t, err)
assert.Equal(t, closeValue["merge-schedule-limit"], 0)
}
- }
- hook.OnJobRunAfterExported = func(job *model.Job) {
+ })
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunAfter", func(job *model.Job) {
assert.Equal(t, model.ActionFlashbackCluster, job.Type)
if job.SchemaState == model.StateWriteReorganization {
// cancel flashback job
job.State = model.JobStateCancelled
job.Error = dbterror.ErrCancelledDDLJob
}
- }
- dom.DDL().SetHook(hook)
+ })
time.Sleep(10 * time.Millisecond)
ts, err := tk.Session().GetStore().GetOracle().GetTimestamp(context.Background(), &oracle.Option{})
require.NoError(t, err)
tk.MustGetErrCode(fmt.Sprintf("flashback cluster to timestamp '%s'", oracle.GetTimeFromTS(ts).Format(types.TimeFSPFormat)), errno.ErrCancelledDDLJob)
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore")
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunAfter")
finishValue, err := infosync.GetPDScheduleConfig(context.Background())
require.NoError(t, err)
@@ -117,8 +83,7 @@ func TestFlashbackCloseAndResetPDSchedule(t *testing.T) {
}
func TestAddDDLDuringFlashback(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
- originHook := dom.DDL().GetHook()
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("create table t(a int)")
@@ -136,26 +101,23 @@ func TestAddDDLDuringFlashback(t *testing.T) {
defer resetGC()
tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop))
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
assert.Equal(t, model.ActionFlashbackCluster, job.Type)
if job.SchemaState == model.StateWriteOnly {
tk1 := testkit.NewTestKit(t, store)
_, err := tk1.Exec("alter table test.t add column b int")
assert.ErrorContains(t, err, "Can't add ddl job, have flashback cluster job")
}
- }
- dom.DDL().SetHook(hook)
+ })
tk.MustExec(fmt.Sprintf("flashback cluster to timestamp '%s'", oracle.GetTimeFromTS(ts).Format(types.TimeFSPFormat)))
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore")
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/mockFlashbackTest"))
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/injectSafeTS"))
}
func TestGlobalVariablesOnFlashback(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
- originHook := dom.DDL().GetHook()
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("create table t(a int)")
@@ -173,8 +135,7 @@ func TestGlobalVariablesOnFlashback(t *testing.T) {
defer resetGC()
tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop))
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
assert.Equal(t, model.ActionFlashbackCluster, job.Type)
if job.SchemaState == model.StateWriteReorganization {
rs, err := tk.Exec("show variables like 'tidb_gc_enable'")
@@ -190,8 +151,7 @@ func TestGlobalVariablesOnFlashback(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, tk.ResultSetToResult(rs, "").Rows()[0][1], variable.Off)
}
- }
- dom.DDL().SetHook(hook)
+ })
// first try with `tidb_gc_enable` = on and `tidb_super_read_only` = off and `tidb_ttl_job_enable` = on
tk.MustExec("set global tidb_gc_enable = on")
tk.MustExec("set global tidb_super_read_only = off")
@@ -227,14 +187,13 @@ func TestGlobalVariablesOnFlashback(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, tk.ResultSetToResult(rs, "").Rows()[0][1], variable.Off)
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore")
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/mockFlashbackTest"))
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/injectSafeTS"))
}
func TestCancelFlashbackCluster(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
- originHook := dom.DDL().GetHook()
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
time.Sleep(10 * time.Millisecond)
@@ -251,10 +210,10 @@ func TestCancelFlashbackCluster(t *testing.T) {
tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop))
// Try canceled on StateDeleteOnly, cancel success
- hook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
+ hook := newCancelJobHook(t, store, func(job *model.Job) bool {
return job.SchemaState == model.StateDeleteOnly
})
- dom.DDL().SetHook(hook)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", hook.OnJobUpdated)
tk.MustExec("set global tidb_ttl_job_enable = on")
tk.MustGetErrCode(fmt.Sprintf("flashback cluster to timestamp '%s'", oracle.GetTimeFromTS(ts).Format(types.TimeFSPFormat)), errno.ErrCancelledDDLJob)
hook.MustCancelDone(t)
@@ -264,10 +223,10 @@ func TestCancelFlashbackCluster(t *testing.T) {
assert.Equal(t, tk.ResultSetToResult(rs, "").Rows()[0][1], variable.On)
// Try canceled on StateWriteReorganization, cancel failed
- hook = newCancelJobHook(t, store, dom, func(job *model.Job) bool {
+ hook = newCancelJobHook(t, store, func(job *model.Job) bool {
return job.SchemaState == model.StateWriteReorganization
})
- dom.DDL().SetHook(hook)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", hook.OnJobUpdated)
tk.MustExec(fmt.Sprintf("flashback cluster to timestamp '%s'", oracle.GetTimeFromTS(ts).Format(types.TimeFSPFormat)))
hook.MustCancelFailed(t)
@@ -275,8 +234,6 @@ func TestCancelFlashbackCluster(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, tk.ResultSetToResult(rs, "").Rows()[0][1], variable.Off)
- dom.DDL().SetHook(originHook)
-
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/mockFlashbackTest"))
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/injectSafeTS"))
}
diff --git a/pkg/ddl/column.go b/pkg/ddl/column.go
index c479c005a8c82..629e6f77d121c 100644
--- a/pkg/ddl/column.go
+++ b/pkg/ddl/column.go
@@ -28,25 +28,20 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/config"
"github.com/pingcap/tidb/pkg/ddl/logutil"
- sess "github.com/pingcap/tidb/pkg/ddl/session"
"github.com/pingcap/tidb/pkg/expression"
- exprctx "github.com/pingcap/tidb/pkg/expression/context"
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/meta"
"github.com/pingcap/tidb/pkg/meta/autoid"
- "github.com/pingcap/tidb/pkg/metrics"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/parser/terror"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
- statsutil "github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/types"
- "github.com/pingcap/tidb/pkg/util"
contextutil "github.com/pingcap/tidb/pkg/util/context"
"github.com/pingcap/tidb/pkg/util/dbterror"
decoder "github.com/pingcap/tidb/pkg/util/rowDecoder"
@@ -104,100 +99,6 @@ func checkAddColumn(t *meta.Meta, job *model.Job) (*model.TableInfo, *model.Colu
return tblInfo, columnInfo, col, pos, false, nil
}
-func onAddColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
- // Handle the rolling back job.
- if job.IsRollingback() {
- ver, err = onDropColumn(d, t, job)
- if err != nil {
- return ver, errors.Trace(err)
- }
- return ver, nil
- }
-
- failpoint.Inject("errorBeforeDecodeArgs", func(val failpoint.Value) {
- //nolint:forcetypeassert
- if val.(bool) {
- failpoint.Return(ver, errors.New("occur an error before decode args"))
- }
- })
-
- tblInfo, columnInfo, colFromArgs, pos, ifNotExists, err := checkAddColumn(t, job)
- if err != nil {
- if ifNotExists && infoschema.ErrColumnExists.Equal(err) {
- job.Warning = toTError(err)
- job.State = model.JobStateDone
- return ver, nil
- }
- return ver, errors.Trace(err)
- }
- if columnInfo == nil {
- columnInfo = InitAndAddColumnToTable(tblInfo, colFromArgs)
- logutil.DDLLogger().Info("run add column job", zap.Stringer("job", job), zap.Reflect("columnInfo", *columnInfo))
- if err = checkAddColumnTooManyColumns(len(tblInfo.Columns)); err != nil {
- job.State = model.JobStateCancelled
- return ver, errors.Trace(err)
- }
- }
-
- originalState := columnInfo.State
- switch columnInfo.State {
- case model.StateNone:
- // none -> delete only
- columnInfo.State = model.StateDeleteOnly
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, originalState != columnInfo.State)
- if err != nil {
- return ver, errors.Trace(err)
- }
- job.SchemaState = model.StateDeleteOnly
- case model.StateDeleteOnly:
- // delete only -> write only
- columnInfo.State = model.StateWriteOnly
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != columnInfo.State)
- if err != nil {
- return ver, errors.Trace(err)
- }
- // Update the job state when all affairs done.
- job.SchemaState = model.StateWriteOnly
- case model.StateWriteOnly:
- // write only -> reorganization
- columnInfo.State = model.StateWriteReorganization
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != columnInfo.State)
- if err != nil {
- return ver, errors.Trace(err)
- }
- // Update the job state when all affairs done.
- job.SchemaState = model.StateWriteReorganization
- job.MarkNonRevertible()
- case model.StateWriteReorganization:
- // reorganization -> public
- // Adjust table column offset.
- failpoint.InjectCall("onAddColumnStateWriteReorg")
- offset, err := LocateOffsetToMove(columnInfo.Offset, pos, tblInfo)
- if err != nil {
- return ver, errors.Trace(err)
- }
- tblInfo.MoveColumnInfo(columnInfo.Offset, offset)
- columnInfo.State = model.StatePublic
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != columnInfo.State)
- if err != nil {
- return ver, errors.Trace(err)
- }
-
- // Finish this job.
- job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tblInfo)
- addColumnEvent := statsutil.NewAddColumnEvent(
- job.SchemaID,
- tblInfo,
- []*model.ColumnInfo{columnInfo},
- )
- asyncNotifyEvent(d, addColumnEvent)
- default:
- err = dbterror.ErrInvalidDDLState.GenWithStackByArgs("column", columnInfo.State)
- }
-
- return ver, errors.Trace(err)
-}
-
// CheckAfterPositionExists makes sure the column specified in AFTER clause is exists.
// For example, ALTER TABLE t ADD COLUMN c3 INT AFTER c1.
func CheckAfterPositionExists(tblInfo *model.TableInfo, pos *ast.ColumnPosition) error {
@@ -236,8 +137,8 @@ func checkDropColumnForStatePublic(colInfo *model.ColumnInfo) (err error) {
return nil
}
-func onDropColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
- tblInfo, colInfo, idxInfos, ifExists, err := checkDropColumn(d, t, job)
+func onDropColumn(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+ tblInfo, colInfo, idxInfos, ifExists, err := checkDropColumn(jobCtx, t, job)
if err != nil {
if ifExists && dbterror.ErrCantDropFieldOrKey.Equal(err) {
// Convert the "not exists" error to a warning.
@@ -251,7 +152,7 @@ func onDropColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error)
job.MarkNonRevertible()
job.SchemaState = colInfo.State
// Store the mark and enter the next DDL handling loop.
- return updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, false)
+ return updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, false)
}
originalState := colInfo.State
@@ -265,7 +166,7 @@ func onDropColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error)
if err != nil {
return ver, errors.Trace(err)
}
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, originalState != colInfo.State)
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, originalState != colInfo.State)
if err != nil {
return ver, errors.Trace(err)
}
@@ -283,7 +184,7 @@ func onDropColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error)
}
tblInfo.Indices = newIndices
}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != colInfo.State)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != colInfo.State)
if err != nil {
return ver, errors.Trace(err)
}
@@ -292,7 +193,7 @@ func onDropColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error)
// delete only -> reorganization
colInfo.State = model.StateDeleteReorganization
tblInfo.MoveColumnInfo(colInfo.Offset, len(tblInfo.Columns)-1)
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != colInfo.State)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != colInfo.State)
if err != nil {
return ver, errors.Trace(err)
}
@@ -302,7 +203,7 @@ func onDropColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error)
tblInfo.MoveColumnInfo(colInfo.Offset, len(tblInfo.Columns)-1)
tblInfo.Columns = tblInfo.Columns[:len(tblInfo.Columns)-1]
colInfo.State = model.StateNone
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != colInfo.State)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != colInfo.State)
if err != nil {
return ver, errors.Trace(err)
}
@@ -322,7 +223,7 @@ func onDropColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error)
return ver, errors.Trace(err)
}
-func checkDropColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (*model.TableInfo, *model.ColumnInfo, []*model.IndexInfo, bool /* ifExists */, error) {
+func checkDropColumn(jobCtx *jobContext, t *meta.Meta, job *model.Job) (*model.TableInfo, *model.ColumnInfo, []*model.IndexInfo, bool /* ifExists */, error) {
schemaID := job.SchemaID
tblInfo, err := GetTableInfoAndCancelFaultJob(t, job, schemaID)
if err != nil {
@@ -348,7 +249,7 @@ func checkDropColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (*model.TableInfo,
job.State = model.JobStateCancelled
return nil, nil, nil, false, errors.Trace(err)
}
- if err = checkDropColumnWithForeignKeyConstraintInOwner(d, t, job, tblInfo, colName.L); err != nil {
+ if err = checkDropColumnWithForeignKeyConstraintInOwner(jobCtx.infoCache, job, tblInfo, colName.L); err != nil {
return nil, nil, nil, false, errors.Trace(err)
}
if err = checkDropColumnWithTTLConfig(tblInfo, colName.L); err != nil {
@@ -358,271 +259,39 @@ func checkDropColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (*model.TableInfo,
return tblInfo, colInfo, idxInfos, false, nil
}
-func onSetDefaultValue(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
- newCol := &model.ColumnInfo{}
- err := job.DecodeArgs(newCol)
- if err != nil {
- job.State = model.JobStateCancelled
- return ver, errors.Trace(err)
- }
-
- return updateColumnDefaultValue(d, t, job, newCol, &newCol.Name)
-}
-
-func needChangeColumnData(oldCol, newCol *model.ColumnInfo) bool {
- toUnsigned := mysql.HasUnsignedFlag(newCol.GetFlag())
- originUnsigned := mysql.HasUnsignedFlag(oldCol.GetFlag())
- needTruncationOrToggleSign := func() bool {
- return (newCol.GetFlen() > 0 && (newCol.GetFlen() < oldCol.GetFlen() || newCol.GetDecimal() < oldCol.GetDecimal())) ||
- (toUnsigned != originUnsigned)
- }
- // Ignore the potential max display length represented by integer's flen, use default flen instead.
- defaultOldColFlen, _ := mysql.GetDefaultFieldLengthAndDecimal(oldCol.GetType())
- defaultNewColFlen, _ := mysql.GetDefaultFieldLengthAndDecimal(newCol.GetType())
- needTruncationOrToggleSignForInteger := func() bool {
- return (defaultNewColFlen > 0 && defaultNewColFlen < defaultOldColFlen) || (toUnsigned != originUnsigned)
- }
-
- // Deal with the same type.
- if oldCol.GetType() == newCol.GetType() {
- switch oldCol.GetType() {
- case mysql.TypeNewDecimal:
- // Since type decimal will encode the precision, frac, negative(signed) and wordBuf into storage together, there is no short
- // cut to eliminate data reorg change for column type change between decimal.
- return oldCol.GetFlen() != newCol.GetFlen() || oldCol.GetDecimal() != newCol.GetDecimal() || toUnsigned != originUnsigned
- case mysql.TypeEnum, mysql.TypeSet:
- return IsElemsChangedToModifyColumn(oldCol.GetElems(), newCol.GetElems())
- case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong:
- return toUnsigned != originUnsigned
- case mysql.TypeString:
- // Due to the behavior of padding \x00 at binary type, always change column data when binary length changed
- if types.IsBinaryStr(&oldCol.FieldType) {
- return newCol.GetFlen() != oldCol.GetFlen()
- }
- case mysql.TypeTiDBVectorFloat32:
- return newCol.GetFlen() != types.UnspecifiedLength && oldCol.GetFlen() != newCol.GetFlen()
- }
-
- return needTruncationOrToggleSign()
- }
-
- if ConvertBetweenCharAndVarchar(oldCol.GetType(), newCol.GetType()) {
- return true
- }
-
- // Deal with the different type.
- switch oldCol.GetType() {
- case mysql.TypeVarchar, mysql.TypeString, mysql.TypeVarString, mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob:
- switch newCol.GetType() {
- case mysql.TypeVarchar, mysql.TypeString, mysql.TypeVarString, mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob:
- return needTruncationOrToggleSign()
- }
- case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong:
- switch newCol.GetType() {
- case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong:
- return needTruncationOrToggleSignForInteger()
- }
- // conversion between float and double needs reorganization, see issue #31372
- }
-
- return true
-}
-
-// ConvertBetweenCharAndVarchar check whether column converted between char and varchar
-// TODO: it is used for plugins. so change plugin's using and remove it.
-func ConvertBetweenCharAndVarchar(oldCol, newCol byte) bool {
- return types.ConvertBetweenCharAndVarchar(oldCol, newCol)
-}
-
-// IsElemsChangedToModifyColumn check elems changed
-func IsElemsChangedToModifyColumn(oldElems, newElems []string) bool {
- if len(newElems) < len(oldElems) {
- return true
- }
- for index, oldElem := range oldElems {
- newElem := newElems[index]
- if oldElem != newElem {
- return true
+func isDroppableColumn(tblInfo *model.TableInfo, colName model.CIStr) error {
+ if ok, dep, isHidden := hasDependentByGeneratedColumn(tblInfo, colName); ok {
+ if isHidden {
+ return dbterror.ErrDependentByFunctionalIndex.GenWithStackByArgs(dep)
}
+ return dbterror.ErrDependentByGeneratedColumn.GenWithStackByArgs(dep)
}
- return false
-}
-
-type modifyingColInfo struct {
- newCol *model.ColumnInfo
- oldColName *model.CIStr
- modifyColumnTp byte
- updatedAutoRandomBits uint64
- changingCol *model.ColumnInfo
- changingIdxs []*model.IndexInfo
- pos *ast.ColumnPosition
- removedIdxs []int64
-}
-func getModifyColumnInfo(t *meta.Meta, job *model.Job) (*model.DBInfo, *model.TableInfo, *model.ColumnInfo, *modifyingColInfo, error) {
- modifyInfo := &modifyingColInfo{pos: &ast.ColumnPosition{}}
- err := job.DecodeArgs(&modifyInfo.newCol, &modifyInfo.oldColName, modifyInfo.pos, &modifyInfo.modifyColumnTp,
- &modifyInfo.updatedAutoRandomBits, &modifyInfo.changingCol, &modifyInfo.changingIdxs, &modifyInfo.removedIdxs)
- if err != nil {
- job.State = model.JobStateCancelled
- return nil, nil, nil, modifyInfo, errors.Trace(err)
+ if len(tblInfo.Columns) == 1 {
+ return dbterror.ErrCantRemoveAllFields.GenWithStack("can't drop only column %s in table %s",
+ colName, tblInfo.Name)
}
-
- dbInfo, err := checkSchemaExistAndCancelNotExistJob(t, job)
+ // We only support dropping column with single-value none Primary Key index covered now.
+ err := isColumnCanDropWithIndex(colName.L, tblInfo.Indices)
if err != nil {
- return nil, nil, nil, modifyInfo, errors.Trace(err)
+ return err
}
-
- tblInfo, err := GetTableInfoAndCancelFaultJob(t, job, job.SchemaID)
+ err = IsColumnDroppableWithCheckConstraint(colName, tblInfo)
if err != nil {
- return nil, nil, nil, modifyInfo, errors.Trace(err)
- }
-
- oldCol := model.FindColumnInfo(tblInfo.Columns, modifyInfo.oldColName.L)
- if oldCol == nil || oldCol.State != model.StatePublic {
- job.State = model.JobStateCancelled
- return nil, nil, nil, modifyInfo, errors.Trace(infoschema.ErrColumnNotExists.GenWithStackByArgs(*(modifyInfo.oldColName), tblInfo.Name))
- }
-
- return dbInfo, tblInfo, oldCol, modifyInfo, errors.Trace(err)
-}
-
-// GetOriginDefaultValueForModifyColumn gets the original default value for modifying column.
-// Since column type change is implemented as adding a new column then substituting the old one.
-// Case exists when update-where statement fetch a NULL for not-null column without any default data,
-// it will errors.
-// So we set original default value here to prevent this error. If the oldCol has the original default value, we use it.
-// Otherwise we set the zero value as original default value.
-// Besides, in insert & update records, we have already implement using the casted value of relative column to insert
-// rather than the original default value.
-func GetOriginDefaultValueForModifyColumn(ctx exprctx.BuildContext, changingCol, oldCol *model.ColumnInfo) (any, error) {
- var err error
- originDefVal := oldCol.GetOriginDefaultValue()
- if originDefVal != nil {
- odv, err := table.CastColumnValue(ctx, types.NewDatum(originDefVal), changingCol, false, false)
- if err != nil {
- logutil.DDLLogger().Info("cast origin default value failed", zap.Error(err))
- }
- if !odv.IsNull() {
- if originDefVal, err = odv.ToString(); err != nil {
- originDefVal = nil
- logutil.DDLLogger().Info("convert default value to string failed", zap.Error(err))
- }
- }
- }
- if originDefVal == nil {
- originDefVal, err = generateOriginDefaultValue(changingCol, nil)
- if err != nil {
- return nil, errors.Trace(err)
- }
+ return err
}
- return originDefVal, nil
+ return nil
}
-func (w *worker) onModifyColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
- dbInfo, tblInfo, oldCol, modifyInfo, err := getModifyColumnInfo(t, job)
- if err != nil {
- return ver, err
- }
-
- if job.IsRollingback() {
- // For those column-type-change jobs which don't reorg the data.
- if !needChangeColumnData(oldCol, modifyInfo.newCol) {
- return rollbackModifyColumnJob(d, t, tblInfo, job, modifyInfo.newCol, oldCol, modifyInfo.modifyColumnTp)
- }
- // For those column-type-change jobs which reorg the data.
- return rollbackModifyColumnJobWithData(d, t, tblInfo, job, oldCol, modifyInfo)
- }
-
- // If we want to rename the column name, we need to check whether it already exists.
- if modifyInfo.newCol.Name.L != modifyInfo.oldColName.L {
- c := model.FindColumnInfo(tblInfo.Columns, modifyInfo.newCol.Name.L)
- if c != nil {
- job.State = model.JobStateCancelled
- return ver, errors.Trace(infoschema.ErrColumnExists.GenWithStackByArgs(modifyInfo.newCol.Name))
- }
- }
-
- failpoint.Inject("uninitializedOffsetAndState", func(val failpoint.Value) {
- //nolint:forcetypeassert
- if val.(bool) {
- if modifyInfo.newCol.State != model.StatePublic {
- failpoint.Return(ver, errors.New("the column state is wrong"))
- }
- }
- })
-
- err = checkAndApplyAutoRandomBits(d, t, dbInfo, tblInfo, oldCol, modifyInfo.newCol, modifyInfo.updatedAutoRandomBits)
+func onSetDefaultValue(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+ newCol := &model.ColumnInfo{}
+ err := job.DecodeArgs(newCol)
if err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
}
- if !needChangeColumnData(oldCol, modifyInfo.newCol) {
- return w.doModifyColumn(d, t, job, dbInfo, tblInfo, modifyInfo.newCol, oldCol, modifyInfo.pos)
- }
-
- if err = isGeneratedRelatedColumn(tblInfo, modifyInfo.newCol, oldCol); err != nil {
- job.State = model.JobStateCancelled
- return ver, errors.Trace(err)
- }
- if tblInfo.Partition != nil {
- job.State = model.JobStateCancelled
- return ver, errors.Trace(dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs("table is partition table"))
- }
-
- changingCol := modifyInfo.changingCol
- if changingCol == nil {
- newColName := model.NewCIStr(genChangingColumnUniqueName(tblInfo, oldCol))
- if mysql.HasPriKeyFlag(oldCol.GetFlag()) {
- job.State = model.JobStateCancelled
- msg := "this column has primary key flag"
- return ver, dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs(msg)
- }
-
- changingCol = modifyInfo.newCol.Clone()
- changingCol.Name = newColName
- changingCol.ChangeStateInfo = &model.ChangeStateInfo{DependencyColumnOffset: oldCol.Offset}
-
- originDefVal, err := GetOriginDefaultValueForModifyColumn(newReorgExprCtx(), changingCol, oldCol)
- if err != nil {
- return ver, errors.Trace(err)
- }
- if err = changingCol.SetOriginDefaultValue(originDefVal); err != nil {
- return ver, errors.Trace(err)
- }
-
- InitAndAddColumnToTable(tblInfo, changingCol)
- indexesToChange := FindRelatedIndexesToChange(tblInfo, oldCol.Name)
- for _, info := range indexesToChange {
- newIdxID := AllocateIndexID(tblInfo)
- if !info.isTemp {
- // We create a temp index for each normal index.
- tmpIdx := info.IndexInfo.Clone()
- tmpIdxName := genChangingIndexUniqueName(tblInfo, info.IndexInfo)
- setIdxIDName(tmpIdx, newIdxID, model.NewCIStr(tmpIdxName))
- SetIdxColNameOffset(tmpIdx.Columns[info.Offset], changingCol)
- tblInfo.Indices = append(tblInfo.Indices, tmpIdx)
- } else {
- // The index is a temp index created by previous modify column job(s).
- // We can overwrite it to reduce reorg cost, because it will be dropped eventually.
- tmpIdx := info.IndexInfo
- oldTempIdxID := tmpIdx.ID
- setIdxIDName(tmpIdx, newIdxID, tmpIdx.Name /* unchanged */)
- SetIdxColNameOffset(tmpIdx.Columns[info.Offset], changingCol)
- modifyInfo.removedIdxs = append(modifyInfo.removedIdxs, oldTempIdxID)
- }
- }
- } else {
- changingCol = model.FindColumnInfoByID(tblInfo.Columns, modifyInfo.changingCol.ID)
- if changingCol == nil {
- logutil.DDLLogger().Error("the changing column has been removed", zap.Error(err))
- job.State = model.JobStateCancelled
- return ver, errors.Trace(infoschema.ErrColumnNotExists.GenWithStackByArgs(oldCol.Name, tblInfo.Name))
- }
- }
-
- return w.doModifyColumnTypeWithData(d, t, job, dbInfo, tblInfo, changingCol, oldCol, modifyInfo.newCol.Name, modifyInfo.pos, modifyInfo.removedIdxs)
+ return updateColumnDefaultValue(jobCtx, t, job, newCol, &newCol.Name)
}
func setIdxIDName(idxInfo *model.IndexInfo, newID int64, newName model.CIStr) {
@@ -640,32 +309,6 @@ func SetIdxColNameOffset(idxCol *model.IndexColumn, changingCol *model.ColumnInf
}
}
-// rollbackModifyColumnJobWithData is used to rollback modify-column job which need to reorg the data.
-func rollbackModifyColumnJobWithData(d *ddlCtx, t *meta.Meta, tblInfo *model.TableInfo, job *model.Job, oldCol *model.ColumnInfo, modifyInfo *modifyingColInfo) (ver int64, err error) {
- // If the not-null change is included, we should clean the flag info in oldCol.
- if modifyInfo.modifyColumnTp == mysql.TypeNull {
- // Reset NotNullFlag flag.
- tblInfo.Columns[oldCol.Offset].SetFlag(oldCol.GetFlag() &^ mysql.NotNullFlag)
- // Reset PreventNullInsertFlag flag.
- tblInfo.Columns[oldCol.Offset].SetFlag(oldCol.GetFlag() &^ mysql.PreventNullInsertFlag)
- }
- var changingIdxIDs []int64
- if modifyInfo.changingCol != nil {
- changingIdxIDs = buildRelatedIndexIDs(tblInfo, modifyInfo.changingCol.ID)
- // The job is in the middle state. The appended changingCol and changingIndex should
- // be removed from the tableInfo as well.
- removeChangingColAndIdxs(tblInfo, modifyInfo.changingCol.ID)
- }
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, true)
- if err != nil {
- return ver, errors.Trace(err)
- }
- job.FinishTableJob(model.JobStateRollbackDone, model.StateNone, ver, tblInfo)
- // Reconstruct the job args to add the temporary index ids into delete range table.
- job.Args = []any{changingIdxIDs, getPartitionIDs(tblInfo)}
- return ver, nil
-}
-
func removeChangingColAndIdxs(tblInfo *model.TableInfo, changingColID int64) {
restIdx := tblInfo.Indices[:0]
for _, idx := range tblInfo.Indices {
@@ -684,234 +327,6 @@ func removeChangingColAndIdxs(tblInfo *model.TableInfo, changingColID int64) {
tblInfo.Columns = restCols
}
-func (w *worker) doModifyColumnTypeWithData(
- d *ddlCtx, t *meta.Meta, job *model.Job,
- dbInfo *model.DBInfo, tblInfo *model.TableInfo, changingCol, oldCol *model.ColumnInfo,
- colName model.CIStr, pos *ast.ColumnPosition, rmIdxIDs []int64) (ver int64, _ error) {
- var err error
- originalState := changingCol.State
- targetCol := changingCol.Clone()
- targetCol.Name = colName
- changingIdxs := buildRelatedIndexInfos(tblInfo, changingCol.ID)
- switch changingCol.State {
- case model.StateNone:
- // Column from null to not null.
- if !mysql.HasNotNullFlag(oldCol.GetFlag()) && mysql.HasNotNullFlag(changingCol.GetFlag()) {
- // Introduce the `mysql.PreventNullInsertFlag` flag to prevent users from inserting or updating null values.
- err := modifyColsFromNull2NotNull(w, dbInfo, tblInfo, []*model.ColumnInfo{oldCol}, targetCol, oldCol.GetType() != changingCol.GetType())
- if err != nil {
- if dbterror.ErrWarnDataTruncated.Equal(err) || dbterror.ErrInvalidUseOfNull.Equal(err) {
- job.State = model.JobStateRollingback
- }
- return ver, err
- }
- }
- // none -> delete only
- updateChangingObjState(changingCol, changingIdxs, model.StateDeleteOnly)
- failpoint.Inject("mockInsertValueAfterCheckNull", func(val failpoint.Value) {
- if valStr, ok := val.(string); ok {
- var sctx sessionctx.Context
- sctx, err := w.sessPool.Get()
- if err != nil {
- failpoint.Return(ver, err)
- }
- defer w.sessPool.Put(sctx)
-
- ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL)
- //nolint:forcetypeassert
- _, _, err = sctx.GetRestrictedSQLExecutor().ExecRestrictedSQL(ctx, nil, valStr)
- if err != nil {
- job.State = model.JobStateCancelled
- failpoint.Return(ver, err)
- }
- }
- })
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, originalState != changingCol.State)
- if err != nil {
- return ver, errors.Trace(err)
- }
- // Make sure job args change after `updateVersionAndTableInfoWithCheck`, otherwise, the job args will
- // be updated in `updateDDLJob` even if it meets an error in `updateVersionAndTableInfoWithCheck`.
- job.SchemaState = model.StateDeleteOnly
- metrics.GetBackfillProgressByLabel(metrics.LblModifyColumn, job.SchemaName, tblInfo.Name.String()).Set(0)
- job.Args = append(job.Args, changingCol, changingIdxs, rmIdxIDs)
- case model.StateDeleteOnly:
- // Column from null to not null.
- if !mysql.HasNotNullFlag(oldCol.GetFlag()) && mysql.HasNotNullFlag(changingCol.GetFlag()) {
- // Introduce the `mysql.PreventNullInsertFlag` flag to prevent users from inserting or updating null values.
- err := modifyColsFromNull2NotNull(w, dbInfo, tblInfo, []*model.ColumnInfo{oldCol}, targetCol, oldCol.GetType() != changingCol.GetType())
- if err != nil {
- if dbterror.ErrWarnDataTruncated.Equal(err) || dbterror.ErrInvalidUseOfNull.Equal(err) {
- job.State = model.JobStateRollingback
- }
- return ver, err
- }
- }
- // delete only -> write only
- updateChangingObjState(changingCol, changingIdxs, model.StateWriteOnly)
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != changingCol.State)
- if err != nil {
- return ver, errors.Trace(err)
- }
- job.SchemaState = model.StateWriteOnly
- failpoint.InjectCall("afterModifyColumnStateDeleteOnly", job.ID)
- case model.StateWriteOnly:
- // write only -> reorganization
- updateChangingObjState(changingCol, changingIdxs, model.StateWriteReorganization)
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != changingCol.State)
- if err != nil {
- return ver, errors.Trace(err)
- }
- // Initialize SnapshotVer to 0 for later reorganization check.
- job.SnapshotVer = 0
- job.SchemaState = model.StateWriteReorganization
- case model.StateWriteReorganization:
- tbl, err := getTable(d.getAutoIDRequirement(), dbInfo.ID, tblInfo)
- if err != nil {
- return ver, errors.Trace(err)
- }
-
- var done bool
- if job.MultiSchemaInfo != nil {
- done, ver, err = doReorgWorkForModifyColumnMultiSchema(w, d, t, job, tbl, oldCol, changingCol, changingIdxs)
- } else {
- done, ver, err = doReorgWorkForModifyColumn(w, d, t, job, tbl, oldCol, changingCol, changingIdxs)
- }
- if !done {
- return ver, err
- }
-
- rmIdxIDs = append(buildRelatedIndexIDs(tblInfo, oldCol.ID), rmIdxIDs...)
-
- err = adjustTableInfoAfterModifyColumnWithData(tblInfo, pos, oldCol, changingCol, colName, changingIdxs)
- if err != nil {
- job.State = model.JobStateRollingback
- return ver, errors.Trace(err)
- }
-
- updateChangingObjState(changingCol, changingIdxs, model.StatePublic)
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != changingCol.State)
- if err != nil {
- return ver, errors.Trace(err)
- }
-
- // Finish this job.
- job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tblInfo)
- // Refactor the job args to add the old index ids into delete range table.
- job.Args = []any{rmIdxIDs, getPartitionIDs(tblInfo)}
- modifyColumnEvent := statsutil.NewModifyColumnEvent(
- job.SchemaID,
- tblInfo,
- []*model.ColumnInfo{changingCol},
- )
- asyncNotifyEvent(d, modifyColumnEvent)
- default:
- err = dbterror.ErrInvalidDDLState.GenWithStackByArgs("column", changingCol.State)
- }
-
- return ver, errors.Trace(err)
-}
-
-func doReorgWorkForModifyColumnMultiSchema(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job, tbl table.Table,
- oldCol, changingCol *model.ColumnInfo, changingIdxs []*model.IndexInfo) (done bool, ver int64, err error) {
- if job.MultiSchemaInfo.Revertible {
- done, ver, err = doReorgWorkForModifyColumn(w, d, t, job, tbl, oldCol, changingCol, changingIdxs)
- if done {
- // We need another round to wait for all the others sub-jobs to finish.
- job.MarkNonRevertible()
- }
- // We need another round to run the reorg process.
- return false, ver, err
- }
- // Non-revertible means all the sub jobs finished.
- return true, ver, err
-}
-
-func doReorgWorkForModifyColumn(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job, tbl table.Table,
- oldCol, changingCol *model.ColumnInfo, changingIdxs []*model.IndexInfo) (done bool, ver int64, err error) {
- job.ReorgMeta.ReorgTp = model.ReorgTypeTxn
- sctx, err1 := w.sessPool.Get()
- if err1 != nil {
- err = errors.Trace(err1)
- return
- }
- defer w.sessPool.Put(sctx)
- rh := newReorgHandler(sess.NewSession(sctx))
- dbInfo, err := t.GetDatabase(job.SchemaID)
- if err != nil {
- return false, ver, errors.Trace(err)
- }
- reorgInfo, err := getReorgInfo(d.jobContext(job.ID, job.ReorgMeta),
- d, rh, job, dbInfo, tbl, BuildElements(changingCol, changingIdxs), false)
- if err != nil || reorgInfo == nil || reorgInfo.first {
- // If we run reorg firstly, we should update the job snapshot version
- // and then run the reorg next time.
- return false, ver, errors.Trace(err)
- }
-
- // Inject a failpoint so that we can pause here and do verification on other components.
- // With a failpoint-enabled version of TiDB, you can trigger this failpoint by the following command:
- // enable: curl -X PUT -d "pause" "http://127.0.0.1:10080/fail/github.com/pingcap/tidb/pkg/ddl/mockDelayInModifyColumnTypeWithData".
- // disable: curl -X DELETE "http://127.0.0.1:10080/fail/github.com/pingcap/tidb/pkg/ddl/mockDelayInModifyColumnTypeWithData"
- failpoint.Inject("mockDelayInModifyColumnTypeWithData", func() {})
- err = w.runReorgJob(reorgInfo, tbl.Meta(), d.lease, func() (addIndexErr error) {
- defer util.Recover(metrics.LabelDDL, "onModifyColumn",
- func() {
- addIndexErr = dbterror.ErrCancelledDDLJob.GenWithStack("modify table `%v` column `%v` panic", tbl.Meta().Name, oldCol.Name)
- }, false)
- // Use old column name to generate less confusing error messages.
- changingColCpy := changingCol.Clone()
- changingColCpy.Name = oldCol.Name
- return w.updateCurrentElement(tbl, reorgInfo)
- })
- if err != nil {
- if dbterror.ErrPausedDDLJob.Equal(err) {
- return false, ver, nil
- }
-
- if dbterror.ErrWaitReorgTimeout.Equal(err) {
- // If timeout, we should return, check for the owner and re-wait job done.
- return false, ver, nil
- }
- if kv.IsTxnRetryableError(err) || dbterror.ErrNotOwner.Equal(err) {
- return false, ver, errors.Trace(err)
- }
- if err1 := rh.RemoveDDLReorgHandle(job, reorgInfo.elements); err1 != nil {
- logutil.DDLLogger().Warn("run modify column job failed, RemoveDDLReorgHandle failed, can't convert job to rollback",
- zap.String("job", job.String()), zap.Error(err1))
- }
- logutil.DDLLogger().Warn("run modify column job failed, convert job to rollback", zap.Stringer("job", job), zap.Error(err))
- job.State = model.JobStateRollingback
- return false, ver, errors.Trace(err)
- }
- return true, ver, nil
-}
-
-func adjustTableInfoAfterModifyColumnWithData(tblInfo *model.TableInfo, pos *ast.ColumnPosition,
- oldCol, changingCol *model.ColumnInfo, newName model.CIStr, changingIdxs []*model.IndexInfo) (err error) {
- if pos != nil && pos.RelativeColumn != nil && oldCol.Name.L == pos.RelativeColumn.Name.L {
- // For cases like `modify column b after b`, it should report this error.
- return errors.Trace(infoschema.ErrColumnNotExists.GenWithStackByArgs(oldCol.Name, tblInfo.Name))
- }
- internalColName := changingCol.Name
- changingCol = replaceOldColumn(tblInfo, oldCol, changingCol, newName)
- if len(changingIdxs) > 0 {
- updateNewIdxColsNameOffset(changingIdxs, internalColName, changingCol)
- indexesToRemove := filterIndexesToRemove(changingIdxs, newName, tblInfo)
- replaceOldIndexes(tblInfo, indexesToRemove)
- }
- if tblInfo.TTLInfo != nil {
- updateTTLInfoWhenModifyColumn(tblInfo, oldCol.Name, changingCol.Name)
- }
- // Move the new column to a correct offset.
- destOffset, err := LocateOffsetToMove(changingCol.Offset, pos, tblInfo)
- if err != nil {
- return errors.Trace(err)
- }
- tblInfo.MoveColumnInfo(changingCol.Offset, destOffset)
- return nil
-}
-
func replaceOldColumn(tblInfo *model.TableInfo, oldCol, changingCol *model.ColumnInfo,
newName model.CIStr) *model.ColumnInfo {
tblInfo.MoveColumnInfo(changingCol.Offset, len(tblInfo.Columns)-1)
@@ -964,30 +379,6 @@ func updateNewIdxColsNameOffset(changingIdxs []*model.IndexInfo,
}
}
-func updateFKInfoWhenModifyColumn(tblInfo *model.TableInfo, oldCol, newCol model.CIStr) {
- if oldCol.L == newCol.L {
- return
- }
- for _, fk := range tblInfo.ForeignKeys {
- for i := range fk.Cols {
- if fk.Cols[i].L == oldCol.L {
- fk.Cols[i] = newCol
- }
- }
- }
-}
-
-func updateTTLInfoWhenModifyColumn(tblInfo *model.TableInfo, oldCol, newCol model.CIStr) {
- if oldCol.L == newCol.L {
- return
- }
- if tblInfo.TTLInfo != nil {
- if tblInfo.TTLInfo.ColumnName.L == oldCol.L {
- tblInfo.TTLInfo.ColumnName = newCol
- }
- }
-}
-
// filterIndexesToRemove filters out the indexes that can be removed.
func filterIndexesToRemove(changingIdxs []*model.IndexInfo, colName model.CIStr, tblInfo *model.TableInfo) []*model.IndexInfo {
indexesToRemove := make([]*model.IndexInfo, 0, len(changingIdxs))
@@ -1149,12 +540,12 @@ func (w *worker) updateCurrentElement(t table.Table, reorgInfo *reorgInfo) error
return dbterror.ErrCancelledDDLJob.GenWithStack("Modify Column on partitioned table / typeUpdateColumnWorker not yet supported.")
}
// Get the original start handle and end handle.
- currentVer, err := getValidCurrentVersion(reorgInfo.d.store)
+ currentVer, err := getValidCurrentVersion(reorgInfo.jobCtx.store)
if err != nil {
return errors.Trace(err)
}
//nolint:forcetypeassert
- originalStartHandle, originalEndHandle, err := getTableRange(reorgInfo.NewJobContext(), reorgInfo.d, t.(table.PhysicalTable), currentVer.Ver, reorgInfo.Job.Priority)
+ originalStartHandle, originalEndHandle, err := getTableRange(reorgInfo.NewJobContext(), reorgInfo.jobCtx.store, t.(table.PhysicalTable), currentVer.Ver, reorgInfo.Job.Priority)
if err != nil {
return errors.Trace(err)
}
@@ -1213,7 +604,7 @@ type updateColumnWorker struct {
checksumNeeded bool
}
-func newUpdateColumnWorker(id int, t table.PhysicalTable, decodeColMap map[int64]decoder.Column, reorgInfo *reorgInfo, jc *JobContext) (*updateColumnWorker, error) {
+func newUpdateColumnWorker(id int, t table.PhysicalTable, decodeColMap map[int64]decoder.Column, reorgInfo *reorgInfo, jc *ReorgContext) (*updateColumnWorker, error) {
bCtx, err := newBackfillCtx(id, reorgInfo, reorgInfo.SchemaName, t, jc, "update_col_rate", false)
if err != nil {
return nil, err
@@ -1504,130 +895,7 @@ func updateChangingObjState(changingCol *model.ColumnInfo, changingIdxs []*model
}
}
-// doModifyColumn updates the column information and reorders all columns. It does not support modifying column data.
-func (w *worker) doModifyColumn(
- d *ddlCtx, t *meta.Meta, job *model.Job, dbInfo *model.DBInfo, tblInfo *model.TableInfo,
- newCol, oldCol *model.ColumnInfo, pos *ast.ColumnPosition) (ver int64, _ error) {
- if oldCol.ID != newCol.ID {
- job.State = model.JobStateRollingback
- return ver, dbterror.ErrColumnInChange.GenWithStackByArgs(oldCol.Name, newCol.ID)
- }
- // Column from null to not null.
- if !mysql.HasNotNullFlag(oldCol.GetFlag()) && mysql.HasNotNullFlag(newCol.GetFlag()) {
- noPreventNullFlag := !mysql.HasPreventNullInsertFlag(oldCol.GetFlag())
-
- // lease = 0 means it's in an integration test. In this case we don't delay so the test won't run too slowly.
- // We need to check after the flag is set
- if d.lease > 0 && !noPreventNullFlag {
- delayForAsyncCommit()
- }
-
- // Introduce the `mysql.PreventNullInsertFlag` flag to prevent users from inserting or updating null values.
- err := modifyColsFromNull2NotNull(w, dbInfo, tblInfo, []*model.ColumnInfo{oldCol}, newCol, oldCol.GetType() != newCol.GetType())
- if err != nil {
- if dbterror.ErrWarnDataTruncated.Equal(err) || dbterror.ErrInvalidUseOfNull.Equal(err) {
- job.State = model.JobStateRollingback
- }
- return ver, err
- }
- // The column should get into prevent null status first.
- if noPreventNullFlag {
- return updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, true)
- }
- }
-
- if job.MultiSchemaInfo != nil && job.MultiSchemaInfo.Revertible {
- job.MarkNonRevertible()
- // Store the mark and enter the next DDL handling loop.
- return updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, false)
- }
-
- if err := adjustTableInfoAfterModifyColumn(tblInfo, newCol, oldCol, pos); err != nil {
- job.State = model.JobStateRollingback
- return ver, errors.Trace(err)
- }
-
- childTableInfos, err := adjustForeignKeyChildTableInfoAfterModifyColumn(d, t, job, tblInfo, newCol, oldCol)
- if err != nil {
- return ver, errors.Trace(err)
- }
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, true, childTableInfos...)
- if err != nil {
- // Modified the type definition of 'null' to 'not null' before this, so rollBack the job when an error occurs.
- job.State = model.JobStateRollingback
- return ver, errors.Trace(err)
- }
-
- job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tblInfo)
- // For those column-type-change type which doesn't need reorg data, we should also mock the job args for delete range.
- job.Args = []any{[]int64{}, []int64{}}
- return ver, nil
-}
-
-func adjustTableInfoAfterModifyColumn(
- tblInfo *model.TableInfo, newCol, oldCol *model.ColumnInfo, pos *ast.ColumnPosition) error {
- // We need the latest column's offset and state. This information can be obtained from the store.
- newCol.Offset = oldCol.Offset
- newCol.State = oldCol.State
- if pos != nil && pos.RelativeColumn != nil && oldCol.Name.L == pos.RelativeColumn.Name.L {
- // For cases like `modify column b after b`, it should report this error.
- return errors.Trace(infoschema.ErrColumnNotExists.GenWithStackByArgs(oldCol.Name, tblInfo.Name))
- }
- destOffset, err := LocateOffsetToMove(oldCol.Offset, pos, tblInfo)
- if err != nil {
- return errors.Trace(infoschema.ErrColumnNotExists.GenWithStackByArgs(oldCol.Name, tblInfo.Name))
- }
- tblInfo.Columns[oldCol.Offset] = newCol
- tblInfo.MoveColumnInfo(oldCol.Offset, destOffset)
- updateNewIdxColsNameOffset(tblInfo.Indices, oldCol.Name, newCol)
- updateFKInfoWhenModifyColumn(tblInfo, oldCol.Name, newCol.Name)
- updateTTLInfoWhenModifyColumn(tblInfo, oldCol.Name, newCol.Name)
- return nil
-}
-
-func adjustForeignKeyChildTableInfoAfterModifyColumn(d *ddlCtx, t *meta.Meta, job *model.Job, tblInfo *model.TableInfo, newCol, oldCol *model.ColumnInfo) ([]schemaIDAndTableInfo, error) {
- if !variable.EnableForeignKey.Load() || newCol.Name.L == oldCol.Name.L {
- return nil, nil
- }
- is, err := getAndCheckLatestInfoSchema(d, t)
- if err != nil {
- return nil, err
- }
- referredFKs := is.GetTableReferredForeignKeys(job.SchemaName, tblInfo.Name.L)
- if len(referredFKs) == 0 {
- return nil, nil
- }
- fkh := newForeignKeyHelper()
- fkh.addLoadedTable(job.SchemaName, tblInfo.Name.L, job.SchemaID, tblInfo)
- for _, referredFK := range referredFKs {
- info, err := fkh.getTableFromStorage(is, t, referredFK.ChildSchema, referredFK.ChildTable)
- if err != nil {
- if infoschema.ErrTableNotExists.Equal(err) || infoschema.ErrDatabaseNotExists.Equal(err) {
- continue
- }
- return nil, err
- }
- fkInfo := model.FindFKInfoByName(info.tblInfo.ForeignKeys, referredFK.ChildFKName.L)
- if fkInfo == nil {
- continue
- }
- for i := range fkInfo.RefCols {
- if fkInfo.RefCols[i].L == oldCol.Name.L {
- fkInfo.RefCols[i] = newCol.Name
- }
- }
- }
- infoList := make([]schemaIDAndTableInfo, 0, len(fkh.loaded))
- for _, info := range fkh.loaded {
- if info.tblInfo.ID == tblInfo.ID {
- continue
- }
- infoList = append(infoList, info)
- }
- return infoList, nil
-}
-
-func checkAndApplyAutoRandomBits(d *ddlCtx, m *meta.Meta, dbInfo *model.DBInfo, tblInfo *model.TableInfo,
+func checkAndApplyAutoRandomBits(jobCtx *jobContext, m *meta.Meta, dbInfo *model.DBInfo, tblInfo *model.TableInfo,
oldCol *model.ColumnInfo, newCol *model.ColumnInfo, newAutoRandBits uint64) error {
if newAutoRandBits == 0 {
return nil
@@ -1637,7 +905,7 @@ func checkAndApplyAutoRandomBits(d *ddlCtx, m *meta.Meta, dbInfo *model.DBInfo,
if err != nil {
return err
}
- return applyNewAutoRandomBits(d, m, dbInfo, tblInfo, oldCol, newAutoRandBits)
+ return applyNewAutoRandomBits(jobCtx, m, dbInfo, tblInfo, oldCol, newAutoRandBits)
}
// checkNewAutoRandomBits checks whether the new auto_random bits number can cause overflow.
@@ -1698,14 +966,14 @@ func (r *asAutoIDRequirement) AutoIDClient() *autoid.ClientDiscover {
// applyNewAutoRandomBits set auto_random bits to TableInfo and
// migrate auto_increment ID to auto_random ID if possible.
-func applyNewAutoRandomBits(d *ddlCtx, m *meta.Meta, dbInfo *model.DBInfo,
+func applyNewAutoRandomBits(jobCtx *jobContext, m *meta.Meta, dbInfo *model.DBInfo,
tblInfo *model.TableInfo, oldCol *model.ColumnInfo, newAutoRandBits uint64) error {
tblInfo.AutoRandomBits = newAutoRandBits
needMigrateFromAutoIncToAutoRand := mysql.HasAutoIncrementFlag(oldCol.GetFlag())
if !needMigrateFromAutoIncToAutoRand {
return nil
}
- autoRandAlloc := autoid.NewAllocatorsFromTblInfo(d.getAutoIDRequirement(), dbInfo.ID, tblInfo).Get(autoid.AutoRandomType)
+ autoRandAlloc := autoid.NewAllocatorsFromTblInfo(jobCtx.getAutoIDRequirement(), dbInfo.ID, tblInfo).Get(autoid.AutoRandomType)
if autoRandAlloc == nil {
errMsg := fmt.Sprintf(autoid.AutoRandomAllocatorNotFound, dbInfo.Name.O, tblInfo.Name.O)
return dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(errMsg)
@@ -1768,7 +1036,7 @@ func checkForNullValue(ctx context.Context, sctx sessionctx.Context, isDataTrunc
return nil
}
-func updateColumnDefaultValue(d *ddlCtx, t *meta.Meta, job *model.Job, newCol *model.ColumnInfo, oldColName *model.CIStr) (ver int64, _ error) {
+func updateColumnDefaultValue(jobCtx *jobContext, t *meta.Meta, job *model.Job, newCol *model.ColumnInfo, oldColName *model.CIStr) (ver int64, _ error) {
tblInfo, err := GetTableInfoAndCancelFaultJob(t, job, job.SchemaID)
if err != nil {
return ver, errors.Trace(err)
@@ -1777,7 +1045,7 @@ func updateColumnDefaultValue(d *ddlCtx, t *meta.Meta, job *model.Job, newCol *m
if job.MultiSchemaInfo != nil && job.MultiSchemaInfo.Revertible {
job.MarkNonRevertible()
// Store the mark and enter the next DDL handling loop.
- return updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, false)
+ return updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, false)
}
oldCol := model.FindColumnInfo(tblInfo.Columns, oldColName.L)
@@ -1809,7 +1077,7 @@ func updateColumnDefaultValue(d *ddlCtx, t *meta.Meta, job *model.Job, newCol *m
}
}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
@@ -1878,25 +1146,6 @@ func checkAddColumnTooManyColumns(colNum int) error {
return nil
}
-// rollbackModifyColumnJob rollbacks the job when an error occurs.
-func rollbackModifyColumnJob(d *ddlCtx, t *meta.Meta, tblInfo *model.TableInfo, job *model.Job, newCol, oldCol *model.ColumnInfo, modifyColumnTp byte) (ver int64, _ error) {
- var err error
- if oldCol.ID == newCol.ID && modifyColumnTp == mysql.TypeNull {
- // field NotNullFlag flag reset.
- tblInfo.Columns[oldCol.Offset].SetFlag(oldCol.GetFlag() &^ mysql.NotNullFlag)
- // field PreventNullInsertFlag flag reset.
- tblInfo.Columns[oldCol.Offset].SetFlag(oldCol.GetFlag() &^ mysql.PreventNullInsertFlag)
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
- if err != nil {
- return ver, errors.Trace(err)
- }
- }
- job.FinishTableJob(model.JobStateRollbackDone, model.StateNone, ver, tblInfo)
- // For those column-type-change type which doesn't need reorg data, we should also mock the job args for delete range.
- job.Args = []any{[]int64{}, []int64{}}
- return ver, nil
-}
-
// modifyColsFromNull2NotNull modifies the type definitions of 'null' to 'not null'.
// Introduce the `mysql.PreventNullInsertFlag` flag to prevent users from inserting or updating null values.
func modifyColsFromNull2NotNull(w *worker, dbInfo *model.DBInfo, tblInfo *model.TableInfo, cols []*model.ColumnInfo, newCol *model.ColumnInfo, isDataTruncated bool) error {
diff --git a/pkg/ddl/column_change_test.go b/pkg/ddl/column_change_test.go
index 5d10c51866b2c..f637ed5a89b69 100644
--- a/pkg/ddl/column_change_test.go
+++ b/pkg/ddl/column_change_test.go
@@ -23,7 +23,6 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/tidb/pkg/ddl"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/meta"
"github.com/pingcap/tidb/pkg/parser/model"
@@ -34,6 +33,7 @@ import (
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/external"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/mock"
"github.com/stretchr/testify/require"
@@ -48,9 +48,6 @@ func TestColumnAdd(t *testing.T) {
tk.MustExec("create table t (c1 int, c2 int);")
tk.MustExec("insert t values (1, 2);")
- d := dom.DDL()
- tc := &callback.TestDDLCallback{Do: dom}
-
ct := testNewContext(store)
// set up hook
var (
@@ -61,9 +58,9 @@ func TestColumnAdd(t *testing.T) {
)
first := true
var jobID int64
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
jobID = job.ID
- tbl, exist := dom.InfoSchema().TableByID(job.TableID)
+ tbl, exist := dom.InfoSchema().TableByID(context.Background(), job.TableID)
require.True(t, exist)
switch job.SchemaState {
case model.StateDeleteOnly:
@@ -79,22 +76,20 @@ func TestColumnAdd(t *testing.T) {
publicTable = tbl
require.NoError(t, checkAddPublic(ct, writeOnlyTable, publicTable))
}
- }
- tc.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(tc.Clone())
+ })
tk.MustExec("alter table t add column c3 int default 3")
tb := publicTable
v := getSchemaVer(t, tk.Session())
checkHistoryJobArgs(t, tk.Session(), jobID, &historyJobArgs{ver: v, tbl: tb.Meta()})
// Drop column.
- tc.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if dropCol == nil {
tbl := external.GetTableByName(t, internal, "test", "t")
dropCol = tbl.VisibleCols()[2]
}
- }
- onJobUpdatedExportedFunc2 := func(job *model.Job) {
+ })
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.NotStarted() {
return
}
@@ -105,9 +100,7 @@ func TestColumnAdd(t *testing.T) {
require.NotEqualf(t, col.ID, dropCol.ID, "column is not dropped")
}
}
- }
- tc.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc2)
- d.SetHook(tc.Clone())
+ })
tk.MustExec("alter table t drop column c3")
v = getSchemaVer(t, tk.Session())
// Don't check column, so it's ok to use tb.
@@ -115,9 +108,9 @@ func TestColumnAdd(t *testing.T) {
// Add column not default.
first = true
- onJobUpdatedExportedFunc3 := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
jobID = job.ID
- tbl, exist := dom.InfoSchema().TableByID(job.TableID)
+ tbl, exist := dom.InfoSchema().TableByID(context.Background(), job.TableID)
require.True(t, exist)
switch job.SchemaState {
case model.StateWriteOnly:
@@ -128,34 +121,29 @@ func TestColumnAdd(t *testing.T) {
}
first = false
sess := testNewContext(store)
- err := sessiontxn.NewTxn(context.Background(), sess)
+ txn, err := newTxn(sess)
require.NoError(t, err)
- _, err = writeOnlyTable.AddRecord(sess.GetTableCtx(), types.MakeDatums(10, 10))
+ _, err = writeOnlyTable.AddRecord(sess.GetTableCtx(), txn, types.MakeDatums(10, 10))
require.NoError(t, err)
}
- }
- tc.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc3)
- d.SetHook(tc)
+ })
tk.MustExec("alter table t add column c3 int")
testCheckJobDone(t, store, jobID, true)
}
func TestModifyAutoRandColumnWithMetaKeyChanged(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
ddl.SetWaitTimeWhenErrorOccurred(1 * time.Microsecond)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("create table t (a bigint primary key clustered AUTO_RANDOM(5));")
- d := dom.DDL()
- tc := &callback.TestDDLCallback{Do: dom}
-
var errCount int32 = 3
var genAutoRandErr error
var dbID int64
var tID int64
var jobID int64
- tc.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
jobID = job.ID
dbID = job.SchemaID
tID = job.TableID
@@ -168,8 +156,7 @@ func TestModifyAutoRandColumnWithMetaKeyChanged(t *testing.T) {
return err1
})
}
- }
- d.SetHook(tc)
+ })
tk.MustExec("alter table t modify column a bigint AUTO_RANDOM(10)")
require.True(t, errCount == 0)
@@ -215,15 +202,15 @@ func seek(t table.PhysicalTable, ctx sessionctx.Context, h kv.Handle) (kv.Handle
func checkAddWriteOnly(ctx sessionctx.Context, deleteOnlyTable, writeOnlyTable table.Table, h kv.Handle) error {
// WriteOnlyTable: insert t values (2, 3)
- err := sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err := newTxn(ctx)
if err != nil {
return errors.Trace(err)
}
- _, err = writeOnlyTable.AddRecord(ctx.GetTableCtx(), types.MakeDatums(2, 3))
+ _, err = writeOnlyTable.AddRecord(ctx.GetTableCtx(), txn, types.MakeDatums(2, 3))
if err != nil {
return errors.Trace(err)
}
- err = sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err = newTxn(ctx)
if err != nil {
return errors.Trace(err)
}
@@ -257,11 +244,11 @@ func checkAddWriteOnly(ctx sessionctx.Context, deleteOnlyTable, writeOnlyTable t
if err != nil {
return errors.Trace(err)
}
- err = writeOnlyTable.UpdateRecord(context.Background(), ctx.GetTableCtx(), h, types.MakeDatums(1, 2, 3), types.MakeDatums(2, 2, 3), touchedSlice(writeOnlyTable))
+ err = writeOnlyTable.UpdateRecord(ctx.GetTableCtx(), txn, h, types.MakeDatums(1, 2, 3), types.MakeDatums(2, 2, 3), touchedSlice(writeOnlyTable))
if err != nil {
return errors.Trace(err)
}
- err = sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err = newTxn(ctx)
if err != nil {
return errors.Trace(err)
}
@@ -274,11 +261,11 @@ func checkAddWriteOnly(ctx sessionctx.Context, deleteOnlyTable, writeOnlyTable t
return errors.Trace(err)
}
// DeleteOnlyTable: delete from t where c2 = 2
- err = deleteOnlyTable.RemoveRecord(ctx.GetTableCtx(), h, types.MakeDatums(2, 2))
+ err = deleteOnlyTable.RemoveRecord(ctx.GetTableCtx(), txn, h, types.MakeDatums(2, 2))
if err != nil {
return errors.Trace(err)
}
- err = sessiontxn.NewTxn(context.Background(), ctx)
+ _, err = newTxn(ctx)
if err != nil {
return errors.Trace(err)
}
@@ -298,17 +285,16 @@ func touchedSlice(t table.Table) []bool {
}
func checkAddPublic(sctx sessionctx.Context, writeOnlyTable, publicTable table.Table) error {
- ctx := context.TODO()
// publicTable Insert t values (4, 4, 4)
- err := sessiontxn.NewTxn(ctx, sctx)
+ txn, err := newTxn(sctx)
if err != nil {
return errors.Trace(err)
}
- h, err := publicTable.AddRecord(sctx.GetTableCtx(), types.MakeDatums(4, 4, 4))
+ h, err := publicTable.AddRecord(sctx.GetTableCtx(), txn, types.MakeDatums(4, 4, 4))
if err != nil {
return errors.Trace(err)
}
- err = sessiontxn.NewTxn(ctx, sctx)
+ txn, err = newTxn(sctx)
if err != nil {
return errors.Trace(err)
}
@@ -321,11 +307,11 @@ func checkAddPublic(sctx sessionctx.Context, writeOnlyTable, publicTable table.T
return errors.Errorf("%v", oldRow)
}
newRow := types.MakeDatums(3, 4, oldRow[2].GetValue())
- err = writeOnlyTable.UpdateRecord(context.Background(), sctx.GetTableCtx(), h, oldRow, newRow, touchedSlice(writeOnlyTable))
+ err = writeOnlyTable.UpdateRecord(sctx.GetTableCtx(), txn, h, oldRow, newRow, touchedSlice(writeOnlyTable))
if err != nil {
return errors.Trace(err)
}
- err = sessiontxn.NewTxn(ctx, sctx)
+ _, err = newTxn(sctx)
if err != nil {
return errors.Trace(err)
}
@@ -373,9 +359,7 @@ type historyJobArgs struct {
}
func getSchemaVer(t *testing.T, ctx sessionctx.Context) int64 {
- err := sessiontxn.NewTxn(context.Background(), ctx)
- require.NoError(t, err)
- txn, err := ctx.Txn(true)
+ txn, err := newTxn(ctx)
require.NoError(t, err)
m := meta.NewMeta(txn)
ver, err := m.GetSchemaVersion()
@@ -438,7 +422,7 @@ func testNewContext(store kv.Storage) sessionctx.Context {
}
func TestIssue40135(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -447,16 +431,22 @@ func TestIssue40135(t *testing.T) {
tk.MustExec("CREATE TABLE t40135 ( a tinyint DEFAULT NULL, b varchar(32) DEFAULT 'md') PARTITION BY HASH (a) PARTITIONS 2")
one := true
- hook := &callback.TestDDLCallback{Do: dom}
var checkErr error
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if one {
one = false
_, checkErr = tk1.Exec("alter table t40135 change column a aNew SMALLINT NULL DEFAULT '-14996'")
}
- }
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("alter table t40135 modify column a MEDIUMINT NULL DEFAULT '6243108' FIRST")
require.ErrorContains(t, checkErr, "[ddl:3855]Column 'a' has a partitioning function dependency and cannot be dropped or renamed")
}
+
+func newTxn(ctx sessionctx.Context) (kv.Transaction, error) {
+ err := sessiontxn.NewTxn(context.Background(), ctx)
+ if err != nil {
+ return nil, err
+ }
+ return ctx.Txn(true)
+}
diff --git a/pkg/ddl/column_modify_test.go b/pkg/ddl/column_modify_test.go
index 90b4a930fb85a..c453e6370f503 100644
--- a/pkg/ddl/column_modify_test.go
+++ b/pkg/ddl/column_modify_test.go
@@ -24,7 +24,6 @@ import (
"github.com/pingcap/errors"
testddlutil "github.com/pingcap/tidb/pkg/ddl/testutil"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/kv"
@@ -37,6 +36,7 @@ import (
"github.com/pingcap/tidb/pkg/table/tables"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/external"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/mock"
"github.com/stretchr/testify/require"
@@ -421,7 +421,7 @@ func TestVirtualColumnDDL(t *testing.T) {
}
func TestTransactionWithWriteOnlyColumn(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, columnModifyLease)
+ store := testkit.CreateMockStoreWithSchemaLease(t, columnModifyLease)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t1")
@@ -436,9 +436,8 @@ func TestTransactionWithWriteOnlyColumn(t *testing.T) {
},
}
- hook := &callback.TestDDLCallback{Do: dom}
var checkErr error
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if checkErr != nil {
return
}
@@ -456,8 +455,7 @@ func TestTransactionWithWriteOnlyColumn(t *testing.T) {
}
}
}
- }
- dom.DDL().SetHook(hook)
+ })
done := make(chan error, 1)
// test transaction on add column.
go backgroundExec(store, "test", "alter table t1 add column c int not null", done)
@@ -477,7 +475,7 @@ func TestTransactionWithWriteOnlyColumn(t *testing.T) {
// For issue #31735.
func TestAddGeneratedColumnAndInsert(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, columnModifyLease)
+ store := testkit.CreateMockStoreWithSchemaLease(t, columnModifyLease)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -487,13 +485,11 @@ func TestAddGeneratedColumnAndInsert(t *testing.T) {
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
- d := dom.DDL()
- hook := &callback.TestDDLCallback{Do: dom}
ctx := mock.NewContext()
ctx.Store = store
times := 0
var checkErr error
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if checkErr != nil {
return
}
@@ -517,9 +513,7 @@ func TestAddGeneratedColumnAndInsert(t *testing.T) {
times++
}
}
- }
- hook.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(hook)
+ })
tk.MustExec("alter table t1 add column gc int as ((a+1))")
tk.MustQuery("select * from t1 order by a").Check(testkit.Rows("4 5", "10 11"))
@@ -527,16 +521,15 @@ func TestAddGeneratedColumnAndInsert(t *testing.T) {
}
func TestColumnTypeChangeGenUniqueChangingName(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, columnModifyLease)
+ store := testkit.CreateMockStoreWithSchemaLease(t, columnModifyLease)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
- hook := &callback.TestDDLCallback{}
var checkErr error
assertChangingColName := "_col$_c2_0"
assertChangingIdxName := "_idx$_idx_0"
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.SchemaState == model.StateDeleteOnly && job.Type == model.ActionModifyColumn {
var (
_newCol *model.ColumnInfo
@@ -559,10 +552,7 @@ func TestColumnTypeChangeGenUniqueChangingName(t *testing.T) {
checkErr = errors.New("changing index name is incorrect")
}
}
- }
- hook.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d := dom.DDL()
- d.SetHook(hook)
+ })
tk.MustExec("create table if not exists t(c1 varchar(256), c2 bigint, `_col$_c2` varchar(10), unique _idx$_idx(c1), unique idx(c2));")
tk.MustExec("alter table test.t change column c2 cC2 tinyint after `_col$_c2`")
@@ -593,7 +583,7 @@ func TestColumnTypeChangeGenUniqueChangingName(t *testing.T) {
assertChangingColName2 := "_col$__col$__col$_c1_0_1"
query1 := "alter table t modify column _col$_c1 tinyint"
query2 := "alter table t modify column _col$__col$_c1_0 tinyint"
- onJobUpdatedExportedFunc2 := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if (job.Query == query1 || job.Query == query2) && job.SchemaState == model.StateDeleteOnly && job.Type == model.ActionModifyColumn {
var (
_newCol *model.ColumnInfo
@@ -616,9 +606,7 @@ func TestColumnTypeChangeGenUniqueChangingName(t *testing.T) {
checkErr = errors.New("changing column name is incorrect")
}
}
- }
- hook.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc2)
- d.SetHook(hook)
+ })
tk.MustExec("drop table if exists t")
tk.MustExec("create table if not exists t(c1 bigint, _col$_c1 bigint, _col$__col$_c1_0 bigint, _col$__col$__col$_c1_0_0 bigint)")
diff --git a/pkg/ddl/column_test.go b/pkg/ddl/column_test.go
index 71f22ea7bb643..caa33dc883314 100644
--- a/pkg/ddl/column_test.go
+++ b/pkg/ddl/column_test.go
@@ -23,18 +23,17 @@ import (
"testing"
"github.com/pingcap/errors"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/terror"
"github.com/pingcap/tidb/pkg/sessionctx"
- "github.com/pingcap/tidb/pkg/sessiontxn"
"github.com/pingcap/tidb/pkg/store/mockstore"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/table/tables"
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/types"
"github.com/stretchr/testify/require"
)
@@ -51,7 +50,7 @@ func testCreateColumn(tk *testkit.TestKit, t *testing.T, ctx sessionctx.Context,
id := int64(idi)
v := getSchemaVer(t, ctx)
require.NoError(t, dom.Reload())
- tblInfo, exist := dom.InfoSchema().TableByID(tblID)
+ tblInfo, exist := dom.InfoSchema().TableByID(context.Background(), tblID)
require.True(t, exist)
checkHistoryJobArgs(t, ctx, id, &historyJobArgs{ver: v, tbl: tblInfo.Meta()})
return id
@@ -74,7 +73,7 @@ func testCreateColumns(tk *testkit.TestKit, t *testing.T, ctx sessionctx.Context
id := int64(idi)
v := getSchemaVer(t, ctx)
require.NoError(t, dom.Reload())
- tblInfo, exist := dom.InfoSchema().TableByID(tblID)
+ tblInfo, exist := dom.InfoSchema().TableByID(context.Background(), tblID)
require.True(t, exist)
checkHistoryJobArgs(t, ctx, id, &historyJobArgs{ver: v, tbl: tblInfo.Meta()})
return id
@@ -93,7 +92,7 @@ func testDropColumnInternal(tk *testkit.TestKit, t *testing.T, ctx sessionctx.Co
id := int64(idi)
v := getSchemaVer(t, ctx)
require.NoError(t, dom.Reload())
- tblInfo, exist := dom.InfoSchema().TableByID(tblID)
+ tblInfo, exist := dom.InfoSchema().TableByID(context.Background(), tblID)
require.True(t, exist)
checkHistoryJobArgs(t, ctx, id, &historyJobArgs{ver: v, tbl: tblInfo.Meta()})
return id
@@ -124,7 +123,7 @@ func testCreateIndex(tk *testkit.TestKit, t *testing.T, ctx sessionctx.Context,
id := int64(idi)
v := getSchemaVer(t, ctx)
require.NoError(t, dom.Reload())
- tblInfo, exist := dom.InfoSchema().TableByID(tblID)
+ tblInfo, exist := dom.InfoSchema().TableByID(context.Background(), tblID)
require.True(t, exist)
checkHistoryJobArgs(t, ctx, id, &historyJobArgs{ver: v, tbl: tblInfo.Meta()})
return id
@@ -149,7 +148,7 @@ func testDropColumns(tk *testkit.TestKit, t *testing.T, ctx sessionctx.Context,
id := int64(idi)
v := getSchemaVer(t, ctx)
require.NoError(t, dom.Reload())
- tblInfo, exist := dom.InfoSchema().TableByID(tblID)
+ tblInfo, exist := dom.InfoSchema().TableByID(context.Background(), tblID)
require.True(t, exist)
checkHistoryJobArgs(t, ctx, id, &historyJobArgs{ver: v, tbl: tblInfo.Meta()})
return id
@@ -168,7 +167,7 @@ func TestColumnBasic(t *testing.T) {
}
ctx := testNewContext(store)
- err := sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err := newTxn(ctx)
require.NoError(t, err)
var tableID int64
@@ -212,9 +211,9 @@ func TestColumnBasic(t *testing.T) {
require.NoError(t, err)
require.Equal(t, i, num)
- h, err := tbl.AddRecord(ctx.GetTableCtx(), types.MakeDatums(11, 12, 13, 14))
+ h, err := tbl.AddRecord(ctx.GetTableCtx(), txn, types.MakeDatums(11, 12, 13, 14))
require.NoError(t, err)
- err = sessiontxn.NewTxn(context.Background(), ctx)
+ _, err = newTxn(ctx)
require.NoError(t, err)
values, err := tables.RowWithCols(tbl, ctx, h, tbl.Cols())
require.NoError(t, err)
@@ -311,23 +310,17 @@ func TestColumnBasic(t *testing.T) {
}
func checkColumnKVExist(ctx sessionctx.Context, t table.Table, handle kv.Handle, col *table.Column, columnValue any, isExist bool) error {
- err := sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err := newTxn(ctx)
if err != nil {
return errors.Trace(err)
}
defer func() {
- if txn, err1 := ctx.Txn(true); err1 == nil {
- err = txn.Commit(context.Background())
- if err != nil {
- panic(err)
- }
+ err = txn.Commit(context.Background())
+ if err != nil {
+ panic(err)
}
}()
key := tablecodec.EncodeRecordKey(t.RecordPrefix(), handle)
- txn, err := ctx.Txn(true)
- if err != nil {
- return errors.Trace(err)
- }
data, err := txn.Get(context.TODO(), key)
if !isExist {
if terror.ErrorEqual(err, kv.ErrNotExist) {
@@ -366,7 +359,7 @@ func checkNoneColumn(t *testing.T, ctx sessionctx.Context, tableID int64, handle
func checkDeleteOnlyColumn(t *testing.T, ctx sessionctx.Context, tableID int64, handle kv.Handle, col *table.Column, row []types.Datum, columnValue any, dom *domain.Domain) {
tbl := testGetTable(t, dom, tableID)
- err := sessiontxn.NewTxn(context.Background(), ctx)
+ _, err := newTxn(ctx)
require.NoError(t, err)
i := 0
err = tables.IterRecords(tbl, ctx, tbl.Cols(), func(_ kv.Handle, data []types.Datum, cols []*table.Column) (bool, error) {
@@ -379,13 +372,13 @@ func checkDeleteOnlyColumn(t *testing.T, ctx sessionctx.Context, tableID int64,
err = checkColumnKVExist(ctx, tbl, handle, col, columnValue, false)
require.NoError(t, err)
// Test add a new row.
- err = sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err := newTxn(ctx)
require.NoError(t, err)
newRow := types.MakeDatums(int64(11), int64(22), int64(33))
- newHandle, err := tbl.AddRecord(ctx.GetTableCtx(), newRow)
+ newHandle, err := tbl.AddRecord(ctx.GetTableCtx(), txn, newRow)
require.NoError(t, err)
- err = sessiontxn.NewTxn(context.Background(), ctx)
+ _, err = newTxn(ctx)
require.NoError(t, err)
rows := [][]types.Datum{row, newRow}
@@ -402,12 +395,12 @@ func checkDeleteOnlyColumn(t *testing.T, ctx sessionctx.Context, tableID int64,
err = checkColumnKVExist(ctx, tbl, handle, col, columnValue, false)
require.NoError(t, err)
// Test remove a row.
- err = sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err = newTxn(ctx)
require.NoError(t, err)
- err = tbl.RemoveRecord(ctx.GetTableCtx(), newHandle, newRow)
+ err = tbl.RemoveRecord(ctx.GetTableCtx(), txn, newHandle, newRow)
require.NoError(t, err)
- err = sessiontxn.NewTxn(context.Background(), ctx)
+ _, err = newTxn(ctx)
require.NoError(t, err)
i = 0
err = tables.IterRecords(tbl, ctx, tbl.Cols(), func(_ kv.Handle, data []types.Datum, cols []*table.Column) (bool, error) {
@@ -425,7 +418,7 @@ func checkDeleteOnlyColumn(t *testing.T, ctx sessionctx.Context, tableID int64,
func checkWriteOnlyColumn(t *testing.T, ctx sessionctx.Context, tableID int64, handle kv.Handle, col *table.Column, row []types.Datum, columnValue any, dom *domain.Domain) {
tbl := testGetTable(t, dom, tableID)
- err := sessiontxn.NewTxn(context.Background(), ctx)
+ _, err := newTxn(ctx)
require.NoError(t, err)
i := 0
@@ -441,13 +434,13 @@ func checkWriteOnlyColumn(t *testing.T, ctx sessionctx.Context, tableID int64, h
require.NoError(t, err)
// Test add a new row.
- err = sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err := newTxn(ctx)
require.NoError(t, err)
newRow := types.MakeDatums(int64(11), int64(22), int64(33))
- newHandle, err := tbl.AddRecord(ctx.GetTableCtx(), newRow)
+ newHandle, err := tbl.AddRecord(ctx.GetTableCtx(), txn, newRow)
require.NoError(t, err)
- err = sessiontxn.NewTxn(context.Background(), ctx)
+ _, err = newTxn(ctx)
require.NoError(t, err)
rows := [][]types.Datum{row, newRow}
@@ -464,12 +457,12 @@ func checkWriteOnlyColumn(t *testing.T, ctx sessionctx.Context, tableID int64, h
err = checkColumnKVExist(ctx, tbl, newHandle, col, columnValue, true)
require.NoError(t, err)
// Test remove a row.
- err = sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err = newTxn(ctx)
require.NoError(t, err)
- err = tbl.RemoveRecord(ctx.GetTableCtx(), newHandle, newRow)
+ err = tbl.RemoveRecord(ctx.GetTableCtx(), txn, newHandle, newRow)
require.NoError(t, err)
- err = sessiontxn.NewTxn(context.Background(), ctx)
+ _, err = newTxn(ctx)
require.NoError(t, err)
i = 0
@@ -488,7 +481,7 @@ func checkWriteOnlyColumn(t *testing.T, ctx sessionctx.Context, tableID int64, h
func checkReorganizationColumn(t *testing.T, ctx sessionctx.Context, tableID int64, col *table.Column, row []types.Datum, columnValue any, dom *domain.Domain) {
tbl := testGetTable(t, dom, tableID)
- err := sessiontxn.NewTxn(context.Background(), ctx)
+ _, err := newTxn(ctx)
require.NoError(t, err)
i := 0
@@ -501,13 +494,13 @@ func checkReorganizationColumn(t *testing.T, ctx sessionctx.Context, tableID int
require.Equalf(t, 1, i, "expect 1, got %v", i)
// Test add a new row.
- err = sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err := newTxn(ctx)
require.NoError(t, err)
newRow := types.MakeDatums(int64(11), int64(22), int64(33))
- newHandle, err := tbl.AddRecord(ctx.GetTableCtx(), newRow)
+ newHandle, err := tbl.AddRecord(ctx.GetTableCtx(), txn, newRow)
require.NoError(t, err)
- err = sessiontxn.NewTxn(context.Background(), ctx)
+ _, err = newTxn(ctx)
require.NoError(t, err)
rows := [][]types.Datum{row, newRow}
@@ -525,12 +518,12 @@ func checkReorganizationColumn(t *testing.T, ctx sessionctx.Context, tableID int
require.NoError(t, err)
// Test remove a row.
- err = sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err = newTxn(ctx)
require.NoError(t, err)
- err = tbl.RemoveRecord(ctx.GetTableCtx(), newHandle, newRow)
+ err = tbl.RemoveRecord(ctx.GetTableCtx(), txn, newHandle, newRow)
require.NoError(t, err)
- err = sessiontxn.NewTxn(context.Background(), ctx)
+ _, err = newTxn(ctx)
require.NoError(t, err)
i = 0
@@ -546,7 +539,7 @@ func checkReorganizationColumn(t *testing.T, ctx sessionctx.Context, tableID int
func checkPublicColumn(t *testing.T, ctx sessionctx.Context, tableID int64, newCol *table.Column, oldRow []types.Datum, columnValue any, dom *domain.Domain, columnCnt int) {
tbl := testGetTable(t, dom, tableID)
- err := sessiontxn.NewTxn(context.Background(), ctx)
+ _, err := newTxn(ctx)
require.NoError(t, err)
i := 0
@@ -564,16 +557,16 @@ func checkPublicColumn(t *testing.T, ctx sessionctx.Context, tableID int64, newC
require.Equalf(t, 1, i, "expect 1, got %v", i)
// Test add a new row.
- err = sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err := newTxn(ctx)
require.NoError(t, err)
newRow := types.MakeDatums(int64(11), int64(22), int64(33), int64(44))
for j := 1; j < columnCnt; j++ {
newRow = append(newRow, types.NewDatum(int64(44)))
}
- handle, err := tbl.AddRecord(ctx.GetTableCtx(), newRow)
+ handle, err := tbl.AddRecord(ctx.GetTableCtx(), txn, newRow)
require.NoError(t, err)
- err = sessiontxn.NewTxn(context.Background(), ctx)
+ _, err = newTxn(ctx)
require.NoError(t, err)
rows := [][]types.Datum{updatedRow, newRow}
@@ -588,13 +581,13 @@ func checkPublicColumn(t *testing.T, ctx sessionctx.Context, tableID int64, newC
require.Equalf(t, 2, i, "expect 2, got %v", i)
// Test remove a row.
- err = sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err = newTxn(ctx)
require.NoError(t, err)
- err = tbl.RemoveRecord(ctx.GetTableCtx(), handle, newRow)
+ err = tbl.RemoveRecord(ctx.GetTableCtx(), txn, handle, newRow)
require.NoError(t, err)
- err = sessiontxn.NewTxn(context.Background(), ctx)
+ _, err = newTxn(ctx)
require.NoError(t, err)
i = 0
@@ -643,8 +636,6 @@ func testGetColumn(t table.Table, name string, isExist bool) error {
func TestAddColumn(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t, mockstore.WithDDLChecker())
- d := dom.DDL()
-
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("create table t1 (c1 int, c2 int, c3 int);")
@@ -656,13 +647,10 @@ func TestAddColumn(t *testing.T) {
tbl := testGetTable(t, dom, tableID)
ctx := testNewContext(store)
- err := sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err := newTxn(ctx)
require.NoError(t, err)
oldRow := types.MakeDatums(int64(1), int64(2), int64(3))
- handle, err := tbl.AddRecord(ctx.GetTableCtx(), oldRow)
- require.NoError(t, err)
-
- txn, err := ctx.Txn(true)
+ handle, err := tbl.AddRecord(ctx.GetTableCtx(), txn, oldRow)
require.NoError(t, err)
err = txn.Commit(context.Background())
require.NoError(t, err)
@@ -672,8 +660,7 @@ func TestAddColumn(t *testing.T) {
checkOK := false
- tc := &callback.TestDDLCallback{Do: dom}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if checkOK {
return
}
@@ -689,9 +676,7 @@ func TestAddColumn(t *testing.T) {
if newCol.State == model.StatePublic {
checkOK = true
}
- }
- tc.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(tc)
+ })
jobID := testCreateColumn(tk, t, testkit.NewTestKit(t, store).Session(), tableID, newColName, "", defaultColValue, dom)
testCheckJobDone(t, store, jobID, true)
@@ -705,8 +690,6 @@ func TestAddColumn(t *testing.T) {
func TestAddColumns(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t, mockstore.WithDDLChecker())
- d := dom.DDL()
-
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("create table t1 (c1 int, c2 int, c3 int);")
@@ -729,19 +712,15 @@ func TestAddColumns(t *testing.T) {
tbl := testGetTable(t, dom, tableID)
ctx := testNewContext(store)
- err := sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err := newTxn(ctx)
require.NoError(t, err)
oldRow := types.MakeDatums(int64(1), int64(2), int64(3))
- handle, err := tbl.AddRecord(ctx.GetTableCtx(), oldRow)
- require.NoError(t, err)
-
- txn, err := ctx.Txn(true)
+ handle, err := tbl.AddRecord(ctx.GetTableCtx(), txn, oldRow)
require.NoError(t, err)
err = txn.Commit(context.Background())
require.NoError(t, err)
- tc := &callback.TestDDLCallback{Do: dom}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
mu.Lock()
defer mu.Unlock()
if checkOK {
@@ -761,9 +740,7 @@ func TestAddColumns(t *testing.T) {
checkOK = true
}
}
- }
- tc.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(tc)
+ })
jobID := testCreateColumns(tk, t, testkit.NewTestKit(t, store).Session(), tableID, newColNames, positions, defaultColValue, dom)
@@ -795,12 +772,9 @@ func TestDropColumnInColumnTest(t *testing.T) {
colName := "c4"
defaultColValue := int64(4)
row := types.MakeDatums(int64(1), int64(2), int64(3))
- err := sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err := newTxn(ctx)
require.NoError(t, err)
- _, err = tbl.AddRecord(ctx.GetTableCtx(), append(row, types.NewDatum(defaultColValue)))
- require.NoError(t, err)
-
- txn, err := ctx.Txn(true)
+ _, err = tbl.AddRecord(ctx.GetTableCtx(), txn, append(row, types.NewDatum(defaultColValue)))
require.NoError(t, err)
err = txn.Commit(context.Background())
require.NoError(t, err)
@@ -809,9 +783,7 @@ func TestDropColumnInColumnTest(t *testing.T) {
var hookErr error
var mu sync.Mutex
- d := dom.DDL()
- tc := &callback.TestDDLCallback{Do: dom}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
mu.Lock()
defer mu.Unlock()
if checkOK {
@@ -823,9 +795,7 @@ func TestDropColumnInColumnTest(t *testing.T) {
checkOK = true
return
}
- }
- tc.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(tc)
+ })
jobID := testDropColumnInternal(tk, t, testkit.NewTestKit(t, store).Session(), tableID, colName, false, dom)
testCheckJobDone(t, store, jobID, false)
@@ -853,16 +823,13 @@ func TestDropColumns(t *testing.T) {
tbl := testGetTable(t, dom, tableID)
ctx := testNewContext(store)
- err := sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err := newTxn(ctx)
require.NoError(t, err)
colNames := []string{"c3", "c4"}
defaultColValue := int64(4)
row := types.MakeDatums(int64(1), int64(2), int64(3))
- _, err = tbl.AddRecord(ctx.GetTableCtx(), append(row, types.NewDatum(defaultColValue)))
- require.NoError(t, err)
-
- txn, err := ctx.Txn(true)
+ _, err = tbl.AddRecord(ctx.GetTableCtx(), txn, append(row, types.NewDatum(defaultColValue)))
require.NoError(t, err)
err = txn.Commit(context.Background())
require.NoError(t, err)
@@ -871,9 +838,7 @@ func TestDropColumns(t *testing.T) {
var hookErr error
var mu sync.Mutex
- d := dom.DDL()
- tc := &callback.TestDDLCallback{Do: dom}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
mu.Lock()
defer mu.Unlock()
if checkOK {
@@ -887,9 +852,7 @@ func TestDropColumns(t *testing.T) {
return
}
}
- }
- tc.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(tc)
+ })
jobID := testDropColumns(tk, t, testkit.NewTestKit(t, store).Session(), tableID, colNames, false, dom)
testCheckJobDone(t, store, jobID, false)
@@ -906,13 +869,13 @@ func TestDropColumns(t *testing.T) {
func testGetTable(t *testing.T, dom *domain.Domain, tableID int64) table.Table {
require.NoError(t, dom.Reload())
- tbl, exist := dom.InfoSchema().TableByID(tableID)
+ tbl, exist := dom.InfoSchema().TableByID(context.Background(), tableID)
require.True(t, exist)
return tbl
}
func TestWriteDataWriteOnlyMode(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, dbTestLease)
+ store := testkit.CreateMockStoreWithSchemaLease(t, dbTestLease)
tk := testkit.NewTestKit(t, store)
tk2 := testkit.NewTestKit(t, store)
@@ -920,29 +883,21 @@ func TestWriteDataWriteOnlyMode(t *testing.T) {
tk2.MustExec("use test")
tk.MustExec("CREATE TABLE t (`col1` bigint(20) DEFAULT 1,`col2` float,UNIQUE KEY `key1` (`col1`))")
- originalCallback := dom.DDL().GetHook()
- defer dom.DDL().SetHook(originalCallback)
-
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.SchemaState != model.StateWriteOnly {
return
}
tk2.MustExec("insert ignore into t values (1, 2)")
tk2.MustExec("insert ignore into t values (2, 2)")
- }
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("alter table t change column `col1` `col1` varchar(20)")
- hook = &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.SchemaState != model.StateWriteOnly {
return
}
tk2.MustExec("insert ignore into t values (1)")
tk2.MustExec("insert ignore into t values (2)")
- }
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("alter table t drop column `col1`")
- dom.DDL().SetHook(originalCallback)
}
diff --git a/pkg/ddl/column_type_change_test.go b/pkg/ddl/column_type_change_test.go
index 2d3bb48963b72..42ef263952afe 100644
--- a/pkg/ddl/column_type_change_test.go
+++ b/pkg/ddl/column_type_change_test.go
@@ -23,7 +23,6 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/ddl"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
@@ -34,13 +33,14 @@ import (
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/external"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/dbterror"
"github.com/stretchr/testify/require"
)
func TestColumnTypeChangeStateBetweenInteger(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("set global tidb_enable_row_level_checksum = 1")
tk.MustExec("use test")
@@ -57,9 +57,8 @@ func TestColumnTypeChangeStateBetweenInteger(t *testing.T) {
require.Equal(t, 2, len(tbl.Cols()))
require.NotNil(t, external.GetModifyColumn(t, tk, "test", "t", "c2", false))
- hook := &callback.TestDDLCallback{Do: dom}
var checkErr error
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if checkErr != nil {
return
}
@@ -87,8 +86,7 @@ func TestColumnTypeChangeStateBetweenInteger(t *testing.T) {
checkErr = errors.New("changingCol is nil")
}
}
- }
- dom.DDL().SetHook(hook)
+ })
// Alter sql will modify column c2 to tinyint not null.
SQL := "alter table t modify column c2 tinyint not null"
tk.MustExec(SQL)
@@ -109,7 +107,7 @@ func TestColumnTypeChangeStateBetweenInteger(t *testing.T) {
}
func TestRollbackColumnTypeChangeBetweenInteger(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("set global tidb_enable_row_level_checksum = 1")
tk.MustExec("use test")
@@ -122,10 +120,8 @@ func TestRollbackColumnTypeChangeBetweenInteger(t *testing.T) {
require.Equal(t, 2, len(tbl.Cols()))
require.NotNil(t, external.GetModifyColumn(t, tk, "test", "t", "c2", false))
- hook := &callback.TestDDLCallback{Do: dom}
// Mock roll back at model.StateNone.
- customizeHookRollbackAtState(hook, tbl, model.StateNone)
- dom.DDL().SetHook(hook)
+ customizeHookRollbackAtState(t, tbl, model.StateNone)
// Alter sql will modify column c2 to bigint not null.
SQL := "alter table t modify column c2 int not null"
err := tk.ExecToErr(SQL)
@@ -133,29 +129,26 @@ func TestRollbackColumnTypeChangeBetweenInteger(t *testing.T) {
assertRollBackedColUnchanged(t, tk)
// Mock roll back at model.StateDeleteOnly.
- customizeHookRollbackAtState(hook, tbl, model.StateDeleteOnly)
- dom.DDL().SetHook(hook)
+ customizeHookRollbackAtState(t, tbl, model.StateDeleteOnly)
err = tk.ExecToErr(SQL)
require.EqualError(t, err, "[ddl:1]MockRollingBackInCallBack-delete only")
assertRollBackedColUnchanged(t, tk)
// Mock roll back at model.StateWriteOnly.
- customizeHookRollbackAtState(hook, tbl, model.StateWriteOnly)
- dom.DDL().SetHook(hook)
+ customizeHookRollbackAtState(t, tbl, model.StateWriteOnly)
err = tk.ExecToErr(SQL)
require.EqualError(t, err, "[ddl:1]MockRollingBackInCallBack-write only")
assertRollBackedColUnchanged(t, tk)
// Mock roll back at model.StateWriteReorg.
- customizeHookRollbackAtState(hook, tbl, model.StateWriteReorganization)
- dom.DDL().SetHook(hook)
+ customizeHookRollbackAtState(t, tbl, model.StateWriteReorganization)
err = tk.ExecToErr(SQL)
require.EqualError(t, err, "[ddl:1]MockRollingBackInCallBack-write reorganization")
assertRollBackedColUnchanged(t, tk)
}
-func customizeHookRollbackAtState(hook *callback.TestDDLCallback, tbl table.Table, state model.SchemaState) {
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+func customizeHookRollbackAtState(t *testing.T, tbl table.Table, state model.SchemaState) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if tbl.Meta().ID != job.TableID {
return
}
@@ -163,7 +156,7 @@ func customizeHookRollbackAtState(hook *callback.TestDDLCallback, tbl table.Tabl
job.State = model.JobStateRollingback
job.Error = mockTerrorMap[state.String()]
}
- }
+ })
}
func assertRollBackedColUnchanged(t *testing.T, tk *testkit.TestKit) {
@@ -190,7 +183,7 @@ func init() {
// Test issue #20529.
func TestColumnTypeChangeIgnoreDisplayLength(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("set global tidb_enable_row_level_checksum = 1")
tk.MustExec("use test")
@@ -199,16 +192,14 @@ func TestColumnTypeChangeIgnoreDisplayLength(t *testing.T) {
assertHasAlterWriteReorg := func(tbl table.Table) {
// Restore assertResult to false.
assertResult = false
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if tbl.Meta().ID != job.TableID {
return
}
if job.SchemaState == model.StateWriteReorganization {
assertResult = true
}
- }
- dom.DDL().SetHook(hook)
+ })
}
// Change int to tinyint.
@@ -305,7 +296,7 @@ func TestRowLevelChecksumWithMultiSchemaChange(t *testing.T) {
// It's good because the insert / update logic will cast the related column to changing column rather than use
// origin default value directly.
func TestChangingColOriginDefaultValue(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("set global tidb_enable_row_level_checksum = 1")
tk.MustExec("use test")
@@ -319,14 +310,12 @@ func TestChangingColOriginDefaultValue(t *testing.T) {
tk.MustExec("insert into t values(2, 2)")
tbl := external.GetTableByName(t, tk, "test", "t")
- originalHook := dom.DDL().GetHook()
- hook := &callback.TestDDLCallback{Do: dom}
var (
once bool
checkErr error
)
i := 0
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if checkErr != nil {
return
}
@@ -371,11 +360,9 @@ func TestChangingColOriginDefaultValue(t *testing.T) {
}
i++
}
- }
- hook.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("alter table t modify column b tinyint NOT NULL")
- dom.DDL().SetHook(originalHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
require.NoError(t, checkErr)
// Since getReorgInfo will stagnate StateWriteReorganization for a ddl round, so insert should exec 3 times.
tk.MustQuery("select * from t order by a").Check(testkit.Rows("1 -1", "2 -2", "3 3", "4 4", "5 5"))
@@ -383,7 +370,7 @@ func TestChangingColOriginDefaultValue(t *testing.T) {
}
func TestChangingColOriginDefaultValueAfterAddColAndCastSucc(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("set global tidb_enable_row_level_checksum = 1")
tk.MustExec("use test")
@@ -400,14 +387,12 @@ func TestChangingColOriginDefaultValueAfterAddColAndCastSucc(t *testing.T) {
tk.MustExec("alter table t add column c timestamp default '1971-06-09' not null")
tbl := external.GetTableByName(t, tk, "test", "t")
- originalHook := dom.DDL().GetHook()
- hook := &callback.TestDDLCallback{Do: dom}
var (
once bool
checkErr error
)
i, stableTimes := 0, 0
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if checkErr != nil {
return
}
@@ -458,11 +443,10 @@ func TestChangingColOriginDefaultValueAfterAddColAndCastSucc(t *testing.T) {
stableTimes++
}
i++
- }
+ })
- dom.DDL().SetHook(hook)
tk.MustExec("alter table t modify column c date NOT NULL")
- dom.DDL().SetHook(originalHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore")
require.NoError(t, checkErr)
// Since getReorgInfo will stagnate StateWriteReorganization for a ddl round, so insert should exec 3 times.
tk.MustQuery("select * from t order by a").Check(
@@ -472,7 +456,7 @@ func TestChangingColOriginDefaultValueAfterAddColAndCastSucc(t *testing.T) {
// TestChangingColOriginDefaultValueAfterAddColAndCastFail tests #25383.
func TestChangingColOriginDefaultValueAfterAddColAndCastFail(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("set global tidb_enable_row_level_checksum = 1")
tk.MustExec("use test")
@@ -486,11 +470,9 @@ func TestChangingColOriginDefaultValueAfterAddColAndCastFail(t *testing.T) {
tk.MustExec("ALTER TABLE t ADD COLUMN x CHAR(218) NULL DEFAULT 'lkittuae'")
tbl := external.GetTableByName(t, tk, "test", "t")
- originalHook := dom.DDL().GetHook()
- hook := &callback.TestDDLCallback{Do: dom}
var checkErr error
var firstJobID int64
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if checkErr != nil {
return
}
@@ -541,13 +523,12 @@ func TestChangingColOriginDefaultValueAfterAddColAndCastFail(t *testing.T) {
}
}
}
- }
+ })
- dom.DDL().SetHook(hook)
tk.MustExec("alter table t modify column x DATETIME NULL DEFAULT '3771-02-28 13:00:11' AFTER b;")
tk.MustExec("insert into t(a) value('1')")
tk.MustExec("alter table t modify column b varchar(256) default (REPLACE(UPPER(UUID()), '-', ''));")
- dom.DDL().SetHook(originalHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore")
require.NoError(t, checkErr)
tk.MustQuery("select * from t order by a").Check(testkit.Rows("18apf -729850476163 3771-02-28 13:00:11"))
tk.MustExec("drop table if exists t")
@@ -555,7 +536,7 @@ func TestChangingColOriginDefaultValueAfterAddColAndCastFail(t *testing.T) {
// Close issue #23202
func TestDDLExitWhenCancelMeetPanic(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("set global tidb_enable_row_level_checksum = 1")
tk.MustExec("use test")
@@ -571,17 +552,15 @@ func TestDDLExitWhenCancelMeetPanic(t *testing.T) {
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/mockExceedErrorLimit"))
}()
- hook := &callback.TestDDLCallback{Do: dom}
var jobID int64
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if jobID != 0 {
return
}
if job.Type == model.ActionDropIndex {
jobID = job.ID
}
- }
- dom.DDL().SetHook(hook)
+ })
// when it panics in write-reorg state, the job will be pulled up as a cancelling job. Since drop-index with
// write-reorg can't be cancelled, so it will be converted to running state and try again (dead loop).
@@ -598,7 +577,7 @@ func TestDDLExitWhenCancelMeetPanic(t *testing.T) {
// Close issue #24584
func TestCancelCTCInReorgStateWillCauseGoroutineLeak(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("set global tidb_enable_row_level_checksum = 1")
tk.MustExec("use test")
@@ -608,18 +587,13 @@ func TestCancelCTCInReorgStateWillCauseGoroutineLeak(t *testing.T) {
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/mockInfiniteReorgLogic"))
}()
- // set ddl hook
- originalHook := dom.DDL().GetHook()
- defer dom.DDL().SetHook(originalHook)
-
tk.MustExec("drop table if exists ctc_goroutine_leak")
tk.MustExec("create table ctc_goroutine_leak (a int)")
tk.MustExec("insert into ctc_goroutine_leak values(1),(2),(3)")
tbl := external.GetTableByName(t, tk, "test", "ctc_goroutine_leak")
- hook := &callback.TestDDLCallback{Do: dom}
var jobID int64
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if jobID != 0 {
return
}
@@ -629,8 +603,7 @@ func TestCancelCTCInReorgStateWillCauseGoroutineLeak(t *testing.T) {
if job.Query == "alter table ctc_goroutine_leak modify column a tinyint" {
jobID = job.ID
}
- }
- dom.DDL().SetHook(hook)
+ })
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
@@ -654,7 +627,7 @@ func TestCancelCTCInReorgStateWillCauseGoroutineLeak(t *testing.T) {
// For select statement, it truncates the string and return no errors. (which is 3977-02-22 00:00:00 here)
// For ddl reorging or changing column in ctc, it needs report some errors.
func TestCastDateToTimestampInReorgAttribute(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, 600*time.Millisecond)
+ store := testkit.CreateMockStoreWithSchemaLease(t, 600*time.Millisecond)
tk := testkit.NewTestKit(t, store)
tk.MustExec("set global tidb_enable_row_level_checksum = 1")
tk.MustExec("use test")
@@ -668,8 +641,7 @@ func TestCastDateToTimestampInReorgAttribute(t *testing.T) {
var checkErr1 error
var checkErr2 error
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if checkErr1 != nil || checkErr2 != nil || tbl.Meta().ID != job.TableID {
return
}
@@ -680,8 +652,7 @@ func TestCastDateToTimestampInReorgAttribute(t *testing.T) {
checkErr1 = tk.ExecToErr("insert into `t` set `a` = '3977-02-22'") // this(string) will be cast to a as date, then cast a(date) as timestamp to changing column.
checkErr2 = tk.ExecToErr("update t set `a` = '3977-02-22'")
}
- }
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("alter table t modify column a TIMESTAMP NULL DEFAULT '2021-04-28 03:35:11' FIRST")
require.EqualError(t, checkErr1, "[types:1292]Incorrect timestamp value: '3977-02-22'")
diff --git a/pkg/ddl/constraint.go b/pkg/ddl/constraint.go
index 572c66439889a..c1919aef1bc9a 100644
--- a/pkg/ddl/constraint.go
+++ b/pkg/ddl/constraint.go
@@ -31,10 +31,10 @@ import (
"github.com/pingcap/tidb/pkg/util/dbterror"
)
-func (w *worker) onAddCheckConstraint(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func (w *worker) onAddCheckConstraint(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
// Handle the rolling back job.
if job.IsRollingback() {
- return rollingBackAddConstraint(d, t, job)
+ return rollingBackAddConstraint(jobCtx, t, job)
}
failpoint.Inject("errorBeforeDecodeArgs", func(val failpoint.Value) {
@@ -80,7 +80,7 @@ func (w *worker) onAddCheckConstraint(d *ddlCtx, t *meta.Meta, job *model.Job) (
// If not enforced, add it directly.
if !constraintInfoInMeta.Enforced {
constraintInfoInMeta.State = model.StatePublic
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -93,11 +93,11 @@ func (w *worker) onAddCheckConstraint(d *ddlCtx, t *meta.Meta, job *model.Job) (
case model.StateNone:
job.SchemaState = model.StateWriteOnly
constraintInfoInMeta.State = model.StateWriteOnly
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, true)
case model.StateWriteOnly:
job.SchemaState = model.StateWriteReorganization
constraintInfoInMeta.State = model.StateWriteReorganization
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, true)
case model.StateWriteReorganization:
err = w.verifyRemainRecordsForCheckConstraint(dbInfo, tblInfo, constraintInfoInMeta)
if err != nil {
@@ -107,7 +107,7 @@ func (w *worker) onAddCheckConstraint(d *ddlCtx, t *meta.Meta, job *model.Job) (
return ver, errors.Trace(err)
}
constraintInfoInMeta.State = model.StatePublic
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -159,7 +159,7 @@ func checkAddCheckConstraint(t *meta.Meta, job *model.Job) (*model.DBInfo, *mode
// onDropCheckConstraint can be called from two case:
// 1: rollback in add constraint.(in rollback function the job.args will be changed)
// 2: user drop constraint ddl.
-func onDropCheckConstraint(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onDropCheckConstraint(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
tblInfo, constraintInfo, err := checkDropCheckConstraint(t, job)
if err != nil {
return ver, errors.Trace(err)
@@ -169,7 +169,7 @@ func onDropCheckConstraint(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64,
case model.StatePublic:
job.SchemaState = model.StateWriteOnly
constraintInfo.State = model.StateWriteOnly
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, true)
case model.StateWriteOnly:
// write only state constraint will still take effect to check the newly inserted data.
// So the dependent column shouldn't be dropped even in this intermediate state.
@@ -180,7 +180,7 @@ func onDropCheckConstraint(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64,
tblInfo.Constraints = append(tblInfo.Constraints[0:i], tblInfo.Constraints[i+1:]...)
}
}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -214,14 +214,14 @@ func checkDropCheckConstraint(t *meta.Meta, job *model.Job) (*model.TableInfo, *
return tblInfo, constraintInfo, nil
}
-func (w *worker) onAlterCheckConstraint(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func (w *worker) onAlterCheckConstraint(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
dbInfo, tblInfo, constraintInfo, enforced, err := checkAlterCheckConstraint(t, job)
if err != nil {
return ver, errors.Trace(err)
}
if job.IsRollingback() {
- return rollingBackAlterConstraint(d, t, job)
+ return rollingBackAlterConstraint(jobCtx, t, job)
}
// Current State is desired.
@@ -237,11 +237,11 @@ func (w *worker) onAlterCheckConstraint(d *ddlCtx, t *meta.Meta, job *model.Job)
job.SchemaState = model.StateWriteReorganization
constraintInfo.State = model.StateWriteReorganization
constraintInfo.Enforced = enforced
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, true)
case model.StateWriteReorganization:
job.SchemaState = model.StateWriteOnly
constraintInfo.State = model.StateWriteOnly
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, true)
case model.StateWriteOnly:
err = w.verifyRemainRecordsForCheckConstraint(dbInfo, tblInfo, constraintInfo)
if err != nil {
@@ -251,7 +251,7 @@ func (w *worker) onAlterCheckConstraint(d *ddlCtx, t *meta.Meta, job *model.Job)
return ver, errors.Trace(err)
}
constraintInfo.State = model.StatePublic
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -259,7 +259,7 @@ func (w *worker) onAlterCheckConstraint(d *ddlCtx, t *meta.Meta, job *model.Job)
}
} else {
constraintInfo.Enforced = enforced
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, true)
if err != nil {
// update version and tableInfo error will cause retry.
return ver, errors.Trace(err)
diff --git a/pkg/ddl/constraint_test.go b/pkg/ddl/constraint_test.go
index 96d1f60bb185e..02047441f2974 100644
--- a/pkg/ddl/constraint_test.go
+++ b/pkg/ddl/constraint_test.go
@@ -20,17 +20,17 @@ import (
"time"
"github.com/pingcap/failpoint"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/table/tables"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/external"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/stretchr/testify/require"
)
func TestAlterConstraintAddDrop(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
@@ -47,18 +47,14 @@ func TestAlterConstraintAddDrop(t *testing.T) {
require.Error(t, err)
var checkErr error
- d := dom.DDL()
- callback := &callback.TestDDLCallback{}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if checkErr != nil {
return
}
if job.SchemaState == model.StateWriteOnly {
_, checkErr = tk1.Exec("insert into t (a, b) values(5,6) ")
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(callback)
+ })
tk.MustExec("alter table t add constraint cc check ( b < 5 )")
require.Errorf(t, err, "[table:3819]Check constraint 'cc' is violated.")
@@ -68,7 +64,7 @@ func TestAlterConstraintAddDrop(t *testing.T) {
}
func TestAlterAddConstraintStateChange(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
@@ -80,9 +76,7 @@ func TestAlterAddConstraintStateChange(t *testing.T) {
tk1.MustExec("insert into t values(12)")
var checkErr error
- d := dom.DDL()
- callback := &callback.TestDDLCallback{}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if checkErr != nil {
return
}
@@ -101,12 +95,10 @@ func TestAlterAddConstraintStateChange(t *testing.T) {
tableCommon.Constraints = originCons
tableCommon.WritableConstraint()
}
- }
+ })
//StatNone StateWriteReorganization
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/mockVerifyRemainDataSuccess", "return(true)"))
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(callback)
tk.MustExec("alter table t add constraint c0 check ( a > 10)")
tk.MustQuery("select * from t").Check(testkit.Rows("12", "1"))
tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n `a` int(11) DEFAULT NULL,\n CONSTRAINT `c0` CHECK ((`a` > 10))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
@@ -116,7 +108,7 @@ func TestAlterAddConstraintStateChange(t *testing.T) {
}
func TestAlterAddConstraintStateChange1(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
@@ -126,10 +118,8 @@ func TestAlterAddConstraintStateChange1(t *testing.T) {
tk1.MustExec("use test")
tk1.MustExec("insert into t values(12)")
- d := dom.DDL()
- callback := &callback.TestDDLCallback{}
// StatNone -> StateWriteOnly
- onJobUpdatedExportedFunc1 := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.SchemaState == model.StateWriteOnly {
// set constraint state
constraintTable := external.GetTableByName(t, tk1, "test", "t")
@@ -144,9 +134,7 @@ func TestAlterAddConstraintStateChange1(t *testing.T) {
tableCommon.Constraints = originCons
tableCommon.WritableConstraint()
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc1)
- d.SetHook(callback)
+ })
tk.MustGetErrMsg("alter table t add constraint c1 check ( a > 10)", "[ddl:3819]Check constraint 'c1' is violated.")
tk.MustQuery("select * from t").Check(testkit.Rows("12", "1"))
tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n `a` int(11) DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
@@ -154,7 +142,7 @@ func TestAlterAddConstraintStateChange1(t *testing.T) {
}
func TestAlterAddConstraintStateChange2(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
@@ -164,10 +152,8 @@ func TestAlterAddConstraintStateChange2(t *testing.T) {
tk1.MustExec("use test")
tk1.MustExec("insert into t values(12)")
- d := dom.DDL()
- callback := &callback.TestDDLCallback{}
// StateWriteOnly -> StateWriteReorganization
- onJobUpdatedExportedFunc2 := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.SchemaState == model.StateWriteReorganization {
// set constraint state
constraintTable := external.GetTableByName(t, tk1, "test", "t")
@@ -180,9 +166,7 @@ func TestAlterAddConstraintStateChange2(t *testing.T) {
tableCommon.Constraints[0].State = model.StateWriteReorganization
tableCommon.WritableConstraint()
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc2)
- d.SetHook(callback)
+ })
tk.MustExec("alter table t add constraint c2 check ( a > 10)")
tk.MustQuery("select * from t").Check(testkit.Rows("12"))
tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n `a` int(11) DEFAULT NULL,\n CONSTRAINT `c2` CHECK ((`a` > 10))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
@@ -190,7 +174,7 @@ func TestAlterAddConstraintStateChange2(t *testing.T) {
}
func TestAlterAddConstraintStateChange3(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
@@ -201,10 +185,8 @@ func TestAlterAddConstraintStateChange3(t *testing.T) {
tk1.MustExec("insert into t values(12)")
addCheckDone := false
- d := dom.DDL()
- callback := &callback.TestDDLCallback{}
// StateWriteReorganization -> StatePublic
- onJobUpdatedExportedFunc3 := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.Type != model.ActionAddCheckConstraint || job.TableName != "t" {
return
}
@@ -221,9 +203,7 @@ func TestAlterAddConstraintStateChange3(t *testing.T) {
tableCommon.WritableConstraint()
addCheckDone = true
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc3)
- d.SetHook(callback)
+ })
tk.MustExec("alter table t add constraint c3 check ( a > 10)")
// Issue TiDB#48123.
for i := 0; i <= 100; i++ {
@@ -237,7 +217,7 @@ func TestAlterAddConstraintStateChange3(t *testing.T) {
}
func TestAlterEnforcedConstraintStateChange(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
@@ -247,10 +227,8 @@ func TestAlterEnforcedConstraintStateChange(t *testing.T) {
tk1.MustExec("use test")
tk1.MustExec("insert into t values(12)")
- d := dom.DDL()
- callback := &callback.TestDDLCallback{}
// StateWriteReorganization -> StatePublic
- onJobUpdatedExportedFunc3 := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.SchemaState == model.StateWriteReorganization {
// set constraint state
constraintTable := external.GetTableByName(t, tk1, "test", "t")
@@ -263,9 +241,7 @@ func TestAlterEnforcedConstraintStateChange(t *testing.T) {
tableCommon.Constraints[0].State = model.StateWriteReorganization
tableCommon.WritableConstraint()
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc3)
- d.SetHook(callback)
+ })
tk.MustExec("alter table t alter constraint c1 enforced")
tk.MustQuery("select * from t").Check(testkit.Rows("12"))
}
diff --git a/pkg/ddl/create_table.go b/pkg/ddl/create_table.go
new file mode 100644
index 0000000000000..d25800058d032
--- /dev/null
+++ b/pkg/ddl/create_table.go
@@ -0,0 +1,1529 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ddl
+
+import (
+ "context"
+ "fmt"
+ "math"
+ "strings"
+ "sync/atomic"
+ "unicode/utf8"
+
+ "github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
+ "github.com/pingcap/tidb/pkg/config"
+ "github.com/pingcap/tidb/pkg/ddl/logutil"
+ "github.com/pingcap/tidb/pkg/ddl/placement"
+ "github.com/pingcap/tidb/pkg/domain/infosync"
+ "github.com/pingcap/tidb/pkg/expression"
+ "github.com/pingcap/tidb/pkg/infoschema"
+ "github.com/pingcap/tidb/pkg/meta"
+ "github.com/pingcap/tidb/pkg/meta/autoid"
+ "github.com/pingcap/tidb/pkg/parser/ast"
+ "github.com/pingcap/tidb/pkg/parser/format"
+ "github.com/pingcap/tidb/pkg/parser/model"
+ "github.com/pingcap/tidb/pkg/parser/mysql"
+ field_types "github.com/pingcap/tidb/pkg/parser/types"
+ "github.com/pingcap/tidb/pkg/sessionctx"
+ "github.com/pingcap/tidb/pkg/sessionctx/variable"
+ statsutil "github.com/pingcap/tidb/pkg/statistics/handle/util"
+ "github.com/pingcap/tidb/pkg/table"
+ "github.com/pingcap/tidb/pkg/table/tables"
+ "github.com/pingcap/tidb/pkg/types"
+ driver "github.com/pingcap/tidb/pkg/types/parser_driver"
+ "github.com/pingcap/tidb/pkg/util/dbterror"
+ "github.com/pingcap/tidb/pkg/util/mock"
+ "github.com/pingcap/tidb/pkg/util/set"
+ "go.uber.org/zap"
+)
+
+// DANGER: it is an internal function used by onCreateTable and onCreateTables, for reusing code. Be careful.
+// 1. it expects the argument of job has been deserialized.
+// 2. it won't call updateSchemaVersion, FinishTableJob and asyncNotifyEvent.
+func createTable(jobCtx *jobContext, t *meta.Meta, job *model.Job, fkCheck bool) (*model.TableInfo, error) {
+ schemaID := job.SchemaID
+ tbInfo := job.Args[0].(*model.TableInfo)
+
+ tbInfo.State = model.StateNone
+ err := checkTableNotExists(jobCtx.infoCache, schemaID, tbInfo.Name.L)
+ if err != nil {
+ if infoschema.ErrDatabaseNotExists.Equal(err) || infoschema.ErrTableExists.Equal(err) {
+ job.State = model.JobStateCancelled
+ }
+ return tbInfo, errors.Trace(err)
+ }
+
+ err = checkConstraintNamesNotExists(t, schemaID, tbInfo.Constraints)
+ if err != nil {
+ if infoschema.ErrCheckConstraintDupName.Equal(err) {
+ job.State = model.JobStateCancelled
+ }
+ return tbInfo, errors.Trace(err)
+ }
+
+ retryable, err := checkTableForeignKeyValidInOwner(jobCtx, job, tbInfo, fkCheck)
+ if err != nil {
+ if !retryable {
+ job.State = model.JobStateCancelled
+ }
+ return tbInfo, errors.Trace(err)
+ }
+ // Allocate foreign key ID.
+ for _, fkInfo := range tbInfo.ForeignKeys {
+ fkInfo.ID = allocateFKIndexID(tbInfo)
+ fkInfo.State = model.StatePublic
+ }
+ switch tbInfo.State {
+ case model.StateNone:
+ // none -> public
+ tbInfo.State = model.StatePublic
+ tbInfo.UpdateTS = t.StartTS
+ err = createTableOrViewWithCheck(t, job, schemaID, tbInfo)
+ if err != nil {
+ return tbInfo, errors.Trace(err)
+ }
+
+ failpoint.Inject("checkOwnerCheckAllVersionsWaitTime", func(val failpoint.Value) {
+ if val.(bool) {
+ failpoint.Return(tbInfo, errors.New("mock create table error"))
+ }
+ })
+
+ // build table & partition bundles if any.
+ if err = checkAllTablePlacementPoliciesExistAndCancelNonExistJob(t, job, tbInfo); err != nil {
+ return tbInfo, errors.Trace(err)
+ }
+
+ if tbInfo.TiFlashReplica != nil {
+ replicaInfo := tbInfo.TiFlashReplica
+ if pi := tbInfo.GetPartitionInfo(); pi != nil {
+ logutil.DDLLogger().Info("Set TiFlash replica pd rule for partitioned table when creating", zap.Int64("tableID", tbInfo.ID))
+ if e := infosync.ConfigureTiFlashPDForPartitions(false, &pi.Definitions, replicaInfo.Count, &replicaInfo.LocationLabels, tbInfo.ID); e != nil {
+ job.State = model.JobStateCancelled
+ return tbInfo, errors.Trace(e)
+ }
+ // Partitions that in adding mid-state. They have high priorities, so we should set accordingly pd rules.
+ if e := infosync.ConfigureTiFlashPDForPartitions(true, &pi.AddingDefinitions, replicaInfo.Count, &replicaInfo.LocationLabels, tbInfo.ID); e != nil {
+ job.State = model.JobStateCancelled
+ return tbInfo, errors.Trace(e)
+ }
+ } else {
+ logutil.DDLLogger().Info("Set TiFlash replica pd rule when creating", zap.Int64("tableID", tbInfo.ID))
+ if e := infosync.ConfigureTiFlashPDForTable(tbInfo.ID, replicaInfo.Count, &replicaInfo.LocationLabels); e != nil {
+ job.State = model.JobStateCancelled
+ return tbInfo, errors.Trace(e)
+ }
+ }
+ }
+
+ bundles, err := placement.NewFullTableBundles(t, tbInfo)
+ if err != nil {
+ job.State = model.JobStateCancelled
+ return tbInfo, errors.Trace(err)
+ }
+
+ // Send the placement bundle to PD.
+ err = infosync.PutRuleBundlesWithDefaultRetry(context.TODO(), bundles)
+ if err != nil {
+ job.State = model.JobStateCancelled
+ return tbInfo, errors.Wrapf(err, "failed to notify PD the placement rules")
+ }
+
+ return tbInfo, nil
+ default:
+ return tbInfo, dbterror.ErrInvalidDDLState.GenWithStackByArgs("table", tbInfo.State)
+ }
+}
+
+func onCreateTable(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+ failpoint.Inject("mockExceedErrorLimit", func(val failpoint.Value) {
+ if val.(bool) {
+ failpoint.Return(ver, errors.New("mock do job error"))
+ }
+ })
+
+ // just decode, createTable will use it as Args[0]
+ tbInfo := &model.TableInfo{}
+ fkCheck := false
+ if err := job.DecodeArgs(tbInfo, &fkCheck); err != nil {
+ // Invalid arguments, cancel this job.
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+
+ if len(tbInfo.ForeignKeys) > 0 {
+ return createTableWithForeignKeys(jobCtx, t, job, tbInfo, fkCheck)
+ }
+
+ tbInfo, err := createTable(jobCtx, t, job, fkCheck)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+
+ ver, err = updateSchemaVersion(jobCtx, t, job)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+
+ // Finish this job.
+ job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tbInfo)
+ createTableEvent := statsutil.NewCreateTableEvent(
+ job.SchemaID,
+ tbInfo,
+ )
+ asyncNotifyEvent(jobCtx, createTableEvent)
+ return ver, errors.Trace(err)
+}
+
+func createTableWithForeignKeys(jobCtx *jobContext, t *meta.Meta, job *model.Job, tbInfo *model.TableInfo, fkCheck bool) (ver int64, err error) {
+ switch tbInfo.State {
+ case model.StateNone, model.StatePublic:
+ // create table in non-public or public state. The function `createTable` will always reset
+ // the `tbInfo.State` with `model.StateNone`, so it's fine to just call the `createTable` with
+ // public state.
+ // when `br` restores table, the state of `tbInfo` will be public.
+ tbInfo, err = createTable(jobCtx, t, job, fkCheck)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ tbInfo.State = model.StateWriteOnly
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tbInfo, true)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ job.SchemaState = model.StateWriteOnly
+ case model.StateWriteOnly:
+ tbInfo.State = model.StatePublic
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tbInfo, true)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tbInfo)
+ createTableEvent := statsutil.NewCreateTableEvent(
+ job.SchemaID,
+ tbInfo,
+ )
+ asyncNotifyEvent(jobCtx, createTableEvent)
+ return ver, nil
+ default:
+ return ver, errors.Trace(dbterror.ErrInvalidDDLJob.GenWithStackByArgs("table", tbInfo.State))
+ }
+ return ver, errors.Trace(err)
+}
+
+func onCreateTables(jobCtx *jobContext, t *meta.Meta, job *model.Job) (int64, error) {
+ var ver int64
+
+ var args []*model.TableInfo
+ fkCheck := false
+ err := job.DecodeArgs(&args, &fkCheck)
+ if err != nil {
+ // Invalid arguments, cancel this job.
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+
+ // We don't construct jobs for every table, but only tableInfo
+ // The following loop creates a stub job for every table
+ //
+ // it clones a stub job from the ActionCreateTables job
+ stubJob := job.Clone()
+ stubJob.Args = make([]any, 1)
+ for i := range args {
+ stubJob.TableID = args[i].ID
+ stubJob.Args[0] = args[i]
+ if args[i].Sequence != nil {
+ err := createSequenceWithCheck(t, stubJob, args[i])
+ if err != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+ } else {
+ tbInfo, err := createTable(jobCtx, t, stubJob, fkCheck)
+ if err != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+ args[i] = tbInfo
+ }
+ }
+
+ ver, err = updateSchemaVersion(jobCtx, t, job)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+
+ job.State = model.JobStateDone
+ job.SchemaState = model.StatePublic
+ job.BinlogInfo.SetTableInfos(ver, args)
+
+ for i := range args {
+ createTableEvent := statsutil.NewCreateTableEvent(
+ job.SchemaID,
+ args[i],
+ )
+ asyncNotifyEvent(jobCtx, createTableEvent)
+ }
+
+ return ver, errors.Trace(err)
+}
+
+func createTableOrViewWithCheck(t *meta.Meta, job *model.Job, schemaID int64, tbInfo *model.TableInfo) error {
+ err := checkTableInfoValid(tbInfo)
+ if err != nil {
+ job.State = model.JobStateCancelled
+ return errors.Trace(err)
+ }
+ return t.CreateTableOrView(schemaID, tbInfo)
+}
+
+func onCreateView(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+ schemaID := job.SchemaID
+ tbInfo := &model.TableInfo{}
+ var orReplace bool
+ var _placeholder int64 // oldTblInfoID
+ if err := job.DecodeArgs(tbInfo, &orReplace, &_placeholder); err != nil {
+ // Invalid arguments, cancel this job.
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+ tbInfo.State = model.StateNone
+
+ oldTableID, err := findTableIDByName(jobCtx.infoCache, t, schemaID, tbInfo.Name.L)
+ if infoschema.ErrTableNotExists.Equal(err) {
+ err = nil
+ }
+ failpoint.InjectCall("onDDLCreateView", job)
+ if err != nil {
+ if infoschema.ErrDatabaseNotExists.Equal(err) {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ } else if !infoschema.ErrTableExists.Equal(err) {
+ return ver, errors.Trace(err)
+ }
+ if !orReplace {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+ }
+ ver, err = updateSchemaVersion(jobCtx, t, job)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ switch tbInfo.State {
+ case model.StateNone:
+ // none -> public
+ tbInfo.State = model.StatePublic
+ tbInfo.UpdateTS = t.StartTS
+ if oldTableID > 0 && orReplace {
+ err = t.DropTableOrView(schemaID, oldTableID)
+ if err != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+ err = t.GetAutoIDAccessors(schemaID, oldTableID).Del()
+ if err != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+ }
+ err = createTableOrViewWithCheck(t, job, schemaID, tbInfo)
+ if err != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+ // Finish this job.
+ job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tbInfo)
+ return ver, nil
+ default:
+ return ver, dbterror.ErrInvalidDDLState.GenWithStackByArgs("table", tbInfo.State)
+ }
+}
+
+func findTableIDByName(infoCache *infoschema.InfoCache, t *meta.Meta, schemaID int64, tableName string) (int64, error) {
+ // Try to use memory schema info to check first.
+ currVer, err := t.GetSchemaVersion()
+ if err != nil {
+ return 0, err
+ }
+ is := infoCache.GetLatest()
+ if is != nil && is.SchemaMetaVersion() == currVer {
+ return findTableIDFromInfoSchema(is, schemaID, tableName)
+ }
+
+ return findTableIDFromStore(t, schemaID, tableName)
+}
+
+func findTableIDFromInfoSchema(is infoschema.InfoSchema, schemaID int64, tableName string) (int64, error) {
+ schema, ok := is.SchemaByID(schemaID)
+ if !ok {
+ return 0, infoschema.ErrDatabaseNotExists.GenWithStackByArgs("")
+ }
+ tbl, err := is.TableByName(context.Background(), schema.Name, model.NewCIStr(tableName))
+ if err != nil {
+ return 0, err
+ }
+ return tbl.Meta().ID, nil
+}
+
+func findTableIDFromStore(t *meta.Meta, schemaID int64, tableName string) (int64, error) {
+ tbls, err := t.ListSimpleTables(schemaID)
+ if err != nil {
+ if meta.ErrDBNotExists.Equal(err) {
+ return 0, infoschema.ErrDatabaseNotExists.GenWithStackByArgs("")
+ }
+ return 0, errors.Trace(err)
+ }
+ for _, tbl := range tbls {
+ if tbl.Name.L == tableName {
+ return tbl.ID, nil
+ }
+ }
+ return 0, infoschema.ErrTableNotExists.FastGenByArgs(tableName)
+}
+
+// BuildTableInfoFromAST builds model.TableInfo from a SQL statement.
+// Note: TableID and PartitionID are left as uninitialized value.
+func BuildTableInfoFromAST(s *ast.CreateTableStmt) (*model.TableInfo, error) {
+ return buildTableInfoWithCheck(mock.NewContext(), s, mysql.DefaultCharset, "", nil)
+}
+
+// buildTableInfoWithCheck builds model.TableInfo from a SQL statement.
+// Note: TableID and PartitionIDs are left as uninitialized value.
+func buildTableInfoWithCheck(ctx sessionctx.Context, s *ast.CreateTableStmt, dbCharset, dbCollate string, placementPolicyRef *model.PolicyRefInfo) (*model.TableInfo, error) {
+ tbInfo, err := BuildTableInfoWithStmt(ctx, s, dbCharset, dbCollate, placementPolicyRef)
+ if err != nil {
+ return nil, err
+ }
+ // Fix issue 17952 which will cause partition range expr can't be parsed as Int.
+ // checkTableInfoValidWithStmt will do the constant fold the partition expression first,
+ // then checkTableInfoValidExtra will pass the tableInfo check successfully.
+ if err = checkTableInfoValidWithStmt(ctx, tbInfo, s); err != nil {
+ return nil, err
+ }
+ if err = checkTableInfoValidExtra(ctx, tbInfo); err != nil {
+ return nil, err
+ }
+ return tbInfo, nil
+}
+
+// CheckTableInfoValidWithStmt exposes checkTableInfoValidWithStmt to SchemaTracker. Maybe one day we can delete it.
+func CheckTableInfoValidWithStmt(ctx sessionctx.Context, tbInfo *model.TableInfo, s *ast.CreateTableStmt) (err error) {
+ return checkTableInfoValidWithStmt(ctx, tbInfo, s)
+}
+
+func checkTableInfoValidWithStmt(ctx sessionctx.Context, tbInfo *model.TableInfo, s *ast.CreateTableStmt) (err error) {
+ // All of these rely on the AST structure of expressions, which were
+ // lost in the model (got serialized into strings).
+ if err := checkGeneratedColumn(ctx, s.Table.Schema, tbInfo.Name, s.Cols); err != nil {
+ return errors.Trace(err)
+ }
+
+ // Check if table has a primary key if required.
+ if !ctx.GetSessionVars().InRestrictedSQL && ctx.GetSessionVars().PrimaryKeyRequired && len(tbInfo.GetPkName().String()) == 0 {
+ return infoschema.ErrTableWithoutPrimaryKey
+ }
+ if tbInfo.Partition != nil {
+ if err := checkPartitionDefinitionConstraints(ctx, tbInfo); err != nil {
+ return errors.Trace(err)
+ }
+ if s.Partition != nil {
+ if err := checkPartitionFuncType(ctx, s.Partition.Expr, s.Table.Schema.O, tbInfo); err != nil {
+ return errors.Trace(err)
+ }
+ if err := checkPartitioningKeysConstraints(ctx, s, tbInfo); err != nil {
+ return errors.Trace(err)
+ }
+ }
+ }
+ if tbInfo.TTLInfo != nil {
+ if err := checkTTLInfoValid(ctx, s.Table.Schema, tbInfo); err != nil {
+ return errors.Trace(err)
+ }
+ }
+
+ return nil
+}
+
+func checkGeneratedColumn(ctx sessionctx.Context, schemaName model.CIStr, tableName model.CIStr, colDefs []*ast.ColumnDef) error {
+ var colName2Generation = make(map[string]columnGenerationInDDL, len(colDefs))
+ var exists bool
+ var autoIncrementColumn string
+ for i, colDef := range colDefs {
+ for _, option := range colDef.Options {
+ if option.Tp == ast.ColumnOptionGenerated {
+ if err := checkIllegalFn4Generated(colDef.Name.Name.L, typeColumn, option.Expr); err != nil {
+ return errors.Trace(err)
+ }
+ }
+ }
+ if containsColumnOption(colDef, ast.ColumnOptionAutoIncrement) {
+ exists, autoIncrementColumn = true, colDef.Name.Name.L
+ }
+ generated, depCols, err := findDependedColumnNames(schemaName, tableName, colDef)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if !generated {
+ colName2Generation[colDef.Name.Name.L] = columnGenerationInDDL{
+ position: i,
+ generated: false,
+ }
+ } else {
+ colName2Generation[colDef.Name.Name.L] = columnGenerationInDDL{
+ position: i,
+ generated: true,
+ dependences: depCols,
+ }
+ }
+ }
+
+ // Check whether the generated column refers to any auto-increment columns
+ if exists {
+ if !ctx.GetSessionVars().EnableAutoIncrementInGenerated {
+ for colName, generated := range colName2Generation {
+ if _, found := generated.dependences[autoIncrementColumn]; found {
+ return dbterror.ErrGeneratedColumnRefAutoInc.GenWithStackByArgs(colName)
+ }
+ }
+ }
+ }
+
+ for _, colDef := range colDefs {
+ colName := colDef.Name.Name.L
+ if err := verifyColumnGeneration(colName2Generation, colName); err != nil {
+ return errors.Trace(err)
+ }
+ }
+ return nil
+}
+
+// checkTableInfoValidExtra is like checkTableInfoValid, but also assumes the
+// table info comes from untrusted source and performs further checks such as
+// name length and column count.
+// (checkTableInfoValid is also used in repairing objects which don't perform
+// these checks. Perhaps the two functions should be merged together regardless?)
+func checkTableInfoValidExtra(ctx sessionctx.Context, tbInfo *model.TableInfo) error {
+ if err := checkTooLongTable(tbInfo.Name); err != nil {
+ return err
+ }
+
+ if err := checkDuplicateColumn(tbInfo.Columns); err != nil {
+ return err
+ }
+ if err := checkTooLongColumns(tbInfo.Columns); err != nil {
+ return err
+ }
+ if err := checkTooManyColumns(tbInfo.Columns); err != nil {
+ return errors.Trace(err)
+ }
+ if err := checkTooManyIndexes(tbInfo.Indices); err != nil {
+ return errors.Trace(err)
+ }
+ if err := checkColumnsAttributes(tbInfo.Columns); err != nil {
+ return errors.Trace(err)
+ }
+ if err := checkGlobalIndexes(ctx, tbInfo); err != nil {
+ return errors.Trace(err)
+ }
+
+ // FIXME: perform checkConstraintNames
+ if err := checkCharsetAndCollation(tbInfo.Charset, tbInfo.Collate); err != nil {
+ return errors.Trace(err)
+ }
+
+ oldState := tbInfo.State
+ tbInfo.State = model.StatePublic
+ err := checkTableInfoValid(tbInfo)
+ tbInfo.State = oldState
+ return err
+}
+
+// checkTableInfoValid uses to check table info valid. This is used to validate table info.
+func checkTableInfoValid(tblInfo *model.TableInfo) error {
+ _, err := tables.TableFromMeta(autoid.NewAllocators(false), tblInfo)
+ if err != nil {
+ return err
+ }
+ return checkInvisibleIndexOnPK(tblInfo)
+}
+
+func checkDuplicateColumn(cols []*model.ColumnInfo) error {
+ colNames := set.StringSet{}
+ for _, col := range cols {
+ colName := col.Name
+ if colNames.Exist(colName.L) {
+ return infoschema.ErrColumnExists.GenWithStackByArgs(colName.O)
+ }
+ colNames.Insert(colName.L)
+ }
+ return nil
+}
+
+func checkTooLongColumns(cols []*model.ColumnInfo) error {
+ for _, col := range cols {
+ if err := checkTooLongColumn(col.Name); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func checkTooManyColumns(colDefs []*model.ColumnInfo) error {
+ if uint32(len(colDefs)) > atomic.LoadUint32(&config.GetGlobalConfig().TableColumnCountLimit) {
+ return dbterror.ErrTooManyFields
+ }
+ return nil
+}
+
+func checkTooManyIndexes(idxDefs []*model.IndexInfo) error {
+ if len(idxDefs) > config.GetGlobalConfig().IndexLimit {
+ return dbterror.ErrTooManyKeys.GenWithStackByArgs(config.GetGlobalConfig().IndexLimit)
+ }
+ return nil
+}
+
+// checkColumnsAttributes checks attributes for multiple columns.
+func checkColumnsAttributes(colDefs []*model.ColumnInfo) error {
+ for _, colDef := range colDefs {
+ if err := checkColumnAttributes(colDef.Name.O, &colDef.FieldType); err != nil {
+ return errors.Trace(err)
+ }
+ }
+ return nil
+}
+
+// checkColumnAttributes check attributes for single column.
+func checkColumnAttributes(colName string, tp *types.FieldType) error {
+ switch tp.GetType() {
+ case mysql.TypeNewDecimal, mysql.TypeDouble, mysql.TypeFloat:
+ if tp.GetFlen() < tp.GetDecimal() {
+ return types.ErrMBiggerThanD.GenWithStackByArgs(colName)
+ }
+ case mysql.TypeDatetime, mysql.TypeDuration, mysql.TypeTimestamp:
+ if tp.GetDecimal() != types.UnspecifiedFsp && (tp.GetDecimal() < types.MinFsp || tp.GetDecimal() > types.MaxFsp) {
+ return types.ErrTooBigPrecision.GenWithStackByArgs(tp.GetDecimal(), colName, types.MaxFsp)
+ }
+ }
+ return nil
+}
+
+// BuildSessionTemporaryTableInfo builds model.TableInfo from a SQL statement.
+func BuildSessionTemporaryTableInfo(ctx sessionctx.Context, is infoschema.InfoSchema, s *ast.CreateTableStmt, dbCharset, dbCollate string, placementPolicyRef *model.PolicyRefInfo) (*model.TableInfo, error) {
+ ident := ast.Ident{Schema: s.Table.Schema, Name: s.Table.Name}
+ //build tableInfo
+ var tbInfo *model.TableInfo
+ var referTbl table.Table
+ var err error
+ if s.ReferTable != nil {
+ referIdent := ast.Ident{Schema: s.ReferTable.Schema, Name: s.ReferTable.Name}
+ _, ok := is.SchemaByName(referIdent.Schema)
+ if !ok {
+ return nil, infoschema.ErrTableNotExists.GenWithStackByArgs(referIdent.Schema, referIdent.Name)
+ }
+ referTbl, err = is.TableByName(context.Background(), referIdent.Schema, referIdent.Name)
+ if err != nil {
+ return nil, infoschema.ErrTableNotExists.GenWithStackByArgs(referIdent.Schema, referIdent.Name)
+ }
+ tbInfo, err = BuildTableInfoWithLike(ctx, ident, referTbl.Meta(), s)
+ } else {
+ tbInfo, err = buildTableInfoWithCheck(ctx, s, dbCharset, dbCollate, placementPolicyRef)
+ }
+ return tbInfo, err
+}
+
+// BuildTableInfoWithStmt builds model.TableInfo from a SQL statement without validity check
+func BuildTableInfoWithStmt(ctx sessionctx.Context, s *ast.CreateTableStmt, dbCharset, dbCollate string, placementPolicyRef *model.PolicyRefInfo) (*model.TableInfo, error) {
+ colDefs := s.Cols
+ tableCharset, tableCollate, err := GetCharsetAndCollateInTableOption(ctx.GetSessionVars(), 0, s.Options)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ tableCharset, tableCollate, err = ResolveCharsetCollation(ctx.GetSessionVars(),
+ ast.CharsetOpt{Chs: tableCharset, Col: tableCollate},
+ ast.CharsetOpt{Chs: dbCharset, Col: dbCollate},
+ )
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ // The column charset haven't been resolved here.
+ cols, newConstraints, err := buildColumnsAndConstraints(ctx, colDefs, s.Constraints, tableCharset, tableCollate)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ err = checkConstraintNames(s.Table.Name, newConstraints)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ var tbInfo *model.TableInfo
+ tbInfo, err = BuildTableInfo(ctx, s.Table.Name, cols, newConstraints, tableCharset, tableCollate)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ if err = setTemporaryType(ctx, tbInfo, s); err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ if err = setTableAutoRandomBits(ctx, tbInfo, colDefs); err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ if err = handleTableOptions(s.Options, tbInfo); err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ sessionVars := ctx.GetSessionVars()
+ if _, err = validateCommentLength(sessionVars.StmtCtx.ErrCtx(), sessionVars.SQLMode, tbInfo.Name.L, &tbInfo.Comment, dbterror.ErrTooLongTableComment); err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ if tbInfo.TempTableType == model.TempTableNone && tbInfo.PlacementPolicyRef == nil && placementPolicyRef != nil {
+ // Set the defaults from Schema. Note: they are mutual exclusive!
+ tbInfo.PlacementPolicyRef = placementPolicyRef
+ }
+
+ // After handleTableOptions, so the partitions can get defaults from Table level
+ err = buildTablePartitionInfo(ctx, s.Partition, tbInfo)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ return tbInfo, nil
+}
+
+func setTableAutoRandomBits(ctx sessionctx.Context, tbInfo *model.TableInfo, colDefs []*ast.ColumnDef) error {
+ for _, col := range colDefs {
+ if containsColumnOption(col, ast.ColumnOptionAutoRandom) {
+ if col.Tp.GetType() != mysql.TypeLonglong {
+ return dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(
+ fmt.Sprintf(autoid.AutoRandomOnNonBigIntColumn, types.TypeStr(col.Tp.GetType())))
+ }
+ switch {
+ case tbInfo.PKIsHandle:
+ if tbInfo.GetPkName().L != col.Name.Name.L {
+ errMsg := fmt.Sprintf(autoid.AutoRandomMustFirstColumnInPK, col.Name.Name.O)
+ return dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(errMsg)
+ }
+ case tbInfo.IsCommonHandle:
+ pk := tables.FindPrimaryIndex(tbInfo)
+ if pk == nil {
+ return dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomNoClusteredPKErrMsg)
+ }
+ if col.Name.Name.L != pk.Columns[0].Name.L {
+ errMsg := fmt.Sprintf(autoid.AutoRandomMustFirstColumnInPK, col.Name.Name.O)
+ return dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(errMsg)
+ }
+ default:
+ return dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomNoClusteredPKErrMsg)
+ }
+
+ if containsColumnOption(col, ast.ColumnOptionAutoIncrement) {
+ return dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomIncompatibleWithAutoIncErrMsg)
+ }
+ if containsColumnOption(col, ast.ColumnOptionDefaultValue) {
+ return dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomIncompatibleWithDefaultValueErrMsg)
+ }
+
+ shardBits, rangeBits, err := extractAutoRandomBitsFromColDef(col)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ tbInfo.AutoRandomBits = shardBits
+ tbInfo.AutoRandomRangeBits = rangeBits
+
+ shardFmt := autoid.NewShardIDFormat(col.Tp, shardBits, rangeBits)
+ if shardFmt.IncrementalBits < autoid.AutoRandomIncBitsMin {
+ return dbterror.ErrInvalidAutoRandom.FastGenByArgs(autoid.AutoRandomIncrementalBitsTooSmall)
+ }
+ msg := fmt.Sprintf(autoid.AutoRandomAvailableAllocTimesNote, shardFmt.IncrementalBitsCapacity())
+ ctx.GetSessionVars().StmtCtx.AppendNote(errors.NewNoStackError(msg))
+ }
+ }
+ return nil
+}
+
+func containsColumnOption(colDef *ast.ColumnDef, opTp ast.ColumnOptionType) bool {
+ for _, option := range colDef.Options {
+ if option.Tp == opTp {
+ return true
+ }
+ }
+ return false
+}
+
+func extractAutoRandomBitsFromColDef(colDef *ast.ColumnDef) (shardBits, rangeBits uint64, err error) {
+ for _, op := range colDef.Options {
+ if op.Tp == ast.ColumnOptionAutoRandom {
+ shardBits, err = autoid.AutoRandomShardBitsNormalize(op.AutoRandOpt.ShardBits, colDef.Name.Name.O)
+ if err != nil {
+ return 0, 0, err
+ }
+ rangeBits, err = autoid.AutoRandomRangeBitsNormalize(op.AutoRandOpt.RangeBits)
+ if err != nil {
+ return 0, 0, err
+ }
+ return shardBits, rangeBits, nil
+ }
+ }
+ return 0, 0, nil
+}
+
+// handleTableOptions updates tableInfo according to table options.
+func handleTableOptions(options []*ast.TableOption, tbInfo *model.TableInfo) error {
+ var ttlOptionsHandled bool
+
+ for _, op := range options {
+ switch op.Tp {
+ case ast.TableOptionAutoIncrement:
+ tbInfo.AutoIncID = int64(op.UintValue)
+ case ast.TableOptionAutoIdCache:
+ if op.UintValue > uint64(math.MaxInt64) {
+ // TODO: Refine this error.
+ return errors.New("table option auto_id_cache overflows int64")
+ }
+ tbInfo.AutoIdCache = int64(op.UintValue)
+ case ast.TableOptionAutoRandomBase:
+ tbInfo.AutoRandID = int64(op.UintValue)
+ case ast.TableOptionComment:
+ tbInfo.Comment = op.StrValue
+ case ast.TableOptionCompression:
+ tbInfo.Compression = op.StrValue
+ case ast.TableOptionShardRowID:
+ if op.UintValue > 0 && tbInfo.HasClusteredIndex() {
+ return dbterror.ErrUnsupportedShardRowIDBits
+ }
+ tbInfo.ShardRowIDBits = op.UintValue
+ if tbInfo.ShardRowIDBits > shardRowIDBitsMax {
+ tbInfo.ShardRowIDBits = shardRowIDBitsMax
+ }
+ tbInfo.MaxShardRowIDBits = tbInfo.ShardRowIDBits
+ case ast.TableOptionPreSplitRegion:
+ if tbInfo.TempTableType != model.TempTableNone {
+ return errors.Trace(dbterror.ErrOptOnTemporaryTable.GenWithStackByArgs("pre split regions"))
+ }
+ tbInfo.PreSplitRegions = op.UintValue
+ case ast.TableOptionCharset, ast.TableOptionCollate:
+ // We don't handle charset and collate here since they're handled in `GetCharsetAndCollateInTableOption`.
+ case ast.TableOptionPlacementPolicy:
+ tbInfo.PlacementPolicyRef = &model.PolicyRefInfo{
+ Name: model.NewCIStr(op.StrValue),
+ }
+ case ast.TableOptionTTL, ast.TableOptionTTLEnable, ast.TableOptionTTLJobInterval:
+ if ttlOptionsHandled {
+ continue
+ }
+
+ ttlInfo, ttlEnable, ttlJobInterval, err := getTTLInfoInOptions(options)
+ if err != nil {
+ return err
+ }
+ // It's impossible that `ttlInfo` and `ttlEnable` are all nil, because we have met this option.
+ // After exclude the situation `ttlInfo == nil && ttlEnable != nil`, we could say `ttlInfo != nil`
+ if ttlInfo == nil {
+ if ttlEnable != nil {
+ return errors.Trace(dbterror.ErrSetTTLOptionForNonTTLTable.FastGenByArgs("TTL_ENABLE"))
+ }
+ if ttlJobInterval != nil {
+ return errors.Trace(dbterror.ErrSetTTLOptionForNonTTLTable.FastGenByArgs("TTL_JOB_INTERVAL"))
+ }
+ }
+
+ tbInfo.TTLInfo = ttlInfo
+ ttlOptionsHandled = true
+ }
+ }
+ shardingBits := shardingBits(tbInfo)
+ if tbInfo.PreSplitRegions > shardingBits {
+ tbInfo.PreSplitRegions = shardingBits
+ }
+ return nil
+}
+
+func setTemporaryType(_ sessionctx.Context, tbInfo *model.TableInfo, s *ast.CreateTableStmt) error {
+ switch s.TemporaryKeyword {
+ case ast.TemporaryGlobal:
+ tbInfo.TempTableType = model.TempTableGlobal
+ // "create global temporary table ... on commit preserve rows"
+ if !s.OnCommitDelete {
+ return errors.Trace(dbterror.ErrUnsupportedOnCommitPreserve)
+ }
+ case ast.TemporaryLocal:
+ tbInfo.TempTableType = model.TempTableLocal
+ default:
+ tbInfo.TempTableType = model.TempTableNone
+ }
+ return nil
+}
+
+func buildColumnsAndConstraints(
+ ctx sessionctx.Context,
+ colDefs []*ast.ColumnDef,
+ constraints []*ast.Constraint,
+ tblCharset string,
+ tblCollate string,
+) ([]*table.Column, []*ast.Constraint, error) {
+ // outPriKeyConstraint is the primary key constraint out of column definition. such as: create table t1 (id int , age int, primary key(id));
+ var outPriKeyConstraint *ast.Constraint
+ for _, v := range constraints {
+ if v.Tp == ast.ConstraintPrimaryKey {
+ outPriKeyConstraint = v
+ break
+ }
+ }
+ cols := make([]*table.Column, 0, len(colDefs))
+ colMap := make(map[string]*table.Column, len(colDefs))
+
+ for i, colDef := range colDefs {
+ if field_types.TiDBStrictIntegerDisplayWidth {
+ switch colDef.Tp.GetType() {
+ case mysql.TypeTiny:
+ // No warning for BOOL-like tinyint(1)
+ if colDef.Tp.GetFlen() != types.UnspecifiedLength && colDef.Tp.GetFlen() != 1 {
+ ctx.GetSessionVars().StmtCtx.AppendWarning(
+ dbterror.ErrWarnDeprecatedIntegerDisplayWidth.FastGenByArgs(),
+ )
+ }
+ case mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong:
+ if colDef.Tp.GetFlen() != types.UnspecifiedLength {
+ ctx.GetSessionVars().StmtCtx.AppendWarning(
+ dbterror.ErrWarnDeprecatedIntegerDisplayWidth.FastGenByArgs(),
+ )
+ }
+ }
+ }
+ col, cts, err := buildColumnAndConstraint(ctx, i, colDef, outPriKeyConstraint, tblCharset, tblCollate)
+ if err != nil {
+ return nil, nil, errors.Trace(err)
+ }
+ col.State = model.StatePublic
+ if mysql.HasZerofillFlag(col.GetFlag()) {
+ ctx.GetSessionVars().StmtCtx.AppendWarning(
+ dbterror.ErrWarnDeprecatedZerofill.FastGenByArgs(),
+ )
+ }
+ constraints = append(constraints, cts...)
+ cols = append(cols, col)
+ colMap[colDef.Name.Name.L] = col
+ }
+ // Traverse table Constraints and set col.flag.
+ for _, v := range constraints {
+ setColumnFlagWithConstraint(colMap, v)
+ }
+ return cols, constraints, nil
+}
+
+func setEmptyConstraintName(namesMap map[string]bool, constr *ast.Constraint) {
+ if constr.Name == "" && len(constr.Keys) > 0 {
+ var colName string
+ for _, keyPart := range constr.Keys {
+ if keyPart.Expr != nil {
+ colName = "expression_index"
+ }
+ }
+ if colName == "" {
+ colName = constr.Keys[0].Column.Name.O
+ }
+ constrName := colName
+ i := 2
+ if strings.EqualFold(constrName, mysql.PrimaryKeyName) {
+ constrName = fmt.Sprintf("%s_%d", constrName, 2)
+ i = 3
+ }
+ for namesMap[constrName] {
+ // We loop forever until we find constrName that haven't been used.
+ constrName = fmt.Sprintf("%s_%d", colName, i)
+ i++
+ }
+ constr.Name = constrName
+ namesMap[constrName] = true
+ }
+}
+
+func checkConstraintNames(tableName model.CIStr, constraints []*ast.Constraint) error {
+ constrNames := map[string]bool{}
+ fkNames := map[string]bool{}
+
+ // Check not empty constraint name whether is duplicated.
+ for _, constr := range constraints {
+ if constr.Tp == ast.ConstraintForeignKey {
+ err := checkDuplicateConstraint(fkNames, constr.Name, constr.Tp)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ } else {
+ err := checkDuplicateConstraint(constrNames, constr.Name, constr.Tp)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ }
+ }
+
+ // Set empty constraint names.
+ checkConstraints := make([]*ast.Constraint, 0, len(constraints))
+ for _, constr := range constraints {
+ if constr.Tp != ast.ConstraintForeignKey {
+ setEmptyConstraintName(constrNames, constr)
+ }
+ if constr.Tp == ast.ConstraintCheck {
+ checkConstraints = append(checkConstraints, constr)
+ }
+ }
+ // Set check constraint name under its order.
+ if len(checkConstraints) > 0 {
+ setEmptyCheckConstraintName(tableName.L, constrNames, checkConstraints)
+ }
+ return nil
+}
+
+func checkDuplicateConstraint(namesMap map[string]bool, name string, constraintType ast.ConstraintType) error {
+ if name == "" {
+ return nil
+ }
+ nameLower := strings.ToLower(name)
+ if namesMap[nameLower] {
+ switch constraintType {
+ case ast.ConstraintForeignKey:
+ return dbterror.ErrFkDupName.GenWithStackByArgs(name)
+ case ast.ConstraintCheck:
+ return dbterror.ErrCheckConstraintDupName.GenWithStackByArgs(name)
+ default:
+ return dbterror.ErrDupKeyName.GenWithStackByArgs(name)
+ }
+ }
+ namesMap[nameLower] = true
+ return nil
+}
+
+func setEmptyCheckConstraintName(tableLowerName string, namesMap map[string]bool, constrs []*ast.Constraint) {
+ cnt := 1
+ constraintPrefix := tableLowerName + "_chk_"
+ for _, constr := range constrs {
+ if constr.Name == "" {
+ constrName := fmt.Sprintf("%s%d", constraintPrefix, cnt)
+ for {
+ // loop until find constrName that haven't been used.
+ if !namesMap[constrName] {
+ namesMap[constrName] = true
+ break
+ }
+ cnt++
+ constrName = fmt.Sprintf("%s%d", constraintPrefix, cnt)
+ }
+ constr.Name = constrName
+ }
+ }
+}
+
+func setColumnFlagWithConstraint(colMap map[string]*table.Column, v *ast.Constraint) {
+ switch v.Tp {
+ case ast.ConstraintPrimaryKey:
+ for _, key := range v.Keys {
+ if key.Expr != nil {
+ continue
+ }
+ c, ok := colMap[key.Column.Name.L]
+ if !ok {
+ continue
+ }
+ c.AddFlag(mysql.PriKeyFlag)
+ // Primary key can not be NULL.
+ c.AddFlag(mysql.NotNullFlag)
+ setNoDefaultValueFlag(c, c.DefaultValue != nil)
+ }
+ case ast.ConstraintUniq, ast.ConstraintUniqIndex, ast.ConstraintUniqKey:
+ for i, key := range v.Keys {
+ if key.Expr != nil {
+ continue
+ }
+ c, ok := colMap[key.Column.Name.L]
+ if !ok {
+ continue
+ }
+ if i == 0 {
+ // Only the first column can be set
+ // if unique index has multi columns,
+ // the flag should be MultipleKeyFlag.
+ // See https://dev.mysql.com/doc/refman/5.7/en/show-columns.html
+ if len(v.Keys) > 1 {
+ c.AddFlag(mysql.MultipleKeyFlag)
+ } else {
+ c.AddFlag(mysql.UniqueKeyFlag)
+ }
+ }
+ }
+ case ast.ConstraintKey, ast.ConstraintIndex:
+ for i, key := range v.Keys {
+ if key.Expr != nil {
+ continue
+ }
+ c, ok := colMap[key.Column.Name.L]
+ if !ok {
+ continue
+ }
+ if i == 0 {
+ // Only the first column can be set.
+ c.AddFlag(mysql.MultipleKeyFlag)
+ }
+ }
+ }
+}
+
+// BuildTableInfoWithLike builds a new table info according to CREATE TABLE ... LIKE statement.
+func BuildTableInfoWithLike(ctx sessionctx.Context, ident ast.Ident, referTblInfo *model.TableInfo, s *ast.CreateTableStmt) (*model.TableInfo, error) {
+ // Check the referred table is a real table object.
+ if referTblInfo.IsSequence() || referTblInfo.IsView() {
+ return nil, dbterror.ErrWrongObject.GenWithStackByArgs(ident.Schema, referTblInfo.Name, "BASE TABLE")
+ }
+ tblInfo := *referTblInfo
+ if err := setTemporaryType(ctx, &tblInfo, s); err != nil {
+ return nil, errors.Trace(err)
+ }
+ // Check non-public column and adjust column offset.
+ newColumns := referTblInfo.Cols()
+ newIndices := make([]*model.IndexInfo, 0, len(tblInfo.Indices))
+ for _, idx := range tblInfo.Indices {
+ if idx.State == model.StatePublic {
+ newIndices = append(newIndices, idx)
+ }
+ }
+ tblInfo.Columns = newColumns
+ tblInfo.Indices = newIndices
+ tblInfo.Name = ident.Name
+ tblInfo.AutoIncID = 0
+ tblInfo.ForeignKeys = nil
+ // Ignore TiFlash replicas for temporary tables.
+ if s.TemporaryKeyword != ast.TemporaryNone {
+ tblInfo.TiFlashReplica = nil
+ } else if tblInfo.TiFlashReplica != nil {
+ replica := *tblInfo.TiFlashReplica
+ // Keep the tiflash replica setting, remove the replica available status.
+ replica.AvailablePartitionIDs = nil
+ replica.Available = false
+ tblInfo.TiFlashReplica = &replica
+ }
+ if referTblInfo.Partition != nil {
+ pi := *referTblInfo.Partition
+ pi.Definitions = make([]model.PartitionDefinition, len(referTblInfo.Partition.Definitions))
+ copy(pi.Definitions, referTblInfo.Partition.Definitions)
+ tblInfo.Partition = &pi
+ }
+
+ if referTblInfo.TTLInfo != nil {
+ tblInfo.TTLInfo = referTblInfo.TTLInfo.Clone()
+ }
+ renameCheckConstraint(&tblInfo)
+ return &tblInfo, nil
+}
+
+func renameCheckConstraint(tblInfo *model.TableInfo) {
+ for _, cons := range tblInfo.Constraints {
+ cons.Name = model.NewCIStr("")
+ cons.Table = tblInfo.Name
+ }
+ setNameForConstraintInfo(tblInfo.Name.L, map[string]bool{}, tblInfo.Constraints)
+}
+
+// BuildTableInfo creates a TableInfo.
+func BuildTableInfo(
+ ctx sessionctx.Context,
+ tableName model.CIStr,
+ cols []*table.Column,
+ constraints []*ast.Constraint,
+ charset string,
+ collate string,
+) (tbInfo *model.TableInfo, err error) {
+ tbInfo = &model.TableInfo{
+ Name: tableName,
+ Version: model.CurrLatestTableInfoVersion,
+ Charset: charset,
+ Collate: collate,
+ }
+ tblColumns := make([]*table.Column, 0, len(cols))
+ existedColsMap := make(map[string]struct{}, len(cols))
+ for _, v := range cols {
+ v.ID = AllocateColumnID(tbInfo)
+ tbInfo.Columns = append(tbInfo.Columns, v.ToInfo())
+ tblColumns = append(tblColumns, table.ToColumn(v.ToInfo()))
+ existedColsMap[v.Name.L] = struct{}{}
+ }
+ foreignKeyID := tbInfo.MaxForeignKeyID
+ for _, constr := range constraints {
+ // Build hidden columns if necessary.
+ hiddenCols, err := buildHiddenColumnInfoWithCheck(ctx, constr.Keys, model.NewCIStr(constr.Name), tbInfo, tblColumns)
+ if err != nil {
+ return nil, err
+ }
+ for _, hiddenCol := range hiddenCols {
+ hiddenCol.State = model.StatePublic
+ hiddenCol.ID = AllocateColumnID(tbInfo)
+ hiddenCol.Offset = len(tbInfo.Columns)
+ tbInfo.Columns = append(tbInfo.Columns, hiddenCol)
+ tblColumns = append(tblColumns, table.ToColumn(hiddenCol))
+ }
+ // Check clustered on non-primary key.
+ if constr.Option != nil && constr.Option.PrimaryKeyTp != model.PrimaryKeyTypeDefault &&
+ constr.Tp != ast.ConstraintPrimaryKey {
+ return nil, dbterror.ErrUnsupportedClusteredSecondaryKey
+ }
+ if constr.Tp == ast.ConstraintForeignKey {
+ var fkName model.CIStr
+ foreignKeyID++
+ if constr.Name != "" {
+ fkName = model.NewCIStr(constr.Name)
+ } else {
+ fkName = model.NewCIStr(fmt.Sprintf("fk_%d", foreignKeyID))
+ }
+ if model.FindFKInfoByName(tbInfo.ForeignKeys, fkName.L) != nil {
+ return nil, infoschema.ErrCannotAddForeign
+ }
+ fk, err := buildFKInfo(fkName, constr.Keys, constr.Refer, cols)
+ if err != nil {
+ return nil, err
+ }
+ fk.State = model.StatePublic
+
+ tbInfo.ForeignKeys = append(tbInfo.ForeignKeys, fk)
+ continue
+ }
+ if constr.Tp == ast.ConstraintPrimaryKey {
+ lastCol, err := CheckPKOnGeneratedColumn(tbInfo, constr.Keys)
+ if err != nil {
+ return nil, err
+ }
+ isSingleIntPK := isSingleIntPK(constr, lastCol)
+ if ShouldBuildClusteredIndex(ctx, constr.Option, isSingleIntPK) {
+ if isSingleIntPK {
+ tbInfo.PKIsHandle = true
+ } else {
+ tbInfo.IsCommonHandle = true
+ tbInfo.CommonHandleVersion = 1
+ }
+ }
+ if tbInfo.HasClusteredIndex() {
+ // Primary key cannot be invisible.
+ if constr.Option != nil && constr.Option.Visibility == ast.IndexVisibilityInvisible {
+ return nil, dbterror.ErrPKIndexCantBeInvisible
+ }
+ }
+ if tbInfo.PKIsHandle {
+ continue
+ }
+ }
+
+ if constr.Tp == ast.ConstraintFulltext {
+ ctx.GetSessionVars().StmtCtx.AppendWarning(dbterror.ErrTableCantHandleFt.FastGenByArgs())
+ continue
+ }
+
+ var (
+ indexName = constr.Name
+ primary, unique bool
+ )
+
+ // Check if the index is primary or unique.
+ switch constr.Tp {
+ case ast.ConstraintPrimaryKey:
+ primary = true
+ unique = true
+ indexName = mysql.PrimaryKeyName
+ case ast.ConstraintUniq, ast.ConstraintUniqKey, ast.ConstraintUniqIndex:
+ unique = true
+ }
+
+ // check constraint
+ if constr.Tp == ast.ConstraintCheck {
+ if !variable.EnableCheckConstraint.Load() {
+ ctx.GetSessionVars().StmtCtx.AppendWarning(errCheckConstraintIsOff)
+ continue
+ }
+ // Since column check constraint dependency has been done in columnDefToCol.
+ // Here do the table check constraint dependency check, table constraint
+ // can only refer the columns in defined columns of the table.
+ // Refer: https://dev.mysql.com/doc/refman/8.0/en/create-table-check-constraints.html
+ if ok, err := table.IsSupportedExpr(constr); !ok {
+ return nil, err
+ }
+ var dependedCols []model.CIStr
+ dependedColsMap := findDependentColsInExpr(constr.Expr)
+ if !constr.InColumn {
+ dependedCols = make([]model.CIStr, 0, len(dependedColsMap))
+ for k := range dependedColsMap {
+ if _, ok := existedColsMap[k]; !ok {
+ // The table constraint depended on a non-existed column.
+ return nil, dbterror.ErrTableCheckConstraintReferUnknown.GenWithStackByArgs(constr.Name, k)
+ }
+ dependedCols = append(dependedCols, model.NewCIStr(k))
+ }
+ } else {
+ // Check the column-type constraint dependency.
+ if len(dependedColsMap) > 1 {
+ return nil, dbterror.ErrColumnCheckConstraintReferOther.GenWithStackByArgs(constr.Name)
+ } else if len(dependedColsMap) == 0 {
+ // If dependedCols is empty, the expression must be true/false.
+ valExpr, ok := constr.Expr.(*driver.ValueExpr)
+ if !ok || !mysql.HasIsBooleanFlag(valExpr.GetType().GetFlag()) {
+ return nil, errors.Trace(errors.New("unsupported expression in check constraint"))
+ }
+ } else {
+ if _, ok := dependedColsMap[constr.InColumnName]; !ok {
+ return nil, dbterror.ErrColumnCheckConstraintReferOther.GenWithStackByArgs(constr.Name)
+ }
+ dependedCols = []model.CIStr{model.NewCIStr(constr.InColumnName)}
+ }
+ }
+ // check auto-increment column
+ if table.ContainsAutoIncrementCol(dependedCols, tbInfo) {
+ return nil, dbterror.ErrCheckConstraintRefersAutoIncrementColumn.GenWithStackByArgs(constr.Name)
+ }
+ // check foreign key
+ if err := table.HasForeignKeyRefAction(tbInfo.ForeignKeys, constraints, constr, dependedCols); err != nil {
+ return nil, err
+ }
+ // build constraint meta info.
+ constraintInfo, err := buildConstraintInfo(tbInfo, dependedCols, constr, model.StatePublic)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ // check if the expression is bool type
+ if err := table.IfCheckConstraintExprBoolType(ctx.GetExprCtx().GetEvalCtx(), constraintInfo, tbInfo); err != nil {
+ return nil, err
+ }
+ constraintInfo.ID = allocateConstraintID(tbInfo)
+ tbInfo.Constraints = append(tbInfo.Constraints, constraintInfo)
+ continue
+ }
+
+ // build index info.
+ idxInfo, err := BuildIndexInfo(
+ ctx,
+ tbInfo.Columns,
+ model.NewCIStr(indexName),
+ primary,
+ unique,
+ constr.Keys,
+ constr.Option,
+ model.StatePublic,
+ )
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ if len(hiddenCols) > 0 {
+ AddIndexColumnFlag(tbInfo, idxInfo)
+ }
+ sessionVars := ctx.GetSessionVars()
+ _, err = validateCommentLength(sessionVars.StmtCtx.ErrCtx(), sessionVars.SQLMode, idxInfo.Name.String(), &idxInfo.Comment, dbterror.ErrTooLongIndexComment)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ idxInfo.ID = AllocateIndexID(tbInfo)
+ tbInfo.Indices = append(tbInfo.Indices, idxInfo)
+ }
+
+ err = addIndexForForeignKey(ctx, tbInfo)
+ return tbInfo, err
+}
+
+func precheckBuildHiddenColumnInfo(
+ indexPartSpecifications []*ast.IndexPartSpecification,
+ indexName model.CIStr,
+) error {
+ for i, idxPart := range indexPartSpecifications {
+ if idxPart.Expr == nil {
+ continue
+ }
+ name := fmt.Sprintf("%s_%s_%d", expressionIndexPrefix, indexName, i)
+ if utf8.RuneCountInString(name) > mysql.MaxColumnNameLength {
+ // TODO: Refine the error message.
+ return dbterror.ErrTooLongIdent.GenWithStackByArgs("hidden column")
+ }
+ // TODO: Refine the error message.
+ if err := checkIllegalFn4Generated(indexName.L, typeIndex, idxPart.Expr); err != nil {
+ return errors.Trace(err)
+ }
+ }
+ return nil
+}
+
+func buildHiddenColumnInfoWithCheck(ctx sessionctx.Context, indexPartSpecifications []*ast.IndexPartSpecification, indexName model.CIStr, tblInfo *model.TableInfo, existCols []*table.Column) ([]*model.ColumnInfo, error) {
+ if err := precheckBuildHiddenColumnInfo(indexPartSpecifications, indexName); err != nil {
+ return nil, err
+ }
+ return BuildHiddenColumnInfo(ctx, indexPartSpecifications, indexName, tblInfo, existCols)
+}
+
+// BuildHiddenColumnInfo builds hidden column info.
+func BuildHiddenColumnInfo(ctx sessionctx.Context, indexPartSpecifications []*ast.IndexPartSpecification, indexName model.CIStr, tblInfo *model.TableInfo, existCols []*table.Column) ([]*model.ColumnInfo, error) {
+ hiddenCols := make([]*model.ColumnInfo, 0, len(indexPartSpecifications))
+ for i, idxPart := range indexPartSpecifications {
+ if idxPart.Expr == nil {
+ continue
+ }
+ idxPart.Column = &ast.ColumnName{Name: model.NewCIStr(fmt.Sprintf("%s_%s_%d", expressionIndexPrefix, indexName, i))}
+ // Check whether the hidden columns have existed.
+ col := table.FindCol(existCols, idxPart.Column.Name.L)
+ if col != nil {
+ // TODO: Use expression index related error.
+ return nil, infoschema.ErrColumnExists.GenWithStackByArgs(col.Name.String())
+ }
+ idxPart.Length = types.UnspecifiedLength
+ // The index part is an expression, prepare a hidden column for it.
+
+ var sb strings.Builder
+ restoreFlags := format.RestoreStringSingleQuotes | format.RestoreKeyWordLowercase | format.RestoreNameBackQuotes |
+ format.RestoreSpacesAroundBinaryOperation | format.RestoreWithoutSchemaName | format.RestoreWithoutTableName
+ restoreCtx := format.NewRestoreCtx(restoreFlags, &sb)
+ sb.Reset()
+ err := idxPart.Expr.Restore(restoreCtx)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ expr, err := expression.BuildSimpleExpr(ctx.GetExprCtx(), idxPart.Expr,
+ expression.WithTableInfo(ctx.GetSessionVars().CurrentDB, tblInfo),
+ expression.WithAllowCastArray(true),
+ )
+ if err != nil {
+ // TODO: refine the error message.
+ return nil, err
+ }
+ if _, ok := expr.(*expression.Column); ok {
+ return nil, dbterror.ErrFunctionalIndexOnField
+ }
+
+ colInfo := &model.ColumnInfo{
+ Name: idxPart.Column.Name,
+ GeneratedExprString: sb.String(),
+ GeneratedStored: false,
+ Version: model.CurrLatestColumnInfoVersion,
+ Dependences: make(map[string]struct{}),
+ Hidden: true,
+ FieldType: *expr.GetType(ctx.GetExprCtx().GetEvalCtx()),
+ }
+ // Reset some flag, it may be caused by wrong type infer. But it's not easy to fix them all, so reset them here for safety.
+ colInfo.DelFlag(mysql.PriKeyFlag | mysql.UniqueKeyFlag | mysql.AutoIncrementFlag)
+
+ if colInfo.GetType() == mysql.TypeDatetime || colInfo.GetType() == mysql.TypeDate || colInfo.GetType() == mysql.TypeTimestamp || colInfo.GetType() == mysql.TypeDuration {
+ if colInfo.FieldType.GetDecimal() == types.UnspecifiedLength {
+ colInfo.FieldType.SetDecimal(types.MaxFsp)
+ }
+ }
+ // For an array, the collation is set to "binary". The collation has no effect on the array itself (as it's usually
+ // regarded as a JSON), but will influence how TiKV handles the index value.
+ if colInfo.FieldType.IsArray() {
+ colInfo.SetCharset("binary")
+ colInfo.SetCollate("binary")
+ }
+ checkDependencies := make(map[string]struct{})
+ for _, colName := range FindColumnNamesInExpr(idxPart.Expr) {
+ colInfo.Dependences[colName.Name.L] = struct{}{}
+ checkDependencies[colName.Name.L] = struct{}{}
+ }
+ if err = checkDependedColExist(checkDependencies, existCols); err != nil {
+ return nil, errors.Trace(err)
+ }
+ if !ctx.GetSessionVars().EnableAutoIncrementInGenerated {
+ if err = checkExpressionIndexAutoIncrement(indexName.O, colInfo.Dependences, tblInfo); err != nil {
+ return nil, errors.Trace(err)
+ }
+ }
+ idxPart.Expr = nil
+ hiddenCols = append(hiddenCols, colInfo)
+ }
+ return hiddenCols, nil
+}
+
+// addIndexForForeignKey uses to auto create an index for the foreign key if the table doesn't have any index cover the
+// foreign key columns.
+func addIndexForForeignKey(ctx sessionctx.Context, tbInfo *model.TableInfo) error {
+ if len(tbInfo.ForeignKeys) == 0 {
+ return nil
+ }
+ var handleCol *model.ColumnInfo
+ if tbInfo.PKIsHandle {
+ handleCol = tbInfo.GetPkColInfo()
+ }
+ for _, fk := range tbInfo.ForeignKeys {
+ if fk.Version < model.FKVersion1 {
+ continue
+ }
+ if handleCol != nil && len(fk.Cols) == 1 && handleCol.Name.L == fk.Cols[0].L {
+ continue
+ }
+ if model.FindIndexByColumns(tbInfo, tbInfo.Indices, fk.Cols...) != nil {
+ continue
+ }
+ idxName := fk.Name
+ if tbInfo.FindIndexByName(idxName.L) != nil {
+ return dbterror.ErrDupKeyName.GenWithStack("duplicate key name %s", fk.Name.O)
+ }
+ keys := make([]*ast.IndexPartSpecification, 0, len(fk.Cols))
+ for _, col := range fk.Cols {
+ keys = append(keys, &ast.IndexPartSpecification{
+ Column: &ast.ColumnName{Name: col},
+ Length: types.UnspecifiedLength,
+ })
+ }
+ idxInfo, err := BuildIndexInfo(ctx, tbInfo.Columns, idxName, false, false, keys, nil, model.StatePublic)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ idxInfo.ID = AllocateIndexID(tbInfo)
+ tbInfo.Indices = append(tbInfo.Indices, idxInfo)
+ }
+ return nil
+}
+
+func isSingleIntPK(constr *ast.Constraint, lastCol *model.ColumnInfo) bool {
+ if len(constr.Keys) != 1 {
+ return false
+ }
+ switch lastCol.GetType() {
+ case mysql.TypeLong, mysql.TypeLonglong,
+ mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24:
+ return true
+ }
+ return false
+}
+
+// ShouldBuildClusteredIndex is used to determine whether the CREATE TABLE statement should build a clustered index table.
+func ShouldBuildClusteredIndex(ctx sessionctx.Context, opt *ast.IndexOption, isSingleIntPK bool) bool {
+ if opt == nil || opt.PrimaryKeyTp == model.PrimaryKeyTypeDefault {
+ switch ctx.GetSessionVars().EnableClusteredIndex {
+ case variable.ClusteredIndexDefModeOn:
+ return true
+ case variable.ClusteredIndexDefModeIntOnly:
+ return !config.GetGlobalConfig().AlterPrimaryKey && isSingleIntPK
+ default:
+ return false
+ }
+ }
+ return opt.PrimaryKeyTp == model.PrimaryKeyTypeClustered
+}
+
+// BuildViewInfo builds a ViewInfo structure from an ast.CreateViewStmt.
+func BuildViewInfo(s *ast.CreateViewStmt) (*model.ViewInfo, error) {
+ // Always Use `format.RestoreNameBackQuotes` to restore `SELECT` statement despite the `ANSI_QUOTES` SQL Mode is enabled or not.
+ restoreFlag := format.RestoreStringSingleQuotes | format.RestoreKeyWordUppercase | format.RestoreNameBackQuotes
+ var sb strings.Builder
+ if err := s.Select.Restore(format.NewRestoreCtx(restoreFlag, &sb)); err != nil {
+ return nil, err
+ }
+
+ return &model.ViewInfo{Definer: s.Definer, Algorithm: s.Algorithm,
+ Security: s.Security, SelectStmt: sb.String(), CheckOption: s.CheckOption, Cols: nil}, nil
+}
diff --git a/pkg/ddl/db_change_failpoints_test.go b/pkg/ddl/db_change_failpoints_test.go
index dfacf2cd05ff4..23059e004f5eb 100644
--- a/pkg/ddl/db_change_failpoints_test.go
+++ b/pkg/ddl/db_change_failpoints_test.go
@@ -86,7 +86,7 @@ func TestParallelUpdateTableReplica(t *testing.T) {
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/infoschema/mockTiFlashStoreCount"))
}()
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("create database test_db_state default charset utf8 default collate utf8_bin")
@@ -95,8 +95,7 @@ func TestParallelUpdateTableReplica(t *testing.T) {
tk.MustExec("create table t1 (a int);")
tk.MustExec("alter table t1 set tiflash replica 3 location labels 'a','b';")
- tk1, tk2, ch, originalCallback := prepareTestControlParallelExecSQL(t, store, dom)
- defer dom.DDL().SetHook(originalCallback)
+ tk1, tk2, ch := prepareTestControlParallelExecSQL(t, store)
t1 := external.GetTableByName(t, tk, "test_db_state", "t1")
diff --git a/pkg/ddl/db_change_test.go b/pkg/ddl/db_change_test.go
index ddb6709bef06b..25a8249a5b9cc 100644
--- a/pkg/ddl/db_change_test.go
+++ b/pkg/ddl/db_change_test.go
@@ -23,8 +23,8 @@ import (
"time"
"github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/ddl"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/executor"
"github.com/pingcap/tidb/pkg/kv"
@@ -46,7 +46,7 @@ import (
// TestShowCreateTable tests the result of "show create table" when we are running "add index" or "add column".
func TestShowCreateTable(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -74,9 +74,8 @@ func TestShowCreateTable(t *testing.T) {
"CREATE TABLE `t2` (\n `a` int(11) DEFAULT NULL,\n `b` varchar(10) COLLATE utf8mb4_general_ci DEFAULT NULL,\n `c` varchar(1) COLLATE utf8mb4_general_ci DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci"},
}
prevState := model.StateNone
- callback := &callback.TestDDLCallback{}
currTestCaseOffset := 0
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.SchemaState == prevState || checkErr != nil {
return
}
@@ -110,12 +109,7 @@ func TestShowCreateTable(t *testing.T) {
}
terror.Log(result.Close())
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- d.SetHook(callback)
+ })
for _, tc := range testCases {
tk.MustExec(tc.sql)
require.NoError(t, checkErr)
@@ -124,7 +118,7 @@ func TestShowCreateTable(t *testing.T) {
// TestDropNotNullColumn is used to test issue #8654.
func TestDropNotNullColumn(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -143,11 +137,8 @@ func TestDropNotNullColumn(t *testing.T) {
tk1.MustExec("use test")
var checkErr error
- d := dom.DDL()
- originalCallback := d.GetHook()
- callback := &callback.TestDDLCallback{}
sqlNum := 0
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if checkErr != nil {
return
}
@@ -165,9 +156,7 @@ func TestDropNotNullColumn(t *testing.T) {
_, checkErr = tk1.Exec("insert into t4 set id = 5")
}
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(callback)
+ })
tk.MustExec("alter table t drop column a")
require.NoError(t, checkErr)
sqlNum++
@@ -182,12 +171,12 @@ func TestDropNotNullColumn(t *testing.T) {
sqlNum++
tk.MustExec("alter table t4 drop column e")
require.NoError(t, checkErr)
- d.SetHook(originalCallback)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
tk.MustExec("drop table t, t1, t2, t3")
}
func TestTwoStates(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, 200*time.Millisecond)
+ store := testkit.CreateMockStoreWithSchemaLease(t, 200*time.Millisecond)
tk := testkit.NewTestKit(t, store)
tk.MustExec("create database test_db_state default charset utf8 default collate utf8_bin")
tk.MustExec("use test_db_state")
@@ -227,13 +216,12 @@ func TestTwoStates(t *testing.T) {
key(c1, c2))`)
tk.MustExec("insert into t values(1, 'a', 'N', '2017-07-01')")
- callback := &callback.TestDDLCallback{}
prevState := model.StateNone
require.NoError(t, testInfo.parseSQLs(parser.New()))
times := 0
var checkErr error
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.SchemaState == prevState || checkErr != nil || times >= 3 {
return
}
@@ -280,12 +268,7 @@ func TestTwoStates(t *testing.T) {
checkErr = err
}
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- d.SetHook(callback)
+ })
tk.MustExec(alterTableSQL)
require.NoError(t, testInfo.compileSQL(4))
require.NoError(t, testInfo.execSQL(4))
@@ -788,7 +771,6 @@ func runTestInSchemaState(
// Make sure these SQLs use the plan of index scan.
tk.MustExec("drop stats t")
- callback := &callback.TestDDLCallback{Do: dom}
prevState := model.StateNone
var checkErr error
se, err := session.CreateSession(store)
@@ -812,16 +794,13 @@ func runTestInSchemaState(
}
}
if isOnJobUpdated {
- callback.OnJobUpdatedExported.Store(&cbFunc)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", cbFunc)
} else {
- callback.OnJobRunBeforeExported = cbFunc
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", cbFunc)
}
- d := dom.DDL()
- originalCallback := d.GetHook()
- d.SetHook(callback)
tk.MustExec(alterTableSQL)
require.NoError(t, checkErr)
- d.SetHook(originalCallback)
+ _ = failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/onJobRunBefore")
if expectQuery != nil {
tk := testkit.NewTestKit(t, store)
@@ -845,17 +824,16 @@ func jobStateOrLastSubJobState(job *model.Job) model.SchemaState {
}
func TestShowIndex(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, 200*time.Millisecond)
+ store := testkit.CreateMockStoreWithSchemaLease(t, 200*time.Millisecond)
tk := testkit.NewTestKit(t, store)
tk.MustExec("create database test_db_state default charset utf8 default collate utf8_bin")
tk.MustExec("use test_db_state")
tk.MustExec(`create table t(c1 int primary key nonclustered, c2 int)`)
- callback := &callback.TestDDLCallback{}
prevState := model.StateNone
showIndexSQL := `show index from t`
var checkErr error
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.SchemaState == prevState || checkErr != nil {
return
}
@@ -873,11 +851,7 @@ func TestShowIndex(t *testing.T) {
checkErr = fmt.Errorf("need %v, but got %v", need, got)
}
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d := dom.DDL()
- originalCallback := d.GetHook()
- d.SetHook(callback)
+ })
alterTableSQL := `alter table t add index c2(c2)`
tk.MustExec(alterTableSQL)
require.NoError(t, checkErr)
@@ -886,7 +860,7 @@ func TestShowIndex(t *testing.T) {
"t 0 PRIMARY 1 c1 A 0 BTREE YES NO",
"t 1 c2 1 c2 A 0 YES BTREE YES NO",
))
- d.SetHook(originalCallback)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
tk.MustExec(`create table tr(
id int, name varchar(50),
@@ -1316,10 +1290,9 @@ func TestParallelAlterAndDropSchema(t *testing.T) {
testControlParallelExecSQL(t, tk, store, dom, "", sql1, sql2, f)
}
-func prepareTestControlParallelExecSQL(t *testing.T, store kv.Storage, dom *domain.Domain) (*testkit.TestKit, *testkit.TestKit, chan struct{}, ddl.Callback) {
- callback := &callback.TestDDLCallback{}
+func prepareTestControlParallelExecSQL(t *testing.T, store kv.Storage) (*testkit.TestKit, *testkit.TestKit, chan struct{}) {
times := 0
- callback.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if times != 0 {
return
}
@@ -1337,10 +1310,7 @@ func prepareTestControlParallelExecSQL(t *testing.T, store kv.Storage, dom *doma
time.Sleep(5 * time.Millisecond)
}
times++
- }
- d := dom.DDL()
- originalCallback := d.GetHook()
- d.SetHook(callback)
+ })
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test_db_state")
@@ -1365,7 +1335,7 @@ func prepareTestControlParallelExecSQL(t *testing.T, store kv.Storage, dom *doma
time.Sleep(5 * time.Millisecond)
}
}()
- return tk1, tk2, ch, originalCallback
+ return tk1, tk2, ch
}
func testControlParallelExecSQL(t *testing.T, tk *testkit.TestKit, store kv.Storage, dom *domain.Domain, preSQL, sql1, sql2 string, f func(e1, e2 error)) {
@@ -1386,8 +1356,7 @@ func testControlParallelExecSQL(t *testing.T, tk *testkit.TestKit, store kv.Stor
partition p1 values less than (20)
);`)
- tk1, tk2, ch, originalCallback := prepareTestControlParallelExecSQL(t, store, dom)
- defer dom.DDL().SetHook(originalCallback)
+ tk1, tk2, ch := prepareTestControlParallelExecSQL(t, store)
var err1 error
var err2 error
@@ -1412,7 +1381,7 @@ func testControlParallelExecSQL(t *testing.T, tk *testkit.TestKit, store kv.Stor
f(err1, err2)
}
-func dbChangeTestParallelExecSQL(t *testing.T, store kv.Storage, dom *domain.Domain, sql string) {
+func dbChangeTestParallelExecSQL(t *testing.T, store kv.Storage, sql string) {
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test_db_state")
tk2 := testkit.NewTestKit(t, store)
@@ -1421,19 +1390,14 @@ func dbChangeTestParallelExecSQL(t *testing.T, store kv.Storage, dom *domain.Dom
var err2, err3 error
var wg util.WaitGroupWrapper
- callback := &callback.TestDDLCallback{}
once := sync.Once{}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
// sleep a while, let other job enqueue.
once.Do(func() {
time.Sleep(time.Millisecond * 10)
})
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- d.SetHook(callback)
+ })
+ defer testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
wg.Run(func() {
err2 = tk1.ExecToErr(sql)
})
@@ -1447,58 +1411,58 @@ func dbChangeTestParallelExecSQL(t *testing.T, store kv.Storage, dom *domain.Dom
// TestCreateTableIfNotExists parallel exec create table if not exists xxx. No error returns is expected.
func TestCreateTableIfNotExists(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("create database test_db_state default charset utf8 default collate utf8_bin")
- dbChangeTestParallelExecSQL(t, store, dom, "create table if not exists test_not_exists(a int)")
+ dbChangeTestParallelExecSQL(t, store, "create table if not exists test_not_exists(a int)")
}
// TestCreateDBIfNotExists parallel exec create database if not exists xxx. No error returns is expected.
func TestCreateDBIfNotExists(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("create database test_db_state default charset utf8 default collate utf8_bin")
- dbChangeTestParallelExecSQL(t, store, dom, "create database if not exists test_not_exists")
+ dbChangeTestParallelExecSQL(t, store, "create database if not exists test_not_exists")
}
// TestDDLIfNotExists parallel exec some DDLs with `if not exists` clause. No error returns is expected.
func TestDDLIfNotExists(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, 200*time.Millisecond)
+ store := testkit.CreateMockStoreWithSchemaLease(t, 200*time.Millisecond)
tk := testkit.NewTestKit(t, store)
tk.MustExec("create database test_db_state default charset utf8 default collate utf8_bin")
tk.MustExec("use test_db_state")
tk.MustExec("create table if not exists test_not_exists(a int)")
// ADD COLUMN
- dbChangeTestParallelExecSQL(t, store, dom, "alter table test_not_exists add column if not exists b int")
+ dbChangeTestParallelExecSQL(t, store, "alter table test_not_exists add column if not exists b int")
// ADD COLUMNS
- dbChangeTestParallelExecSQL(t, store, dom, "alter table test_not_exists add column if not exists (c11 int, d11 int)")
+ dbChangeTestParallelExecSQL(t, store, "alter table test_not_exists add column if not exists (c11 int, d11 int)")
// ADD INDEX
- dbChangeTestParallelExecSQL(t, store, dom, "alter table test_not_exists add index if not exists idx_b (b)")
+ dbChangeTestParallelExecSQL(t, store, "alter table test_not_exists add index if not exists idx_b (b)")
// CREATE INDEX
- dbChangeTestParallelExecSQL(t, store, dom, "create index if not exists idx_b on test_not_exists (b)")
+ dbChangeTestParallelExecSQL(t, store, "create index if not exists idx_b on test_not_exists (b)")
}
// TestDDLIfExists parallel exec some DDLs with `if exists` clause. No error returns is expected.
func TestDDLIfExists(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, 200*time.Millisecond)
+ store := testkit.CreateMockStoreWithSchemaLease(t, 200*time.Millisecond)
tk := testkit.NewTestKit(t, store)
tk.MustExec("create database test_db_state default charset utf8 default collate utf8_bin")
tk.MustExec("use test_db_state")
tk.MustExec("create table if not exists test_exists (a int key, b int)")
// DROP COLUMNS
- dbChangeTestParallelExecSQL(t, store, dom, "alter table test_exists drop column if exists c, drop column if exists d")
+ dbChangeTestParallelExecSQL(t, store, "alter table test_exists drop column if exists c, drop column if exists d")
// DROP COLUMN
- dbChangeTestParallelExecSQL(t, store, dom, "alter table test_exists drop column if exists b") // only `a` exists now
+ dbChangeTestParallelExecSQL(t, store, "alter table test_exists drop column if exists b") // only `a` exists now
// CHANGE COLUMN
- dbChangeTestParallelExecSQL(t, store, dom, "alter table test_exists change column if exists a c int") // only, `c` exists now
+ dbChangeTestParallelExecSQL(t, store, "alter table test_exists change column if exists a c int") // only, `c` exists now
// MODIFY COLUMN
- dbChangeTestParallelExecSQL(t, store, dom, "alter table test_exists modify column if exists a bigint")
+ dbChangeTestParallelExecSQL(t, store, "alter table test_exists modify column if exists a bigint")
// DROP INDEX
tk.MustExec("alter table test_exists add index idx_c (c)")
- dbChangeTestParallelExecSQL(t, store, dom, "alter table test_exists drop index if exists idx_c")
+ dbChangeTestParallelExecSQL(t, store, "alter table test_exists drop index if exists idx_c")
// DROP PARTITION (ADD PARTITION tested in TestParallelAlterAddPartition)
tk.MustExec("create table test_exists_2 (a int key) partition by range(a) (partition p0 values less than (10), partition p1 values less than (20), partition p2 values less than (30))")
- dbChangeTestParallelExecSQL(t, store, dom, "alter table test_exists_2 drop partition if exists p1")
+ dbChangeTestParallelExecSQL(t, store, "alter table test_exists_2 drop partition if exists p1")
}
// TestParallelDDLBeforeRunDDLJob tests a session to execute DDL with an outdated information schema.
@@ -1622,7 +1586,7 @@ func TestWriteReorgForColumnTypeChange(t *testing.T) {
}
func TestCreateExpressionIndex(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -1640,11 +1604,7 @@ func TestCreateExpressionIndex(t *testing.T) {
// If waitReorg timeout, the worker may enter writeReorg more than 2 times.
reorgTime := 0
var checkErr error
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- callback := &callback.TestDDLCallback{}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if checkErr != nil {
return
}
@@ -1678,10 +1638,8 @@ func TestCreateExpressionIndex(t *testing.T) {
}
// (1, 7), (2, 7), (5, 7), (8, 8), (0, 9), (10, 10), (10, 10), (0, 11), (0, 11)
}
- }
+ })
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(callback)
tk.MustExec("alter table t add index idx((b+1))")
require.NoError(t, checkErr)
tk.MustExec("admin check table t")
@@ -1697,7 +1655,7 @@ func TestCreateExpressionIndex(t *testing.T) {
}
func TestCreateUniqueExpressionIndex(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -1712,11 +1670,7 @@ func TestCreateUniqueExpressionIndex(t *testing.T) {
// If waitReorg timeout, the worker may enter writeReorg more than 2 times.
reorgTime := 0
var checkErr error
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- callback := &callback.TestDDLCallback{}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if checkErr != nil {
return
}
@@ -1796,9 +1750,7 @@ func TestCreateUniqueExpressionIndex(t *testing.T) {
}
// (1, 7), (2, 7), (5, 7), (8, 8), (13, 9), (11, 10), (0, 11)
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(callback)
+ })
tk.MustExec("alter table t add unique index idx((a*b+1))")
require.NoError(t, checkErr)
tk.MustExec("admin check table t")
@@ -1806,7 +1758,7 @@ func TestCreateUniqueExpressionIndex(t *testing.T) {
}
func TestDropExpressionIndex(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -1820,11 +1772,7 @@ func TestDropExpressionIndex(t *testing.T) {
stateWriteReorganizationSQLs := []string{"insert into t values (10, 10)", "begin pessimistic;", "insert into t select * from t", "rollback", "insert into t set b = 11", "update t set b = 7 where a = 5", "delete from t where b = 6"}
var checkErr error
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- callback := &callback.TestDDLCallback{}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if checkErr != nil {
return
}
@@ -1854,9 +1802,7 @@ func TestDropExpressionIndex(t *testing.T) {
}
// (1, 7), (2, 7), (5, 7), (8, 8), (0, 9), (10, 10), (0, 11)
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(callback)
+ })
tk.MustExec("alter table t drop index idx")
require.NoError(t, checkErr)
tk.MustExec("admin check table t")
@@ -1864,7 +1810,7 @@ func TestDropExpressionIndex(t *testing.T) {
}
func TestParallelRenameTable(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("create database test2")
@@ -1881,11 +1827,7 @@ func TestParallelRenameTable(t *testing.T) {
var wg sync.WaitGroup
var checkErr error
- d2 := dom.DDL()
- originalCallback := d2.GetHook()
- defer d2.SetHook(originalCallback)
- callback := &callback.TestDDLCallback{Do: dom}
- callback.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
switch job.SchemaState {
case model.StateNone:
if !firstDDL {
@@ -1911,9 +1853,7 @@ func TestParallelRenameTable(t *testing.T) {
}()
time.Sleep(10 * time.Millisecond)
}
- }
-
- d2.SetHook(callback)
+ })
// rename then add column
concurrentDDLQuery = "alter table t add column g int"
@@ -1979,7 +1919,7 @@ func TestParallelRenameTable(t *testing.T) {
}
func TestConcurrentSetDefaultValue(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -1995,12 +1935,8 @@ func TestConcurrentSetDefaultValue(t *testing.T) {
setdefaultSQLOffset := 0
var wg sync.WaitGroup
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- callback := &callback.TestDDLCallback{Do: dom}
skip := false
- callback.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
switch job.SchemaState {
case model.StateDeleteOnly:
if skip {
@@ -2016,9 +1952,8 @@ func TestConcurrentSetDefaultValue(t *testing.T) {
wg.Done()
}()
}
- }
+ })
- d.SetHook(callback)
tk.MustExec("alter table t modify column a MEDIUMINT NULL DEFAULT '-8145111'")
wg.Wait()
diff --git a/pkg/ddl/db_integration_test.go b/pkg/ddl/db_integration_test.go
index 91808b4fee99f..e675b1696e9af 100644
--- a/pkg/ddl/db_integration_test.go
+++ b/pkg/ddl/db_integration_test.go
@@ -30,7 +30,6 @@ import (
"github.com/pingcap/tidb/pkg/config"
"github.com/pingcap/tidb/pkg/ddl/schematracker"
ddlutil "github.com/pingcap/tidb/pkg/ddl/util"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/infoschema"
@@ -374,7 +373,7 @@ func TestTableDDLWithTimeType(t *testing.T) {
}
func TestUpdateMultipleTable(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("create table t1 (c1 int, c2 int)")
@@ -384,17 +383,13 @@ func TestUpdateMultipleTable(t *testing.T) {
tk2 := testkit.NewTestKit(t, store)
tk2.MustExec("use test")
- d := dom.DDL()
- hook := &callback.TestDDLCallback{Do: dom}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.SchemaState == model.StateWriteOnly {
tk2.MustExec("update t1, t2 set t1.c1 = 8, t2.c2 = 10 where t1.c2 = t2.c1")
tk2.MustQuery("select * from t1").Check(testkit.Rows("8 1", "8 2"))
tk2.MustQuery("select * from t2").Check(testkit.Rows("1 10", "2 10"))
}
- }
- hook.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(hook)
+ })
tk.MustExec("alter table t1 add column c3 bigint default 9")
@@ -3098,16 +3093,13 @@ func TestIssue52680(t *testing.T) {
}
func TestCreateIndexWithChangeMaxIndexLength(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
originCfg := config.GetGlobalConfig()
defer func() {
config.StoreGlobalConfig(originCfg)
}()
- originHook := dom.DDL().GetHook()
- defer dom.DDL().SetHook(originHook)
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.Type != model.ActionAddIndex {
return
}
@@ -3116,8 +3108,7 @@ func TestCreateIndexWithChangeMaxIndexLength(t *testing.T) {
newCfg.MaxIndexLength = 1000
config.StoreGlobalConfig(&newCfg)
}
- }
- dom.DDL().SetHook(hook)
+ })
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test;")
diff --git a/pkg/ddl/db_table_test.go b/pkg/ddl/db_table_test.go
index defa179374184..c5aa1fee85a3b 100644
--- a/pkg/ddl/db_table_test.go
+++ b/pkg/ddl/db_table_test.go
@@ -27,7 +27,6 @@ import (
"github.com/pingcap/tidb/pkg/config"
"github.com/pingcap/tidb/pkg/ddl"
testddlutil "github.com/pingcap/tidb/pkg/ddl/testutil"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/infoschema"
@@ -37,12 +36,12 @@ import (
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/terror"
"github.com/pingcap/tidb/pkg/sessionctx"
- "github.com/pingcap/tidb/pkg/sessiontxn"
"github.com/pingcap/tidb/pkg/store/mockstore"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/table/tables"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/external"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util"
"github.com/stretchr/testify/require"
@@ -106,7 +105,7 @@ func TestAddNotNullColumnWhileInsertOnDupUpdate(t *testing.T) {
}
func TestTransactionOnAddDropColumn(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("set @@global.tidb_max_delta_schema_count= 4096")
tk.MustExec("use test")
@@ -130,11 +129,8 @@ func TestTransactionOnAddDropColumn(t *testing.T) {
},
}
- originHook := dom.DDL().GetHook()
- defer dom.DDL().SetHook(originHook)
- hook := &callback.TestDDLCallback{Do: dom}
var checkErr error
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if checkErr != nil {
return
}
@@ -152,8 +148,7 @@ func TestTransactionOnAddDropColumn(t *testing.T) {
}
}
}
- }
- dom.DDL().SetHook(hook)
+ })
done := make(chan error, 1)
// test transaction on add column.
go backgroundExec(store, "test", "alter table t1 add column c int not null after a", done)
@@ -749,16 +744,12 @@ func TestAddColumn2(t *testing.T) {
tk.MustExec("create table t1 (a int key, b int);")
defer tk.MustExec("drop table if exists t1, t2")
- originHook := dom.DDL().GetHook()
- defer dom.DDL().SetHook(originHook)
- hook := &callback.TestDDLCallback{Do: dom}
var writeOnlyTable table.Table
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.SchemaState == model.StateWriteOnly {
- writeOnlyTable, _ = dom.InfoSchema().TableByID(job.TableID)
+ writeOnlyTable, _ = dom.InfoSchema().TableByID(context.Background(), job.TableID)
}
- }
- dom.DDL().SetHook(hook)
+ })
done := make(chan error, 1)
// test transaction on add column.
go backgroundExec(store, "test", "alter table t1 add column c int not null", done)
@@ -771,14 +762,14 @@ func TestAddColumn2(t *testing.T) {
// mock for outdated tidb update record.
require.NotNil(t, writeOnlyTable)
ctx := context.Background()
- err = sessiontxn.NewTxn(ctx, tk.Session())
+ txn, err := newTxn(tk.Session())
require.NoError(t, err)
oldRow, err := tables.RowWithCols(writeOnlyTable, tk.Session(), kv.IntHandle(1), writeOnlyTable.WritableCols())
require.NoError(t, err)
require.Equal(t, 3, len(oldRow))
- err = writeOnlyTable.RemoveRecord(tk.Session().GetTableCtx(), kv.IntHandle(1), oldRow)
+ err = writeOnlyTable.RemoveRecord(tk.Session().GetTableCtx(), txn, kv.IntHandle(1), oldRow)
require.NoError(t, err)
- _, err = writeOnlyTable.AddRecord(tk.Session().GetTableCtx(), types.MakeDatums(oldRow[0].GetInt64(), 2, oldRow[2].GetInt64()), table.IsUpdate)
+ _, err = writeOnlyTable.AddRecord(tk.Session().GetTableCtx(), txn, types.MakeDatums(oldRow[0].GetInt64(), 2, oldRow[2].GetInt64()), table.IsUpdate)
require.NoError(t, err)
tk.Session().StmtCommit(ctx)
err = tk.Session().CommitTxn(ctx)
@@ -789,24 +780,26 @@ func TestAddColumn2(t *testing.T) {
// Test for _tidb_rowid
var re *testkit.Result
tk.MustExec("create table t2 (a int);")
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.SchemaState != model.StateWriteOnly {
return
}
// allow write _tidb_rowid first
- tk.MustExec("set @@tidb_opt_write_row_id=1")
- tk.MustExec("begin")
- tk.MustExec("insert into t2 (a,_tidb_rowid) values (1,2);")
- re = tk.MustQuery(" select a,_tidb_rowid from t2;")
- tk.MustExec("commit")
- }
- dom.DDL().SetHook(hook)
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("use test")
+ tk2.MustExec("set @@tidb_opt_write_row_id=1")
+ tk2.MustExec("begin")
+ tk2.MustExec("insert into t2 (a,_tidb_rowid) values (1,2);")
+ re = tk2.MustQuery(" select a,_tidb_rowid from t2;")
+ tk2.MustExec("commit")
+ })
go backgroundExec(store, "test", "alter table t2 add column b int not null default 3", done)
err = <-done
require.NoError(t, err)
re.Check(testkit.Rows("1 2"))
tk.MustQuery("select a,b,_tidb_rowid from t2").Check(testkit.Rows("1 3 2"))
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore")
}
func TestDropTables(t *testing.T) {
diff --git a/pkg/ddl/db_test.go b/pkg/ddl/db_test.go
index c0ec7e732bb81..5bc4eb1d17bb6 100644
--- a/pkg/ddl/db_test.go
+++ b/pkg/ddl/db_test.go
@@ -29,7 +29,6 @@ import (
"github.com/pingcap/tidb/pkg/ddl"
"github.com/pingcap/tidb/pkg/ddl/testutil"
ddlutil "github.com/pingcap/tidb/pkg/ddl/util"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/kv"
@@ -45,6 +44,7 @@ import (
"github.com/pingcap/tidb/pkg/sessiontxn"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/external"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/dbterror"
"github.com/pingcap/tidb/pkg/util/mock"
@@ -133,7 +133,7 @@ func TestIssue22819(t *testing.T) {
}
func TestIssue22307(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, dbTestLease)
+ store := testkit.CreateMockStoreWithSchemaLease(t, dbTestLease)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -141,16 +141,14 @@ func TestIssue22307(t *testing.T) {
tk.MustExec("create table t (a int, b int)")
tk.MustExec("insert into t values(1, 1);")
- hook := &callback.TestDDLCallback{Do: dom}
var checkErr1, checkErr2 error
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.SchemaState != model.StateWriteOnly {
return
}
_, checkErr1 = tk.Exec("update t set a = 3 where b = 1;")
_, checkErr2 = tk.Exec("update t set a = 3 order by b;")
- }
- dom.DDL().SetHook(hook)
+ })
done := make(chan error, 1)
// test transaction on add column.
go backgroundExec(store, "test", "alter table t drop column b;", done)
@@ -161,7 +159,7 @@ func TestIssue22307(t *testing.T) {
}
func TestAddExpressionIndexRollback(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, dbTestLease)
+ store := testkit.CreateMockStoreWithSchemaLease(t, dbTestLease)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("create table t1 (c1 int, c2 int, c3 int, unique key(c1))")
@@ -171,13 +169,11 @@ func TestAddExpressionIndexRollback(t *testing.T) {
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
- d := dom.DDL()
- hook := &callback.TestDDLCallback{Do: dom}
var currJob *model.Job
ctx := mock.NewContext()
ctx.Store = store
times := 0
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if checkErr != nil {
return
}
@@ -209,9 +205,7 @@ func TestAddExpressionIndexRollback(t *testing.T) {
times++
}
}
- }
- hook.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(hook)
+ })
tk.MustGetErrMsg("alter table t1 add index expr_idx ((pow(c1, c2)));", "[types:1690]DOUBLE value is out of range in 'pow(160, 160)'")
require.NoError(t, checkErr)
@@ -379,7 +373,7 @@ func TestAlterShardRowIDBits(t *testing.T) {
}
func TestDDLJobErrorCount(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, dbTestLease)
+ store := testkit.CreateMockStoreWithSchemaLease(t, dbTestLease)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("drop table if exists ddl_error_table, new_ddl_error_table")
@@ -391,14 +385,9 @@ func TestDDLJobErrorCount(t *testing.T) {
}()
var jobID int64
- hook := &callback.TestDDLCallback{}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
jobID = job.ID
- }
- hook.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- originHook := dom.DDL().GetHook()
- dom.DDL().SetHook(hook)
- defer dom.DDL().SetHook(originHook)
+ })
tk.MustGetErrCode("rename table ddl_error_table to new_ddl_error_table", errno.ErrEntryTooLarge)
@@ -515,7 +504,7 @@ func TestShowCountWarningsOrErrors(t *testing.T) {
// Close issue #24172.
// See https://github.com/pingcap/tidb/issues/24172
func TestCancelJobWriteConflict(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, dbTestLease)
+ store := testkit.CreateMockStoreWithSchemaLease(t, dbTestLease)
tk1 := testkit.NewTestKit(t, store)
tk2 := testkit.NewTestKit(t, store)
@@ -526,14 +515,9 @@ func TestCancelJobWriteConflict(t *testing.T) {
var cancelErr error
var rs []sqlexec.RecordSet
- hook := &callback.TestDDLCallback{Do: dom}
- d := dom.DDL()
- originalHook := d.GetHook()
- d.SetHook(hook)
- defer d.SetHook(originalHook)
// Test when cancelling cannot be retried and adding index succeeds.
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.Type == model.ActionAddIndex && job.State == model.JobStateRunning && job.SchemaState == model.StateWriteReorganization {
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/kv/mockCommitErrorInNewTxn", `return("no_retry")`))
defer func() {
@@ -547,13 +531,13 @@ func TestCancelJobWriteConflict(t *testing.T) {
stmt := fmt.Sprintf("admin cancel ddl jobs %d", job.ID)
rs, cancelErr = tk2.Session().Execute(context.Background(), stmt)
}
- }
+ })
tk1.MustExec("alter table t add index (id)")
require.EqualError(t, cancelErr, "mock failed admin command on ddl jobs")
// Test when cancelling is retried only once and adding index is cancelled in the end.
var jobID int64
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.Type == model.ActionAddIndex && job.State == model.JobStateRunning && job.SchemaState == model.StateWriteReorganization {
jobID = job.ID
stmt := fmt.Sprintf("admin cancel ddl jobs %d", job.ID)
@@ -563,7 +547,7 @@ func TestCancelJobWriteConflict(t *testing.T) {
}()
rs, cancelErr = tk2.Session().Execute(context.Background(), stmt)
}
- }
+ })
tk1.MustGetErrCode("alter table t add index (id)", errno.ErrCancelledDDLJob)
require.NoError(t, cancelErr)
result := tk2.ResultSetToResultWithCtx(context.Background(), rs[0], "cancel ddl job fails")
@@ -629,7 +613,7 @@ func TestSnapshotVersion(t *testing.T) {
dd := dom.DDL()
ddl.DisableTiFlashPoll(dd)
- require.Equal(t, dbTestLease, dd.GetLease())
+ require.Equal(t, dbTestLease, dom.GetSchemaLease())
snapTS := oracle.GoTimeToTS(time.Now())
tk.MustExec("create database test2")
@@ -689,7 +673,7 @@ func TestSchemaValidator(t *testing.T) {
dd := dom.DDL()
ddl.DisableTiFlashPoll(dd)
- require.Equal(t, dbTestLease, dd.GetLease())
+ require.Equal(t, dbTestLease, dom.GetSchemaLease())
tk.MustExec("create table test.t(a int)")
@@ -939,15 +923,14 @@ func TestTiDBDownBeforeUpdateGlobalVersion(t *testing.T) {
}
func TestDDLBlockedCreateView(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("create table t(a int)")
- hook := &callback.TestDDLCallback{Do: dom}
first := true
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.SchemaState != model.StateWriteOnly {
return
}
@@ -958,33 +941,30 @@ func TestDDLBlockedCreateView(t *testing.T) {
tk2 := testkit.NewTestKit(t, store)
tk2.MustExec("use test")
tk2.MustExec("create view v as select * from t")
- }
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("alter table t modify column a char(10)")
}
func TestHashPartitionAddColumn(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("create table t(a int, b int) partition by hash(a) partitions 4")
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.SchemaState != model.StateWriteOnly {
return
}
tk2 := testkit.NewTestKit(t, store)
tk2.MustExec("use test")
tk2.MustExec("delete from t")
- }
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("alter table t add column c int")
}
func TestSetInvalidDefaultValueAfterModifyColumn(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -993,8 +973,7 @@ func TestSetInvalidDefaultValueAfterModifyColumn(t *testing.T) {
var wg sync.WaitGroup
var checkErr error
one := false
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.SchemaState != model.StateDeleteOnly {
return
}
@@ -1009,15 +988,14 @@ func TestSetInvalidDefaultValueAfterModifyColumn(t *testing.T) {
_, checkErr = tk2.Exec("alter table t alter column a set default 1")
wg.Done()
}()
- }
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("alter table t modify column a text(100)")
wg.Wait()
require.EqualError(t, checkErr, "[ddl:1101]BLOB/TEXT/JSON column 'a' can't have a default value")
}
func TestMDLTruncateTable(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk2 := testkit.NewTestKit(t, store)
@@ -1029,13 +1007,12 @@ func TestMDLTruncateTable(t *testing.T) {
var wg sync.WaitGroup
- hook := &callback.TestDDLCallback{Do: dom}
wg.Add(2)
var timetk2 time.Time
var timetk3 time.Time
one := false
- f := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if one {
return
}
@@ -1045,10 +1022,7 @@ func TestMDLTruncateTable(t *testing.T) {
timetk3 = time.Now()
wg.Done()
}()
- }
-
- hook.OnJobUpdatedExported.Store(&f)
- dom.DDL().SetHook(hook)
+ })
go func() {
tk2.MustExec("truncate table test.t")
@@ -1065,7 +1039,7 @@ func TestMDLTruncateTable(t *testing.T) {
}
func TestTruncateTableAndSchemaDependence(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk2 := testkit.NewTestKit(t, store)
@@ -1080,7 +1054,7 @@ func TestTruncateTableAndSchemaDependence(t *testing.T) {
var timetk3 time.Time
first := false
- f := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if first || job.Type != model.ActionTruncateTable {
return
}
@@ -1091,11 +1065,7 @@ func TestTruncateTableAndSchemaDependence(t *testing.T) {
wg.Done()
}()
time.Sleep(3 * time.Second)
- }
-
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobUpdatedExported.Store(&f)
- dom.DDL().SetHook(hook)
+ })
go func() {
tk2.MustExec("truncate table test.t")
@@ -1118,12 +1088,7 @@ func TestInsertIgnore(t *testing.T) {
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- callback := &callback.TestDDLCallback{}
-
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
switch job.SchemaState {
case model.StateDeleteOnly:
_, err := tk1.Exec("INSERT INTO t VALUES (-18585,'aaa',1), (-18585,'0',1), (-18585,'1',1), (-18585,'duplicatevalue',1);")
@@ -1136,9 +1101,7 @@ func TestInsertIgnore(t *testing.T) {
return
}
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(callback)
+ })
tk.MustExec("alter table t add unique index idx(b);")
tk.MustExec("admin check table t;")
diff --git a/pkg/ddl/ddl.go b/pkg/ddl/ddl.go
index 67b27238456e6..f0c041e28291c 100644
--- a/pkg/ddl/ddl.go
+++ b/pkg/ddl/ddl.go
@@ -21,23 +21,21 @@ package ddl
import (
"context"
"fmt"
- "runtime"
"strconv"
"strings"
"sync"
- "sync/atomic"
"time"
"github.com/google/uuid"
"github.com/ngaut/pools"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
- "github.com/pingcap/kvproto/pkg/kvrpcpb"
"github.com/pingcap/tidb/pkg/config"
"github.com/pingcap/tidb/pkg/ddl/ingest"
"github.com/pingcap/tidb/pkg/ddl/logutil"
+ "github.com/pingcap/tidb/pkg/ddl/schemaver"
+ "github.com/pingcap/tidb/pkg/ddl/serverstate"
sess "github.com/pingcap/tidb/pkg/ddl/session"
- "github.com/pingcap/tidb/pkg/ddl/syncer"
"github.com/pingcap/tidb/pkg/ddl/systable"
"github.com/pingcap/tidb/pkg/ddl/util"
"github.com/pingcap/tidb/pkg/disttask/framework/proto"
@@ -58,16 +56,13 @@ import (
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/statistics/handle"
statsutil "github.com/pingcap/tidb/pkg/statistics/handle/util"
- "github.com/pingcap/tidb/pkg/table"
pumpcli "github.com/pingcap/tidb/pkg/tidb-binlog/pump_client"
tidbutil "github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/dbterror"
- "github.com/pingcap/tidb/pkg/util/dbterror/exeerrors"
"github.com/pingcap/tidb/pkg/util/gcutil"
"github.com/pingcap/tidb/pkg/util/generic"
"github.com/tikv/client-go/v2/tikvrpc"
clientv3 "go.etcd.io/etcd/client/v3"
- "go.etcd.io/etcd/client/v3/concurrency"
atomicutil "go.uber.org/atomic"
"go.uber.org/zap"
)
@@ -84,11 +79,10 @@ const (
shardRowIDBitsMax = 15
- batchAddingJobs = 10
+ batchAddingJobs = 100
reorgWorkerCnt = 10
generalWorkerCnt = 10
- localWorkerCnt = 10
// checkFlagIndexInJobArgs is the recoverCheckFlag index used in RecoverTable/RecoverSchema job arg list.
checkFlagIndexInJobArgs = 1
@@ -170,8 +164,6 @@ type DDL interface {
// Start campaigns the owner and starts workers.
// ctxPool is used for the worker's delRangeManager and creates sessions.
Start(ctxPool *pools.ResourcePool) error
- // GetLease returns current schema lease time.
- GetLease() time.Duration
// Stats returns the DDL statistics.
Stats(vars *variable.SessionVars) (map[string]any, error)
// GetScope gets the status variables scope.
@@ -181,25 +173,28 @@ type DDL interface {
// RegisterStatsHandle registers statistics handle and its corresponding event channel for ddl.
RegisterStatsHandle(*handle.Handle)
// SchemaSyncer gets the schema syncer.
- SchemaSyncer() syncer.SchemaSyncer
+ SchemaSyncer() schemaver.Syncer
// StateSyncer gets the cluster state syncer.
- StateSyncer() syncer.StateSyncer
+ StateSyncer() serverstate.Syncer
// OwnerManager gets the owner manager.
OwnerManager() owner.Manager
// GetID gets the ddl ID.
GetID() string
- // GetTableMaxHandle gets the max row ID of a normal table or a partition.
- GetTableMaxHandle(ctx *JobContext, startTS uint64, tbl table.PhysicalTable) (kv.Handle, bool, error)
// SetBinlogClient sets the binlog client for DDL worker. It's exported for testing.
SetBinlogClient(*pumpcli.PumpsClient)
- // GetHook gets the hook. It's exported for testing.
- GetHook() Callback
- // SetHook sets the hook.
- SetHook(h Callback)
// GetMinJobIDRefresher gets the MinJobIDRefresher, this api only works after Start.
GetMinJobIDRefresher() *systable.MinJobIDRefresher
}
+type jobSubmitResult struct {
+ err error
+ jobID int64
+ // merged indicates whether the job is merged into another job together with
+ // other jobs. we only merge multiple create table jobs into one job when fast
+ // create table is enabled.
+ merged bool
+}
+
// JobWrapper is used to wrap a job and some other information.
// exported for testing.
type JobWrapper struct {
@@ -208,9 +203,9 @@ type JobWrapper struct {
// exported for test.
IDAllocated bool
// job submission is run in async, we use this channel to notify the caller.
- // for local job we might combine multiple jobs into one, append the ErrChs to
- // this slice.
- ErrChs []chan error
+ // when fast create table enabled, we might combine multiple jobs into one, and
+ // append the channel to this slice.
+ ResultCh []chan jobSubmitResult
cacheErr error
}
@@ -220,14 +215,19 @@ func NewJobWrapper(job *model.Job, idAllocated bool) *JobWrapper {
return &JobWrapper{
Job: job,
IDAllocated: idAllocated,
- ErrChs: []chan error{make(chan error)},
+ ResultCh: []chan jobSubmitResult{make(chan jobSubmitResult)},
}
}
-// NotifyError notifies the error to all error channels.
-func (t *JobWrapper) NotifyError(err error) {
- for _, errCh := range t.ErrChs {
- errCh <- err
+// NotifyResult notifies the job submit result.
+func (t *JobWrapper) NotifyResult(err error) {
+ merged := len(t.ResultCh) > 1
+ for _, resultCh := range t.ResultCh {
+ resultCh <- jobSubmitResult{
+ err: err,
+ jobID: t.ID,
+ merged: merged,
+ }
}
}
@@ -236,72 +236,67 @@ type ddl struct {
m sync.RWMutex
wg tidbutil.WaitGroupWrapper // It's only used to deal with data race in restart_test.
limitJobCh chan *JobWrapper
- // limitJobChV2 is used to limit the number of jobs being executed in local worker.
- limitJobChV2 chan *JobWrapper
*ddlCtx
sessPool *sess.Pool
delRangeMgr delRangeManager
enableTiFlashPoll *atomicutil.Bool
- // used in the concurrency ddl.
- localWorkerPool *workerPool
// get notification if any DDL job submitted or finished.
ddlJobNotifyCh chan struct{}
sysTblMgr systable.Manager
minJobIDRefresher *systable.MinJobIDRefresher
- // localJobCh is used to delivery job in local TiDB nodes.
- localJobCh chan *JobWrapper
- // globalIDLocal locks global id to reduce write conflict.
+ // globalIDLock locks global id to reduce write conflict.
globalIDLock sync.Mutex
executor *executor
}
-// waitSchemaSyncedController is to control whether to waitSchemaSynced or not.
-type waitSchemaSyncedController struct {
- mu sync.RWMutex
- job map[int64]struct{}
+// unSyncedJobTracker is to track whether changes of a DDL job are synced to all
+// TiDB instances.
+type unSyncedJobTracker struct {
+ mu sync.RWMutex
+ unSyncedJobs map[int64]struct{}
// Use to check if the DDL job is the first run on this owner.
onceMap map[int64]struct{}
}
-func newWaitSchemaSyncedController() *waitSchemaSyncedController {
- return &waitSchemaSyncedController{
- job: make(map[int64]struct{}, jobRecordCapacity),
- onceMap: make(map[int64]struct{}, jobOnceCapacity),
+func newUnSyncedJobTracker() *unSyncedJobTracker {
+ return &unSyncedJobTracker{
+ unSyncedJobs: make(map[int64]struct{}, jobRecordCapacity),
+ onceMap: make(map[int64]struct{}, jobOnceCapacity),
}
}
-func (w *waitSchemaSyncedController) registerSync(job *model.Job) {
+func (w *unSyncedJobTracker) addUnSynced(jobID int64) {
w.mu.Lock()
defer w.mu.Unlock()
- w.job[job.ID] = struct{}{}
+ w.unSyncedJobs[jobID] = struct{}{}
}
-func (w *waitSchemaSyncedController) isSynced(job *model.Job) bool {
+func (w *unSyncedJobTracker) isUnSynced(jobID int64) bool {
w.mu.RLock()
defer w.mu.RUnlock()
- _, ok := w.job[job.ID]
- return !ok
+ _, ok := w.unSyncedJobs[jobID]
+ return ok
}
-func (w *waitSchemaSyncedController) synced(job *model.Job) {
+func (w *unSyncedJobTracker) removeUnSynced(jobID int64) {
w.mu.Lock()
defer w.mu.Unlock()
- delete(w.job, job.ID)
+ delete(w.unSyncedJobs, jobID)
}
// maybeAlreadyRunOnce returns true means that the job may be the first run on this owner.
// Returns false means that the job must not be the first run on this owner.
-func (w *waitSchemaSyncedController) maybeAlreadyRunOnce(id int64) bool {
+func (w *unSyncedJobTracker) maybeAlreadyRunOnce(id int64) bool {
w.mu.Lock()
defer w.mu.Unlock()
_, ok := w.onceMap[id]
return ok
}
-func (w *waitSchemaSyncedController) setAlreadyRunOnce(id int64) {
+func (w *unSyncedJobTracker) setAlreadyRunOnce(id int64) {
w.mu.Lock()
defer w.mu.Unlock()
if len(w.onceMap) > jobOnceCapacity {
@@ -311,21 +306,15 @@ func (w *waitSchemaSyncedController) setAlreadyRunOnce(id int64) {
w.onceMap[id] = struct{}{}
}
-func (w *waitSchemaSyncedController) clearOnceMap() {
- w.mu.Lock()
- defer w.mu.Unlock()
- w.onceMap = make(map[int64]struct{}, jobOnceCapacity)
-}
-
// ddlCtx is the context when we use worker to handle DDL jobs.
type ddlCtx struct {
- ctx context.Context
- cancel context.CancelFunc
- uuid string
- store kv.Storage
- ownerManager owner.Manager
- schemaSyncer syncer.SchemaSyncer
- stateSyncer syncer.StateSyncer
+ ctx context.Context
+ cancel context.CancelFunc
+ uuid string
+ store kv.Storage
+ ownerManager owner.Manager
+ schemaVerSyncer schemaver.Syncer
+ serverStateSyncer serverstate.Syncer
// ddlJobDoneChMap is used to notify the session that the DDL job is finished.
// jobID -> chan struct{}
ddlJobDoneChMap generic.SyncMap[int64, chan struct{}]
@@ -339,63 +328,49 @@ type ddlCtx struct {
autoidCli *autoid.ClientDiscover
schemaLoader SchemaLoader
- *waitSchemaSyncedController
- *schemaVersionManager
-
// reorgCtx is used for reorganization.
reorgCtx reorgContexts
jobCtx struct {
sync.RWMutex
// jobCtxMap maps job ID to job's ctx.
- jobCtxMap map[int64]*JobContext
+ jobCtxMap map[int64]*ReorgContext
}
-
- // hook may be modified.
- mu hookStruct
-}
-
-// TODO remove it after we remove hook.
-type hookStruct struct {
- sync.RWMutex
- // see newDefaultCallBack for its value in normal flow.
- hook Callback
}
-// the schema synchronization mechanism now requires strict incremental schema versions.
-// Therefore, we require a distributed lock to ensure the sequential commit of schema diffs from different TiDB nodes.
-type etcdLockInfo struct {
- se *concurrency.Session
- mu *concurrency.Mutex
+// SchemaLoader is used to reload info schema, the only impl is domain currently.
+type SchemaLoader interface {
+ Reload() error
}
-// schemaVersionManager is used to manage the schema version. To prevent the conflicts on this key between different DDL job,
-// we use another transaction to update the schema version, so that we need to lock the schema version and unlock it until the job is committed.
-// for version2, we use etcd lock to lock the schema version between TiDB nodes now.
+// schemaVersionManager is used to manage the schema version. info schema cache
+// only load differences between its version and current version, so we must
+// make sure increment version & set differ run in a single transaction. we are
+// using memory lock to make sure this right now, as we are using optimistic
+// transaction here, and there will be many write conflict if we allow those
+// transactions run in parallel. we can change this to lock TiKV key inside the
+// transaction later.
type schemaVersionManager struct {
schemaVersionMu sync.Mutex
// lockOwner stores the job ID that is holding the lock.
lockOwner atomicutil.Int64
-
- ctx context.Context
- etcdClient *clientv3.Client
- lockInfoMaps map[int64]*etcdLockInfo
+ store kv.Storage
}
-func newSchemaVersionManager(ctx context.Context, etcdClient *clientv3.Client) *schemaVersionManager {
+func newSchemaVersionManager(store kv.Storage) *schemaVersionManager {
return &schemaVersionManager{
- ctx: ctx,
- etcdClient: etcdClient,
- lockInfoMaps: make(map[int64]*etcdLockInfo),
+ store: store,
}
}
-func (sv *schemaVersionManager) setSchemaVersion(job *model.Job, store kv.Storage) (schemaVersion int64, err error) {
+func (sv *schemaVersionManager) setSchemaVersion(job *model.Job) (schemaVersion int64, err error) {
err = sv.lockSchemaVersion(job.ID)
if err != nil {
return schemaVersion, errors.Trace(err)
}
- err = kv.RunInNewTxn(kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL), store, true, func(_ context.Context, txn kv.Transaction) error {
+ // TODO we can merge this txn into job transaction to avoid schema version
+ // without differ.
+ err = kv.RunInNewTxn(kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL), sv.store, true, func(_ context.Context, txn kv.Transaction) error {
var err error
m := meta.NewMeta(txn)
schemaVersion, err = m.GenSchemaVersion()
@@ -412,17 +387,6 @@ func (sv *schemaVersionManager) lockSchemaVersion(jobID int64) error {
if ownerID != jobID {
sv.schemaVersionMu.Lock()
sv.lockOwner.Store(jobID)
- if sv.etcdClient != nil && variable.EnableFastCreateTable.Load() {
- se, err := concurrency.NewSession(sv.etcdClient)
- if err != nil {
- return errors.Trace(err)
- }
- mu := concurrency.NewMutex(se, ddlSchemaVersionKeyLock)
- if err := mu.Lock(sv.ctx); err != nil {
- return errors.Trace(err)
- }
- sv.lockInfoMaps[jobID] = &etcdLockInfo{se: se, mu: mu}
- }
}
return nil
}
@@ -431,24 +395,6 @@ func (sv *schemaVersionManager) lockSchemaVersion(jobID int64) error {
func (sv *schemaVersionManager) unlockSchemaVersion(jobID int64) {
ownerID := sv.lockOwner.Load()
if ownerID == jobID {
- if lockInfo, ok := sv.lockInfoMaps[jobID]; ok {
- delete(sv.lockInfoMaps, jobID)
- err := lockInfo.mu.Unlock(sv.ctx)
- outer:
- for err != nil {
- logutil.DDLLogger().Error("unlock schema version", zap.Error(err))
- select {
- case <-sv.ctx.Done():
- break outer
- case <-time.After(time.Second):
- }
- // retry unlock
- err = lockInfo.mu.Unlock(sv.ctx)
- }
- if err := lockInfo.se.Close(); err != nil {
- logutil.DDLLogger().Error("close etcd session", zap.Error(err))
- }
- }
sv.lockOwner.Store(0)
sv.schemaVersionMu.Unlock()
}
@@ -468,7 +414,7 @@ func (dc *ddlCtx) setDDLLabelForTopSQL(jobID int64, jobQuery string) {
defer dc.jobCtx.Unlock()
ctx, exists := dc.jobCtx.jobCtxMap[jobID]
if !exists {
- ctx = NewJobContext()
+ ctx = NewReorgContext()
dc.jobCtx.jobCtxMap[jobID] = ctx
}
ctx.setDDLLabelForTopSQL(jobQuery)
@@ -479,7 +425,7 @@ func (dc *ddlCtx) setDDLSourceForDiagnosis(jobID int64, jobType model.ActionType
defer dc.jobCtx.Unlock()
ctx, exists := dc.jobCtx.jobCtxMap[jobID]
if !exists {
- ctx = NewJobContext()
+ ctx = NewReorgContext()
dc.jobCtx.jobCtxMap[jobID] = ctx
}
ctx.setDDLLabelForDiagnosis(jobType)
@@ -501,14 +447,14 @@ func (dc *ddlCtx) removeJobCtx(job *model.Job) {
delete(dc.jobCtx.jobCtxMap, job.ID)
}
-func (dc *ddlCtx) jobContext(jobID int64, reorgMeta *model.DDLReorgMeta) *JobContext {
+func (dc *ddlCtx) jobContext(jobID int64, reorgMeta *model.DDLReorgMeta) *ReorgContext {
dc.jobCtx.RLock()
defer dc.jobCtx.RUnlock()
- var ctx *JobContext
+ var ctx *ReorgContext
if jobContext, exists := dc.jobCtx.jobCtxMap[jobID]; exists {
ctx = jobContext
} else {
- ctx = NewJobContext()
+ ctx = NewReorgContext()
}
if reorgMeta != nil && len(ctx.resourceGroupName) == 0 {
ctx.resourceGroupName = reorgMeta.ResourceGroupName
@@ -520,6 +466,19 @@ type reorgContexts struct {
sync.RWMutex
// reorgCtxMap maps job ID to reorg context.
reorgCtxMap map[int64]*reorgCtx
+ beOwnerTS int64
+}
+
+func (r *reorgContexts) getOwnerTS() int64 {
+ r.RLock()
+ defer r.RUnlock()
+ return r.beOwnerTS
+}
+
+func (r *reorgContexts) setOwnerTS(ts int64) {
+ r.Lock()
+ r.beOwnerTS = ts
+ r.Unlock()
}
func (dc *ddlCtx) getReorgCtx(jobID int64) *reorgCtx {
@@ -537,7 +496,7 @@ func (dc *ddlCtx) newReorgCtx(jobID int64, rowCount int64) *reorgCtx {
return existedRC
}
rc := &reorgCtx{}
- rc.doneCh = make(chan error, 1)
+ rc.doneCh = make(chan reorgFnResult, 1)
// initial reorgCtx
rc.setRowCount(rowCount)
rc.mu.warnings = make(map[errors.ErrorID]*terror.Error)
@@ -571,24 +530,11 @@ func (dc *ddlCtx) notifyReorgWorkerJobStateChange(job *model.Job) {
rc.notifyJobState(job.State)
}
-func (dc *ddlCtx) initJobDoneCh(jobID int64) {
- dc.ddlJobDoneChMap.Store(jobID, make(chan struct{}, 1))
-}
-
-func (e *executor) getJobDoneCh(jobID int64) (chan struct{}, bool) {
- return e.ddlJobDoneChMap.Load(jobID)
-}
-
-func (e *executor) delJobDoneCh(jobID int64) {
- e.ddlJobDoneChMap.Delete(jobID)
-}
-
func (dc *ddlCtx) notifyJobDone(jobID int64) {
- if ch, ok := dc.ddlJobDoneChMap.Load(jobID); ok {
- select {
- case ch <- struct{}{}:
- default:
- }
+ if ch, ok := dc.ddlJobDoneChMap.Delete(jobID); ok {
+ // broadcast done event as we might merge multiple jobs into one when fast
+ // create table is enabled.
+ close(ch)
}
}
@@ -612,6 +558,8 @@ func (d *ddl) IsTiFlashPollEnabled() bool {
}
// RegisterStatsHandle registers statistics handle and its corresponding even channel for ddl.
+// TODO this is called after ddl started, will cause panic if related DDL are executed
+// in between.
func (d *ddl) RegisterStatsHandle(h *handle.Handle) {
d.ddlCtx.statsHandle = h
d.executor.statsHandle = h
@@ -620,19 +568,12 @@ func (d *ddl) RegisterStatsHandle(h *handle.Handle) {
// asyncNotifyEvent will notify the ddl event to outside world, say statistic handle. When the channel is full, we may
// give up notify and log it.
-func asyncNotifyEvent(d *ddlCtx, e *statsutil.DDLEvent) {
- if d.ddlEventCh != nil {
- if d.lease == 0 {
- // If lease is 0, it's always used in test.
- select {
- case d.ddlEventCh <- e:
- default:
- }
- return
- }
+func asyncNotifyEvent(jobCtx *jobContext, e *statsutil.DDLEvent) {
+ ch := jobCtx.oldDDLCtx.ddlEventCh
+ if ch != nil {
for i := 0; i < 10; i++ {
select {
- case d.ddlEventCh <- e:
+ case ch <- e:
return
default:
time.Sleep(time.Microsecond * 10)
@@ -649,28 +590,26 @@ func NewDDL(ctx context.Context, options ...Option) (DDL, Executor) {
}
func newDDL(ctx context.Context, options ...Option) (*ddl, *executor) {
- opt := &Options{
- Hook: &BaseCallback{},
- }
+ opt := &Options{}
for _, o := range options {
o(opt)
}
id := uuid.New().String()
var manager owner.Manager
- var schemaSyncer syncer.SchemaSyncer
- var stateSyncer syncer.StateSyncer
+ var schemaVerSyncer schemaver.Syncer
+ var serverStateSyncer serverstate.Syncer
var deadLockCkr util.DeadTableLockChecker
if etcdCli := opt.EtcdCli; etcdCli == nil {
// The etcdCli is nil if the store is localstore which is only used for testing.
- // So we use mockOwnerManager and MockSchemaSyncer.
+ // So we use mockOwnerManager and memSyncer.
manager = owner.NewMockManager(ctx, id, opt.Store, DDLOwnerKey)
- schemaSyncer = NewMockSchemaSyncer()
- stateSyncer = NewMockStateSyncer()
+ schemaVerSyncer = schemaver.NewMemSyncer()
+ serverStateSyncer = serverstate.NewMemSyncer()
} else {
manager = owner.NewOwnerManager(ctx, etcdCli, ddlPrompt, id, DDLOwnerKey)
- schemaSyncer = syncer.NewSchemaSyncer(etcdCli, id)
- stateSyncer = syncer.NewStateSyncer(etcdCli, util.ServerGlobalState)
+ schemaVerSyncer = schemaver.NewEtcdSyncer(etcdCli, id)
+ serverStateSyncer = serverstate.NewEtcdSyncer(etcdCli, util.ServerGlobalState)
deadLockCkr = util.NewDeadTableLockChecker(etcdCli)
}
@@ -684,35 +623,30 @@ func newDDL(ctx context.Context, options ...Option) (*ddl, *executor) {
}
ddlCtx := &ddlCtx{
- uuid: id,
- store: opt.Store,
- lease: opt.Lease,
- ddlJobDoneChMap: generic.NewSyncMap[int64, chan struct{}](10),
- ownerManager: manager,
- schemaSyncer: schemaSyncer,
- stateSyncer: stateSyncer,
- binlogCli: binloginfo.GetPumpsClient(),
- infoCache: opt.InfoCache,
- tableLockCkr: deadLockCkr,
- etcdCli: opt.EtcdCli,
- autoidCli: opt.AutoIDClient,
- schemaLoader: opt.SchemaLoader,
- waitSchemaSyncedController: newWaitSchemaSyncedController(),
+ uuid: id,
+ store: opt.Store,
+ lease: opt.Lease,
+ ddlJobDoneChMap: generic.NewSyncMap[int64, chan struct{}](10),
+ ownerManager: manager,
+ schemaVerSyncer: schemaVerSyncer,
+ serverStateSyncer: serverStateSyncer,
+ binlogCli: binloginfo.GetPumpsClient(),
+ infoCache: opt.InfoCache,
+ tableLockCkr: deadLockCkr,
+ etcdCli: opt.EtcdCli,
+ autoidCli: opt.AutoIDClient,
+ schemaLoader: opt.SchemaLoader,
}
ddlCtx.reorgCtx.reorgCtxMap = make(map[int64]*reorgCtx)
- ddlCtx.jobCtx.jobCtxMap = make(map[int64]*JobContext)
- ddlCtx.mu.hook = opt.Hook
+ ddlCtx.jobCtx.jobCtxMap = make(map[int64]*ReorgContext)
ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnDDL)
ddlCtx.ctx, ddlCtx.cancel = context.WithCancel(ctx)
- ddlCtx.schemaVersionManager = newSchemaVersionManager(ddlCtx.ctx, opt.EtcdCli)
d := &ddl{
ddlCtx: ddlCtx,
limitJobCh: make(chan *JobWrapper, batchAddingJobs),
- limitJobChV2: make(chan *JobWrapper, batchAddingJobs),
enableTiFlashPoll: atomicutil.NewBool(true),
ddlJobNotifyCh: make(chan struct{}, 100),
- localJobCh: make(chan *JobWrapper, 1),
}
taskexecutor.RegisterTaskType(proto.Backfill,
@@ -730,7 +664,6 @@ func newDDL(ctx context.Context, options ...Option) (*ddl, *executor) {
variable.EnableDDL = d.EnableDDL
variable.DisableDDL = d.DisableDDL
variable.SwitchMDL = d.SwitchMDL
- variable.SwitchFastCreateTable = d.SwitchFastCreateTable
e := &executor{
ctx: d.ctx,
@@ -743,10 +676,8 @@ func newDDL(ctx context.Context, options ...Option) (*ddl, *executor) {
schemaLoader: d.schemaLoader,
lease: d.lease,
ownerManager: d.ownerManager,
- limitJobChV2: d.limitJobChV2,
ddlJobDoneChMap: &d.ddlJobDoneChMap,
ddlJobNotifyCh: d.ddlJobNotifyCh,
- mu: &d.mu,
globalIDLock: &d.globalIDLock,
}
d.executor = e
@@ -777,33 +708,6 @@ func (d *ddl) newDeleteRangeManager(mock bool) delRangeManager {
return delRangeMgr
}
-func (d *ddl) prepareLocalModeWorkers() {
- var idAllocator atomic.Uint64
- workerFactory := func(tp workerType) func() (pools.Resource, error) {
- return func() (pools.Resource, error) {
- wk := newWorker(d.ctx, tp, d.sessPool, d.delRangeMgr, d.ddlCtx)
- sessForJob, err := d.sessPool.Get()
- if err != nil {
- return nil, err
- }
- sessForJob.GetSessionVars().SetDiskFullOpt(kvrpcpb.DiskFullOpt_AllowedOnAlmostFull)
- wk.sess = sess.NewSession(sessForJob)
- wk.seqAllocator = &idAllocator
- metrics.DDLCounter.WithLabelValues(fmt.Sprintf("%s_%s", metrics.CreateDDL, wk.String())).Inc()
- return wk, nil
- }
- }
- // local worker count at least 2 at most 10.
- localCnt := min(max(runtime.GOMAXPROCS(0)/4, 2), localWorkerCnt)
- d.localWorkerPool = newDDLWorkerPool(pools.NewResourcePool(workerFactory(localWorker), localCnt, localCnt, 0), jobTypeLocal)
- failpoint.Inject("NoDDLDispatchLoop", func(val failpoint.Value) {
- if val.(bool) {
- failpoint.Return()
- }
- })
- d.wg.Run(d.startLocalWorkerLoop)
-}
-
// Start implements DDL.Start interface.
func (d *ddl) Start(ctxPool *pools.ResourcePool) error {
logutil.DDLLogger().Info("start DDL", zap.String("ID", d.uuid), zap.Bool("runWorker", config.GetGlobalConfig().Instance.TiDBEnableDDL.Load()))
@@ -813,10 +717,7 @@ func (d *ddl) Start(ctxPool *pools.ResourcePool) error {
d.sysTblMgr = systable.NewManager(d.sessPool)
d.minJobIDRefresher = systable.NewMinJobIDRefresher(d.sysTblMgr)
d.wg.Run(func() {
- d.limitDDLJobs(d.limitJobCh, d.addBatchDDLJobsV1)
- })
- d.wg.Run(func() {
- d.limitDDLJobs(d.limitJobChV2, d.addBatchLocalDDLJobs)
+ d.limitDDLJobs()
})
d.wg.Run(func() {
d.minJobIDRefresher.Start(d.ctx)
@@ -824,7 +725,7 @@ func (d *ddl) Start(ctxPool *pools.ResourcePool) error {
d.delRangeMgr = d.newDeleteRangeManager(ctxPool == nil)
- if err := d.stateSyncer.Init(d.ctx); err != nil {
+ if err := d.serverStateSyncer.Init(d.ctx); err != nil {
logutil.DDLLogger().Warn("start DDL init state syncer failed", zap.Error(err))
return errors.Trace(err)
}
@@ -832,8 +733,6 @@ func (d *ddl) Start(ctxPool *pools.ResourcePool) error {
ddl: d,
})
- d.prepareLocalModeWorkers()
-
if config.TableLockEnabled() {
d.wg.Add(1)
go d.startCleanDeadTableLock()
@@ -927,10 +826,7 @@ func (d *ddl) close() {
d.cancel()
d.wg.Wait()
d.ownerManager.Cancel()
- d.schemaSyncer.Close()
- if d.localWorkerPool != nil {
- d.localWorkerPool.close()
- }
+ d.schemaVerSyncer.Close()
// d.delRangeMgr using sessions from d.sessPool.
// Put it before d.sessPool.close to reduce the time spent by d.sessPool.close.
@@ -945,49 +841,14 @@ func (d *ddl) close() {
logutil.DDLLogger().Info("DDL closed", zap.String("ID", d.uuid), zap.Duration("take time", time.Since(startTime)))
}
-// GetLease implements DDL.GetLease interface.
-func (d *ddl) GetLease() time.Duration {
- lease := d.lease
- return lease
-}
-
-func (e *executor) genGlobalIDs(count int) ([]int64, error) {
- var ret []int64
- ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL)
- // lock to reduce conflict
- e.globalIDLock.Lock()
- defer e.globalIDLock.Unlock()
- err := kv.RunInNewTxn(ctx, e.store, true, func(_ context.Context, txn kv.Transaction) error {
- m := meta.NewMeta(txn)
- var err error
- ret, err = m.GenGlobalIDs(count)
- return err
- })
-
- return ret, err
-}
-
-func (e *executor) genPlacementPolicyID() (int64, error) {
- var ret int64
- ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL)
- err := kv.RunInNewTxn(ctx, e.store, true, func(_ context.Context, txn kv.Transaction) error {
- m := meta.NewMeta(txn)
- var err error
- ret, err = m.GenPlacementPolicyID()
- return err
- })
-
- return ret, err
-}
-
// SchemaSyncer implements DDL.SchemaSyncer interface.
-func (d *ddl) SchemaSyncer() syncer.SchemaSyncer {
- return d.schemaSyncer
+func (d *ddl) SchemaSyncer() schemaver.Syncer {
+ return d.schemaVerSyncer
}
// StateSyncer implements DDL.StateSyncer interface.
-func (d *ddl) StateSyncer() syncer.StateSyncer {
- return d.stateSyncer
+func (d *ddl) StateSyncer() serverstate.Syncer {
+ return d.serverStateSyncer
}
// OwnerManager implements DDL.OwnerManager interface.
@@ -1000,307 +861,11 @@ func (d *ddl) GetID() string {
return d.uuid
}
-var (
- fastDDLIntervalPolicy = []time.Duration{
- 500 * time.Millisecond,
- }
- normalDDLIntervalPolicy = []time.Duration{
- 500 * time.Millisecond,
- 500 * time.Millisecond,
- 1 * time.Second,
- }
- slowDDLIntervalPolicy = []time.Duration{
- 500 * time.Millisecond,
- 500 * time.Millisecond,
- 1 * time.Second,
- 1 * time.Second,
- 3 * time.Second,
- }
-)
-
-func getIntervalFromPolicy(policy []time.Duration, i int) (time.Duration, bool) {
- plen := len(policy)
- if i < plen {
- return policy[i], true
- }
- return policy[plen-1], false
-}
-
-func getJobCheckInterval(job *model.Job, i int) (time.Duration, bool) {
- switch job.Type {
- case model.ActionAddIndex, model.ActionAddPrimaryKey, model.ActionModifyColumn,
- model.ActionReorganizePartition,
- model.ActionRemovePartitioning,
- model.ActionAlterTablePartitioning:
- return getIntervalFromPolicy(slowDDLIntervalPolicy, i)
- case model.ActionCreateTable, model.ActionCreateSchema:
- return getIntervalFromPolicy(fastDDLIntervalPolicy, i)
- default:
- return getIntervalFromPolicy(normalDDLIntervalPolicy, i)
- }
-}
-
-func (e *executor) notifyNewJobSubmitted(ch chan struct{}, etcdPath string, jobID int64, jobType string) {
- // If the workers don't run, we needn't notify workers.
- // TODO: It does not affect informing the backfill worker.
- if !config.GetGlobalConfig().Instance.TiDBEnableDDL.Load() {
- return
- }
- if e.ownerManager.IsOwner() {
- asyncNotify(ch)
- } else {
- e.notifyNewJobByEtcd(etcdPath, jobID, jobType)
- }
-}
-
-func updateTickerInterval(ticker *time.Ticker, lease time.Duration, job *model.Job, i int) *time.Ticker {
- interval, changed := getJobCheckInterval(job, i)
- if !changed {
- return ticker
- }
- // For now we should stop old ticker and create a new ticker
- ticker.Stop()
- return time.NewTicker(chooseLeaseTime(lease, interval))
-}
-
-func recordLastDDLInfo(ctx sessionctx.Context, job *model.Job) {
- if job == nil {
- return
- }
- ctx.GetSessionVars().LastDDLInfo.Query = job.Query
- ctx.GetSessionVars().LastDDLInfo.SeqNum = job.SeqNum
-}
-
-func setDDLJobQuery(ctx sessionctx.Context, job *model.Job) {
- switch job.Type {
- case model.ActionUpdateTiFlashReplicaStatus, model.ActionUnlockTable:
- job.Query = ""
- default:
- job.Query, _ = ctx.Value(sessionctx.QueryString).(string)
- }
-}
-
-func setDDLJobMode(job *model.Job) {
- if !variable.EnableFastCreateTable.Load() {
- job.LocalMode = false
- return
- }
-
- switch job.Type {
- // currently, v2 only support CreateTable without foreign keys.
- case model.ActionCreateTable:
- tbInfo, ok := job.Args[0].(*model.TableInfo)
- if ok && len(tbInfo.ForeignKeys) == 0 {
- job.LocalMode = true
- return
- }
- case model.ActionCreateSchema:
- job.LocalMode = true
- return
- default:
- }
- job.LocalMode = false
-}
-
-func (e *executor) deliverJobTask(task *JobWrapper) {
- if task.LocalMode {
- e.limitJobChV2 <- task
- } else {
- e.limitJobCh <- task
- }
-}
-
-// DoDDLJob will return
-// - nil: found in history DDL job and no job error
-// - context.Cancel: job has been sent to worker, but not found in history DDL job before cancel
-// - other: found in history DDL job and return that job error
-func (e *executor) DoDDLJob(ctx sessionctx.Context, job *model.Job) error {
- return e.DoDDLJobWrapper(ctx, NewJobWrapper(job, false))
-}
-
-func (e *executor) DoDDLJobWrapper(ctx sessionctx.Context, jobW *JobWrapper) error {
- job := jobW.Job
- job.TraceInfo = &model.TraceInfo{
- ConnectionID: ctx.GetSessionVars().ConnectionID,
- SessionAlias: ctx.GetSessionVars().SessionAlias,
- }
- if mci := ctx.GetSessionVars().StmtCtx.MultiSchemaInfo; mci != nil {
- // In multiple schema change, we don't run the job.
- // Instead, we merge all the jobs into one pending job.
- return appendToSubJobs(mci, job)
- }
- // Get a global job ID and put the DDL job in the queue.
- setDDLJobQuery(ctx, job)
- setDDLJobMode(job)
- e.deliverJobTask(jobW)
-
- failpoint.Inject("mockParallelSameDDLJobTwice", func(val failpoint.Value) {
- if val.(bool) {
- <-jobW.ErrChs[0]
- // The same job will be put to the DDL queue twice.
- job = job.Clone()
- newJobW := NewJobWrapper(job, jobW.IDAllocated)
- e.deliverJobTask(newJobW)
- // The second job result is used for test.
- jobW = newJobW
- }
- })
-
- // worker should restart to continue handling tasks in limitJobCh, and send back through jobW.err
- err := <-jobW.ErrChs[0]
- // job.ID must be allocated after previous channel receive returns nil.
- defer e.delJobDoneCh(job.ID)
- if err != nil {
- // The transaction of enqueuing job is failed.
- return errors.Trace(err)
- }
- failpoint.InjectCall("waitJobSubmitted")
-
- sessVars := ctx.GetSessionVars()
- sessVars.StmtCtx.IsDDLJobInQueue = true
-
- // Notice worker that we push a new job and wait the job done.
- e.notifyNewJobSubmitted(e.ddlJobNotifyCh, addingDDLJobNotifyKey, job.ID, job.Type.String())
- logutil.DDLLogger().Info("start DDL job", zap.Stringer("job", job), zap.String("query", job.Query))
-
- // for local mode job, we add the history job directly now, so no need to check it.
- // fast-create doesn't wait schema version synced, we must reload info-schema
- // here to make sure later statements can see the created table/database.
- if job.LocalMode {
- return e.schemaLoader.Reload()
- }
-
- var historyJob *model.Job
- jobID := job.ID
-
- // Attach the context of the jobId to the calling session so that
- // KILL can cancel this DDL job.
- ctx.GetSessionVars().StmtCtx.DDLJobID = jobID
-
- // For a job from start to end, the state of it will be none -> delete only -> write only -> reorganization -> public
- // For every state changes, we will wait as lease 2 * lease time, so here the ticker check is 10 * lease.
- // But we use etcd to speed up, normally it takes less than 0.5s now, so we use 0.5s or 1s or 3s as the max value.
- initInterval, _ := getJobCheckInterval(job, 0)
- ticker := time.NewTicker(chooseLeaseTime(10*e.lease, initInterval))
- startTime := time.Now()
- metrics.JobsGauge.WithLabelValues(job.Type.String()).Inc()
- defer func() {
- ticker.Stop()
- metrics.JobsGauge.WithLabelValues(job.Type.String()).Dec()
- metrics.HandleJobHistogram.WithLabelValues(job.Type.String(), metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
- recordLastDDLInfo(ctx, historyJob)
- }()
- i := 0
- notifyCh, _ := e.getJobDoneCh(job.ID)
- for {
- failpoint.InjectCall("storeCloseInLoop")
- select {
- case <-notifyCh:
- case <-ticker.C:
- i++
- ticker = updateTickerInterval(ticker, 10*e.lease, job, i)
- case <-e.ctx.Done():
- logutil.DDLLogger().Info("DoDDLJob will quit because context done")
- return context.Canceled
- }
-
- // If the connection being killed, we need to CANCEL the DDL job.
- if sessVars.SQLKiller.HandleSignal() == exeerrors.ErrQueryInterrupted {
- if atomic.LoadInt32(&sessVars.ConnectionStatus) == variable.ConnStatusShutdown {
- logutil.DDLLogger().Info("DoDDLJob will quit because context done")
- return context.Canceled
- }
- if sessVars.StmtCtx.DDLJobID != 0 {
- se, err := e.sessPool.Get()
- if err != nil {
- logutil.DDLLogger().Error("get session failed, check again", zap.Error(err))
- continue
- }
- sessVars.StmtCtx.DDLJobID = 0 // Avoid repeat.
- errs, err := CancelJobsBySystem(se, []int64{jobID})
- e.sessPool.Put(se)
- if len(errs) > 0 {
- logutil.DDLLogger().Warn("error canceling DDL job", zap.Error(errs[0]))
- }
- if err != nil {
- logutil.DDLLogger().Warn("Kill command could not cancel DDL job", zap.Error(err))
- continue
- }
- }
- }
-
- se, err := e.sessPool.Get()
- if err != nil {
- logutil.DDLLogger().Error("get session failed, check again", zap.Error(err))
- continue
- }
- historyJob, err = GetHistoryJobByID(se, jobID)
- e.sessPool.Put(se)
- if err != nil {
- logutil.DDLLogger().Error("get history DDL job failed, check again", zap.Error(err))
- continue
- }
- if historyJob == nil {
- logutil.DDLLogger().Debug("DDL job is not in history, maybe not run", zap.Int64("jobID", jobID))
- continue
- }
-
- e.checkHistoryJobInTest(ctx, historyJob)
-
- // If a job is a history job, the state must be JobStateSynced or JobStateRollbackDone or JobStateCancelled.
- if historyJob.IsSynced() {
- // Judge whether there are some warnings when executing DDL under the certain SQL mode.
- if historyJob.ReorgMeta != nil && len(historyJob.ReorgMeta.Warnings) != 0 {
- if len(historyJob.ReorgMeta.Warnings) != len(historyJob.ReorgMeta.WarningsCount) {
- logutil.DDLLogger().Info("DDL warnings doesn't match the warnings count", zap.Int64("jobID", jobID))
- } else {
- for key, warning := range historyJob.ReorgMeta.Warnings {
- keyCount := historyJob.ReorgMeta.WarningsCount[key]
- if keyCount == 1 {
- ctx.GetSessionVars().StmtCtx.AppendWarning(warning)
- } else {
- newMsg := fmt.Sprintf("%d warnings with this error code, first warning: "+warning.GetMsg(), keyCount)
- newWarning := dbterror.ClassTypes.Synthesize(terror.ErrCode(warning.Code()), newMsg)
- ctx.GetSessionVars().StmtCtx.AppendWarning(newWarning)
- }
- }
- }
- }
- appendMultiChangeWarningsToOwnerCtx(ctx, historyJob)
-
- logutil.DDLLogger().Info("DDL job is finished", zap.Int64("jobID", jobID))
- return nil
- }
-
- if historyJob.Error != nil {
- logutil.DDLLogger().Info("DDL job is failed", zap.Int64("jobID", jobID))
- return errors.Trace(historyJob.Error)
- }
- panic("When the state is JobStateRollbackDone or JobStateCancelled, historyJob.Error should never be nil")
- }
-}
-
// SetBinlogClient implements DDL.SetBinlogClient interface.
func (d *ddl) SetBinlogClient(binlogCli *pumpcli.PumpsClient) {
d.binlogCli = binlogCli
}
-// GetHook implements DDL.GetHook interface.
-func (d *ddl) GetHook() Callback {
- d.mu.Lock()
- defer d.mu.Unlock()
-
- return d.mu.hook
-}
-
-// SetHook set the customized hook.
-func (d *ddl) SetHook(h Callback) {
- d.mu.Lock()
- defer d.mu.Unlock()
-
- d.mu.hook = h
-}
-
func (d *ddl) GetMinJobIDRefresher() *systable.MinJobIDRefresher {
return d.minJobIDRefresher
}
@@ -1320,13 +885,13 @@ func (d *ddl) startCleanDeadTableLock() {
if !d.ownerManager.IsOwner() {
continue
}
- deadLockTables, err := d.tableLockCkr.GetDeadLockedTables(d.ctx, d.infoCache.GetLatest().AllSchemas())
+ deadLockTables, err := d.tableLockCkr.GetDeadLockedTables(d.ctx, d.infoCache.GetLatest())
if err != nil {
logutil.DDLLogger().Info("get dead table lock failed.", zap.Error(err))
continue
}
for se, tables := range deadLockTables {
- err := d.CleanDeadTableLock(tables, se)
+ err := d.cleanDeadTableLock(tables, se)
if err != nil {
logutil.DDLLogger().Info("clean dead table lock failed.", zap.Error(err))
}
@@ -1337,6 +902,32 @@ func (d *ddl) startCleanDeadTableLock() {
}
}
+// cleanDeadTableLock uses to clean dead table locks.
+func (d *ddl) cleanDeadTableLock(unlockTables []model.TableLockTpInfo, se model.SessionInfo) error {
+ if len(unlockTables) == 0 {
+ return nil
+ }
+ arg := &LockTablesArg{
+ UnlockTables: unlockTables,
+ SessionInfo: se,
+ }
+ job := &model.Job{
+ SchemaID: unlockTables[0].SchemaID,
+ TableID: unlockTables[0].TableID,
+ Type: model.ActionUnlockTable,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []any{arg},
+ }
+
+ ctx, err := d.sessPool.Get()
+ if err != nil {
+ return err
+ }
+ defer d.sessPool.Put(ctx)
+ err = d.executor.DoDDLJob(ctx, job)
+ return errors.Trace(err)
+}
+
// SwitchMDL enables MDL or disable MDL.
func (d *ddl) SwitchMDL(enable bool) error {
isEnableBefore := variable.EnableMDL.Load()
@@ -1383,114 +974,6 @@ func (d *ddl) SwitchMDL(enable bool) error {
return nil
}
-// SwitchFastCreateTable switch fast create table
-func (d *ddl) SwitchFastCreateTable(val bool) error {
- old := variable.EnableFastCreateTable.Load()
- if old == val {
- return nil
- }
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
- defer cancel()
-
- // Check if there is any DDL running.
- // This check can not cover every corner cases, so users need to guarantee that there is no DDL running by themselves.
- sessCtx, err := d.sessPool.Get()
- if err != nil {
- return errors.Trace(err)
- }
- defer d.sessPool.Put(sessCtx)
- se := sess.NewSession(sessCtx)
- rows, err := se.Execute(ctx, "select 1 from mysql.tidb_ddl_job", "check job")
- if err != nil {
- return errors.Trace(err)
- }
- if len(rows) != 0 {
- return errors.New("please wait for all jobs done")
- }
-
- if err := d.switchFastCreateTable(val); err != nil {
- return errors.Trace(err)
- }
-
- variable.EnableFastCreateTable.Store(val)
- logutil.DDLLogger().Info("switch fast create table", zap.Bool("val", val))
- return nil
-}
-
-// disableFastCreateTable disable fast create table feature.
-func (*ddl) disableFastCreateTable(m *meta.Meta) error {
- fastCreateTableInitialized, err := m.GetFastCreateTableInitialized()
- if err != nil {
- return errors.Trace(err)
- }
- if !fastCreateTableInitialized {
- return nil
- }
- if err := m.ClearAllDatabaseNames(); err != nil {
- return errors.Trace(err)
- }
- if err := m.ClearAllTableNames(); err != nil {
- return errors.Trace(err)
- }
- return errors.Trace(m.SetFastCreateTableInitialized(false))
-}
-
-// enableFastCreateTable enable fast create table feature.
-func (*ddl) enableFastCreateTable(m *meta.Meta) error {
- fastCreateTableInitialized, err := m.GetFastCreateTableInitialized()
- if err != nil {
- return errors.Trace(err)
- }
- if fastCreateTableInitialized {
- return nil
- }
-
- if err := m.ClearAllDatabaseNames(); err != nil {
- return errors.Trace(err)
- }
- if err := m.ClearAllTableNames(); err != nil {
- return errors.Trace(err)
- }
-
- dbs, err := m.ListDatabases()
- if err != nil {
- return errors.Trace(err)
- }
-
- for _, dbInfo := range dbs {
- if err := m.CreateDatabaseName(dbInfo.Name.L, dbInfo.ID); err != nil {
- return errors.Trace(err)
- }
- }
-
- for _, dbInfo := range dbs {
- tables, err := m.ListTables(dbInfo.ID)
- if err != nil {
- return errors.Trace(err)
- }
- for _, tableInfo := range tables {
- if err := m.CreateTableName(dbInfo.Name.L, tableInfo.Name.L, tableInfo.ID); err != nil {
- return errors.Trace(err)
- }
- }
- }
-
- return errors.Trace(m.SetFastCreateTableInitialized(true))
-}
-
-func (d *ddl) switchFastCreateTable(val bool) (err error) {
- return kv.RunInNewTxn(kv.WithInternalSourceType(d.ctx, kv.InternalTxnDDL), d.store, true, func(_ context.Context, txn kv.Transaction) error {
- m := meta.NewMeta(txn)
-
- if val {
- err = d.enableFastCreateTable(m)
- } else {
- err = d.disableFastCreateTable(m)
- }
- return errors.Trace(err)
- })
-}
-
// RecoverInfo contains information needed by DDL.RecoverTable.
type RecoverInfo struct {
SchemaID int64
@@ -1520,9 +1003,19 @@ type RecoverSchemaInfo struct {
// This provides a safe window for async commit and 1PC to commit with an old schema.
func delayForAsyncCommit() {
if variable.EnableMDL.Load() {
- // If metadata lock is enabled. The transaction of DDL must begin after prewrite of the async commit transaction,
- // then the commit ts of DDL must be greater than the async commit transaction. In this case, the corresponding schema of the async commit transaction
- // is correct. But if metadata lock is disabled, we can't ensure that the corresponding schema of the async commit transaction isn't change.
+ // If metadata lock is enabled. The transaction of DDL must begin after
+ // pre-write of the async commit transaction, then the commit ts of DDL
+ // must be greater than the async commit transaction. In this case, the
+ // corresponding schema of the async commit transaction is correct.
+ // suppose we're adding index:
+ // - schema state -> StateWriteOnly with version V
+ // - some txn T started using async commit and version V,
+ // and T do pre-write before or after V+1
+ // - schema state -> StateWriteReorganization with version V+1
+ // - T commit finish, with TS
+ // - 'wait schema synced' finish
+ // - schema state -> Done with version V+2, commit-ts of this
+ // transaction must > TS, so it's safe for T to commit.
return
}
cfg := config.GetGlobalConfig().TiKVClient.AsyncCommit
diff --git a/pkg/ddl/ddl_test.go b/pkg/ddl/ddl_test.go
index 6a88570f222ea..314d4ab0d27a2 100644
--- a/pkg/ddl/ddl_test.go
+++ b/pkg/ddl/ddl_test.go
@@ -30,6 +30,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/terror"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/store/mockstore"
+ "github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/dbterror"
"github.com/pingcap/tidb/pkg/util/mock"
@@ -294,3 +295,146 @@ func TestCheckDuplicateConstraint(t *testing.T) {
err = checkDuplicateConstraint(constrNames, "u1", ast.ConstraintUniq)
require.EqualError(t, err, "[ddl:1061]Duplicate key name 'u1'")
}
+
+func TestGetTableDataKeyRanges(t *testing.T) {
+ // case 1, empty flashbackIDs
+ keyRanges := getTableDataKeyRanges([]int64{})
+ require.Len(t, keyRanges, 1)
+ require.Equal(t, keyRanges[0].StartKey, tablecodec.EncodeTablePrefix(0))
+ require.Equal(t, keyRanges[0].EndKey, tablecodec.EncodeTablePrefix(meta.MaxGlobalID))
+
+ // case 2, insert a execluded table ID
+ keyRanges = getTableDataKeyRanges([]int64{3})
+ require.Len(t, keyRanges, 2)
+ require.Equal(t, keyRanges[0].StartKey, tablecodec.EncodeTablePrefix(0))
+ require.Equal(t, keyRanges[0].EndKey, tablecodec.EncodeTablePrefix(3))
+ require.Equal(t, keyRanges[1].StartKey, tablecodec.EncodeTablePrefix(4))
+ require.Equal(t, keyRanges[1].EndKey, tablecodec.EncodeTablePrefix(meta.MaxGlobalID))
+
+ // case 3, insert some execluded table ID
+ keyRanges = getTableDataKeyRanges([]int64{3, 5, 9})
+ require.Len(t, keyRanges, 4)
+ require.Equal(t, keyRanges[0].StartKey, tablecodec.EncodeTablePrefix(0))
+ require.Equal(t, keyRanges[0].EndKey, tablecodec.EncodeTablePrefix(3))
+ require.Equal(t, keyRanges[1].StartKey, tablecodec.EncodeTablePrefix(4))
+ require.Equal(t, keyRanges[1].EndKey, tablecodec.EncodeTablePrefix(5))
+ require.Equal(t, keyRanges[2].StartKey, tablecodec.EncodeTablePrefix(6))
+ require.Equal(t, keyRanges[2].EndKey, tablecodec.EncodeTablePrefix(9))
+ require.Equal(t, keyRanges[3].StartKey, tablecodec.EncodeTablePrefix(10))
+ require.Equal(t, keyRanges[3].EndKey, tablecodec.EncodeTablePrefix(meta.MaxGlobalID))
+}
+
+func TestAppendContinuousKeyRanges(t *testing.T) {
+ cases := []struct {
+ input []keyRangeMayExclude
+ expect []kv.KeyRange
+ }{
+ {
+ []keyRangeMayExclude{
+ {
+ r: kv.KeyRange{StartKey: []byte{1}, EndKey: []byte{2}},
+ exclude: true,
+ },
+ },
+ []kv.KeyRange{},
+ },
+ {
+ []keyRangeMayExclude{
+ {
+ r: kv.KeyRange{StartKey: []byte{1}, EndKey: []byte{2}},
+ exclude: false,
+ },
+ },
+ []kv.KeyRange{{StartKey: []byte{1}, EndKey: []byte{2}}},
+ },
+ {
+ []keyRangeMayExclude{
+ {
+ r: kv.KeyRange{StartKey: []byte{1}, EndKey: []byte{2}},
+ exclude: false,
+ },
+ {
+ r: kv.KeyRange{StartKey: []byte{3}, EndKey: []byte{4}},
+ exclude: false,
+ },
+ },
+ []kv.KeyRange{{StartKey: []byte{1}, EndKey: []byte{4}}},
+ },
+ {
+ []keyRangeMayExclude{
+ {
+ r: kv.KeyRange{StartKey: []byte{1}, EndKey: []byte{2}},
+ exclude: false,
+ },
+ {
+ r: kv.KeyRange{StartKey: []byte{3}, EndKey: []byte{4}},
+ exclude: true,
+ },
+ {
+ r: kv.KeyRange{StartKey: []byte{5}, EndKey: []byte{6}},
+ exclude: false,
+ },
+ },
+ []kv.KeyRange{
+ {StartKey: []byte{1}, EndKey: []byte{2}},
+ {StartKey: []byte{5}, EndKey: []byte{6}},
+ },
+ },
+ {
+ []keyRangeMayExclude{
+ {
+ r: kv.KeyRange{StartKey: []byte{1}, EndKey: []byte{2}},
+ exclude: true,
+ },
+ {
+ r: kv.KeyRange{StartKey: []byte{3}, EndKey: []byte{4}},
+ exclude: true,
+ },
+ {
+ r: kv.KeyRange{StartKey: []byte{5}, EndKey: []byte{6}},
+ exclude: false,
+ },
+ },
+ []kv.KeyRange{{StartKey: []byte{5}, EndKey: []byte{6}}},
+ },
+ {
+ []keyRangeMayExclude{
+ {
+ r: kv.KeyRange{StartKey: []byte{1}, EndKey: []byte{2}},
+ exclude: false,
+ },
+ {
+ r: kv.KeyRange{StartKey: []byte{3}, EndKey: []byte{4}},
+ exclude: true,
+ },
+ {
+ r: kv.KeyRange{StartKey: []byte{5}, EndKey: []byte{6}},
+ exclude: true,
+ },
+ },
+ []kv.KeyRange{{StartKey: []byte{1}, EndKey: []byte{2}}},
+ },
+ {
+ []keyRangeMayExclude{
+ {
+ r: kv.KeyRange{StartKey: []byte{1}, EndKey: []byte{2}},
+ exclude: true,
+ },
+ {
+ r: kv.KeyRange{StartKey: []byte{3}, EndKey: []byte{4}},
+ exclude: false,
+ },
+ {
+ r: kv.KeyRange{StartKey: []byte{5}, EndKey: []byte{6}},
+ exclude: true,
+ },
+ },
+ []kv.KeyRange{{StartKey: []byte{3}, EndKey: []byte{4}}},
+ },
+ }
+
+ for i, ca := range cases {
+ ranges := appendContinuousKeyRanges([]kv.KeyRange{}, ca.input)
+ require.Equal(t, ca.expect, ranges, "case %d", i)
+ }
+}
diff --git a/pkg/ddl/ddl_tiflash_api.go b/pkg/ddl/ddl_tiflash_api.go
index 9f4c2512f0019..9813ce612f7e3 100644
--- a/pkg/ddl/ddl_tiflash_api.go
+++ b/pkg/ddl/ddl_tiflash_api.go
@@ -370,7 +370,7 @@ func PollAvailableTableProgress(schemas infoschema.InfoSchema, _ sessionctx.Cont
}
} else {
var ok bool
- table, ok = schemas.TableByID(availableTableID.ID)
+ table, ok = schemas.TableByID(context.Background(), availableTableID.ID)
if !ok {
logutil.DDLLogger().Info("get table id failed, may be dropped or truncated",
zap.Int64("tableID", availableTableID.ID),
@@ -461,7 +461,7 @@ func (d *ddl) refreshTiFlashTicker(ctx sessionctx.Context, pollTiFlashContext *T
failpoint.Inject("waitForAddPartition", func(val failpoint.Value) {
for _, phyTable := range tableList {
is := d.infoCache.GetLatest()
- _, ok := is.TableByID(phyTable.ID)
+ _, ok := is.TableByID(d.ctx, phyTable.ID)
if !ok {
tb, _, _ := is.FindTableByPartitionID(phyTable.ID)
if tb == nil {
diff --git a/pkg/ddl/delete_range.go b/pkg/ddl/delete_range.go
index 68ddfe092982f..bafa01c847977 100644
--- a/pkg/ddl/delete_range.go
+++ b/pkg/ddl/delete_range.go
@@ -324,9 +324,10 @@ func insertJobIntoDeleteRangeTable(ctx context.Context, wrapper DelRangeExecWrap
case model.ActionAddIndex, model.ActionAddPrimaryKey:
allIndexIDs := make([]int64, 1)
ifExists := make([]bool, 1)
+ isGlobal := make([]bool, 0, 1)
var partitionIDs []int64
if err := job.DecodeArgs(&allIndexIDs[0], &ifExists[0], &partitionIDs); err != nil {
- if err = job.DecodeArgs(&allIndexIDs, &ifExists, &partitionIDs); err != nil {
+ if err = job.DecodeArgs(&allIndexIDs, &ifExists, &partitionIDs, &isGlobal); err != nil {
return errors.Trace(err)
}
}
@@ -335,7 +336,7 @@ func insertJobIntoDeleteRangeTable(ctx context.Context, wrapper DelRangeExecWrap
if len(partitionIDs) > 0 {
physicalIDs = partitionIDs
}
- for _, indexID := range allIndexIDs {
+ for i, indexID := range allIndexIDs {
// Determine the index IDs to be added.
tempIdxID := tablecodec.TempIndexPrefix | indexID
var indexIDs []int64
@@ -344,10 +345,16 @@ func insertJobIntoDeleteRangeTable(ctx context.Context, wrapper DelRangeExecWrap
} else {
indexIDs = []int64{tempIdxID}
}
- for _, pid := range physicalIDs {
- if err := doBatchDeleteIndiceRange(ctx, wrapper, job.ID, pid, indexIDs, ea, "add index: physical table ID(s)"); err != nil {
+ if len(isGlobal) != 0 && isGlobal[i] {
+ if err := doBatchDeleteIndiceRange(ctx, wrapper, job.ID, job.TableID, indexIDs, ea, "add index: physical table ID(s)"); err != nil {
return errors.Trace(err)
}
+ } else {
+ for _, pid := range physicalIDs {
+ if err := doBatchDeleteIndiceRange(ctx, wrapper, job.ID, pid, indexIDs, ea, "add index: physical table ID(s)"); err != nil {
+ return errors.Trace(err)
+ }
+ }
}
}
case model.ActionDropIndex, model.ActionDropPrimaryKey:
diff --git a/pkg/ddl/executor.go b/pkg/ddl/executor.go
index 0086afbfc86c4..8b7e4cbd1afe6 100644
--- a/pkg/ddl/executor.go
+++ b/pkg/ddl/executor.go
@@ -23,8 +23,6 @@ import (
"context"
"fmt"
"math"
- "slices"
- "strconv"
"strings"
"sync"
"sync/atomic"
@@ -42,20 +40,17 @@ import (
rg "github.com/pingcap/tidb/pkg/domain/resourcegroup"
"github.com/pingcap/tidb/pkg/errctx"
"github.com/pingcap/tidb/pkg/expression"
- exprctx "github.com/pingcap/tidb/pkg/expression/context"
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/meta"
"github.com/pingcap/tidb/pkg/meta/autoid"
+ "github.com/pingcap/tidb/pkg/metrics"
"github.com/pingcap/tidb/pkg/owner"
- "github.com/pingcap/tidb/pkg/parser"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/charset"
- "github.com/pingcap/tidb/pkg/parser/format"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/parser/terror"
- field_types "github.com/pingcap/tidb/pkg/parser/types"
"github.com/pingcap/tidb/pkg/privilege"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
@@ -65,22 +60,16 @@ import (
"github.com/pingcap/tidb/pkg/table/tables"
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/types"
- driver "github.com/pingcap/tidb/pkg/types/parser_driver"
"github.com/pingcap/tidb/pkg/util"
- "github.com/pingcap/tidb/pkg/util/chunk"
"github.com/pingcap/tidb/pkg/util/collate"
"github.com/pingcap/tidb/pkg/util/dbterror"
"github.com/pingcap/tidb/pkg/util/dbterror/exeerrors"
"github.com/pingcap/tidb/pkg/util/domainutil"
"github.com/pingcap/tidb/pkg/util/generic"
- "github.com/pingcap/tidb/pkg/util/hack"
+ "github.com/pingcap/tidb/pkg/util/intest"
"github.com/pingcap/tidb/pkg/util/mathutil"
- "github.com/pingcap/tidb/pkg/util/mock"
- "github.com/pingcap/tidb/pkg/util/set"
- "github.com/pingcap/tidb/pkg/util/sqlkiller"
"github.com/pingcap/tidb/pkg/util/stringutil"
"github.com/tikv/client-go/v2/oracle"
- kvutil "github.com/tikv/client-go/v2/util"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
)
@@ -195,10 +184,8 @@ type executor struct {
schemaLoader SchemaLoader
lease time.Duration // lease is schema lease, default 45s, see config.Lease.
ownerManager owner.Manager
- limitJobChV2 chan *JobWrapper
ddlJobDoneChMap *generic.SyncMap[int64, chan struct{}]
ddlJobNotifyCh chan struct{}
- mu *hookStruct // TODO remove it.
globalIDLock *sync.Mutex
}
@@ -430,7 +417,7 @@ func (e *executor) getPendingTiFlashTableCount(originVersion int64, pendingCount
cnt := uint32(0)
dbs := is.ListTablesWithSpecialAttribute(infoschema.TiFlashAttribute)
for _, db := range dbs {
- if util.IsMemOrSysDB(db.DBName) {
+ if util.IsMemOrSysDB(db.DBName.L) {
continue
}
for _, tbl := range db.TableInfos {
@@ -665,25 +652,6 @@ func (e *executor) AlterTablePlacement(ctx sessionctx.Context, ident ast.Ident,
return errors.Trace(err)
}
-func checkAndNormalizePlacementPolicy(ctx sessionctx.Context, placementPolicyRef *model.PolicyRefInfo) (*model.PolicyRefInfo, error) {
- if placementPolicyRef == nil {
- return nil, nil
- }
-
- if placementPolicyRef.Name.L == defaultPlacementPolicyName {
- // When policy name is 'default', it means to remove the placement settings
- return nil, nil
- }
-
- policy, ok := sessiontxn.GetTxnManager(ctx).GetTxnInfoSchema().PolicyByName(placementPolicyRef.Name)
- if !ok {
- return nil, errors.Trace(infoschema.ErrPlacementPolicyNotExists.GenWithStackByArgs(placementPolicyRef.Name))
- }
-
- placementPolicyRef.ID = policy.ID
- return placementPolicyRef, nil
-}
-
func checkMultiSchemaSpecs(_ sessionctx.Context, specs []*ast.DatabaseOption) error {
hasSetTiFlashReplica := false
if len(specs) == 1 {
@@ -877,117 +845,6 @@ func checkTooLongForeignKey(fk model.CIStr) error {
return nil
}
-func setColumnFlagWithConstraint(colMap map[string]*table.Column, v *ast.Constraint) {
- switch v.Tp {
- case ast.ConstraintPrimaryKey:
- for _, key := range v.Keys {
- if key.Expr != nil {
- continue
- }
- c, ok := colMap[key.Column.Name.L]
- if !ok {
- continue
- }
- c.AddFlag(mysql.PriKeyFlag)
- // Primary key can not be NULL.
- c.AddFlag(mysql.NotNullFlag)
- setNoDefaultValueFlag(c, c.DefaultValue != nil)
- }
- case ast.ConstraintUniq, ast.ConstraintUniqIndex, ast.ConstraintUniqKey:
- for i, key := range v.Keys {
- if key.Expr != nil {
- continue
- }
- c, ok := colMap[key.Column.Name.L]
- if !ok {
- continue
- }
- if i == 0 {
- // Only the first column can be set
- // if unique index has multi columns,
- // the flag should be MultipleKeyFlag.
- // See https://dev.mysql.com/doc/refman/5.7/en/show-columns.html
- if len(v.Keys) > 1 {
- c.AddFlag(mysql.MultipleKeyFlag)
- } else {
- c.AddFlag(mysql.UniqueKeyFlag)
- }
- }
- }
- case ast.ConstraintKey, ast.ConstraintIndex:
- for i, key := range v.Keys {
- if key.Expr != nil {
- continue
- }
- c, ok := colMap[key.Column.Name.L]
- if !ok {
- continue
- }
- if i == 0 {
- // Only the first column can be set.
- c.AddFlag(mysql.MultipleKeyFlag)
- }
- }
- }
-}
-
-func buildColumnsAndConstraints(
- ctx sessionctx.Context,
- colDefs []*ast.ColumnDef,
- constraints []*ast.Constraint,
- tblCharset string,
- tblCollate string,
-) ([]*table.Column, []*ast.Constraint, error) {
- // outPriKeyConstraint is the primary key constraint out of column definition. such as: create table t1 (id int , age int, primary key(id));
- var outPriKeyConstraint *ast.Constraint
- for _, v := range constraints {
- if v.Tp == ast.ConstraintPrimaryKey {
- outPriKeyConstraint = v
- break
- }
- }
- cols := make([]*table.Column, 0, len(colDefs))
- colMap := make(map[string]*table.Column, len(colDefs))
-
- for i, colDef := range colDefs {
- if field_types.TiDBStrictIntegerDisplayWidth {
- switch colDef.Tp.GetType() {
- case mysql.TypeTiny:
- // No warning for BOOL-like tinyint(1)
- if colDef.Tp.GetFlen() != types.UnspecifiedLength && colDef.Tp.GetFlen() != 1 {
- ctx.GetSessionVars().StmtCtx.AppendWarning(
- dbterror.ErrWarnDeprecatedIntegerDisplayWidth.FastGenByArgs(),
- )
- }
- case mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong:
- if colDef.Tp.GetFlen() != types.UnspecifiedLength {
- ctx.GetSessionVars().StmtCtx.AppendWarning(
- dbterror.ErrWarnDeprecatedIntegerDisplayWidth.FastGenByArgs(),
- )
- }
- }
- }
- col, cts, err := buildColumnAndConstraint(ctx, i, colDef, outPriKeyConstraint, tblCharset, tblCollate)
- if err != nil {
- return nil, nil, errors.Trace(err)
- }
- col.State = model.StatePublic
- if mysql.HasZerofillFlag(col.GetFlag()) {
- ctx.GetSessionVars().StmtCtx.AppendWarning(
- dbterror.ErrWarnDeprecatedZerofill.FastGenByArgs(),
- )
- }
- constraints = append(constraints, cts...)
- cols = append(cols, col)
- colMap[colDef.Name.Name.L] = col
- }
- // Traverse table Constraints and set col.flag.
- for _, v := range constraints {
- setColumnFlagWithConstraint(colMap, v)
- }
- return cols, constraints, nil
-}
-
func getDefaultCollationForUTF8MB4(sessVars *variable.SessionVars, cs string) (string, error) {
if sessVars == nil || cs != charset.CharsetUTF8MB4 {
return "", nil
@@ -1050,4155 +907,1369 @@ func ResolveCharsetCollation(sessVars *variable.SessionVars, charsetOpts ...ast.
return chs, coll, nil
}
-// OverwriteCollationWithBinaryFlag is used to handle the case like
-//
-// CREATE TABLE t (a VARCHAR(255) BINARY) CHARSET utf8 COLLATE utf8_general_ci;
-//
-// The 'BINARY' sets the column collation to *_bin according to the table charset.
-func OverwriteCollationWithBinaryFlag(sessVars *variable.SessionVars, colDef *ast.ColumnDef, chs, coll string) (newChs string, newColl string) {
- ignoreBinFlag := colDef.Tp.GetCharset() != "" && (colDef.Tp.GetCollate() != "" || containsColumnOption(colDef, ast.ColumnOptionCollate))
- if ignoreBinFlag {
- return chs, coll
- }
- needOverwriteBinColl := types.IsString(colDef.Tp.GetType()) && mysql.HasBinaryFlag(colDef.Tp.GetFlag())
- if needOverwriteBinColl {
- newColl, err := GetDefaultCollation(sessVars, chs)
- if err != nil {
- return chs, coll
- }
- return chs, newColl
+// IsAutoRandomColumnID returns true if the given column ID belongs to an auto_random column.
+func IsAutoRandomColumnID(tblInfo *model.TableInfo, colID int64) bool {
+ if !tblInfo.ContainsAutoRandomBits() {
+ return false
}
- return chs, coll
-}
-
-func typesNeedCharset(tp byte) bool {
- switch tp {
- case mysql.TypeString, mysql.TypeVarchar, mysql.TypeVarString,
- mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob,
- mysql.TypeEnum, mysql.TypeSet:
- return true
+ if tblInfo.PKIsHandle {
+ return tblInfo.GetPkColInfo().ID == colID
+ } else if tblInfo.IsCommonHandle {
+ pk := tables.FindPrimaryIndex(tblInfo)
+ if pk == nil {
+ return false
+ }
+ offset := pk.Columns[0].Offset
+ return tblInfo.Columns[offset].ID == colID
}
return false
}
-func setCharsetCollationFlenDecimal(tp *types.FieldType, colName, colCharset, colCollate string, sessVars *variable.SessionVars) error {
- var err error
- if typesNeedCharset(tp.GetType()) {
- tp.SetCharset(colCharset)
- tp.SetCollate(colCollate)
- } else {
- tp.SetCharset(charset.CharsetBin)
- tp.SetCollate(charset.CharsetBin)
- }
-
- // Use default value for flen or decimal when they are unspecified.
- defaultFlen, defaultDecimal := mysql.GetDefaultFieldLengthAndDecimal(tp.GetType())
- if tp.GetDecimal() == types.UnspecifiedLength {
- tp.SetDecimal(defaultDecimal)
+// checkInvisibleIndexOnPK check if primary key is invisible index.
+// Note: PKIsHandle == true means the table already has a visible primary key,
+// we do not need do a check for this case and return directly,
+// because whether primary key is invisible has been check when creating table.
+func checkInvisibleIndexOnPK(tblInfo *model.TableInfo) error {
+ if tblInfo.PKIsHandle {
+ return nil
}
- if tp.GetFlen() == types.UnspecifiedLength {
- tp.SetFlen(defaultFlen)
- if mysql.HasUnsignedFlag(tp.GetFlag()) && tp.GetType() != mysql.TypeLonglong && mysql.IsIntegerType(tp.GetType()) {
- // Issue #4684: the flen of unsigned integer(except bigint) is 1 digit shorter than signed integer
- // because it has no prefix "+" or "-" character.
- tp.SetFlen(tp.GetFlen() - 1)
- }
- } else {
- // Adjust the field type for blob/text types if the flen is set.
- if err = adjustBlobTypesFlen(tp, colCharset); err != nil {
- return err
- }
+ pk := tblInfo.GetPrimaryKey()
+ if pk != nil && pk.Invisible {
+ return dbterror.ErrPKIndexCantBeInvisible
}
- return checkTooBigFieldLengthAndTryAutoConvert(tp, colName, sessVars)
+ return nil
}
-func decodeEnumSetBinaryLiteralToUTF8(tp *types.FieldType, chs string) {
- if tp.GetType() != mysql.TypeEnum && tp.GetType() != mysql.TypeSet {
- return
- }
- enc := charset.FindEncoding(chs)
- for i, elem := range tp.GetElems() {
- if !tp.GetElemIsBinaryLit(i) {
- continue
+// checkGlobalIndex check if the index is allowed to have global index
+func checkGlobalIndex(ctx sessionctx.Context, tblInfo *model.TableInfo, indexInfo *model.IndexInfo) error {
+ pi := tblInfo.GetPartitionInfo()
+ isPartitioned := pi != nil && pi.Type != model.PartitionTypeNone
+ if indexInfo.Global {
+ if !isPartitioned {
+ // Makes no sense with LOCAL/GLOBAL index for non-partitioned tables, since we don't support
+ // partitioning an index differently from the table partitioning.
+ return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("Global Index on non-partitioned table")
+ }
+ if !ctx.GetSessionVars().EnableGlobalIndex {
+ return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("GLOBAL IndexOption when tidb_enable_global_index is disabled")
}
- s, err := enc.Transform(nil, hack.Slice(elem), charset.OpDecodeReplace)
+ // TODO: remove limitation
+ if !indexInfo.Unique {
+ return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("GLOBAL IndexOption on non-unique index")
+ }
+ // TODO: remove limitation
+ // check that not all partitioned columns are included.
+ inAllPartitionColumns, err := checkPartitionKeysConstraint(pi, indexInfo.Columns, tblInfo)
if err != nil {
- logutil.DDLLogger().Warn("decode enum binary literal to utf-8 failed", zap.Error(err))
+ return errors.Trace(err)
+ }
+ if inAllPartitionColumns {
+ return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("Global Index including all columns in the partitioning expression")
}
- tp.SetElem(i, string(hack.String(s)))
}
- tp.CleanElemIsBinaryLit()
+ return nil
}
-// buildColumnAndConstraint builds table.Column and ast.Constraint from the parameters.
-// outPriKeyConstraint is the primary key constraint out of column definition. For example:
-// `create table t1 (id int , age int, primary key(id));`
-func buildColumnAndConstraint(
- ctx sessionctx.Context,
- offset int,
- colDef *ast.ColumnDef,
- outPriKeyConstraint *ast.Constraint,
- tblCharset string,
- tblCollate string,
-) (*table.Column, []*ast.Constraint, error) {
- if colName := colDef.Name.Name.L; colName == model.ExtraHandleName.L {
- return nil, nil, dbterror.ErrWrongColumnName.GenWithStackByArgs(colName)
+// checkGlobalIndexes check if global index is supported.
+func checkGlobalIndexes(ctx sessionctx.Context, tblInfo *model.TableInfo) error {
+ for _, indexInfo := range tblInfo.Indices {
+ err := checkGlobalIndex(ctx, tblInfo, indexInfo)
+ if err != nil {
+ return err
+ }
}
+ return nil
+}
- // specifiedCollate refers to the last collate specified in colDef.Options.
- chs, coll, err := getCharsetAndCollateInColumnDef(ctx.GetSessionVars(), colDef)
+func (e *executor) assignPartitionIDs(defs []model.PartitionDefinition) error {
+ genIDs, err := e.genGlobalIDs(len(defs))
if err != nil {
- return nil, nil, errors.Trace(err)
+ return errors.Trace(err)
}
- chs, coll, err = ResolveCharsetCollation(ctx.GetSessionVars(),
- ast.CharsetOpt{Chs: chs, Col: coll},
- ast.CharsetOpt{Chs: tblCharset, Col: tblCollate},
- )
- chs, coll = OverwriteCollationWithBinaryFlag(ctx.GetSessionVars(), colDef, chs, coll)
- if err != nil {
- return nil, nil, errors.Trace(err)
+ for i := range defs {
+ defs[i].ID = genIDs[i]
}
+ return nil
+}
- if err := setCharsetCollationFlenDecimal(colDef.Tp, colDef.Name.Name.O, chs, coll, ctx.GetSessionVars()); err != nil {
- return nil, nil, errors.Trace(err)
- }
- decodeEnumSetBinaryLiteralToUTF8(colDef.Tp, chs)
- col, cts, err := columnDefToCol(ctx, offset, colDef, outPriKeyConstraint)
- if err != nil {
- return nil, nil, errors.Trace(err)
+func (e *executor) CreateTable(ctx sessionctx.Context, s *ast.CreateTableStmt) (err error) {
+ ident := ast.Ident{Schema: s.Table.Schema, Name: s.Table.Name}
+ is := e.infoCache.GetLatest()
+ schema, ok := is.SchemaByName(ident.Schema)
+ if !ok {
+ return infoschema.ErrDatabaseNotExists.GenWithStackByArgs(ident.Schema)
}
- return col, cts, nil
-}
-// checkColumnDefaultValue checks the default value of the column.
-// In non-strict SQL mode, if the default value of the column is an empty string, the default value can be ignored.
-// In strict SQL mode, TEXT/BLOB/JSON can't have not null default values.
-// In NO_ZERO_DATE SQL mode, TIMESTAMP/DATE/DATETIME type can't have zero date like '0000-00-00' or '0000-00-00 00:00:00'.
-func checkColumnDefaultValue(ctx exprctx.BuildContext, col *table.Column, value any) (bool, any, error) {
- hasDefaultValue := true
- if value != nil && (col.GetType() == mysql.TypeJSON ||
- col.GetType() == mysql.TypeTinyBlob || col.GetType() == mysql.TypeMediumBlob ||
- col.GetType() == mysql.TypeLongBlob || col.GetType() == mysql.TypeBlob) {
- // In non-strict SQL mode.
- if !ctx.GetEvalCtx().SQLMode().HasStrictMode() && value == "" {
- if col.GetType() == mysql.TypeBlob || col.GetType() == mysql.TypeLongBlob {
- // The TEXT/BLOB default value can be ignored.
- hasDefaultValue = false
- }
- // In non-strict SQL mode, if the column type is json and the default value is null, it is initialized to an empty array.
- if col.GetType() == mysql.TypeJSON {
- value = `null`
- }
- ctx.GetEvalCtx().AppendWarning(dbterror.ErrBlobCantHaveDefault.FastGenByArgs(col.Name.O))
- return hasDefaultValue, value, nil
+ var (
+ referTbl table.Table
+ involvingRef []model.InvolvingSchemaInfo
+ )
+ if s.ReferTable != nil {
+ referIdent := ast.Ident{Schema: s.ReferTable.Schema, Name: s.ReferTable.Name}
+ _, ok := is.SchemaByName(referIdent.Schema)
+ if !ok {
+ return infoschema.ErrTableNotExists.GenWithStackByArgs(referIdent.Schema, referIdent.Name)
}
- // In strict SQL mode or default value is not an empty string.
- return hasDefaultValue, value, dbterror.ErrBlobCantHaveDefault.GenWithStackByArgs(col.Name.O)
- }
- if value != nil && ctx.GetEvalCtx().SQLMode().HasNoZeroDateMode() &&
- ctx.GetEvalCtx().SQLMode().HasStrictMode() && types.IsTypeTime(col.GetType()) {
- if vv, ok := value.(string); ok {
- timeValue, err := expression.GetTimeValue(ctx, vv, col.GetType(), col.GetDecimal(), nil)
- if err != nil {
- return hasDefaultValue, value, errors.Trace(err)
- }
- if timeValue.GetMysqlTime().CoreTime() == types.ZeroCoreTime {
- return hasDefaultValue, value, types.ErrInvalidDefault.GenWithStackByArgs(col.Name.O)
- }
+ referTbl, err = is.TableByName(e.ctx, referIdent.Schema, referIdent.Name)
+ if err != nil {
+ return infoschema.ErrTableNotExists.GenWithStackByArgs(referIdent.Schema, referIdent.Name)
}
+ involvingRef = append(involvingRef, model.InvolvingSchemaInfo{
+ Database: s.ReferTable.Schema.L,
+ Table: s.ReferTable.Name.L,
+ Mode: model.SharedInvolving,
+ })
}
- return hasDefaultValue, value, nil
-}
-
-func checkSequenceDefaultValue(col *table.Column) error {
- if mysql.IsIntegerType(col.GetType()) {
- return nil
- }
- return dbterror.ErrColumnTypeUnsupportedNextValue.GenWithStackByArgs(col.ColumnInfo.Name.O)
-}
-func convertTimestampDefaultValToUTC(ctx sessionctx.Context, defaultVal any, col *table.Column) (any, error) {
- if defaultVal == nil || col.GetType() != mysql.TypeTimestamp {
- return defaultVal, nil
+ // build tableInfo
+ var tbInfo *model.TableInfo
+ if s.ReferTable != nil {
+ tbInfo, err = BuildTableInfoWithLike(ctx, ident, referTbl.Meta(), s)
+ } else {
+ tbInfo, err = BuildTableInfoWithStmt(ctx, s, schema.Charset, schema.Collate, schema.PlacementPolicyRef)
}
- if vv, ok := defaultVal.(string); ok {
- if vv != types.ZeroDatetimeStr && !strings.EqualFold(vv, ast.CurrentTimestamp) {
- t, err := types.ParseTime(ctx.GetSessionVars().StmtCtx.TypeCtx(), vv, col.GetType(), col.GetDecimal())
- if err != nil {
- return defaultVal, errors.Trace(err)
- }
- err = t.ConvertTimeZone(ctx.GetSessionVars().Location(), time.UTC)
- if err != nil {
- return defaultVal, errors.Trace(err)
- }
- defaultVal = t.String()
- }
+ if err != nil {
+ return errors.Trace(err)
}
- return defaultVal, nil
-}
-// isExplicitTimeStamp is used to check if explicit_defaults_for_timestamp is on or off.
-// Check out this link for more details.
-// https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_explicit_defaults_for_timestamp
-func isExplicitTimeStamp() bool {
- // TODO: implement the behavior as MySQL when explicit_defaults_for_timestamp = off, then this function could return false.
- return true
-}
-
-// processColumnFlags is used by columnDefToCol and processColumnOptions. It is intended to unify behaviors on `create/add` and `modify/change` statements. Check tidb#issue#19342.
-func processColumnFlags(col *table.Column) {
- if col.FieldType.EvalType().IsStringKind() {
- if col.GetCharset() == charset.CharsetBin {
- col.AddFlag(mysql.BinaryFlag)
- } else {
- col.DelFlag(mysql.BinaryFlag)
- }
- }
- if col.GetType() == mysql.TypeBit {
- // For BIT field, it's charset is binary but does not have binary flag.
- col.DelFlag(mysql.BinaryFlag)
- col.AddFlag(mysql.UnsignedFlag)
+ if err = checkTableInfoValidWithStmt(ctx, tbInfo, s); err != nil {
+ return err
}
- if col.GetType() == mysql.TypeYear {
- // For Year field, it's charset is binary but does not have binary flag.
- col.DelFlag(mysql.BinaryFlag)
- col.AddFlag(mysql.ZerofillFlag)
+ if err = checkTableForeignKeysValid(ctx, is, schema.Name.L, tbInfo); err != nil {
+ return err
}
- // If you specify ZEROFILL for a numeric column, MySQL automatically adds the UNSIGNED attribute to the column.
- // See https://dev.mysql.com/doc/refman/5.7/en/numeric-type-overview.html for more details.
- // But some types like bit and year, won't show its unsigned flag in `show create table`.
- if mysql.HasZerofillFlag(col.GetFlag()) {
- col.AddFlag(mysql.UnsignedFlag)
+ onExist := OnExistError
+ if s.IfNotExists {
+ onExist = OnExistIgnore
}
-}
-func adjustBlobTypesFlen(tp *types.FieldType, colCharset string) error {
- cs, err := charset.GetCharsetInfo(colCharset)
- // when we meet the unsupported charset, we do not adjust.
- if err != nil {
- return err
- }
- l := tp.GetFlen() * cs.Maxlen
- if tp.GetType() == mysql.TypeBlob {
- if l <= tinyBlobMaxLength {
- logutil.DDLLogger().Info(fmt.Sprintf("Automatically convert BLOB(%d) to TINYBLOB", tp.GetFlen()))
- tp.SetFlen(tinyBlobMaxLength)
- tp.SetType(mysql.TypeTinyBlob)
- } else if l <= blobMaxLength {
- tp.SetFlen(blobMaxLength)
- } else if l <= mediumBlobMaxLength {
- logutil.DDLLogger().Info(fmt.Sprintf("Automatically convert BLOB(%d) to MEDIUMBLOB", tp.GetFlen()))
- tp.SetFlen(mediumBlobMaxLength)
- tp.SetType(mysql.TypeMediumBlob)
- } else if l <= longBlobMaxLength {
- logutil.DDLLogger().Info(fmt.Sprintf("Automatically convert BLOB(%d) to LONGBLOB", tp.GetFlen()))
- tp.SetFlen(longBlobMaxLength)
- tp.SetType(mysql.TypeLongBlob)
- }
- }
- return nil
+ return e.CreateTableWithInfo(ctx, schema.Name, tbInfo, involvingRef, WithOnExist(onExist))
}
-// columnDefToCol converts ColumnDef to Col and TableConstraints.
-// outPriKeyConstraint is the primary key constraint out of column definition. such as: create table t1 (id int , age int, primary key(id));
-func columnDefToCol(ctx sessionctx.Context, offset int, colDef *ast.ColumnDef, outPriKeyConstraint *ast.Constraint) (*table.Column, []*ast.Constraint, error) {
- var constraints = make([]*ast.Constraint, 0)
- col := table.ToColumn(&model.ColumnInfo{
- Offset: offset,
- Name: colDef.Name.Name,
- FieldType: *colDef.Tp,
- // TODO: remove this version field after there is no old version.
- Version: model.CurrLatestColumnInfoVersion,
- })
-
- if !isExplicitTimeStamp() {
- // Check and set TimestampFlag, OnUpdateNowFlag and NotNullFlag.
- if col.GetType() == mysql.TypeTimestamp {
- col.AddFlag(mysql.TimestampFlag | mysql.OnUpdateNowFlag | mysql.NotNullFlag)
- }
+// createTableWithInfoJob returns the table creation job.
+// WARNING: it may return a nil job, which means you don't need to submit any DDL job.
+func (e *executor) createTableWithInfoJob(
+ ctx sessionctx.Context,
+ dbName model.CIStr,
+ tbInfo *model.TableInfo,
+ involvingRef []model.InvolvingSchemaInfo,
+ onExist OnExist,
+) (job *model.Job, err error) {
+ is := e.infoCache.GetLatest()
+ schema, ok := is.SchemaByName(dbName)
+ if !ok {
+ return nil, infoschema.ErrDatabaseNotExists.GenWithStackByArgs(dbName)
}
- var err error
- setOnUpdateNow := false
- hasDefaultValue := false
- hasNullFlag := false
- if colDef.Options != nil {
- length := types.UnspecifiedLength
- keys := []*ast.IndexPartSpecification{
- {
- Column: colDef.Name,
- Length: length,
- },
- }
+ if err = handleTablePlacement(ctx, tbInfo); err != nil {
+ return nil, errors.Trace(err)
+ }
- var sb strings.Builder
- restoreFlags := format.RestoreStringSingleQuotes | format.RestoreKeyWordLowercase | format.RestoreNameBackQuotes |
- format.RestoreSpacesAroundBinaryOperation | format.RestoreWithoutSchemaName | format.RestoreWithoutTableName
- restoreCtx := format.NewRestoreCtx(restoreFlags, &sb)
-
- for _, v := range colDef.Options {
- switch v.Tp {
- case ast.ColumnOptionNotNull:
- col.AddFlag(mysql.NotNullFlag)
- case ast.ColumnOptionNull:
- col.DelFlag(mysql.NotNullFlag)
- removeOnUpdateNowFlag(col)
- hasNullFlag = true
- case ast.ColumnOptionAutoIncrement:
- col.AddFlag(mysql.AutoIncrementFlag | mysql.NotNullFlag)
- case ast.ColumnOptionPrimaryKey:
- // Check PriKeyFlag first to avoid extra duplicate constraints.
- if col.GetFlag()&mysql.PriKeyFlag == 0 {
- constraint := &ast.Constraint{Tp: ast.ConstraintPrimaryKey, Keys: keys,
- Option: &ast.IndexOption{PrimaryKeyTp: v.PrimaryKeyTp}}
- constraints = append(constraints, constraint)
- col.AddFlag(mysql.PriKeyFlag)
- // Add NotNullFlag early so that processColumnFlags() can see it.
- col.AddFlag(mysql.NotNullFlag)
- }
- case ast.ColumnOptionUniqKey:
- // Check UniqueFlag first to avoid extra duplicate constraints.
- if col.GetFlag()&mysql.UniqueFlag == 0 {
- constraint := &ast.Constraint{Tp: ast.ConstraintUniqKey, Keys: keys}
- constraints = append(constraints, constraint)
- col.AddFlag(mysql.UniqueKeyFlag)
- }
- case ast.ColumnOptionDefaultValue:
- hasDefaultValue, err = SetDefaultValue(ctx, col, v)
- if err != nil {
- return nil, nil, errors.Trace(err)
- }
- removeOnUpdateNowFlag(col)
- case ast.ColumnOptionOnUpdate:
- // TODO: Support other time functions.
- if !(col.GetType() == mysql.TypeTimestamp || col.GetType() == mysql.TypeDatetime) {
- return nil, nil, dbterror.ErrInvalidOnUpdate.GenWithStackByArgs(col.Name)
- }
- if !expression.IsValidCurrentTimestampExpr(v.Expr, colDef.Tp) {
- return nil, nil, dbterror.ErrInvalidOnUpdate.GenWithStackByArgs(col.Name)
- }
- col.AddFlag(mysql.OnUpdateNowFlag)
- setOnUpdateNow = true
- case ast.ColumnOptionComment:
- err := setColumnComment(ctx, col, v)
- if err != nil {
- return nil, nil, errors.Trace(err)
- }
- case ast.ColumnOptionGenerated:
- sb.Reset()
- err = v.Expr.Restore(restoreCtx)
- if err != nil {
- return nil, nil, errors.Trace(err)
- }
- col.GeneratedExprString = sb.String()
- col.GeneratedStored = v.Stored
- _, dependColNames, err := findDependedColumnNames(model.NewCIStr(""), model.NewCIStr(""), colDef)
- if err != nil {
- return nil, nil, errors.Trace(err)
- }
- col.Dependences = dependColNames
- case ast.ColumnOptionCollate:
- if field_types.HasCharset(colDef.Tp) {
- col.FieldType.SetCollate(v.StrValue)
- }
- case ast.ColumnOptionFulltext:
- ctx.GetSessionVars().StmtCtx.AppendWarning(dbterror.ErrTableCantHandleFt.FastGenByArgs())
- case ast.ColumnOptionCheck:
- if !variable.EnableCheckConstraint.Load() {
- ctx.GetSessionVars().StmtCtx.AppendWarning(errCheckConstraintIsOff)
- } else {
- // Check the column CHECK constraint dependency lazily, after fill all the name.
- // Extract column constraint from column option.
- constraint := &ast.Constraint{
- Tp: ast.ConstraintCheck,
- Expr: v.Expr,
- Enforced: v.Enforced,
- Name: v.ConstraintName,
- InColumn: true,
- InColumnName: colDef.Name.Name.O,
- }
- constraints = append(constraints, constraint)
+ var oldViewTblID int64
+ if oldTable, err := is.TableByName(e.ctx, schema.Name, tbInfo.Name); err == nil {
+ err = infoschema.ErrTableExists.GenWithStackByArgs(ast.Ident{Schema: schema.Name, Name: tbInfo.Name})
+ switch onExist {
+ case OnExistIgnore:
+ ctx.GetSessionVars().StmtCtx.AppendNote(err)
+ return nil, nil
+ case OnExistReplace:
+ // only CREATE OR REPLACE VIEW is supported at the moment.
+ if tbInfo.View != nil {
+ if oldTable.Meta().IsView() {
+ oldViewTblID = oldTable.Meta().ID
+ break
}
+ // The object to replace isn't a view.
+ return nil, dbterror.ErrWrongObject.GenWithStackByArgs(dbName, tbInfo.Name, "VIEW")
}
+ return nil, err
+ default:
+ return nil, err
}
}
- if err = processAndCheckDefaultValueAndColumn(ctx, col, outPriKeyConstraint, hasDefaultValue, setOnUpdateNow, hasNullFlag); err != nil {
- return nil, nil, errors.Trace(err)
+ if err := checkTableInfoValidExtra(ctx, tbInfo); err != nil {
+ return nil, err
}
- return col, constraints, nil
-}
-func restoreFuncCall(expr *ast.FuncCallExpr) (string, error) {
- var sb strings.Builder
- restoreFlags := format.RestoreStringSingleQuotes | format.RestoreKeyWordLowercase | format.RestoreNameBackQuotes |
- format.RestoreSpacesAroundBinaryOperation
- restoreCtx := format.NewRestoreCtx(restoreFlags, &sb)
- if err := expr.Restore(restoreCtx); err != nil {
- return "", err
+ var actionType model.ActionType
+ args := []any{tbInfo}
+ switch {
+ case tbInfo.View != nil:
+ actionType = model.ActionCreateView
+ args = append(args, onExist == OnExistReplace, oldViewTblID)
+ case tbInfo.Sequence != nil:
+ actionType = model.ActionCreateSequence
+ default:
+ actionType = model.ActionCreateTable
+ args = append(args, ctx.GetSessionVars().ForeignKeyChecks)
}
- return sb.String(), nil
-}
-// getFuncCallDefaultValue gets the default column value of function-call expression.
-func getFuncCallDefaultValue(col *table.Column, option *ast.ColumnOption, expr *ast.FuncCallExpr) (any, bool, error) {
- switch expr.FnName.L {
- case ast.CurrentTimestamp, ast.CurrentDate: // CURRENT_TIMESTAMP() and CURRENT_DATE()
- tp, fsp := col.FieldType.GetType(), col.FieldType.GetDecimal()
- if tp == mysql.TypeTimestamp || tp == mysql.TypeDatetime {
- defaultFsp := 0
- if len(expr.Args) == 1 {
- if val := expr.Args[0].(*driver.ValueExpr); val != nil {
- defaultFsp = int(val.GetInt64())
- }
- }
- if defaultFsp != fsp {
- return nil, false, dbterror.ErrInvalidDefaultValue.GenWithStackByArgs(col.Name.O)
- }
- }
- return nil, false, nil
- case ast.NextVal:
- // handle default next value of sequence. (keep the expr string)
- str, err := getSequenceDefaultValue(option)
- if err != nil {
- return nil, false, errors.Trace(err)
- }
- return str, true, nil
- case ast.Rand, ast.UUID, ast.UUIDToBin: // RAND(), UUID() and UUID_TO_BIN()
- if err := expression.VerifyArgsWrapper(expr.FnName.L, len(expr.Args)); err != nil {
- return nil, false, errors.Trace(err)
- }
- str, err := restoreFuncCall(expr)
- if err != nil {
- return nil, false, errors.Trace(err)
- }
- col.DefaultIsExpr = true
- return str, false, nil
- case ast.DateFormat: // DATE_FORMAT()
- if err := expression.VerifyArgsWrapper(expr.FnName.L, len(expr.Args)); err != nil {
- return nil, false, errors.Trace(err)
- }
- // Support DATE_FORMAT(NOW(),'%Y-%m'), DATE_FORMAT(NOW(),'%Y-%m-%d'),
- // DATE_FORMAT(NOW(),'%Y-%m-%d %H.%i.%s'), DATE_FORMAT(NOW(),'%Y-%m-%d %H:%i:%s').
- nowFunc, ok := expr.Args[0].(*ast.FuncCallExpr)
- if ok && nowFunc.FnName.L == ast.Now {
- if err := expression.VerifyArgsWrapper(nowFunc.FnName.L, len(nowFunc.Args)); err != nil {
- return nil, false, errors.Trace(err)
- }
- valExpr, isValue := expr.Args[1].(ast.ValueExpr)
- if !isValue || (valExpr.GetString() != "%Y-%m" && valExpr.GetString() != "%Y-%m-%d" &&
- valExpr.GetString() != "%Y-%m-%d %H.%i.%s" && valExpr.GetString() != "%Y-%m-%d %H:%i:%s") {
- return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(), valExpr)
- }
- str, err := restoreFuncCall(expr)
- if err != nil {
- return nil, false, errors.Trace(err)
- }
- col.DefaultIsExpr = true
- return str, false, nil
- }
- return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(),
- fmt.Sprintf("%s with disallowed args", expr.FnName.String()))
- case ast.Replace:
- if err := expression.VerifyArgsWrapper(expr.FnName.L, len(expr.Args)); err != nil {
- return nil, false, errors.Trace(err)
- }
- funcCall := expr.Args[0]
- // Support REPLACE(CONVERT(UPPER(UUID()) USING UTF8MB4), '-', ''))
- if convertFunc, ok := funcCall.(*ast.FuncCallExpr); ok && convertFunc.FnName.L == ast.Convert {
- if err := expression.VerifyArgsWrapper(convertFunc.FnName.L, len(convertFunc.Args)); err != nil {
- return nil, false, errors.Trace(err)
- }
- funcCall = convertFunc.Args[0]
- }
- // Support REPLACE(UPPER(UUID()), '-', '').
- if upperFunc, ok := funcCall.(*ast.FuncCallExpr); ok && upperFunc.FnName.L == ast.Upper {
- if err := expression.VerifyArgsWrapper(upperFunc.FnName.L, len(upperFunc.Args)); err != nil {
- return nil, false, errors.Trace(err)
- }
- if uuidFunc, ok := upperFunc.Args[0].(*ast.FuncCallExpr); ok && uuidFunc.FnName.L == ast.UUID {
- if err := expression.VerifyArgsWrapper(uuidFunc.FnName.L, len(uuidFunc.Args)); err != nil {
- return nil, false, errors.Trace(err)
- }
- str, err := restoreFuncCall(expr)
- if err != nil {
- return nil, false, errors.Trace(err)
- }
- col.DefaultIsExpr = true
- return str, false, nil
- }
- }
- return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(),
- fmt.Sprintf("%s with disallowed args", expr.FnName.String()))
- case ast.Upper:
- if err := expression.VerifyArgsWrapper(expr.FnName.L, len(expr.Args)); err != nil {
- return nil, false, errors.Trace(err)
- }
- // Support UPPER(SUBSTRING_INDEX(USER(), '@', 1)).
- if substringIndexFunc, ok := expr.Args[0].(*ast.FuncCallExpr); ok && substringIndexFunc.FnName.L == ast.SubstringIndex {
- if err := expression.VerifyArgsWrapper(substringIndexFunc.FnName.L, len(substringIndexFunc.Args)); err != nil {
- return nil, false, errors.Trace(err)
- }
- if userFunc, ok := substringIndexFunc.Args[0].(*ast.FuncCallExpr); ok && userFunc.FnName.L == ast.User {
- if err := expression.VerifyArgsWrapper(userFunc.FnName.L, len(userFunc.Args)); err != nil {
- return nil, false, errors.Trace(err)
- }
- valExpr, isValue := substringIndexFunc.Args[1].(ast.ValueExpr)
- if !isValue || valExpr.GetString() != "@" {
- return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(), valExpr)
- }
- str, err := restoreFuncCall(expr)
- if err != nil {
- return nil, false, errors.Trace(err)
- }
- col.DefaultIsExpr = true
- return str, false, nil
- }
- }
- return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(),
- fmt.Sprintf("%s with disallowed args", expr.FnName.String()))
- case ast.StrToDate: // STR_TO_DATE()
- if err := expression.VerifyArgsWrapper(expr.FnName.L, len(expr.Args)); err != nil {
- return nil, false, errors.Trace(err)
- }
- // Support STR_TO_DATE('1980-01-01', '%Y-%m-%d').
- if _, ok1 := expr.Args[0].(ast.ValueExpr); ok1 {
- if _, ok2 := expr.Args[1].(ast.ValueExpr); ok2 {
- str, err := restoreFuncCall(expr)
- if err != nil {
- return nil, false, errors.Trace(err)
- }
- col.DefaultIsExpr = true
- return str, false, nil
- }
- }
- return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(),
- fmt.Sprintf("%s with disallowed args", expr.FnName.String()))
- case ast.JSONObject, ast.JSONArray, ast.JSONQuote: // JSON_OBJECT(), JSON_ARRAY(), JSON_QUOTE()
- if err := expression.VerifyArgsWrapper(expr.FnName.L, len(expr.Args)); err != nil {
- return nil, false, errors.Trace(err)
- }
- str, err := restoreFuncCall(expr)
- if err != nil {
- return nil, false, errors.Trace(err)
- }
- col.DefaultIsExpr = true
- return str, false, nil
+ var involvingSchemas []model.InvolvingSchemaInfo
+ sharedInvolvingFromTableInfo := getSharedInvolvingSchemaInfo(tbInfo)
- default:
- return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(), expr.FnName.String())
+ if sum := len(involvingRef) + len(sharedInvolvingFromTableInfo); sum > 0 {
+ involvingSchemas = make([]model.InvolvingSchemaInfo, 0, sum+1)
+ involvingSchemas = append(involvingSchemas, model.InvolvingSchemaInfo{
+ Database: schema.Name.L,
+ Table: tbInfo.Name.L,
+ })
+ involvingSchemas = append(involvingSchemas, involvingRef...)
+ involvingSchemas = append(involvingSchemas, sharedInvolvingFromTableInfo...)
}
-}
-// getDefaultValue will get the default value for column.
-// 1: get the expr restored string for the column which uses sequence next value as default value.
-// 2: get specific default value for the other column.
-func getDefaultValue(ctx exprctx.BuildContext, col *table.Column, option *ast.ColumnOption) (any, bool, error) {
- // handle default value with function call
- tp, fsp := col.FieldType.GetType(), col.FieldType.GetDecimal()
- if x, ok := option.Expr.(*ast.FuncCallExpr); ok {
- val, isSeqExpr, err := getFuncCallDefaultValue(col, option, x)
- if val != nil || isSeqExpr || err != nil {
- return val, isSeqExpr, err
- }
- // If the function call is ast.CurrentTimestamp, it needs to be continuously processed.
+ job = &model.Job{
+ SchemaID: schema.ID,
+ SchemaName: schema.Name.L,
+ TableName: tbInfo.Name.L,
+ Type: actionType,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: args,
+ CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
+ InvolvingSchemaInfo: involvingSchemas,
+ SQLMode: ctx.GetSessionVars().SQLMode,
}
+ return job, nil
+}
- if tp == mysql.TypeTimestamp || tp == mysql.TypeDatetime || tp == mysql.TypeDate {
- vd, err := expression.GetTimeValue(ctx, option.Expr, tp, fsp, nil)
- value := vd.GetValue()
- if err != nil {
- return nil, false, dbterror.ErrInvalidDefaultValue.GenWithStackByArgs(col.Name.O)
- }
-
- // Value is nil means `default null`.
- if value == nil {
- return nil, false, nil
- }
-
- // If value is types.Time, convert it to string.
- if vv, ok := value.(types.Time); ok {
- return vv.String(), false, nil
- }
-
- return value, false, nil
+func getSharedInvolvingSchemaInfo(info *model.TableInfo) []model.InvolvingSchemaInfo {
+ ret := make([]model.InvolvingSchemaInfo, 0, len(info.ForeignKeys)+1)
+ for _, fk := range info.ForeignKeys {
+ ret = append(ret, model.InvolvingSchemaInfo{
+ Database: fk.RefSchema.L,
+ Table: fk.RefTable.L,
+ Mode: model.SharedInvolving,
+ })
}
-
- // evaluate the non-function-call expr to a certain value.
- v, err := expression.EvalSimpleAst(ctx, option.Expr)
- if err != nil {
- return nil, false, errors.Trace(err)
+ if ref := info.PlacementPolicyRef; ref != nil {
+ ret = append(ret, model.InvolvingSchemaInfo{
+ Policy: ref.Name.L,
+ Mode: model.SharedInvolving,
+ })
}
+ return ret
+}
- if v.IsNull() {
- return nil, false, nil
+func (e *executor) createTableWithInfoPost(
+ ctx sessionctx.Context,
+ tbInfo *model.TableInfo,
+ schemaID int64,
+) error {
+ var err error
+ var partitions []model.PartitionDefinition
+ if pi := tbInfo.GetPartitionInfo(); pi != nil {
+ partitions = pi.Definitions
}
-
- if v.Kind() == types.KindBinaryLiteral || v.Kind() == types.KindMysqlBit {
- if types.IsTypeBlob(tp) || tp == mysql.TypeJSON || tp == mysql.TypeTiDBVectorFloat32 {
- // BLOB/TEXT/JSON column cannot have a default value.
- // Skip the unnecessary decode procedure.
- return v.GetString(), false, err
+ preSplitAndScatter(ctx, e.store, tbInfo, partitions)
+ if tbInfo.AutoIncID > 1 {
+ // Default tableAutoIncID base is 0.
+ // If the first ID is expected to greater than 1, we need to do rebase.
+ newEnd := tbInfo.AutoIncID - 1
+ var allocType autoid.AllocatorType
+ if tbInfo.SepAutoInc() {
+ allocType = autoid.AutoIncrementType
+ } else {
+ allocType = autoid.RowIDAllocType
}
- if tp == mysql.TypeBit || tp == mysql.TypeString || tp == mysql.TypeVarchar ||
- tp == mysql.TypeVarString || tp == mysql.TypeEnum || tp == mysql.TypeSet {
- // For BinaryLiteral or bit fields, we decode the default value to utf8 string.
- str, err := v.GetBinaryStringDecoded(types.StrictFlags, col.GetCharset())
- if err != nil {
- // Overwrite the decoding error with invalid default value error.
- err = dbterror.ErrInvalidDefaultValue.GenWithStackByArgs(col.Name.O)
- }
- return str, false, err
+ if err = e.handleAutoIncID(tbInfo, schemaID, newEnd, allocType); err != nil {
+ return errors.Trace(err)
}
- // For other kind of fields (e.g. INT), we supply its integer as string value.
- value, err := v.GetBinaryLiteral().ToInt(ctx.GetEvalCtx().TypeCtx())
- if err != nil {
- return nil, false, err
+ }
+ // For issue https://github.com/pingcap/tidb/issues/46093
+ if tbInfo.AutoIncIDExtra != 0 {
+ if err = e.handleAutoIncID(tbInfo, schemaID, tbInfo.AutoIncIDExtra-1, autoid.RowIDAllocType); err != nil {
+ return errors.Trace(err)
}
- return strconv.FormatUint(value, 10), false, nil
}
-
- switch tp {
- case mysql.TypeSet:
- val, err := getSetDefaultValue(v, col)
- return val, false, err
- case mysql.TypeEnum:
- val, err := getEnumDefaultValue(v, col)
- return val, false, err
- case mysql.TypeDuration, mysql.TypeDate:
- if v, err = v.ConvertTo(ctx.GetEvalCtx().TypeCtx(), &col.FieldType); err != nil {
- return "", false, errors.Trace(err)
- }
- case mysql.TypeBit:
- if v.Kind() == types.KindInt64 || v.Kind() == types.KindUint64 {
- // For BIT fields, convert int into BinaryLiteral.
- return types.NewBinaryLiteralFromUint(v.GetUint64(), -1).ToString(), false, nil
- }
- case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong, mysql.TypeFloat, mysql.TypeDouble:
- // For these types, convert it to standard format firstly.
- // like integer fields, convert it into integer string literals. like convert "1.25" into "1" and "2.8" into "3".
- // if raise a error, we will use original expression. We will handle it in check phase
- if temp, err := v.ConvertTo(ctx.GetEvalCtx().TypeCtx(), &col.FieldType); err == nil {
- v = temp
- }
- }
-
- val, err := v.ToString()
- return val, false, err
-}
-
-func getSequenceDefaultValue(c *ast.ColumnOption) (expr string, err error) {
- var sb strings.Builder
- restoreFlags := format.RestoreStringSingleQuotes | format.RestoreKeyWordLowercase | format.RestoreNameBackQuotes |
- format.RestoreSpacesAroundBinaryOperation
- restoreCtx := format.NewRestoreCtx(restoreFlags, &sb)
- if err := c.Expr.Restore(restoreCtx); err != nil {
- return "", err
+ if tbInfo.AutoRandID > 1 {
+ // Default tableAutoRandID base is 0.
+ // If the first ID is expected to greater than 1, we need to do rebase.
+ newEnd := tbInfo.AutoRandID - 1
+ err = e.handleAutoIncID(tbInfo, schemaID, newEnd, autoid.AutoRandomType)
}
- return sb.String(), nil
+ return err
}
-// getSetDefaultValue gets the default value for the set type. See https://dev.mysql.com/doc/refman/5.7/en/set.html.
-func getSetDefaultValue(v types.Datum, col *table.Column) (string, error) {
- if v.Kind() == types.KindInt64 {
- setCnt := len(col.GetElems())
- maxLimit := int64(1< maxLimit {
- return "", dbterror.ErrInvalidDefaultValue.GenWithStackByArgs(col.Name.O)
- }
- setVal, err := types.ParseSetValue(col.GetElems(), uint64(val))
- if err != nil {
- return "", errors.Trace(err)
- }
- v.SetMysqlSet(setVal, col.GetCollate())
- return v.ToString()
- }
+func (e *executor) CreateTableWithInfo(
+ ctx sessionctx.Context,
+ dbName model.CIStr,
+ tbInfo *model.TableInfo,
+ involvingRef []model.InvolvingSchemaInfo,
+ cs ...CreateTableOption,
+) (err error) {
+ c := GetCreateTableConfig(cs)
- str, err := v.ToString()
+ job, err := e.createTableWithInfoJob(
+ ctx, dbName, tbInfo, involvingRef, c.OnExist,
+ )
if err != nil {
- return "", errors.Trace(err)
- }
- if str == "" {
- return str, nil
+ return err
}
- setVal, err := types.ParseSetName(col.GetElems(), str, col.GetCollate())
- if err != nil {
- return "", dbterror.ErrInvalidDefaultValue.GenWithStackByArgs(col.Name.O)
+ if job == nil {
+ return nil
}
- v.SetMysqlSet(setVal, col.GetCollate())
- return v.ToString()
-}
+ jobW := NewJobWrapper(job, c.IDAllocated)
-// getEnumDefaultValue gets the default value for the enum type. See https://dev.mysql.com/doc/refman/5.7/en/enum.html.
-func getEnumDefaultValue(v types.Datum, col *table.Column) (string, error) {
- if v.Kind() == types.KindInt64 {
- val := v.GetInt64()
- if val < 1 || val > int64(len(col.GetElems())) {
- return "", dbterror.ErrInvalidDefaultValue.GenWithStackByArgs(col.Name.O)
- }
- enumVal, err := types.ParseEnumValue(col.GetElems(), uint64(val))
- if err != nil {
- return "", errors.Trace(err)
- }
- v.SetMysqlEnum(enumVal, col.GetCollate())
- return v.ToString()
- }
- str, err := v.ToString()
- if err != nil {
- return "", errors.Trace(err)
- }
- // Ref: https://dev.mysql.com/doc/refman/8.0/en/enum.html
- // Trailing spaces are automatically deleted from ENUM member values in the table definition when a table is created.
- str = strings.TrimRight(str, " ")
- enumVal, err := types.ParseEnumName(col.GetElems(), str, col.GetCollate())
+ err = e.DoDDLJobWrapper(ctx, jobW)
if err != nil {
- return "", dbterror.ErrInvalidDefaultValue.GenWithStackByArgs(col.Name.O)
+ // table exists, but if_not_exists flags is true, so we ignore this error.
+ if c.OnExist == OnExistIgnore && infoschema.ErrTableExists.Equal(err) {
+ ctx.GetSessionVars().StmtCtx.AppendNote(err)
+ err = nil
+ }
+ } else {
+ err = e.createTableWithInfoPost(ctx, tbInfo, job.SchemaID)
}
- v.SetMysqlEnum(enumVal, col.GetCollate())
- return v.ToString()
+ return errors.Trace(err)
}
-func removeOnUpdateNowFlag(c *table.Column) {
- // For timestamp Col, if it is set null or default value,
- // OnUpdateNowFlag should be removed.
- if mysql.HasTimestampFlag(c.GetFlag()) {
- c.DelFlag(mysql.OnUpdateNowFlag)
- }
-}
+func (e *executor) BatchCreateTableWithInfo(ctx sessionctx.Context,
+ dbName model.CIStr,
+ infos []*model.TableInfo,
+ cs ...CreateTableOption,
+) error {
+ failpoint.Inject("RestoreBatchCreateTableEntryTooLarge", func(val failpoint.Value) {
+ injectBatchSize := val.(int)
+ if len(infos) > injectBatchSize {
+ failpoint.Return(kv.ErrEntryTooLarge)
+ }
+ })
+ c := GetCreateTableConfig(cs)
-func processDefaultValue(c *table.Column, hasDefaultValue bool, setOnUpdateNow bool) {
- setTimestampDefaultValue(c, hasDefaultValue, setOnUpdateNow)
+ jobW := NewJobWrapper(
+ &model.Job{
+ BinlogInfo: &model.HistoryInfo{},
+ CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
+ SQLMode: ctx.GetSessionVars().SQLMode,
+ },
+ c.IDAllocated,
+ )
+ args := make([]*model.TableInfo, 0, len(infos))
- setYearDefaultValue(c, hasDefaultValue)
+ var err error
- // Set `NoDefaultValueFlag` if this field doesn't have a default value and
- // it is `not null` and not an `AUTO_INCREMENT` field or `TIMESTAMP` field.
- setNoDefaultValueFlag(c, hasDefaultValue)
-}
+ // check if there are any duplicated table names
+ duplication := make(map[string]struct{})
+ // TODO filter those duplicated info out.
+ for _, info := range infos {
+ if _, ok := duplication[info.Name.L]; ok {
+ err = infoschema.ErrTableExists.FastGenByArgs("can not batch create tables with same name")
+ if c.OnExist == OnExistIgnore && infoschema.ErrTableExists.Equal(err) {
+ ctx.GetSessionVars().StmtCtx.AppendNote(err)
+ err = nil
+ }
+ }
+ if err != nil {
+ return errors.Trace(err)
+ }
-func setYearDefaultValue(c *table.Column, hasDefaultValue bool) {
- if hasDefaultValue {
- return
+ duplication[info.Name.L] = struct{}{}
}
- if c.GetType() == mysql.TypeYear && mysql.HasNotNullFlag(c.GetFlag()) {
- if err := c.SetDefaultValue("0000"); err != nil {
- logutil.DDLLogger().Error("set default value failed", zap.Error(err))
+ for _, info := range infos {
+ job, err := e.createTableWithInfoJob(ctx, dbName, info, nil, c.OnExist)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if job == nil {
+ continue
}
- }
-}
-func setTimestampDefaultValue(c *table.Column, hasDefaultValue bool, setOnUpdateNow bool) {
- if hasDefaultValue {
- return
- }
+ // if jobW.Type == model.ActionCreateTables, it is initialized
+ // if not, initialize jobW by job.XXXX
+ if jobW.Type != model.ActionCreateTables {
+ jobW.Type = model.ActionCreateTables
+ jobW.SchemaID = job.SchemaID
+ jobW.SchemaName = job.SchemaName
+ }
- // For timestamp Col, if is not set default value or not set null, use current timestamp.
- if mysql.HasTimestampFlag(c.GetFlag()) && mysql.HasNotNullFlag(c.GetFlag()) {
- if setOnUpdateNow {
- if err := c.SetDefaultValue(types.ZeroDatetimeStr); err != nil {
- logutil.DDLLogger().Error("set default value failed", zap.Error(err))
- }
- } else {
- if err := c.SetDefaultValue(strings.ToUpper(ast.CurrentTimestamp)); err != nil {
- logutil.DDLLogger().Error("set default value failed", zap.Error(err))
- }
+ // append table job args
+ info, ok := job.Args[0].(*model.TableInfo)
+ if !ok {
+ return errors.Trace(fmt.Errorf("except table info"))
+ }
+ args = append(args, info)
+ jobW.InvolvingSchemaInfo = append(jobW.InvolvingSchemaInfo, model.InvolvingSchemaInfo{
+ Database: dbName.L,
+ Table: info.Name.L,
+ })
+ if sharedInv := getSharedInvolvingSchemaInfo(info); len(sharedInv) > 0 {
+ jobW.InvolvingSchemaInfo = append(jobW.InvolvingSchemaInfo, sharedInv...)
}
}
-}
-
-func setNoDefaultValueFlag(c *table.Column, hasDefaultValue bool) {
- if hasDefaultValue {
- return
+ if len(args) == 0 {
+ return nil
}
+ jobW.Args = append(jobW.Args, args)
+ jobW.Args = append(jobW.Args, ctx.GetSessionVars().ForeignKeyChecks)
- if !mysql.HasNotNullFlag(c.GetFlag()) {
- return
+ err = e.DoDDLJobWrapper(ctx, jobW)
+ if err != nil {
+ // table exists, but if_not_exists flags is true, so we ignore this error.
+ if c.OnExist == OnExistIgnore && infoschema.ErrTableExists.Equal(err) {
+ ctx.GetSessionVars().StmtCtx.AppendNote(err)
+ err = nil
+ }
+ return errors.Trace(err)
}
- // Check if it is an `AUTO_INCREMENT` field or `TIMESTAMP` field.
- if !mysql.HasAutoIncrementFlag(c.GetFlag()) && !mysql.HasTimestampFlag(c.GetFlag()) {
- c.AddFlag(mysql.NoDefaultValueFlag)
+ for j := range args {
+ if err = e.createTableWithInfoPost(ctx, args[j], jobW.SchemaID); err != nil {
+ return errors.Trace(err)
+ }
}
-}
-func checkDefaultValue(ctx exprctx.BuildContext, c *table.Column, hasDefaultValue bool) (err error) {
- if !hasDefaultValue {
+ return nil
+}
+
+func (e *executor) CreatePlacementPolicyWithInfo(ctx sessionctx.Context, policy *model.PolicyInfo, onExist OnExist) error {
+ if checkIgnorePlacementDDL(ctx) {
return nil
}
- if c.GetDefaultValue() != nil {
- if c.DefaultIsExpr {
- if mysql.HasAutoIncrementFlag(c.GetFlag()) {
- return types.ErrInvalidDefault.GenWithStackByArgs(c.Name)
- }
+ policyName := policy.Name
+ if policyName.L == defaultPlacementPolicyName {
+ return errors.Trace(infoschema.ErrReservedSyntax.GenWithStackByArgs(policyName))
+ }
+
+ // Check policy existence.
+ _, ok := e.infoCache.GetLatest().PolicyByName(policyName)
+ if ok {
+ err := infoschema.ErrPlacementPolicyExists.GenWithStackByArgs(policyName)
+ switch onExist {
+ case OnExistIgnore:
+ ctx.GetSessionVars().StmtCtx.AppendNote(err)
return nil
+ case OnExistError:
+ return err
}
- _, err = table.GetColDefaultValue(
- exprctx.CtxWithHandleTruncateErrLevel(ctx, errctx.LevelError),
- c.ToInfo(),
- )
- if err != nil {
- return types.ErrInvalidDefault.GenWithStackByArgs(c.Name)
- }
- return nil
}
- // Primary key default null is invalid.
- if mysql.HasPriKeyFlag(c.GetFlag()) {
- return dbterror.ErrPrimaryCantHaveNull
+
+ if err := checkPolicyValidation(policy.PlacementSettings); err != nil {
+ return err
}
- // Set not null but default null is invalid.
- if mysql.HasNotNullFlag(c.GetFlag()) {
- return types.ErrInvalidDefault.GenWithStackByArgs(c.Name)
+ policyID, err := e.genPlacementPolicyID()
+ if err != nil {
+ return err
}
+ policy.ID = policyID
- return nil
+ job := &model.Job{
+ SchemaName: policy.Name.L,
+ Type: model.ActionCreatePlacementPolicy,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []any{policy, onExist == OnExistReplace},
+ InvolvingSchemaInfo: []model.InvolvingSchemaInfo{{
+ Policy: policy.Name.L,
+ }},
+ CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
+ SQLMode: ctx.GetSessionVars().SQLMode,
+ }
+ err = e.DoDDLJob(ctx, job)
+ return errors.Trace(err)
}
-// checkPriKeyConstraint check all parts of a PRIMARY KEY must be NOT NULL
-func checkPriKeyConstraint(col *table.Column, hasDefaultValue, hasNullFlag bool, outPriKeyConstraint *ast.Constraint) error {
- // Primary key should not be null.
- if mysql.HasPriKeyFlag(col.GetFlag()) && hasDefaultValue && col.GetDefaultValue() == nil {
- return types.ErrInvalidDefault.GenWithStackByArgs(col.Name)
+// preSplitAndScatter performs pre-split and scatter of the table's regions.
+// If `pi` is not nil, will only split region for `pi`, this is used when add partition.
+func preSplitAndScatter(ctx sessionctx.Context, store kv.Storage, tbInfo *model.TableInfo, parts []model.PartitionDefinition) {
+ if tbInfo.TempTableType != model.TempTableNone {
+ return
}
- // Set primary key flag for outer primary key constraint.
- // Such as: create table t1 (id int , age int, primary key(id))
- if !mysql.HasPriKeyFlag(col.GetFlag()) && outPriKeyConstraint != nil {
- for _, key := range outPriKeyConstraint.Keys {
- if key.Expr == nil && key.Column.Name.L != col.Name.L {
- continue
- }
- col.AddFlag(mysql.PriKeyFlag)
- break
- }
+ sp, ok := store.(kv.SplittableStore)
+ if !ok || atomic.LoadUint32(&EnableSplitTableRegion) == 0 {
+ return
+ }
+ var (
+ preSplit func()
+ scatterRegion bool
+ )
+ val, err := ctx.GetSessionVars().GetGlobalSystemVar(context.Background(), variable.TiDBScatterRegion)
+ if err != nil {
+ logutil.DDLLogger().Warn("won't scatter region", zap.Error(err))
+ } else {
+ scatterRegion = variable.TiDBOptOn(val)
}
- // Primary key should not be null.
- if mysql.HasPriKeyFlag(col.GetFlag()) && hasNullFlag {
- return dbterror.ErrPrimaryCantHaveNull
+ if len(parts) > 0 {
+ preSplit = func() { splitPartitionTableRegion(ctx, sp, tbInfo, parts, scatterRegion) }
+ } else {
+ preSplit = func() { splitTableRegion(ctx, sp, tbInfo, scatterRegion) }
+ }
+ if scatterRegion {
+ preSplit()
+ } else {
+ go preSplit()
}
- return nil
}
-func checkColumnValueConstraint(col *table.Column, collation string) error {
- if col.GetType() != mysql.TypeEnum && col.GetType() != mysql.TypeSet {
- return nil
- }
- valueMap := make(map[string]bool, len(col.GetElems()))
- ctor := collate.GetCollator(collation)
- enumLengthLimit := config.GetGlobalConfig().EnableEnumLengthLimit
- desc, err := charset.GetCharsetInfo(col.GetCharset())
+func (e *executor) FlashbackCluster(ctx sessionctx.Context, flashbackTS uint64) error {
+ logutil.DDLLogger().Info("get flashback cluster job", zap.Stringer("flashbackTS", oracle.GetTimeFromTS(flashbackTS)))
+ nowTS, err := ctx.GetStore().GetOracle().GetTimestamp(e.ctx, &oracle.Option{})
if err != nil {
return errors.Trace(err)
}
- for i := range col.GetElems() {
- val := string(ctor.Key(col.GetElems()[i]))
- // According to MySQL 8.0 Refman:
- // The maximum supported length of an individual ENUM element is M <= 255 and (M x w) <= 1020,
- // where M is the element literal length and w is the number of bytes required for the maximum-length character in the character set.
- // See https://dev.mysql.com/doc/refman/8.0/en/string-type-syntax.html for more details.
- if enumLengthLimit && (len(val) > 255 || len(val)*desc.Maxlen > 1020) {
- return dbterror.ErrTooLongValueForType.GenWithStackByArgs(col.Name)
- }
- if _, ok := valueMap[val]; ok {
- tpStr := "ENUM"
- if col.GetType() == mysql.TypeSet {
- tpStr = "SET"
- }
- return types.ErrDuplicatedValueInType.GenWithStackByArgs(col.Name, col.GetElems()[i], tpStr)
- }
- valueMap[val] = true
+ gap := time.Until(oracle.GetTimeFromTS(nowTS)).Abs()
+ if gap > 1*time.Second {
+ ctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackErrorf("Gap between local time and PD TSO is %s, please check PD/system time", gap))
}
- return nil
+ job := &model.Job{
+ Type: model.ActionFlashbackCluster,
+ BinlogInfo: &model.HistoryInfo{},
+ // The value for global variables is meaningless, it will cover during flashback cluster.
+ Args: []any{
+ flashbackTS,
+ map[string]any{},
+ true, /* tidb_gc_enable */
+ variable.On, /* tidb_enable_auto_analyze */
+ variable.Off, /* tidb_super_read_only */
+ 0, /* totalRegions */
+ 0, /* startTS */
+ 0, /* commitTS */
+ variable.On, /* tidb_ttl_job_enable */
+ []kv.KeyRange{} /* flashback key_ranges */},
+ CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
+ // FLASHBACK CLUSTER affects all schemas and tables.
+ InvolvingSchemaInfo: []model.InvolvingSchemaInfo{{
+ Database: model.InvolvingAll,
+ Table: model.InvolvingAll,
+ }},
+ SQLMode: ctx.GetSessionVars().SQLMode,
+ }
+ err = e.DoDDLJob(ctx, job)
+ return errors.Trace(err)
}
-func checkDuplicateColumn(cols []*model.ColumnInfo) error {
- colNames := set.StringSet{}
- for _, col := range cols {
- colName := col.Name
- if colNames.Exist(colName.L) {
- return infoschema.ErrColumnExists.GenWithStackByArgs(colName.O)
- }
- colNames.Insert(colName.L)
+func (e *executor) RecoverTable(ctx sessionctx.Context, recoverInfo *RecoverInfo) (err error) {
+ is := e.infoCache.GetLatest()
+ schemaID, tbInfo := recoverInfo.SchemaID, recoverInfo.TableInfo
+ // Check schema exist.
+ schema, ok := is.SchemaByID(schemaID)
+ if !ok {
+ return errors.Trace(infoschema.ErrDatabaseNotExists.GenWithStackByArgs(
+ fmt.Sprintf("(Schema ID %d)", schemaID),
+ ))
+ }
+ // Check not exist table with same name.
+ if ok := is.TableExists(schema.Name, tbInfo.Name); ok {
+ return infoschema.ErrTableExists.GenWithStackByArgs(tbInfo.Name)
}
- return nil
-}
-func containsColumnOption(colDef *ast.ColumnDef, opTp ast.ColumnOptionType) bool {
- for _, option := range colDef.Options {
- if option.Tp == opTp {
- return true
+ // for "flashback table xxx to yyy"
+ // Note: this case only allow change table name, schema remains the same.
+ var involvedSchemas []model.InvolvingSchemaInfo
+ if recoverInfo.OldTableName != tbInfo.Name.L {
+ involvedSchemas = []model.InvolvingSchemaInfo{
+ {Database: schema.Name.L, Table: recoverInfo.OldTableName},
+ {Database: schema.Name.L, Table: tbInfo.Name.L},
}
}
- return false
-}
-// IsAutoRandomColumnID returns true if the given column ID belongs to an auto_random column.
-func IsAutoRandomColumnID(tblInfo *model.TableInfo, colID int64) bool {
- if !tblInfo.ContainsAutoRandomBits() {
- return false
- }
- if tblInfo.PKIsHandle {
- return tblInfo.GetPkColInfo().ID == colID
- } else if tblInfo.IsCommonHandle {
- pk := tables.FindPrimaryIndex(tblInfo)
- if pk == nil {
- return false
- }
- offset := pk.Columns[0].Offset
- return tblInfo.Columns[offset].ID == colID
+ tbInfo.State = model.StateNone
+ job := &model.Job{
+ SchemaID: schemaID,
+ TableID: tbInfo.ID,
+ SchemaName: schema.Name.L,
+ TableName: tbInfo.Name.L,
+
+ Type: model.ActionRecoverTable,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []any{recoverInfo, recoverCheckFlagNone},
+ InvolvingSchemaInfo: involvedSchemas,
+ CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
+ SQLMode: ctx.GetSessionVars().SQLMode,
}
- return false
+ err = e.DoDDLJob(ctx, job)
+ return errors.Trace(err)
}
-func checkGeneratedColumn(ctx sessionctx.Context, schemaName model.CIStr, tableName model.CIStr, colDefs []*ast.ColumnDef) error {
- var colName2Generation = make(map[string]columnGenerationInDDL, len(colDefs))
- var exists bool
- var autoIncrementColumn string
- for i, colDef := range colDefs {
- for _, option := range colDef.Options {
- if option.Tp == ast.ColumnOptionGenerated {
- if err := checkIllegalFn4Generated(colDef.Name.Name.L, typeColumn, option.Expr); err != nil {
- return errors.Trace(err)
- }
- }
- }
- if containsColumnOption(colDef, ast.ColumnOptionAutoIncrement) {
- exists, autoIncrementColumn = true, colDef.Name.Name.L
- }
- generated, depCols, err := findDependedColumnNames(schemaName, tableName, colDef)
- if err != nil {
- return errors.Trace(err)
- }
- if !generated {
- colName2Generation[colDef.Name.Name.L] = columnGenerationInDDL{
- position: i,
- generated: false,
- }
- } else {
- colName2Generation[colDef.Name.Name.L] = columnGenerationInDDL{
- position: i,
- generated: true,
- dependences: depCols,
- }
- }
+func (e *executor) CreateView(ctx sessionctx.Context, s *ast.CreateViewStmt) (err error) {
+ viewInfo, err := BuildViewInfo(s)
+ if err != nil {
+ return err
}
- // Check whether the generated column refers to any auto-increment columns
- if exists {
- if !ctx.GetSessionVars().EnableAutoIncrementInGenerated {
- for colName, generated := range colName2Generation {
- if _, found := generated.dependences[autoIncrementColumn]; found {
- return dbterror.ErrGeneratedColumnRefAutoInc.GenWithStackByArgs(colName)
- }
- }
- }
+ cols := make([]*table.Column, len(s.Cols))
+ for i, v := range s.Cols {
+ cols[i] = table.ToColumn(&model.ColumnInfo{
+ Name: v,
+ ID: int64(i),
+ Offset: i,
+ State: model.StatePublic,
+ })
}
- for _, colDef := range colDefs {
- colName := colDef.Name.Name.L
- if err := verifyColumnGeneration(colName2Generation, colName); err != nil {
- return errors.Trace(err)
- }
+ tblCharset := ""
+ tblCollate := ""
+ if v, ok := ctx.GetSessionVars().GetSystemVar(variable.CharacterSetConnection); ok {
+ tblCharset = v
}
- return nil
-}
-
-func checkTooLongColumns(cols []*model.ColumnInfo) error {
- for _, col := range cols {
- if err := checkTooLongColumn(col.Name); err != nil {
- return err
- }
+ if v, ok := ctx.GetSessionVars().GetSystemVar(variable.CollationConnection); ok {
+ tblCollate = v
}
- return nil
-}
-func checkTooManyColumns(colDefs []*model.ColumnInfo) error {
- if uint32(len(colDefs)) > atomic.LoadUint32(&config.GetGlobalConfig().TableColumnCountLimit) {
- return dbterror.ErrTooManyFields
+ tbInfo, err := BuildTableInfo(ctx, s.ViewName.Name, cols, nil, tblCharset, tblCollate)
+ if err != nil {
+ return err
}
- return nil
-}
+ tbInfo.View = viewInfo
-func checkTooManyIndexes(idxDefs []*model.IndexInfo) error {
- if len(idxDefs) > config.GetGlobalConfig().IndexLimit {
- return dbterror.ErrTooManyKeys.GenWithStackByArgs(config.GetGlobalConfig().IndexLimit)
+ onExist := OnExistError
+ if s.OrReplace {
+ onExist = OnExistReplace
}
- return nil
+
+ return e.CreateTableWithInfo(ctx, s.ViewName.Schema, tbInfo, nil, WithOnExist(onExist))
}
-// checkColumnsAttributes checks attributes for multiple columns.
-func checkColumnsAttributes(colDefs []*model.ColumnInfo) error {
- for _, colDef := range colDefs {
- if err := checkColumnAttributes(colDef.Name.O, &colDef.FieldType); err != nil {
+func checkCharsetAndCollation(cs string, co string) error {
+ if !charset.ValidCharsetAndCollation(cs, co) {
+ return dbterror.ErrUnknownCharacterSet.GenWithStackByArgs(cs)
+ }
+ if co != "" {
+ if _, err := collate.GetCollationByName(co); err != nil {
return errors.Trace(err)
}
}
return nil
}
-func checkColumnFieldLength(col *table.Column) error {
- if col.GetType() == mysql.TypeVarchar {
- if err := types.IsVarcharTooBigFieldLength(col.GetFlen(), col.Name.O, col.GetCharset()); err != nil {
+// handleAutoIncID handles auto_increment option in DDL. It creates a ID counter for the table and initiates the counter to a proper value.
+// For example if the option sets auto_increment to 10. The counter will be set to 9. So the next allocated ID will be 10.
+func (e *executor) handleAutoIncID(tbInfo *model.TableInfo, schemaID int64, newEnd int64, tp autoid.AllocatorType) error {
+ allocs := autoid.NewAllocatorsFromTblInfo(e.getAutoIDRequirement(), schemaID, tbInfo)
+ if alloc := allocs.Get(tp); alloc != nil {
+ err := alloc.Rebase(context.Background(), newEnd, false)
+ if err != nil {
return errors.Trace(err)
}
}
-
return nil
}
-// checkColumnAttributes check attributes for single column.
-func checkColumnAttributes(colName string, tp *types.FieldType) error {
- switch tp.GetType() {
- case mysql.TypeNewDecimal, mysql.TypeDouble, mysql.TypeFloat:
- if tp.GetFlen() < tp.GetDecimal() {
- return types.ErrMBiggerThanD.GenWithStackByArgs(colName)
- }
- case mysql.TypeDatetime, mysql.TypeDuration, mysql.TypeTimestamp:
- if tp.GetDecimal() != types.UnspecifiedFsp && (tp.GetDecimal() < types.MinFsp || tp.GetDecimal() > types.MaxFsp) {
- return types.ErrTooBigPrecision.GenWithStackByArgs(tp.GetDecimal(), colName, types.MaxFsp)
- }
+// TODO we can unify this part with ddlCtx.
+func (e *executor) getAutoIDRequirement() autoid.Requirement {
+ return &asAutoIDRequirement{
+ store: e.store,
+ autoidCli: e.autoidCli,
}
- return nil
}
-func checkDuplicateConstraint(namesMap map[string]bool, name string, constraintType ast.ConstraintType) error {
- if name == "" {
- return nil
- }
- nameLower := strings.ToLower(name)
- if namesMap[nameLower] {
- switch constraintType {
- case ast.ConstraintForeignKey:
- return dbterror.ErrFkDupName.GenWithStackByArgs(name)
- case ast.ConstraintCheck:
- return dbterror.ErrCheckConstraintDupName.GenWithStackByArgs(name)
- default:
- return dbterror.ErrDupKeyName.GenWithStackByArgs(name)
- }
+func shardingBits(tblInfo *model.TableInfo) uint64 {
+ if tblInfo.ShardRowIDBits > 0 {
+ return tblInfo.ShardRowIDBits
}
- namesMap[nameLower] = true
- return nil
+ return tblInfo.AutoRandomBits
}
-func setEmptyCheckConstraintName(tableLowerName string, namesMap map[string]bool, constrs []*ast.Constraint) {
- cnt := 1
- constraintPrefix := tableLowerName + "_chk_"
- for _, constr := range constrs {
- if constr.Name == "" {
- constrName := fmt.Sprintf("%s%d", constraintPrefix, cnt)
- for {
- // loop until find constrName that haven't been used.
- if !namesMap[constrName] {
- namesMap[constrName] = true
- break
- }
- cnt++
- constrName = fmt.Sprintf("%s%d", constraintPrefix, cnt)
- }
- constr.Name = constrName
- }
- }
-}
-
-func setEmptyConstraintName(namesMap map[string]bool, constr *ast.Constraint) {
- if constr.Name == "" && len(constr.Keys) > 0 {
- var colName string
- for _, keyPart := range constr.Keys {
- if keyPart.Expr != nil {
- colName = "expression_index"
- }
- }
- if colName == "" {
- colName = constr.Keys[0].Column.Name.O
- }
- constrName := colName
- i := 2
- if strings.EqualFold(constrName, mysql.PrimaryKeyName) {
- constrName = fmt.Sprintf("%s_%d", constrName, 2)
- i = 3
- }
- for namesMap[constrName] {
- // We loop forever until we find constrName that haven't been used.
- constrName = fmt.Sprintf("%s_%d", colName, i)
- i++
- }
- constr.Name = constrName
- namesMap[constrName] = true
- }
+// isIgnorableSpec checks if the spec type is ignorable.
+// Some specs are parsed by ignored. This is for compatibility.
+func isIgnorableSpec(tp ast.AlterTableType) bool {
+ // AlterTableLock/AlterTableAlgorithm are ignored.
+ return tp == ast.AlterTableLock || tp == ast.AlterTableAlgorithm
}
-func checkConstraintNames(tableName model.CIStr, constraints []*ast.Constraint) error {
- constrNames := map[string]bool{}
- fkNames := map[string]bool{}
-
- // Check not empty constraint name whether is duplicated.
- for _, constr := range constraints {
- if constr.Tp == ast.ConstraintForeignKey {
- err := checkDuplicateConstraint(fkNames, constr.Name, constr.Tp)
- if err != nil {
- return errors.Trace(err)
- }
- } else {
- err := checkDuplicateConstraint(constrNames, constr.Name, constr.Tp)
+// GetCharsetAndCollateInTableOption will iterate the charset and collate in the options,
+// and returns the last charset and collate in options. If there is no charset in the options,
+// the returns charset will be "", the same as collate.
+func GetCharsetAndCollateInTableOption(sessVars *variable.SessionVars, startIdx int, options []*ast.TableOption) (chs, coll string, err error) {
+ for i := startIdx; i < len(options); i++ {
+ opt := options[i]
+ // we set the charset to the last option. example: alter table t charset latin1 charset utf8 collate utf8_bin;
+ // the charset will be utf8, collate will be utf8_bin
+ switch opt.Tp {
+ case ast.TableOptionCharset:
+ info, err := charset.GetCharsetInfo(opt.StrValue)
if err != nil {
- return errors.Trace(err)
+ return "", "", err
}
- }
- }
-
- // Set empty constraint names.
- checkConstraints := make([]*ast.Constraint, 0, len(constraints))
- for _, constr := range constraints {
- if constr.Tp != ast.ConstraintForeignKey {
- setEmptyConstraintName(constrNames, constr)
- }
- if constr.Tp == ast.ConstraintCheck {
- checkConstraints = append(checkConstraints, constr)
- }
- }
- // Set check constraint name under its order.
- if len(checkConstraints) > 0 {
- setEmptyCheckConstraintName(tableName.L, constrNames, checkConstraints)
- }
- return nil
-}
-
-// checkInvisibleIndexOnPK check if primary key is invisible index.
-// Note: PKIsHandle == true means the table already has a visible primary key,
-// we do not need do a check for this case and return directly,
-// because whether primary key is invisible has been check when creating table.
-func checkInvisibleIndexOnPK(tblInfo *model.TableInfo) error {
- if tblInfo.PKIsHandle {
- return nil
- }
- pk := tblInfo.GetPrimaryKey()
- if pk != nil && pk.Invisible {
- return dbterror.ErrPKIndexCantBeInvisible
- }
- return nil
-}
-
-func setTableAutoRandomBits(ctx sessionctx.Context, tbInfo *model.TableInfo, colDefs []*ast.ColumnDef) error {
- for _, col := range colDefs {
- if containsColumnOption(col, ast.ColumnOptionAutoRandom) {
- if col.Tp.GetType() != mysql.TypeLonglong {
- return dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(
- fmt.Sprintf(autoid.AutoRandomOnNonBigIntColumn, types.TypeStr(col.Tp.GetType())))
+ if len(chs) == 0 {
+ chs = info.Name
+ } else if chs != info.Name {
+ return "", "", dbterror.ErrConflictingDeclarations.GenWithStackByArgs(chs, info.Name)
}
- switch {
- case tbInfo.PKIsHandle:
- if tbInfo.GetPkName().L != col.Name.Name.L {
- errMsg := fmt.Sprintf(autoid.AutoRandomMustFirstColumnInPK, col.Name.Name.O)
- return dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(errMsg)
- }
- case tbInfo.IsCommonHandle:
- pk := tables.FindPrimaryIndex(tbInfo)
- if pk == nil {
- return dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomNoClusteredPKErrMsg)
+ if len(coll) == 0 {
+ defaultColl, err := getDefaultCollationForUTF8MB4(sessVars, chs)
+ if err != nil {
+ return "", "", errors.Trace(err)
}
- if col.Name.Name.L != pk.Columns[0].Name.L {
- errMsg := fmt.Sprintf(autoid.AutoRandomMustFirstColumnInPK, col.Name.Name.O)
- return dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(errMsg)
+ if len(defaultColl) == 0 {
+ coll = info.DefaultCollation
+ } else {
+ coll = defaultColl
}
- default:
- return dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomNoClusteredPKErrMsg)
- }
-
- if containsColumnOption(col, ast.ColumnOptionAutoIncrement) {
- return dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomIncompatibleWithAutoIncErrMsg)
- }
- if containsColumnOption(col, ast.ColumnOptionDefaultValue) {
- return dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomIncompatibleWithDefaultValueErrMsg)
}
-
- shardBits, rangeBits, err := extractAutoRandomBitsFromColDef(col)
+ case ast.TableOptionCollate:
+ info, err := collate.GetCollationByName(opt.StrValue)
if err != nil {
- return errors.Trace(err)
+ return "", "", err
}
- tbInfo.AutoRandomBits = shardBits
- tbInfo.AutoRandomRangeBits = rangeBits
-
- shardFmt := autoid.NewShardIDFormat(col.Tp, shardBits, rangeBits)
- if shardFmt.IncrementalBits < autoid.AutoRandomIncBitsMin {
- return dbterror.ErrInvalidAutoRandom.FastGenByArgs(autoid.AutoRandomIncrementalBitsTooSmall)
+ if len(chs) == 0 {
+ chs = info.CharsetName
+ } else if chs != info.CharsetName {
+ return "", "", dbterror.ErrCollationCharsetMismatch.GenWithStackByArgs(info.Name, chs)
}
- msg := fmt.Sprintf(autoid.AutoRandomAvailableAllocTimesNote, shardFmt.IncrementalBitsCapacity())
- ctx.GetSessionVars().StmtCtx.AppendNote(errors.NewNoStackError(msg))
+ coll = info.Name
}
}
- return nil
+ return
}
-func extractAutoRandomBitsFromColDef(colDef *ast.ColumnDef) (shardBits, rangeBits uint64, err error) {
- for _, op := range colDef.Options {
- if op.Tp == ast.ColumnOptionAutoRandom {
- shardBits, err = autoid.AutoRandomShardBitsNormalize(op.AutoRandOpt.ShardBits, colDef.Name.Name.O)
- if err != nil {
- return 0, 0, err
- }
- rangeBits, err = autoid.AutoRandomRangeBitsNormalize(op.AutoRandOpt.RangeBits)
- if err != nil {
- return 0, 0, err
- }
- return shardBits, rangeBits, nil
+// NeedToOverwriteColCharset return true for altering charset and specified CONVERT TO.
+func NeedToOverwriteColCharset(options []*ast.TableOption) bool {
+ for i := len(options) - 1; i >= 0; i-- {
+ opt := options[i]
+ if opt.Tp == ast.TableOptionCharset {
+ // Only overwrite columns charset if the option contains `CONVERT TO`.
+ return opt.UintValue == ast.TableOptionCharsetWithConvertTo
}
}
- return 0, 0, nil
+ return false
}
-// BuildTableInfo creates a TableInfo.
-func BuildTableInfo(
- ctx sessionctx.Context,
- tableName model.CIStr,
- cols []*table.Column,
- constraints []*ast.Constraint,
- charset string,
- collate string,
-) (tbInfo *model.TableInfo, err error) {
- tbInfo = &model.TableInfo{
- Name: tableName,
- Version: model.CurrLatestTableInfoVersion,
- Charset: charset,
- Collate: collate,
- }
- tblColumns := make([]*table.Column, 0, len(cols))
- existedColsMap := make(map[string]struct{}, len(cols))
- for _, v := range cols {
- v.ID = AllocateColumnID(tbInfo)
- tbInfo.Columns = append(tbInfo.Columns, v.ToInfo())
- tblColumns = append(tblColumns, table.ToColumn(v.ToInfo()))
- existedColsMap[v.Name.L] = struct{}{}
+// resolveAlterTableAddColumns splits "add columns" to multiple spec. For example,
+// `ALTER TABLE ADD COLUMN (c1 INT, c2 INT)` is split into
+// `ALTER TABLE ADD COLUMN c1 INT, ADD COLUMN c2 INT`.
+func resolveAlterTableAddColumns(spec *ast.AlterTableSpec) []*ast.AlterTableSpec {
+ specs := make([]*ast.AlterTableSpec, 0, len(spec.NewColumns)+len(spec.NewConstraints))
+ for _, col := range spec.NewColumns {
+ t := *spec
+ t.NewColumns = []*ast.ColumnDef{col}
+ t.NewConstraints = []*ast.Constraint{}
+ specs = append(specs, &t)
}
- foreignKeyID := tbInfo.MaxForeignKeyID
- for _, constr := range constraints {
- // Build hidden columns if necessary.
- hiddenCols, err := buildHiddenColumnInfoWithCheck(ctx, constr.Keys, model.NewCIStr(constr.Name), tbInfo, tblColumns)
- if err != nil {
- return nil, err
- }
- for _, hiddenCol := range hiddenCols {
- hiddenCol.State = model.StatePublic
- hiddenCol.ID = AllocateColumnID(tbInfo)
- hiddenCol.Offset = len(tbInfo.Columns)
- tbInfo.Columns = append(tbInfo.Columns, hiddenCol)
- tblColumns = append(tblColumns, table.ToColumn(hiddenCol))
- }
- // Check clustered on non-primary key.
- if constr.Option != nil && constr.Option.PrimaryKeyTp != model.PrimaryKeyTypeDefault &&
- constr.Tp != ast.ConstraintPrimaryKey {
- return nil, dbterror.ErrUnsupportedClusteredSecondaryKey
- }
- if constr.Tp == ast.ConstraintForeignKey {
- var fkName model.CIStr
- foreignKeyID++
- if constr.Name != "" {
- fkName = model.NewCIStr(constr.Name)
- } else {
- fkName = model.NewCIStr(fmt.Sprintf("fk_%d", foreignKeyID))
- }
- if model.FindFKInfoByName(tbInfo.ForeignKeys, fkName.L) != nil {
- return nil, infoschema.ErrCannotAddForeign
- }
- fk, err := buildFKInfo(fkName, constr.Keys, constr.Refer, cols)
- if err != nil {
- return nil, err
- }
- fk.State = model.StatePublic
-
- tbInfo.ForeignKeys = append(tbInfo.ForeignKeys, fk)
- continue
- }
- if constr.Tp == ast.ConstraintPrimaryKey {
- lastCol, err := CheckPKOnGeneratedColumn(tbInfo, constr.Keys)
- if err != nil {
- return nil, err
- }
- isSingleIntPK := isSingleIntPK(constr, lastCol)
- if ShouldBuildClusteredIndex(ctx, constr.Option, isSingleIntPK) {
- if isSingleIntPK {
- tbInfo.PKIsHandle = true
- } else {
- tbInfo.IsCommonHandle = true
- tbInfo.CommonHandleVersion = 1
- }
- }
- if tbInfo.HasClusteredIndex() {
- // Primary key cannot be invisible.
- if constr.Option != nil && constr.Option.Visibility == ast.IndexVisibilityInvisible {
- return nil, dbterror.ErrPKIndexCantBeInvisible
- }
- }
- if tbInfo.PKIsHandle {
- continue
- }
- }
-
- if constr.Tp == ast.ConstraintFulltext {
- ctx.GetSessionVars().StmtCtx.AppendWarning(dbterror.ErrTableCantHandleFt.FastGenByArgs())
- continue
- }
-
- var (
- indexName = constr.Name
- primary, unique bool
- )
-
- // Check if the index is primary or unique.
- switch constr.Tp {
- case ast.ConstraintPrimaryKey:
- primary = true
- unique = true
- indexName = mysql.PrimaryKeyName
- case ast.ConstraintUniq, ast.ConstraintUniqKey, ast.ConstraintUniqIndex:
- unique = true
- }
-
- // check constraint
- if constr.Tp == ast.ConstraintCheck {
- if !variable.EnableCheckConstraint.Load() {
- ctx.GetSessionVars().StmtCtx.AppendWarning(errCheckConstraintIsOff)
- continue
- }
- // Since column check constraint dependency has been done in columnDefToCol.
- // Here do the table check constraint dependency check, table constraint
- // can only refer the columns in defined columns of the table.
- // Refer: https://dev.mysql.com/doc/refman/8.0/en/create-table-check-constraints.html
- if ok, err := table.IsSupportedExpr(constr); !ok {
- return nil, err
- }
- var dependedCols []model.CIStr
- dependedColsMap := findDependentColsInExpr(constr.Expr)
- if !constr.InColumn {
- dependedCols = make([]model.CIStr, 0, len(dependedColsMap))
- for k := range dependedColsMap {
- if _, ok := existedColsMap[k]; !ok {
- // The table constraint depended on a non-existed column.
- return nil, dbterror.ErrTableCheckConstraintReferUnknown.GenWithStackByArgs(constr.Name, k)
- }
- dependedCols = append(dependedCols, model.NewCIStr(k))
- }
- } else {
- // Check the column-type constraint dependency.
- if len(dependedColsMap) > 1 {
- return nil, dbterror.ErrColumnCheckConstraintReferOther.GenWithStackByArgs(constr.Name)
- } else if len(dependedColsMap) == 0 {
- // If dependedCols is empty, the expression must be true/false.
- valExpr, ok := constr.Expr.(*driver.ValueExpr)
- if !ok || !mysql.HasIsBooleanFlag(valExpr.GetType().GetFlag()) {
- return nil, errors.Trace(errors.New("unsupported expression in check constraint"))
- }
- } else {
- if _, ok := dependedColsMap[constr.InColumnName]; !ok {
- return nil, dbterror.ErrColumnCheckConstraintReferOther.GenWithStackByArgs(constr.Name)
- }
- dependedCols = []model.CIStr{model.NewCIStr(constr.InColumnName)}
- }
- }
- // check auto-increment column
- if table.ContainsAutoIncrementCol(dependedCols, tbInfo) {
- return nil, dbterror.ErrCheckConstraintRefersAutoIncrementColumn.GenWithStackByArgs(constr.Name)
- }
- // check foreign key
- if err := table.HasForeignKeyRefAction(tbInfo.ForeignKeys, constraints, constr, dependedCols); err != nil {
- return nil, err
- }
- // build constraint meta info.
- constraintInfo, err := buildConstraintInfo(tbInfo, dependedCols, constr, model.StatePublic)
- if err != nil {
- return nil, errors.Trace(err)
- }
- // check if the expression is bool type
- if err := table.IfCheckConstraintExprBoolType(ctx.GetExprCtx().GetEvalCtx(), constraintInfo, tbInfo); err != nil {
- return nil, err
- }
- constraintInfo.ID = allocateConstraintID(tbInfo)
- tbInfo.Constraints = append(tbInfo.Constraints, constraintInfo)
- continue
- }
-
- // build index info.
- idxInfo, err := BuildIndexInfo(
- ctx,
- tbInfo.Columns,
- model.NewCIStr(indexName),
- primary,
- unique,
- false,
- constr.Keys,
- constr.Option,
- model.StatePublic,
- )
- if err != nil {
- return nil, errors.Trace(err)
- }
-
- if len(hiddenCols) > 0 {
- AddIndexColumnFlag(tbInfo, idxInfo)
- }
- sessionVars := ctx.GetSessionVars()
- _, err = validateCommentLength(sessionVars.StmtCtx.ErrCtx(), sessionVars.SQLMode, idxInfo.Name.String(), &idxInfo.Comment, dbterror.ErrTooLongIndexComment)
- if err != nil {
- return nil, errors.Trace(err)
- }
- idxInfo.ID = AllocateIndexID(tbInfo)
- tbInfo.Indices = append(tbInfo.Indices, idxInfo)
+ // Split the add constraints from AlterTableSpec.
+ for _, con := range spec.NewConstraints {
+ t := *spec
+ t.NewColumns = []*ast.ColumnDef{}
+ t.NewConstraints = []*ast.Constraint{}
+ t.Constraint = con
+ t.Tp = ast.AlterTableAddConstraint
+ specs = append(specs, &t)
}
-
- err = addIndexForForeignKey(ctx, tbInfo)
- return tbInfo, err
+ return specs
}
-// addIndexForForeignKey uses to auto create an index for the foreign key if the table doesn't have any index cover the
-// foreign key columns.
-func addIndexForForeignKey(ctx sessionctx.Context, tbInfo *model.TableInfo) error {
- if len(tbInfo.ForeignKeys) == 0 {
- return nil
- }
- var handleCol *model.ColumnInfo
- if tbInfo.PKIsHandle {
- handleCol = tbInfo.GetPkColInfo()
- }
- for _, fk := range tbInfo.ForeignKeys {
- if fk.Version < model.FKVersion1 {
- continue
- }
- if handleCol != nil && len(fk.Cols) == 1 && handleCol.Name.L == fk.Cols[0].L {
- continue
+// ResolveAlterTableSpec resolves alter table algorithm and removes ignore table spec in specs.
+// returns valid specs, and the occurred error.
+func ResolveAlterTableSpec(ctx sessionctx.Context, specs []*ast.AlterTableSpec) ([]*ast.AlterTableSpec, error) {
+ validSpecs := make([]*ast.AlterTableSpec, 0, len(specs))
+ algorithm := ast.AlgorithmTypeDefault
+ for _, spec := range specs {
+ if spec.Tp == ast.AlterTableAlgorithm {
+ // Find the last AlterTableAlgorithm.
+ algorithm = spec.Algorithm
}
- if model.FindIndexByColumns(tbInfo, tbInfo.Indices, fk.Cols...) != nil {
+ if isIgnorableSpec(spec.Tp) {
continue
}
- idxName := fk.Name
- if tbInfo.FindIndexByName(idxName.L) != nil {
- return dbterror.ErrDupKeyName.GenWithStack("duplicate key name %s", fk.Name.O)
- }
- keys := make([]*ast.IndexPartSpecification, 0, len(fk.Cols))
- for _, col := range fk.Cols {
- keys = append(keys, &ast.IndexPartSpecification{
- Column: &ast.ColumnName{Name: col},
- Length: types.UnspecifiedLength,
- })
- }
- idxInfo, err := BuildIndexInfo(ctx, tbInfo.Columns, idxName, false, false, false, keys, nil, model.StatePublic)
- if err != nil {
- return errors.Trace(err)
+ if spec.Tp == ast.AlterTableAddColumns && (len(spec.NewColumns) > 1 || len(spec.NewConstraints) > 0) {
+ validSpecs = append(validSpecs, resolveAlterTableAddColumns(spec)...)
+ } else {
+ validSpecs = append(validSpecs, spec)
}
- idxInfo.ID = AllocateIndexID(tbInfo)
- tbInfo.Indices = append(tbInfo.Indices, idxInfo)
+ // TODO: Only allow REMOVE PARTITIONING as a single ALTER TABLE statement?
}
- return nil
-}
-func indexColumnsLen(cols []*model.ColumnInfo, idxCols []*model.IndexColumn) (colLen int, err error) {
- for _, idxCol := range idxCols {
- col := model.FindColumnInfo(cols, idxCol.Name.L)
- if col == nil {
- err = dbterror.ErrKeyColumnDoesNotExits.GenWithStack("column does not exist: %s", idxCol.Name.L)
- return
- }
- var l int
- l, err = getIndexColumnLength(col, idxCol.Length)
+ // Verify whether the algorithm is supported.
+ for _, spec := range validSpecs {
+ resolvedAlgorithm, err := ResolveAlterAlgorithm(spec, algorithm)
if err != nil {
- return
+ // If TiDB failed to choose a better algorithm, report the error
+ if resolvedAlgorithm == ast.AlgorithmTypeDefault {
+ return nil, errors.Trace(err)
+ }
+ // For the compatibility, we return warning instead of error when a better algorithm is chosed by TiDB
+ ctx.GetSessionVars().StmtCtx.AppendError(err)
}
- colLen += l
+
+ spec.Algorithm = resolvedAlgorithm
}
- return
+
+ // Only handle valid specs.
+ return validSpecs, nil
}
-func isSingleIntPK(constr *ast.Constraint, lastCol *model.ColumnInfo) bool {
- if len(constr.Keys) != 1 {
- return false
+func isMultiSchemaChanges(specs []*ast.AlterTableSpec) bool {
+ if len(specs) > 1 {
+ return true
}
- switch lastCol.GetType() {
- case mysql.TypeLong, mysql.TypeLonglong,
- mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24:
+ if len(specs) == 1 && len(specs[0].NewColumns) > 1 && specs[0].Tp == ast.AlterTableAddColumns {
return true
}
return false
}
-// ShouldBuildClusteredIndex is used to determine whether the CREATE TABLE statement should build a clustered index table.
-func ShouldBuildClusteredIndex(ctx sessionctx.Context, opt *ast.IndexOption, isSingleIntPK bool) bool {
- if opt == nil || opt.PrimaryKeyTp == model.PrimaryKeyTypeDefault {
- switch ctx.GetSessionVars().EnableClusteredIndex {
- case variable.ClusteredIndexDefModeOn:
- return true
- case variable.ClusteredIndexDefModeIntOnly:
- return !config.GetGlobalConfig().AlterPrimaryKey && isSingleIntPK
- default:
- return false
- }
- }
- return opt.PrimaryKeyTp == model.PrimaryKeyTypeClustered
-}
-
-// checkTableInfoValidExtra is like checkTableInfoValid, but also assumes the
-// table info comes from untrusted source and performs further checks such as
-// name length and column count.
-// (checkTableInfoValid is also used in repairing objects which don't perform
-// these checks. Perhaps the two functions should be merged together regardless?)
-func checkTableInfoValidExtra(tbInfo *model.TableInfo) error {
- if err := checkTooLongTable(tbInfo.Name); err != nil {
- return err
+func (e *executor) AlterTable(ctx context.Context, sctx sessionctx.Context, stmt *ast.AlterTableStmt) (err error) {
+ ident := ast.Ident{Schema: stmt.Table.Schema, Name: stmt.Table.Name}
+ validSpecs, err := ResolveAlterTableSpec(sctx, stmt.Specs)
+ if err != nil {
+ return errors.Trace(err)
}
- if err := checkDuplicateColumn(tbInfo.Columns); err != nil {
- return err
- }
- if err := checkTooLongColumns(tbInfo.Columns); err != nil {
- return err
- }
- if err := checkTooManyColumns(tbInfo.Columns); err != nil {
- return errors.Trace(err)
- }
- if err := checkTooManyIndexes(tbInfo.Indices); err != nil {
- return errors.Trace(err)
- }
- if err := checkColumnsAttributes(tbInfo.Columns); err != nil {
- return errors.Trace(err)
- }
-
- // FIXME: perform checkConstraintNames
- if err := checkCharsetAndCollation(tbInfo.Charset, tbInfo.Collate); err != nil {
- return errors.Trace(err)
- }
-
- oldState := tbInfo.State
- tbInfo.State = model.StatePublic
- err := checkTableInfoValid(tbInfo)
- tbInfo.State = oldState
- return err
-}
-
-// CheckTableInfoValidWithStmt exposes checkTableInfoValidWithStmt to SchemaTracker. Maybe one day we can delete it.
-func CheckTableInfoValidWithStmt(ctx sessionctx.Context, tbInfo *model.TableInfo, s *ast.CreateTableStmt) (err error) {
- return checkTableInfoValidWithStmt(ctx, tbInfo, s)
-}
-
-func checkTableInfoValidWithStmt(ctx sessionctx.Context, tbInfo *model.TableInfo, s *ast.CreateTableStmt) (err error) {
- // All of these rely on the AST structure of expressions, which were
- // lost in the model (got serialized into strings).
- if err := checkGeneratedColumn(ctx, s.Table.Schema, tbInfo.Name, s.Cols); err != nil {
- return errors.Trace(err)
- }
-
- // Check if table has a primary key if required.
- if !ctx.GetSessionVars().InRestrictedSQL && ctx.GetSessionVars().PrimaryKeyRequired && len(tbInfo.GetPkName().String()) == 0 {
- return infoschema.ErrTableWithoutPrimaryKey
- }
- if tbInfo.Partition != nil {
- if err := checkPartitionDefinitionConstraints(ctx, tbInfo); err != nil {
- return errors.Trace(err)
- }
- if s.Partition != nil {
- if err := checkPartitionFuncType(ctx, s.Partition.Expr, s.Table.Schema.O, tbInfo); err != nil {
- return errors.Trace(err)
- }
- if err := checkPartitioningKeysConstraints(ctx, s, tbInfo); err != nil {
- return errors.Trace(err)
- }
- }
- }
- if tbInfo.TTLInfo != nil {
- if err := checkTTLInfoValid(ctx, s.Table.Schema, tbInfo); err != nil {
- return errors.Trace(err)
- }
- }
-
- return nil
-}
-
-func checkPartitionDefinitionConstraints(ctx sessionctx.Context, tbInfo *model.TableInfo) error {
- var err error
- if err = checkPartitionNameUnique(tbInfo.Partition); err != nil {
- return errors.Trace(err)
- }
- if err = checkAddPartitionTooManyPartitions(uint64(len(tbInfo.Partition.Definitions))); err != nil {
- return err
- }
- if err = checkAddPartitionOnTemporaryMode(tbInfo); err != nil {
- return err
- }
- if err = checkPartitionColumnsUnique(tbInfo); err != nil {
- return err
- }
-
- switch tbInfo.Partition.Type {
- case model.PartitionTypeRange:
- err = checkPartitionByRange(ctx, tbInfo)
- case model.PartitionTypeHash, model.PartitionTypeKey:
- err = checkPartitionByHash(ctx, tbInfo)
- case model.PartitionTypeList:
- err = checkPartitionByList(ctx, tbInfo)
- }
- return errors.Trace(err)
-}
-
-// checkTableInfoValid uses to check table info valid. This is used to validate table info.
-func checkTableInfoValid(tblInfo *model.TableInfo) error {
- _, err := tables.TableFromMeta(autoid.NewAllocators(false), tblInfo)
- if err != nil {
- return err
- }
- return checkInvisibleIndexOnPK(tblInfo)
-}
-
-// BuildTableInfoWithLike builds a new table info according to CREATE TABLE ... LIKE statement.
-func BuildTableInfoWithLike(ctx sessionctx.Context, ident ast.Ident, referTblInfo *model.TableInfo, s *ast.CreateTableStmt) (*model.TableInfo, error) {
- // Check the referred table is a real table object.
- if referTblInfo.IsSequence() || referTblInfo.IsView() {
- return nil, dbterror.ErrWrongObject.GenWithStackByArgs(ident.Schema, referTblInfo.Name, "BASE TABLE")
- }
- tblInfo := *referTblInfo
- if err := setTemporaryType(ctx, &tblInfo, s); err != nil {
- return nil, errors.Trace(err)
- }
- // Check non-public column and adjust column offset.
- newColumns := referTblInfo.Cols()
- newIndices := make([]*model.IndexInfo, 0, len(tblInfo.Indices))
- for _, idx := range tblInfo.Indices {
- if idx.State == model.StatePublic {
- newIndices = append(newIndices, idx)
- }
- }
- tblInfo.Columns = newColumns
- tblInfo.Indices = newIndices
- tblInfo.Name = ident.Name
- tblInfo.AutoIncID = 0
- tblInfo.ForeignKeys = nil
- // Ignore TiFlash replicas for temporary tables.
- if s.TemporaryKeyword != ast.TemporaryNone {
- tblInfo.TiFlashReplica = nil
- } else if tblInfo.TiFlashReplica != nil {
- replica := *tblInfo.TiFlashReplica
- // Keep the tiflash replica setting, remove the replica available status.
- replica.AvailablePartitionIDs = nil
- replica.Available = false
- tblInfo.TiFlashReplica = &replica
- }
- if referTblInfo.Partition != nil {
- pi := *referTblInfo.Partition
- pi.Definitions = make([]model.PartitionDefinition, len(referTblInfo.Partition.Definitions))
- copy(pi.Definitions, referTblInfo.Partition.Definitions)
- tblInfo.Partition = &pi
- }
-
- if referTblInfo.TTLInfo != nil {
- tblInfo.TTLInfo = referTblInfo.TTLInfo.Clone()
- }
- renameCheckConstraint(&tblInfo)
- return &tblInfo, nil
-}
-
-func renameCheckConstraint(tblInfo *model.TableInfo) {
- for _, cons := range tblInfo.Constraints {
- cons.Name = model.NewCIStr("")
- cons.Table = tblInfo.Name
- }
- setNameForConstraintInfo(tblInfo.Name.L, map[string]bool{}, tblInfo.Constraints)
-}
-
-// BuildTableInfoFromAST builds model.TableInfo from a SQL statement.
-// Note: TableID and PartitionID are left as uninitialized value.
-func BuildTableInfoFromAST(s *ast.CreateTableStmt) (*model.TableInfo, error) {
- return buildTableInfoWithCheck(mock.NewContext(), s, mysql.DefaultCharset, "", nil)
-}
-
-// buildTableInfoWithCheck builds model.TableInfo from a SQL statement.
-// Note: TableID and PartitionIDs are left as uninitialized value.
-func buildTableInfoWithCheck(ctx sessionctx.Context, s *ast.CreateTableStmt, dbCharset, dbCollate string, placementPolicyRef *model.PolicyRefInfo) (*model.TableInfo, error) {
- tbInfo, err := BuildTableInfoWithStmt(ctx, s, dbCharset, dbCollate, placementPolicyRef)
- if err != nil {
- return nil, err
- }
- // Fix issue 17952 which will cause partition range expr can't be parsed as Int.
- // checkTableInfoValidWithStmt will do the constant fold the partition expression first,
- // then checkTableInfoValidExtra will pass the tableInfo check successfully.
- if err = checkTableInfoValidWithStmt(ctx, tbInfo, s); err != nil {
- return nil, err
- }
- if err = checkTableInfoValidExtra(tbInfo); err != nil {
- return nil, err
- }
- return tbInfo, nil
-}
-
-// BuildSessionTemporaryTableInfo builds model.TableInfo from a SQL statement.
-func BuildSessionTemporaryTableInfo(ctx sessionctx.Context, is infoschema.InfoSchema, s *ast.CreateTableStmt, dbCharset, dbCollate string, placementPolicyRef *model.PolicyRefInfo) (*model.TableInfo, error) {
- ident := ast.Ident{Schema: s.Table.Schema, Name: s.Table.Name}
- //build tableInfo
- var tbInfo *model.TableInfo
- var referTbl table.Table
- var err error
- if s.ReferTable != nil {
- referIdent := ast.Ident{Schema: s.ReferTable.Schema, Name: s.ReferTable.Name}
- _, ok := is.SchemaByName(referIdent.Schema)
- if !ok {
- return nil, infoschema.ErrTableNotExists.GenWithStackByArgs(referIdent.Schema, referIdent.Name)
- }
- referTbl, err = is.TableByName(context.Background(), referIdent.Schema, referIdent.Name)
- if err != nil {
- return nil, infoschema.ErrTableNotExists.GenWithStackByArgs(referIdent.Schema, referIdent.Name)
- }
- tbInfo, err = BuildTableInfoWithLike(ctx, ident, referTbl.Meta(), s)
- } else {
- tbInfo, err = buildTableInfoWithCheck(ctx, s, dbCharset, dbCollate, placementPolicyRef)
- }
- return tbInfo, err
-}
-
-// BuildTableInfoWithStmt builds model.TableInfo from a SQL statement without validity check
-func BuildTableInfoWithStmt(ctx sessionctx.Context, s *ast.CreateTableStmt, dbCharset, dbCollate string, placementPolicyRef *model.PolicyRefInfo) (*model.TableInfo, error) {
- colDefs := s.Cols
- tableCharset, tableCollate, err := GetCharsetAndCollateInTableOption(ctx.GetSessionVars(), 0, s.Options)
- if err != nil {
- return nil, errors.Trace(err)
- }
- tableCharset, tableCollate, err = ResolveCharsetCollation(ctx.GetSessionVars(),
- ast.CharsetOpt{Chs: tableCharset, Col: tableCollate},
- ast.CharsetOpt{Chs: dbCharset, Col: dbCollate},
- )
- if err != nil {
- return nil, errors.Trace(err)
- }
-
- // The column charset haven't been resolved here.
- cols, newConstraints, err := buildColumnsAndConstraints(ctx, colDefs, s.Constraints, tableCharset, tableCollate)
- if err != nil {
- return nil, errors.Trace(err)
- }
- err = checkConstraintNames(s.Table.Name, newConstraints)
- if err != nil {
- return nil, errors.Trace(err)
- }
-
- var tbInfo *model.TableInfo
- tbInfo, err = BuildTableInfo(ctx, s.Table.Name, cols, newConstraints, tableCharset, tableCollate)
- if err != nil {
- return nil, errors.Trace(err)
- }
- if err = setTemporaryType(ctx, tbInfo, s); err != nil {
- return nil, errors.Trace(err)
- }
-
- if err = setTableAutoRandomBits(ctx, tbInfo, colDefs); err != nil {
- return nil, errors.Trace(err)
- }
-
- if err = handleTableOptions(s.Options, tbInfo); err != nil {
- return nil, errors.Trace(err)
- }
-
- sessionVars := ctx.GetSessionVars()
- if _, err = validateCommentLength(sessionVars.StmtCtx.ErrCtx(), sessionVars.SQLMode, tbInfo.Name.L, &tbInfo.Comment, dbterror.ErrTooLongTableComment); err != nil {
- return nil, errors.Trace(err)
- }
-
- if tbInfo.TempTableType == model.TempTableNone && tbInfo.PlacementPolicyRef == nil && placementPolicyRef != nil {
- // Set the defaults from Schema. Note: they are mutual exclusive!
- tbInfo.PlacementPolicyRef = placementPolicyRef
- }
-
- // After handleTableOptions, so the partitions can get defaults from Table level
- err = buildTablePartitionInfo(ctx, s.Partition, tbInfo)
- if err != nil {
- return nil, errors.Trace(err)
- }
-
- return tbInfo, nil
-}
-
-func (e *executor) assignPartitionIDs(defs []model.PartitionDefinition) error {
- genIDs, err := e.genGlobalIDs(len(defs))
- if err != nil {
- return errors.Trace(err)
- }
- for i := range defs {
- defs[i].ID = genIDs[i]
- }
- return nil
-}
-
-func (e *executor) CreateTable(ctx sessionctx.Context, s *ast.CreateTableStmt) (err error) {
- ident := ast.Ident{Schema: s.Table.Schema, Name: s.Table.Name}
- is := e.infoCache.GetLatest()
- schema, ok := is.SchemaByName(ident.Schema)
- if !ok {
- return infoschema.ErrDatabaseNotExists.GenWithStackByArgs(ident.Schema)
- }
-
- var (
- referTbl table.Table
- involvingRef []model.InvolvingSchemaInfo
- )
- if s.ReferTable != nil {
- referIdent := ast.Ident{Schema: s.ReferTable.Schema, Name: s.ReferTable.Name}
- _, ok := is.SchemaByName(referIdent.Schema)
- if !ok {
- return infoschema.ErrTableNotExists.GenWithStackByArgs(referIdent.Schema, referIdent.Name)
- }
- referTbl, err = is.TableByName(e.ctx, referIdent.Schema, referIdent.Name)
- if err != nil {
- return infoschema.ErrTableNotExists.GenWithStackByArgs(referIdent.Schema, referIdent.Name)
- }
- involvingRef = append(involvingRef, model.InvolvingSchemaInfo{
- Database: s.ReferTable.Schema.L,
- Table: s.ReferTable.Name.L,
- Mode: model.SharedInvolving,
- })
- }
-
- // build tableInfo
- var tbInfo *model.TableInfo
- if s.ReferTable != nil {
- tbInfo, err = BuildTableInfoWithLike(ctx, ident, referTbl.Meta(), s)
- } else {
- tbInfo, err = BuildTableInfoWithStmt(ctx, s, schema.Charset, schema.Collate, schema.PlacementPolicyRef)
- }
- if err != nil {
- return errors.Trace(err)
- }
-
- if err = checkTableInfoValidWithStmt(ctx, tbInfo, s); err != nil {
- return err
- }
- if err = checkTableForeignKeysValid(ctx, is, schema.Name.L, tbInfo); err != nil {
- return err
- }
-
- onExist := OnExistError
- if s.IfNotExists {
- onExist = OnExistIgnore
- }
-
- return e.CreateTableWithInfo(ctx, schema.Name, tbInfo, involvingRef, WithOnExist(onExist))
-}
-
-func setTemporaryType(_ sessionctx.Context, tbInfo *model.TableInfo, s *ast.CreateTableStmt) error {
- switch s.TemporaryKeyword {
- case ast.TemporaryGlobal:
- tbInfo.TempTableType = model.TempTableGlobal
- // "create global temporary table ... on commit preserve rows"
- if !s.OnCommitDelete {
- return errors.Trace(dbterror.ErrUnsupportedOnCommitPreserve)
- }
- case ast.TemporaryLocal:
- tbInfo.TempTableType = model.TempTableLocal
- default:
- tbInfo.TempTableType = model.TempTableNone
- }
- return nil
-}
-
-// createTableWithInfoJob returns the table creation job.
-// WARNING: it may return a nil job, which means you don't need to submit any DDL job.
-func (e *executor) createTableWithInfoJob(
- ctx sessionctx.Context,
- dbName model.CIStr,
- tbInfo *model.TableInfo,
- involvingRef []model.InvolvingSchemaInfo,
- onExist OnExist,
-) (job *model.Job, err error) {
- is := e.infoCache.GetLatest()
- schema, ok := is.SchemaByName(dbName)
- if !ok {
- return nil, infoschema.ErrDatabaseNotExists.GenWithStackByArgs(dbName)
- }
-
- if err = handleTablePlacement(ctx, tbInfo); err != nil {
- return nil, errors.Trace(err)
- }
-
- var oldViewTblID int64
- if oldTable, err := is.TableByName(e.ctx, schema.Name, tbInfo.Name); err == nil {
- err = infoschema.ErrTableExists.GenWithStackByArgs(ast.Ident{Schema: schema.Name, Name: tbInfo.Name})
- switch onExist {
- case OnExistIgnore:
- ctx.GetSessionVars().StmtCtx.AppendNote(err)
- return nil, nil
- case OnExistReplace:
- // only CREATE OR REPLACE VIEW is supported at the moment.
- if tbInfo.View != nil {
- if oldTable.Meta().IsView() {
- oldViewTblID = oldTable.Meta().ID
- break
- }
- // The object to replace isn't a view.
- return nil, dbterror.ErrWrongObject.GenWithStackByArgs(dbName, tbInfo.Name, "VIEW")
- }
- return nil, err
- default:
- return nil, err
- }
- }
-
- if err := checkTableInfoValidExtra(tbInfo); err != nil {
- return nil, err
- }
-
- var actionType model.ActionType
- args := []any{tbInfo}
- switch {
- case tbInfo.View != nil:
- actionType = model.ActionCreateView
- args = append(args, onExist == OnExistReplace, oldViewTblID)
- case tbInfo.Sequence != nil:
- actionType = model.ActionCreateSequence
- default:
- actionType = model.ActionCreateTable
- args = append(args, ctx.GetSessionVars().ForeignKeyChecks)
- }
-
- var involvingSchemas []model.InvolvingSchemaInfo
- sharedInvolvingFromTableInfo := getSharedInvolvingSchemaInfo(tbInfo)
-
- if sum := len(involvingRef) + len(sharedInvolvingFromTableInfo); sum > 0 {
- involvingSchemas = make([]model.InvolvingSchemaInfo, 0, sum+1)
- involvingSchemas = append(involvingSchemas, model.InvolvingSchemaInfo{
- Database: schema.Name.L,
- Table: tbInfo.Name.L,
- })
- involvingSchemas = append(involvingSchemas, involvingRef...)
- involvingSchemas = append(involvingSchemas, sharedInvolvingFromTableInfo...)
- }
-
- job = &model.Job{
- SchemaID: schema.ID,
- SchemaName: schema.Name.L,
- TableName: tbInfo.Name.L,
- Type: actionType,
- BinlogInfo: &model.HistoryInfo{},
- Args: args,
- CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
- InvolvingSchemaInfo: involvingSchemas,
- SQLMode: ctx.GetSessionVars().SQLMode,
- }
- return job, nil
-}
-
-func getSharedInvolvingSchemaInfo(info *model.TableInfo) []model.InvolvingSchemaInfo {
- ret := make([]model.InvolvingSchemaInfo, 0, len(info.ForeignKeys)+1)
- for _, fk := range info.ForeignKeys {
- ret = append(ret, model.InvolvingSchemaInfo{
- Database: fk.RefSchema.L,
- Table: fk.RefTable.L,
- Mode: model.SharedInvolving,
- })
- }
- if ref := info.PlacementPolicyRef; ref != nil {
- ret = append(ret, model.InvolvingSchemaInfo{
- Policy: ref.Name.L,
- Mode: model.SharedInvolving,
- })
- }
- return ret
-}
-
-func (e *executor) createTableWithInfoPost(
- ctx sessionctx.Context,
- tbInfo *model.TableInfo,
- schemaID int64,
-) error {
- var err error
- var partitions []model.PartitionDefinition
- if pi := tbInfo.GetPartitionInfo(); pi != nil {
- partitions = pi.Definitions
- }
- preSplitAndScatter(ctx, e.store, tbInfo, partitions)
- if tbInfo.AutoIncID > 1 {
- // Default tableAutoIncID base is 0.
- // If the first ID is expected to greater than 1, we need to do rebase.
- newEnd := tbInfo.AutoIncID - 1
- var allocType autoid.AllocatorType
- if tbInfo.SepAutoInc() {
- allocType = autoid.AutoIncrementType
- } else {
- allocType = autoid.RowIDAllocType
- }
- if err = e.handleAutoIncID(tbInfo, schemaID, newEnd, allocType); err != nil {
- return errors.Trace(err)
- }
- }
- // For issue https://github.com/pingcap/tidb/issues/46093
- if tbInfo.AutoIncIDExtra != 0 {
- if err = e.handleAutoIncID(tbInfo, schemaID, tbInfo.AutoIncIDExtra-1, autoid.RowIDAllocType); err != nil {
- return errors.Trace(err)
- }
- }
- if tbInfo.AutoRandID > 1 {
- // Default tableAutoRandID base is 0.
- // If the first ID is expected to greater than 1, we need to do rebase.
- newEnd := tbInfo.AutoRandID - 1
- err = e.handleAutoIncID(tbInfo, schemaID, newEnd, autoid.AutoRandomType)
- }
- return err
-}
-
-func (e *executor) CreateTableWithInfo(
- ctx sessionctx.Context,
- dbName model.CIStr,
- tbInfo *model.TableInfo,
- involvingRef []model.InvolvingSchemaInfo,
- cs ...CreateTableOption,
-) (err error) {
- c := GetCreateTableConfig(cs)
-
- job, err := e.createTableWithInfoJob(
- ctx, dbName, tbInfo, involvingRef, c.OnExist,
- )
- if err != nil {
- return err
- }
- if job == nil {
- return nil
- }
-
- jobW := NewJobWrapper(job, c.IDAllocated)
-
- err = e.DoDDLJobWrapper(ctx, jobW)
- if err != nil {
- // table exists, but if_not_exists flags is true, so we ignore this error.
- if c.OnExist == OnExistIgnore && infoschema.ErrTableExists.Equal(err) {
- ctx.GetSessionVars().StmtCtx.AppendNote(err)
- err = nil
- }
- } else {
- err = e.createTableWithInfoPost(ctx, tbInfo, job.SchemaID)
- }
-
- return errors.Trace(err)
-}
-
-func (e *executor) BatchCreateTableWithInfo(ctx sessionctx.Context,
- dbName model.CIStr,
- infos []*model.TableInfo,
- cs ...CreateTableOption,
-) error {
- failpoint.Inject("RestoreBatchCreateTableEntryTooLarge", func(val failpoint.Value) {
- injectBatchSize := val.(int)
- if len(infos) > injectBatchSize {
- failpoint.Return(kv.ErrEntryTooLarge)
- }
- })
- c := GetCreateTableConfig(cs)
-
- jobW := NewJobWrapper(
- &model.Job{
- BinlogInfo: &model.HistoryInfo{},
- CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
- SQLMode: ctx.GetSessionVars().SQLMode,
- },
- c.IDAllocated,
- )
- args := make([]*model.TableInfo, 0, len(infos))
-
- var err error
-
- // check if there are any duplicated table names
- duplication := make(map[string]struct{})
- // TODO filter those duplicated info out.
- for _, info := range infos {
- if _, ok := duplication[info.Name.L]; ok {
- err = infoschema.ErrTableExists.FastGenByArgs("can not batch create tables with same name")
- if c.OnExist == OnExistIgnore && infoschema.ErrTableExists.Equal(err) {
- ctx.GetSessionVars().StmtCtx.AppendNote(err)
- err = nil
- }
- }
- if err != nil {
- return errors.Trace(err)
- }
-
- duplication[info.Name.L] = struct{}{}
- }
-
- for _, info := range infos {
- job, err := e.createTableWithInfoJob(ctx, dbName, info, nil, c.OnExist)
- if err != nil {
- return errors.Trace(err)
- }
- if job == nil {
- continue
- }
-
- // if jobW.Type == model.ActionCreateTables, it is initialized
- // if not, initialize jobW by job.XXXX
- if jobW.Type != model.ActionCreateTables {
- jobW.Type = model.ActionCreateTables
- jobW.SchemaID = job.SchemaID
- jobW.SchemaName = job.SchemaName
- }
-
- // append table job args
- info, ok := job.Args[0].(*model.TableInfo)
- if !ok {
- return errors.Trace(fmt.Errorf("except table info"))
- }
- args = append(args, info)
- jobW.InvolvingSchemaInfo = append(jobW.InvolvingSchemaInfo, model.InvolvingSchemaInfo{
- Database: dbName.L,
- Table: info.Name.L,
- })
- if sharedInv := getSharedInvolvingSchemaInfo(info); len(sharedInv) > 0 {
- jobW.InvolvingSchemaInfo = append(jobW.InvolvingSchemaInfo, sharedInv...)
- }
- }
- if len(args) == 0 {
- return nil
- }
- jobW.Args = append(jobW.Args, args)
- jobW.Args = append(jobW.Args, ctx.GetSessionVars().ForeignKeyChecks)
-
- err = e.DoDDLJobWrapper(ctx, jobW)
- if err != nil {
- // table exists, but if_not_exists flags is true, so we ignore this error.
- if c.OnExist == OnExistIgnore && infoschema.ErrTableExists.Equal(err) {
- ctx.GetSessionVars().StmtCtx.AppendNote(err)
- err = nil
- }
- return errors.Trace(err)
- }
-
- for j := range args {
- if err = e.createTableWithInfoPost(ctx, args[j], jobW.SchemaID); err != nil {
- return errors.Trace(err)
- }
- }
-
- return nil
-}
-
-// BuildQueryStringFromJobs takes a slice of Jobs and concatenates their
-// queries into a single query string.
-// Each query is separated by a semicolon and a space.
-// Trailing spaces are removed from each query, and a semicolon is appended
-// if it's not already present.
-func BuildQueryStringFromJobs(jobs []*model.Job) string {
- var queryBuilder strings.Builder
- for i, job := range jobs {
- q := strings.TrimSpace(job.Query)
- if !strings.HasSuffix(q, ";") {
- q += ";"
- }
- queryBuilder.WriteString(q)
-
- if i < len(jobs)-1 {
- queryBuilder.WriteString(" ")
- }
- }
- return queryBuilder.String()
-}
-
-// BatchCreateTableWithJobs combine CreateTableJobs to BatchCreateTableJob.
-func BatchCreateTableWithJobs(jobs []*model.Job) (*model.Job, error) {
- if len(jobs) == 0 {
- return nil, errors.Trace(fmt.Errorf("expect non-empty jobs"))
- }
-
- var combinedJob *model.Job
-
- args := make([]*model.TableInfo, 0, len(jobs))
- involvingSchemaInfo := make([]model.InvolvingSchemaInfo, 0, len(jobs))
- var foreignKeyChecks bool
-
- // if there is any duplicated table name
- duplication := make(map[string]struct{})
- for _, job := range jobs {
- if combinedJob == nil {
- combinedJob = job.Clone()
- combinedJob.Type = model.ActionCreateTables
- combinedJob.Args = combinedJob.Args[:0]
- foreignKeyChecks = job.Args[1].(bool)
- }
- // append table job args
- info, ok := job.Args[0].(*model.TableInfo)
- if !ok {
- return nil, errors.Trace(fmt.Errorf("expect model.TableInfo, but got %T", job.Args[0]))
- }
- args = append(args, info)
-
- if _, ok := duplication[info.Name.L]; ok {
- // return err even if create table if not exists
- return nil, infoschema.ErrTableExists.FastGenByArgs("can not batch create tables with same name")
- }
-
- duplication[info.Name.L] = struct{}{}
-
- involvingSchemaInfo = append(involvingSchemaInfo,
- model.InvolvingSchemaInfo{
- Database: job.SchemaName,
- Table: info.Name.L,
- })
- }
-
- combinedJob.Args = append(combinedJob.Args, args)
- combinedJob.Args = append(combinedJob.Args, foreignKeyChecks)
- combinedJob.InvolvingSchemaInfo = involvingSchemaInfo
- combinedJob.Query = BuildQueryStringFromJobs(jobs)
-
- return combinedJob, nil
-}
-
-func (e *executor) CreatePlacementPolicyWithInfo(ctx sessionctx.Context, policy *model.PolicyInfo, onExist OnExist) error {
- if checkIgnorePlacementDDL(ctx) {
- return nil
- }
-
- policyName := policy.Name
- if policyName.L == defaultPlacementPolicyName {
- return errors.Trace(infoschema.ErrReservedSyntax.GenWithStackByArgs(policyName))
- }
-
- // Check policy existence.
- _, ok := e.infoCache.GetLatest().PolicyByName(policyName)
- if ok {
- err := infoschema.ErrPlacementPolicyExists.GenWithStackByArgs(policyName)
- switch onExist {
- case OnExistIgnore:
- ctx.GetSessionVars().StmtCtx.AppendNote(err)
- return nil
- case OnExistError:
- return err
- }
- }
-
- if err := checkPolicyValidation(policy.PlacementSettings); err != nil {
- return err
- }
-
- policyID, err := e.genPlacementPolicyID()
- if err != nil {
- return err
- }
- policy.ID = policyID
-
- job := &model.Job{
- SchemaName: policy.Name.L,
- Type: model.ActionCreatePlacementPolicy,
- BinlogInfo: &model.HistoryInfo{},
- Args: []any{policy, onExist == OnExistReplace},
- InvolvingSchemaInfo: []model.InvolvingSchemaInfo{{
- Policy: policy.Name.L,
- }},
- CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
- SQLMode: ctx.GetSessionVars().SQLMode,
- }
- err = e.DoDDLJob(ctx, job)
- return errors.Trace(err)
-}
-
-// preSplitAndScatter performs pre-split and scatter of the table's regions.
-// If `pi` is not nil, will only split region for `pi`, this is used when add partition.
-func preSplitAndScatter(ctx sessionctx.Context, store kv.Storage, tbInfo *model.TableInfo, parts []model.PartitionDefinition) {
- if tbInfo.TempTableType != model.TempTableNone {
- return
- }
- sp, ok := store.(kv.SplittableStore)
- if !ok || atomic.LoadUint32(&EnableSplitTableRegion) == 0 {
- return
- }
- var (
- preSplit func()
- scatterRegion bool
- )
- val, err := ctx.GetSessionVars().GetGlobalSystemVar(context.Background(), variable.TiDBScatterRegion)
- if err != nil {
- logutil.DDLLogger().Warn("won't scatter region", zap.Error(err))
- } else {
- scatterRegion = variable.TiDBOptOn(val)
- }
- if len(parts) > 0 {
- preSplit = func() { splitPartitionTableRegion(ctx, sp, tbInfo, parts, scatterRegion) }
- } else {
- preSplit = func() { splitTableRegion(ctx, sp, tbInfo, scatterRegion) }
- }
- if scatterRegion {
- preSplit()
- } else {
- go preSplit()
- }
-}
-
-func (e *executor) FlashbackCluster(ctx sessionctx.Context, flashbackTS uint64) error {
- logutil.DDLLogger().Info("get flashback cluster job", zap.Stringer("flashbackTS", oracle.GetTimeFromTS(flashbackTS)))
- nowTS, err := ctx.GetStore().GetOracle().GetTimestamp(e.ctx, &oracle.Option{})
- if err != nil {
- return errors.Trace(err)
- }
- gap := time.Until(oracle.GetTimeFromTS(nowTS)).Abs()
- if gap > 1*time.Second {
- ctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackErrorf("Gap between local time and PD TSO is %s, please check PD/system time", gap))
- }
- job := &model.Job{
- Type: model.ActionFlashbackCluster,
- BinlogInfo: &model.HistoryInfo{},
- // The value for global variables is meaningless, it will cover during flashback cluster.
- Args: []any{
- flashbackTS,
- map[string]any{},
- true, /* tidb_gc_enable */
- variable.On, /* tidb_enable_auto_analyze */
- variable.Off, /* tidb_super_read_only */
- 0, /* totalRegions */
- 0, /* startTS */
- 0, /* commitTS */
- variable.On, /* tidb_ttl_job_enable */
- []kv.KeyRange{} /* flashback key_ranges */},
- CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
- // FLASHBACK CLUSTER affects all schemas and tables.
- InvolvingSchemaInfo: []model.InvolvingSchemaInfo{{
- Database: model.InvolvingAll,
- Table: model.InvolvingAll,
- }},
- SQLMode: ctx.GetSessionVars().SQLMode,
- }
- err = e.DoDDLJob(ctx, job)
- return errors.Trace(err)
-}
-
-func (e *executor) RecoverTable(ctx sessionctx.Context, recoverInfo *RecoverInfo) (err error) {
- is := e.infoCache.GetLatest()
- schemaID, tbInfo := recoverInfo.SchemaID, recoverInfo.TableInfo
- // Check schema exist.
- schema, ok := is.SchemaByID(schemaID)
- if !ok {
- return errors.Trace(infoschema.ErrDatabaseNotExists.GenWithStackByArgs(
- fmt.Sprintf("(Schema ID %d)", schemaID),
- ))
- }
- // Check not exist table with same name.
- if ok := is.TableExists(schema.Name, tbInfo.Name); ok {
- return infoschema.ErrTableExists.GenWithStackByArgs(tbInfo.Name)
- }
-
- // for "flashback table xxx to yyy"
- // Note: this case only allow change table name, schema remains the same.
- var involvedSchemas []model.InvolvingSchemaInfo
- if recoverInfo.OldTableName != tbInfo.Name.L {
- involvedSchemas = []model.InvolvingSchemaInfo{
- {Database: schema.Name.L, Table: recoverInfo.OldTableName},
- {Database: schema.Name.L, Table: tbInfo.Name.L},
- }
- }
-
- tbInfo.State = model.StateNone
- job := &model.Job{
- SchemaID: schemaID,
- TableID: tbInfo.ID,
- SchemaName: schema.Name.L,
- TableName: tbInfo.Name.L,
-
- Type: model.ActionRecoverTable,
- BinlogInfo: &model.HistoryInfo{},
- Args: []any{recoverInfo, recoverCheckFlagNone},
- InvolvingSchemaInfo: involvedSchemas,
- CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
- SQLMode: ctx.GetSessionVars().SQLMode,
- }
- err = e.DoDDLJob(ctx, job)
- return errors.Trace(err)
-}
-
-func (e *executor) CreateView(ctx sessionctx.Context, s *ast.CreateViewStmt) (err error) {
- viewInfo, err := BuildViewInfo(s)
- if err != nil {
- return err
- }
-
- cols := make([]*table.Column, len(s.Cols))
- for i, v := range s.Cols {
- cols[i] = table.ToColumn(&model.ColumnInfo{
- Name: v,
- ID: int64(i),
- Offset: i,
- State: model.StatePublic,
- })
- }
-
- tblCharset := ""
- tblCollate := ""
- if v, ok := ctx.GetSessionVars().GetSystemVar(variable.CharacterSetConnection); ok {
- tblCharset = v
- }
- if v, ok := ctx.GetSessionVars().GetSystemVar(variable.CollationConnection); ok {
- tblCollate = v
- }
-
- tbInfo, err := BuildTableInfo(ctx, s.ViewName.Name, cols, nil, tblCharset, tblCollate)
- if err != nil {
- return err
- }
- tbInfo.View = viewInfo
-
- onExist := OnExistError
- if s.OrReplace {
- onExist = OnExistReplace
- }
-
- return e.CreateTableWithInfo(ctx, s.ViewName.Schema, tbInfo, nil, WithOnExist(onExist))
-}
-
-// BuildViewInfo builds a ViewInfo structure from an ast.CreateViewStmt.
-func BuildViewInfo(s *ast.CreateViewStmt) (*model.ViewInfo, error) {
- // Always Use `format.RestoreNameBackQuotes` to restore `SELECT` statement despite the `ANSI_QUOTES` SQL Mode is enabled or not.
- restoreFlag := format.RestoreStringSingleQuotes | format.RestoreKeyWordUppercase | format.RestoreNameBackQuotes
- var sb strings.Builder
- if err := s.Select.Restore(format.NewRestoreCtx(restoreFlag, &sb)); err != nil {
- return nil, err
- }
-
- return &model.ViewInfo{Definer: s.Definer, Algorithm: s.Algorithm,
- Security: s.Security, SelectStmt: sb.String(), CheckOption: s.CheckOption, Cols: nil}, nil
-}
-
-func checkPartitionByHash(ctx sessionctx.Context, tbInfo *model.TableInfo) error {
- return checkNoHashPartitions(ctx, tbInfo.Partition.Num)
-}
-
-// checkPartitionByRange checks validity of a "BY RANGE" partition.
-func checkPartitionByRange(ctx sessionctx.Context, tbInfo *model.TableInfo) error {
- failpoint.Inject("CheckPartitionByRangeErr", func() {
- ctx.GetSessionVars().SQLKiller.SendKillSignal(sqlkiller.QueryMemoryExceeded)
- panic(ctx.GetSessionVars().SQLKiller.HandleSignal())
- })
- pi := tbInfo.Partition
-
- if len(pi.Columns) == 0 {
- return checkRangePartitionValue(ctx, tbInfo)
- }
-
- return checkRangeColumnsPartitionValue(ctx, tbInfo)
-}
-
-// checkPartitionByList checks validity of a "BY LIST" partition.
-func checkPartitionByList(ctx sessionctx.Context, tbInfo *model.TableInfo) error {
- return checkListPartitionValue(ctx.GetExprCtx(), tbInfo)
-}
-
-func isValidKeyPartitionColType(fieldType types.FieldType) bool {
- switch fieldType.GetType() {
- case mysql.TypeBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob, mysql.TypeJSON, mysql.TypeGeometry,
- mysql.TypeTiDBVectorFloat32:
- return false
- default:
- return true
- }
-}
-
-func isColTypeAllowedAsPartitioningCol(partType model.PartitionType, fieldType types.FieldType) bool {
- // For key partition, the permitted partition field types can be all field types except
- // BLOB, JSON, Geometry
- if partType == model.PartitionTypeKey {
- return isValidKeyPartitionColType(fieldType)
- }
- // The permitted data types are shown in the following list:
- // All integer types
- // DATE and DATETIME
- // CHAR, VARCHAR, BINARY, and VARBINARY
- // See https://dev.mysql.com/doc/mysql-partitioning-excerpt/5.7/en/partitioning-columns.html
- // Note that also TIME is allowed in MySQL. Also see https://bugs.mysql.com/bug.php?id=84362
- switch fieldType.GetType() {
- case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong:
- case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeDuration:
- case mysql.TypeVarchar, mysql.TypeString:
- default:
- return false
- }
- return true
-}
-
-func checkColumnsPartitionType(tbInfo *model.TableInfo) error {
- for _, col := range tbInfo.Partition.Columns {
- colInfo := tbInfo.FindPublicColumnByName(col.L)
- if colInfo == nil {
- return errors.Trace(dbterror.ErrFieldNotFoundPart)
- }
- if !isColTypeAllowedAsPartitioningCol(tbInfo.Partition.Type, colInfo.FieldType) {
- return dbterror.ErrNotAllowedTypeInPartition.GenWithStackByArgs(col.O)
- }
- }
- return nil
-}
-
-func checkRangeColumnsPartitionValue(ctx sessionctx.Context, tbInfo *model.TableInfo) error {
- // Range columns partition key supports multiple data types with integer、datetime、string.
- pi := tbInfo.Partition
- defs := pi.Definitions
- if len(defs) < 1 {
- return ast.ErrPartitionsMustBeDefined.GenWithStackByArgs("RANGE")
- }
-
- curr := &defs[0]
- if len(curr.LessThan) != len(pi.Columns) {
- return errors.Trace(ast.ErrPartitionColumnList)
- }
- var prev *model.PartitionDefinition
- for i := 1; i < len(defs); i++ {
- prev, curr = curr, &defs[i]
- succ, err := checkTwoRangeColumns(ctx, curr, prev, pi, tbInfo)
- if err != nil {
- return err
- }
- if !succ {
- return errors.Trace(dbterror.ErrRangeNotIncreasing)
- }
- }
- return nil
-}
-
-func checkTwoRangeColumns(ctx sessionctx.Context, curr, prev *model.PartitionDefinition, pi *model.PartitionInfo, tbInfo *model.TableInfo) (bool, error) {
- if len(curr.LessThan) != len(pi.Columns) {
- return false, errors.Trace(ast.ErrPartitionColumnList)
- }
- for i := 0; i < len(pi.Columns); i++ {
- // Special handling for MAXVALUE.
- if strings.EqualFold(curr.LessThan[i], partitionMaxValue) && !strings.EqualFold(prev.LessThan[i], partitionMaxValue) {
- // If current is maxvalue, it certainly >= previous.
- return true, nil
- }
- if strings.EqualFold(prev.LessThan[i], partitionMaxValue) {
- // Current is not maxvalue, and previous is maxvalue.
- return false, nil
- }
-
- // The tuples of column values used to define the partitions are strictly increasing:
- // PARTITION p0 VALUES LESS THAN (5,10,'ggg')
- // PARTITION p1 VALUES LESS THAN (10,20,'mmm')
- // PARTITION p2 VALUES LESS THAN (15,30,'sss')
- colInfo := findColumnByName(pi.Columns[i].L, tbInfo)
- cmp, err := parseAndEvalBoolExpr(ctx.GetExprCtx(), curr.LessThan[i], prev.LessThan[i], colInfo, tbInfo)
- if err != nil {
- return false, err
- }
-
- if cmp > 0 {
- return true, nil
- }
-
- if cmp < 0 {
- return false, nil
- }
- }
- return false, nil
-}
-
-// equal, return 0
-// greater, return 1
-// less, return -1
-func parseAndEvalBoolExpr(ctx expression.BuildContext, l, r string, colInfo *model.ColumnInfo, tbInfo *model.TableInfo) (int64, error) {
- lexpr, err := expression.ParseSimpleExpr(ctx, l, expression.WithTableInfo("", tbInfo), expression.WithCastExprTo(&colInfo.FieldType))
- if err != nil {
- return 0, err
- }
- rexpr, err := expression.ParseSimpleExpr(ctx, r, expression.WithTableInfo("", tbInfo), expression.WithCastExprTo(&colInfo.FieldType))
- if err != nil {
- return 0, err
- }
-
- e, err := expression.NewFunctionBase(ctx, ast.EQ, field_types.NewFieldType(mysql.TypeLonglong), lexpr, rexpr)
- if err != nil {
- return 0, err
- }
- e.SetCharsetAndCollation(colInfo.GetCharset(), colInfo.GetCollate())
- res, _, err1 := e.EvalInt(ctx.GetEvalCtx(), chunk.Row{})
- if err1 != nil {
- return 0, err1
- }
- if res == 1 {
- return 0, nil
- }
-
- e, err = expression.NewFunctionBase(ctx, ast.GT, field_types.NewFieldType(mysql.TypeLonglong), lexpr, rexpr)
- if err != nil {
- return 0, err
- }
- e.SetCharsetAndCollation(colInfo.GetCharset(), colInfo.GetCollate())
- res, _, err1 = e.EvalInt(ctx.GetEvalCtx(), chunk.Row{})
- if err1 != nil {
- return 0, err1
- }
- if res > 0 {
- return 1, nil
- }
- return -1, nil
-}
-
-func checkCharsetAndCollation(cs string, co string) error {
- if !charset.ValidCharsetAndCollation(cs, co) {
- return dbterror.ErrUnknownCharacterSet.GenWithStackByArgs(cs)
- }
- if co != "" {
- if _, err := collate.GetCollationByName(co); err != nil {
- return errors.Trace(err)
- }
- }
- return nil
-}
-
-// handleAutoIncID handles auto_increment option in DDL. It creates a ID counter for the table and initiates the counter to a proper value.
-// For example if the option sets auto_increment to 10. The counter will be set to 9. So the next allocated ID will be 10.
-func (e *executor) handleAutoIncID(tbInfo *model.TableInfo, schemaID int64, newEnd int64, tp autoid.AllocatorType) error {
- allocs := autoid.NewAllocatorsFromTblInfo(e.getAutoIDRequirement(), schemaID, tbInfo)
- if alloc := allocs.Get(tp); alloc != nil {
- err := alloc.Rebase(context.Background(), newEnd, false)
- if err != nil {
- return errors.Trace(err)
- }
- }
- return nil
-}
-
-// TODO we can unify this part with ddlCtx.
-func (e *executor) getAutoIDRequirement() autoid.Requirement {
- return &asAutoIDRequirement{
- store: e.store,
- autoidCli: e.autoidCli,
- }
-}
-
-// SetDirectPlacementOpt tries to make the PlacementSettings assignments generic for Schema/Table/Partition
-func SetDirectPlacementOpt(placementSettings *model.PlacementSettings, placementOptionType ast.PlacementOptionType, stringVal string, uintVal uint64) error {
- switch placementOptionType {
- case ast.PlacementOptionPrimaryRegion:
- placementSettings.PrimaryRegion = stringVal
- case ast.PlacementOptionRegions:
- placementSettings.Regions = stringVal
- case ast.PlacementOptionFollowerCount:
- placementSettings.Followers = uintVal
- case ast.PlacementOptionVoterCount:
- placementSettings.Voters = uintVal
- case ast.PlacementOptionLearnerCount:
- placementSettings.Learners = uintVal
- case ast.PlacementOptionSchedule:
- placementSettings.Schedule = stringVal
- case ast.PlacementOptionConstraints:
- placementSettings.Constraints = stringVal
- case ast.PlacementOptionLeaderConstraints:
- placementSettings.LeaderConstraints = stringVal
- case ast.PlacementOptionLearnerConstraints:
- placementSettings.LearnerConstraints = stringVal
- case ast.PlacementOptionFollowerConstraints:
- placementSettings.FollowerConstraints = stringVal
- case ast.PlacementOptionVoterConstraints:
- placementSettings.VoterConstraints = stringVal
- case ast.PlacementOptionSurvivalPreferences:
- placementSettings.SurvivalPreferences = stringVal
- default:
- return errors.Trace(errors.New("unknown placement policy option"))
- }
- return nil
-}
-
-// SetDirectResourceGroupSettings tries to set the ResourceGroupSettings.
-func SetDirectResourceGroupSettings(groupInfo *model.ResourceGroupInfo, opt *ast.ResourceGroupOption) error {
- resourceGroupSettings := groupInfo.ResourceGroupSettings
- switch opt.Tp {
- case ast.ResourceRURate:
- resourceGroupSettings.RURate = opt.UintValue
- case ast.ResourcePriority:
- resourceGroupSettings.Priority = opt.UintValue
- case ast.ResourceUnitCPU:
- resourceGroupSettings.CPULimiter = opt.StrValue
- case ast.ResourceUnitIOReadBandwidth:
- resourceGroupSettings.IOReadBandwidth = opt.StrValue
- case ast.ResourceUnitIOWriteBandwidth:
- resourceGroupSettings.IOWriteBandwidth = opt.StrValue
- case ast.ResourceBurstableOpiton:
- // Some about BurstLimit(b):
- // - If b == 0, that means the limiter is unlimited capacity. default use in resource controller (burst with a rate within a unlimited capacity).
- // - If b < 0, that means the limiter is unlimited capacity and fillrate(r) is ignored, can be seen as r == Inf (burst with a inf rate within a unlimited capacity).
- // - If b > 0, that means the limiter is limited capacity. (current not used).
- limit := int64(0)
- if opt.BoolValue {
- limit = -1
- }
- resourceGroupSettings.BurstLimit = limit
- case ast.ResourceGroupRunaway:
- if len(opt.RunawayOptionList) == 0 {
- resourceGroupSettings.Runaway = nil
- }
- for _, opt := range opt.RunawayOptionList {
- if err := SetDirectResourceGroupRunawayOption(resourceGroupSettings, opt); err != nil {
- return err
- }
- }
- case ast.ResourceGroupBackground:
- if groupInfo.Name.L != rg.DefaultResourceGroupName {
- // FIXME: this is a temporary restriction, so we don't add a error-code for it.
- return errors.New("unsupported operation. Currently, only the default resource group support change background settings")
- }
- if len(opt.BackgroundOptions) == 0 {
- resourceGroupSettings.Background = nil
- }
- for _, opt := range opt.BackgroundOptions {
- if err := SetDirectResourceGroupBackgroundOption(resourceGroupSettings, opt); err != nil {
- return err
- }
- }
- default:
- return errors.Trace(errors.New("unknown resource unit type"))
- }
- return nil
-}
-
-// SetDirectResourceGroupRunawayOption tries to set runaway part of the ResourceGroupSettings.
-func SetDirectResourceGroupRunawayOption(resourceGroupSettings *model.ResourceGroupSettings, opt *ast.ResourceGroupRunawayOption) error {
- if resourceGroupSettings.Runaway == nil {
- resourceGroupSettings.Runaway = &model.ResourceGroupRunawaySettings{}
- }
- settings := resourceGroupSettings.Runaway
- switch opt.Tp {
- case ast.RunawayRule:
- // because execute time won't be too long, we use `time` pkg which does not support to parse unit 'd'.
- dur, err := time.ParseDuration(opt.RuleOption.ExecElapsed)
- if err != nil {
- return err
- }
- settings.ExecElapsedTimeMs = uint64(dur.Milliseconds())
- case ast.RunawayAction:
- settings.Action = opt.ActionOption.Type
- case ast.RunawayWatch:
- settings.WatchType = opt.WatchOption.Type
- if dur := opt.WatchOption.Duration; len(dur) > 0 {
- dur, err := time.ParseDuration(dur)
- if err != nil {
- return err
- }
- settings.WatchDurationMs = dur.Milliseconds()
- } else {
- settings.WatchDurationMs = 0
- }
- default:
- return errors.Trace(errors.New("unknown runaway option type"))
- }
- return nil
-}
-
-// SetDirectResourceGroupBackgroundOption set background configs of the ResourceGroupSettings.
-func SetDirectResourceGroupBackgroundOption(resourceGroupSettings *model.ResourceGroupSettings, opt *ast.ResourceGroupBackgroundOption) error {
- if resourceGroupSettings.Background == nil {
- resourceGroupSettings.Background = &model.ResourceGroupBackgroundSettings{}
- }
- switch opt.Type {
- case ast.BackgroundOptionTaskNames:
- jobTypes, err := parseBackgroundJobTypes(opt.StrValue)
- if err != nil {
- return err
- }
- resourceGroupSettings.Background.JobTypes = jobTypes
- default:
- return errors.Trace(errors.New("unknown background option type"))
- }
- return nil
-}
-
-func parseBackgroundJobTypes(t string) ([]string, error) {
- if len(t) == 0 {
- return []string{}, nil
- }
-
- segs := strings.Split(t, ",")
- res := make([]string, 0, len(segs))
- for _, s := range segs {
- ty := strings.ToLower(strings.TrimSpace(s))
- if len(ty) > 0 {
- if !slices.Contains(kvutil.ExplicitTypeList, ty) {
- return nil, infoschema.ErrResourceGroupInvalidBackgroundTaskName.GenWithStackByArgs(ty)
- }
- res = append(res, ty)
- }
- }
- return res, nil
-}
-
-// handleTableOptions updates tableInfo according to table options.
-func handleTableOptions(options []*ast.TableOption, tbInfo *model.TableInfo) error {
- var ttlOptionsHandled bool
-
- for _, op := range options {
- switch op.Tp {
- case ast.TableOptionAutoIncrement:
- tbInfo.AutoIncID = int64(op.UintValue)
- case ast.TableOptionAutoIdCache:
- if op.UintValue > uint64(math.MaxInt64) {
- // TODO: Refine this error.
- return errors.New("table option auto_id_cache overflows int64")
- }
- tbInfo.AutoIdCache = int64(op.UintValue)
- case ast.TableOptionAutoRandomBase:
- tbInfo.AutoRandID = int64(op.UintValue)
- case ast.TableOptionComment:
- tbInfo.Comment = op.StrValue
- case ast.TableOptionCompression:
- tbInfo.Compression = op.StrValue
- case ast.TableOptionShardRowID:
- if op.UintValue > 0 && tbInfo.HasClusteredIndex() {
- return dbterror.ErrUnsupportedShardRowIDBits
- }
- tbInfo.ShardRowIDBits = op.UintValue
- if tbInfo.ShardRowIDBits > shardRowIDBitsMax {
- tbInfo.ShardRowIDBits = shardRowIDBitsMax
- }
- tbInfo.MaxShardRowIDBits = tbInfo.ShardRowIDBits
- case ast.TableOptionPreSplitRegion:
- if tbInfo.TempTableType != model.TempTableNone {
- return errors.Trace(dbterror.ErrOptOnTemporaryTable.GenWithStackByArgs("pre split regions"))
- }
- tbInfo.PreSplitRegions = op.UintValue
- case ast.TableOptionCharset, ast.TableOptionCollate:
- // We don't handle charset and collate here since they're handled in `GetCharsetAndCollateInTableOption`.
- case ast.TableOptionPlacementPolicy:
- tbInfo.PlacementPolicyRef = &model.PolicyRefInfo{
- Name: model.NewCIStr(op.StrValue),
- }
- case ast.TableOptionTTL, ast.TableOptionTTLEnable, ast.TableOptionTTLJobInterval:
- if ttlOptionsHandled {
- continue
- }
-
- ttlInfo, ttlEnable, ttlJobInterval, err := getTTLInfoInOptions(options)
- if err != nil {
- return err
- }
- // It's impossible that `ttlInfo` and `ttlEnable` are all nil, because we have met this option.
- // After exclude the situation `ttlInfo == nil && ttlEnable != nil`, we could say `ttlInfo != nil`
- if ttlInfo == nil {
- if ttlEnable != nil {
- return errors.Trace(dbterror.ErrSetTTLOptionForNonTTLTable.FastGenByArgs("TTL_ENABLE"))
- }
- if ttlJobInterval != nil {
- return errors.Trace(dbterror.ErrSetTTLOptionForNonTTLTable.FastGenByArgs("TTL_JOB_INTERVAL"))
- }
- }
-
- tbInfo.TTLInfo = ttlInfo
- ttlOptionsHandled = true
- }
- }
- shardingBits := shardingBits(tbInfo)
- if tbInfo.PreSplitRegions > shardingBits {
- tbInfo.PreSplitRegions = shardingBits
- }
- return nil
-}
-
-func shardingBits(tblInfo *model.TableInfo) uint64 {
- if tblInfo.ShardRowIDBits > 0 {
- return tblInfo.ShardRowIDBits
- }
- return tblInfo.AutoRandomBits
-}
-
-// isIgnorableSpec checks if the spec type is ignorable.
-// Some specs are parsed by ignored. This is for compatibility.
-func isIgnorableSpec(tp ast.AlterTableType) bool {
- // AlterTableLock/AlterTableAlgorithm are ignored.
- return tp == ast.AlterTableLock || tp == ast.AlterTableAlgorithm
-}
-
-// getCharsetAndCollateInColumnDef will iterate collate in the options, validate it by checking the charset
-// of column definition. If there's no collate in the option, the default collate of column's charset will be used.
-func getCharsetAndCollateInColumnDef(sessVars *variable.SessionVars, def *ast.ColumnDef) (chs, coll string, err error) {
- chs = def.Tp.GetCharset()
- coll = def.Tp.GetCollate()
- if chs != "" && coll == "" {
- if coll, err = GetDefaultCollation(sessVars, chs); err != nil {
- return "", "", errors.Trace(err)
- }
- }
- for _, opt := range def.Options {
- if opt.Tp == ast.ColumnOptionCollate {
- info, err := collate.GetCollationByName(opt.StrValue)
- if err != nil {
- return "", "", errors.Trace(err)
- }
- if chs == "" {
- chs = info.CharsetName
- } else if chs != info.CharsetName {
- return "", "", dbterror.ErrCollationCharsetMismatch.GenWithStackByArgs(info.Name, chs)
- }
- coll = info.Name
- }
- }
- return
-}
-
-// GetCharsetAndCollateInTableOption will iterate the charset and collate in the options,
-// and returns the last charset and collate in options. If there is no charset in the options,
-// the returns charset will be "", the same as collate.
-func GetCharsetAndCollateInTableOption(sessVars *variable.SessionVars, startIdx int, options []*ast.TableOption) (chs, coll string, err error) {
- for i := startIdx; i < len(options); i++ {
- opt := options[i]
- // we set the charset to the last option. example: alter table t charset latin1 charset utf8 collate utf8_bin;
- // the charset will be utf8, collate will be utf8_bin
- switch opt.Tp {
- case ast.TableOptionCharset:
- info, err := charset.GetCharsetInfo(opt.StrValue)
- if err != nil {
- return "", "", err
- }
- if len(chs) == 0 {
- chs = info.Name
- } else if chs != info.Name {
- return "", "", dbterror.ErrConflictingDeclarations.GenWithStackByArgs(chs, info.Name)
- }
- if len(coll) == 0 {
- defaultColl, err := getDefaultCollationForUTF8MB4(sessVars, chs)
- if err != nil {
- return "", "", errors.Trace(err)
- }
- if len(defaultColl) == 0 {
- coll = info.DefaultCollation
- } else {
- coll = defaultColl
- }
- }
- case ast.TableOptionCollate:
- info, err := collate.GetCollationByName(opt.StrValue)
- if err != nil {
- return "", "", err
- }
- if len(chs) == 0 {
- chs = info.CharsetName
- } else if chs != info.CharsetName {
- return "", "", dbterror.ErrCollationCharsetMismatch.GenWithStackByArgs(info.Name, chs)
- }
- coll = info.Name
- }
- }
- return
-}
-
-// NeedToOverwriteColCharset return true for altering charset and specified CONVERT TO.
-func NeedToOverwriteColCharset(options []*ast.TableOption) bool {
- for i := len(options) - 1; i >= 0; i-- {
- opt := options[i]
- if opt.Tp == ast.TableOptionCharset {
- // Only overwrite columns charset if the option contains `CONVERT TO`.
- return opt.UintValue == ast.TableOptionCharsetWithConvertTo
- }
- }
- return false
-}
-
-// resolveAlterTableAddColumns splits "add columns" to multiple spec. For example,
-// `ALTER TABLE ADD COLUMN (c1 INT, c2 INT)` is split into
-// `ALTER TABLE ADD COLUMN c1 INT, ADD COLUMN c2 INT`.
-func resolveAlterTableAddColumns(spec *ast.AlterTableSpec) []*ast.AlterTableSpec {
- specs := make([]*ast.AlterTableSpec, 0, len(spec.NewColumns)+len(spec.NewConstraints))
- for _, col := range spec.NewColumns {
- t := *spec
- t.NewColumns = []*ast.ColumnDef{col}
- t.NewConstraints = []*ast.Constraint{}
- specs = append(specs, &t)
- }
- // Split the add constraints from AlterTableSpec.
- for _, con := range spec.NewConstraints {
- t := *spec
- t.NewColumns = []*ast.ColumnDef{}
- t.NewConstraints = []*ast.Constraint{}
- t.Constraint = con
- t.Tp = ast.AlterTableAddConstraint
- specs = append(specs, &t)
- }
- return specs
-}
-
-// ResolveAlterTableSpec resolves alter table algorithm and removes ignore table spec in specs.
-// returns valid specs, and the occurred error.
-func ResolveAlterTableSpec(ctx sessionctx.Context, specs []*ast.AlterTableSpec) ([]*ast.AlterTableSpec, error) {
- validSpecs := make([]*ast.AlterTableSpec, 0, len(specs))
- algorithm := ast.AlgorithmTypeDefault
- for _, spec := range specs {
- if spec.Tp == ast.AlterTableAlgorithm {
- // Find the last AlterTableAlgorithm.
- algorithm = spec.Algorithm
- }
- if isIgnorableSpec(spec.Tp) {
- continue
- }
- if spec.Tp == ast.AlterTableAddColumns && (len(spec.NewColumns) > 1 || len(spec.NewConstraints) > 0) {
- validSpecs = append(validSpecs, resolveAlterTableAddColumns(spec)...)
- } else {
- validSpecs = append(validSpecs, spec)
- }
- // TODO: Only allow REMOVE PARTITIONING as a single ALTER TABLE statement?
- }
-
- // Verify whether the algorithm is supported.
- for _, spec := range validSpecs {
- resolvedAlgorithm, err := ResolveAlterAlgorithm(spec, algorithm)
- if err != nil {
- // If TiDB failed to choose a better algorithm, report the error
- if resolvedAlgorithm == ast.AlgorithmTypeDefault {
- return nil, errors.Trace(err)
- }
- // For the compatibility, we return warning instead of error when a better algorithm is chosed by TiDB
- ctx.GetSessionVars().StmtCtx.AppendError(err)
- }
-
- spec.Algorithm = resolvedAlgorithm
- }
-
- // Only handle valid specs.
- return validSpecs, nil
-}
-
-func isMultiSchemaChanges(specs []*ast.AlterTableSpec) bool {
- if len(specs) > 1 {
- return true
- }
- if len(specs) == 1 && len(specs[0].NewColumns) > 1 && specs[0].Tp == ast.AlterTableAddColumns {
- return true
- }
- return false
-}
-
-func (e *executor) AlterTable(ctx context.Context, sctx sessionctx.Context, stmt *ast.AlterTableStmt) (err error) {
- ident := ast.Ident{Schema: stmt.Table.Schema, Name: stmt.Table.Name}
- validSpecs, err := ResolveAlterTableSpec(sctx, stmt.Specs)
- if err != nil {
- return errors.Trace(err)
- }
-
- is := e.infoCache.GetLatest()
- tb, err := is.TableByName(ctx, ident.Schema, ident.Name)
- if err != nil {
- return errors.Trace(err)
- }
- if tb.Meta().IsView() || tb.Meta().IsSequence() {
- return dbterror.ErrWrongObject.GenWithStackByArgs(ident.Schema, ident.Name, "BASE TABLE")
- }
- if tb.Meta().TableCacheStatusType != model.TableCacheStatusDisable {
- if len(validSpecs) != 1 {
- return dbterror.ErrOptOnCacheTable.GenWithStackByArgs("Alter Table")
- }
- if validSpecs[0].Tp != ast.AlterTableCache && validSpecs[0].Tp != ast.AlterTableNoCache {
- return dbterror.ErrOptOnCacheTable.GenWithStackByArgs("Alter Table")
- }
- }
- if isMultiSchemaChanges(validSpecs) && (sctx.GetSessionVars().EnableRowLevelChecksum || variable.EnableRowLevelChecksum.Load()) {
- return dbterror.ErrRunMultiSchemaChanges.GenWithStack("Unsupported multi schema change when row level checksum is enabled")
- }
- // set name for anonymous foreign key.
- maxForeignKeyID := tb.Meta().MaxForeignKeyID
- for _, spec := range validSpecs {
- if spec.Tp == ast.AlterTableAddConstraint && spec.Constraint.Tp == ast.ConstraintForeignKey && spec.Constraint.Name == "" {
- maxForeignKeyID++
- spec.Constraint.Name = fmt.Sprintf("fk_%d", maxForeignKeyID)
- }
- }
-
- if len(validSpecs) > 1 {
- // after MultiSchemaInfo is set, DoDDLJob will collect all jobs into
- // MultiSchemaInfo and skip running them. Then we will run them in
- // d.MultiSchemaChange all at once.
- sctx.GetSessionVars().StmtCtx.MultiSchemaInfo = model.NewMultiSchemaInfo()
- }
- for _, spec := range validSpecs {
- var handledCharsetOrCollate bool
- var ttlOptionsHandled bool
- switch spec.Tp {
- case ast.AlterTableAddColumns:
- err = e.AddColumn(sctx, ident, spec)
- case ast.AlterTableAddPartitions, ast.AlterTableAddLastPartition:
- err = e.AddTablePartitions(sctx, ident, spec)
- case ast.AlterTableCoalescePartitions:
- err = e.CoalescePartitions(sctx, ident, spec)
- case ast.AlterTableReorganizePartition:
- err = e.ReorganizePartitions(sctx, ident, spec)
- case ast.AlterTableReorganizeFirstPartition:
- err = dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("MERGE FIRST PARTITION")
- case ast.AlterTableReorganizeLastPartition:
- err = dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("SPLIT LAST PARTITION")
- case ast.AlterTableCheckPartitions:
- err = errors.Trace(dbterror.ErrUnsupportedCheckPartition)
- case ast.AlterTableRebuildPartition:
- err = errors.Trace(dbterror.ErrUnsupportedRebuildPartition)
- case ast.AlterTableOptimizePartition:
- err = errors.Trace(dbterror.ErrUnsupportedOptimizePartition)
- case ast.AlterTableRemovePartitioning:
- err = e.RemovePartitioning(sctx, ident, spec)
- case ast.AlterTableRepairPartition:
- err = errors.Trace(dbterror.ErrUnsupportedRepairPartition)
- case ast.AlterTableDropColumn:
- err = e.DropColumn(sctx, ident, spec)
- case ast.AlterTableDropIndex:
- err = e.dropIndex(sctx, ident, model.NewCIStr(spec.Name), spec.IfExists, false)
- case ast.AlterTableDropPrimaryKey:
- err = e.dropIndex(sctx, ident, model.NewCIStr(mysql.PrimaryKeyName), spec.IfExists, false)
- case ast.AlterTableRenameIndex:
- err = e.RenameIndex(sctx, ident, spec)
- case ast.AlterTableDropPartition, ast.AlterTableDropFirstPartition:
- err = e.DropTablePartition(sctx, ident, spec)
- case ast.AlterTableTruncatePartition:
- err = e.TruncateTablePartition(sctx, ident, spec)
- case ast.AlterTableWriteable:
- if !config.TableLockEnabled() {
- return nil
- }
- tName := &ast.TableName{Schema: ident.Schema, Name: ident.Name}
- if spec.Writeable {
- err = e.CleanupTableLock(sctx, []*ast.TableName{tName})
- } else {
- lockStmt := &ast.LockTablesStmt{
- TableLocks: []ast.TableLock{
- {
- Table: tName,
- Type: model.TableLockReadOnly,
- },
- },
- }
- err = e.LockTables(sctx, lockStmt)
- }
- case ast.AlterTableExchangePartition:
- err = e.ExchangeTablePartition(sctx, ident, spec)
- case ast.AlterTableAddConstraint:
- constr := spec.Constraint
- switch spec.Constraint.Tp {
- case ast.ConstraintKey, ast.ConstraintIndex:
- err = e.createIndex(sctx, ident, ast.IndexKeyTypeNone, model.NewCIStr(constr.Name),
- spec.Constraint.Keys, constr.Option, constr.IfNotExists)
- case ast.ConstraintUniq, ast.ConstraintUniqIndex, ast.ConstraintUniqKey:
- err = e.createIndex(sctx, ident, ast.IndexKeyTypeUnique, model.NewCIStr(constr.Name),
- spec.Constraint.Keys, constr.Option, false) // IfNotExists should be not applied
- case ast.ConstraintForeignKey:
- // NOTE: we do not handle `symbol` and `index_name` well in the parser and we do not check ForeignKey already exists,
- // so we just also ignore the `if not exists` check.
- err = e.CreateForeignKey(sctx, ident, model.NewCIStr(constr.Name), spec.Constraint.Keys, spec.Constraint.Refer)
- case ast.ConstraintPrimaryKey:
- err = e.CreatePrimaryKey(sctx, ident, model.NewCIStr(constr.Name), spec.Constraint.Keys, constr.Option)
- case ast.ConstraintFulltext:
- sctx.GetSessionVars().StmtCtx.AppendWarning(dbterror.ErrTableCantHandleFt)
- case ast.ConstraintCheck:
- if !variable.EnableCheckConstraint.Load() {
- sctx.GetSessionVars().StmtCtx.AppendWarning(errCheckConstraintIsOff)
- } else {
- err = e.CreateCheckConstraint(sctx, ident, model.NewCIStr(constr.Name), spec.Constraint)
- }
- default:
- // Nothing to do now.
- }
- case ast.AlterTableDropForeignKey:
- // NOTE: we do not check `if not exists` and `if exists` for ForeignKey now.
- err = e.DropForeignKey(sctx, ident, model.NewCIStr(spec.Name))
- case ast.AlterTableModifyColumn:
- err = e.ModifyColumn(ctx, sctx, ident, spec)
- case ast.AlterTableChangeColumn:
- err = e.ChangeColumn(ctx, sctx, ident, spec)
- case ast.AlterTableRenameColumn:
- err = e.RenameColumn(sctx, ident, spec)
- case ast.AlterTableAlterColumn:
- err = e.AlterColumn(sctx, ident, spec)
- case ast.AlterTableRenameTable:
- newIdent := ast.Ident{Schema: spec.NewTable.Schema, Name: spec.NewTable.Name}
- isAlterTable := true
- err = e.renameTable(sctx, ident, newIdent, isAlterTable)
- case ast.AlterTablePartition:
- err = e.AlterTablePartitioning(sctx, ident, spec)
- case ast.AlterTableOption:
- var placementPolicyRef *model.PolicyRefInfo
- for i, opt := range spec.Options {
- switch opt.Tp {
- case ast.TableOptionShardRowID:
- if opt.UintValue > shardRowIDBitsMax {
- opt.UintValue = shardRowIDBitsMax
- }
- err = e.ShardRowID(sctx, ident, opt.UintValue)
- case ast.TableOptionAutoIncrement:
- err = e.RebaseAutoID(sctx, ident, int64(opt.UintValue), autoid.AutoIncrementType, opt.BoolValue)
- case ast.TableOptionAutoIdCache:
- if opt.UintValue > uint64(math.MaxInt64) {
- // TODO: Refine this error.
- return errors.New("table option auto_id_cache overflows int64")
- }
- err = e.AlterTableAutoIDCache(sctx, ident, int64(opt.UintValue))
- case ast.TableOptionAutoRandomBase:
- err = e.RebaseAutoID(sctx, ident, int64(opt.UintValue), autoid.AutoRandomType, opt.BoolValue)
- case ast.TableOptionComment:
- spec.Comment = opt.StrValue
- err = e.AlterTableComment(sctx, ident, spec)
- case ast.TableOptionCharset, ast.TableOptionCollate:
- // GetCharsetAndCollateInTableOption will get the last charset and collate in the options,
- // so it should be handled only once.
- if handledCharsetOrCollate {
- continue
- }
- var toCharset, toCollate string
- toCharset, toCollate, err = GetCharsetAndCollateInTableOption(sctx.GetSessionVars(), i, spec.Options)
- if err != nil {
- return err
- }
- needsOverwriteCols := NeedToOverwriteColCharset(spec.Options)
- err = e.AlterTableCharsetAndCollate(sctx, ident, toCharset, toCollate, needsOverwriteCols)
- handledCharsetOrCollate = true
- case ast.TableOptionPlacementPolicy:
- placementPolicyRef = &model.PolicyRefInfo{
- Name: model.NewCIStr(opt.StrValue),
- }
- case ast.TableOptionEngine:
- case ast.TableOptionRowFormat:
- case ast.TableOptionTTL, ast.TableOptionTTLEnable, ast.TableOptionTTLJobInterval:
- var ttlInfo *model.TTLInfo
- var ttlEnable *bool
- var ttlJobInterval *string
-
- if ttlOptionsHandled {
- continue
- }
- ttlInfo, ttlEnable, ttlJobInterval, err = getTTLInfoInOptions(spec.Options)
- if err != nil {
- return err
- }
- err = e.AlterTableTTLInfoOrEnable(sctx, ident, ttlInfo, ttlEnable, ttlJobInterval)
-
- ttlOptionsHandled = true
- default:
- err = dbterror.ErrUnsupportedAlterTableOption
- }
-
- if err != nil {
- return errors.Trace(err)
- }
- }
-
- if placementPolicyRef != nil {
- err = e.AlterTablePlacement(sctx, ident, placementPolicyRef)
- }
- case ast.AlterTableSetTiFlashReplica:
- err = e.AlterTableSetTiFlashReplica(sctx, ident, spec.TiFlashReplica)
- case ast.AlterTableOrderByColumns:
- err = e.OrderByColumns(sctx, ident)
- case ast.AlterTableIndexInvisible:
- err = e.AlterIndexVisibility(sctx, ident, spec.IndexName, spec.Visibility)
- case ast.AlterTableAlterCheck:
- if !variable.EnableCheckConstraint.Load() {
- sctx.GetSessionVars().StmtCtx.AppendWarning(errCheckConstraintIsOff)
- } else {
- err = e.AlterCheckConstraint(sctx, ident, model.NewCIStr(spec.Constraint.Name), spec.Constraint.Enforced)
- }
- case ast.AlterTableDropCheck:
- if !variable.EnableCheckConstraint.Load() {
- sctx.GetSessionVars().StmtCtx.AppendWarning(errCheckConstraintIsOff)
- } else {
- err = e.DropCheckConstraint(sctx, ident, model.NewCIStr(spec.Constraint.Name))
- }
- case ast.AlterTableWithValidation:
- sctx.GetSessionVars().StmtCtx.AppendWarning(dbterror.ErrUnsupportedAlterTableWithValidation)
- case ast.AlterTableWithoutValidation:
- sctx.GetSessionVars().StmtCtx.AppendWarning(dbterror.ErrUnsupportedAlterTableWithoutValidation)
- case ast.AlterTableAddStatistics:
- err = e.AlterTableAddStatistics(sctx, ident, spec.Statistics, spec.IfNotExists)
- case ast.AlterTableDropStatistics:
- err = e.AlterTableDropStatistics(sctx, ident, spec.Statistics, spec.IfExists)
- case ast.AlterTableAttributes:
- err = e.AlterTableAttributes(sctx, ident, spec)
- case ast.AlterTablePartitionAttributes:
- err = e.AlterTablePartitionAttributes(sctx, ident, spec)
- case ast.AlterTablePartitionOptions:
- err = e.AlterTablePartitionOptions(sctx, ident, spec)
- case ast.AlterTableCache:
- err = e.AlterTableCache(sctx, ident)
- case ast.AlterTableNoCache:
- err = e.AlterTableNoCache(sctx, ident)
- case ast.AlterTableDisableKeys, ast.AlterTableEnableKeys:
- // Nothing to do now, see https://github.com/pingcap/tidb/issues/1051
- // MyISAM specific
- case ast.AlterTableRemoveTTL:
- // the parser makes sure we have only one `ast.AlterTableRemoveTTL` in an alter statement
- err = e.AlterTableRemoveTTL(sctx, ident)
- default:
- err = errors.Trace(dbterror.ErrUnsupportedAlterTableSpec)
- }
-
- if err != nil {
- return errors.Trace(err)
- }
- }
-
- if sctx.GetSessionVars().StmtCtx.MultiSchemaInfo != nil {
- info := sctx.GetSessionVars().StmtCtx.MultiSchemaInfo
- sctx.GetSessionVars().StmtCtx.MultiSchemaInfo = nil
- err = e.MultiSchemaChange(sctx, ident, info)
- if err != nil {
- return errors.Trace(err)
- }
- }
-
- return nil
-}
-
-func (e *executor) RebaseAutoID(ctx sessionctx.Context, ident ast.Ident, newBase int64, tp autoid.AllocatorType, force bool) error {
- schema, t, err := e.getSchemaAndTableByIdent(ident)
- if err != nil {
- return errors.Trace(err)
- }
- tbInfo := t.Meta()
- var actionType model.ActionType
- switch tp {
- case autoid.AutoRandomType:
- pkCol := tbInfo.GetPkColInfo()
- if tbInfo.AutoRandomBits == 0 || pkCol == nil {
- return errors.Trace(dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomRebaseNotApplicable))
- }
- shardFmt := autoid.NewShardIDFormat(&pkCol.FieldType, tbInfo.AutoRandomBits, tbInfo.AutoRandomRangeBits)
- if shardFmt.IncrementalMask()&newBase != newBase {
- errMsg := fmt.Sprintf(autoid.AutoRandomRebaseOverflow, newBase, shardFmt.IncrementalBitsCapacity())
- return errors.Trace(dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(errMsg))
- }
- actionType = model.ActionRebaseAutoRandomBase
- case autoid.RowIDAllocType:
- actionType = model.ActionRebaseAutoID
- case autoid.AutoIncrementType:
- actionType = model.ActionRebaseAutoID
- default:
- panic(fmt.Sprintf("unimplemented rebase autoid type %s", tp))
- }
-
- if !force {
- newBaseTemp, err := adjustNewBaseToNextGlobalID(ctx.GetTableCtx(), t, tp, newBase)
- if err != nil {
- return err
- }
- if newBase != newBaseTemp {
- ctx.GetSessionVars().StmtCtx.AppendWarning(
- errors.NewNoStackErrorf("Can't reset AUTO_INCREMENT to %d without FORCE option, using %d instead",
- newBase, newBaseTemp,
- ))
- }
- newBase = newBaseTemp
- }
- job := &model.Job{
- SchemaID: schema.ID,
- TableID: tbInfo.ID,
- SchemaName: schema.Name.L,
- TableName: tbInfo.Name.L,
- Type: actionType,
- BinlogInfo: &model.HistoryInfo{},
- Args: []any{newBase, force},
- CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
- SQLMode: ctx.GetSessionVars().SQLMode,
- }
- err = e.DoDDLJob(ctx, job)
- return errors.Trace(err)
-}
-
-func adjustNewBaseToNextGlobalID(ctx table.AllocatorContext, t table.Table, tp autoid.AllocatorType, newBase int64) (int64, error) {
- alloc := t.Allocators(ctx).Get(tp)
- if alloc == nil {
- return newBase, nil
- }
- autoID, err := alloc.NextGlobalAutoID()
- if err != nil {
- return newBase, errors.Trace(err)
- }
- // If newBase < autoID, we need to do a rebase before returning.
- // Assume there are 2 TiDB servers: TiDB-A with allocator range of 0 ~ 30000; TiDB-B with allocator range of 30001 ~ 60000.
- // If the user sends SQL `alter table t1 auto_increment = 100` to TiDB-B,
- // and TiDB-B finds 100 < 30001 but returns without any handling,
- // then TiDB-A may still allocate 99 for auto_increment column. This doesn't make sense for the user.
- return int64(mathutil.Max(uint64(newBase), uint64(autoID))), nil
-}
-
-// ShardRowID shards the implicit row ID by adding shard value to the row ID's first few bits.
-func (e *executor) ShardRowID(ctx sessionctx.Context, tableIdent ast.Ident, uVal uint64) error {
- schema, t, err := e.getSchemaAndTableByIdent(tableIdent)
- if err != nil {
- return errors.Trace(err)
- }
- tbInfo := t.Meta()
- if tbInfo.TempTableType != model.TempTableNone {
- return dbterror.ErrOptOnTemporaryTable.GenWithStackByArgs("shard_row_id_bits")
- }
- if uVal == tbInfo.ShardRowIDBits {
- // Nothing need to do.
- return nil
- }
- if uVal > 0 && tbInfo.HasClusteredIndex() {
- return dbterror.ErrUnsupportedShardRowIDBits
- }
- err = verifyNoOverflowShardBits(e.sessPool, t, uVal)
- if err != nil {
- return err
- }
- job := &model.Job{
- Type: model.ActionShardRowID,
- SchemaID: schema.ID,
- TableID: tbInfo.ID,
- SchemaName: schema.Name.L,
- TableName: tbInfo.Name.L,
- BinlogInfo: &model.HistoryInfo{},
- Args: []any{uVal},
- CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
- SQLMode: ctx.GetSessionVars().SQLMode,
- }
- err = e.DoDDLJob(ctx, job)
- return errors.Trace(err)
-}
-
-func (e *executor) getSchemaAndTableByIdent(tableIdent ast.Ident) (dbInfo *model.DBInfo, t table.Table, err error) {
- is := e.infoCache.GetLatest()
- schema, ok := is.SchemaByName(tableIdent.Schema)
- if !ok {
- return nil, nil, infoschema.ErrDatabaseNotExists.GenWithStackByArgs(tableIdent.Schema)
- }
- t, err = is.TableByName(e.ctx, tableIdent.Schema, tableIdent.Name)
- if err != nil {
- return nil, nil, infoschema.ErrTableNotExists.GenWithStackByArgs(tableIdent.Schema, tableIdent.Name)
- }
- return schema, t, nil
-}
-
-func checkUnsupportedColumnConstraint(col *ast.ColumnDef, ti ast.Ident) error {
- for _, constraint := range col.Options {
- switch constraint.Tp {
- case ast.ColumnOptionAutoIncrement:
- return dbterror.ErrUnsupportedAddColumn.GenWithStack("unsupported add column '%s' constraint AUTO_INCREMENT when altering '%s.%s'", col.Name, ti.Schema, ti.Name)
- case ast.ColumnOptionPrimaryKey:
- return dbterror.ErrUnsupportedAddColumn.GenWithStack("unsupported add column '%s' constraint PRIMARY KEY when altering '%s.%s'", col.Name, ti.Schema, ti.Name)
- case ast.ColumnOptionUniqKey:
- return dbterror.ErrUnsupportedAddColumn.GenWithStack("unsupported add column '%s' constraint UNIQUE KEY when altering '%s.%s'", col.Name, ti.Schema, ti.Name)
- case ast.ColumnOptionAutoRandom:
- errMsg := fmt.Sprintf(autoid.AutoRandomAlterAddColumn, col.Name, ti.Schema, ti.Name)
- return dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(errMsg)
- }
- }
-
- return nil
-}
-
-func checkAndCreateNewColumn(ctx sessionctx.Context, ti ast.Ident, schema *model.DBInfo, spec *ast.AlterTableSpec, t table.Table, specNewColumn *ast.ColumnDef) (*table.Column, error) {
- err := checkUnsupportedColumnConstraint(specNewColumn, ti)
- if err != nil {
- return nil, errors.Trace(err)
- }
-
- colName := specNewColumn.Name.Name.O
- // Check whether added column has existed.
- col := table.FindCol(t.Cols(), colName)
- if col != nil {
- err = infoschema.ErrColumnExists.GenWithStackByArgs(colName)
- if spec.IfNotExists {
- ctx.GetSessionVars().StmtCtx.AppendNote(err)
- return nil, nil
- }
- return nil, err
- }
- if err = checkColumnAttributes(colName, specNewColumn.Tp); err != nil {
- return nil, errors.Trace(err)
- }
- if utf8.RuneCountInString(colName) > mysql.MaxColumnNameLength {
- return nil, dbterror.ErrTooLongIdent.GenWithStackByArgs(colName)
- }
-
- return CreateNewColumn(ctx, schema, spec, t, specNewColumn)
-}
-
-// CreateNewColumn creates a new column according to the column information.
-func CreateNewColumn(ctx sessionctx.Context, schema *model.DBInfo, spec *ast.AlterTableSpec, t table.Table, specNewColumn *ast.ColumnDef) (*table.Column, error) {
- // If new column is a generated column, do validation.
- // NOTE: we do check whether the column refers other generated
- // columns occurring later in a table, but we don't handle the col offset.
- for _, option := range specNewColumn.Options {
- if option.Tp == ast.ColumnOptionGenerated {
- if err := checkIllegalFn4Generated(specNewColumn.Name.Name.L, typeColumn, option.Expr); err != nil {
- return nil, errors.Trace(err)
- }
-
- if option.Stored {
- return nil, dbterror.ErrUnsupportedOnGeneratedColumn.GenWithStackByArgs("Adding generated stored column through ALTER TABLE")
- }
-
- _, dependColNames, err := findDependedColumnNames(schema.Name, t.Meta().Name, specNewColumn)
- if err != nil {
- return nil, errors.Trace(err)
- }
- if !ctx.GetSessionVars().EnableAutoIncrementInGenerated {
- if err := checkAutoIncrementRef(specNewColumn.Name.Name.L, dependColNames, t.Meta()); err != nil {
- return nil, errors.Trace(err)
- }
- }
- duplicateColNames := make(map[string]struct{}, len(dependColNames))
- for k := range dependColNames {
- duplicateColNames[k] = struct{}{}
- }
- cols := t.Cols()
-
- if err := checkDependedColExist(dependColNames, cols); err != nil {
- return nil, errors.Trace(err)
- }
-
- if err := verifyColumnGenerationSingle(duplicateColNames, cols, spec.Position); err != nil {
- return nil, errors.Trace(err)
- }
- }
- // Specially, since sequence has been supported, if a newly added column has a
- // sequence nextval function as it's default value option, it won't fill the
- // known rows with specific sequence next value under current add column logic.
- // More explanation can refer: TestSequenceDefaultLogic's comment in sequence_test.go
- if option.Tp == ast.ColumnOptionDefaultValue {
- if f, ok := option.Expr.(*ast.FuncCallExpr); ok {
- switch f.FnName.L {
- case ast.NextVal:
- if _, err := getSequenceDefaultValue(option); err != nil {
- return nil, errors.Trace(err)
- }
- return nil, errors.Trace(dbterror.ErrAddColumnWithSequenceAsDefault.GenWithStackByArgs(specNewColumn.Name.Name.O))
- case ast.Rand, ast.UUID, ast.UUIDToBin, ast.Replace, ast.Upper:
- return nil, errors.Trace(dbterror.ErrBinlogUnsafeSystemFunction.GenWithStackByArgs())
- }
- }
- }
- }
-
- tableCharset, tableCollate, err := ResolveCharsetCollation(ctx.GetSessionVars(),
- ast.CharsetOpt{Chs: t.Meta().Charset, Col: t.Meta().Collate},
- ast.CharsetOpt{Chs: schema.Charset, Col: schema.Collate},
- )
- if err != nil {
- return nil, errors.Trace(err)
- }
- // Ignore table constraints now, they will be checked later.
- // We use length(t.Cols()) as the default offset firstly, we will change the column's offset later.
- col, _, err := buildColumnAndConstraint(
- ctx,
- len(t.Cols()),
- specNewColumn,
- nil,
- tableCharset,
- tableCollate,
- )
- if err != nil {
- return nil, errors.Trace(err)
- }
-
- originDefVal, err := generateOriginDefaultValue(col.ToInfo(), ctx)
- if err != nil {
- return nil, errors.Trace(err)
- }
-
- err = col.SetOriginDefaultValue(originDefVal)
- return col, err
-}
-
-// AddColumn will add a new column to the table.
-func (e *executor) AddColumn(ctx sessionctx.Context, ti ast.Ident, spec *ast.AlterTableSpec) error {
- specNewColumn := spec.NewColumns[0]
- schema, t, err := e.getSchemaAndTableByIdent(ti)
- if err != nil {
- return errors.Trace(err)
- }
- failpoint.InjectCall("afterGetSchemaAndTableByIdent", ctx)
- tbInfo := t.Meta()
- if err = checkAddColumnTooManyColumns(len(t.Cols()) + 1); err != nil {
- return errors.Trace(err)
- }
- col, err := checkAndCreateNewColumn(ctx, ti, schema, spec, t, specNewColumn)
- if err != nil {
- return errors.Trace(err)
- }
- // Added column has existed and if_not_exists flag is true.
- if col == nil {
- return nil
- }
- err = CheckAfterPositionExists(tbInfo, spec.Position)
- if err != nil {
- return errors.Trace(err)
- }
-
- txn, err := ctx.Txn(true)
- if err != nil {
- return errors.Trace(err)
- }
- bdrRole, err := meta.NewMeta(txn).GetBDRRole()
- if err != nil {
- return errors.Trace(err)
- }
- if bdrRole == string(ast.BDRRolePrimary) && deniedByBDRWhenAddColumn(specNewColumn.Options) {
- return dbterror.ErrBDRRestrictedDDL.FastGenByArgs(bdrRole)
- }
-
- job := &model.Job{
- SchemaID: schema.ID,
- TableID: tbInfo.ID,
- SchemaName: schema.Name.L,
- TableName: tbInfo.Name.L,
- Type: model.ActionAddColumn,
- BinlogInfo: &model.HistoryInfo{},
- Args: []any{col, spec.Position, 0, spec.IfNotExists},
- CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
- SQLMode: ctx.GetSessionVars().SQLMode,
- }
-
- err = e.DoDDLJob(ctx, job)
- return errors.Trace(err)
-}
-
-// AddTablePartitions will add a new partition to the table.
-func (e *executor) AddTablePartitions(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
- is := e.infoCache.GetLatest()
- schema, ok := is.SchemaByName(ident.Schema)
- if !ok {
- return errors.Trace(infoschema.ErrDatabaseNotExists.GenWithStackByArgs(schema))
- }
- t, err := is.TableByName(e.ctx, ident.Schema, ident.Name)
+ is := e.infoCache.GetLatest()
+ tb, err := is.TableByName(ctx, ident.Schema, ident.Name)
if err != nil {
- return errors.Trace(infoschema.ErrTableNotExists.GenWithStackByArgs(ident.Schema, ident.Name))
+ return errors.Trace(err)
}
-
- meta := t.Meta()
- pi := meta.GetPartitionInfo()
- if pi == nil {
- return errors.Trace(dbterror.ErrPartitionMgmtOnNonpartitioned)
+ if tb.Meta().IsView() || tb.Meta().IsSequence() {
+ return dbterror.ErrWrongObject.GenWithStackByArgs(ident.Schema, ident.Name, "BASE TABLE")
}
- if pi.Type == model.PartitionTypeHash || pi.Type == model.PartitionTypeKey {
- // Add partition for hash/key is actually a reorganize partition
- // operation and not a metadata only change!
- switch spec.Tp {
- case ast.AlterTableAddLastPartition:
- return errors.Trace(dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("LAST PARTITION of HASH/KEY partitioned table"))
- case ast.AlterTableAddPartitions:
- // only thing supported
- default:
- return errors.Trace(dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("ADD PARTITION of HASH/KEY partitioned table"))
+ if tb.Meta().TableCacheStatusType != model.TableCacheStatusDisable {
+ if len(validSpecs) != 1 {
+ return dbterror.ErrOptOnCacheTable.GenWithStackByArgs("Alter Table")
}
- return e.hashPartitionManagement(ctx, ident, spec, pi)
- }
-
- partInfo, err := BuildAddedPartitionInfo(ctx.GetExprCtx(), meta, spec)
- if err != nil {
- return errors.Trace(err)
- }
- if pi.Type == model.PartitionTypeList {
- // TODO: make sure that checks in ddl_api and ddl_worker is the same.
- err = checkAddListPartitions(meta)
- if err != nil {
- return errors.Trace(err)
+ if validSpecs[0].Tp != ast.AlterTableCache && validSpecs[0].Tp != ast.AlterTableNoCache {
+ return dbterror.ErrOptOnCacheTable.GenWithStackByArgs("Alter Table")
}
}
- if err := e.assignPartitionIDs(partInfo.Definitions); err != nil {
- return errors.Trace(err)
+ if isMultiSchemaChanges(validSpecs) && (sctx.GetSessionVars().EnableRowLevelChecksum || variable.EnableRowLevelChecksum.Load()) {
+ return dbterror.ErrRunMultiSchemaChanges.GenWithStack("Unsupported multi schema change when row level checksum is enabled")
}
-
- // partInfo contains only the new added partition, we have to combine it with the
- // old partitions to check all partitions is strictly increasing.
- clonedMeta := meta.Clone()
- tmp := *partInfo
- tmp.Definitions = append(pi.Definitions, tmp.Definitions...)
- clonedMeta.Partition = &tmp
- if err := checkPartitionDefinitionConstraints(ctx, clonedMeta); err != nil {
- if dbterror.ErrSameNamePartition.Equal(err) && spec.IfNotExists {
- ctx.GetSessionVars().StmtCtx.AppendNote(err)
- return nil
+ // set name for anonymous foreign key.
+ maxForeignKeyID := tb.Meta().MaxForeignKeyID
+ for _, spec := range validSpecs {
+ if spec.Tp == ast.AlterTableAddConstraint && spec.Constraint.Tp == ast.ConstraintForeignKey && spec.Constraint.Name == "" {
+ maxForeignKeyID++
+ spec.Constraint.Name = fmt.Sprintf("fk_%d", maxForeignKeyID)
}
- return errors.Trace(err)
- }
-
- if err = handlePartitionPlacement(ctx, partInfo); err != nil {
- return errors.Trace(err)
}
- job := &model.Job{
- SchemaID: schema.ID,
- TableID: meta.ID,
- SchemaName: schema.Name.L,
- TableName: t.Meta().Name.L,
- Type: model.ActionAddTablePartition,
- BinlogInfo: &model.HistoryInfo{},
- Args: []any{partInfo},
- CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
- SQLMode: ctx.GetSessionVars().SQLMode,
+ if len(validSpecs) > 1 {
+ // after MultiSchemaInfo is set, DoDDLJob will collect all jobs into
+ // MultiSchemaInfo and skip running them. Then we will run them in
+ // d.multiSchemaChange all at once.
+ sctx.GetSessionVars().StmtCtx.MultiSchemaInfo = model.NewMultiSchemaInfo()
}
+ for _, spec := range validSpecs {
+ var handledCharsetOrCollate bool
+ var ttlOptionsHandled bool
+ switch spec.Tp {
+ case ast.AlterTableAddColumns:
+ err = e.AddColumn(sctx, ident, spec)
+ case ast.AlterTableAddPartitions, ast.AlterTableAddLastPartition:
+ err = e.AddTablePartitions(sctx, ident, spec)
+ case ast.AlterTableCoalescePartitions:
+ err = e.CoalescePartitions(sctx, ident, spec)
+ case ast.AlterTableReorganizePartition:
+ err = e.ReorganizePartitions(sctx, ident, spec)
+ case ast.AlterTableReorganizeFirstPartition:
+ err = dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("MERGE FIRST PARTITION")
+ case ast.AlterTableReorganizeLastPartition:
+ err = dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("SPLIT LAST PARTITION")
+ case ast.AlterTableCheckPartitions:
+ err = errors.Trace(dbterror.ErrUnsupportedCheckPartition)
+ case ast.AlterTableRebuildPartition:
+ err = errors.Trace(dbterror.ErrUnsupportedRebuildPartition)
+ case ast.AlterTableOptimizePartition:
+ err = errors.Trace(dbterror.ErrUnsupportedOptimizePartition)
+ case ast.AlterTableRemovePartitioning:
+ err = e.RemovePartitioning(sctx, ident, spec)
+ case ast.AlterTableRepairPartition:
+ err = errors.Trace(dbterror.ErrUnsupportedRepairPartition)
+ case ast.AlterTableDropColumn:
+ err = e.DropColumn(sctx, ident, spec)
+ case ast.AlterTableDropIndex:
+ err = e.dropIndex(sctx, ident, model.NewCIStr(spec.Name), spec.IfExists, false)
+ case ast.AlterTableDropPrimaryKey:
+ err = e.dropIndex(sctx, ident, model.NewCIStr(mysql.PrimaryKeyName), spec.IfExists, false)
+ case ast.AlterTableRenameIndex:
+ err = e.RenameIndex(sctx, ident, spec)
+ case ast.AlterTableDropPartition, ast.AlterTableDropFirstPartition:
+ err = e.DropTablePartition(sctx, ident, spec)
+ case ast.AlterTableTruncatePartition:
+ err = e.TruncateTablePartition(sctx, ident, spec)
+ case ast.AlterTableWriteable:
+ if !config.TableLockEnabled() {
+ return nil
+ }
+ tName := &ast.TableName{Schema: ident.Schema, Name: ident.Name}
+ if spec.Writeable {
+ err = e.CleanupTableLock(sctx, []*ast.TableName{tName})
+ } else {
+ lockStmt := &ast.LockTablesStmt{
+ TableLocks: []ast.TableLock{
+ {
+ Table: tName,
+ Type: model.TableLockReadOnly,
+ },
+ },
+ }
+ err = e.LockTables(sctx, lockStmt)
+ }
+ case ast.AlterTableExchangePartition:
+ err = e.ExchangeTablePartition(sctx, ident, spec)
+ case ast.AlterTableAddConstraint:
+ constr := spec.Constraint
+ switch spec.Constraint.Tp {
+ case ast.ConstraintKey, ast.ConstraintIndex:
+ err = e.createIndex(sctx, ident, ast.IndexKeyTypeNone, model.NewCIStr(constr.Name),
+ spec.Constraint.Keys, constr.Option, constr.IfNotExists)
+ case ast.ConstraintUniq, ast.ConstraintUniqIndex, ast.ConstraintUniqKey:
+ err = e.createIndex(sctx, ident, ast.IndexKeyTypeUnique, model.NewCIStr(constr.Name),
+ spec.Constraint.Keys, constr.Option, false) // IfNotExists should be not applied
+ case ast.ConstraintForeignKey:
+ // NOTE: we do not handle `symbol` and `index_name` well in the parser and we do not check ForeignKey already exists,
+ // so we just also ignore the `if not exists` check.
+ err = e.CreateForeignKey(sctx, ident, model.NewCIStr(constr.Name), spec.Constraint.Keys, spec.Constraint.Refer)
+ case ast.ConstraintPrimaryKey:
+ err = e.CreatePrimaryKey(sctx, ident, model.NewCIStr(constr.Name), spec.Constraint.Keys, constr.Option)
+ case ast.ConstraintFulltext:
+ sctx.GetSessionVars().StmtCtx.AppendWarning(dbterror.ErrTableCantHandleFt)
+ case ast.ConstraintCheck:
+ if !variable.EnableCheckConstraint.Load() {
+ sctx.GetSessionVars().StmtCtx.AppendWarning(errCheckConstraintIsOff)
+ } else {
+ err = e.CreateCheckConstraint(sctx, ident, model.NewCIStr(constr.Name), spec.Constraint)
+ }
+ default:
+ // Nothing to do now.
+ }
+ case ast.AlterTableDropForeignKey:
+ // NOTE: we do not check `if not exists` and `if exists` for ForeignKey now.
+ err = e.DropForeignKey(sctx, ident, model.NewCIStr(spec.Name))
+ case ast.AlterTableModifyColumn:
+ err = e.ModifyColumn(ctx, sctx, ident, spec)
+ case ast.AlterTableChangeColumn:
+ err = e.ChangeColumn(ctx, sctx, ident, spec)
+ case ast.AlterTableRenameColumn:
+ err = e.RenameColumn(sctx, ident, spec)
+ case ast.AlterTableAlterColumn:
+ err = e.AlterColumn(sctx, ident, spec)
+ case ast.AlterTableRenameTable:
+ newIdent := ast.Ident{Schema: spec.NewTable.Schema, Name: spec.NewTable.Name}
+ isAlterTable := true
+ err = e.renameTable(sctx, ident, newIdent, isAlterTable)
+ case ast.AlterTablePartition:
+ err = e.AlterTablePartitioning(sctx, ident, spec)
+ case ast.AlterTableOption:
+ var placementPolicyRef *model.PolicyRefInfo
+ for i, opt := range spec.Options {
+ switch opt.Tp {
+ case ast.TableOptionShardRowID:
+ if opt.UintValue > shardRowIDBitsMax {
+ opt.UintValue = shardRowIDBitsMax
+ }
+ err = e.ShardRowID(sctx, ident, opt.UintValue)
+ case ast.TableOptionAutoIncrement:
+ err = e.RebaseAutoID(sctx, ident, int64(opt.UintValue), autoid.AutoIncrementType, opt.BoolValue)
+ case ast.TableOptionAutoIdCache:
+ if opt.UintValue > uint64(math.MaxInt64) {
+ // TODO: Refine this error.
+ return errors.New("table option auto_id_cache overflows int64")
+ }
+ err = e.AlterTableAutoIDCache(sctx, ident, int64(opt.UintValue))
+ case ast.TableOptionAutoRandomBase:
+ err = e.RebaseAutoID(sctx, ident, int64(opt.UintValue), autoid.AutoRandomType, opt.BoolValue)
+ case ast.TableOptionComment:
+ spec.Comment = opt.StrValue
+ err = e.AlterTableComment(sctx, ident, spec)
+ case ast.TableOptionCharset, ast.TableOptionCollate:
+ // GetCharsetAndCollateInTableOption will get the last charset and collate in the options,
+ // so it should be handled only once.
+ if handledCharsetOrCollate {
+ continue
+ }
+ var toCharset, toCollate string
+ toCharset, toCollate, err = GetCharsetAndCollateInTableOption(sctx.GetSessionVars(), i, spec.Options)
+ if err != nil {
+ return err
+ }
+ needsOverwriteCols := NeedToOverwriteColCharset(spec.Options)
+ err = e.AlterTableCharsetAndCollate(sctx, ident, toCharset, toCollate, needsOverwriteCols)
+ handledCharsetOrCollate = true
+ case ast.TableOptionPlacementPolicy:
+ placementPolicyRef = &model.PolicyRefInfo{
+ Name: model.NewCIStr(opt.StrValue),
+ }
+ case ast.TableOptionEngine:
+ case ast.TableOptionRowFormat:
+ case ast.TableOptionTTL, ast.TableOptionTTLEnable, ast.TableOptionTTLJobInterval:
+ var ttlInfo *model.TTLInfo
+ var ttlEnable *bool
+ var ttlJobInterval *string
- if spec.Tp == ast.AlterTableAddLastPartition && spec.Partition != nil {
- query, ok := ctx.Value(sessionctx.QueryString).(string)
- if ok {
- sqlMode := ctx.GetSessionVars().SQLMode
- var buf bytes.Buffer
- AppendPartitionDefs(partInfo, &buf, sqlMode)
+ if ttlOptionsHandled {
+ continue
+ }
+ ttlInfo, ttlEnable, ttlJobInterval, err = getTTLInfoInOptions(spec.Options)
+ if err != nil {
+ return err
+ }
+ err = e.AlterTableTTLInfoOrEnable(sctx, ident, ttlInfo, ttlEnable, ttlJobInterval)
- syntacticSugar := spec.Partition.PartitionMethod.OriginalText()
- syntacticStart := spec.Partition.PartitionMethod.OriginTextPosition()
- newQuery := query[:syntacticStart] + "ADD PARTITION (" + buf.String() + ")" + query[syntacticStart+len(syntacticSugar):]
- defer ctx.SetValue(sessionctx.QueryString, query)
- ctx.SetValue(sessionctx.QueryString, newQuery)
- }
- }
- err = e.DoDDLJob(ctx, job)
- if dbterror.ErrSameNamePartition.Equal(err) && spec.IfNotExists {
- ctx.GetSessionVars().StmtCtx.AppendNote(err)
- return nil
- }
- return errors.Trace(err)
-}
+ ttlOptionsHandled = true
+ default:
+ err = dbterror.ErrUnsupportedAlterTableOption
+ }
-// getReorganizedDefinitions return the definitions as they would look like after the REORGANIZE PARTITION is done.
-func getReorganizedDefinitions(pi *model.PartitionInfo, firstPartIdx, lastPartIdx int, idMap map[int]struct{}) []model.PartitionDefinition {
- tmpDefs := make([]model.PartitionDefinition, 0, len(pi.Definitions)+len(pi.AddingDefinitions)-len(idMap))
- if pi.Type == model.PartitionTypeList {
- replaced := false
- for i := range pi.Definitions {
- if _, ok := idMap[i]; ok {
- if !replaced {
- tmpDefs = append(tmpDefs, pi.AddingDefinitions...)
- replaced = true
+ if err != nil {
+ return errors.Trace(err)
}
- continue
}
- tmpDefs = append(tmpDefs, pi.Definitions[i])
- }
- if !replaced {
- // For safety, for future non-partitioned table -> partitioned
- tmpDefs = append(tmpDefs, pi.AddingDefinitions...)
- }
- return tmpDefs
- }
- // Range
- tmpDefs = append(tmpDefs, pi.Definitions[:firstPartIdx]...)
- tmpDefs = append(tmpDefs, pi.AddingDefinitions...)
- if len(pi.Definitions) > (lastPartIdx + 1) {
- tmpDefs = append(tmpDefs, pi.Definitions[lastPartIdx+1:]...)
- }
- return tmpDefs
-}
-func getReplacedPartitionIDs(names []string, pi *model.PartitionInfo) (firstPartIdx int, lastPartIdx int, idMap map[int]struct{}, err error) {
- idMap = make(map[int]struct{})
- firstPartIdx, lastPartIdx = -1, -1
- for _, name := range names {
- nameL := strings.ToLower(name)
- partIdx := pi.FindPartitionDefinitionByName(nameL)
- if partIdx == -1 {
- return 0, 0, nil, errors.Trace(dbterror.ErrWrongPartitionName)
- }
- if _, ok := idMap[partIdx]; ok {
- return 0, 0, nil, errors.Trace(dbterror.ErrSameNamePartition)
- }
- idMap[partIdx] = struct{}{}
- if firstPartIdx == -1 {
- firstPartIdx = partIdx
- } else {
- firstPartIdx = mathutil.Min[int](firstPartIdx, partIdx)
+ if placementPolicyRef != nil {
+ err = e.AlterTablePlacement(sctx, ident, placementPolicyRef)
+ }
+ case ast.AlterTableSetTiFlashReplica:
+ err = e.AlterTableSetTiFlashReplica(sctx, ident, spec.TiFlashReplica)
+ case ast.AlterTableOrderByColumns:
+ err = e.OrderByColumns(sctx, ident)
+ case ast.AlterTableIndexInvisible:
+ err = e.AlterIndexVisibility(sctx, ident, spec.IndexName, spec.Visibility)
+ case ast.AlterTableAlterCheck:
+ if !variable.EnableCheckConstraint.Load() {
+ sctx.GetSessionVars().StmtCtx.AppendWarning(errCheckConstraintIsOff)
+ } else {
+ err = e.AlterCheckConstraint(sctx, ident, model.NewCIStr(spec.Constraint.Name), spec.Constraint.Enforced)
+ }
+ case ast.AlterTableDropCheck:
+ if !variable.EnableCheckConstraint.Load() {
+ sctx.GetSessionVars().StmtCtx.AppendWarning(errCheckConstraintIsOff)
+ } else {
+ err = e.DropCheckConstraint(sctx, ident, model.NewCIStr(spec.Constraint.Name))
+ }
+ case ast.AlterTableWithValidation:
+ sctx.GetSessionVars().StmtCtx.AppendWarning(dbterror.ErrUnsupportedAlterTableWithValidation)
+ case ast.AlterTableWithoutValidation:
+ sctx.GetSessionVars().StmtCtx.AppendWarning(dbterror.ErrUnsupportedAlterTableWithoutValidation)
+ case ast.AlterTableAddStatistics:
+ err = e.AlterTableAddStatistics(sctx, ident, spec.Statistics, spec.IfNotExists)
+ case ast.AlterTableDropStatistics:
+ err = e.AlterTableDropStatistics(sctx, ident, spec.Statistics, spec.IfExists)
+ case ast.AlterTableAttributes:
+ err = e.AlterTableAttributes(sctx, ident, spec)
+ case ast.AlterTablePartitionAttributes:
+ err = e.AlterTablePartitionAttributes(sctx, ident, spec)
+ case ast.AlterTablePartitionOptions:
+ err = e.AlterTablePartitionOptions(sctx, ident, spec)
+ case ast.AlterTableCache:
+ err = e.AlterTableCache(sctx, ident)
+ case ast.AlterTableNoCache:
+ err = e.AlterTableNoCache(sctx, ident)
+ case ast.AlterTableDisableKeys, ast.AlterTableEnableKeys:
+ // Nothing to do now, see https://github.com/pingcap/tidb/issues/1051
+ // MyISAM specific
+ case ast.AlterTableRemoveTTL:
+ // the parser makes sure we have only one `ast.AlterTableRemoveTTL` in an alter statement
+ err = e.AlterTableRemoveTTL(sctx, ident)
+ default:
+ err = errors.Trace(dbterror.ErrUnsupportedAlterTableSpec)
}
- if lastPartIdx == -1 {
- lastPartIdx = partIdx
- } else {
- lastPartIdx = mathutil.Max[int](lastPartIdx, partIdx)
+
+ if err != nil {
+ return errors.Trace(err)
}
}
- switch pi.Type {
- case model.PartitionTypeRange:
- if len(idMap) != (lastPartIdx - firstPartIdx + 1) {
- return 0, 0, nil, errors.Trace(dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs(
- "REORGANIZE PARTITION of RANGE; not adjacent partitions"))
- }
- case model.PartitionTypeHash, model.PartitionTypeKey:
- if len(idMap) != len(pi.Definitions) {
- return 0, 0, nil, errors.Trace(dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs(
- "REORGANIZE PARTITION of HASH/RANGE; must reorganize all partitions"))
+
+ if sctx.GetSessionVars().StmtCtx.MultiSchemaInfo != nil {
+ info := sctx.GetSessionVars().StmtCtx.MultiSchemaInfo
+ sctx.GetSessionVars().StmtCtx.MultiSchemaInfo = nil
+ err = e.multiSchemaChange(sctx, ident, info)
+ if err != nil {
+ return errors.Trace(err)
}
}
- return firstPartIdx, lastPartIdx, idMap, nil
+ return nil
}
-func getPartitionInfoTypeNone() *model.PartitionInfo {
- return &model.PartitionInfo{
- Type: model.PartitionTypeNone,
- Enable: true,
- Definitions: []model.PartitionDefinition{{
- Name: model.NewCIStr("pFullTable"),
- Comment: "Intermediate partition during ALTER TABLE ... PARTITION BY ...",
- }},
- Num: 1,
+func (e *executor) multiSchemaChange(ctx sessionctx.Context, ti ast.Ident, info *model.MultiSchemaInfo) error {
+ subJobs := info.SubJobs
+ if len(subJobs) == 0 {
+ return nil
}
-}
-
-// AlterTablePartitioning reorganize one set of partitions to a new set of partitions.
-func (e *executor) AlterTablePartitioning(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
- schema, t, err := e.getSchemaAndTableByIdent(ident)
+ schema, t, err := e.getSchemaAndTableByIdent(ti)
if err != nil {
- return errors.Trace(infoschema.ErrTableNotExists.FastGenByArgs(ident.Schema, ident.Name))
+ return errors.Trace(err)
}
- meta := t.Meta().Clone()
- piOld := meta.GetPartitionInfo()
- var partNames []string
- if piOld != nil {
- partNames = make([]string, 0, len(piOld.Definitions))
- for i := range piOld.Definitions {
- partNames = append(partNames, piOld.Definitions[i].Name.L)
- }
- } else {
- piOld = getPartitionInfoTypeNone()
- meta.Partition = piOld
- partNames = append(partNames, piOld.Definitions[0].Name.L)
+ logFn := logutil.DDLLogger().Warn
+ if intest.InTest {
+ logFn = logutil.DDLLogger().Fatal
}
- newMeta := meta.Clone()
- err = buildTablePartitionInfo(ctx, spec.Partition, newMeta)
- if err != nil {
- return err
- }
- newPartInfo := newMeta.Partition
- for _, index := range newMeta.Indices {
- if index.Unique {
- ck, err := checkPartitionKeysConstraint(newMeta.GetPartitionInfo(), index.Columns, newMeta)
- if err != nil {
- return err
+ var involvingSchemaInfo []model.InvolvingSchemaInfo
+ for _, j := range subJobs {
+ switch j.Type {
+ case model.ActionAlterTablePlacement:
+ ref, ok := j.Args[0].(*model.PolicyRefInfo)
+ if !ok {
+ logFn("unexpected type of policy reference info",
+ zap.Any("args[0]", j.Args[0]),
+ zap.String("type", fmt.Sprintf("%T", j.Args[0])))
+ continue
}
- if !ck {
- indexTp := ""
- if !ctx.GetSessionVars().EnableGlobalIndex {
- if index.Primary {
- indexTp = "PRIMARY KEY"
- } else {
- indexTp = "UNIQUE INDEX"
- }
- } else if t.Meta().IsCommonHandle {
- indexTp = "CLUSTERED INDEX"
- }
- if indexTp != "" {
- return dbterror.ErrUniqueKeyNeedAllFieldsInPf.GenWithStackByArgs(indexTp)
- }
- // Also mark the unique index as global index
- index.Global = true
+ if ref == nil {
+ continue
}
- }
- }
- if newMeta.PKIsHandle {
- // This case is covers when the Handle is the PK (only ints), since it would not
- // have an entry in the tblInfo.Indices
- indexCols := []*model.IndexColumn{{
- Name: newMeta.GetPkName(),
- Length: types.UnspecifiedLength,
- }}
- ck, err := checkPartitionKeysConstraint(newMeta.GetPartitionInfo(), indexCols, newMeta)
- if err != nil {
- return err
- }
- if !ck {
- if !ctx.GetSessionVars().EnableGlobalIndex {
- return dbterror.ErrUniqueKeyNeedAllFieldsInPf.GenWithStackByArgs("PRIMARY KEY")
+ involvingSchemaInfo = append(involvingSchemaInfo, model.InvolvingSchemaInfo{
+ Policy: ref.Name.L,
+ Mode: model.SharedInvolving,
+ })
+ case model.ActionAddForeignKey:
+ ref, ok := j.Args[0].(*model.FKInfo)
+ if !ok {
+ logFn("unexpected type of foreign key info",
+ zap.Any("args[0]", j.Args[0]),
+ zap.String("type", fmt.Sprintf("%T", j.Args[0])))
+ continue
+ }
+ involvingSchemaInfo = append(involvingSchemaInfo, model.InvolvingSchemaInfo{
+ Database: ref.RefSchema.L,
+ Table: ref.RefTable.L,
+ Mode: model.SharedInvolving,
+ })
+ case model.ActionAlterTablePartitionPlacement:
+ if len(j.Args) < 2 {
+ logFn("unexpected number of arguments for partition placement",
+ zap.Int("len(args)", len(j.Args)),
+ zap.Any("args", j.Args))
+ continue
+ }
+ ref, ok := j.Args[1].(*model.PolicyRefInfo)
+ if !ok {
+ logFn("unexpected type of policy reference info",
+ zap.Any("args[0]", j.Args[0]),
+ zap.String("type", fmt.Sprintf("%T", j.Args[0])))
+ continue
+ }
+ if ref == nil {
+ continue
}
- return dbterror.ErrUniqueKeyNeedAllFieldsInPf.GenWithStackByArgs("CLUSTERED INDEX")
+ involvingSchemaInfo = append(involvingSchemaInfo, model.InvolvingSchemaInfo{
+ Policy: ref.Name.L,
+ Mode: model.SharedInvolving,
+ })
}
}
- if err = handlePartitionPlacement(ctx, newPartInfo); err != nil {
- return errors.Trace(err)
- }
-
- if err = e.assignPartitionIDs(newPartInfo.Definitions); err != nil {
- return errors.Trace(err)
- }
- // A new table ID would be needed for
- // the global index, which cannot be the same as the current table id,
- // since this table id will be removed in the final state when removing
- // all the data with this table id.
- var newID []int64
- newID, err = e.genGlobalIDs(1)
- if err != nil {
- return errors.Trace(err)
- }
- newPartInfo.NewTableID = newID[0]
- newPartInfo.DDLType = piOld.Type
-
- job := &model.Job{
- SchemaID: schema.ID,
- TableID: meta.ID,
- SchemaName: schema.Name.L,
- TableName: t.Meta().Name.L,
- Type: model.ActionAlterTablePartitioning,
- BinlogInfo: &model.HistoryInfo{},
- Args: []any{partNames, newPartInfo},
- ReorgMeta: NewDDLReorgMeta(ctx),
- CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
- SQLMode: ctx.GetSessionVars().SQLMode,
- }
-
- // No preSplitAndScatter here, it will be done by the worker in onReorganizePartition instead.
- err = e.DoDDLJob(ctx, job)
- if err == nil {
- ctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackError("The statistics of new partitions will be outdated after reorganizing partitions. Please use 'ANALYZE TABLE' statement if you want to update it now"))
- }
- return errors.Trace(err)
-}
-
-// ReorganizePartitions reorganize one set of partitions to a new set of partitions.
-func (e *executor) ReorganizePartitions(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
- schema, t, err := e.getSchemaAndTableByIdent(ident)
- if err != nil {
- return errors.Trace(infoschema.ErrTableNotExists.FastGenByArgs(ident.Schema, ident.Name))
+ if len(involvingSchemaInfo) > 0 {
+ involvingSchemaInfo = append(involvingSchemaInfo, model.InvolvingSchemaInfo{
+ Database: schema.Name.L,
+ Table: t.Meta().Name.L,
+ })
}
- meta := t.Meta()
- pi := meta.GetPartitionInfo()
- if pi == nil {
- return dbterror.ErrPartitionMgmtOnNonpartitioned
- }
- switch pi.Type {
- case model.PartitionTypeRange, model.PartitionTypeList:
- case model.PartitionTypeHash, model.PartitionTypeKey:
- if spec.Tp != ast.AlterTableCoalescePartitions &&
- spec.Tp != ast.AlterTableAddPartitions {
- return errors.Trace(dbterror.ErrUnsupportedReorganizePartition)
- }
- default:
- return errors.Trace(dbterror.ErrUnsupportedReorganizePartition)
- }
- partNames := make([]string, 0, len(spec.PartitionNames))
- for _, name := range spec.PartitionNames {
- partNames = append(partNames, name.L)
+ job := &model.Job{
+ SchemaID: schema.ID,
+ TableID: t.Meta().ID,
+ SchemaName: schema.Name.L,
+ TableName: t.Meta().Name.L,
+ Type: model.ActionMultiSchemaChange,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: nil,
+ MultiSchemaInfo: info,
+ ReorgMeta: nil,
+ CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
+ InvolvingSchemaInfo: involvingSchemaInfo,
+ SQLMode: ctx.GetSessionVars().SQLMode,
}
- firstPartIdx, lastPartIdx, idMap, err := getReplacedPartitionIDs(partNames, pi)
- if err != nil {
- return errors.Trace(err)
+ if containsDistTaskSubJob(subJobs) {
+ job.ReorgMeta, err = newReorgMetaFromVariables(job, ctx)
+ if err != nil {
+ return err
+ }
+ } else {
+ job.ReorgMeta = NewDDLReorgMeta(ctx)
}
- partInfo, err := BuildAddedPartitionInfo(ctx.GetExprCtx(), meta, spec)
+
+ err = checkMultiSchemaInfo(info, t)
if err != nil {
return errors.Trace(err)
}
- if err = e.assignPartitionIDs(partInfo.Definitions); err != nil {
- return errors.Trace(err)
+ mergeAddIndex(info)
+ return e.DoDDLJob(ctx, job)
+}
+
+func containsDistTaskSubJob(subJobs []*model.SubJob) bool {
+ for _, sub := range subJobs {
+ if sub.Type == model.ActionAddIndex ||
+ sub.Type == model.ActionAddPrimaryKey {
+ return true
+ }
}
- if err = checkReorgPartitionDefs(ctx, model.ActionReorganizePartition, meta, partInfo, firstPartIdx, lastPartIdx, idMap); err != nil {
+ return false
+}
+
+func (e *executor) RebaseAutoID(ctx sessionctx.Context, ident ast.Ident, newBase int64, tp autoid.AllocatorType, force bool) error {
+ schema, t, err := e.getSchemaAndTableByIdent(ident)
+ if err != nil {
return errors.Trace(err)
}
- if err = handlePartitionPlacement(ctx, partInfo); err != nil {
- return errors.Trace(err)
+ tbInfo := t.Meta()
+ var actionType model.ActionType
+ switch tp {
+ case autoid.AutoRandomType:
+ pkCol := tbInfo.GetPkColInfo()
+ if tbInfo.AutoRandomBits == 0 || pkCol == nil {
+ return errors.Trace(dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomRebaseNotApplicable))
+ }
+ shardFmt := autoid.NewShardIDFormat(&pkCol.FieldType, tbInfo.AutoRandomBits, tbInfo.AutoRandomRangeBits)
+ if shardFmt.IncrementalMask()&newBase != newBase {
+ errMsg := fmt.Sprintf(autoid.AutoRandomRebaseOverflow, newBase, shardFmt.IncrementalBitsCapacity())
+ return errors.Trace(dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(errMsg))
+ }
+ actionType = model.ActionRebaseAutoRandomBase
+ case autoid.RowIDAllocType:
+ actionType = model.ActionRebaseAutoID
+ case autoid.AutoIncrementType:
+ actionType = model.ActionRebaseAutoID
+ default:
+ panic(fmt.Sprintf("unimplemented rebase autoid type %s", tp))
}
+ if !force {
+ newBaseTemp, err := adjustNewBaseToNextGlobalID(ctx.GetTableCtx(), t, tp, newBase)
+ if err != nil {
+ return err
+ }
+ if newBase != newBaseTemp {
+ ctx.GetSessionVars().StmtCtx.AppendWarning(
+ errors.NewNoStackErrorf("Can't reset AUTO_INCREMENT to %d without FORCE option, using %d instead",
+ newBase, newBaseTemp,
+ ))
+ }
+ newBase = newBaseTemp
+ }
job := &model.Job{
SchemaID: schema.ID,
- TableID: meta.ID,
+ TableID: tbInfo.ID,
SchemaName: schema.Name.L,
- TableName: t.Meta().Name.L,
- Type: model.ActionReorganizePartition,
+ TableName: tbInfo.Name.L,
+ Type: actionType,
BinlogInfo: &model.HistoryInfo{},
- Args: []any{partNames, partInfo},
- ReorgMeta: NewDDLReorgMeta(ctx),
+ Args: []any{newBase, force},
CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
SQLMode: ctx.GetSessionVars().SQLMode,
}
-
- // No preSplitAndScatter here, it will be done by the worker in onReorganizePartition instead.
err = e.DoDDLJob(ctx, job)
- failpoint.InjectCall("afterReorganizePartition")
- if err == nil {
- ctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackError("The statistics of related partitions will be outdated after reorganizing partitions. Please use 'ANALYZE TABLE' statement if you want to update it now"))
- }
return errors.Trace(err)
}
-// RemovePartitioning removes partitioning from a table.
-func (e *executor) RemovePartitioning(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
- schema, t, err := e.getSchemaAndTableByIdent(ident)
- if err != nil {
- return errors.Trace(infoschema.ErrTableNotExists.FastGenByArgs(ident.Schema, ident.Name))
+func adjustNewBaseToNextGlobalID(ctx table.AllocatorContext, t table.Table, tp autoid.AllocatorType, newBase int64) (int64, error) {
+ alloc := t.Allocators(ctx).Get(tp)
+ if alloc == nil {
+ return newBase, nil
}
-
- meta := t.Meta().Clone()
- pi := meta.GetPartitionInfo()
- if pi == nil {
- return dbterror.ErrPartitionMgmtOnNonpartitioned
+ autoID, err := alloc.NextGlobalAutoID()
+ if err != nil {
+ return newBase, errors.Trace(err)
}
- // TODO: Optimize for remove partitioning with a single partition
- // TODO: Add the support for this in onReorganizePartition
- // skip if only one partition
- // If there are only one partition, then we can do:
- // change the table id to the partition id
- // and keep the statistics for the partition id (which should be similar to the global statistics)
- // and it let the GC clean up the old table metadata including possible global index.
+ // If newBase < autoID, we need to do a rebase before returning.
+ // Assume there are 2 TiDB servers: TiDB-A with allocator range of 0 ~ 30000; TiDB-B with allocator range of 30001 ~ 60000.
+ // If the user sends SQL `alter table t1 auto_increment = 100` to TiDB-B,
+ // and TiDB-B finds 100 < 30001 but returns without any handling,
+ // then TiDB-A may still allocate 99 for auto_increment column. This doesn't make sense for the user.
+ return int64(mathutil.Max(uint64(newBase), uint64(autoID))), nil
+}
- newSpec := &ast.AlterTableSpec{}
- newSpec.Tp = spec.Tp
- defs := make([]*ast.PartitionDefinition, 1)
- defs[0] = &ast.PartitionDefinition{}
- defs[0].Name = model.NewCIStr("CollapsedPartitions")
- newSpec.PartDefinitions = defs
- partNames := make([]string, len(pi.Definitions))
- for i := range pi.Definitions {
- partNames[i] = pi.Definitions[i].Name.L
- }
- meta.Partition.Type = model.PartitionTypeNone
- partInfo, err := BuildAddedPartitionInfo(ctx.GetExprCtx(), meta, newSpec)
+// ShardRowID shards the implicit row ID by adding shard value to the row ID's first few bits.
+func (e *executor) ShardRowID(ctx sessionctx.Context, tableIdent ast.Ident, uVal uint64) error {
+ schema, t, err := e.getSchemaAndTableByIdent(tableIdent)
if err != nil {
return errors.Trace(err)
}
- if err = e.assignPartitionIDs(partInfo.Definitions); err != nil {
- return errors.Trace(err)
+ tbInfo := t.Meta()
+ if tbInfo.TempTableType != model.TempTableNone {
+ return dbterror.ErrOptOnTemporaryTable.GenWithStackByArgs("shard_row_id_bits")
}
- // TODO: check where the default placement comes from (i.e. table level)
- if err = handlePartitionPlacement(ctx, partInfo); err != nil {
- return errors.Trace(err)
+ if uVal == tbInfo.ShardRowIDBits {
+ // Nothing need to do.
+ return nil
+ }
+ if uVal > 0 && tbInfo.HasClusteredIndex() {
+ return dbterror.ErrUnsupportedShardRowIDBits
+ }
+ err = verifyNoOverflowShardBits(e.sessPool, t, uVal)
+ if err != nil {
+ return err
}
- partInfo.NewTableID = partInfo.Definitions[0].ID
-
job := &model.Job{
+ Type: model.ActionShardRowID,
SchemaID: schema.ID,
- TableID: meta.ID,
+ TableID: tbInfo.ID,
SchemaName: schema.Name.L,
- TableName: meta.Name.L,
- Type: model.ActionRemovePartitioning,
+ TableName: tbInfo.Name.L,
BinlogInfo: &model.HistoryInfo{},
- Args: []any{partNames, partInfo},
- ReorgMeta: NewDDLReorgMeta(ctx),
+ Args: []any{uVal},
CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
SQLMode: ctx.GetSessionVars().SQLMode,
}
-
- // No preSplitAndScatter here, it will be done by the worker in onReorganizePartition instead.
err = e.DoDDLJob(ctx, job)
return errors.Trace(err)
}
-func checkReorgPartitionDefs(ctx sessionctx.Context, action model.ActionType, tblInfo *model.TableInfo, partInfo *model.PartitionInfo, firstPartIdx, lastPartIdx int, idMap map[int]struct{}) error {
- // partInfo contains only the new added partition, we have to combine it with the
- // old partitions to check all partitions is strictly increasing.
- pi := tblInfo.Partition
- clonedMeta := tblInfo.Clone()
- switch action {
- case model.ActionRemovePartitioning, model.ActionAlterTablePartitioning:
- clonedMeta.Partition = partInfo
- clonedMeta.ID = partInfo.NewTableID
- case model.ActionReorganizePartition:
- clonedMeta.Partition.AddingDefinitions = partInfo.Definitions
- clonedMeta.Partition.Definitions = getReorganizedDefinitions(clonedMeta.Partition, firstPartIdx, lastPartIdx, idMap)
- default:
- return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("partition type")
- }
- if err := checkPartitionDefinitionConstraints(ctx, clonedMeta); err != nil {
- return errors.Trace(err)
+func (e *executor) getSchemaAndTableByIdent(tableIdent ast.Ident) (dbInfo *model.DBInfo, t table.Table, err error) {
+ is := e.infoCache.GetLatest()
+ schema, ok := is.SchemaByName(tableIdent.Schema)
+ if !ok {
+ return nil, nil, infoschema.ErrDatabaseNotExists.GenWithStackByArgs(tableIdent.Schema)
}
- if action == model.ActionReorganizePartition {
- if pi.Type == model.PartitionTypeRange {
- if lastPartIdx == len(pi.Definitions)-1 {
- // Last partition dropped, OK to change the end range
- // Also includes MAXVALUE
- return nil
- }
- // Check if the replaced end range is the same as before
- lastAddingPartition := partInfo.Definitions[len(partInfo.Definitions)-1]
- lastOldPartition := pi.Definitions[lastPartIdx]
- if len(pi.Columns) > 0 {
- newGtOld, err := checkTwoRangeColumns(ctx, &lastAddingPartition, &lastOldPartition, pi, tblInfo)
- if err != nil {
- return errors.Trace(err)
- }
- if newGtOld {
- return errors.Trace(dbterror.ErrRangeNotIncreasing)
- }
- oldGtNew, err := checkTwoRangeColumns(ctx, &lastOldPartition, &lastAddingPartition, pi, tblInfo)
- if err != nil {
- return errors.Trace(err)
- }
- if oldGtNew {
- return errors.Trace(dbterror.ErrRangeNotIncreasing)
- }
- return nil
- }
-
- isUnsigned := isPartExprUnsigned(ctx.GetExprCtx().GetEvalCtx(), tblInfo)
- currentRangeValue, _, err := getRangeValue(ctx.GetExprCtx(), pi.Definitions[lastPartIdx].LessThan[0], isUnsigned)
- if err != nil {
- return errors.Trace(err)
- }
- newRangeValue, _, err := getRangeValue(ctx.GetExprCtx(), partInfo.Definitions[len(partInfo.Definitions)-1].LessThan[0], isUnsigned)
- if err != nil {
- return errors.Trace(err)
- }
-
- if currentRangeValue != newRangeValue {
- return errors.Trace(dbterror.ErrRangeNotIncreasing)
- }
- }
- } else {
- if len(pi.Definitions) != (lastPartIdx - firstPartIdx + 1) {
- // if not ActionReorganizePartition, require all partitions to be changed.
- return errors.Trace(dbterror.ErrAlterOperationNotSupported)
- }
+ t, err = is.TableByName(e.ctx, tableIdent.Schema, tableIdent.Name)
+ if err != nil {
+ return nil, nil, infoschema.ErrTableNotExists.GenWithStackByArgs(tableIdent.Schema, tableIdent.Name)
}
- return nil
+ return schema, t, nil
}
-// CoalescePartitions coalesce partitions can be used with a table that is partitioned by hash or key to reduce the number of partitions by number.
-func (e *executor) CoalescePartitions(sctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
- is := e.infoCache.GetLatest()
- schema, ok := is.SchemaByName(ident.Schema)
- if !ok {
- return errors.Trace(infoschema.ErrDatabaseNotExists.GenWithStackByArgs(schema))
+// AddColumn will add a new column to the table.
+func (e *executor) AddColumn(ctx sessionctx.Context, ti ast.Ident, spec *ast.AlterTableSpec) error {
+ specNewColumn := spec.NewColumns[0]
+ schema, t, err := e.getSchemaAndTableByIdent(ti)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ failpoint.InjectCall("afterGetSchemaAndTableByIdent", ctx)
+ tbInfo := t.Meta()
+ if err = checkAddColumnTooManyColumns(len(t.Cols()) + 1); err != nil {
+ return errors.Trace(err)
+ }
+ col, err := checkAndCreateNewColumn(ctx, ti, schema, spec, t, specNewColumn)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ // Added column has existed and if_not_exists flag is true.
+ if col == nil {
+ return nil
}
- t, err := is.TableByName(e.ctx, ident.Schema, ident.Name)
+ err = CheckAfterPositionExists(tbInfo, spec.Position)
if err != nil {
- return errors.Trace(infoschema.ErrTableNotExists.GenWithStackByArgs(ident.Schema, ident.Name))
+ return errors.Trace(err)
}
- pi := t.Meta().GetPartitionInfo()
- if pi == nil {
- return errors.Trace(dbterror.ErrPartitionMgmtOnNonpartitioned)
+ txn, err := ctx.Txn(true)
+ if err != nil {
+ return errors.Trace(err)
}
-
- switch pi.Type {
- case model.PartitionTypeHash, model.PartitionTypeKey:
- return e.hashPartitionManagement(sctx, ident, spec, pi)
-
- // Coalesce partition can only be used on hash/key partitions.
- default:
- return errors.Trace(dbterror.ErrCoalesceOnlyOnHashPartition)
+ bdrRole, err := meta.NewMeta(txn).GetBDRRole()
+ if err != nil {
+ return errors.Trace(err)
}
-}
-
-func (e *executor) hashPartitionManagement(sctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec, pi *model.PartitionInfo) error {
- newSpec := *spec
- newSpec.PartitionNames = make([]model.CIStr, len(pi.Definitions))
- for i := 0; i < len(pi.Definitions); i++ {
- // reorganize ALL partitions into the new number of partitions
- newSpec.PartitionNames[i] = pi.Definitions[i].Name
+ if bdrRole == string(ast.BDRRolePrimary) && deniedByBDRWhenAddColumn(specNewColumn.Options) {
+ return dbterror.ErrBDRRestrictedDDL.FastGenByArgs(bdrRole)
}
- for i := 0; i < len(newSpec.PartDefinitions); i++ {
- switch newSpec.PartDefinitions[i].Clause.(type) {
- case *ast.PartitionDefinitionClauseNone:
- // OK, expected
- case *ast.PartitionDefinitionClauseIn:
- return errors.Trace(ast.ErrPartitionWrongValues.FastGenByArgs("LIST", "IN"))
- case *ast.PartitionDefinitionClauseLessThan:
- return errors.Trace(ast.ErrPartitionWrongValues.FastGenByArgs("RANGE", "LESS THAN"))
- case *ast.PartitionDefinitionClauseHistory:
- return errors.Trace(ast.ErrPartitionWrongValues.FastGenByArgs("SYSTEM_TIME", "HISTORY"))
- default:
- return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs(
- "partitioning clause")
- }
- }
- if newSpec.Num < uint64(len(newSpec.PartDefinitions)) {
- newSpec.Num = uint64(len(newSpec.PartDefinitions))
- }
- if spec.Tp == ast.AlterTableCoalescePartitions {
- if newSpec.Num < 1 {
- return ast.ErrCoalescePartitionNoPartition
- }
- if newSpec.Num >= uint64(len(pi.Definitions)) {
- return dbterror.ErrDropLastPartition
- }
- if isNonDefaultPartitionOptionsUsed(pi.Definitions) {
- // The partition definitions will be copied in buildHashPartitionDefinitions()
- // if there is a non-empty list of definitions
- newSpec.PartDefinitions = []*ast.PartitionDefinition{{}}
- }
+ job := &model.Job{
+ SchemaID: schema.ID,
+ TableID: tbInfo.ID,
+ SchemaName: schema.Name.L,
+ TableName: tbInfo.Name.L,
+ Type: model.ActionAddColumn,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []any{col, spec.Position, 0, spec.IfNotExists},
+ CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
+ SQLMode: ctx.GetSessionVars().SQLMode,
}
- return e.ReorganizePartitions(sctx, ident, &newSpec)
+ err = e.DoDDLJob(ctx, job)
+ return errors.Trace(err)
}
-func (e *executor) TruncateTablePartition(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
+// AddTablePartitions will add a new partition to the table.
+func (e *executor) AddTablePartitions(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
is := e.infoCache.GetLatest()
schema, ok := is.SchemaByName(ident.Schema)
if !ok {
@@ -5208,44 +2279,56 @@ func (e *executor) TruncateTablePartition(ctx sessionctx.Context, ident ast.Iden
if err != nil {
return errors.Trace(infoschema.ErrTableNotExists.GenWithStackByArgs(ident.Schema, ident.Name))
}
+
meta := t.Meta()
- if meta.GetPartitionInfo() == nil {
+ pi := meta.GetPartitionInfo()
+ if pi == nil {
return errors.Trace(dbterror.ErrPartitionMgmtOnNonpartitioned)
}
-
- getTruncatedParts := func(pi *model.PartitionInfo) (*model.PartitionInfo, error) {
- if spec.OnAllPartitions {
- return pi.Clone(), nil
- }
- var defs []model.PartitionDefinition
- // MySQL allows duplicate partition names in truncate partition
- // so we filter them out through a hash
- posMap := make(map[int]bool)
- for _, name := range spec.PartitionNames {
- pos := pi.FindPartitionDefinitionByName(name.L)
- if pos < 0 {
- return nil, errors.Trace(table.ErrUnknownPartition.GenWithStackByArgs(name.L, ident.Name.O))
- }
- if _, ok := posMap[pos]; !ok {
- defs = append(defs, pi.Definitions[pos])
- posMap[pos] = true
- }
+ if pi.Type == model.PartitionTypeHash || pi.Type == model.PartitionTypeKey {
+ // Add partition for hash/key is actually a reorganize partition
+ // operation and not a metadata only change!
+ switch spec.Tp {
+ case ast.AlterTableAddLastPartition:
+ return errors.Trace(dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("LAST PARTITION of HASH/KEY partitioned table"))
+ case ast.AlterTableAddPartitions:
+ // only thing supported
+ default:
+ return errors.Trace(dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("ADD PARTITION of HASH/KEY partitioned table"))
}
- pi = pi.Clone()
- pi.Definitions = defs
- return pi, nil
+ return e.hashPartitionManagement(ctx, ident, spec, pi)
}
- pi, err := getTruncatedParts(meta.GetPartitionInfo())
+
+ partInfo, err := BuildAddedPartitionInfo(ctx.GetExprCtx(), meta, spec)
if err != nil {
- return err
+ return errors.Trace(err)
}
- pids := make([]int64, 0, len(pi.Definitions))
- for i := range pi.Definitions {
- pids = append(pids, pi.Definitions[i].ID)
+ if pi.Type == model.PartitionTypeList {
+ // TODO: make sure that checks in ddl_api and ddl_worker is the same.
+ err = checkAddListPartitions(meta)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ }
+ if err := e.assignPartitionIDs(partInfo.Definitions); err != nil {
+ return errors.Trace(err)
}
- genIDs, err := e.genGlobalIDs(len(pids))
- if err != nil {
+ // partInfo contains only the new added partition, we have to combine it with the
+ // old partitions to check all partitions is strictly increasing.
+ clonedMeta := meta.Clone()
+ tmp := *partInfo
+ tmp.Definitions = append(pi.Definitions, tmp.Definitions...)
+ clonedMeta.Partition = &tmp
+ if err := checkPartitionDefinitionConstraints(ctx, clonedMeta); err != nil {
+ if dbterror.ErrSameNamePartition.Equal(err) && spec.IfNotExists {
+ ctx.GetSessionVars().StmtCtx.AppendNote(err)
+ return nil
+ }
+ return errors.Trace(err)
+ }
+
+ if err = handlePartitionPlacement(ctx, partInfo); err != nil {
return errors.Trace(err)
}
@@ -5253,1172 +2336,993 @@ func (e *executor) TruncateTablePartition(ctx sessionctx.Context, ident ast.Iden
SchemaID: schema.ID,
TableID: meta.ID,
SchemaName: schema.Name.L,
- SchemaState: model.StatePublic,
TableName: t.Meta().Name.L,
- Type: model.ActionTruncateTablePartition,
+ Type: model.ActionAddTablePartition,
BinlogInfo: &model.HistoryInfo{},
- Args: []any{pids, genIDs},
+ Args: []any{partInfo},
CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
SQLMode: ctx.GetSessionVars().SQLMode,
}
- err = e.DoDDLJob(ctx, job)
+ if spec.Tp == ast.AlterTableAddLastPartition && spec.Partition != nil {
+ query, ok := ctx.Value(sessionctx.QueryString).(string)
+ if ok {
+ sqlMode := ctx.GetSessionVars().SQLMode
+ var buf bytes.Buffer
+ AppendPartitionDefs(partInfo, &buf, sqlMode)
+
+ syntacticSugar := spec.Partition.PartitionMethod.OriginalText()
+ syntacticStart := spec.Partition.PartitionMethod.OriginTextPosition()
+ newQuery := query[:syntacticStart] + "ADD PARTITION (" + buf.String() + ")" + query[syntacticStart+len(syntacticSugar):]
+ defer ctx.SetValue(sessionctx.QueryString, query)
+ ctx.SetValue(sessionctx.QueryString, newQuery)
+ }
+ }
+ err = e.DoDDLJob(ctx, job)
+ if dbterror.ErrSameNamePartition.Equal(err) && spec.IfNotExists {
+ ctx.GetSessionVars().StmtCtx.AppendNote(err)
+ return nil
+ }
+ return errors.Trace(err)
+}
+
+// getReorganizedDefinitions return the definitions as they would look like after the REORGANIZE PARTITION is done.
+func getReorganizedDefinitions(pi *model.PartitionInfo, firstPartIdx, lastPartIdx int, idMap map[int]struct{}) []model.PartitionDefinition {
+ tmpDefs := make([]model.PartitionDefinition, 0, len(pi.Definitions)+len(pi.AddingDefinitions)-len(idMap))
+ if pi.Type == model.PartitionTypeList {
+ replaced := false
+ for i := range pi.Definitions {
+ if _, ok := idMap[i]; ok {
+ if !replaced {
+ tmpDefs = append(tmpDefs, pi.AddingDefinitions...)
+ replaced = true
+ }
+ continue
+ }
+ tmpDefs = append(tmpDefs, pi.Definitions[i])
+ }
+ if !replaced {
+ // For safety, for future non-partitioned table -> partitioned
+ tmpDefs = append(tmpDefs, pi.AddingDefinitions...)
+ }
+ return tmpDefs
+ }
+ // Range
+ tmpDefs = append(tmpDefs, pi.Definitions[:firstPartIdx]...)
+ tmpDefs = append(tmpDefs, pi.AddingDefinitions...)
+ if len(pi.Definitions) > (lastPartIdx + 1) {
+ tmpDefs = append(tmpDefs, pi.Definitions[lastPartIdx+1:]...)
+ }
+ return tmpDefs
+}
+
+func getReplacedPartitionIDs(names []string, pi *model.PartitionInfo) (firstPartIdx int, lastPartIdx int, idMap map[int]struct{}, err error) {
+ idMap = make(map[int]struct{})
+ firstPartIdx, lastPartIdx = -1, -1
+ for _, name := range names {
+ nameL := strings.ToLower(name)
+ partIdx := pi.FindPartitionDefinitionByName(nameL)
+ if partIdx == -1 {
+ return 0, 0, nil, errors.Trace(dbterror.ErrWrongPartitionName)
+ }
+ if _, ok := idMap[partIdx]; ok {
+ return 0, 0, nil, errors.Trace(dbterror.ErrSameNamePartition)
+ }
+ idMap[partIdx] = struct{}{}
+ if firstPartIdx == -1 {
+ firstPartIdx = partIdx
+ } else {
+ firstPartIdx = mathutil.Min[int](firstPartIdx, partIdx)
+ }
+ if lastPartIdx == -1 {
+ lastPartIdx = partIdx
+ } else {
+ lastPartIdx = mathutil.Max[int](lastPartIdx, partIdx)
+ }
+ }
+ switch pi.Type {
+ case model.PartitionTypeRange:
+ if len(idMap) != (lastPartIdx - firstPartIdx + 1) {
+ return 0, 0, nil, errors.Trace(dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs(
+ "REORGANIZE PARTITION of RANGE; not adjacent partitions"))
+ }
+ case model.PartitionTypeHash, model.PartitionTypeKey:
+ if len(idMap) != len(pi.Definitions) {
+ return 0, 0, nil, errors.Trace(dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs(
+ "REORGANIZE PARTITION of HASH/RANGE; must reorganize all partitions"))
+ }
+ }
+
+ return firstPartIdx, lastPartIdx, idMap, nil
+}
+
+func getPartitionInfoTypeNone() *model.PartitionInfo {
+ return &model.PartitionInfo{
+ Type: model.PartitionTypeNone,
+ Enable: true,
+ Definitions: []model.PartitionDefinition{{
+ Name: model.NewCIStr("pFullTable"),
+ Comment: "Intermediate partition during ALTER TABLE ... PARTITION BY ...",
+ }},
+ Num: 1,
+ }
+}
+
+// AlterTablePartitioning reorganize one set of partitions to a new set of partitions.
+func (e *executor) AlterTablePartitioning(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
+ schema, t, err := e.getSchemaAndTableByIdent(ident)
if err != nil {
- return errors.Trace(err)
+ return errors.Trace(infoschema.ErrTableNotExists.FastGenByArgs(ident.Schema, ident.Name))
}
- return nil
-}
-func (e *executor) DropTablePartition(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
- is := e.infoCache.GetLatest()
- schema, ok := is.SchemaByName(ident.Schema)
- if !ok {
- return errors.Trace(infoschema.ErrDatabaseNotExists.GenWithStackByArgs(schema))
+ meta := t.Meta().Clone()
+ piOld := meta.GetPartitionInfo()
+ var partNames []string
+ if piOld != nil {
+ partNames = make([]string, 0, len(piOld.Definitions))
+ for i := range piOld.Definitions {
+ partNames = append(partNames, piOld.Definitions[i].Name.L)
+ }
+ } else {
+ piOld = getPartitionInfoTypeNone()
+ meta.Partition = piOld
+ partNames = append(partNames, piOld.Definitions[0].Name.L)
}
- t, err := is.TableByName(e.ctx, ident.Schema, ident.Name)
+ newMeta := meta.Clone()
+
+ err = buildTablePartitionInfo(ctx, spec.Partition, newMeta)
if err != nil {
- return errors.Trace(infoschema.ErrTableNotExists.GenWithStackByArgs(ident.Schema, ident.Name))
- }
- meta := t.Meta()
- if meta.GetPartitionInfo() == nil {
- return errors.Trace(dbterror.ErrPartitionMgmtOnNonpartitioned)
+ return err
}
- if spec.Tp == ast.AlterTableDropFirstPartition {
- intervalOptions := getPartitionIntervalFromTable(ctx.GetExprCtx(), meta)
- if intervalOptions == nil {
- return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs(
- "FIRST PARTITION, does not seem like an INTERVAL partitioned table")
- }
- if len(spec.Partition.Definitions) != 0 {
- return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs(
- "FIRST PARTITION, table info already contains partition definitions")
- }
- spec.Partition.Interval = intervalOptions
- err = GeneratePartDefsFromInterval(ctx.GetExprCtx(), spec.Tp, meta, spec.Partition)
- if err != nil {
- return err
- }
- pNullOffset := 0
- if intervalOptions.NullPart {
- pNullOffset = 1
- }
- if len(spec.Partition.Definitions) == 0 ||
- len(spec.Partition.Definitions) >= len(meta.Partition.Definitions)-pNullOffset {
- return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs(
- "FIRST PARTITION, number of partitions does not match")
- }
- if len(spec.PartitionNames) != 0 || len(spec.Partition.Definitions) <= 1 {
- return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs(
- "FIRST PARTITION, given value does not generate a list of partition names to be dropped")
- }
- for i := range spec.Partition.Definitions {
- spec.PartitionNames = append(spec.PartitionNames, meta.Partition.Definitions[i+pNullOffset].Name)
- }
- // Use the last generated partition as First, i.e. do not drop the last name in the slice
- spec.PartitionNames = spec.PartitionNames[:len(spec.PartitionNames)-1]
+ newPartInfo := newMeta.Partition
- query, ok := ctx.Value(sessionctx.QueryString).(string)
- if ok {
- partNames := make([]string, 0, len(spec.PartitionNames))
- sqlMode := ctx.GetSessionVars().SQLMode
- for i := range spec.PartitionNames {
- partNames = append(partNames, stringutil.Escape(spec.PartitionNames[i].O, sqlMode))
- }
- syntacticSugar := spec.Partition.PartitionMethod.OriginalText()
- syntacticStart := spec.Partition.PartitionMethod.OriginTextPosition()
- newQuery := query[:syntacticStart] + "DROP PARTITION " + strings.Join(partNames, ", ") + query[syntacticStart+len(syntacticSugar):]
- defer ctx.SetValue(sessionctx.QueryString, query)
- ctx.SetValue(sessionctx.QueryString, newQuery)
- }
+ if err = handlePartitionPlacement(ctx, newPartInfo); err != nil {
+ return errors.Trace(err)
}
- partNames := make([]string, len(spec.PartitionNames))
- for i, partCIName := range spec.PartitionNames {
- partNames[i] = partCIName.L
+
+ if err = e.assignPartitionIDs(newPartInfo.Definitions); err != nil {
+ return errors.Trace(err)
}
- err = CheckDropTablePartition(meta, partNames)
+ // A new table ID would be needed for
+ // the global index, which cannot be the same as the current table id,
+ // since this table id will be removed in the final state when removing
+ // all the data with this table id.
+ var newID []int64
+ newID, err = e.genGlobalIDs(1)
if err != nil {
- if dbterror.ErrDropPartitionNonExistent.Equal(err) && spec.IfExists {
- ctx.GetSessionVars().StmtCtx.AppendNote(err)
- return nil
- }
return errors.Trace(err)
}
+ newPartInfo.NewTableID = newID[0]
+ newPartInfo.DDLType = piOld.Type
job := &model.Job{
SchemaID: schema.ID,
TableID: meta.ID,
SchemaName: schema.Name.L,
- SchemaState: model.StatePublic,
- TableName: meta.Name.L,
- Type: model.ActionDropTablePartition,
+ TableName: t.Meta().Name.L,
+ Type: model.ActionAlterTablePartitioning,
BinlogInfo: &model.HistoryInfo{},
- Args: []any{partNames},
+ Args: []any{partNames, newPartInfo},
+ ReorgMeta: NewDDLReorgMeta(ctx),
CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
SQLMode: ctx.GetSessionVars().SQLMode,
}
+ // No preSplitAndScatter here, it will be done by the worker in onReorganizePartition instead.
err = e.DoDDLJob(ctx, job)
- if err != nil {
- if dbterror.ErrDropPartitionNonExistent.Equal(err) && spec.IfExists {
- ctx.GetSessionVars().StmtCtx.AppendNote(err)
- return nil
- }
- return errors.Trace(err)
+ if err == nil {
+ ctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackError("The statistics of new partitions will be outdated after reorganizing partitions. Please use 'ANALYZE TABLE' statement if you want to update it now"))
}
return errors.Trace(err)
}
-func checkFieldTypeCompatible(ft *types.FieldType, other *types.FieldType) bool {
- // int(1) could match the type with int(8)
- partialEqual := ft.GetType() == other.GetType() &&
- ft.GetDecimal() == other.GetDecimal() &&
- ft.GetCharset() == other.GetCharset() &&
- ft.GetCollate() == other.GetCollate() &&
- (ft.GetFlen() == other.GetFlen() || ft.StorageLength() != types.VarStorageLen) &&
- mysql.HasUnsignedFlag(ft.GetFlag()) == mysql.HasUnsignedFlag(other.GetFlag()) &&
- mysql.HasAutoIncrementFlag(ft.GetFlag()) == mysql.HasAutoIncrementFlag(other.GetFlag()) &&
- mysql.HasNotNullFlag(ft.GetFlag()) == mysql.HasNotNullFlag(other.GetFlag()) &&
- mysql.HasZerofillFlag(ft.GetFlag()) == mysql.HasZerofillFlag(other.GetFlag()) &&
- mysql.HasBinaryFlag(ft.GetFlag()) == mysql.HasBinaryFlag(other.GetFlag()) &&
- mysql.HasPriKeyFlag(ft.GetFlag()) == mysql.HasPriKeyFlag(other.GetFlag())
- if !partialEqual || len(ft.GetElems()) != len(other.GetElems()) {
- return false
- }
- for i := range ft.GetElems() {
- if ft.GetElems()[i] != other.GetElems()[i] {
- return false
- }
- }
- return true
-}
-
-func checkTiFlashReplicaCompatible(source *model.TiFlashReplicaInfo, target *model.TiFlashReplicaInfo) bool {
- if source == target {
- return true
- }
- if source == nil || target == nil {
- return false
- }
- if source.Count != target.Count ||
- source.Available != target.Available || len(source.LocationLabels) != len(target.LocationLabels) {
- return false
- }
- for i, lable := range source.LocationLabels {
- if target.LocationLabels[i] != lable {
- return false
- }
- }
- return true
-}
-
-func checkTableDefCompatible(source *model.TableInfo, target *model.TableInfo) error {
- // check temp table
- if target.TempTableType != model.TempTableNone {
- return errors.Trace(dbterror.ErrPartitionExchangeTempTable.FastGenByArgs(target.Name))
+// ReorganizePartitions reorganize one set of partitions to a new set of partitions.
+func (e *executor) ReorganizePartitions(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
+ schema, t, err := e.getSchemaAndTableByIdent(ident)
+ if err != nil {
+ return errors.Trace(infoschema.ErrTableNotExists.FastGenByArgs(ident.Schema, ident.Name))
}
- // check auto_random
- if source.AutoRandomBits != target.AutoRandomBits ||
- source.AutoRandomRangeBits != target.AutoRandomRangeBits ||
- source.Charset != target.Charset ||
- source.Collate != target.Collate ||
- source.ShardRowIDBits != target.ShardRowIDBits ||
- source.MaxShardRowIDBits != target.MaxShardRowIDBits ||
- !checkTiFlashReplicaCompatible(source.TiFlashReplica, target.TiFlashReplica) {
- return errors.Trace(dbterror.ErrTablesDifferentMetadata)
- }
- if len(source.Cols()) != len(target.Cols()) {
- return errors.Trace(dbterror.ErrTablesDifferentMetadata)
+ meta := t.Meta()
+ pi := meta.GetPartitionInfo()
+ if pi == nil {
+ return dbterror.ErrPartitionMgmtOnNonpartitioned
}
- // Col compatible check
- for i, sourceCol := range source.Cols() {
- targetCol := target.Cols()[i]
- if sourceCol.IsVirtualGenerated() != targetCol.IsVirtualGenerated() {
- return dbterror.ErrUnsupportedOnGeneratedColumn.GenWithStackByArgs("Exchanging partitions for non-generated columns")
- }
- // It should strictyle compare expressions for generated columns
- if sourceCol.Name.L != targetCol.Name.L ||
- sourceCol.Hidden != targetCol.Hidden ||
- !checkFieldTypeCompatible(&sourceCol.FieldType, &targetCol.FieldType) ||
- sourceCol.GeneratedExprString != targetCol.GeneratedExprString {
- return errors.Trace(dbterror.ErrTablesDifferentMetadata)
- }
- if sourceCol.State != model.StatePublic ||
- targetCol.State != model.StatePublic {
- return errors.Trace(dbterror.ErrTablesDifferentMetadata)
- }
- if sourceCol.ID != targetCol.ID {
- return dbterror.ErrPartitionExchangeDifferentOption.GenWithStackByArgs(fmt.Sprintf("column: %s", sourceCol.Name))
+ switch pi.Type {
+ case model.PartitionTypeRange, model.PartitionTypeList:
+ case model.PartitionTypeHash, model.PartitionTypeKey:
+ if spec.Tp != ast.AlterTableCoalescePartitions &&
+ spec.Tp != ast.AlterTableAddPartitions {
+ return errors.Trace(dbterror.ErrUnsupportedReorganizePartition)
}
+ default:
+ return errors.Trace(dbterror.ErrUnsupportedReorganizePartition)
}
- if len(source.Indices) != len(target.Indices) {
- return errors.Trace(dbterror.ErrTablesDifferentMetadata)
+ partNames := make([]string, 0, len(spec.PartitionNames))
+ for _, name := range spec.PartitionNames {
+ partNames = append(partNames, name.L)
}
- for _, sourceIdx := range source.Indices {
- if sourceIdx.Global {
- return dbterror.ErrPartitionExchangeDifferentOption.GenWithStackByArgs(fmt.Sprintf("global index: %s", sourceIdx.Name))
- }
- var compatIdx *model.IndexInfo
- for _, targetIdx := range target.Indices {
- if strings.EqualFold(sourceIdx.Name.L, targetIdx.Name.L) {
- compatIdx = targetIdx
- }
- }
- // No match index
- if compatIdx == nil {
- return errors.Trace(dbterror.ErrTablesDifferentMetadata)
- }
- // Index type is not compatible
- if sourceIdx.Tp != compatIdx.Tp ||
- sourceIdx.Unique != compatIdx.Unique ||
- sourceIdx.Primary != compatIdx.Primary {
- return errors.Trace(dbterror.ErrTablesDifferentMetadata)
- }
- // The index column
- if len(sourceIdx.Columns) != len(compatIdx.Columns) {
- return errors.Trace(dbterror.ErrTablesDifferentMetadata)
- }
- for i, sourceIdxCol := range sourceIdx.Columns {
- compatIdxCol := compatIdx.Columns[i]
- if sourceIdxCol.Length != compatIdxCol.Length ||
- sourceIdxCol.Name.L != compatIdxCol.Name.L {
- return errors.Trace(dbterror.ErrTablesDifferentMetadata)
- }
- }
- if sourceIdx.ID != compatIdx.ID {
- return dbterror.ErrPartitionExchangeDifferentOption.GenWithStackByArgs(fmt.Sprintf("index: %s", sourceIdx.Name))
- }
+ firstPartIdx, lastPartIdx, idMap, err := getReplacedPartitionIDs(partNames, pi)
+ if err != nil {
+ return errors.Trace(err)
}
-
- return nil
-}
-
-func checkExchangePartition(pt *model.TableInfo, nt *model.TableInfo) error {
- if nt.IsView() || nt.IsSequence() {
- return errors.Trace(dbterror.ErrCheckNoSuchTable)
+ partInfo, err := BuildAddedPartitionInfo(ctx.GetExprCtx(), meta, spec)
+ if err != nil {
+ return errors.Trace(err)
}
- if pt.GetPartitionInfo() == nil {
- return errors.Trace(dbterror.ErrPartitionMgmtOnNonpartitioned)
+ if err = e.assignPartitionIDs(partInfo.Definitions); err != nil {
+ return errors.Trace(err)
}
- if nt.GetPartitionInfo() != nil {
- return errors.Trace(dbterror.ErrPartitionExchangePartTable.GenWithStackByArgs(nt.Name))
+ if err = checkReorgPartitionDefs(ctx, model.ActionReorganizePartition, meta, partInfo, firstPartIdx, lastPartIdx, idMap); err != nil {
+ return errors.Trace(err)
+ }
+ if err = handlePartitionPlacement(ctx, partInfo); err != nil {
+ return errors.Trace(err)
}
- if len(nt.ForeignKeys) > 0 {
- return errors.Trace(dbterror.ErrPartitionExchangeForeignKey.GenWithStackByArgs(nt.Name))
+ job := &model.Job{
+ SchemaID: schema.ID,
+ TableID: meta.ID,
+ SchemaName: schema.Name.L,
+ TableName: t.Meta().Name.L,
+ Type: model.ActionReorganizePartition,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []any{partNames, partInfo},
+ ReorgMeta: NewDDLReorgMeta(ctx),
+ CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
+ SQLMode: ctx.GetSessionVars().SQLMode,
}
- return nil
+ // No preSplitAndScatter here, it will be done by the worker in onReorganizePartition instead.
+ err = e.DoDDLJob(ctx, job)
+ failpoint.InjectCall("afterReorganizePartition")
+ if err == nil {
+ ctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackError("The statistics of related partitions will be outdated after reorganizing partitions. Please use 'ANALYZE TABLE' statement if you want to update it now"))
+ }
+ return errors.Trace(err)
}
-func (e *executor) ExchangeTablePartition(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
- ptSchema, pt, err := e.getSchemaAndTableByIdent(ident)
+// RemovePartitioning removes partitioning from a table.
+func (e *executor) RemovePartitioning(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
+ schema, t, err := e.getSchemaAndTableByIdent(ident)
if err != nil {
- return errors.Trace(err)
+ return errors.Trace(infoschema.ErrTableNotExists.FastGenByArgs(ident.Schema, ident.Name))
}
- ptMeta := pt.Meta()
-
- ntIdent := ast.Ident{Schema: spec.NewTable.Schema, Name: spec.NewTable.Name}
-
- // We should check local temporary here using session's info schema because the local temporary tables are only stored in session.
- ntLocalTempTable, err := sessiontxn.GetTxnManager(ctx).GetTxnInfoSchema().TableByName(context.Background(), ntIdent.Schema, ntIdent.Name)
- if err == nil && ntLocalTempTable.Meta().TempTableType == model.TempTableLocal {
- return errors.Trace(dbterror.ErrPartitionExchangeTempTable.FastGenByArgs(ntLocalTempTable.Meta().Name))
+ meta := t.Meta().Clone()
+ pi := meta.GetPartitionInfo()
+ if pi == nil {
+ return dbterror.ErrPartitionMgmtOnNonpartitioned
}
+ // TODO: Optimize for remove partitioning with a single partition
+ // TODO: Add the support for this in onReorganizePartition
+ // skip if only one partition
+ // If there are only one partition, then we can do:
+ // change the table id to the partition id
+ // and keep the statistics for the partition id (which should be similar to the global statistics)
+ // and it let the GC clean up the old table metadata including possible global index.
- ntSchema, nt, err := e.getSchemaAndTableByIdent(ntIdent)
+ newSpec := &ast.AlterTableSpec{}
+ newSpec.Tp = spec.Tp
+ defs := make([]*ast.PartitionDefinition, 1)
+ defs[0] = &ast.PartitionDefinition{}
+ defs[0].Name = model.NewCIStr("CollapsedPartitions")
+ newSpec.PartDefinitions = defs
+ partNames := make([]string, len(pi.Definitions))
+ for i := range pi.Definitions {
+ partNames[i] = pi.Definitions[i].Name.L
+ }
+ meta.Partition.Type = model.PartitionTypeNone
+ partInfo, err := BuildAddedPartitionInfo(ctx.GetExprCtx(), meta, newSpec)
if err != nil {
return errors.Trace(err)
}
+ if err = e.assignPartitionIDs(partInfo.Definitions); err != nil {
+ return errors.Trace(err)
+ }
+ // TODO: check where the default placement comes from (i.e. table level)
+ if err = handlePartitionPlacement(ctx, partInfo); err != nil {
+ return errors.Trace(err)
+ }
+ partInfo.NewTableID = partInfo.Definitions[0].ID
- ntMeta := nt.Meta()
+ job := &model.Job{
+ SchemaID: schema.ID,
+ TableID: meta.ID,
+ SchemaName: schema.Name.L,
+ TableName: meta.Name.L,
+ Type: model.ActionRemovePartitioning,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []any{partNames, partInfo},
+ ReorgMeta: NewDDLReorgMeta(ctx),
+ CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
+ SQLMode: ctx.GetSessionVars().SQLMode,
+ }
- err = checkExchangePartition(ptMeta, ntMeta)
- if err != nil {
+ // No preSplitAndScatter here, it will be done by the worker in onReorganizePartition instead.
+ err = e.DoDDLJob(ctx, job)
+ return errors.Trace(err)
+}
+
+func checkReorgPartitionDefs(ctx sessionctx.Context, action model.ActionType, tblInfo *model.TableInfo, partInfo *model.PartitionInfo, firstPartIdx, lastPartIdx int, idMap map[int]struct{}) error {
+ // partInfo contains only the new added partition, we have to combine it with the
+ // old partitions to check all partitions is strictly increasing.
+ pi := tblInfo.Partition
+ clonedMeta := tblInfo.Clone()
+ switch action {
+ case model.ActionRemovePartitioning, model.ActionAlterTablePartitioning:
+ clonedMeta.Partition = partInfo
+ clonedMeta.ID = partInfo.NewTableID
+ case model.ActionReorganizePartition:
+ clonedMeta.Partition.AddingDefinitions = partInfo.Definitions
+ clonedMeta.Partition.Definitions = getReorganizedDefinitions(clonedMeta.Partition, firstPartIdx, lastPartIdx, idMap)
+ default:
+ return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("partition type")
+ }
+ if err := checkPartitionDefinitionConstraints(ctx, clonedMeta); err != nil {
return errors.Trace(err)
}
+ if action == model.ActionReorganizePartition {
+ if pi.Type == model.PartitionTypeRange {
+ if lastPartIdx == len(pi.Definitions)-1 {
+ // Last partition dropped, OK to change the end range
+ // Also includes MAXVALUE
+ return nil
+ }
+ // Check if the replaced end range is the same as before
+ lastAddingPartition := partInfo.Definitions[len(partInfo.Definitions)-1]
+ lastOldPartition := pi.Definitions[lastPartIdx]
+ if len(pi.Columns) > 0 {
+ newGtOld, err := checkTwoRangeColumns(ctx, &lastAddingPartition, &lastOldPartition, pi, tblInfo)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if newGtOld {
+ return errors.Trace(dbterror.ErrRangeNotIncreasing)
+ }
+ oldGtNew, err := checkTwoRangeColumns(ctx, &lastOldPartition, &lastAddingPartition, pi, tblInfo)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if oldGtNew {
+ return errors.Trace(dbterror.ErrRangeNotIncreasing)
+ }
+ return nil
+ }
- partName := spec.PartitionNames[0].L
+ isUnsigned := isPartExprUnsigned(ctx.GetExprCtx().GetEvalCtx(), tblInfo)
+ currentRangeValue, _, err := getRangeValue(ctx.GetExprCtx(), pi.Definitions[lastPartIdx].LessThan[0], isUnsigned)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ newRangeValue, _, err := getRangeValue(ctx.GetExprCtx(), partInfo.Definitions[len(partInfo.Definitions)-1].LessThan[0], isUnsigned)
+ if err != nil {
+ return errors.Trace(err)
+ }
- // NOTE: if pt is subPartitioned, it should be checked
+ if currentRangeValue != newRangeValue {
+ return errors.Trace(dbterror.ErrRangeNotIncreasing)
+ }
+ }
+ } else {
+ if len(pi.Definitions) != (lastPartIdx - firstPartIdx + 1) {
+ // if not ActionReorganizePartition, require all partitions to be changed.
+ return errors.Trace(dbterror.ErrAlterOperationNotSupported)
+ }
+ }
+ return nil
+}
- defID, err := tables.FindPartitionByName(ptMeta, partName)
+// CoalescePartitions coalesce partitions can be used with a table that is partitioned by hash or key to reduce the number of partitions by number.
+func (e *executor) CoalescePartitions(sctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
+ is := e.infoCache.GetLatest()
+ schema, ok := is.SchemaByName(ident.Schema)
+ if !ok {
+ return errors.Trace(infoschema.ErrDatabaseNotExists.GenWithStackByArgs(schema))
+ }
+ t, err := is.TableByName(e.ctx, ident.Schema, ident.Name)
if err != nil {
- return errors.Trace(err)
+ return errors.Trace(infoschema.ErrTableNotExists.GenWithStackByArgs(ident.Schema, ident.Name))
}
- err = checkTableDefCompatible(ptMeta, ntMeta)
- if err != nil {
- return errors.Trace(err)
+ pi := t.Meta().GetPartitionInfo()
+ if pi == nil {
+ return errors.Trace(dbterror.ErrPartitionMgmtOnNonpartitioned)
}
- job := &model.Job{
- SchemaID: ntSchema.ID,
- TableID: ntMeta.ID,
- SchemaName: ntSchema.Name.L,
- TableName: ntMeta.Name.L,
- Type: model.ActionExchangeTablePartition,
- BinlogInfo: &model.HistoryInfo{},
- Args: []any{defID, ptSchema.ID, ptMeta.ID, partName, spec.WithValidation},
- CtxVars: []any{[]int64{ntSchema.ID, ptSchema.ID}, []int64{ntMeta.ID, ptMeta.ID}},
- CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
- InvolvingSchemaInfo: []model.InvolvingSchemaInfo{
- {Database: ptSchema.Name.L, Table: ptMeta.Name.L},
- {Database: ntSchema.Name.L, Table: ntMeta.Name.L},
- },
- SQLMode: ctx.GetSessionVars().SQLMode,
+ switch pi.Type {
+ case model.PartitionTypeHash, model.PartitionTypeKey:
+ return e.hashPartitionManagement(sctx, ident, spec, pi)
+
+ // Coalesce partition can only be used on hash/key partitions.
+ default:
+ return errors.Trace(dbterror.ErrCoalesceOnlyOnHashPartition)
}
+}
- err = e.DoDDLJob(ctx, job)
- if err != nil {
- return errors.Trace(err)
+func (e *executor) hashPartitionManagement(sctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec, pi *model.PartitionInfo) error {
+ newSpec := *spec
+ newSpec.PartitionNames = make([]model.CIStr, len(pi.Definitions))
+ for i := 0; i < len(pi.Definitions); i++ {
+ // reorganize ALL partitions into the new number of partitions
+ newSpec.PartitionNames[i] = pi.Definitions[i].Name
+ }
+ for i := 0; i < len(newSpec.PartDefinitions); i++ {
+ switch newSpec.PartDefinitions[i].Clause.(type) {
+ case *ast.PartitionDefinitionClauseNone:
+ // OK, expected
+ case *ast.PartitionDefinitionClauseIn:
+ return errors.Trace(ast.ErrPartitionWrongValues.FastGenByArgs("LIST", "IN"))
+ case *ast.PartitionDefinitionClauseLessThan:
+ return errors.Trace(ast.ErrPartitionWrongValues.FastGenByArgs("RANGE", "LESS THAN"))
+ case *ast.PartitionDefinitionClauseHistory:
+ return errors.Trace(ast.ErrPartitionWrongValues.FastGenByArgs("SYSTEM_TIME", "HISTORY"))
+
+ default:
+ return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs(
+ "partitioning clause")
+ }
+ }
+ if newSpec.Num < uint64(len(newSpec.PartDefinitions)) {
+ newSpec.Num = uint64(len(newSpec.PartDefinitions))
+ }
+ if spec.Tp == ast.AlterTableCoalescePartitions {
+ if newSpec.Num < 1 {
+ return ast.ErrCoalescePartitionNoPartition
+ }
+ if newSpec.Num >= uint64(len(pi.Definitions)) {
+ return dbterror.ErrDropLastPartition
+ }
+ if isNonDefaultPartitionOptionsUsed(pi.Definitions) {
+ // The partition definitions will be copied in buildHashPartitionDefinitions()
+ // if there is a non-empty list of definitions
+ newSpec.PartDefinitions = []*ast.PartitionDefinition{{}}
+ }
}
- ctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackError("after the exchange, please analyze related table of the exchange to update statistics"))
- return nil
+
+ return e.ReorganizePartitions(sctx, ident, &newSpec)
}
-// DropColumn will drop a column from the table, now we don't support drop the column with index covered.
-func (e *executor) DropColumn(ctx sessionctx.Context, ti ast.Ident, spec *ast.AlterTableSpec) error {
- schema, t, err := e.getSchemaAndTableByIdent(ti)
+func (e *executor) TruncateTablePartition(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
+ is := e.infoCache.GetLatest()
+ schema, ok := is.SchemaByName(ident.Schema)
+ if !ok {
+ return errors.Trace(infoschema.ErrDatabaseNotExists.GenWithStackByArgs(schema))
+ }
+ t, err := is.TableByName(e.ctx, ident.Schema, ident.Name)
if err != nil {
- return errors.Trace(err)
+ return errors.Trace(infoschema.ErrTableNotExists.GenWithStackByArgs(ident.Schema, ident.Name))
+ }
+ meta := t.Meta()
+ if meta.GetPartitionInfo() == nil {
+ return errors.Trace(dbterror.ErrPartitionMgmtOnNonpartitioned)
}
- failpoint.InjectCall("afterGetSchemaAndTableByIdent", ctx)
- isDropable, err := checkIsDroppableColumn(ctx, e.infoCache.GetLatest(), schema, t, spec)
+ getTruncatedParts := func(pi *model.PartitionInfo) (*model.PartitionInfo, error) {
+ if spec.OnAllPartitions {
+ return pi.Clone(), nil
+ }
+ var defs []model.PartitionDefinition
+ // MySQL allows duplicate partition names in truncate partition
+ // so we filter them out through a hash
+ posMap := make(map[int]bool)
+ for _, name := range spec.PartitionNames {
+ pos := pi.FindPartitionDefinitionByName(name.L)
+ if pos < 0 {
+ return nil, errors.Trace(table.ErrUnknownPartition.GenWithStackByArgs(name.L, ident.Name.O))
+ }
+ if _, ok := posMap[pos]; !ok {
+ defs = append(defs, pi.Definitions[pos])
+ posMap[pos] = true
+ }
+ }
+ pi = pi.Clone()
+ pi.Definitions = defs
+ return pi, nil
+ }
+ pi, err := getTruncatedParts(meta.GetPartitionInfo())
if err != nil {
return err
}
- if !isDropable {
- return nil
+ pids := make([]int64, 0, len(pi.Definitions))
+ for i := range pi.Definitions {
+ pids = append(pids, pi.Definitions[i].ID)
}
- colName := spec.OldColumnName.Name
- err = checkVisibleColumnCnt(t, 0, 1)
+
+ genIDs, err := e.genGlobalIDs(len(pids))
if err != nil {
- return err
+ return errors.Trace(err)
}
job := &model.Job{
SchemaID: schema.ID,
- TableID: t.Meta().ID,
+ TableID: meta.ID,
SchemaName: schema.Name.L,
SchemaState: model.StatePublic,
TableName: t.Meta().Name.L,
- Type: model.ActionDropColumn,
+ Type: model.ActionTruncateTablePartition,
BinlogInfo: &model.HistoryInfo{},
- Args: []any{colName, spec.IfExists},
+ Args: []any{pids, genIDs},
CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
SQLMode: ctx.GetSessionVars().SQLMode,
}
err = e.DoDDLJob(ctx, job)
- return errors.Trace(err)
-}
-
-func checkIsDroppableColumn(ctx sessionctx.Context, is infoschema.InfoSchema, schema *model.DBInfo, t table.Table, spec *ast.AlterTableSpec) (isDrapable bool, err error) {
- tblInfo := t.Meta()
- // Check whether dropped column has existed.
- colName := spec.OldColumnName.Name
- col := table.FindCol(t.VisibleCols(), colName.L)
- if col == nil {
- err = dbterror.ErrCantDropFieldOrKey.GenWithStackByArgs(colName)
- if spec.IfExists {
- ctx.GetSessionVars().StmtCtx.AppendNote(err)
- return false, nil
- }
- return false, err
- }
-
- if err = isDroppableColumn(tblInfo, colName); err != nil {
- return false, errors.Trace(err)
- }
- if err = checkDropColumnWithPartitionConstraint(t, colName); err != nil {
- return false, errors.Trace(err)
- }
- // Check the column with foreign key.
- err = checkDropColumnWithForeignKeyConstraint(is, schema.Name.L, tblInfo, colName.L)
if err != nil {
- return false, errors.Trace(err)
- }
- // Check the column with TTL config
- err = checkDropColumnWithTTLConfig(tblInfo, colName.L)
- if err != nil {
- return false, errors.Trace(err)
- }
- // We don't support dropping column with PK handle covered now.
- if col.IsPKHandleColumn(tblInfo) {
- return false, dbterror.ErrUnsupportedPKHandle
- }
- if mysql.HasAutoIncrementFlag(col.GetFlag()) && !ctx.GetSessionVars().AllowRemoveAutoInc {
- return false, dbterror.ErrCantDropColWithAutoInc
- }
- return true, nil
-}
-
-// checkDropColumnWithPartitionConstraint is used to check the partition constraint of the drop column.
-func checkDropColumnWithPartitionConstraint(t table.Table, colName model.CIStr) error {
- if t.Meta().Partition == nil {
- return nil
- }
- pt, ok := t.(table.PartitionedTable)
- if !ok {
- // Should never happen!
- return errors.Trace(dbterror.ErrDependentByPartitionFunctional.GenWithStackByArgs(colName.L))
- }
- for _, name := range pt.GetPartitionColumnNames() {
- if strings.EqualFold(name.L, colName.L) {
- return errors.Trace(dbterror.ErrDependentByPartitionFunctional.GenWithStackByArgs(colName.L))
- }
+ return errors.Trace(err)
}
return nil
}
-func checkVisibleColumnCnt(t table.Table, addCnt, dropCnt int) error {
- tblInfo := t.Meta()
- visibleColumCnt := 0
- for _, column := range tblInfo.Columns {
- if !column.Hidden {
- visibleColumCnt++
- }
- }
- if visibleColumCnt+addCnt > dropCnt {
- return nil
- }
- if len(tblInfo.Columns)-visibleColumCnt > 0 {
- // There are only invisible columns.
- return dbterror.ErrTableMustHaveColumns
- }
- return dbterror.ErrCantRemoveAllFields
-}
-
-// checkModifyCharsetAndCollation returns error when the charset or collation is not modifiable.
-// needRewriteCollationData is used when trying to modify the collation of a column, it is true when the column is with
-// index because index of a string column is collation-aware.
-func checkModifyCharsetAndCollation(toCharset, toCollate, origCharset, origCollate string, needRewriteCollationData bool) error {
- if !charset.ValidCharsetAndCollation(toCharset, toCollate) {
- return dbterror.ErrUnknownCharacterSet.GenWithStack("Unknown character set: '%s', collation: '%s'", toCharset, toCollate)
- }
-
- if needRewriteCollationData && collate.NewCollationEnabled() && !collate.CompatibleCollate(origCollate, toCollate) {
- return dbterror.ErrUnsupportedModifyCollation.GenWithStackByArgs(origCollate, toCollate)
- }
-
- if (origCharset == charset.CharsetUTF8 && toCharset == charset.CharsetUTF8MB4) ||
- (origCharset == charset.CharsetUTF8 && toCharset == charset.CharsetUTF8) ||
- (origCharset == charset.CharsetUTF8MB4 && toCharset == charset.CharsetUTF8MB4) ||
- (origCharset == charset.CharsetLatin1 && toCharset == charset.CharsetUTF8MB4) {
- // TiDB only allow utf8/latin1 to be changed to utf8mb4, or changing the collation when the charset is utf8/utf8mb4/latin1.
- return nil
+func (e *executor) DropTablePartition(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
+ is := e.infoCache.GetLatest()
+ schema, ok := is.SchemaByName(ident.Schema)
+ if !ok {
+ return errors.Trace(infoschema.ErrDatabaseNotExists.GenWithStackByArgs(schema))
}
-
- if toCharset != origCharset {
- msg := fmt.Sprintf("charset from %s to %s", origCharset, toCharset)
- return dbterror.ErrUnsupportedModifyCharset.GenWithStackByArgs(msg)
+ t, err := is.TableByName(e.ctx, ident.Schema, ident.Name)
+ if err != nil {
+ return errors.Trace(infoschema.ErrTableNotExists.GenWithStackByArgs(ident.Schema, ident.Name))
}
- if toCollate != origCollate {
- msg := fmt.Sprintf("change collate from %s to %s", origCollate, toCollate)
- return dbterror.ErrUnsupportedModifyCharset.GenWithStackByArgs(msg)
+ meta := t.Meta()
+ if meta.GetPartitionInfo() == nil {
+ return errors.Trace(dbterror.ErrPartitionMgmtOnNonpartitioned)
}
- return nil
-}
-// checkModifyTypes checks if the 'origin' type can be modified to 'to' type no matter directly change
-// or change by reorg. It returns error if the two types are incompatible and correlated change are not
-// supported. However, even the two types can be change, if the "origin" type contains primary key, error will be returned.
-func checkModifyTypes(origin *types.FieldType, to *types.FieldType, needRewriteCollationData bool) error {
- canReorg, err := types.CheckModifyTypeCompatible(origin, to)
- if err != nil {
- if !canReorg {
- return errors.Trace(dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs(err.Error()))
+ if spec.Tp == ast.AlterTableDropFirstPartition {
+ intervalOptions := getPartitionIntervalFromTable(ctx.GetExprCtx(), meta)
+ if intervalOptions == nil {
+ return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs(
+ "FIRST PARTITION, does not seem like an INTERVAL partitioned table")
}
- if mysql.HasPriKeyFlag(origin.GetFlag()) {
- msg := "this column has primary key flag"
- return dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs(msg)
+ if len(spec.Partition.Definitions) != 0 {
+ return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs(
+ "FIRST PARTITION, table info already contains partition definitions")
}
- }
-
- err = checkModifyCharsetAndCollation(to.GetCharset(), to.GetCollate(), origin.GetCharset(), origin.GetCollate(), needRewriteCollationData)
-
- if err != nil {
- if to.GetCharset() == charset.CharsetGBK || origin.GetCharset() == charset.CharsetGBK {
- return errors.Trace(err)
+ spec.Partition.Interval = intervalOptions
+ err = GeneratePartDefsFromInterval(ctx.GetExprCtx(), spec.Tp, meta, spec.Partition)
+ if err != nil {
+ return err
}
- // column type change can handle the charset change between these two types in the process of the reorg.
- if dbterror.ErrUnsupportedModifyCharset.Equal(err) && canReorg {
- return nil
+ pNullOffset := 0
+ if intervalOptions.NullPart {
+ pNullOffset = 1
}
- }
- return errors.Trace(err)
-}
-
-// SetDefaultValue sets the default value of the column.
-func SetDefaultValue(ctx sessionctx.Context, col *table.Column, option *ast.ColumnOption) (hasDefaultValue bool, err error) {
- var value any
- var isSeqExpr bool
- value, isSeqExpr, err = getDefaultValue(
- exprctx.CtxWithHandleTruncateErrLevel(ctx.GetExprCtx(), errctx.LevelError),
- col, option,
- )
- if err != nil {
- return false, errors.Trace(err)
- }
- if isSeqExpr {
- if err := checkSequenceDefaultValue(col); err != nil {
- return false, errors.Trace(err)
+ if len(spec.Partition.Definitions) == 0 ||
+ len(spec.Partition.Definitions) >= len(meta.Partition.Definitions)-pNullOffset {
+ return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs(
+ "FIRST PARTITION, number of partitions does not match")
}
- col.DefaultIsExpr = isSeqExpr
- }
-
- // When the default value is expression, we skip check and convert.
- if !col.DefaultIsExpr {
- if hasDefaultValue, value, err = checkColumnDefaultValue(ctx.GetExprCtx(), col, value); err != nil {
- return hasDefaultValue, errors.Trace(err)
+ if len(spec.PartitionNames) != 0 || len(spec.Partition.Definitions) <= 1 {
+ return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs(
+ "FIRST PARTITION, given value does not generate a list of partition names to be dropped")
}
- value, err = convertTimestampDefaultValToUTC(ctx, value, col)
- if err != nil {
- return hasDefaultValue, errors.Trace(err)
+ for i := range spec.Partition.Definitions {
+ spec.PartitionNames = append(spec.PartitionNames, meta.Partition.Definitions[i+pNullOffset].Name)
}
- } else {
- hasDefaultValue = true
+ // Use the last generated partition as First, i.e. do not drop the last name in the slice
+ spec.PartitionNames = spec.PartitionNames[:len(spec.PartitionNames)-1]
+
+ query, ok := ctx.Value(sessionctx.QueryString).(string)
+ if ok {
+ partNames := make([]string, 0, len(spec.PartitionNames))
+ sqlMode := ctx.GetSessionVars().SQLMode
+ for i := range spec.PartitionNames {
+ partNames = append(partNames, stringutil.Escape(spec.PartitionNames[i].O, sqlMode))
+ }
+ syntacticSugar := spec.Partition.PartitionMethod.OriginalText()
+ syntacticStart := spec.Partition.PartitionMethod.OriginTextPosition()
+ newQuery := query[:syntacticStart] + "DROP PARTITION " + strings.Join(partNames, ", ") + query[syntacticStart+len(syntacticSugar):]
+ defer ctx.SetValue(sessionctx.QueryString, query)
+ ctx.SetValue(sessionctx.QueryString, newQuery)
+ }
+ }
+ partNames := make([]string, len(spec.PartitionNames))
+ for i, partCIName := range spec.PartitionNames {
+ partNames[i] = partCIName.L
}
- err = setDefaultValueWithBinaryPadding(col, value)
+ err = CheckDropTablePartition(meta, partNames)
if err != nil {
- return hasDefaultValue, errors.Trace(err)
+ if dbterror.ErrDropPartitionNonExistent.Equal(err) && spec.IfExists {
+ ctx.GetSessionVars().StmtCtx.AppendNote(err)
+ return nil
+ }
+ return errors.Trace(err)
+ }
+
+ job := &model.Job{
+ SchemaID: schema.ID,
+ TableID: meta.ID,
+ SchemaName: schema.Name.L,
+ SchemaState: model.StatePublic,
+ TableName: meta.Name.L,
+ Type: model.ActionDropTablePartition,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []any{partNames},
+ CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
+ SQLMode: ctx.GetSessionVars().SQLMode,
}
- return hasDefaultValue, nil
-}
-func setDefaultValueWithBinaryPadding(col *table.Column, value any) error {
- err := col.SetDefaultValue(value)
+ err = e.DoDDLJob(ctx, job)
if err != nil {
- return err
- }
- // https://dev.mysql.com/doc/refman/8.0/en/binary-varbinary.html
- // Set the default value for binary type should append the paddings.
- if value != nil {
- if col.GetType() == mysql.TypeString && types.IsBinaryStr(&col.FieldType) && len(value.(string)) < col.GetFlen() {
- padding := make([]byte, col.GetFlen()-len(value.(string)))
- col.DefaultValue = string(append([]byte(col.DefaultValue.(string)), padding...))
+ if dbterror.ErrDropPartitionNonExistent.Equal(err) && spec.IfExists {
+ ctx.GetSessionVars().StmtCtx.AppendNote(err)
+ return nil
}
- }
- return nil
-}
-
-func setColumnComment(ctx sessionctx.Context, col *table.Column, option *ast.ColumnOption) error {
- value, err := expression.EvalSimpleAst(ctx.GetExprCtx(), option.Expr)
- if err != nil {
- return errors.Trace(err)
- }
- if col.Comment, err = value.ToString(); err != nil {
return errors.Trace(err)
}
-
- sessionVars := ctx.GetSessionVars()
- col.Comment, err = validateCommentLength(sessionVars.StmtCtx.ErrCtx(), sessionVars.SQLMode, col.Name.L, &col.Comment, dbterror.ErrTooLongFieldComment)
return errors.Trace(err)
}
-// ProcessModifyColumnOptions process column options.
-func ProcessModifyColumnOptions(ctx sessionctx.Context, col *table.Column, options []*ast.ColumnOption) error {
- var sb strings.Builder
- restoreFlags := format.RestoreStringSingleQuotes | format.RestoreKeyWordLowercase | format.RestoreNameBackQuotes |
- format.RestoreSpacesAroundBinaryOperation | format.RestoreWithoutSchemaName | format.RestoreWithoutSchemaName
- restoreCtx := format.NewRestoreCtx(restoreFlags, &sb)
-
- var hasDefaultValue, setOnUpdateNow bool
- var err error
- var hasNullFlag bool
- for _, opt := range options {
- switch opt.Tp {
- case ast.ColumnOptionDefaultValue:
- hasDefaultValue, err = SetDefaultValue(ctx, col, opt)
- if err != nil {
- return errors.Trace(err)
- }
- case ast.ColumnOptionComment:
- err := setColumnComment(ctx, col, opt)
- if err != nil {
- return errors.Trace(err)
- }
- case ast.ColumnOptionNotNull:
- col.AddFlag(mysql.NotNullFlag)
- case ast.ColumnOptionNull:
- hasNullFlag = true
- col.DelFlag(mysql.NotNullFlag)
- case ast.ColumnOptionAutoIncrement:
- col.AddFlag(mysql.AutoIncrementFlag)
- case ast.ColumnOptionPrimaryKey:
- return errors.Trace(dbterror.ErrUnsupportedModifyColumn.GenWithStack("can't change column constraint (PRIMARY KEY)"))
- case ast.ColumnOptionUniqKey:
- return errors.Trace(dbterror.ErrUnsupportedModifyColumn.GenWithStack("can't change column constraint (UNIQUE KEY)"))
- case ast.ColumnOptionOnUpdate:
- // TODO: Support other time functions.
- if !(col.GetType() == mysql.TypeTimestamp || col.GetType() == mysql.TypeDatetime) {
- return dbterror.ErrInvalidOnUpdate.GenWithStackByArgs(col.Name)
- }
- if !expression.IsValidCurrentTimestampExpr(opt.Expr, &col.FieldType) {
- return dbterror.ErrInvalidOnUpdate.GenWithStackByArgs(col.Name)
- }
- col.AddFlag(mysql.OnUpdateNowFlag)
- setOnUpdateNow = true
- case ast.ColumnOptionGenerated:
- sb.Reset()
- err = opt.Expr.Restore(restoreCtx)
- if err != nil {
- return errors.Trace(err)
- }
- col.GeneratedExprString = sb.String()
- col.GeneratedStored = opt.Stored
- col.Dependences = make(map[string]struct{})
- // Only used by checkModifyGeneratedColumn, there is no need to set a ctor for it.
- col.GeneratedExpr = table.NewClonableExprNode(nil, opt.Expr)
- for _, colName := range FindColumnNamesInExpr(opt.Expr) {
- col.Dependences[colName.Name.L] = struct{}{}
- }
- case ast.ColumnOptionCollate:
- col.SetCollate(opt.StrValue)
- case ast.ColumnOptionReference:
- return errors.Trace(dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs("can't modify with references"))
- case ast.ColumnOptionFulltext:
- return errors.Trace(dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs("can't modify with full text"))
- case ast.ColumnOptionCheck:
- return errors.Trace(dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs("can't modify with check"))
- // Ignore ColumnOptionAutoRandom. It will be handled later.
- case ast.ColumnOptionAutoRandom:
- default:
- return errors.Trace(dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs(fmt.Sprintf("unknown column option type: %d", opt.Tp)))
- }
+func checkFieldTypeCompatible(ft *types.FieldType, other *types.FieldType) bool {
+ // int(1) could match the type with int(8)
+ partialEqual := ft.GetType() == other.GetType() &&
+ ft.GetDecimal() == other.GetDecimal() &&
+ ft.GetCharset() == other.GetCharset() &&
+ ft.GetCollate() == other.GetCollate() &&
+ (ft.GetFlen() == other.GetFlen() || ft.StorageLength() != types.VarStorageLen) &&
+ mysql.HasUnsignedFlag(ft.GetFlag()) == mysql.HasUnsignedFlag(other.GetFlag()) &&
+ mysql.HasAutoIncrementFlag(ft.GetFlag()) == mysql.HasAutoIncrementFlag(other.GetFlag()) &&
+ mysql.HasNotNullFlag(ft.GetFlag()) == mysql.HasNotNullFlag(other.GetFlag()) &&
+ mysql.HasZerofillFlag(ft.GetFlag()) == mysql.HasZerofillFlag(other.GetFlag()) &&
+ mysql.HasBinaryFlag(ft.GetFlag()) == mysql.HasBinaryFlag(other.GetFlag()) &&
+ mysql.HasPriKeyFlag(ft.GetFlag()) == mysql.HasPriKeyFlag(other.GetFlag())
+ if !partialEqual || len(ft.GetElems()) != len(other.GetElems()) {
+ return false
}
-
- if err = processAndCheckDefaultValueAndColumn(ctx, col, nil, hasDefaultValue, setOnUpdateNow, hasNullFlag); err != nil {
- return errors.Trace(err)
+ for i := range ft.GetElems() {
+ if ft.GetElems()[i] != other.GetElems()[i] {
+ return false
+ }
}
-
- return nil
+ return true
}
-func processAndCheckDefaultValueAndColumn(ctx sessionctx.Context, col *table.Column,
- outPriKeyConstraint *ast.Constraint, hasDefaultValue, setOnUpdateNow, hasNullFlag bool) error {
- processDefaultValue(col, hasDefaultValue, setOnUpdateNow)
- processColumnFlags(col)
-
- err := checkPriKeyConstraint(col, hasDefaultValue, hasNullFlag, outPriKeyConstraint)
- if err != nil {
- return errors.Trace(err)
+func checkTiFlashReplicaCompatible(source *model.TiFlashReplicaInfo, target *model.TiFlashReplicaInfo) bool {
+ if source == target {
+ return true
}
- if err = checkColumnValueConstraint(col, col.GetCollate()); err != nil {
- return errors.Trace(err)
+ if source == nil || target == nil {
+ return false
}
- if err = checkDefaultValue(ctx.GetExprCtx(), col, hasDefaultValue); err != nil {
- return errors.Trace(err)
+ if source.Count != target.Count ||
+ source.Available != target.Available || len(source.LocationLabels) != len(target.LocationLabels) {
+ return false
}
- if err = checkColumnFieldLength(col); err != nil {
- return errors.Trace(err)
+ for i, lable := range source.LocationLabels {
+ if target.LocationLabels[i] != lable {
+ return false
+ }
}
- return nil
+ return true
}
-func (e *executor) getModifiableColumnJob(ctx context.Context, sctx sessionctx.Context, ident ast.Ident, originalColName model.CIStr,
- spec *ast.AlterTableSpec) (*model.Job, error) {
- is := e.infoCache.GetLatest()
- schema, ok := is.SchemaByName(ident.Schema)
- if !ok {
- return nil, errors.Trace(infoschema.ErrDatabaseNotExists)
- }
- t, err := is.TableByName(ctx, ident.Schema, ident.Name)
- if err != nil {
- return nil, errors.Trace(infoschema.ErrTableNotExists.GenWithStackByArgs(ident.Schema, ident.Name))
+func checkTableDefCompatible(source *model.TableInfo, target *model.TableInfo) error {
+ // check temp table
+ if target.TempTableType != model.TempTableNone {
+ return errors.Trace(dbterror.ErrPartitionExchangeTempTable.FastGenByArgs(target.Name))
}
- return GetModifiableColumnJob(ctx, sctx, is, ident, originalColName, schema, t, spec)
-}
-
-func checkModifyColumnWithGeneratedColumnsConstraint(allCols []*table.Column, oldColName model.CIStr) error {
- for _, col := range allCols {
- if col.GeneratedExpr == nil {
- continue
+ // check auto_random
+ if source.AutoRandomBits != target.AutoRandomBits ||
+ source.AutoRandomRangeBits != target.AutoRandomRangeBits ||
+ source.Charset != target.Charset ||
+ source.Collate != target.Collate ||
+ source.ShardRowIDBits != target.ShardRowIDBits ||
+ source.MaxShardRowIDBits != target.MaxShardRowIDBits ||
+ !checkTiFlashReplicaCompatible(source.TiFlashReplica, target.TiFlashReplica) {
+ return errors.Trace(dbterror.ErrTablesDifferentMetadata)
+ }
+ if len(source.Cols()) != len(target.Cols()) {
+ return errors.Trace(dbterror.ErrTablesDifferentMetadata)
+ }
+ // Col compatible check
+ for i, sourceCol := range source.Cols() {
+ targetCol := target.Cols()[i]
+ if sourceCol.IsVirtualGenerated() != targetCol.IsVirtualGenerated() {
+ return dbterror.ErrUnsupportedOnGeneratedColumn.GenWithStackByArgs("Exchanging partitions for non-generated columns")
}
- dependedColNames := FindColumnNamesInExpr(col.GeneratedExpr.Internal())
- for _, name := range dependedColNames {
- if name.Name.L == oldColName.L {
- if col.Hidden {
- return dbterror.ErrDependentByFunctionalIndex.GenWithStackByArgs(oldColName.O)
- }
- return dbterror.ErrDependentByGeneratedColumn.GenWithStackByArgs(oldColName.O)
- }
+ // It should strictyle compare expressions for generated columns
+ if sourceCol.Name.L != targetCol.Name.L ||
+ sourceCol.Hidden != targetCol.Hidden ||
+ !checkFieldTypeCompatible(&sourceCol.FieldType, &targetCol.FieldType) ||
+ sourceCol.GeneratedExprString != targetCol.GeneratedExprString {
+ return errors.Trace(dbterror.ErrTablesDifferentMetadata)
+ }
+ if sourceCol.State != model.StatePublic ||
+ targetCol.State != model.StatePublic {
+ return errors.Trace(dbterror.ErrTablesDifferentMetadata)
+ }
+ if sourceCol.ID != targetCol.ID {
+ return dbterror.ErrPartitionExchangeDifferentOption.GenWithStackByArgs(fmt.Sprintf("column: %s", sourceCol.Name))
}
}
- return nil
-}
-
-// ProcessColumnCharsetAndCollation process column charset and collation
-func ProcessColumnCharsetAndCollation(sctx sessionctx.Context, col *table.Column, newCol *table.Column, meta *model.TableInfo, specNewColumn *ast.ColumnDef, schema *model.DBInfo) error {
- var chs, coll string
- var err error
- // TODO: Remove it when all table versions are greater than or equal to TableInfoVersion1.
- // If newCol's charset is empty and the table's version less than TableInfoVersion1,
- // we will not modify the charset of the column. This behavior is not compatible with MySQL.
- if len(newCol.FieldType.GetCharset()) == 0 && meta.Version < model.TableInfoVersion1 {
- chs = col.FieldType.GetCharset()
- coll = col.FieldType.GetCollate()
- } else {
- chs, coll, err = getCharsetAndCollateInColumnDef(sctx.GetSessionVars(), specNewColumn)
- if err != nil {
- return errors.Trace(err)
+ if len(source.Indices) != len(target.Indices) {
+ return errors.Trace(dbterror.ErrTablesDifferentMetadata)
+ }
+ for _, sourceIdx := range source.Indices {
+ if sourceIdx.Global {
+ return dbterror.ErrPartitionExchangeDifferentOption.GenWithStackByArgs(fmt.Sprintf("global index: %s", sourceIdx.Name))
}
- chs, coll, err = ResolveCharsetCollation(sctx.GetSessionVars(),
- ast.CharsetOpt{Chs: chs, Col: coll},
- ast.CharsetOpt{Chs: meta.Charset, Col: meta.Collate},
- ast.CharsetOpt{Chs: schema.Charset, Col: schema.Collate},
- )
- chs, coll = OverwriteCollationWithBinaryFlag(sctx.GetSessionVars(), specNewColumn, chs, coll)
- if err != nil {
- return errors.Trace(err)
+ var compatIdx *model.IndexInfo
+ for _, targetIdx := range target.Indices {
+ if strings.EqualFold(sourceIdx.Name.L, targetIdx.Name.L) {
+ compatIdx = targetIdx
+ }
+ }
+ // No match index
+ if compatIdx == nil {
+ return errors.Trace(dbterror.ErrTablesDifferentMetadata)
+ }
+ // Index type is not compatible
+ if sourceIdx.Tp != compatIdx.Tp ||
+ sourceIdx.Unique != compatIdx.Unique ||
+ sourceIdx.Primary != compatIdx.Primary {
+ return errors.Trace(dbterror.ErrTablesDifferentMetadata)
+ }
+ // The index column
+ if len(sourceIdx.Columns) != len(compatIdx.Columns) {
+ return errors.Trace(dbterror.ErrTablesDifferentMetadata)
+ }
+ for i, sourceIdxCol := range sourceIdx.Columns {
+ compatIdxCol := compatIdx.Columns[i]
+ if sourceIdxCol.Length != compatIdxCol.Length ||
+ sourceIdxCol.Name.L != compatIdxCol.Name.L {
+ return errors.Trace(dbterror.ErrTablesDifferentMetadata)
+ }
+ }
+ if sourceIdx.ID != compatIdx.ID {
+ return dbterror.ErrPartitionExchangeDifferentOption.GenWithStackByArgs(fmt.Sprintf("index: %s", sourceIdx.Name))
}
}
- if err = setCharsetCollationFlenDecimal(&newCol.FieldType, newCol.Name.O, chs, coll, sctx.GetSessionVars()); err != nil {
- return errors.Trace(err)
- }
- decodeEnumSetBinaryLiteralToUTF8(&newCol.FieldType, chs)
return nil
}
-// GetModifiableColumnJob returns a DDL job of model.ActionModifyColumn.
-func GetModifiableColumnJob(
- ctx context.Context,
- sctx sessionctx.Context,
- is infoschema.InfoSchema, // WARN: is maybe nil here.
- ident ast.Ident,
- originalColName model.CIStr,
- schema *model.DBInfo,
- t table.Table,
- spec *ast.AlterTableSpec,
-) (*model.Job, error) {
- var err error
- specNewColumn := spec.NewColumns[0]
-
- col := table.FindCol(t.Cols(), originalColName.L)
- if col == nil {
- return nil, infoschema.ErrColumnNotExists.GenWithStackByArgs(originalColName, ident.Name)
- }
- newColName := specNewColumn.Name.Name
- if newColName.L == model.ExtraHandleName.L {
- return nil, dbterror.ErrWrongColumnName.GenWithStackByArgs(newColName.L)
+func checkExchangePartition(pt *model.TableInfo, nt *model.TableInfo) error {
+ if nt.IsView() || nt.IsSequence() {
+ return errors.Trace(dbterror.ErrCheckNoSuchTable)
}
- errG := checkModifyColumnWithGeneratedColumnsConstraint(t.Cols(), originalColName)
-
- // If we want to rename the column name, we need to check whether it already exists.
- if newColName.L != originalColName.L {
- c := table.FindCol(t.Cols(), newColName.L)
- if c != nil {
- return nil, infoschema.ErrColumnExists.GenWithStackByArgs(newColName)
- }
-
- // And also check the generated columns dependency, if some generated columns
- // depend on this column, we can't rename the column name.
- if errG != nil {
- return nil, errors.Trace(errG)
- }
+ if pt.GetPartitionInfo() == nil {
+ return errors.Trace(dbterror.ErrPartitionMgmtOnNonpartitioned)
}
-
- // Constraints in the new column means adding new constraints. Errors should thrown,
- // which will be done by `processColumnOptions` later.
- if specNewColumn.Tp == nil {
- // Make sure the column definition is simple field type.
- return nil, errors.Trace(dbterror.ErrUnsupportedModifyColumn)
+ if nt.GetPartitionInfo() != nil {
+ return errors.Trace(dbterror.ErrPartitionExchangePartTable.GenWithStackByArgs(nt.Name))
}
- if err = checkColumnAttributes(specNewColumn.Name.OrigColName(), specNewColumn.Tp); err != nil {
- return nil, errors.Trace(err)
+ if len(nt.ForeignKeys) > 0 {
+ return errors.Trace(dbterror.ErrPartitionExchangeForeignKey.GenWithStackByArgs(nt.Name))
}
- newCol := table.ToColumn(&model.ColumnInfo{
- ID: col.ID,
- // We use this PR(https://github.com/pingcap/tidb/pull/6274) as the dividing line to define whether it is a new version or an old version TiDB.
- // The old version TiDB initializes the column's offset and state here.
- // The new version TiDB doesn't initialize the column's offset and state, and it will do the initialization in run DDL function.
- // When we do the rolling upgrade the following may happen:
- // a new version TiDB builds the DDL job that doesn't be set the column's offset and state,
- // and the old version TiDB is the DDL owner, it doesn't get offset and state from the store. Then it will encounter errors.
- // So here we set offset and state to support the rolling upgrade.
- Offset: col.Offset,
- State: col.State,
- OriginDefaultValue: col.OriginDefaultValue,
- OriginDefaultValueBit: col.OriginDefaultValueBit,
- FieldType: *specNewColumn.Tp,
- Name: newColName,
- Version: col.Version,
- })
+ return nil
+}
- if err = ProcessColumnCharsetAndCollation(sctx, col, newCol, t.Meta(), specNewColumn, schema); err != nil {
- return nil, err
+func (e *executor) ExchangeTablePartition(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
+ ptSchema, pt, err := e.getSchemaAndTableByIdent(ident)
+ if err != nil {
+ return errors.Trace(err)
}
- if err = checkModifyColumnWithForeignKeyConstraint(is, schema.Name.L, t.Meta(), col.ColumnInfo, newCol.ColumnInfo); err != nil {
- return nil, errors.Trace(err)
- }
+ ptMeta := pt.Meta()
- // Copy index related options to the new spec.
- indexFlags := col.FieldType.GetFlag() & (mysql.PriKeyFlag | mysql.UniqueKeyFlag | mysql.MultipleKeyFlag)
- newCol.FieldType.AddFlag(indexFlags)
- if mysql.HasPriKeyFlag(col.FieldType.GetFlag()) {
- newCol.FieldType.AddFlag(mysql.NotNullFlag)
- // TODO: If user explicitly set NULL, we should throw error ErrPrimaryCantHaveNull.
- }
+ ntIdent := ast.Ident{Schema: spec.NewTable.Schema, Name: spec.NewTable.Name}
- if err = ProcessModifyColumnOptions(sctx, newCol, specNewColumn.Options); err != nil {
- return nil, errors.Trace(err)
+ // We should check local temporary here using session's info schema because the local temporary tables are only stored in session.
+ ntLocalTempTable, err := sessiontxn.GetTxnManager(ctx).GetTxnInfoSchema().TableByName(context.Background(), ntIdent.Schema, ntIdent.Name)
+ if err == nil && ntLocalTempTable.Meta().TempTableType == model.TempTableLocal {
+ return errors.Trace(dbterror.ErrPartitionExchangeTempTable.FastGenByArgs(ntLocalTempTable.Meta().Name))
}
- if err = checkModifyTypes(&col.FieldType, &newCol.FieldType, isColumnWithIndex(col.Name.L, t.Meta().Indices)); err != nil {
- if strings.Contains(err.Error(), "Unsupported modifying collation") {
- colErrMsg := "Unsupported modifying collation of column '%s' from '%s' to '%s' when index is defined on it."
- err = dbterror.ErrUnsupportedModifyCollation.GenWithStack(colErrMsg, col.Name.L, col.GetCollate(), newCol.GetCollate())
- }
- return nil, errors.Trace(err)
- }
- needChangeColData := needChangeColumnData(col.ColumnInfo, newCol.ColumnInfo)
- if needChangeColData {
- if err = isGeneratedRelatedColumn(t.Meta(), newCol.ColumnInfo, col.ColumnInfo); err != nil {
- return nil, errors.Trace(err)
- }
- if t.Meta().Partition != nil {
- return nil, dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs("table is partition table")
- }
+ ntSchema, nt, err := e.getSchemaAndTableByIdent(ntIdent)
+ if err != nil {
+ return errors.Trace(err)
}
- // Check that the column change does not affect the partitioning column
- // It must keep the same type, int [unsigned], [var]char, date[time]
- if t.Meta().Partition != nil {
- pt, ok := t.(table.PartitionedTable)
- if !ok {
- // Should never happen!
- return nil, dbterror.ErrNotAllowedTypeInPartition.GenWithStackByArgs(newCol.Name.O)
- }
- isPartitioningColumn := false
- for _, name := range pt.GetPartitionColumnNames() {
- if strings.EqualFold(name.L, col.Name.L) {
- isPartitioningColumn = true
- break
- }
- }
- if isPartitioningColumn {
- // TODO: update the partitioning columns with new names if column is renamed
- // Would be an extension from MySQL which does not support it.
- if col.Name.L != newCol.Name.L {
- return nil, dbterror.ErrDependentByPartitionFunctional.GenWithStackByArgs(col.Name.L)
- }
- if !isColTypeAllowedAsPartitioningCol(t.Meta().Partition.Type, newCol.FieldType) {
- return nil, dbterror.ErrNotAllowedTypeInPartition.GenWithStackByArgs(newCol.Name.O)
- }
- pi := pt.Meta().GetPartitionInfo()
- if len(pi.Columns) == 0 {
- // non COLUMNS partitioning, only checks INTs, not their actual range
- // There are many edge cases, like when truncating SQL Mode is allowed
- // which will change the partitioning expression value resulting in a
- // different partition. Better be safe and not allow decreasing of length.
- // TODO: Should we allow it in strict mode? Wait for a use case / request.
- if newCol.FieldType.GetFlen() < col.FieldType.GetFlen() {
- return nil, dbterror.ErrUnsupportedModifyCollation.GenWithStack("Unsupported modify column, decreasing length of int may result in truncation and change of partition")
- }
- }
- // Basically only allow changes of the length/decimals for the column
- // Note that enum is not allowed, so elems are not checked
- // TODO: support partition by ENUM
- if newCol.FieldType.EvalType() != col.FieldType.EvalType() ||
- newCol.FieldType.GetFlag() != col.FieldType.GetFlag() ||
- newCol.FieldType.GetCollate() != col.FieldType.GetCollate() ||
- newCol.FieldType.GetCharset() != col.FieldType.GetCharset() {
- return nil, dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs("can't change the partitioning column, since it would require reorganize all partitions")
- }
- // Generate a new PartitionInfo and validate it together with the new column definition
- // Checks if all partition definition values are compatible.
- // Similar to what buildRangePartitionDefinitions would do in terms of checks.
-
- tblInfo := pt.Meta()
- newTblInfo := *tblInfo
- // Replace col with newCol and see if we can generate a new SHOW CREATE TABLE
- // and reparse it and build new partition definitions (which will do additional
- // checks columns vs partition definition values
- newCols := make([]*model.ColumnInfo, 0, len(newTblInfo.Columns))
- for _, c := range newTblInfo.Columns {
- if c.ID == col.ID {
- newCols = append(newCols, newCol.ColumnInfo)
- continue
- }
- newCols = append(newCols, c)
- }
- newTblInfo.Columns = newCols
+ ntMeta := nt.Meta()
- var buf bytes.Buffer
- AppendPartitionInfo(tblInfo.GetPartitionInfo(), &buf, mysql.ModeNone)
- // The parser supports ALTER TABLE ... PARTITION BY ... even if the ddl code does not yet :)
- // Ignoring warnings
- stmt, _, err := parser.New().ParseSQL("ALTER TABLE t " + buf.String())
- if err != nil {
- // Should never happen!
- return nil, dbterror.ErrUnsupportedModifyColumn.GenWithStack("cannot parse generated PartitionInfo")
- }
- at, ok := stmt[0].(*ast.AlterTableStmt)
- if !ok || len(at.Specs) != 1 || at.Specs[0].Partition == nil {
- return nil, dbterror.ErrUnsupportedModifyColumn.GenWithStack("cannot parse generated PartitionInfo")
- }
- pAst := at.Specs[0].Partition
- _, err = buildPartitionDefinitionsInfo(
- exprctx.CtxWithHandleTruncateErrLevel(sctx.GetExprCtx(), errctx.LevelError),
- pAst.Definitions, &newTblInfo, uint64(len(newTblInfo.Partition.Definitions)),
- )
- if err != nil {
- return nil, dbterror.ErrUnsupportedModifyColumn.GenWithStack("New column does not match partition definitions: %s", err.Error())
- }
- }
+ err = checkExchangePartition(ptMeta, ntMeta)
+ if err != nil {
+ return errors.Trace(err)
}
- // We don't support modifying column from not_auto_increment to auto_increment.
- if !mysql.HasAutoIncrementFlag(col.GetFlag()) && mysql.HasAutoIncrementFlag(newCol.GetFlag()) {
- return nil, dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs("can't set auto_increment")
- }
- // Not support auto id with default value.
- if mysql.HasAutoIncrementFlag(newCol.GetFlag()) && newCol.GetDefaultValue() != nil {
- return nil, dbterror.ErrInvalidDefaultValue.GenWithStackByArgs(newCol.Name)
- }
- // Disallow modifying column from auto_increment to not auto_increment if the session variable `AllowRemoveAutoInc` is false.
- if !sctx.GetSessionVars().AllowRemoveAutoInc && mysql.HasAutoIncrementFlag(col.GetFlag()) && !mysql.HasAutoIncrementFlag(newCol.GetFlag()) {
- return nil, dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs("can't remove auto_increment without @@tidb_allow_remove_auto_inc enabled")
- }
+ partName := spec.PartitionNames[0].L
- // We support modifying the type definitions of 'null' to 'not null' now.
- var modifyColumnTp byte
- if !mysql.HasNotNullFlag(col.GetFlag()) && mysql.HasNotNullFlag(newCol.GetFlag()) {
- if err = checkForNullValue(ctx, sctx, true, ident.Schema, ident.Name, newCol.ColumnInfo, col.ColumnInfo); err != nil {
- return nil, errors.Trace(err)
- }
- // `modifyColumnTp` indicates that there is a type modification.
- modifyColumnTp = mysql.TypeNull
- }
+ // NOTE: if pt is subPartitioned, it should be checked
- if err = checkColumnWithIndexConstraint(t.Meta(), col.ColumnInfo, newCol.ColumnInfo); err != nil {
- return nil, err
+ defID, err := tables.FindPartitionByName(ptMeta, partName)
+ if err != nil {
+ return errors.Trace(err)
}
- // As same with MySQL, we don't support modifying the stored status for generated columns.
- if err = checkModifyGeneratedColumn(sctx, schema.Name, t, col, newCol, specNewColumn, spec.Position); err != nil {
- return nil, errors.Trace(err)
- }
- if errG != nil {
- // According to issue https://github.com/pingcap/tidb/issues/24321,
- // changing the type of a column involving generating a column is prohibited.
- return nil, dbterror.ErrUnsupportedOnGeneratedColumn.GenWithStackByArgs(errG.Error())
+ err = checkTableDefCompatible(ptMeta, ntMeta)
+ if err != nil {
+ return errors.Trace(err)
}
- if t.Meta().TTLInfo != nil {
- // the column referenced by TTL should be a time type
- if t.Meta().TTLInfo.ColumnName.L == originalColName.L && !types.IsTypeTime(newCol.ColumnInfo.FieldType.GetType()) {
- return nil, errors.Trace(dbterror.ErrUnsupportedColumnInTTLConfig.GenWithStackByArgs(newCol.ColumnInfo.Name.O))
- }
+ job := &model.Job{
+ SchemaID: ntSchema.ID,
+ TableID: ntMeta.ID,
+ SchemaName: ntSchema.Name.L,
+ TableName: ntMeta.Name.L,
+ Type: model.ActionExchangeTablePartition,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []any{defID, ptSchema.ID, ptMeta.ID, partName, spec.WithValidation},
+ CtxVars: []any{[]int64{ntSchema.ID, ptSchema.ID}, []int64{ntMeta.ID, ptMeta.ID}},
+ CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
+ InvolvingSchemaInfo: []model.InvolvingSchemaInfo{
+ {Database: ptSchema.Name.L, Table: ptMeta.Name.L},
+ {Database: ntSchema.Name.L, Table: ntMeta.Name.L},
+ },
+ SQLMode: ctx.GetSessionVars().SQLMode,
}
- var newAutoRandBits uint64
- if newAutoRandBits, err = checkAutoRandom(t.Meta(), col, specNewColumn); err != nil {
- return nil, errors.Trace(err)
+ err = e.DoDDLJob(ctx, job)
+ if err != nil {
+ return errors.Trace(err)
}
+ ctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackError("after the exchange, please analyze related table of the exchange to update statistics"))
+ return nil
+}
- txn, err := sctx.Txn(true)
+// DropColumn will drop a column from the table, now we don't support drop the column with index covered.
+func (e *executor) DropColumn(ctx sessionctx.Context, ti ast.Ident, spec *ast.AlterTableSpec) error {
+ schema, t, err := e.getSchemaAndTableByIdent(ti)
if err != nil {
- return nil, errors.Trace(err)
+ return errors.Trace(err)
}
- bdrRole, err := meta.NewMeta(txn).GetBDRRole()
+ failpoint.InjectCall("afterGetSchemaAndTableByIdent", ctx)
+
+ isDropable, err := checkIsDroppableColumn(ctx, e.infoCache.GetLatest(), schema, t, spec)
if err != nil {
- return nil, errors.Trace(err)
+ return err
+ }
+ if !isDropable {
+ return nil
}
- if bdrRole == string(ast.BDRRolePrimary) &&
- deniedByBDRWhenModifyColumn(newCol.FieldType, col.FieldType, specNewColumn.Options) {
- return nil, dbterror.ErrBDRRestrictedDDL.FastGenByArgs(bdrRole)
+ colName := spec.OldColumnName.Name
+ err = checkVisibleColumnCnt(t, 0, 1)
+ if err != nil {
+ return err
}
job := &model.Job{
SchemaID: schema.ID,
TableID: t.Meta().ID,
SchemaName: schema.Name.L,
+ SchemaState: model.StatePublic,
TableName: t.Meta().Name.L,
- Type: model.ActionModifyColumn,
+ Type: model.ActionDropColumn,
BinlogInfo: &model.HistoryInfo{},
- ReorgMeta: NewDDLReorgMeta(sctx),
- CtxVars: []any{needChangeColData},
- Args: []any{&newCol.ColumnInfo, originalColName, spec.Position, modifyColumnTp, newAutoRandBits},
- CDCWriteSource: sctx.GetSessionVars().CDCWriteSource,
- SQLMode: sctx.GetSessionVars().SQLMode,
+ Args: []any{colName, spec.IfExists},
+ CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
+ SQLMode: ctx.GetSessionVars().SQLMode,
}
- return job, nil
+
+ err = e.DoDDLJob(ctx, job)
+ return errors.Trace(err)
}
-// checkColumnWithIndexConstraint is used to check the related index constraint of the modified column.
-// Index has a max-prefix-length constraint. eg: a varchar(100), index idx(a), modifying column a to a varchar(4000)
-// will cause index idx to break the max-prefix-length constraint.
-func checkColumnWithIndexConstraint(tbInfo *model.TableInfo, originalCol, newCol *model.ColumnInfo) error {
- columns := make([]*model.ColumnInfo, 0, len(tbInfo.Columns))
- columns = append(columns, tbInfo.Columns...)
- // Replace old column with new column.
- for i, col := range columns {
- if col.Name.L != originalCol.Name.L {
- continue
+func checkIsDroppableColumn(ctx sessionctx.Context, is infoschema.InfoSchema, schema *model.DBInfo, t table.Table, spec *ast.AlterTableSpec) (isDrapable bool, err error) {
+ tblInfo := t.Meta()
+ // Check whether dropped column has existed.
+ colName := spec.OldColumnName.Name
+ col := table.FindCol(t.VisibleCols(), colName.L)
+ if col == nil {
+ err = dbterror.ErrCantDropFieldOrKey.GenWithStackByArgs(colName)
+ if spec.IfExists {
+ ctx.GetSessionVars().StmtCtx.AppendNote(err)
+ return false, nil
}
- columns[i] = newCol.Clone()
- columns[i].Name = originalCol.Name
- break
+ return false, err
}
- pkIndex := tables.FindPrimaryIndex(tbInfo)
-
- checkOneIndex := func(indexInfo *model.IndexInfo) (err error) {
- var modified bool
- for _, col := range indexInfo.Columns {
- if col.Name.L == originalCol.Name.L {
- modified = true
- break
- }
- }
- if !modified {
- return
- }
- err = checkIndexInModifiableColumns(columns, indexInfo.Columns)
- if err != nil {
- return
- }
- err = checkIndexPrefixLength(columns, indexInfo.Columns)
- return
+ if err = isDroppableColumn(tblInfo, colName); err != nil {
+ return false, errors.Trace(err)
}
-
- // Check primary key first.
- var err error
-
- if pkIndex != nil {
- err = checkOneIndex(pkIndex)
- if err != nil {
- return err
- }
+ if err = checkDropColumnWithPartitionConstraint(t, colName); err != nil {
+ return false, errors.Trace(err)
}
-
- // Check secondary indexes.
- for _, indexInfo := range tbInfo.Indices {
- if indexInfo.Primary {
- continue
- }
- // the second param should always be set to true, check index length only if it was modified
- // checkOneIndex needs one param only.
- err = checkOneIndex(indexInfo)
- if err != nil {
- return err
- }
+ // Check the column with foreign key.
+ err = checkDropColumnWithForeignKeyConstraint(is, schema.Name.L, tblInfo, colName.L)
+ if err != nil {
+ return false, errors.Trace(err)
}
- return nil
+ // Check the column with TTL config
+ err = checkDropColumnWithTTLConfig(tblInfo, colName.L)
+ if err != nil {
+ return false, errors.Trace(err)
+ }
+ // We don't support dropping column with PK handle covered now.
+ if col.IsPKHandleColumn(tblInfo) {
+ return false, dbterror.ErrUnsupportedPKHandle
+ }
+ if mysql.HasAutoIncrementFlag(col.GetFlag()) && !ctx.GetSessionVars().AllowRemoveAutoInc {
+ return false, dbterror.ErrCantDropColWithAutoInc
+ }
+ return true, nil
}
-func checkIndexInModifiableColumns(columns []*model.ColumnInfo, idxColumns []*model.IndexColumn) error {
- for _, ic := range idxColumns {
- col := model.FindColumnInfo(columns, ic.Name.L)
- if col == nil {
- return dbterror.ErrKeyColumnDoesNotExits.GenWithStack("column does not exist: %s", ic.Name)
- }
-
- prefixLength := types.UnspecifiedLength
- if types.IsTypePrefixable(col.FieldType.GetType()) && col.FieldType.GetFlen() > ic.Length {
- // When the index column is changed, prefix length is only valid
- // if the type is still prefixable and larger than old prefix length.
- prefixLength = ic.Length
- }
- if err := checkIndexColumn(nil, col, prefixLength); err != nil {
- return err
+// checkDropColumnWithPartitionConstraint is used to check the partition constraint of the drop column.
+func checkDropColumnWithPartitionConstraint(t table.Table, colName model.CIStr) error {
+ if t.Meta().Partition == nil {
+ return nil
+ }
+ pt, ok := t.(table.PartitionedTable)
+ if !ok {
+ // Should never happen!
+ return errors.Trace(dbterror.ErrDependentByPartitionFunctional.GenWithStackByArgs(colName.L))
+ }
+ for _, name := range pt.GetPartitionColumnNames() {
+ if strings.EqualFold(name.L, colName.L) {
+ return errors.Trace(dbterror.ErrDependentByPartitionFunctional.GenWithStackByArgs(colName.L))
}
}
return nil
}
-func isClusteredPKColumn(col *table.Column, tblInfo *model.TableInfo) bool {
- switch {
- case tblInfo.PKIsHandle:
- return mysql.HasPriKeyFlag(col.GetFlag())
- case tblInfo.IsCommonHandle:
- pk := tables.FindPrimaryIndex(tblInfo)
- for _, c := range pk.Columns {
- if c.Name.L == col.Name.L {
- return true
- }
+func checkVisibleColumnCnt(t table.Table, addCnt, dropCnt int) error {
+ tblInfo := t.Meta()
+ visibleColumCnt := 0
+ for _, column := range tblInfo.Columns {
+ if !column.Hidden {
+ visibleColumCnt++
}
- return false
- default:
- return false
}
+ if visibleColumCnt+addCnt > dropCnt {
+ return nil
+ }
+ if len(tblInfo.Columns)-visibleColumCnt > 0 {
+ // There are only invisible columns.
+ return dbterror.ErrTableMustHaveColumns
+ }
+ return dbterror.ErrCantRemoveAllFields
}
-func checkAutoRandom(tableInfo *model.TableInfo, originCol *table.Column, specNewColumn *ast.ColumnDef) (uint64, error) {
- var oldShardBits, oldRangeBits uint64
- if isClusteredPKColumn(originCol, tableInfo) {
- oldShardBits = tableInfo.AutoRandomBits
- oldRangeBits = tableInfo.AutoRandomRangeBits
+// checkModifyCharsetAndCollation returns error when the charset or collation is not modifiable.
+// needRewriteCollationData is used when trying to modify the collation of a column, it is true when the column is with
+// index because index of a string column is collation-aware.
+func checkModifyCharsetAndCollation(toCharset, toCollate, origCharset, origCollate string, needRewriteCollationData bool) error {
+ if !charset.ValidCharsetAndCollation(toCharset, toCollate) {
+ return dbterror.ErrUnknownCharacterSet.GenWithStack("Unknown character set: '%s', collation: '%s'", toCharset, toCollate)
}
- newShardBits, newRangeBits, err := extractAutoRandomBitsFromColDef(specNewColumn)
- if err != nil {
- return 0, errors.Trace(err)
+
+ if needRewriteCollationData && collate.NewCollationEnabled() && !collate.CompatibleCollate(origCollate, toCollate) {
+ return dbterror.ErrUnsupportedModifyCollation.GenWithStackByArgs(origCollate, toCollate)
}
- switch {
- case oldShardBits == newShardBits:
- case oldShardBits < newShardBits:
- addingAutoRandom := oldShardBits == 0
- if addingAutoRandom {
- convFromAutoInc := mysql.HasAutoIncrementFlag(originCol.GetFlag()) && originCol.IsPKHandleColumn(tableInfo)
- if !convFromAutoInc {
- return 0, dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomAlterChangeFromAutoInc)
- }
- }
- if autoid.AutoRandomShardBitsMax < newShardBits {
- errMsg := fmt.Sprintf(autoid.AutoRandomOverflowErrMsg,
- autoid.AutoRandomShardBitsMax, newShardBits, specNewColumn.Name.Name.O)
- return 0, dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(errMsg)
- }
- // increasing auto_random shard bits is allowed.
- case oldShardBits > newShardBits:
- if newShardBits == 0 {
- return 0, dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomAlterErrMsg)
- }
- return 0, dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomDecreaseBitErrMsg)
+
+ if (origCharset == charset.CharsetUTF8 && toCharset == charset.CharsetUTF8MB4) ||
+ (origCharset == charset.CharsetUTF8 && toCharset == charset.CharsetUTF8) ||
+ (origCharset == charset.CharsetUTF8MB4 && toCharset == charset.CharsetUTF8MB4) ||
+ (origCharset == charset.CharsetLatin1 && toCharset == charset.CharsetUTF8MB4) {
+ // TiDB only allow utf8/latin1 to be changed to utf8mb4, or changing the collation when the charset is utf8/utf8mb4/latin1.
+ return nil
}
- modifyingAutoRandCol := oldShardBits > 0 || newShardBits > 0
- if modifyingAutoRandCol {
- // Disallow changing the column field type.
- if originCol.GetType() != specNewColumn.Tp.GetType() {
- return 0, dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomModifyColTypeErrMsg)
- }
- if originCol.GetType() != mysql.TypeLonglong {
- return 0, dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(fmt.Sprintf(autoid.AutoRandomOnNonBigIntColumn, types.TypeStr(originCol.GetType())))
- }
- // Disallow changing from auto_random to auto_increment column.
- if containsColumnOption(specNewColumn, ast.ColumnOptionAutoIncrement) {
- return 0, dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomIncompatibleWithAutoIncErrMsg)
- }
- // Disallow specifying a default value on auto_random column.
- if containsColumnOption(specNewColumn, ast.ColumnOptionDefaultValue) {
- return 0, dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomIncompatibleWithDefaultValueErrMsg)
- }
+ if toCharset != origCharset {
+ msg := fmt.Sprintf("charset from %s to %s", origCharset, toCharset)
+ return dbterror.ErrUnsupportedModifyCharset.GenWithStackByArgs(msg)
}
- if rangeBitsIsChanged(oldRangeBits, newRangeBits) {
- return 0, dbterror.ErrInvalidAutoRandom.FastGenByArgs(autoid.AutoRandomUnsupportedAlterRangeBits)
+ if toCollate != origCollate {
+ msg := fmt.Sprintf("change collate from %s to %s", origCollate, toCollate)
+ return dbterror.ErrUnsupportedModifyCharset.GenWithStackByArgs(msg)
}
- return newShardBits, nil
+ return nil
}
-func rangeBitsIsChanged(oldBits, newBits uint64) bool {
- if oldBits == 0 {
- oldBits = autoid.AutoRandomRangeBitsDefault
+func (e *executor) getModifiableColumnJob(ctx context.Context, sctx sessionctx.Context, ident ast.Ident, originalColName model.CIStr,
+ spec *ast.AlterTableSpec) (*model.Job, error) {
+ is := e.infoCache.GetLatest()
+ schema, ok := is.SchemaByName(ident.Schema)
+ if !ok {
+ return nil, errors.Trace(infoschema.ErrDatabaseNotExists)
}
- if newBits == 0 {
- newBits = autoid.AutoRandomRangeBitsDefault
+ t, err := is.TableByName(ctx, ident.Schema, ident.Name)
+ if err != nil {
+ return nil, errors.Trace(infoschema.ErrTableNotExists.GenWithStackByArgs(ident.Schema, ident.Name))
}
- return oldBits != newBits
+
+ return GetModifiableColumnJob(ctx, sctx, is, ident, originalColName, schema, t, spec)
}
// ChangeColumn renames an existing column and modifies the column's definition,
@@ -6982,7 +3886,7 @@ func (e *executor) AlterTableDropStatistics(ctx sessionctx.Context, ident ast.Id
// UpdateTableReplicaInfo updates the table flash replica infos.
func (e *executor) UpdateTableReplicaInfo(ctx sessionctx.Context, physicalID int64, available bool) error {
is := e.infoCache.GetLatest()
- tb, ok := is.TableByID(physicalID)
+ tb, ok := is.TableByID(e.ctx, physicalID)
if !ok {
tb, _, _ = is.FindTableByPartitionID(physicalID)
if tb == nil {
@@ -7398,7 +4302,7 @@ func (e *executor) renameTable(ctx sessionctx.Context, oldIdent, newIdent ast.Id
return nil
}
- if tbl, ok := is.TableByID(tableID); ok {
+ if tbl, ok := is.TableByID(e.ctx, tableID); ok {
if tbl.Meta().TableCacheStatusType != model.TableCacheStatusDisable {
return errors.Trace(dbterror.ErrOptOnCacheTable.GenWithStackByArgs("Rename Table"))
}
@@ -7446,7 +4350,7 @@ func (e *executor) renameTables(ctx sessionctx.Context, oldIdents, newIdents []a
return err
}
- if t, ok := is.TableByID(tableID); ok {
+ if t, ok := is.TableByID(e.ctx, tableID); ok {
if t.Meta().TableCacheStatusType != model.TableCacheStatusDisable {
return errors.Trace(dbterror.ErrOptOnCacheTable.GenWithStackByArgs("Rename Tables"))
}
@@ -7638,7 +4542,6 @@ func (e *executor) CreatePrimaryKey(ctx sessionctx.Context, ti ast.Ident, indexN
return err
}
- global := false
if tblInfo.GetPartitionInfo() != nil {
ck, err := checkPartitionKeysConstraint(tblInfo.GetPartitionInfo(), indexColumns, tblInfo)
if err != nil {
@@ -7649,7 +4552,9 @@ func (e *executor) CreatePrimaryKey(ctx sessionctx.Context, ti ast.Ident, indexN
return dbterror.ErrUniqueKeyNeedAllFieldsInPf.GenWithStackByArgs("PRIMARY")
}
// index columns does not contain all partition columns, must set global
- global = true
+ if indexOption == nil || !indexOption.Global {
+ return dbterror.ErrGlobalIndexNotExplicitlySet.GenWithStackByArgs("PRIMARY")
+ }
}
}
@@ -7663,135 +4568,30 @@ func (e *executor) CreatePrimaryKey(ctx sessionctx.Context, ti ast.Ident, indexN
unique := true
sqlMode := ctx.GetSessionVars().SQLMode
+ // global is set to 'false' is just there to be backwards compatible,
+ // to avoid unmarshal issues, it is now part of indexOption.
+ global := false
job := &model.Job{
SchemaID: schema.ID,
TableID: t.Meta().ID,
- SchemaName: schema.Name.L,
- TableName: t.Meta().Name.L,
- Type: model.ActionAddPrimaryKey,
- BinlogInfo: &model.HistoryInfo{},
- ReorgMeta: nil,
- Args: []any{unique, indexName, indexPartSpecifications, indexOption, sqlMode, nil, global},
- Priority: ctx.GetSessionVars().DDLReorgPriority,
- CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
- SQLMode: ctx.GetSessionVars().SQLMode,
- }
- reorgMeta, err := newReorgMetaFromVariables(job, ctx)
- if err != nil {
- return err
- }
- job.ReorgMeta = reorgMeta
-
- err = e.DoDDLJob(ctx, job)
- return errors.Trace(err)
-}
-
-func precheckBuildHiddenColumnInfo(
- indexPartSpecifications []*ast.IndexPartSpecification,
- indexName model.CIStr,
-) error {
- for i, idxPart := range indexPartSpecifications {
- if idxPart.Expr == nil {
- continue
- }
- name := fmt.Sprintf("%s_%s_%d", expressionIndexPrefix, indexName, i)
- if utf8.RuneCountInString(name) > mysql.MaxColumnNameLength {
- // TODO: Refine the error message.
- return dbterror.ErrTooLongIdent.GenWithStackByArgs("hidden column")
- }
- // TODO: Refine the error message.
- if err := checkIllegalFn4Generated(indexName.L, typeIndex, idxPart.Expr); err != nil {
- return errors.Trace(err)
- }
- }
- return nil
-}
-
-func buildHiddenColumnInfoWithCheck(ctx sessionctx.Context, indexPartSpecifications []*ast.IndexPartSpecification, indexName model.CIStr, tblInfo *model.TableInfo, existCols []*table.Column) ([]*model.ColumnInfo, error) {
- if err := precheckBuildHiddenColumnInfo(indexPartSpecifications, indexName); err != nil {
- return nil, err
- }
- return BuildHiddenColumnInfo(ctx, indexPartSpecifications, indexName, tblInfo, existCols)
-}
-
-// BuildHiddenColumnInfo builds hidden column info.
-func BuildHiddenColumnInfo(ctx sessionctx.Context, indexPartSpecifications []*ast.IndexPartSpecification, indexName model.CIStr, tblInfo *model.TableInfo, existCols []*table.Column) ([]*model.ColumnInfo, error) {
- hiddenCols := make([]*model.ColumnInfo, 0, len(indexPartSpecifications))
- for i, idxPart := range indexPartSpecifications {
- if idxPart.Expr == nil {
- continue
- }
- idxPart.Column = &ast.ColumnName{Name: model.NewCIStr(fmt.Sprintf("%s_%s_%d", expressionIndexPrefix, indexName, i))}
- // Check whether the hidden columns have existed.
- col := table.FindCol(existCols, idxPart.Column.Name.L)
- if col != nil {
- // TODO: Use expression index related error.
- return nil, infoschema.ErrColumnExists.GenWithStackByArgs(col.Name.String())
- }
- idxPart.Length = types.UnspecifiedLength
- // The index part is an expression, prepare a hidden column for it.
-
- var sb strings.Builder
- restoreFlags := format.RestoreStringSingleQuotes | format.RestoreKeyWordLowercase | format.RestoreNameBackQuotes |
- format.RestoreSpacesAroundBinaryOperation | format.RestoreWithoutSchemaName | format.RestoreWithoutTableName
- restoreCtx := format.NewRestoreCtx(restoreFlags, &sb)
- sb.Reset()
- err := idxPart.Expr.Restore(restoreCtx)
- if err != nil {
- return nil, errors.Trace(err)
- }
- expr, err := expression.BuildSimpleExpr(ctx.GetExprCtx(), idxPart.Expr,
- expression.WithTableInfo(ctx.GetSessionVars().CurrentDB, tblInfo),
- expression.WithAllowCastArray(true),
- )
- if err != nil {
- // TODO: refine the error message.
- return nil, err
- }
- if _, ok := expr.(*expression.Column); ok {
- return nil, dbterror.ErrFunctionalIndexOnField
- }
-
- colInfo := &model.ColumnInfo{
- Name: idxPart.Column.Name,
- GeneratedExprString: sb.String(),
- GeneratedStored: false,
- Version: model.CurrLatestColumnInfoVersion,
- Dependences: make(map[string]struct{}),
- Hidden: true,
- FieldType: *expr.GetType(ctx.GetExprCtx().GetEvalCtx()),
- }
- // Reset some flag, it may be caused by wrong type infer. But it's not easy to fix them all, so reset them here for safety.
- colInfo.DelFlag(mysql.PriKeyFlag | mysql.UniqueKeyFlag | mysql.AutoIncrementFlag)
-
- if colInfo.GetType() == mysql.TypeDatetime || colInfo.GetType() == mysql.TypeDate || colInfo.GetType() == mysql.TypeTimestamp || colInfo.GetType() == mysql.TypeDuration {
- if colInfo.FieldType.GetDecimal() == types.UnspecifiedLength {
- colInfo.FieldType.SetDecimal(types.MaxFsp)
- }
- }
- // For an array, the collation is set to "binary". The collation has no effect on the array itself (as it's usually
- // regarded as a JSON), but will influence how TiKV handles the index value.
- if colInfo.FieldType.IsArray() {
- colInfo.SetCharset("binary")
- colInfo.SetCollate("binary")
- }
- checkDependencies := make(map[string]struct{})
- for _, colName := range FindColumnNamesInExpr(idxPart.Expr) {
- colInfo.Dependences[colName.Name.L] = struct{}{}
- checkDependencies[colName.Name.L] = struct{}{}
- }
- if err = checkDependedColExist(checkDependencies, existCols); err != nil {
- return nil, errors.Trace(err)
- }
- if !ctx.GetSessionVars().EnableAutoIncrementInGenerated {
- if err = checkExpressionIndexAutoIncrement(indexName.O, colInfo.Dependences, tblInfo); err != nil {
- return nil, errors.Trace(err)
- }
- }
- idxPart.Expr = nil
- hiddenCols = append(hiddenCols, colInfo)
+ SchemaName: schema.Name.L,
+ TableName: t.Meta().Name.L,
+ Type: model.ActionAddPrimaryKey,
+ BinlogInfo: &model.HistoryInfo{},
+ ReorgMeta: nil,
+ Args: []any{unique, indexName, indexPartSpecifications, indexOption, sqlMode, nil, global},
+ Priority: ctx.GetSessionVars().DDLReorgPriority,
+ CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
+ SQLMode: ctx.GetSessionVars().SQLMode,
+ }
+ reorgMeta, err := newReorgMetaFromVariables(job, ctx)
+ if err != nil {
+ return err
}
- return hiddenCols, nil
+ job.ReorgMeta = reorgMeta
+
+ err = e.DoDDLJob(ctx, job)
+ return errors.Trace(err)
}
func (e *executor) CreateIndex(ctx sessionctx.Context, stmt *ast.CreateIndexStmt) error {
@@ -7891,7 +4691,19 @@ func (e *executor) createIndex(ctx sessionctx.Context, ti ast.Ident, keyType ast
return errors.Trace(err)
}
- global := false
+ globalIndex := false
+ if indexOption != nil && indexOption.Global {
+ globalIndex = true
+ }
+ if globalIndex {
+ if tblInfo.GetPartitionInfo() == nil {
+ return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("Global Index on non-partitioned table")
+ }
+ if !unique {
+ // TODO: remove this limitation
+ return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("Global IndexOption on non-unique index")
+ }
+ }
if unique && tblInfo.GetPartitionInfo() != nil {
ck, err := checkPartitionKeysConstraint(tblInfo.GetPartitionInfo(), indexColumns, tblInfo)
if err != nil {
@@ -7902,7 +4714,12 @@ func (e *executor) createIndex(ctx sessionctx.Context, ti ast.Ident, keyType ast
return dbterror.ErrUniqueKeyNeedAllFieldsInPf.GenWithStackByArgs("UNIQUE INDEX")
}
// index columns does not contain all partition columns, must set global
- global = true
+ if !globalIndex {
+ return dbterror.ErrGlobalIndexNotExplicitlySet.GenWithStackByArgs(indexName.O)
+ }
+ } else if globalIndex {
+ // TODO: remove this restriction
+ return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("Global IndexOption on index including all columns in the partitioning expression")
}
}
// May be truncate comment here, when index comment too long and sql_mode is't strict.
@@ -7914,7 +4731,7 @@ func (e *executor) createIndex(ctx sessionctx.Context, ti ast.Ident, keyType ast
}
if indexOption != nil && indexOption.Tp == model.IndexTypeHypo { // for hypo-index
- indexInfo, err := BuildIndexInfo(ctx, tblInfo.Columns, indexName, false, unique, global,
+ indexInfo, err := BuildIndexInfo(ctx, tblInfo.Columns, indexName, false, unique,
indexPartSpecifications, indexOption, model.StatePublic)
if err != nil {
return err
@@ -7923,6 +4740,9 @@ func (e *executor) createIndex(ctx sessionctx.Context, ti ast.Ident, keyType ast
}
chs, coll := ctx.GetSessionVars().GetCharsetInfo()
+ // global is set to 'false' is just there to be backwards compatible,
+ // to avoid unmarshal issues, it is now part of indexOption.
+ global := false
job := &model.Job{
SchemaID: schema.ID,
TableID: t.Meta().ID,
@@ -7958,6 +4778,12 @@ func newReorgMetaFromVariables(job *model.Job, sctx sessionctx.Context) (*model.
reorgMeta.IsDistReorg = variable.EnableDistTask.Load()
reorgMeta.IsFastReorg = variable.EnableFastReorg.Load()
reorgMeta.TargetScope = variable.ServiceScope.Load()
+ if sv, ok := sctx.GetSessionVars().GetSystemVar(variable.TiDBDDLReorgWorkerCount); ok {
+ reorgMeta.Concurrency = variable.TidbOptInt(sv, 0)
+ }
+ if sv, ok := sctx.GetSessionVars().GetSystemVar(variable.TiDBDDLReorgBatchSize); ok {
+ reorgMeta.BatchSize = variable.TidbOptInt(sv, 0)
+ }
if reorgMeta.IsDistReorg && !reorgMeta.IsFastReorg {
return nil, dbterror.ErrUnsupportedDistTask
@@ -7973,6 +4799,17 @@ func newReorgMetaFromVariables(job *model.Job, sctx sessionctx.Context) (*model.
LastReorgMetaFastReorgDisabled = true
})
}
+
+ logutil.DDLLogger().Info("initialize reorg meta",
+ zap.String("jobSchema", job.SchemaName),
+ zap.String("jobTable", job.TableName),
+ zap.Stringer("jobType", job.Type),
+ zap.Bool("enableDistTask", reorgMeta.IsDistReorg),
+ zap.Bool("enableFastReorg", reorgMeta.IsFastReorg),
+ zap.String("targetScope", reorgMeta.TargetScope),
+ zap.Int("concurrency", reorgMeta.Concurrency),
+ zap.Int("batchSize", reorgMeta.BatchSize),
+ )
return reorgMeta, nil
}
@@ -8294,30 +5131,6 @@ func CheckIsDropPrimaryKey(indexName model.CIStr, indexInfo *model.IndexInfo, t
return isPK, nil
}
-func isDroppableColumn(tblInfo *model.TableInfo, colName model.CIStr) error {
- if ok, dep, isHidden := hasDependentByGeneratedColumn(tblInfo, colName); ok {
- if isHidden {
- return dbterror.ErrDependentByFunctionalIndex.GenWithStackByArgs(dep)
- }
- return dbterror.ErrDependentByGeneratedColumn.GenWithStackByArgs(dep)
- }
-
- if len(tblInfo.Columns) == 1 {
- return dbterror.ErrCantRemoveAllFields.GenWithStack("can't drop only column %s in table %s",
- colName, tblInfo.Name)
- }
- // We only support dropping column with single-value none Primary Key index covered now.
- err := isColumnCanDropWithIndex(colName.L, tblInfo.Indices)
- if err != nil {
- return err
- }
- err = IsColumnDroppableWithCheckConstraint(colName, tblInfo)
- if err != nil {
- return err
- }
- return nil
-}
-
// validateCommentLength checks comment length of table, column, or index
// If comment length is more than the standard length truncate it
// and store the comment length upto the standard comment length size.
@@ -8433,66 +5246,6 @@ func buildAddedPartitionDefs(ctx expression.BuildContext, meta *model.TableInfo,
return GeneratePartDefsFromInterval(ctx, spec.Tp, meta, spec.Partition)
}
-func checkAndGetColumnsTypeAndValuesMatch(ctx expression.BuildContext, colTypes []types.FieldType, exprs []ast.ExprNode) ([]types.Datum, error) {
- // Validate() has already checked len(colNames) = len(exprs)
- // create table ... partition by range columns (cols)
- // partition p0 values less than (expr)
- // check the type of cols[i] and expr is consistent.
- valDatums := make([]types.Datum, 0, len(colTypes))
- for i, colExpr := range exprs {
- if _, ok := colExpr.(*ast.MaxValueExpr); ok {
- valDatums = append(valDatums, types.NewStringDatum(partitionMaxValue))
- continue
- }
- if d, ok := colExpr.(*ast.DefaultExpr); ok {
- if d.Name != nil {
- return nil, dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
- }
- continue
- }
- colType := colTypes[i]
- val, err := expression.EvalSimpleAst(ctx, colExpr)
- if err != nil {
- return nil, err
- }
- // Check val.ConvertTo(colType) doesn't work, so we need this case by case check.
- vkind := val.Kind()
- switch colType.GetType() {
- case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeDuration:
- switch vkind {
- case types.KindString, types.KindBytes, types.KindNull:
- default:
- return nil, dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
- }
- case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong:
- switch vkind {
- case types.KindInt64, types.KindUint64, types.KindNull:
- default:
- return nil, dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
- }
- case mysql.TypeFloat, mysql.TypeDouble:
- switch vkind {
- case types.KindFloat32, types.KindFloat64, types.KindNull:
- default:
- return nil, dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
- }
- case mysql.TypeString, mysql.TypeVarString:
- switch vkind {
- case types.KindString, types.KindBytes, types.KindNull, types.KindBinaryLiteral:
- default:
- return nil, dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
- }
- }
- evalCtx := ctx.GetEvalCtx()
- newVal, err := val.ConvertTo(evalCtx.TypeCtx(), &colType)
- if err != nil {
- return nil, dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
- }
- valDatums = append(valDatums, newVal)
- }
- return valDatums, nil
-}
-
// LockTables uses to execute lock tables statement.
func (e *executor) LockTables(ctx sessionctx.Context, stmt *ast.LockTablesStmt) error {
lockTables := make([]model.TableLockTpInfo, 0, len(stmt.TableLocks))
@@ -8578,7 +5331,7 @@ func (e *executor) UnlockTables(ctx sessionctx.Context, unlockTables []model.Tab
if !ok {
continue
}
- tbl, ok := is.TableByID(t.TableID)
+ tbl, ok := is.TableByID(e.ctx, t.TableID)
if !ok {
continue
}
@@ -8605,32 +5358,6 @@ func (e *executor) UnlockTables(ctx sessionctx.Context, unlockTables []model.Tab
return errors.Trace(err)
}
-// CleanDeadTableLock uses to clean dead table locks.
-func (d *ddl) CleanDeadTableLock(unlockTables []model.TableLockTpInfo, se model.SessionInfo) error {
- if len(unlockTables) == 0 {
- return nil
- }
- arg := &LockTablesArg{
- UnlockTables: unlockTables,
- SessionInfo: se,
- }
- job := &model.Job{
- SchemaID: unlockTables[0].SchemaID,
- TableID: unlockTables[0].TableID,
- Type: model.ActionUnlockTable,
- BinlogInfo: &model.HistoryInfo{},
- Args: []any{arg},
- }
-
- ctx, err := d.sessPool.Get()
- if err != nil {
- return err
- }
- defer d.sessPool.Put(ctx)
- err = d.executor.DoDDLJob(ctx, job)
- return errors.Trace(err)
-}
-
func throwErrIfInMemOrSysDB(ctx sessionctx.Context, dbLowerName string) error {
if util.IsMemOrSysDB(dbLowerName) {
if ctx.GetSessionVars().User != nil {
@@ -9066,125 +5793,6 @@ func (e *executor) AlterTablePartitionPlacement(ctx sessionctx.Context, tableIde
return errors.Trace(err)
}
-func buildPolicyInfo(name model.CIStr, options []*ast.PlacementOption) (*model.PolicyInfo, error) {
- policyInfo := &model.PolicyInfo{PlacementSettings: &model.PlacementSettings{}}
- policyInfo.Name = name
- for _, opt := range options {
- err := SetDirectPlacementOpt(policyInfo.PlacementSettings, opt.Tp, opt.StrValue, opt.UintValue)
- if err != nil {
- return nil, err
- }
- }
- return policyInfo, nil
-}
-
-func removeTablePlacement(tbInfo *model.TableInfo) bool {
- hasPlacementSettings := false
- if tbInfo.PlacementPolicyRef != nil {
- tbInfo.PlacementPolicyRef = nil
- hasPlacementSettings = true
- }
-
- if removePartitionPlacement(tbInfo.Partition) {
- hasPlacementSettings = true
- }
-
- return hasPlacementSettings
-}
-
-func removePartitionPlacement(partInfo *model.PartitionInfo) bool {
- if partInfo == nil {
- return false
- }
-
- hasPlacementSettings := false
- for i := range partInfo.Definitions {
- def := &partInfo.Definitions[i]
- if def.PlacementPolicyRef != nil {
- def.PlacementPolicyRef = nil
- hasPlacementSettings = true
- }
- }
- return hasPlacementSettings
-}
-
-func handleDatabasePlacement(ctx sessionctx.Context, dbInfo *model.DBInfo) error {
- if dbInfo.PlacementPolicyRef == nil {
- return nil
- }
-
- sessVars := ctx.GetSessionVars()
- if sessVars.PlacementMode == variable.PlacementModeIgnore {
- dbInfo.PlacementPolicyRef = nil
- sessVars.StmtCtx.AppendNote(
- errors.NewNoStackErrorf("Placement is ignored when TIDB_PLACEMENT_MODE is '%s'", variable.PlacementModeIgnore),
- )
- return nil
- }
-
- var err error
- dbInfo.PlacementPolicyRef, err = checkAndNormalizePlacementPolicy(ctx, dbInfo.PlacementPolicyRef)
- return err
-}
-
-func handleTablePlacement(ctx sessionctx.Context, tbInfo *model.TableInfo) error {
- sessVars := ctx.GetSessionVars()
- if sessVars.PlacementMode == variable.PlacementModeIgnore && removeTablePlacement(tbInfo) {
- sessVars.StmtCtx.AppendNote(
- errors.NewNoStackErrorf("Placement is ignored when TIDB_PLACEMENT_MODE is '%s'", variable.PlacementModeIgnore),
- )
- return nil
- }
-
- var err error
- tbInfo.PlacementPolicyRef, err = checkAndNormalizePlacementPolicy(ctx, tbInfo.PlacementPolicyRef)
- if err != nil {
- return err
- }
-
- if tbInfo.Partition != nil {
- for i := range tbInfo.Partition.Definitions {
- partition := &tbInfo.Partition.Definitions[i]
- partition.PlacementPolicyRef, err = checkAndNormalizePlacementPolicy(ctx, partition.PlacementPolicyRef)
- if err != nil {
- return err
- }
- }
- }
- return nil
-}
-
-func handlePartitionPlacement(ctx sessionctx.Context, partInfo *model.PartitionInfo) error {
- sessVars := ctx.GetSessionVars()
- if sessVars.PlacementMode == variable.PlacementModeIgnore && removePartitionPlacement(partInfo) {
- sessVars.StmtCtx.AppendNote(
- errors.NewNoStackErrorf("Placement is ignored when TIDB_PLACEMENT_MODE is '%s'", variable.PlacementModeIgnore),
- )
- return nil
- }
-
- var err error
- for i := range partInfo.Definitions {
- partition := &partInfo.Definitions[i]
- partition.PlacementPolicyRef, err = checkAndNormalizePlacementPolicy(ctx, partition.PlacementPolicyRef)
- if err != nil {
- return err
- }
- }
- return nil
-}
-
-func checkIgnorePlacementDDL(ctx sessionctx.Context) bool {
- sessVars := ctx.GetSessionVars()
- if sessVars.PlacementMode == variable.PlacementModeIgnore {
- sessVars.StmtCtx.AppendNote(
- errors.NewNoStackErrorf("Placement is ignored when TIDB_PLACEMENT_MODE is '%s'", variable.PlacementModeIgnore),
- )
- return true
- }
- return false
-}
-
// AddResourceGroup implements the DDL interface, creates a resource group.
func (e *executor) AddResourceGroup(ctx sessionctx.Context, stmt *ast.CreateResourceGroupStmt) (err error) {
groupName := stmt.ResourceGroupName
@@ -9203,7 +5811,7 @@ func (e *executor) AddResourceGroup(ctx sessionctx.Context, stmt *ast.CreateReso
return infoschema.ErrResourceGroupExists.GenWithStackByArgs(groupName)
}
- if err := e.checkResourceGroupValidation(groupInfo); err != nil {
+ if err := checkResourceGroupValidation(groupInfo); err != nil {
return err
}
@@ -9229,11 +5837,6 @@ func (e *executor) AddResourceGroup(ctx sessionctx.Context, stmt *ast.CreateReso
return err
}
-func (*executor) checkResourceGroupValidation(groupInfo *model.ResourceGroupInfo) error {
- _, err := resourcegroup.NewGroupFromOptions(groupInfo.Name.L, groupInfo.ResourceGroupSettings)
- return err
-}
-
// DropResourceGroup implements the DDL interface.
func (e *executor) DropResourceGroup(ctx sessionctx.Context, stmt *ast.DropResourceGroupStmt) (err error) {
groupName := stmt.ResourceGroupName
@@ -9279,21 +5882,6 @@ func (e *executor) DropResourceGroup(ctx sessionctx.Context, stmt *ast.DropResou
return err
}
-func buildResourceGroup(oldGroup *model.ResourceGroupInfo, options []*ast.ResourceGroupOption) (*model.ResourceGroupInfo, error) {
- groupInfo := &model.ResourceGroupInfo{Name: oldGroup.Name, ID: oldGroup.ID, ResourceGroupSettings: model.NewResourceGroupSettings()}
- if oldGroup.ResourceGroupSettings != nil {
- *groupInfo.ResourceGroupSettings = *oldGroup.ResourceGroupSettings
- }
- for _, opt := range options {
- err := SetDirectResourceGroupSettings(groupInfo, opt)
- if err != nil {
- return nil, err
- }
- }
- groupInfo.ResourceGroupSettings.Adjust()
- return groupInfo, nil
-}
-
// AlterResourceGroup implements the DDL interface.
func (e *executor) AlterResourceGroup(ctx sessionctx.Context, stmt *ast.AlterResourceGroupStmt) (err error) {
groupName := stmt.ResourceGroupName
@@ -9313,7 +5901,7 @@ func (e *executor) AlterResourceGroup(ctx sessionctx.Context, stmt *ast.AlterRes
return errors.Trace(err)
}
- if err := e.checkResourceGroupValidation(newGroupInfo); err != nil {
+ if err := checkResourceGroupValidation(newGroupInfo); err != nil {
return err
}
@@ -9554,26 +6142,6 @@ func (e *executor) AlterTableNoCache(ctx sessionctx.Context, ti ast.Ident) (err
return e.DoDDLJob(ctx, job)
}
-// checkTooBigFieldLengthAndTryAutoConvert will check whether the field length is too big
-// in non-strict mode and varchar column. If it is, will try to adjust to blob or text, see issue #30328
-func checkTooBigFieldLengthAndTryAutoConvert(tp *types.FieldType, colName string, sessVars *variable.SessionVars) error {
- if sessVars != nil && !sessVars.SQLMode.HasStrictMode() && tp.GetType() == mysql.TypeVarchar {
- err := types.IsVarcharTooBigFieldLength(tp.GetFlen(), colName, tp.GetCharset())
- if err != nil && terror.ErrorEqual(types.ErrTooBigFieldLength, err) {
- tp.SetType(mysql.TypeBlob)
- if err = adjustBlobTypesFlen(tp, tp.GetCharset()); err != nil {
- return err
- }
- if tp.GetCharset() == charset.CharsetBin {
- sessVars.StmtCtx.AppendWarning(dbterror.ErrAutoConvert.FastGenByArgs(colName, "VARBINARY", "BLOB"))
- } else {
- sessVars.StmtCtx.AppendWarning(dbterror.ErrAutoConvert.FastGenByArgs(colName, "VARCHAR", "TEXT"))
- }
- }
- }
- return nil
-}
-
func (e *executor) CreateCheckConstraint(ctx sessionctx.Context, ti ast.Ident, constrName model.CIStr, constr *ast.Constraint) error {
schema, t, err := e.getSchemaAndTableByIdent(ti)
if err != nil {
@@ -9713,6 +6281,291 @@ func (e *executor) AlterCheckConstraint(ctx sessionctx.Context, ti ast.Ident, co
return errors.Trace(err)
}
+func (e *executor) genGlobalIDs(count int) ([]int64, error) {
+ var ret []int64
+ ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL)
+ // lock to reduce conflict
+ e.globalIDLock.Lock()
+ defer e.globalIDLock.Unlock()
+ err := kv.RunInNewTxn(ctx, e.store, true, func(_ context.Context, txn kv.Transaction) error {
+ m := meta.NewMeta(txn)
+ var err error
+ ret, err = m.GenGlobalIDs(count)
+ return err
+ })
+
+ return ret, err
+}
+
+func (e *executor) genPlacementPolicyID() (int64, error) {
+ var ret int64
+ ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL)
+ err := kv.RunInNewTxn(ctx, e.store, true, func(_ context.Context, txn kv.Transaction) error {
+ m := meta.NewMeta(txn)
+ var err error
+ ret, err = m.GenPlacementPolicyID()
+ return err
+ })
+
+ return ret, err
+}
+
+// DoDDLJob will return
+// - nil: found in history DDL job and no job error
+// - context.Cancel: job has been sent to worker, but not found in history DDL job before cancel
+// - other: found in history DDL job and return that job error
+func (e *executor) DoDDLJob(ctx sessionctx.Context, job *model.Job) error {
+ return e.DoDDLJobWrapper(ctx, NewJobWrapper(job, false))
+}
+
+// DoDDLJobWrapper submit DDL job and wait it finishes.
+// When fast create is enabled, we might merge multiple jobs into one, so do not
+// depend on job.ID, use JobID from jobSubmitResult.
+func (e *executor) DoDDLJobWrapper(ctx sessionctx.Context, jobW *JobWrapper) error {
+ job := jobW.Job
+ job.TraceInfo = &model.TraceInfo{
+ ConnectionID: ctx.GetSessionVars().ConnectionID,
+ SessionAlias: ctx.GetSessionVars().SessionAlias,
+ }
+ if mci := ctx.GetSessionVars().StmtCtx.MultiSchemaInfo; mci != nil {
+ // In multiple schema change, we don't run the job.
+ // Instead, we merge all the jobs into one pending job.
+ return appendToSubJobs(mci, job)
+ }
+ // Get a global job ID and put the DDL job in the queue.
+ setDDLJobQuery(ctx, job)
+ e.deliverJobTask(jobW)
+
+ failpoint.Inject("mockParallelSameDDLJobTwice", func(val failpoint.Value) {
+ if val.(bool) {
+ <-jobW.ResultCh[0]
+ // The same job will be put to the DDL queue twice.
+ job = job.Clone()
+ newJobW := NewJobWrapper(job, jobW.IDAllocated)
+ e.deliverJobTask(newJobW)
+ // The second job result is used for test.
+ jobW = newJobW
+ }
+ })
+
+ // worker should restart to continue handling tasks in limitJobCh, and send back through jobW.err
+ result := <-jobW.ResultCh[0]
+ // job.ID must be allocated after previous channel receive returns nil.
+ jobID, err := result.jobID, result.err
+ defer e.delJobDoneCh(jobID)
+ if err != nil {
+ // The transaction of enqueuing job is failed.
+ return errors.Trace(err)
+ }
+ failpoint.InjectCall("waitJobSubmitted")
+
+ sessVars := ctx.GetSessionVars()
+ sessVars.StmtCtx.IsDDLJobInQueue = true
+
+ ddlAction := job.Type
+ // Notice worker that we push a new job and wait the job done.
+ e.notifyNewJobSubmitted(e.ddlJobNotifyCh, addingDDLJobNotifyKey, jobID, ddlAction.String())
+ if result.merged {
+ logutil.DDLLogger().Info("DDL job submitted", zap.Int64("job_id", jobID), zap.String("query", job.Query), zap.String("merged", "true"))
+ } else {
+ logutil.DDLLogger().Info("DDL job submitted", zap.Stringer("job", job), zap.String("query", job.Query))
+ }
+
+ var historyJob *model.Job
+
+ // Attach the context of the jobId to the calling session so that
+ // KILL can cancel this DDL job.
+ ctx.GetSessionVars().StmtCtx.DDLJobID = jobID
+
+ // For a job from start to end, the state of it will be none -> delete only -> write only -> reorganization -> public
+ // For every state changes, we will wait as lease 2 * lease time, so here the ticker check is 10 * lease.
+ // But we use etcd to speed up, normally it takes less than 0.5s now, so we use 0.5s or 1s or 3s as the max value.
+ initInterval, _ := getJobCheckInterval(ddlAction, 0)
+ ticker := time.NewTicker(chooseLeaseTime(10*e.lease, initInterval))
+ startTime := time.Now()
+ metrics.JobsGauge.WithLabelValues(ddlAction.String()).Inc()
+ defer func() {
+ ticker.Stop()
+ metrics.JobsGauge.WithLabelValues(ddlAction.String()).Dec()
+ metrics.HandleJobHistogram.WithLabelValues(ddlAction.String(), metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
+ recordLastDDLInfo(ctx, historyJob)
+ }()
+ i := 0
+ notifyCh, _ := e.getJobDoneCh(jobID)
+ for {
+ failpoint.InjectCall("storeCloseInLoop")
+ select {
+ case _, ok := <-notifyCh:
+ if !ok {
+ // when fast create enabled, jobs might be merged, and we broadcast
+ // the result by closing the channel, to avoid this loop keeps running
+ // without sleeping on retryable error, we set it to nil.
+ notifyCh = nil
+ }
+ case <-ticker.C:
+ i++
+ ticker = updateTickerInterval(ticker, 10*e.lease, ddlAction, i)
+ case <-e.ctx.Done():
+ logutil.DDLLogger().Info("DoDDLJob will quit because context done")
+ return context.Canceled
+ }
+
+ // If the connection being killed, we need to CANCEL the DDL job.
+ if sessVars.SQLKiller.HandleSignal() == exeerrors.ErrQueryInterrupted {
+ if atomic.LoadInt32(&sessVars.ConnectionStatus) == variable.ConnStatusShutdown {
+ logutil.DDLLogger().Info("DoDDLJob will quit because context done")
+ return context.Canceled
+ }
+ if sessVars.StmtCtx.DDLJobID != 0 {
+ se, err := e.sessPool.Get()
+ if err != nil {
+ logutil.DDLLogger().Error("get session failed, check again", zap.Error(err))
+ continue
+ }
+ sessVars.StmtCtx.DDLJobID = 0 // Avoid repeat.
+ errs, err := CancelJobsBySystem(se, []int64{jobID})
+ e.sessPool.Put(se)
+ if len(errs) > 0 {
+ logutil.DDLLogger().Warn("error canceling DDL job", zap.Error(errs[0]))
+ }
+ if err != nil {
+ logutil.DDLLogger().Warn("Kill command could not cancel DDL job", zap.Error(err))
+ continue
+ }
+ }
+ }
+
+ se, err := e.sessPool.Get()
+ if err != nil {
+ logutil.DDLLogger().Error("get session failed, check again", zap.Error(err))
+ continue
+ }
+ historyJob, err = GetHistoryJobByID(se, jobID)
+ e.sessPool.Put(se)
+ if err != nil {
+ logutil.DDLLogger().Error("get history DDL job failed, check again", zap.Error(err))
+ continue
+ }
+ if historyJob == nil {
+ logutil.DDLLogger().Debug("DDL job is not in history, maybe not run", zap.Int64("jobID", jobID))
+ continue
+ }
+
+ e.checkHistoryJobInTest(ctx, historyJob)
+
+ // If a job is a history job, the state must be JobStateSynced or JobStateRollbackDone or JobStateCancelled.
+ if historyJob.IsSynced() {
+ // Judge whether there are some warnings when executing DDL under the certain SQL mode.
+ if historyJob.ReorgMeta != nil && len(historyJob.ReorgMeta.Warnings) != 0 {
+ if len(historyJob.ReorgMeta.Warnings) != len(historyJob.ReorgMeta.WarningsCount) {
+ logutil.DDLLogger().Info("DDL warnings doesn't match the warnings count", zap.Int64("jobID", jobID))
+ } else {
+ for key, warning := range historyJob.ReorgMeta.Warnings {
+ keyCount := historyJob.ReorgMeta.WarningsCount[key]
+ if keyCount == 1 {
+ ctx.GetSessionVars().StmtCtx.AppendWarning(warning)
+ } else {
+ newMsg := fmt.Sprintf("%d warnings with this error code, first warning: "+warning.GetMsg(), keyCount)
+ newWarning := dbterror.ClassTypes.Synthesize(terror.ErrCode(warning.Code()), newMsg)
+ ctx.GetSessionVars().StmtCtx.AppendWarning(newWarning)
+ }
+ }
+ }
+ }
+ appendMultiChangeWarningsToOwnerCtx(ctx, historyJob)
+
+ logutil.DDLLogger().Info("DDL job is finished", zap.Int64("jobID", jobID))
+ return nil
+ }
+
+ if historyJob.Error != nil {
+ logutil.DDLLogger().Info("DDL job is failed", zap.Int64("jobID", jobID))
+ return errors.Trace(historyJob.Error)
+ }
+ panic("When the state is JobStateRollbackDone or JobStateCancelled, historyJob.Error should never be nil")
+ }
+}
+
+func (e *executor) getJobDoneCh(jobID int64) (chan struct{}, bool) {
+ return e.ddlJobDoneChMap.Load(jobID)
+}
+
+func (e *executor) delJobDoneCh(jobID int64) {
+ e.ddlJobDoneChMap.Delete(jobID)
+}
+
+func (e *executor) deliverJobTask(task *JobWrapper) {
+ // TODO this might block forever, as the consumer part considers context cancel.
+ e.limitJobCh <- task
+}
+
+func updateTickerInterval(ticker *time.Ticker, lease time.Duration, action model.ActionType, i int) *time.Ticker {
+ interval, changed := getJobCheckInterval(action, i)
+ if !changed {
+ return ticker
+ }
+ // For now we should stop old ticker and create a new ticker
+ ticker.Stop()
+ return time.NewTicker(chooseLeaseTime(lease, interval))
+}
+
+func recordLastDDLInfo(ctx sessionctx.Context, job *model.Job) {
+ if job == nil {
+ return
+ }
+ ctx.GetSessionVars().LastDDLInfo.Query = job.Query
+ ctx.GetSessionVars().LastDDLInfo.SeqNum = job.SeqNum
+}
+
+func setDDLJobQuery(ctx sessionctx.Context, job *model.Job) {
+ switch job.Type {
+ case model.ActionUpdateTiFlashReplicaStatus, model.ActionUnlockTable:
+ job.Query = ""
+ default:
+ job.Query, _ = ctx.Value(sessionctx.QueryString).(string)
+ }
+}
+
+var (
+ fastDDLIntervalPolicy = []time.Duration{
+ 500 * time.Millisecond,
+ }
+ normalDDLIntervalPolicy = []time.Duration{
+ 500 * time.Millisecond,
+ 500 * time.Millisecond,
+ 1 * time.Second,
+ }
+ slowDDLIntervalPolicy = []time.Duration{
+ 500 * time.Millisecond,
+ 500 * time.Millisecond,
+ 1 * time.Second,
+ 1 * time.Second,
+ 3 * time.Second,
+ }
+)
+
+func getIntervalFromPolicy(policy []time.Duration, i int) (time.Duration, bool) {
+ plen := len(policy)
+ if i < plen {
+ return policy[i], true
+ }
+ return policy[plen-1], false
+}
+
+func getJobCheckInterval(action model.ActionType, i int) (time.Duration, bool) {
+ switch action {
+ case model.ActionAddIndex, model.ActionAddPrimaryKey, model.ActionModifyColumn,
+ model.ActionReorganizePartition,
+ model.ActionRemovePartitioning,
+ model.ActionAlterTablePartitioning:
+ return getIntervalFromPolicy(slowDDLIntervalPolicy, i)
+ case model.ActionCreateTable, model.ActionCreateSchema:
+ return getIntervalFromPolicy(fastDDLIntervalPolicy, i)
+ default:
+ return getIntervalFromPolicy(normalDDLIntervalPolicy, i)
+ }
+}
+
// NewDDLReorgMeta create a DDL ReorgMeta.
func NewDDLReorgMeta(ctx sessionctx.Context) *model.DDLReorgMeta {
tzName, tzOffset := ddlutil.GetTimeZone(ctx)
diff --git a/pkg/ddl/executor_nokit_test.go b/pkg/ddl/executor_nokit_test.go
new file mode 100644
index 0000000000000..e683e286f0665
--- /dev/null
+++ b/pkg/ddl/executor_nokit_test.go
@@ -0,0 +1,195 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ddl
+
+import (
+ "fmt"
+ "slices"
+ "testing"
+
+ "github.com/pingcap/tidb/pkg/parser/model"
+ "github.com/stretchr/testify/require"
+)
+
+func TestBuildQueryStringFromJobs(t *testing.T) {
+ testCases := []struct {
+ name string
+ jobs []*JobWrapper
+ expected string
+ }{
+ {
+ name: "Empty jobs",
+ jobs: []*JobWrapper{},
+ expected: "",
+ },
+ {
+ name: "Single create table job",
+ jobs: []*JobWrapper{{Job: &model.Job{Query: "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255));"}}},
+ expected: "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255));",
+ },
+ {
+ name: "Multiple create table jobs with trailing semicolons",
+ jobs: []*JobWrapper{
+ {Job: &model.Job{Query: "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255));"}},
+ {Job: &model.Job{Query: "CREATE TABLE products (id INT PRIMARY KEY, description TEXT);"}},
+ },
+ expected: "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255)); CREATE TABLE products (id INT PRIMARY KEY, description TEXT);",
+ },
+ {
+ name: "Multiple create table jobs with and without trailing semicolons",
+ jobs: []*JobWrapper{
+ {Job: &model.Job{Query: "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255))"}},
+ {Job: &model.Job{Query: "CREATE TABLE products (id INT PRIMARY KEY, description TEXT);"}},
+ {Job: &model.Job{Query: " CREATE TABLE orders (id INT PRIMARY KEY, user_id INT, product_id INT) "}},
+ },
+ expected: "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255)); CREATE TABLE products (id INT PRIMARY KEY, description TEXT); CREATE TABLE orders (id INT PRIMARY KEY, user_id INT, product_id INT);",
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ actual := buildQueryStringFromJobs(tc.jobs)
+ require.Equal(t, tc.expected, actual, "Query strings do not match")
+ })
+ }
+}
+
+func TestMergeCreateTableJobsOfSameSchema(t *testing.T) {
+ job1 := NewJobWrapper(&model.Job{
+ SchemaID: 1,
+ Type: model.ActionCreateTable,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []any{&model.TableInfo{Name: model.CIStr{O: "t1", L: "t1"}}, false},
+ Query: "create table db1.t1 (c1 int, c2 int)",
+ }, false)
+ job2 := NewJobWrapper(&model.Job{
+ SchemaID: 1,
+ Type: model.ActionCreateTable,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []any{&model.TableInfo{Name: model.CIStr{O: "t2", L: "t2"}}, &model.TableInfo{}},
+ Query: "create table db1.t2 (c1 int, c2 int);",
+ }, false)
+ job, err := mergeCreateTableJobsOfSameSchema([]*JobWrapper{job1, job2})
+ require.NoError(t, err)
+ require.Equal(t, "create table db1.t1 (c1 int, c2 int); create table db1.t2 (c1 int, c2 int);", job.Query)
+}
+
+func TestMergeCreateTableJobs(t *testing.T) {
+ t.Run("0 or 1 job", func(t *testing.T) {
+ newWs, err := mergeCreateTableJobs([]*JobWrapper{})
+ require.NoError(t, err)
+ require.Empty(t, newWs)
+ jobWs := []*JobWrapper{{Job: &model.Job{}}}
+ newWs, err = mergeCreateTableJobs(jobWs)
+ require.NoError(t, err)
+ require.EqualValues(t, jobWs, newWs)
+ })
+
+ t.Run("non create table are not merged", func(t *testing.T) {
+ jobWs := []*JobWrapper{
+ {Job: &model.Job{SchemaName: "db", Type: model.ActionCreateTable,
+ Args: []any{&model.TableInfo{Name: model.NewCIStr("t1")}, false}}},
+ {Job: &model.Job{SchemaName: "db", Type: model.ActionAddColumn}},
+ {Job: &model.Job{SchemaName: "db", Type: model.ActionCreateTable,
+ Args: []any{&model.TableInfo{Name: model.NewCIStr("t2")}, false}}},
+ }
+ newWs, err := mergeCreateTableJobs(jobWs)
+ require.NoError(t, err)
+ require.Len(t, newWs, 2)
+ require.Equal(t, model.ActionAddColumn, newWs[0].Type)
+ require.Equal(t, model.ActionCreateTables, newWs[1].Type)
+ })
+
+ t.Run("jobs of pre allocated ids are not merged", func(t *testing.T) {
+ jobWs := []*JobWrapper{
+ {Job: &model.Job{SchemaName: "db", Type: model.ActionCreateTable,
+ Args: []any{&model.TableInfo{Name: model.NewCIStr("t1")}, false}}, IDAllocated: true},
+ {Job: &model.Job{SchemaName: "db", Type: model.ActionCreateTable,
+ Args: []any{&model.TableInfo{Name: model.NewCIStr("t2")}, false}}},
+ }
+ newWs, err := mergeCreateTableJobs(jobWs)
+ require.NoError(t, err)
+ require.EqualValues(t, jobWs, newWs)
+ })
+
+ t.Run("jobs of foreign keys are not merged", func(t *testing.T) {
+ jobWs := []*JobWrapper{
+ {Job: &model.Job{SchemaName: "db", Type: model.ActionCreateTable,
+ Args: []any{&model.TableInfo{ForeignKeys: []*model.FKInfo{{}}}, false}}},
+ {Job: &model.Job{SchemaName: "db", Type: model.ActionCreateTable,
+ Args: []any{&model.TableInfo{Name: model.NewCIStr("t2")}, false}}},
+ }
+ newWs, err := mergeCreateTableJobs(jobWs)
+ require.NoError(t, err)
+ require.EqualValues(t, jobWs, newWs)
+ })
+
+ t.Run("jobs of different schema are not merged", func(t *testing.T) {
+ jobWs := []*JobWrapper{
+ {Job: &model.Job{SchemaName: "db1", Type: model.ActionCreateTable,
+ Args: []any{&model.TableInfo{Name: model.NewCIStr("t1")}, false}}},
+ {Job: &model.Job{SchemaName: "db2", Type: model.ActionCreateTable,
+ Args: []any{&model.TableInfo{Name: model.NewCIStr("t2")}, false}}},
+ }
+ newWs, err := mergeCreateTableJobs(jobWs)
+ require.NoError(t, err)
+ require.EqualValues(t, jobWs, newWs)
+ })
+
+ t.Run("max batch size 8", func(t *testing.T) {
+ jobWs := make([]*JobWrapper, 0, 100)
+ for db, cnt := range map[string]int{
+ "db0": 9,
+ "db1": 7,
+ "db2": 22,
+ } {
+ for i := 0; i < cnt; i++ {
+ tblName := fmt.Sprintf("t%d", i)
+ jobWs = append(jobWs, NewJobWrapper(&model.Job{SchemaName: db, Type: model.ActionCreateTable,
+ Args: []any{&model.TableInfo{Name: model.NewCIStr(tblName)}, false}}, false))
+ }
+ }
+ jobWs = append(jobWs, NewJobWrapper(&model.Job{SchemaName: "dbx", Type: model.ActionAddColumn}, false))
+ jobWs = append(jobWs, NewJobWrapper(&model.Job{SchemaName: "dbxx", Type: model.ActionCreateTable,
+ Args: []any{&model.TableInfo{Name: model.NewCIStr("t1")}, false}}, true))
+ jobWs = append(jobWs, NewJobWrapper(&model.Job{SchemaName: "dbxxx", Type: model.ActionCreateTable,
+ Args: []any{&model.TableInfo{ForeignKeys: []*model.FKInfo{{}}}, false}}, false))
+ newWs, err := mergeCreateTableJobs(jobWs)
+ require.NoError(t, err)
+ // 3 non-mergeable + 2 + 1 + 3
+ require.Len(t, newWs, 9)
+ require.Equal(t, model.ActionAddColumn, newWs[0].Type)
+ require.Equal(t, model.ActionCreateTable, newWs[1].Type)
+ require.Equal(t, "dbxx", newWs[1].SchemaName)
+ require.Equal(t, model.ActionCreateTable, newWs[2].Type)
+ require.Equal(t, "dbxxx", newWs[2].SchemaName)
+
+ schemaCnts := make(map[string][]int, 3)
+ for i := 3; i < 9; i++ {
+ require.Equal(t, model.ActionCreateTables, newWs[i].Type)
+ infos := newWs[i].Args[0].([]*model.TableInfo)
+ schemaCnts[newWs[i].SchemaName] = append(schemaCnts[newWs[i].SchemaName], len(infos))
+ require.Equal(t, len(infos), len(newWs[i].ResultCh))
+ }
+ for k := range schemaCnts {
+ slices.Sort(schemaCnts[k])
+ }
+ require.Equal(t, map[string][]int{
+ "db0": {4, 5},
+ "db1": {7},
+ "db2": {7, 7, 8},
+ }, schemaCnts)
+ })
+}
diff --git a/pkg/ddl/executor_test.go b/pkg/ddl/executor_test.go
index d9fa206ee4b30..66c3c74c568d6 100644
--- a/pkg/ddl/executor_test.go
+++ b/pkg/ddl/executor_test.go
@@ -29,7 +29,6 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/ddl"
sess "github.com/pingcap/tidb/pkg/ddl/session"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/meta"
@@ -73,7 +72,7 @@ func TestGetDDLJobs(t *testing.T) {
currJobs2 = currJobs2[:0]
err = ddl.IterAllDDLJobs(sess, txn, func(jobs []*model.Job) (b bool, e error) {
for _, job := range jobs {
- if !job.NotStarted() {
+ if job.Started() {
return true, nil
}
currJobs2 = append(currJobs2, job)
@@ -203,7 +202,7 @@ func TestCreateViewConcurrently(t *testing.T) {
}
func TestCreateDropCreateTable(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk1 := testkit.NewTestKit(t, store)
@@ -216,10 +215,9 @@ func TestCreateDropCreateTable(t *testing.T) {
var fpErr error
var createTable bool
- originHook := dom.DDL().GetHook()
- onJobUpdated := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.Type == model.ActionDropTable && job.SchemaState == model.StateWriteOnly && !createTable {
- fpErr = failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/mockOwnerCheckAllVersionSlow", fmt.Sprintf("return(%d)", job.ID))
+ fpErr = failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/schemaver/mockOwnerCheckAllVersionSlow", fmt.Sprintf("return(%d)", job.ID))
wg.Add(1)
go func() {
_, createErr = tk1.Exec("create table t (b int);")
@@ -227,17 +225,14 @@ func TestCreateDropCreateTable(t *testing.T) {
}()
createTable = true
}
- }
- hook := &callback.TestDDLCallback{}
- hook.OnJobUpdatedExported.Store(&onJobUpdated)
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("drop table t;")
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
wg.Wait()
require.NoError(t, createErr)
require.NoError(t, fpErr)
- require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/mockOwnerCheckAllVersionSlow"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/schemaver/mockOwnerCheckAllVersionSlow"))
rs := tk.MustQuery("admin show ddl jobs 3;").Rows()
create1JobID := rs[0][0].(string)
@@ -265,69 +260,6 @@ func TestCreateDropCreateTable(t *testing.T) {
require.Less(t, dropTS, create1TS, "second create should finish after drop")
}
-func TestBuildQueryStringFromJobs(t *testing.T) {
- testCases := []struct {
- name string
- jobs []*model.Job
- expected string
- }{
- {
- name: "Empty jobs",
- jobs: []*model.Job{},
- expected: "",
- },
- {
- name: "Single create table job",
- jobs: []*model.Job{{Query: "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255));"}},
- expected: "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255));",
- },
- {
- name: "Multiple create table jobs with trailing semicolons",
- jobs: []*model.Job{
- {Query: "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255));"},
- {Query: "CREATE TABLE products (id INT PRIMARY KEY, description TEXT);"},
- },
- expected: "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255)); CREATE TABLE products (id INT PRIMARY KEY, description TEXT);",
- },
- {
- name: "Multiple create table jobs with and without trailing semicolons",
- jobs: []*model.Job{
- {Query: "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255))"},
- {Query: "CREATE TABLE products (id INT PRIMARY KEY, description TEXT);"},
- {Query: " CREATE TABLE orders (id INT PRIMARY KEY, user_id INT, product_id INT) "},
- },
- expected: "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255)); CREATE TABLE products (id INT PRIMARY KEY, description TEXT); CREATE TABLE orders (id INT PRIMARY KEY, user_id INT, product_id INT);",
- },
- }
-
- for _, tc := range testCases {
- t.Run(tc.name, func(t *testing.T) {
- actual := ddl.BuildQueryStringFromJobs(tc.jobs)
- require.Equal(t, tc.expected, actual, "Query strings do not match")
- })
- }
-}
-
-func TestBatchCreateTableWithJobs(t *testing.T) {
- job1 := &model.Job{
- SchemaID: 1,
- Type: model.ActionCreateTable,
- BinlogInfo: &model.HistoryInfo{},
- Args: []any{&model.TableInfo{Name: model.CIStr{O: "t1", L: "t1"}}, false},
- Query: "create table db1.t1 (c1 int, c2 int)",
- }
- job2 := &model.Job{
- SchemaID: 1,
- Type: model.ActionCreateTable,
- BinlogInfo: &model.HistoryInfo{},
- Args: []any{&model.TableInfo{Name: model.CIStr{O: "t2", L: "t2"}}, &model.TableInfo{}},
- Query: "create table db1.t2 (c1 int, c2 int);",
- }
- job, err := ddl.BatchCreateTableWithJobs([]*model.Job{job1, job2})
- require.NoError(t, err)
- require.Equal(t, "create table db1.t1 (c1 int, c2 int); create table db1.t2 (c1 int, c2 int);", job.Query)
-}
-
func getGlobalID(ctx context.Context, t *testing.T, store kv.Storage) int64 {
res := int64(0)
require.NoError(t, kv.RunInNewTxn(ctx, store, true, func(_ context.Context, txn kv.Transaction) error {
@@ -365,10 +297,15 @@ func TestGenIDAndInsertJobsWithRetry(t *testing.T) {
}}
initialGID := getGlobalID(ctx, t, store)
threads, iterations := 10, 500
+ tks := make([]*testkit.TestKit, threads)
+ for i := 0; i < threads; i++ {
+ tks[i] = testkit.NewTestKit(t, store)
+ }
var wg util.WaitGroupWrapper
for i := 0; i < threads; i++ {
+ idx := i
wg.Run(func() {
- kit := testkit.NewTestKit(t, store)
+ kit := tks[idx]
ddlSe := sess.NewSession(kit.Session())
for j := 0; j < iterations; j++ {
require.NoError(t, ddl.GenGIDAndInsertJobsWithRetry(ctx, ddlSe, jobs))
diff --git a/pkg/ddl/export_test.go b/pkg/ddl/export_test.go
index d814000eff867..4aa98bb40fb0a 100644
--- a/pkg/ddl/export_test.go
+++ b/pkg/ddl/export_test.go
@@ -12,59 +12,59 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package ddl
+package ddl_test
import (
"context"
"time"
"github.com/ngaut/pools"
+ "github.com/pingcap/tidb/pkg/ddl"
"github.com/pingcap/tidb/pkg/ddl/copr"
"github.com/pingcap/tidb/pkg/ddl/session"
+ "github.com/pingcap/tidb/pkg/ddl/testutil"
+ "github.com/pingcap/tidb/pkg/disttask/operator"
"github.com/pingcap/tidb/pkg/errctx"
"github.com/pingcap/tidb/pkg/expression"
"github.com/pingcap/tidb/pkg/kv"
- "github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/chunk"
"github.com/pingcap/tidb/pkg/util/mock"
)
-type resultChanForTest struct {
- ch chan IndexRecordChunk
-}
-
-func (r *resultChanForTest) AddTask(rs IndexRecordChunk) {
- r.ch <- rs
-}
-
func FetchChunk4Test(copCtx copr.CopContext, tbl table.PhysicalTable, startKey, endKey kv.Key, store kv.Storage,
- batchSize int) *chunk.Chunk {
- variable.SetDDLReorgBatchSize(int32(batchSize))
- task := &reorgBackfillTask{
- id: 1,
- startKey: startKey,
- endKey: endKey,
- physicalTable: tbl,
- }
- taskCh := make(chan *reorgBackfillTask, 5)
- resultCh := make(chan IndexRecordChunk, 5)
+ batchSize int) (*chunk.Chunk, error) {
resPool := pools.NewResourcePool(func() (pools.Resource, error) {
ctx := mock.NewContext()
ctx.Store = store
return ctx, nil
}, 8, 8, 0)
sessPool := session.NewSessionPool(resPool)
- pool := newCopReqSenderPool(context.Background(), copCtx, store, taskCh, sessPool, nil)
- pool.chunkSender = &resultChanForTest{ch: resultCh}
- pool.adjustSize(1)
- pool.tasksCh <- task
- rs := <-resultCh
- close(taskCh)
- pool.close(false)
- sessPool.Close()
- return rs.Chunk
+ srcChkPool := make(chan *chunk.Chunk, 10)
+ for i := 0; i < 10; i++ {
+ srcChkPool <- chunk.NewChunkWithCapacity(copCtx.GetBase().FieldTypes, batchSize)
+ }
+ opCtx := ddl.NewLocalOperatorCtx(context.Background(), 1)
+ src := testutil.NewOperatorTestSource(ddl.TableScanTask{1, startKey, endKey})
+ scanOp := ddl.NewTableScanOperator(opCtx, sessPool, copCtx, srcChkPool, 1, nil, 0)
+ sink := testutil.NewOperatorTestSink[ddl.IndexRecordChunk]()
+
+ operator.Compose[ddl.TableScanTask](src, scanOp)
+ operator.Compose[ddl.IndexRecordChunk](scanOp, sink)
+
+ pipeline := operator.NewAsyncPipeline(src, scanOp, sink)
+ err := pipeline.Execute()
+ if err != nil {
+ return nil, err
+ }
+ err = pipeline.Close()
+ if err != nil {
+ return nil, err
+ }
+
+ results := sink.Collect()
+ return results[0].Chunk, nil
}
func ConvertRowToHandleAndIndexDatum(
@@ -72,14 +72,8 @@ func ConvertRowToHandleAndIndexDatum(
handleDataBuf, idxDataBuf []types.Datum,
row chunk.Row, copCtx copr.CopContext, idxID int64) (kv.Handle, []types.Datum, error) {
c := copCtx.GetBase()
- idxData := extractDatumByOffsets(ctx, row, copCtx.IndexColumnOutputOffsets(idxID), c.ExprColumnInfos, idxDataBuf)
- handleData := extractDatumByOffsets(ctx, row, c.HandleOutputOffsets, c.ExprColumnInfos, handleDataBuf)
- handle, err := buildHandle(handleData, c.TableInfo, c.PrimaryKeyInfo, time.Local, errctx.StrictNoWarningContext)
+ idxData := ddl.ExtractDatumByOffsets(ctx, row, copCtx.IndexColumnOutputOffsets(idxID), c.ExprColumnInfos, idxDataBuf)
+ handleData := ddl.ExtractDatumByOffsets(ctx, row, c.HandleOutputOffsets, c.ExprColumnInfos, handleDataBuf)
+ handle, err := ddl.BuildHandle(handleData, c.TableInfo, c.PrimaryKeyInfo, time.Local, errctx.StrictNoWarningContext)
return handle, idxData, err
}
-
-// ExtractDatumByOffsetsForTest is used for test.
-var ExtractDatumByOffsetsForTest = extractDatumByOffsets
-
-// CalculateRegionBatchForTest is used for test.
-var CalculateRegionBatchForTest = calculateRegionBatch
diff --git a/pkg/ddl/fail_test.go b/pkg/ddl/fail_test.go
index 64aa9b435709d..e961aadcaff07 100644
--- a/pkg/ddl/fail_test.go
+++ b/pkg/ddl/fail_test.go
@@ -18,9 +18,9 @@ import (
"testing"
"github.com/pingcap/failpoint"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/stretchr/testify/require"
)
@@ -37,12 +37,9 @@ func TestFailBeforeDecodeArgs(t *testing.T) {
tableIDi, _ := strconv.Atoi(rs.Rows()[0][0].(string))
tableID = int64(tableIDi)
- d := dom.DDL()
- tc := &callback.TestDDLCallback{Do: dom}
-
first := true
stateCnt := 0
- tc.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
// It can be other schema states except failed schema state.
// This schema state can only appear once.
if job.SchemaState == model.StateWriteOnly {
@@ -55,8 +52,7 @@ func TestFailBeforeDecodeArgs(t *testing.T) {
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/errorBeforeDecodeArgs"))
}
}
- }
- d.SetHook(tc)
+ })
defaultValue := int64(3)
jobID := testCreateColumn(tk, t, testkit.NewTestKit(t, store).Session(), tableID, "c3", "", defaultValue, dom)
// Make sure the schema state only appears once.
diff --git a/pkg/ddl/foreign_key.go b/pkg/ddl/foreign_key.go
index 62308440a7099..1adc206570868 100644
--- a/pkg/ddl/foreign_key.go
+++ b/pkg/ddl/foreign_key.go
@@ -31,7 +31,7 @@ import (
"github.com/pingcap/tidb/pkg/util/sqlexec"
)
-func (w *worker) onCreateForeignKey(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func (w *worker) onCreateForeignKey(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
schemaID := job.SchemaID
tblInfo, err := GetTableInfoAndCancelFaultJob(t, job, schemaID)
if err != nil {
@@ -46,11 +46,11 @@ func (w *worker) onCreateForeignKey(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
return ver, errors.Trace(err)
}
if job.IsRollingback() {
- return dropForeignKey(d, t, job, tblInfo, fkInfo.Name)
+ return dropForeignKey(jobCtx, t, job, tblInfo, fkInfo.Name)
}
switch job.SchemaState {
case model.StateNone:
- err = checkAddForeignKeyValidInOwner(d, t, job.SchemaName, tblInfo, &fkInfo, fkCheck)
+ err = checkAddForeignKeyValidInOwner(jobCtx.infoCache, job.SchemaName, tblInfo, &fkInfo, fkCheck)
if err != nil {
job.State = model.JobStateCancelled
return ver, err
@@ -58,7 +58,7 @@ func (w *worker) onCreateForeignKey(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
fkInfo.State = model.StateWriteOnly
fkInfo.ID = allocateFKIndexID(tblInfo)
tblInfo.ForeignKeys = append(tblInfo.ForeignKeys, &fkInfo)
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -71,14 +71,14 @@ func (w *worker) onCreateForeignKey(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
return ver, err
}
tblInfo.ForeignKeys[len(tblInfo.ForeignKeys)-1].State = model.StateWriteReorganization
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
job.SchemaState = model.StateWriteReorganization
case model.StateWriteReorganization:
tblInfo.ForeignKeys[len(tblInfo.ForeignKeys)-1].State = model.StatePublic
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -91,7 +91,7 @@ func (w *worker) onCreateForeignKey(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
return ver, nil
}
-func onDropForeignKey(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onDropForeignKey(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
schemaID := job.SchemaID
tblInfo, err := GetTableInfoAndCancelFaultJob(t, job, schemaID)
if err != nil {
@@ -104,10 +104,10 @@ func onDropForeignKey(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ err
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
}
- return dropForeignKey(d, t, job, tblInfo, fkName)
+ return dropForeignKey(jobCtx, t, job, tblInfo, fkName)
}
-func dropForeignKey(d *ddlCtx, t *meta.Meta, job *model.Job, tblInfo *model.TableInfo, fkName model.CIStr) (ver int64, err error) {
+func dropForeignKey(jobCtx *jobContext, t *meta.Meta, job *model.Job, tblInfo *model.TableInfo, fkName model.CIStr) (ver int64, err error) {
var fkInfo *model.FKInfo
for _, fk := range tblInfo.ForeignKeys {
if fk.Name.L == fkName.L {
@@ -126,7 +126,7 @@ func dropForeignKey(d *ddlCtx, t *meta.Meta, job *model.Job, tblInfo *model.Tabl
}
}
tblInfo.ForeignKeys = nfks
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -161,8 +161,9 @@ func checkTableForeignKeysValid(sctx sessionctx.Context, is infoschema.InfoSchem
}
referredFKInfos := is.GetTableReferredForeignKeys(schema, tbInfo.Name.L)
+ ctx := infoschema.WithRefillOption(context.Background(), false)
for _, referredFK := range referredFKInfos {
- childTable, err := is.TableByName(context.Background(), referredFK.ChildSchema, referredFK.ChildTable)
+ childTable, err := is.TableByName(ctx, referredFK.ChildSchema, referredFK.ChildTable)
if err != nil {
return err
}
@@ -206,19 +207,11 @@ func checkTableForeignKeyValid(is infoschema.InfoSchema, schema string, tbInfo *
return checkTableForeignKey(referTblInfo, tbInfo, fk)
}
-func getAndCheckLatestInfoSchema(d *ddlCtx, _ *meta.Meta) (infoschema.InfoSchema, error) {
- // TODO(crazycs520): fix me, need to make sure the `d.infoCache` is the latest infoschema.
- return d.infoCache.GetLatest(), nil
-}
-
-func checkTableForeignKeyValidInOwner(d *ddlCtx, t *meta.Meta, job *model.Job, tbInfo *model.TableInfo, fkCheck bool) (retryable bool, _ error) {
+func checkTableForeignKeyValidInOwner(jobCtx *jobContext, job *model.Job, tbInfo *model.TableInfo, fkCheck bool) (retryable bool, _ error) {
if !variable.EnableForeignKey.Load() {
return false, nil
}
- is, err := getAndCheckLatestInfoSchema(d, t)
- if err != nil {
- return true, err
- }
+ is := jobCtx.infoCache.GetLatest()
for _, fk := range tbInfo.ForeignKeys {
if fk.Version < model.FKVersion1 {
continue
@@ -227,7 +220,7 @@ func checkTableForeignKeyValidInOwner(d *ddlCtx, t *meta.Meta, job *model.Job, t
if fk.RefSchema.L == job.SchemaName && fk.RefTable.L == tbInfo.Name.L {
referTableInfo = tbInfo
} else {
- referTable, err := is.TableByName(d.ctx, fk.RefSchema, fk.RefTable)
+ referTable, err := is.TableByName(jobCtx.ctx, fk.RefSchema, fk.RefTable)
if err != nil {
if !fkCheck && (infoschema.ErrTableNotExists.Equal(err) || infoschema.ErrDatabaseNotExists.Equal(err)) {
continue
@@ -244,7 +237,7 @@ func checkTableForeignKeyValidInOwner(d *ddlCtx, t *meta.Meta, job *model.Job, t
}
referredFKInfos := is.GetTableReferredForeignKeys(job.SchemaName, tbInfo.Name.L)
for _, referredFK := range referredFKInfos {
- childTable, err := is.TableByName(d.ctx, referredFK.ChildSchema, referredFK.ChildTable)
+ childTable, err := is.TableByName(jobCtx.ctx, referredFK.ChildSchema, referredFK.ChildTable)
if err != nil {
return false, err
}
@@ -407,7 +400,7 @@ func checkTableHasForeignKeyReferred(is infoschema.InfoSchema, schema, tbl strin
return nil
}
-func checkDropTableHasForeignKeyReferredInOwner(d *ddlCtx, t *meta.Meta, job *model.Job) error {
+func checkDropTableHasForeignKeyReferredInOwner(infoCache *infoschema.InfoCache, job *model.Job) error {
if !variable.EnableForeignKey.Load() {
return nil
}
@@ -418,7 +411,7 @@ func checkDropTableHasForeignKeyReferredInOwner(d *ddlCtx, t *meta.Meta, job *mo
job.State = model.JobStateCancelled
return errors.Trace(err)
}
- referredFK, err := checkTableHasForeignKeyReferredInOwner(d, t, job.SchemaName, job.TableName, objectIdents, fkCheck)
+ referredFK, err := checkTableHasForeignKeyReferredInOwner(infoCache, job.SchemaName, job.TableName, objectIdents, fkCheck)
if err != nil {
return err
}
@@ -430,8 +423,8 @@ func checkDropTableHasForeignKeyReferredInOwner(d *ddlCtx, t *meta.Meta, job *mo
return nil
}
-func checkTruncateTableHasForeignKeyReferredInOwner(d *ddlCtx, t *meta.Meta, job *model.Job, tblInfo *model.TableInfo, fkCheck bool) error {
- referredFK, err := checkTableHasForeignKeyReferredInOwner(d, t, job.SchemaName, job.TableName, []ast.Ident{{Name: tblInfo.Name, Schema: model.NewCIStr(job.SchemaName)}}, fkCheck)
+func checkTruncateTableHasForeignKeyReferredInOwner(infoCache *infoschema.InfoCache, job *model.Job, tblInfo *model.TableInfo, fkCheck bool) error {
+ referredFK, err := checkTableHasForeignKeyReferredInOwner(infoCache, job.SchemaName, job.TableName, []ast.Ident{{Name: tblInfo.Name, Schema: model.NewCIStr(job.SchemaName)}}, fkCheck)
if err != nil {
return err
}
@@ -443,14 +436,11 @@ func checkTruncateTableHasForeignKeyReferredInOwner(d *ddlCtx, t *meta.Meta, job
return nil
}
-func checkTableHasForeignKeyReferredInOwner(d *ddlCtx, t *meta.Meta, schema, tbl string, ignoreTables []ast.Ident, fkCheck bool) (_ *model.ReferredFKInfo, _ error) {
+func checkTableHasForeignKeyReferredInOwner(infoCache *infoschema.InfoCache, schema, tbl string, ignoreTables []ast.Ident, fkCheck bool) (_ *model.ReferredFKInfo, _ error) {
if !variable.EnableForeignKey.Load() {
return nil, nil
}
- is, err := getAndCheckLatestInfoSchema(d, t)
- if err != nil {
- return nil, err
- }
+ is := infoCache.GetLatest()
referredFK := checkTableHasForeignKeyReferred(is, schema, tbl, ignoreTables, fkCheck)
return referredFK, nil
}
@@ -502,15 +492,12 @@ func checkIndexNeededInForeignKey(is infoschema.InfoSchema, dbName string, tbInf
return nil
}
-func checkIndexNeededInForeignKeyInOwner(d *ddlCtx, t *meta.Meta, job *model.Job, dbName string, tbInfo *model.TableInfo, idxInfo *model.IndexInfo) error {
+func checkIndexNeededInForeignKeyInOwner(infoCache *infoschema.InfoCache, job *model.Job, dbName string, tbInfo *model.TableInfo, idxInfo *model.IndexInfo) error {
if !variable.EnableForeignKey.Load() {
return nil
}
- is, err := getAndCheckLatestInfoSchema(d, t)
- if err != nil {
- return err
- }
- err = checkIndexNeededInForeignKey(is, dbName, tbInfo, idxInfo)
+ is := infoCache.GetLatest()
+ err := checkIndexNeededInForeignKey(is, dbName, tbInfo, idxInfo)
if err != nil {
job.State = model.JobStateCancelled
return err
@@ -537,15 +524,12 @@ func checkDropColumnWithForeignKeyConstraint(is infoschema.InfoSchema, dbName st
return nil
}
-func checkDropColumnWithForeignKeyConstraintInOwner(d *ddlCtx, t *meta.Meta, job *model.Job, tbInfo *model.TableInfo, colName string) error {
+func checkDropColumnWithForeignKeyConstraintInOwner(infoCache *infoschema.InfoCache, job *model.Job, tbInfo *model.TableInfo, colName string) error {
if !variable.EnableForeignKey.Load() {
return nil
}
- is, err := getAndCheckLatestInfoSchema(d, t)
- if err != nil {
- return errors.Trace(err)
- }
- err = checkDropColumnWithForeignKeyConstraint(is, job.SchemaName, tbInfo, colName)
+ is := infoCache.GetLatest()
+ err := checkDropColumnWithForeignKeyConstraint(is, job.SchemaName, tbInfo, colName)
if err != nil {
job.State = model.JobStateCancelled
return errors.Trace(err)
@@ -621,7 +605,7 @@ func checkDatabaseHasForeignKeyReferred(ctx context.Context, is infoschema.InfoS
return nil
}
-func checkDatabaseHasForeignKeyReferredInOwner(d *ddlCtx, t *meta.Meta, job *model.Job) error {
+func checkDatabaseHasForeignKeyReferredInOwner(jobCtx *jobContext, job *model.Job) error {
if !variable.EnableForeignKey.Load() {
return nil
}
@@ -634,11 +618,8 @@ func checkDatabaseHasForeignKeyReferredInOwner(d *ddlCtx, t *meta.Meta, job *mod
if !fkCheck {
return nil
}
- is, err := getAndCheckLatestInfoSchema(d, t)
- if err != nil {
- return errors.Trace(err)
- }
- err = checkDatabaseHasForeignKeyReferred(d.ctx, is, model.NewCIStr(job.SchemaName), fkCheck)
+ is := jobCtx.infoCache.GetLatest()
+ err = checkDatabaseHasForeignKeyReferred(jobCtx.ctx, is, model.NewCIStr(job.SchemaName), fkCheck)
if err != nil {
job.State = model.JobStateCancelled
}
@@ -665,7 +646,7 @@ func checkAddForeignKeyValid(is infoschema.InfoSchema, schema string, tbInfo *mo
return nil
}
-func checkAddForeignKeyValidInOwner(d *ddlCtx, t *meta.Meta, schema string, tbInfo *model.TableInfo, fk *model.FKInfo, fkCheck bool) error {
+func checkAddForeignKeyValidInOwner(infoCache *infoschema.InfoCache, schema string, tbInfo *model.TableInfo, fk *model.FKInfo, fkCheck bool) error {
err := checkFKDupName(tbInfo, fk.Name)
if err != nil {
return err
@@ -673,10 +654,7 @@ func checkAddForeignKeyValidInOwner(d *ddlCtx, t *meta.Meta, schema string, tbIn
if !variable.EnableForeignKey.Load() {
return nil
}
- is, err := getAndCheckLatestInfoSchema(d, t)
- if err != nil {
- return errors.Trace(err)
- }
+ is := infoCache.GetLatest()
err = checkAddForeignKeyValid(is, schema, tbInfo, fk, fkCheck)
if err != nil {
return errors.Trace(err)
diff --git a/pkg/ddl/foreign_key_test.go b/pkg/ddl/foreign_key_test.go
index d9affa1e58054..5441e3df1da62 100644
--- a/pkg/ddl/foreign_key_test.go
+++ b/pkg/ddl/foreign_key_test.go
@@ -23,12 +23,12 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/tidb/pkg/ddl"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessiontxn"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/types"
"github.com/stretchr/testify/require"
)
@@ -107,7 +107,6 @@ func getForeignKey(t table.Table, name string) *model.FKInfo {
func TestForeignKey(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease)
- d := dom.DDL()
dbInfo, err := testSchemaInfo(store, "test_foreign")
require.NoError(t, err)
de := dom.DDLExecutor().(ddl.ExecutorForTest)
@@ -131,8 +130,7 @@ func TestForeignKey(t *testing.T) {
var mu sync.Mutex
checkOK := false
var hookErr error
- tc := &callback.TestDDLCallback{}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.State != model.JobStateDone {
return
}
@@ -150,11 +148,7 @@ func TestForeignKey(t *testing.T) {
return
}
checkOK = true
- }
- tc.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- originalHook := d.GetHook()
- defer d.SetHook(originalHook)
- d.SetHook(tc)
+ })
ctx := testkit.NewTestKit(t, store).Session()
job := testCreateForeignKey(t, de, ctx, dbInfo, tblInfo, "c1_fk", []string{"c1"}, "t2", []string{"c1"}, model.ReferOptionCascade, model.ReferOptionSetNull)
@@ -173,8 +167,7 @@ func TestForeignKey(t *testing.T) {
checkOK = false
mu.Unlock()
// fix data race pr/#9491
- tc2 := &callback.TestDDLCallback{}
- onJobUpdatedExportedFunc2 := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.State != model.JobStateDone {
return
}
@@ -192,9 +185,7 @@ func TestForeignKey(t *testing.T) {
return
}
checkOK = true
- }
- tc2.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc2)
- d.SetHook(tc2)
+ })
job = testDropForeignKey(t, ctx, de, dbInfo, tblInfo, "c1_fk")
testCheckJobDone(t, store, job.ID, false)
@@ -204,7 +195,7 @@ func TestForeignKey(t *testing.T) {
mu.Unlock()
require.NoError(t, hErr)
require.True(t, ok)
- d.SetHook(originalHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
tk := testkit.NewTestKit(t, store)
jobID := testDropTable(tk, t, dbInfo.Name.L, tblInfo.Name.L, dom)
@@ -214,8 +205,7 @@ func TestForeignKey(t *testing.T) {
}
func TestTruncateOrDropTableWithForeignKeyReferred2(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease)
- d := dom.DDL()
+ store := testkit.CreateMockStoreWithSchemaLease(t, testLease)
tk := testkit.NewTestKit(t, store)
tk.MustExec("set @@global.tidb_enable_foreign_key=1")
tk.MustExec("set @@foreign_key_checks=1;")
@@ -230,8 +220,7 @@ func TestTruncateOrDropTableWithForeignKeyReferred2(t *testing.T) {
var wg sync.WaitGroup
var truncateErr, dropErr error
testTruncate := true
- tc := &callback.TestDDLCallback{}
- tc.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.SchemaState != model.StateNone {
return
}
@@ -252,10 +241,7 @@ func TestTruncateOrDropTableWithForeignKeyReferred2(t *testing.T) {
}
// make sure tk2's ddl job already put into ddl job queue.
time.Sleep(time.Millisecond * 100)
- }
- originalHook := d.GetHook()
- defer d.SetHook(originalHook)
- d.SetHook(tc)
+ })
tk.MustExec("create table t2 (a int, b int, foreign key fk(b) references t1(id));")
wg.Wait()
@@ -271,8 +257,7 @@ func TestTruncateOrDropTableWithForeignKeyReferred2(t *testing.T) {
}
func TestDropIndexNeededInForeignKey2(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease)
- d := dom.DDL()
+ store := testkit.CreateMockStoreWithSchemaLease(t, testLease)
tk := testkit.NewTestKit(t, store)
tk.MustExec("set @@global.tidb_enable_foreign_key=1")
tk.MustExec("set @@foreign_key_checks=1;")
@@ -286,8 +271,7 @@ func TestDropIndexNeededInForeignKey2(t *testing.T) {
var wg sync.WaitGroup
var dropErr error
- tc := &callback.TestDDLCallback{}
- tc.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.SchemaState != model.StatePublic || job.Type != model.ActionDropIndex {
return
}
@@ -298,10 +282,7 @@ func TestDropIndexNeededInForeignKey2(t *testing.T) {
}()
// make sure tk2's ddl job already put into ddl job queue.
time.Sleep(time.Millisecond * 100)
- }
- originalHook := d.GetHook()
- defer d.SetHook(originalHook)
- d.SetHook(tc)
+ })
tk.MustExec("alter table t2 drop index idx1")
wg.Wait()
@@ -310,8 +291,7 @@ func TestDropIndexNeededInForeignKey2(t *testing.T) {
}
func TestDropDatabaseWithForeignKeyReferred2(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease)
- d := dom.DDL()
+ store := testkit.CreateMockStoreWithSchemaLease(t, testLease)
tk := testkit.NewTestKit(t, store)
tk.MustExec("set @@global.tidb_enable_foreign_key=1")
tk.MustExec("set @@foreign_key_checks=1;")
@@ -325,8 +305,7 @@ func TestDropDatabaseWithForeignKeyReferred2(t *testing.T) {
tk.MustExec("create database test2")
var wg sync.WaitGroup
var dropErr error
- tc := &callback.TestDDLCallback{}
- tc.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.SchemaState != model.StateNone {
return
}
@@ -340,10 +319,7 @@ func TestDropDatabaseWithForeignKeyReferred2(t *testing.T) {
}()
// make sure tk2's ddl job already put into ddl job queue.
time.Sleep(time.Millisecond * 100)
- }
- originalHook := d.GetHook()
- defer d.SetHook(originalHook)
- d.SetHook(tc)
+ })
tk.MustExec("create table test2.t3 (id int key, b int, foreign key fk_b(b) references test.t2(id));")
wg.Wait()
@@ -354,8 +330,7 @@ func TestDropDatabaseWithForeignKeyReferred2(t *testing.T) {
}
func TestAddForeignKey2(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease)
- d := dom.DDL()
+ store := testkit.CreateMockStoreWithSchemaLease(t, testLease)
tk := testkit.NewTestKit(t, store)
tk.MustExec("set @@global.tidb_enable_foreign_key=1")
tk.MustExec("set @@foreign_key_checks=1;")
@@ -366,8 +341,7 @@ func TestAddForeignKey2(t *testing.T) {
tk.MustExec("create table t2 (id int key, b int, index(b));")
var wg sync.WaitGroup
var addErr error
- tc := &callback.TestDDLCallback{}
- tc.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.SchemaState != model.StatePublic || job.Type != model.ActionDropIndex {
return
}
@@ -378,10 +352,7 @@ func TestAddForeignKey2(t *testing.T) {
}()
// make sure tk2's ddl job already put into ddl job queue.
time.Sleep(time.Millisecond * 100)
- }
- originalHook := d.GetHook()
- defer d.SetHook(originalHook)
- d.SetHook(tc)
+ })
tk.MustExec("alter table t2 drop index b")
wg.Wait()
@@ -390,8 +361,7 @@ func TestAddForeignKey2(t *testing.T) {
}
func TestAddForeignKey3(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease)
- d := dom.DDL()
+ store := testkit.CreateMockStoreWithSchemaLease(t, testLease)
tk := testkit.NewTestKit(t, store)
tk.MustExec("set @@global.tidb_enable_foreign_key=1")
tk.MustExec("set @@foreign_key_checks=1;")
@@ -406,8 +376,7 @@ func TestAddForeignKey3(t *testing.T) {
var insertErrs []error
var deleteErrs []error
- tc := &callback.TestDDLCallback{}
- tc.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.Type != model.ActionAddForeignKey {
return
}
@@ -417,10 +386,7 @@ func TestAddForeignKey3(t *testing.T) {
err = tk2.ExecToErr("delete from t1 where id = 1")
deleteErrs = append(deleteErrs, err)
}
- }
- originalHook := d.GetHook()
- defer d.SetHook(originalHook)
- d.SetHook(tc)
+ })
tk.MustExec("alter table t2 add foreign key (id) references t1(id) on delete cascade")
require.Equal(t, 2, len(insertErrs))
diff --git a/pkg/ddl/index.go b/pkg/ddl/index.go
index 4c3aa25bba417..ade5a5b9fa4f2 100644
--- a/pkg/ddl/index.go
+++ b/pkg/ddl/index.go
@@ -88,7 +88,10 @@ var (
SuppressErrorTooLongKeyKey stringutil.StringerStr = "suppressErrorTooLongKeyKey"
)
-func suppressErrorTooLongKeyKey(sctx sessionctx.Context) bool {
+func suppressErrorTooLongKeyForSchemaTracker(sctx sessionctx.Context) bool {
+ if sctx == nil {
+ return false
+ }
if suppress, ok := sctx.Value(SuppressErrorTooLongKeyKey).(bool); ok && suppress {
return true
}
@@ -130,11 +133,12 @@ func buildIndexColumns(ctx sessionctx.Context, columns []*model.ColumnInfo, inde
}
sumLength += indexColumnLength
- // The sum of all lengths must be shorter than the max length for prefix.
- if sumLength > maxIndexLength {
+ if !suppressErrorTooLongKeyForSchemaTracker(ctx) && sumLength > maxIndexLength {
+ // The sum of all lengths must be shorter than the max length for prefix.
+
// The multiple column index and the unique index in which the length sum exceeds the maximum size
// will return an error instead produce a warning.
- if ctx == nil || (ctx.GetSessionVars().SQLMode.HasStrictMode() && !suppressErrorTooLongKeyKey(ctx)) || mysql.HasUniKeyFlag(col.GetFlag()) || len(indexPartSpecifications) > 1 {
+ if ctx == nil || ctx.GetSessionVars().SQLMode.HasStrictMode() || mysql.HasUniKeyFlag(col.GetFlag()) || len(indexPartSpecifications) > 1 {
return nil, false, dbterror.ErrTooLongKey.GenWithStackByArgs(sumLength, maxIndexLength)
}
// truncate index length and produce warning message in non-restrict sql mode.
@@ -188,6 +192,23 @@ func checkIndexPrefixLength(columns []*model.ColumnInfo, idxColumns []*model.Ind
return nil
}
+func indexColumnsLen(cols []*model.ColumnInfo, idxCols []*model.IndexColumn) (colLen int, err error) {
+ for _, idxCol := range idxCols {
+ col := model.FindColumnInfo(cols, idxCol.Name.L)
+ if col == nil {
+ err = dbterror.ErrKeyColumnDoesNotExits.GenWithStack("column does not exist: %s", idxCol.Name.L)
+ return
+ }
+ var l int
+ l, err = getIndexColumnLength(col, idxCol.Length)
+ if err != nil {
+ return
+ }
+ colLen += l
+ }
+ return
+}
+
func checkIndexColumn(ctx sessionctx.Context, col *model.ColumnInfo, indexColumnLen int) error {
if col.GetFlen() == 0 && (types.IsTypeChar(col.FieldType.GetType()) || types.IsTypeVarchar(col.FieldType.GetType())) {
if col.Hidden {
@@ -252,7 +273,7 @@ func checkIndexColumn(ctx sessionctx.Context, col *model.ColumnInfo, indexColumn
// Specified length must be shorter than the max length for prefix.
maxIndexLength := config.GetGlobalConfig().MaxIndexLength
if indexColumnLen > maxIndexLength {
- if ctx == nil || (ctx.GetSessionVars().SQLMode.HasStrictMode() && !suppressErrorTooLongKeyKey(ctx)) {
+ if ctx == nil || (ctx.GetSessionVars().SQLMode.HasStrictMode() && !suppressErrorTooLongKeyForSchemaTracker(ctx)) {
// return error in strict sql mode
return dbterror.ErrTooLongKey.GenWithStackByArgs(indexColumnLen, maxIndexLength)
}
@@ -307,7 +328,6 @@ func BuildIndexInfo(
indexName model.CIStr,
isPrimary bool,
isUnique bool,
- isGlobal bool,
indexPartSpecifications []*ast.IndexPartSpecification,
indexOption *ast.IndexOption,
state model.SchemaState,
@@ -328,7 +348,6 @@ func BuildIndexInfo(
State: state,
Primary: isPrimary,
Unique: isUnique,
- Global: isGlobal,
MVIndex: mvIndex,
}
@@ -343,6 +362,7 @@ func BuildIndexInfo(
} else {
idxInfo.Tp = indexOption.Tp
}
+ idxInfo.Global = indexOption.Global
} else {
// Use btree as default index type.
idxInfo.Tp = model.IndexTypeBtree
@@ -413,7 +433,7 @@ func ValidateRenameIndex(from, to model.CIStr, tbl *model.TableInfo) (ignore boo
return false, nil
}
-func onRenameIndex(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onRenameIndex(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
tblInfo, from, to, err := checkRenameIndex(t, job)
if err != nil || tblInfo == nil {
return ver, errors.Trace(err)
@@ -425,13 +445,13 @@ func onRenameIndex(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error)
if job.MultiSchemaInfo != nil && job.MultiSchemaInfo.Revertible {
job.MarkNonRevertible()
// Store the mark and enter the next DDL handling loop.
- return updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, false)
+ return updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, false)
}
renameIndexes(tblInfo, from, to)
renameHiddenColumns(tblInfo, from, to)
- if ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true); err != nil {
+ if ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true); err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
}
@@ -453,7 +473,7 @@ func validateAlterIndexVisibility(ctx sessionctx.Context, indexName model.CIStr,
return false, nil
}
-func onAlterIndexVisibility(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onAlterIndexVisibility(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
tblInfo, from, invisible, err := checkAlterIndexVisibility(t, job)
if err != nil || tblInfo == nil {
return ver, errors.Trace(err)
@@ -461,11 +481,11 @@ func onAlterIndexVisibility(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64,
if job.MultiSchemaInfo != nil && job.MultiSchemaInfo.Revertible {
job.MarkNonRevertible()
- return updateVersionAndTableInfo(d, t, job, tblInfo, false)
+ return updateVersionAndTableInfo(jobCtx, t, job, tblInfo, false)
}
setIndexVisibility(tblInfo, from, invisible)
- if ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, true); err != nil {
+ if ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, true); err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
}
@@ -492,7 +512,7 @@ func getNullColInfos(tblInfo *model.TableInfo, indexInfo *model.IndexInfo) ([]*m
return nullCols, nil
}
-func checkPrimaryKeyNotNull(d *ddlCtx, w *worker, t *meta.Meta, job *model.Job,
+func checkPrimaryKeyNotNull(jobCtx *jobContext, w *worker, t *meta.Meta, job *model.Job,
tblInfo *model.TableInfo, indexInfo *model.IndexInfo) (warnings []string, err error) {
if !indexInfo.Primary {
return nil, nil
@@ -514,7 +534,7 @@ func checkPrimaryKeyNotNull(d *ddlCtx, w *worker, t *meta.Meta, job *model.Job,
if err == nil {
return nil, nil
}
- _, err = convertAddIdxJob2RollbackJob(d, t, job, tblInfo, []*model.IndexInfo{indexInfo}, err)
+ _, err = convertAddIdxJob2RollbackJob(jobCtx, t, job, tblInfo, []*model.IndexInfo{indexInfo}, err)
// TODO: Support non-strict mode.
// warnings = append(warnings, ErrWarnDataTruncated.GenWithStackByArgs(oldCol.Name.L, 0).Error())
return nil, err
@@ -555,7 +575,6 @@ func decodeAddIndexArgs(job *model.Job) (
indexPartSpecifications [][]*ast.IndexPartSpecification,
indexOptions []*ast.IndexOption,
hiddenCols [][]*model.ColumnInfo,
- globals []bool,
err error,
) {
var (
@@ -564,27 +583,25 @@ func decodeAddIndexArgs(job *model.Job) (
indexPartSpecification []*ast.IndexPartSpecification
indexOption *ast.IndexOption
hiddenCol []*model.ColumnInfo
- global bool
)
- err = job.DecodeArgs(&unique, &indexName, &indexPartSpecification, &indexOption, &hiddenCol, &global)
+ err = job.DecodeArgs(&unique, &indexName, &indexPartSpecification, &indexOption, &hiddenCol)
if err == nil {
return []bool{unique},
[]model.CIStr{indexName},
[][]*ast.IndexPartSpecification{indexPartSpecification},
[]*ast.IndexOption{indexOption},
[][]*model.ColumnInfo{hiddenCol},
- []bool{global},
nil
}
- err = job.DecodeArgs(&uniques, &indexNames, &indexPartSpecifications, &indexOptions, &hiddenCols, &globals)
+ err = job.DecodeArgs(&uniques, &indexNames, &indexPartSpecifications, &indexOptions, &hiddenCols)
return
}
-func (w *worker) onCreateIndex(d *ddlCtx, t *meta.Meta, job *model.Job, isPK bool) (ver int64, err error) {
+func (w *worker) onCreateIndex(jobCtx *jobContext, t *meta.Meta, job *model.Job, isPK bool) (ver int64, err error) {
// Handle the rolling back job.
if job.IsRollingback() {
- ver, err = onDropIndex(d, t, job)
+ ver, err = onDropIndex(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -602,7 +619,6 @@ func (w *worker) onCreateIndex(d *ddlCtx, t *meta.Meta, job *model.Job, isPK boo
}
uniques := make([]bool, 1)
- global := make([]bool, 1)
indexNames := make([]model.CIStr, 1)
indexPartSpecifications := make([][]*ast.IndexPartSpecification, 1)
indexOption := make([]*ast.IndexOption, 1)
@@ -612,9 +628,9 @@ func (w *worker) onCreateIndex(d *ddlCtx, t *meta.Meta, job *model.Job, isPK boo
if isPK {
// Notice: sqlMode and warnings is used to support non-strict mode.
- err = job.DecodeArgs(&uniques[0], &indexNames[0], &indexPartSpecifications[0], &indexOption[0], &sqlMode, &warnings, &global[0])
+ err = job.DecodeArgs(&uniques[0], &indexNames[0], &indexPartSpecifications[0], &indexOption[0], &sqlMode, &warnings)
} else {
- uniques, indexNames, indexPartSpecifications, indexOption, hiddenCols, global, err = decodeAddIndexArgs(job)
+ uniques, indexNames, indexPartSpecifications, indexOption, hiddenCols, err = decodeAddIndexArgs(job)
}
if err != nil {
job.State = model.JobStateCancelled
@@ -659,7 +675,6 @@ func (w *worker) onCreateIndex(d *ddlCtx, t *meta.Meta, job *model.Job, isPK boo
indexName,
isPK,
uniques[i],
- global[i],
indexPartSpecifications[i],
indexOption[i],
model.StateNone,
@@ -716,7 +731,7 @@ SwitchIndexState:
indexInfo.State = model.StateDeleteOnly
moveAndUpdateHiddenColumnsToPublic(tblInfo, indexInfo)
}
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, originalState != model.StateDeleteOnly)
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, originalState != model.StateDeleteOnly)
if err != nil {
return ver, err
}
@@ -725,13 +740,13 @@ SwitchIndexState:
// delete only -> write only
for _, indexInfo := range allIndexInfos {
indexInfo.State = model.StateWriteOnly
- _, err = checkPrimaryKeyNotNull(d, w, t, job, tblInfo, indexInfo)
+ _, err = checkPrimaryKeyNotNull(jobCtx, w, t, job, tblInfo, indexInfo)
if err != nil {
break SwitchIndexState
}
}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != model.StateWriteOnly)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != model.StateWriteOnly)
if err != nil {
return ver, err
}
@@ -740,13 +755,13 @@ SwitchIndexState:
// write only -> reorganization
for _, indexInfo := range allIndexInfos {
indexInfo.State = model.StateWriteReorganization
- _, err = checkPrimaryKeyNotNull(d, w, t, job, tblInfo, indexInfo)
+ _, err = checkPrimaryKeyNotNull(jobCtx, w, t, job, tblInfo, indexInfo)
if err != nil {
break SwitchIndexState
}
}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != model.StateWriteReorganization)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != model.StateWriteReorganization)
if err != nil {
return ver, err
}
@@ -755,16 +770,16 @@ SwitchIndexState:
job.SchemaState = model.StateWriteReorganization
case model.StateWriteReorganization:
// reorganization -> public
- tbl, err := getTable(d.getAutoIDRequirement(), schemaID, tblInfo)
+ tbl, err := getTable(jobCtx.getAutoIDRequirement(), schemaID, tblInfo)
if err != nil {
return ver, errors.Trace(err)
}
var done bool
if job.MultiSchemaInfo != nil {
- done, ver, err = doReorgWorkForCreateIndexMultiSchema(w, d, t, job, tbl, allIndexInfos)
+ done, ver, err = doReorgWorkForCreateIndexMultiSchema(w, jobCtx, t, job, tbl, allIndexInfos)
} else {
- done, ver, err = doReorgWorkForCreateIndex(w, d, t, job, tbl, allIndexInfos)
+ done, ver, err = doReorgWorkForCreateIndex(w, jobCtx, t, job, tbl, allIndexInfos)
}
if !done {
return ver, err
@@ -797,18 +812,20 @@ SwitchIndexState:
}
})
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != model.StatePublic)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != model.StatePublic)
if err != nil {
return ver, errors.Trace(err)
}
allIndexIDs := make([]int64, 0, len(allIndexInfos))
ifExists := make([]bool, 0, len(allIndexInfos))
+ isGlobal := make([]bool, 0, len(allIndexInfos))
for _, indexInfo := range allIndexInfos {
allIndexIDs = append(allIndexIDs, indexInfo.ID)
ifExists = append(ifExists, false)
+ isGlobal = append(isGlobal, indexInfo.Global)
}
- job.Args = []any{allIndexIDs, ifExists, getPartitionIDs(tbl.Meta())}
+ job.Args = []any{allIndexIDs, ifExists, getPartitionIDs(tbl.Meta()), isGlobal}
// Finish this job.
job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tblInfo)
if !job.ReorgMeta.IsDistReorg && job.ReorgMeta.ReorgTp == model.ReorgTypeLitMerge {
@@ -863,14 +880,14 @@ func loadCloudStorageURI(w *worker, job *model.Job) {
job.ReorgMeta.UseCloudStorage = len(jc.cloudStorageURI) > 0
}
-func doReorgWorkForCreateIndexMultiSchema(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job,
+func doReorgWorkForCreateIndexMultiSchema(w *worker, jobCtx *jobContext, t *meta.Meta, job *model.Job,
tbl table.Table, allIndexInfos []*model.IndexInfo) (done bool, ver int64, err error) {
if job.MultiSchemaInfo.Revertible {
- done, ver, err = doReorgWorkForCreateIndex(w, d, t, job, tbl, allIndexInfos)
+ done, ver, err = doReorgWorkForCreateIndex(w, jobCtx, t, job, tbl, allIndexInfos)
if done {
job.MarkNonRevertible()
if err == nil {
- ver, err = updateVersionAndTableInfo(d, t, job, tbl.Meta(), true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tbl.Meta(), true)
}
}
// We need another round to wait for all the others sub-jobs to finish.
@@ -881,7 +898,7 @@ func doReorgWorkForCreateIndexMultiSchema(w *worker, d *ddlCtx, t *meta.Meta, jo
func doReorgWorkForCreateIndex(
w *worker,
- d *ddlCtx,
+ jobCtx *jobContext,
t *meta.Meta,
job *model.Job,
tbl table.Table,
@@ -893,7 +910,7 @@ func doReorgWorkForCreateIndex(
return false, ver, err
}
if !reorgTp.NeedMergeProcess() {
- return runReorgJobAndHandleErr(w, d, t, job, tbl, allIndexInfos, false)
+ return runReorgJobAndHandleErr(w, jobCtx, t, job, tbl, allIndexInfos, false)
}
switch allIndexInfos[0].BackfillState {
case model.BackfillStateRunning:
@@ -904,12 +921,12 @@ func doReorgWorkForCreateIndex(
switch reorgTp {
case model.ReorgTypeLitMerge:
if job.ReorgMeta.IsDistReorg {
- done, ver, err = runIngestReorgJobDist(w, d, t, job, tbl, allIndexInfos)
+ done, ver, err = runIngestReorgJobDist(w, jobCtx, t, job, tbl, allIndexInfos)
} else {
- done, ver, err = runIngestReorgJob(w, d, t, job, tbl, allIndexInfos)
+ done, ver, err = runIngestReorgJob(w, jobCtx, t, job, tbl, allIndexInfos)
}
case model.ReorgTypeTxnMerge:
- done, ver, err = runReorgJobAndHandleErr(w, d, t, job, tbl, allIndexInfos, false)
+ done, ver, err = runReorgJobAndHandleErr(w, jobCtx, t, job, tbl, allIndexInfos, false)
}
if err != nil || !done {
return false, ver, errors.Trace(err)
@@ -917,7 +934,7 @@ func doReorgWorkForCreateIndex(
for _, indexInfo := range allIndexInfos {
indexInfo.BackfillState = model.BackfillStateReadyToMerge
}
- ver, err = updateVersionAndTableInfo(d, t, job, tbl.Meta(), true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tbl.Meta(), true)
return false, ver, errors.Trace(err)
case model.BackfillStateReadyToMerge:
failpoint.Inject("mockDMLExecutionStateBeforeMerge", func(_ failpoint.Value) {
@@ -936,10 +953,10 @@ func doReorgWorkForCreateIndex(
ingest.LitBackCtxMgr.Unregister(job.ID)
}
job.SnapshotVer = 0 // Reset the snapshot version for merge index reorg.
- ver, err = updateVersionAndTableInfo(d, t, job, tbl.Meta(), true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tbl.Meta(), true)
return false, ver, errors.Trace(err)
case model.BackfillStateMerging:
- done, ver, err = runReorgJobAndHandleErr(w, d, t, job, tbl, allIndexInfos, true)
+ done, ver, err = runReorgJobAndHandleErr(w, jobCtx, t, job, tbl, allIndexInfos, true)
if !done {
return false, ver, err
}
@@ -952,9 +969,9 @@ func doReorgWorkForCreateIndex(
}
}
-func runIngestReorgJobDist(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job,
+func runIngestReorgJobDist(w *worker, jobCtx *jobContext, t *meta.Meta, job *model.Job,
tbl table.Table, allIndexInfos []*model.IndexInfo) (done bool, ver int64, err error) {
- done, ver, err = runReorgJobAndHandleErr(w, d, t, job, tbl, allIndexInfos, false)
+ done, ver, err = runReorgJobAndHandleErr(w, jobCtx, t, job, tbl, allIndexInfos, false)
if err != nil {
return false, ver, errors.Trace(err)
}
@@ -966,17 +983,17 @@ func runIngestReorgJobDist(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job,
return true, ver, nil
}
-func runIngestReorgJob(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job,
+func runIngestReorgJob(w *worker, jobCtx *jobContext, t *meta.Meta, job *model.Job,
tbl table.Table, allIndexInfos []*model.IndexInfo) (done bool, ver int64, err error) {
- done, ver, err = runReorgJobAndHandleErr(w, d, t, job, tbl, allIndexInfos, false)
+ done, ver, err = runReorgJobAndHandleErr(w, jobCtx, t, job, tbl, allIndexInfos, false)
if err != nil {
if kv.ErrKeyExists.Equal(err) {
logutil.DDLLogger().Warn("import index duplicate key, convert job to rollback", zap.Stringer("job", job), zap.Error(err))
- ver, err = convertAddIdxJob2RollbackJob(d, t, job, tbl.Meta(), allIndexInfos, err)
+ ver, err = convertAddIdxJob2RollbackJob(jobCtx, t, job, tbl.Meta(), allIndexInfos, err)
} else if !errorIsRetryable(err, job) {
logutil.DDLLogger().Warn("run reorg job failed, convert job to rollback",
zap.String("job", job.String()), zap.Error(err))
- ver, err = convertAddIdxJob2RollbackJob(d, t, job, tbl.Meta(), allIndexInfos, err)
+ ver, err = convertAddIdxJob2RollbackJob(jobCtx, t, job, tbl.Meta(), allIndexInfos, err)
} else {
logutil.DDLLogger().Warn("run add index ingest job error", zap.Error(err))
}
@@ -1002,7 +1019,7 @@ func errorIsRetryable(err error, job *model.Job) bool {
func runReorgJobAndHandleErr(
w *worker,
- d *ddlCtx,
+ jobCtx *jobContext,
t *meta.Meta,
job *model.Job,
tbl table.Table,
@@ -1033,7 +1050,7 @@ func runReorgJobAndHandleErr(
if err != nil {
return false, ver, errors.Trace(err)
}
- reorgInfo, err := getReorgInfo(d.jobContext(job.ID, job.ReorgMeta), d, rh, job, dbInfo, tbl, elements, mergingTmpIdx)
+ reorgInfo, err := getReorgInfo(jobCtx.oldDDLCtx.jobContext(job.ID, job.ReorgMeta), jobCtx, rh, job, dbInfo, tbl, elements, mergingTmpIdx)
if err != nil || reorgInfo == nil || reorgInfo.first {
// If we run reorg firstly, we should update the job snapshot version
// and then run the reorg next time.
@@ -1043,7 +1060,7 @@ func runReorgJobAndHandleErr(
if err != nil {
return false, ver, errors.Trace(err)
}
- err = w.runReorgJob(reorgInfo, tbl.Meta(), d.lease, func() (addIndexErr error) {
+ err = w.runReorgJob(reorgInfo, tbl.Meta(), func() (addIndexErr error) {
defer util.Recover(metrics.LabelDDL, "onCreateIndex",
func() {
addIndexErr = dbterror.ErrCancelledDDLJob.GenWithStack("add table `%v` index `%v` panic", tbl.Meta().Name, allIndexInfos[0].Name)
@@ -1062,7 +1079,7 @@ func runReorgJobAndHandleErr(
err = ingest.TryConvertToKeyExistsErr(err, allIndexInfos[0], tbl.Meta())
if !errorIsRetryable(err, job) {
logutil.DDLLogger().Warn("run add index job failed, convert job to rollback", zap.Stringer("job", job), zap.Error(err))
- ver, err = convertAddIdxJob2RollbackJob(d, t, job, tbl.Meta(), allIndexInfos, err)
+ ver, err = convertAddIdxJob2RollbackJob(jobCtx, t, job, tbl.Meta(), allIndexInfos, err)
if err1 := rh.RemoveDDLReorgHandle(job, reorgInfo.elements); err1 != nil {
logutil.DDLLogger().Warn("run add index job failed, convert job to rollback, RemoveDDLReorgHandle failed", zap.Stringer("job", job), zap.Error(err1))
}
@@ -1077,8 +1094,8 @@ func runReorgJobAndHandleErr(
return true, ver, nil
}
-func onDropIndex(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
- tblInfo, allIndexInfos, ifExists, err := checkDropIndex(d, t, job)
+func onDropIndex(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+ tblInfo, allIndexInfos, ifExists, err := checkDropIndex(jobCtx.infoCache, t, job)
if err != nil {
if ifExists && dbterror.ErrCantDropFieldOrKey.Equal(err) {
job.Warning = toTError(err)
@@ -1094,7 +1111,7 @@ func onDropIndex(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
if job.MultiSchemaInfo != nil && !job.IsRollingback() && job.MultiSchemaInfo.Revertible {
job.MarkNonRevertible()
job.SchemaState = allIndexInfos[0].State
- return updateVersionAndTableInfo(d, t, job, tblInfo, false)
+ return updateVersionAndTableInfo(jobCtx, t, job, tblInfo, false)
}
originalState := allIndexInfos[0].State
@@ -1104,7 +1121,7 @@ func onDropIndex(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
for _, indexInfo := range allIndexInfos {
indexInfo.State = model.StateWriteOnly
}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != model.StateWriteOnly)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != model.StateWriteOnly)
if err != nil {
return ver, errors.Trace(err)
}
@@ -1113,7 +1130,7 @@ func onDropIndex(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
for _, indexInfo := range allIndexInfos {
indexInfo.State = model.StateDeleteOnly
}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != model.StateDeleteOnly)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != model.StateDeleteOnly)
if err != nil {
return ver, errors.Trace(err)
}
@@ -1122,7 +1139,7 @@ func onDropIndex(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
for _, indexInfo := range allIndexInfos {
indexInfo.State = model.StateDeleteReorganization
}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != model.StateDeleteReorganization)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != model.StateDeleteReorganization)
if err != nil {
return ver, errors.Trace(err)
}
@@ -1145,7 +1162,7 @@ func onDropIndex(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
}
})
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, originalState != model.StateNone)
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, originalState != model.StateNone)
if err != nil {
return ver, errors.Trace(err)
}
@@ -1209,7 +1226,7 @@ func removeIndexInfo(tblInfo *model.TableInfo, idxInfo *model.IndexInfo) {
tblInfo.Indices = append(tblInfo.Indices[:offset], tblInfo.Indices[offset+1:]...)
}
-func checkDropIndex(d *ddlCtx, t *meta.Meta, job *model.Job) (*model.TableInfo, []*model.IndexInfo, bool /* ifExists */, error) {
+func checkDropIndex(infoCache *infoschema.InfoCache, t *meta.Meta, job *model.Job) (*model.TableInfo, []*model.IndexInfo, bool /* ifExists */, error) {
schemaID := job.SchemaID
tblInfo, err := GetTableInfoAndCancelFaultJob(t, job, schemaID)
if err != nil {
@@ -1240,7 +1257,7 @@ func checkDropIndex(d *ddlCtx, t *meta.Meta, job *model.Job) (*model.TableInfo,
}
// Double check for drop index needed in foreign key.
- if err := checkIndexNeededInForeignKeyInOwner(d, t, job, job.SchemaName, tblInfo, indexInfo); err != nil {
+ if err := checkIndexNeededInForeignKeyInOwner(infoCache, job, job.SchemaName, tblInfo, indexInfo); err != nil {
return nil, nil, false, errors.Trace(err)
}
indexInfos = append(indexInfos, indexInfo)
@@ -1635,8 +1652,6 @@ func (w *addIndexTxnWorker) batchCheckUniqueKey(txn kv.Transaction, idxRecords [
}
idxRecords[w.recordIdx[i]].skip = found && idxRecords[w.recordIdx[i]].skip
}
- // Constrains is already checked.
- w.tblCtx.GetSessionVars().StmtCtx.BatchCheck = true
return nil
}
@@ -1701,20 +1716,20 @@ func writeChunkToLocal(
restoreDataBuf = make([]types.Datum, len(c.HandleOutputOffsets))
}
for row := iter.Begin(); row != iter.End(); row = iter.Next() {
- handleDataBuf := extractDatumByOffsets(ectx, row, c.HandleOutputOffsets, c.ExprColumnInfos, handleDataBuf)
+ handleDataBuf := ExtractDatumByOffsets(ectx, row, c.HandleOutputOffsets, c.ExprColumnInfos, handleDataBuf)
if restore {
// restoreDataBuf should not truncate index values.
for i, datum := range handleDataBuf {
restoreDataBuf[i] = *datum.Clone()
}
}
- h, err := buildHandle(handleDataBuf, c.TableInfo, c.PrimaryKeyInfo, loc, errCtx)
+ h, err := BuildHandle(handleDataBuf, c.TableInfo, c.PrimaryKeyInfo, loc, errCtx)
if err != nil {
return 0, nil, errors.Trace(err)
}
for i, index := range indexes {
idxID := index.Meta().ID
- idxDataBuf = extractDatumByOffsets(ectx,
+ idxDataBuf = ExtractDatumByOffsets(ectx,
row, copCtx.IndexColumnOutputOffsets(idxID), c.ExprColumnInfos, idxDataBuf)
idxData := idxDataBuf[:len(index.Meta().Columns)]
var rsData []types.Datum
@@ -1830,7 +1845,12 @@ func (w *addIndexTxnWorker) BackfillData(handleRange reorgBackfillTask) (taskCtx
}
handle, err := w.indexes[i%len(w.indexes)].Create(
- w.tblCtx, txn, idxRecord.vals, idxRecord.handle, idxRecord.rsData, table.WithIgnoreAssertion, table.FromBackfill)
+ w.tblCtx, txn, idxRecord.vals, idxRecord.handle, idxRecord.rsData,
+ table.WithIgnoreAssertion,
+ table.FromBackfill,
+ // Constrains is already checked in batchCheckUniqueKey
+ table.DupKeyCheckSkip,
+ )
if err != nil {
if kv.ErrKeyExists.Equal(err) && idxRecord.handle.Equal(handle) {
// Index already exists, skip it.
@@ -1940,7 +1960,7 @@ func checkDuplicateForUniqueIndex(ctx context.Context, t table.Table, reorgInfo
if indexInfo.Unique {
ctx := tidblogutil.WithCategory(ctx, "ddl-ingest")
if bc == nil {
- bc, err = ingest.LitBackCtxMgr.Register(ctx, reorgInfo.ID, indexInfo.Unique, nil, discovery, reorgInfo.ReorgMeta.ResourceGroupName)
+ bc, err = ingest.LitBackCtxMgr.Register(ctx, reorgInfo.ID, indexInfo.Unique, nil, discovery, reorgInfo.ReorgMeta.ResourceGroupName, 1)
if err != nil {
return err
}
@@ -2015,7 +2035,7 @@ func (w *worker) executeDistTask(t table.Table, reorgInfo *reorgInfo) error {
})
} else {
job := reorgInfo.Job
- workerCntLimit := int(variable.GetDDLReorgWorkerCounter())
+ workerCntLimit := job.ReorgMeta.GetConcurrencyOrDefault(int(variable.GetDDLReorgWorkerCounter()))
cpuCount, err := handle.GetCPUCountOfNode(ctx)
if err != nil {
return err
@@ -2255,7 +2275,7 @@ func getNextPartitionInfo(reorg *reorgInfo, t table.PartitionedTable, currPhysic
if val.(bool) {
ts := oracle.GoTimeToTS(time.Now())
//nolint:forcetypeassert
- s := reorg.d.store.(tikv.Storage)
+ s := reorg.jobCtx.store.(tikv.Storage)
s.UpdateSPCache(ts, time.Now())
time.Sleep(time.Second * 3)
}
@@ -2269,11 +2289,11 @@ func getNextPartitionInfo(reorg *reorgInfo, t table.PartitionedTable, currPhysic
startKey = tablecodec.EncodeIndexSeekKey(pid, firstElemTempID, nil)
endKey = tablecodec.EncodeIndexSeekKey(pid, lastElemTempID, []byte{255})
} else {
- currentVer, err := getValidCurrentVersion(reorg.d.store)
+ currentVer, err := getValidCurrentVersion(reorg.jobCtx.store)
if err != nil {
return 0, nil, nil, errors.Trace(err)
}
- startKey, endKey, err = getTableRange(reorg.NewJobContext(), reorg.d, t.GetPartition(pid), currentVer.Ver, reorg.Job.Priority)
+ startKey, endKey, err = getTableRange(reorg.NewJobContext(), reorg.jobCtx.store, t.GetPartition(pid), currentVer.Ver, reorg.Job.Priority)
if err != nil {
return 0, nil, nil, errors.Trace(err)
}
@@ -2375,7 +2395,7 @@ type cleanUpIndexWorker struct {
baseIndexWorker
}
-func newCleanUpIndexWorker(id int, t table.PhysicalTable, decodeColMap map[int64]decoder.Column, reorgInfo *reorgInfo, jc *JobContext) (*cleanUpIndexWorker, error) {
+func newCleanUpIndexWorker(id int, t table.PhysicalTable, decodeColMap map[int64]decoder.Column, reorgInfo *reorgInfo, jc *ReorgContext) (*cleanUpIndexWorker, error) {
bCtx, err := newBackfillCtx(id, reorgInfo, reorgInfo.SchemaName, t, jc, "cleanup_idx_rate", false)
if err != nil {
return nil, err
@@ -2501,11 +2521,11 @@ func (w *worker) updateReorgInfoForPartitions(t table.PartitionedTable, reorg *r
}
}
- currentVer, err := getValidCurrentVersion(reorg.d.store)
+ currentVer, err := getValidCurrentVersion(reorg.jobCtx.store)
if err != nil {
return false, errors.Trace(err)
}
- start, end, err := getTableRange(reorg.NewJobContext(), reorg.d, t.GetPartition(pid), currentVer.Ver, reorg.Job.Priority)
+ start, end, err := getTableRange(reorg.NewJobContext(), reorg.jobCtx.store, t.GetPartition(pid), currentVer.Ver, reorg.Job.Priority)
if err != nil {
return false, errors.Trace(err)
}
diff --git a/pkg/ddl/index_change_test.go b/pkg/ddl/index_change_test.go
index 8ec5d34fd1403..882d2fecffe23 100644
--- a/pkg/ddl/index_change_test.go
+++ b/pkg/ddl/index_change_test.go
@@ -22,15 +22,14 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/tidb/pkg/ddl"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
- "github.com/pingcap/tidb/pkg/sessiontxn"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/table/tables"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/types"
"github.com/stretchr/testify/require"
)
@@ -43,8 +42,6 @@ func TestIndexChange(t *testing.T) {
tk.MustExec("create table t (c1 int primary key, c2 int)")
tk.MustExec("insert t values (1, 1), (2, 2), (3, 3);")
- d := dom.DDL()
- tc := &callback.TestDDLCallback{Do: dom}
// set up hook
prevState := model.StateNone
addIndexDone := false
@@ -54,7 +51,7 @@ func TestIndexChange(t *testing.T) {
writeOnlyTable table.Table
publicTable table.Table
)
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.Type != model.ActionAddIndex || job.TableName != "t" {
return
}
@@ -65,7 +62,7 @@ func TestIndexChange(t *testing.T) {
ctx1 := testNewContext(store)
prevState = job.SchemaState
require.NoError(t, dom.Reload())
- tbl, exist := dom.InfoSchema().TableByID(job.TableID)
+ tbl, exist := dom.InfoSchema().TableByID(context.Background(), job.TableID)
require.True(t, exist)
switch job.SchemaState {
case model.StateDeleteOnly:
@@ -83,9 +80,7 @@ func TestIndexChange(t *testing.T) {
addIndexDone = true
}
}
- }
- tc.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(tc)
+ })
tk.MustExec("alter table t add index c2(c2)")
// We need to make sure onJobUpdated is called in the first hook.
// After testCreateIndex(), onJobUpdated() may not be called when job.state is Sync.
@@ -101,7 +96,7 @@ func TestIndexChange(t *testing.T) {
prevState = model.StateNone
var noneTable table.Table
- onJobUpdatedExportedFunc2 := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
jobID.Store(job.ID)
if job.SchemaState == prevState {
return
@@ -109,7 +104,7 @@ func TestIndexChange(t *testing.T) {
prevState = job.SchemaState
var err error
require.NoError(t, dom.Reload())
- tbl, exist := dom.InfoSchema().TableByID(job.TableID)
+ tbl, exist := dom.InfoSchema().TableByID(context.Background(), job.TableID)
require.True(t, exist)
ctx1 := testNewContext(store)
switch job.SchemaState {
@@ -125,8 +120,7 @@ func TestIndexChange(t *testing.T) {
noneTable = tbl
require.Equalf(t, 0, len(noneTable.Indices()), "index should have been dropped")
}
- }
- tc.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc2)
+ })
tk.MustExec("alter table t drop index c2")
v = getSchemaVer(t, tk.Session())
checkHistoryJobArgs(t, tk.Session(), jobID.Load(), &historyJobArgs{ver: v, tbl: noneTable.Meta()})
@@ -154,11 +148,11 @@ func checkIndexExists(ctx sessionctx.Context, tbl table.Table, indexValue any, h
func checkAddWriteOnlyForAddIndex(ctx sessionctx.Context, delOnlyTbl, writeOnlyTbl table.Table) error {
// DeleteOnlyTable: insert t values (4, 4);
- err := sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err := newTxn(ctx)
if err != nil {
return errors.Trace(err)
}
- _, err = delOnlyTbl.AddRecord(ctx.GetTableCtx(), types.MakeDatums(4, 4))
+ _, err = delOnlyTbl.AddRecord(ctx.GetTableCtx(), txn, types.MakeDatums(4, 4))
if err != nil {
return errors.Trace(err)
}
@@ -168,7 +162,7 @@ func checkAddWriteOnlyForAddIndex(ctx sessionctx.Context, delOnlyTbl, writeOnlyT
}
// WriteOnlyTable: insert t values (5, 5);
- _, err = writeOnlyTbl.AddRecord(ctx.GetTableCtx(), types.MakeDatums(5, 5))
+ _, err = writeOnlyTbl.AddRecord(ctx.GetTableCtx(), txn, types.MakeDatums(5, 5))
if err != nil {
return errors.Trace(err)
}
@@ -178,7 +172,7 @@ func checkAddWriteOnlyForAddIndex(ctx sessionctx.Context, delOnlyTbl, writeOnlyT
}
// WriteOnlyTable: update t set c2 = 1 where c1 = 4 and c2 = 4
- err = writeOnlyTbl.UpdateRecord(context.Background(), ctx.GetTableCtx(), kv.IntHandle(4), types.MakeDatums(4, 4), types.MakeDatums(4, 1), touchedSlice(writeOnlyTbl))
+ err = writeOnlyTbl.UpdateRecord(ctx.GetTableCtx(), txn, kv.IntHandle(4), types.MakeDatums(4, 4), types.MakeDatums(4, 1), touchedSlice(writeOnlyTbl))
if err != nil {
return errors.Trace(err)
}
@@ -188,7 +182,7 @@ func checkAddWriteOnlyForAddIndex(ctx sessionctx.Context, delOnlyTbl, writeOnlyT
}
// DeleteOnlyTable: update t set c2 = 3 where c1 = 4 and c2 = 1
- err = delOnlyTbl.UpdateRecord(context.Background(), ctx.GetTableCtx(), kv.IntHandle(4), types.MakeDatums(4, 1), types.MakeDatums(4, 3), touchedSlice(writeOnlyTbl))
+ err = delOnlyTbl.UpdateRecord(ctx.GetTableCtx(), txn, kv.IntHandle(4), types.MakeDatums(4, 1), types.MakeDatums(4, 3), touchedSlice(writeOnlyTbl))
if err != nil {
return errors.Trace(err)
}
@@ -204,7 +198,7 @@ func checkAddWriteOnlyForAddIndex(ctx sessionctx.Context, delOnlyTbl, writeOnlyT
}
// WriteOnlyTable: delete t where c1 = 4 and c2 = 3
- err = writeOnlyTbl.RemoveRecord(ctx.GetTableCtx(), kv.IntHandle(4), types.MakeDatums(4, 3))
+ err = writeOnlyTbl.RemoveRecord(ctx.GetTableCtx(), txn, kv.IntHandle(4), types.MakeDatums(4, 3))
if err != nil {
return errors.Trace(err)
}
@@ -214,7 +208,7 @@ func checkAddWriteOnlyForAddIndex(ctx sessionctx.Context, delOnlyTbl, writeOnlyT
}
// DeleteOnlyTable: delete t where c1 = 5
- err = delOnlyTbl.RemoveRecord(ctx.GetTableCtx(), kv.IntHandle(5), types.MakeDatums(5, 5))
+ err = delOnlyTbl.RemoveRecord(ctx.GetTableCtx(), txn, kv.IntHandle(5), types.MakeDatums(5, 5))
if err != nil {
return errors.Trace(err)
}
@@ -228,11 +222,11 @@ func checkAddWriteOnlyForAddIndex(ctx sessionctx.Context, delOnlyTbl, writeOnlyT
func checkAddPublicForAddIndex(ctx sessionctx.Context, writeTbl, publicTbl table.Table) error {
var err1 error
// WriteOnlyTable: insert t values (6, 6)
- err := sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err := newTxn(ctx)
if err != nil {
return errors.Trace(err)
}
- _, err = writeTbl.AddRecord(ctx.GetTableCtx(), types.MakeDatums(6, 6))
+ _, err = writeTbl.AddRecord(ctx.GetTableCtx(), txn, types.MakeDatums(6, 6))
if err != nil {
return errors.Trace(err)
}
@@ -245,7 +239,7 @@ func checkAddPublicForAddIndex(ctx sessionctx.Context, writeTbl, publicTbl table
return errors.Trace(err)
}
// PublicTable: insert t values (7, 7)
- _, err = publicTbl.AddRecord(ctx.GetTableCtx(), types.MakeDatums(7, 7))
+ _, err = publicTbl.AddRecord(ctx.GetTableCtx(), txn, types.MakeDatums(7, 7))
if err != nil {
return errors.Trace(err)
}
@@ -255,7 +249,7 @@ func checkAddPublicForAddIndex(ctx sessionctx.Context, writeTbl, publicTbl table
}
// WriteOnlyTable: update t set c2 = 5 where c1 = 7 and c2 = 7
- err = writeTbl.UpdateRecord(context.Background(), ctx.GetTableCtx(), kv.IntHandle(7), types.MakeDatums(7, 7), types.MakeDatums(7, 5), touchedSlice(writeTbl))
+ err = writeTbl.UpdateRecord(ctx.GetTableCtx(), txn, kv.IntHandle(7), types.MakeDatums(7, 7), types.MakeDatums(7, 5), touchedSlice(writeTbl))
if err != nil {
return errors.Trace(err)
}
@@ -276,7 +270,7 @@ func checkAddPublicForAddIndex(ctx sessionctx.Context, writeTbl, publicTbl table
return errors.Trace(err)
}
// WriteOnlyTable: delete t where c1 = 6
- err = writeTbl.RemoveRecord(ctx.GetTableCtx(), kv.IntHandle(6), types.MakeDatums(6, 6))
+ err = writeTbl.RemoveRecord(ctx.GetTableCtx(), txn, kv.IntHandle(6), types.MakeDatums(6, 6))
if err != nil {
return errors.Trace(err)
}
@@ -309,20 +303,16 @@ func checkAddPublicForAddIndex(ctx sessionctx.Context, writeTbl, publicTbl table
return errors.Trace(err)
}
}
- txn, err := ctx.Txn(true)
- if err != nil {
- return errors.Trace(err)
- }
return txn.Commit(context.Background())
}
func checkDropWriteOnly(ctx sessionctx.Context, publicTbl, writeTbl table.Table) error {
// WriteOnlyTable insert t values (8, 8)
- err := sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err := newTxn(ctx)
if err != nil {
return errors.Trace(err)
}
- _, err = writeTbl.AddRecord(ctx.GetTableCtx(), types.MakeDatums(8, 8))
+ _, err = writeTbl.AddRecord(ctx.GetTableCtx(), txn, types.MakeDatums(8, 8))
if err != nil {
return errors.Trace(err)
}
@@ -333,7 +323,7 @@ func checkDropWriteOnly(ctx sessionctx.Context, publicTbl, writeTbl table.Table)
}
// WriteOnlyTable update t set c2 = 7 where c1 = 8 and c2 = 8
- err = writeTbl.UpdateRecord(context.Background(), ctx.GetTableCtx(), kv.IntHandle(8), types.MakeDatums(8, 8), types.MakeDatums(8, 7), touchedSlice(writeTbl))
+ err = writeTbl.UpdateRecord(ctx.GetTableCtx(), txn, kv.IntHandle(8), types.MakeDatums(8, 8), types.MakeDatums(8, 7), touchedSlice(writeTbl))
if err != nil {
return errors.Trace(err)
}
@@ -344,7 +334,7 @@ func checkDropWriteOnly(ctx sessionctx.Context, publicTbl, writeTbl table.Table)
}
// WriteOnlyTable delete t where c1 = 8
- err = writeTbl.RemoveRecord(ctx.GetTableCtx(), kv.IntHandle(8), types.MakeDatums(8, 7))
+ err = writeTbl.RemoveRecord(ctx.GetTableCtx(), txn, kv.IntHandle(8), types.MakeDatums(8, 7))
if err != nil {
return errors.Trace(err)
}
@@ -353,20 +343,16 @@ func checkDropWriteOnly(ctx sessionctx.Context, publicTbl, writeTbl table.Table)
if err != nil {
return errors.Trace(err)
}
- txn, err := ctx.Txn(true)
- if err != nil {
- return errors.Trace(err)
- }
return txn.Commit(context.Background())
}
func checkDropDeleteOnly(ctx sessionctx.Context, writeTbl, delTbl table.Table) error {
// WriteOnlyTable insert t values (9, 9)
- err := sessiontxn.NewTxn(context.Background(), ctx)
+ txn, err := newTxn(ctx)
if err != nil {
return errors.Trace(err)
}
- _, err = writeTbl.AddRecord(ctx.GetTableCtx(), types.MakeDatums(9, 9))
+ _, err = writeTbl.AddRecord(ctx.GetTableCtx(), txn, types.MakeDatums(9, 9))
if err != nil {
return errors.Trace(err)
}
@@ -377,7 +363,7 @@ func checkDropDeleteOnly(ctx sessionctx.Context, writeTbl, delTbl table.Table) e
}
// DeleteOnlyTable insert t values (10, 10)
- _, err = delTbl.AddRecord(ctx.GetTableCtx(), types.MakeDatums(10, 10))
+ _, err = delTbl.AddRecord(ctx.GetTableCtx(), txn, types.MakeDatums(10, 10))
if err != nil {
return errors.Trace(err)
}
@@ -388,7 +374,7 @@ func checkDropDeleteOnly(ctx sessionctx.Context, writeTbl, delTbl table.Table) e
}
// DeleteOnlyTable update t set c2 = 10 where c1 = 9
- err = delTbl.UpdateRecord(context.Background(), ctx.GetTableCtx(), kv.IntHandle(9), types.MakeDatums(9, 9), types.MakeDatums(9, 10), touchedSlice(delTbl))
+ err = delTbl.UpdateRecord(ctx.GetTableCtx(), txn, kv.IntHandle(9), types.MakeDatums(9, 9), types.MakeDatums(9, 10), touchedSlice(delTbl))
if err != nil {
return errors.Trace(err)
}
@@ -402,9 +388,5 @@ func checkDropDeleteOnly(ctx sessionctx.Context, writeTbl, delTbl table.Table) e
if err != nil {
return errors.Trace(err)
}
- txn, err := ctx.Txn(true)
- if err != nil {
- return errors.Trace(err)
- }
return txn.Commit(context.Background())
}
diff --git a/pkg/ddl/index_cop.go b/pkg/ddl/index_cop.go
index 30d6f70c8b9e0..547ee400d1a41 100644
--- a/pkg/ddl/index_cop.go
+++ b/pkg/ddl/index_cop.go
@@ -16,14 +16,10 @@ package ddl
import (
"context"
- "encoding/hex"
- "sync"
"time"
"github.com/pingcap/errors"
- "github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/ddl/copr"
- "github.com/pingcap/tidb/pkg/ddl/ingest"
sess "github.com/pingcap/tidb/pkg/ddl/session"
"github.com/pingcap/tidb/pkg/distsql"
distsqlctx "github.com/pingcap/tidb/pkg/distsql/context"
@@ -31,155 +27,19 @@ import (
"github.com/pingcap/tidb/pkg/expression"
exprctx "github.com/pingcap/tidb/pkg/expression/context"
"github.com/pingcap/tidb/pkg/kv"
- "github.com/pingcap/tidb/pkg/metrics"
"github.com/pingcap/tidb/pkg/parser/model"
- "github.com/pingcap/tidb/pkg/parser/terror"
- "github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/table/tables"
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/types"
- "github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/chunk"
"github.com/pingcap/tidb/pkg/util/codec"
"github.com/pingcap/tidb/pkg/util/collate"
- "github.com/pingcap/tidb/pkg/util/dbterror"
- "github.com/pingcap/tidb/pkg/util/logutil"
"github.com/pingcap/tidb/pkg/util/timeutil"
"github.com/pingcap/tipb/go-tipb"
kvutil "github.com/tikv/client-go/v2/util"
- "go.uber.org/zap"
)
-// copReadBatchSize is the batch size of coprocessor read.
-// It multiplies the tidb_ddl_reorg_batch_size by 10 to avoid
-// sending too many cop requests for the same handle range.
-func copReadBatchSize() int {
- return 10 * int(variable.GetDDLReorgBatchSize())
-}
-
-// copReadChunkPoolSize is the size of chunk pool, which
-// represents the max concurrent ongoing coprocessor requests.
-// It multiplies the tidb_ddl_reorg_worker_cnt by 10.
-func copReadChunkPoolSize() int {
- return 10 * int(variable.GetDDLReorgWorkerCounter())
-}
-
-// chunkSender is used to receive the result of coprocessor request.
-type chunkSender interface {
- AddTask(IndexRecordChunk)
-}
-
-type copReqSenderPool struct {
- tasksCh chan *reorgBackfillTask
- chunkSender chunkSender
- checkpointMgr *ingest.CheckpointManager
- sessPool *sess.Pool
-
- ctx context.Context
- copCtx copr.CopContext
- store kv.Storage
-
- senders []*copReqSender
- wg sync.WaitGroup
- closed bool
-
- srcChkPool chan *chunk.Chunk
-}
-
-type copReqSender struct {
- senderPool *copReqSenderPool
-
- ctx context.Context
- cancel context.CancelFunc
-}
-
-func (c *copReqSender) run() {
- p := c.senderPool
- defer p.wg.Done()
- defer util.Recover(metrics.LabelDDL, "copReqSender.run", func() {
- p.chunkSender.AddTask(IndexRecordChunk{Err: dbterror.ErrReorgPanic})
- }, false)
- sessCtx, err := p.sessPool.Get()
- if err != nil {
- logutil.Logger(p.ctx).Error("copReqSender get session from pool failed", zap.Error(err))
- p.chunkSender.AddTask(IndexRecordChunk{Err: err})
- return
- }
- se := sess.NewSession(sessCtx)
- defer p.sessPool.Put(sessCtx)
- var (
- task *reorgBackfillTask
- ok bool
- )
-
- for {
- select {
- case <-c.ctx.Done():
- return
- case task, ok = <-p.tasksCh:
- }
- if !ok {
- return
- }
- if p.checkpointMgr != nil && p.checkpointMgr.IsKeyProcessed(task.endKey) {
- logutil.Logger(p.ctx).Info("checkpoint detected, skip a cop-request task",
- zap.Int("task ID", task.id),
- zap.String("task end key", hex.EncodeToString(task.endKey)))
- continue
- }
- err := scanRecords(p, task, se)
- if err != nil {
- p.chunkSender.AddTask(IndexRecordChunk{ID: task.id, Err: err})
- return
- }
- }
-}
-
-func scanRecords(p *copReqSenderPool, task *reorgBackfillTask, se *sess.Session) error {
- logutil.Logger(p.ctx).Info("start a cop-request task",
- zap.Int("id", task.id), zap.Stringer("task", task))
-
- return wrapInBeginRollback(se, func(startTS uint64) error {
- rs, err := buildTableScan(p.ctx, p.copCtx.GetBase(), startTS, task.startKey, task.endKey)
- if err != nil {
- return err
- }
- failpoint.Inject("mockCopSenderPanic", func(val failpoint.Value) {
- if val.(bool) {
- panic("mock panic")
- }
- })
- if p.checkpointMgr != nil {
- p.checkpointMgr.Register(task.id, task.endKey)
- }
- var done bool
- startTime := time.Now()
- for !done {
- srcChk := p.getChunk()
- done, err = fetchTableScanResult(p.ctx, p.copCtx.GetBase(), rs, srcChk)
- if err != nil {
- p.recycleChunk(srcChk)
- terror.Call(rs.Close)
- return err
- }
- if p.checkpointMgr != nil {
- p.checkpointMgr.UpdateTotalKeys(task.id, srcChk.NumRows(), done)
- }
- idxRs := IndexRecordChunk{ID: task.id, Chunk: srcChk, Done: done}
- rate := float64(srcChk.MemoryUsage()) / 1024.0 / 1024.0 / time.Since(startTime).Seconds()
- metrics.AddIndexScanRate.WithLabelValues(metrics.LblAddIndex).Observe(rate)
- failpoint.Inject("mockCopSenderError", func() {
- idxRs.Err = errors.New("mock cop error")
- })
- p.chunkSender.AddTask(idxRs)
- startTime = time.Now()
- }
- terror.Call(rs.Close)
- return nil
- })
-}
-
func wrapInBeginRollback(se *sess.Session, f func(startTS uint64) error) error {
err := se.Begin(context.Background())
if err != nil {
@@ -194,81 +54,6 @@ func wrapInBeginRollback(se *sess.Session, f func(startTS uint64) error) error {
return f(startTS)
}
-func newCopReqSenderPool(ctx context.Context, copCtx copr.CopContext, store kv.Storage,
- taskCh chan *reorgBackfillTask, sessPool *sess.Pool,
- checkpointMgr *ingest.CheckpointManager) *copReqSenderPool {
- poolSize := copReadChunkPoolSize()
- srcChkPool := make(chan *chunk.Chunk, poolSize)
- for i := 0; i < poolSize; i++ {
- srcChkPool <- chunk.NewChunkWithCapacity(copCtx.GetBase().FieldTypes, copReadBatchSize())
- }
- return &copReqSenderPool{
- tasksCh: taskCh,
- ctx: ctx,
- copCtx: copCtx,
- store: store,
- senders: make([]*copReqSender, 0, variable.GetDDLReorgWorkerCounter()),
- wg: sync.WaitGroup{},
- srcChkPool: srcChkPool,
- sessPool: sessPool,
- checkpointMgr: checkpointMgr,
- }
-}
-
-func (c *copReqSenderPool) adjustSize(n int) {
- // Add some senders.
- for i := len(c.senders); i < n; i++ {
- ctx, cancel := context.WithCancel(c.ctx)
- c.senders = append(c.senders, &copReqSender{
- senderPool: c,
- ctx: ctx,
- cancel: cancel,
- })
- c.wg.Add(1)
- go c.senders[i].run()
- }
- // Remove some senders.
- if n < len(c.senders) {
- for i := n; i < len(c.senders); i++ {
- c.senders[i].cancel()
- }
- c.senders = c.senders[:n]
- }
-}
-
-func (c *copReqSenderPool) close(force bool) {
- if c.closed {
- return
- }
- logutil.Logger(c.ctx).Info("close cop-request sender pool", zap.Bool("force", force))
- if force {
- for _, w := range c.senders {
- w.cancel()
- }
- }
- // Wait for all cop-req senders to exit.
- c.wg.Wait()
- c.closed = true
-}
-
-func (c *copReqSenderPool) getChunk() *chunk.Chunk {
- chk := <-c.srcChkPool
- newCap := copReadBatchSize()
- if chk.Capacity() != newCap {
- chk = chunk.NewChunkWithCapacity(c.copCtx.GetBase().FieldTypes, newCap)
- }
- chk.Reset()
- return chk
-}
-
-// recycleChunk puts the index record slice and the chunk back to the pool for reuse.
-func (c *copReqSenderPool) recycleChunk(chk *chunk.Chunk) {
- if chk == nil {
- return
- }
- c.srcChkPool <- chk
-}
-
func buildTableScan(ctx context.Context, c *copr.CopContextBase, startTS uint64, start, end kv.Key) (distsql.SelectResult, error) {
dagPB, err := buildDAGPB(c.ExprCtx, c.DistSQLCtx, c.PushDownFlags, c.TableInfo, c.ColumnInfos)
if err != nil {
@@ -363,13 +148,14 @@ func buildDAGPB(exprCtx exprctx.BuildContext, distSQLCtx *distsqlctx.DistSQLCont
}
func constructTableScanPB(ctx exprctx.BuildContext, tblInfo *model.TableInfo, colInfos []*model.ColumnInfo) (*tipb.Executor, error) {
- tblScan := tables.BuildTableScanFromInfos(tblInfo, colInfos)
+ tblScan := tables.BuildTableScanFromInfos(tblInfo, colInfos, false)
tblScan.TableId = tblInfo.ID
err := tables.SetPBColumnsDefaultValue(ctx, tblScan.Columns, colInfos)
return &tipb.Executor{Tp: tipb.ExecType_TypeTableScan, TblScan: tblScan}, err
}
-func extractDatumByOffsets(ctx expression.EvalContext, row chunk.Row, offsets []int, expCols []*expression.Column, buf []types.Datum) []types.Datum {
+// ExtractDatumByOffsets is exported for test.
+func ExtractDatumByOffsets(ctx expression.EvalContext, row chunk.Row, offsets []int, expCols []*expression.Column, buf []types.Datum) []types.Datum {
for i, offset := range offsets {
c := expCols[offset]
row.DatumWithBuffer(offset, c.GetType(ctx), &buf[i])
@@ -377,7 +163,8 @@ func extractDatumByOffsets(ctx expression.EvalContext, row chunk.Row, offsets []
return buf
}
-func buildHandle(pkDts []types.Datum, tblInfo *model.TableInfo,
+// BuildHandle is exported for test.
+func BuildHandle(pkDts []types.Datum, tblInfo *model.TableInfo,
pkInfo *model.IndexInfo, loc *time.Location, errCtx errctx.Context) (kv.Handle, error) {
if tblInfo.IsCommonHandle {
tablecodec.TruncateIndexValues(tblInfo, pkInfo, pkDts)
diff --git a/pkg/ddl/index_cop_test.go b/pkg/ddl/index_cop_test.go
index 28f07752ab6ed..7c2f8fb2c15d3 100644
--- a/pkg/ddl/index_cop_test.go
+++ b/pkg/ddl/index_cop_test.go
@@ -50,7 +50,7 @@ func TestAddIndexFetchRowsFromCoprocessor(t *testing.T) {
endKey := startKey.PrefixNext()
txn, err := store.Begin()
require.NoError(t, err)
- copChunk := ddl.FetchChunk4Test(copCtx, tbl.(table.PhysicalTable), startKey, endKey, store, 10)
+ copChunk, err := FetchChunk4Test(copCtx, tbl.(table.PhysicalTable), startKey, endKey, store, 10)
require.NoError(t, err)
require.NoError(t, txn.Rollback())
@@ -61,7 +61,7 @@ func TestAddIndexFetchRowsFromCoprocessor(t *testing.T) {
idxDataBuf := make([]types.Datum, len(idxInfo.Columns))
for row := iter.Begin(); row != iter.End(); row = iter.Next() {
- handle, idxDatum, err := ddl.ConvertRowToHandleAndIndexDatum(tk.Session().GetExprCtx().GetEvalCtx(), handleDataBuf, idxDataBuf, row, copCtx, idxInfo.ID)
+ handle, idxDatum, err := ConvertRowToHandleAndIndexDatum(tk.Session().GetExprCtx().GetEvalCtx(), handleDataBuf, idxDataBuf, row, copCtx, idxInfo.ID)
require.NoError(t, err)
handles = append(handles, handle)
copiedIdxDatum := make([]types.Datum, len(idxDatum))
diff --git a/pkg/ddl/index_modify_test.go b/pkg/ddl/index_modify_test.go
index 7639e8c2a55f0..e0a38c17919f6 100644
--- a/pkg/ddl/index_modify_test.go
+++ b/pkg/ddl/index_modify_test.go
@@ -689,7 +689,7 @@ func TestAddGlobalIndex(t *testing.T) {
" (partition p0 values less than (10), " +
" partition p1 values less than (maxvalue));")
tk.MustExec("insert test_t1 values (1, 1)")
- tk.MustExec("alter table test_t1 add unique index p_a (a);")
+ tk.MustExec("alter table test_t1 add unique index p_a (a) global")
tk.MustExec("insert test_t1 values (2, 11)")
tbl := external.GetTableByName(t, tk, "test", "test_t1")
tblInfo := tbl.Meta()
@@ -719,7 +719,7 @@ func TestAddGlobalIndex(t *testing.T) {
" (partition p0 values less than (10), " +
" partition p1 values less than (maxvalue));")
tk.MustExec("insert test_t2 values (1, 1)")
- tk.MustExec("alter table test_t2 add primary key (a) nonclustered;")
+ tk.MustExec("alter table test_t2 add primary key (a) nonclustered global")
tk.MustExec("insert test_t2 values (2, 11)")
tbl = external.GetTableByName(t, tk, "test", "test_t2")
tblInfo = tbl.Meta()
@@ -744,6 +744,24 @@ func TestAddGlobalIndex(t *testing.T) {
checkGlobalIndexRow(t, tk.Session(), tblInfo, indexInfo, pid, idxVals, rowVals)
require.NoError(t, txn.Commit(context.Background()))
+
+ // `sanity_check.go` will check the del_range numbers are correct or not.
+ // normal index
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(a int, b int) partition by hash(b) partitions 64")
+ tk.MustExec("alter table t add unique index idx(a) global")
+
+ // meets duplicate
+ tk.MustExec("drop table t")
+ tk.MustExec("create table t(a int, b int) partition by hash(b) partitions 64")
+ tk.MustExec("insert into t values (1, 2), (1, 3)")
+ // Duplicate
+ tk.MustContainErrMsg("alter table t add unique index idx(a) global", "[kv:1062]Duplicate entry '1' for key 't.idx'")
+
+ // with multi schema change
+ tk.MustExec("drop table t")
+ tk.MustExec("create table t(a int, b int) partition by hash(b) partitions 64")
+ tk.MustExec("alter table t add unique index idx(a) global, add index idx1(b)")
}
// checkGlobalIndexRow reads one record from global index and check. Only support int handle.
@@ -1029,7 +1047,7 @@ func TestAddIndexUniqueFailOnDuplicate(t *testing.T) {
tk.MustExec("create table t (a bigint primary key clustered, b int);")
// The subtask execution order is not guaranteed in distributed reorg. We need to disable it first.
tk.MustExec("set @@global.tidb_enable_dist_task = 0;")
- tk.MustExec("set @@global.tidb_ddl_reorg_worker_cnt = 1;")
+ tk.MustExec("set @@tidb_ddl_reorg_worker_cnt = 1;")
for i := 1; i <= 12; i++ {
tk.MustExec("insert into t values (?, ?)", i, i)
}
diff --git a/pkg/ddl/index_test.go b/pkg/ddl/index_test.go
index 5e5fe1a0564a6..ba89046bf6fa7 100644
--- a/pkg/ddl/index_test.go
+++ b/pkg/ddl/index_test.go
@@ -31,7 +31,6 @@ func TestDecodeAddIndexArgsCompatibility(t *testing.T) {
indexPartSpecifications [][]*ast.IndexPartSpecification
indexOptions []*ast.IndexOption
hiddenCols [][]*model.ColumnInfo
- globals []bool
}{
{
raw: json.RawMessage(`[
@@ -74,7 +73,6 @@ false]`),
},
indexOptions: []*ast.IndexOption{nil},
hiddenCols: [][]*model.ColumnInfo{{}},
- globals: []bool{false},
},
{
raw: json.RawMessage(`[
@@ -134,19 +132,17 @@ false]`),
},
indexOptions: []*ast.IndexOption{nil, nil},
hiddenCols: [][]*model.ColumnInfo{{}, {}},
- globals: []bool{false, false},
},
}
for _, c := range cases {
job := &model.Job{RawArgs: c.raw}
- uniques, indexNames, specs, indexOptions, hiddenCols, globals, err := decodeAddIndexArgs(job)
+ uniques, indexNames, specs, indexOptions, hiddenCols, err := decodeAddIndexArgs(job)
require.NoError(t, err)
require.Equal(t, c.uniques, uniques)
require.Equal(t, c.indexNames, indexNames)
require.Equal(t, c.indexPartSpecifications, specs)
require.Equal(t, c.indexOptions, indexOptions)
require.Equal(t, c.hiddenCols, hiddenCols)
- require.Equal(t, c.globals, globals)
}
}
diff --git a/pkg/ddl/ingest/BUILD.bazel b/pkg/ddl/ingest/BUILD.bazel
index 82c5b95074937..d9cffd0c1a3a8 100644
--- a/pkg/ddl/ingest/BUILD.bazel
+++ b/pkg/ddl/ingest/BUILD.bazel
@@ -31,7 +31,6 @@ go_library(
"//pkg/lightning/checkpoints",
"//pkg/lightning/common",
"//pkg/lightning/config",
- "//pkg/lightning/errormanager",
"//pkg/lightning/log",
"//pkg/meta",
"//pkg/parser/model",
@@ -76,12 +75,9 @@ go_test(
shard_count = 20,
deps = [
"//pkg/config",
- "//pkg/ddl",
"//pkg/ddl/ingest/testutil",
"//pkg/ddl/session",
"//pkg/ddl/testutil",
- "//pkg/ddl/util/callback",
- "//pkg/domain",
"//pkg/errno",
"//pkg/parser/model",
"//pkg/testkit",
diff --git a/pkg/ddl/ingest/backend.go b/pkg/ddl/ingest/backend.go
index 6a8a4e0e6666a..5098192551fb1 100644
--- a/pkg/ddl/ingest/backend.go
+++ b/pkg/ddl/ingest/backend.go
@@ -29,7 +29,6 @@ import (
"github.com/pingcap/tidb/pkg/lightning/backend/local"
"github.com/pingcap/tidb/pkg/lightning/common"
lightning "github.com/pingcap/tidb/pkg/lightning/config"
- "github.com/pingcap/tidb/pkg/lightning/errormanager"
"github.com/pingcap/tidb/pkg/lightning/log"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
@@ -95,7 +94,7 @@ type litBackendCtx struct {
tbl table.Table
backend *local.Backend
ctx context.Context
- cfg *lightning.Config
+ cfg *local.BackendConfig
sysVars map[string]string
flushing atomic.Bool
@@ -148,8 +147,7 @@ func (bc *litBackendCtx) CollectRemoteDuplicateRows(indexID int64, tbl table.Tab
}
func (bc *litBackendCtx) collectRemoteDuplicateRows(indexID int64, tbl table.Table) error {
- errorMgr := errormanager.New(nil, bc.cfg, log.Logger{Logger: logutil.Logger(bc.ctx)})
- dupeController := bc.backend.GetDupeController(bc.cfg.TikvImporter.RangeConcurrency*2, errorMgr)
+ dupeController := bc.backend.GetDupeController(bc.cfg.WorkerConcurrency, nil)
hasDupe, err := dupeController.CollectRemoteDuplicateRows(bc.ctx, tbl, tbl.Meta().Name.L, &encode.SessionOptions{
SQLMode: mysql.ModeStrictAllTables,
SysVars: bc.sysVars,
diff --git a/pkg/ddl/ingest/backend_mgr.go b/pkg/ddl/ingest/backend_mgr.go
index 068047e5a8710..74300de29869f 100644
--- a/pkg/ddl/ingest/backend_mgr.go
+++ b/pkg/ddl/ingest/backend_mgr.go
@@ -17,6 +17,7 @@ package ingest
import (
"context"
"math"
+ "net"
"os"
"path/filepath"
"strconv"
@@ -24,11 +25,11 @@ import (
"time"
"github.com/pingcap/failpoint"
+ "github.com/pingcap/tidb/pkg/config"
ddllogutil "github.com/pingcap/tidb/pkg/ddl/logutil"
"github.com/pingcap/tidb/pkg/lightning/backend/local"
- "github.com/pingcap/tidb/pkg/lightning/config"
+ "github.com/pingcap/tidb/pkg/lightning/common"
"github.com/pingcap/tidb/pkg/util/logutil"
- kvutil "github.com/tikv/client-go/v2/util"
pd "github.com/tikv/pd/client"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/atomic"
@@ -49,6 +50,7 @@ type BackendCtxMgr interface {
etcdClient *clientv3.Client,
pdSvcDiscovery pd.ServiceDiscovery,
resourceGroupName string,
+ importConc int,
) (BackendCtx, error)
Unregister(jobID int64)
// EncodeJobSortPath encodes the job ID to the local disk sort path.
@@ -114,6 +116,7 @@ func (m *litBackendCtxMgr) Register(
etcdClient *clientv3.Client,
pdSvcDiscovery pd.ServiceDiscovery,
resourceGroupName string,
+ concurrency int,
) (BackendCtx, error) {
bc, exist := m.Load(jobID)
if exist {
@@ -131,7 +134,7 @@ func (m *litBackendCtxMgr) Register(
logutil.Logger(ctx).Error(LitErrCreateDirFail, zap.Error(err))
return nil, err
}
- cfg, err := genConfig(ctx, sortPath, m.memRoot, hasUnique, resourceGroupName)
+ cfg, err := genConfig(ctx, sortPath, m.memRoot, hasUnique, resourceGroupName, concurrency)
if err != nil {
logutil.Logger(ctx).Warn(LitWarnConfigError, zap.Int64("job ID", jobID), zap.Error(err))
return nil, err
@@ -150,7 +153,7 @@ func (m *litBackendCtxMgr) Register(
return nil, err
}
- bcCtx := newBackendContext(ctx, jobID, bd, cfg.lightning, defaultImportantVariables, m.memRoot, m.diskRoot, etcdClient)
+ bcCtx := newBackendContext(ctx, jobID, bd, cfg, defaultImportantVariables, m.memRoot, m.diskRoot, etcdClient)
m.backends.m[jobID] = bcCtx
m.memRoot.Consume(structSizeBackendCtx)
m.backends.mu.Unlock()
@@ -169,23 +172,26 @@ func (m *litBackendCtxMgr) EncodeJobSortPath(jobID int64) string {
func createLocalBackend(
ctx context.Context,
- cfg *litConfig,
+ cfg *local.BackendConfig,
pdSvcDiscovery pd.ServiceDiscovery,
) (*local.Backend, error) {
- tls, err := cfg.lightning.ToTLS()
+ tidbCfg := config.GetGlobalConfig()
+ tls, err := common.NewTLS(
+ tidbCfg.Security.ClusterSSLCA,
+ tidbCfg.Security.ClusterSSLCert,
+ tidbCfg.Security.ClusterSSLKey,
+ net.JoinHostPort("127.0.0.1", strconv.Itoa(int(tidbCfg.Status.StatusPort))),
+ nil, nil, nil,
+ )
if err != nil {
logutil.Logger(ctx).Error(LitErrCreateBackendFail, zap.Error(err))
return nil, err
}
ddllogutil.DDLIngestLogger().Info("create local backend for adding index",
- zap.String("sortDir", cfg.lightning.TikvImporter.SortedKVDir),
- zap.String("keyspaceName", cfg.keyspaceName))
- // We disable the switch TiKV mode feature for now,
- // because the impact is not fully tested.
- var raftKV2SwitchModeDuration time.Duration
- backendConfig := local.NewBackendConfig(cfg.lightning, int(litRLimit), cfg.keyspaceName, cfg.resourceGroup, kvutil.ExplicitTypeDDL, raftKV2SwitchModeDuration)
- return local.NewBackend(ctx, tls, backendConfig, pdSvcDiscovery)
+ zap.String("sortDir", cfg.LocalStoreDir),
+ zap.String("keyspaceName", cfg.KeyspaceName))
+ return local.NewBackend(ctx, tls, *cfg, pdSvcDiscovery)
}
const checkpointUpdateInterval = 10 * time.Minute
@@ -194,7 +200,7 @@ func newBackendContext(
ctx context.Context,
jobID int64,
be *local.Backend,
- cfg *config.Config,
+ cfg *local.BackendConfig,
vars map[string]string,
memRoot MemRoot,
diskRoot DiskRoot,
diff --git a/pkg/ddl/ingest/checkpoint.go b/pkg/ddl/ingest/checkpoint.go
index 2d1536905736c..0575dd27af966 100644
--- a/pkg/ddl/ingest/checkpoint.go
+++ b/pkg/ddl/ingest/checkpoint.go
@@ -156,9 +156,9 @@ func (s *CheckpointManager) IsKeyProcessed(end kv.Key) bool {
return s.localDataIsValid && len(s.flushedKeyLowWatermark) > 0 && end.Cmp(s.flushedKeyLowWatermark) <= 0
}
-// LastProcessedKey finds the last processed key in checkpoint.
-// If there is no processed key, it returns nil.
-func (s *CheckpointManager) LastProcessedKey() kv.Key {
+// NextKeyToProcess finds the next unprocessed key in checkpoint.
+// If there is no such key, it returns nil.
+func (s *CheckpointManager) NextKeyToProcess() kv.Key {
s.mu.Lock()
defer s.mu.Unlock()
diff --git a/pkg/ddl/ingest/config.go b/pkg/ddl/ingest/config.go
index c6b59a531c5cd..58686c1d918c2 100644
--- a/pkg/ddl/ingest/config.go
+++ b/pkg/ddl/ingest/config.go
@@ -17,78 +17,91 @@ package ingest
import (
"context"
"net"
+ "runtime"
"strconv"
"sync/atomic"
tidb "github.com/pingcap/tidb/pkg/config"
"github.com/pingcap/tidb/pkg/lightning/backend"
+ "github.com/pingcap/tidb/pkg/lightning/backend/local"
"github.com/pingcap/tidb/pkg/lightning/checkpoints"
"github.com/pingcap/tidb/pkg/lightning/common"
lightning "github.com/pingcap/tidb/pkg/lightning/config"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/util/logutil"
"github.com/pingcap/tidb/pkg/util/size"
+ kvutil "github.com/tikv/client-go/v2/util"
"go.uber.org/zap"
)
// ImporterRangeConcurrencyForTest is only used for test.
var ImporterRangeConcurrencyForTest *atomic.Int32
-// litConfig is the configuration for the lightning local backend used in DDL.
-type litConfig struct {
- lightning *lightning.Config
- keyspaceName string
- isRaftKV2 bool
- resourceGroup string
-}
-
func genConfig(
ctx context.Context,
jobSortPath string,
memRoot MemRoot,
unique bool,
resourceGroup string,
-) (*litConfig, error) {
- tidbCfg := tidb.GetGlobalConfig()
- cfg := lightning.NewConfig()
- cfg.TikvImporter.Backend = lightning.BackendLocal
+ concurrency int,
+) (*local.BackendConfig, error) {
+ cfg := &local.BackendConfig{
+ LocalStoreDir: jobSortPath,
+ ResourceGroupName: resourceGroup,
+ MaxConnPerStore: concurrency,
+ WorkerConcurrency: concurrency * 2,
+ KeyspaceName: tidb.GetGlobalKeyspaceName(),
+ // We disable the switch TiKV mode feature for now, because the impact is not
+ // fully tested.
+ ShouldCheckWriteStall: true,
+
+ // lighting default values
+ CheckpointEnabled: true,
+ BlockSize: lightning.DefaultBlockSize,
+ KVWriteBatchSize: lightning.KVWriteBatchSize,
+ RegionSplitBatchSize: lightning.DefaultRegionSplitBatchSize,
+ RegionSplitConcurrency: runtime.GOMAXPROCS(0),
+ MemTableSize: lightning.DefaultEngineMemCacheSize,
+ LocalWriterMemCacheSize: lightning.DefaultLocalWriterMemCacheSize,
+ ShouldCheckTiKV: true,
+ MaxOpenFiles: int(litRLimit),
+ PausePDSchedulerScope: lightning.PausePDSchedulerScopeTable,
+ TaskType: kvutil.ExplicitTypeDDL,
+ DisableAutomaticCompactions: true,
+ }
// Each backend will build a single dir in lightning dir.
- cfg.TikvImporter.SortedKVDir = jobSortPath
if ImporterRangeConcurrencyForTest != nil {
- cfg.TikvImporter.RangeConcurrency = int(ImporterRangeConcurrencyForTest.Load())
- } else {
- cfg.TikvImporter.RangeConcurrency = int(variable.GetDDLReorgWorkerCounter())
- }
- err := cfg.AdjustForDDL()
- if err != nil {
- logutil.Logger(ctx).Warn(LitWarnConfigError, zap.Error(err))
- return nil, err
+ cfg.WorkerConcurrency = int(ImporterRangeConcurrencyForTest.Load()) * 2
}
adjustImportMemory(ctx, memRoot, cfg)
- cfg.Checkpoint.Enable = true
if unique {
- cfg.Conflict.Strategy = lightning.ErrorOnDup
- cfg.Conflict.Threshold = lightning.DefaultRecordDuplicateThreshold
+ cfg.DupeDetectEnabled = true
+ cfg.DuplicateDetectOpt = common.DupDetectOpt{ReportErrOnDup: true}
} else {
- cfg.Conflict.Strategy = lightning.NoneOnDup
+ cfg.DupeDetectEnabled = false
}
- cfg.TiDB.Host = "127.0.0.1"
- cfg.TiDB.StatusPort = int(tidbCfg.Status.StatusPort)
- // Set TLS related information
- cfg.Security.CAPath = tidbCfg.Security.ClusterSSLCA
- cfg.Security.CertPath = tidbCfg.Security.ClusterSSLCert
- cfg.Security.KeyPath = tidbCfg.Security.ClusterSSLKey
- // in DDL scenario, we don't switch import mode
- cfg.Cron.SwitchMode = lightning.Duration{Duration: 0}
-
- c := &litConfig{
- lightning: cfg,
- keyspaceName: tidb.GetGlobalKeyspaceName(),
- isRaftKV2: false,
- resourceGroup: resourceGroup,
+
+ return cfg, nil
+}
+
+// CopReadBatchSize is the batch size of coprocessor read.
+// It multiplies the tidb_ddl_reorg_batch_size by 10 to avoid
+// sending too many cop requests for the same handle range.
+func CopReadBatchSize(hintSize int) int {
+ if hintSize > 0 {
+ return hintSize
}
+ return 10 * int(variable.GetDDLReorgBatchSize())
+}
- return c, nil
+// CopReadChunkPoolSize is the size of chunk pool, which
+// represents the max concurrent ongoing coprocessor requests.
+// It multiplies the tidb_ddl_reorg_worker_cnt by 10.
+func CopReadChunkPoolSize(hintConc int) int {
+ if hintConc > 0 {
+ return 10 * hintConc
+ }
+ return 10 * int(variable.GetDDLReorgWorkerCounter())
}
// NewDDLTLS creates a common.TLS from the tidb config for DDL.
@@ -124,19 +137,19 @@ func generateLocalEngineConfig(ts uint64) *backend.EngineConfig {
}
// adjustImportMemory adjusts the lightning memory parameters according to the memory root's max limitation.
-func adjustImportMemory(ctx context.Context, memRoot MemRoot, cfg *lightning.Config) {
+func adjustImportMemory(ctx context.Context, memRoot MemRoot, cfg *local.BackendConfig) {
var scale int64
// Try aggressive resource usage successful.
if tryAggressiveMemory(ctx, memRoot, cfg) {
return
}
- defaultMemSize := int64(cfg.TikvImporter.LocalWriterMemCacheSize) * int64(cfg.TikvImporter.RangeConcurrency)
- defaultMemSize += 4 * int64(cfg.TikvImporter.EngineMemCacheSize)
+ defaultMemSize := int64(int(cfg.LocalWriterMemCacheSize) * cfg.WorkerConcurrency / 2)
+ defaultMemSize += 4 * int64(cfg.MemTableSize)
logutil.Logger(ctx).Info(LitInfoInitMemSetting,
- zap.Int64("local writer memory cache size", int64(cfg.TikvImporter.LocalWriterMemCacheSize)),
- zap.Int64("engine memory cache size", int64(cfg.TikvImporter.EngineMemCacheSize)),
- zap.Int("range concurrency", cfg.TikvImporter.RangeConcurrency))
+ zap.Int64("local writer memory cache size", cfg.LocalWriterMemCacheSize),
+ zap.Int("engine memory cache size", cfg.MemTableSize),
+ zap.Int("worker concurrency", cfg.WorkerConcurrency))
maxLimit := memRoot.MaxMemoryQuota()
scale = defaultMemSize / maxLimit
@@ -145,28 +158,28 @@ func adjustImportMemory(ctx context.Context, memRoot MemRoot, cfg *lightning.Con
return
}
- cfg.TikvImporter.LocalWriterMemCacheSize /= lightning.ByteSize(scale)
- cfg.TikvImporter.EngineMemCacheSize /= lightning.ByteSize(scale)
+ cfg.LocalWriterMemCacheSize /= scale
+ cfg.MemTableSize /= int(scale)
logutil.Logger(ctx).Info(LitInfoChgMemSetting,
- zap.Int64("local writer memory cache size", int64(cfg.TikvImporter.LocalWriterMemCacheSize)),
- zap.Int64("engine memory cache size", int64(cfg.TikvImporter.EngineMemCacheSize)),
- zap.Int("range concurrency", cfg.TikvImporter.RangeConcurrency))
+ zap.Int64("local writer memory cache size", cfg.LocalWriterMemCacheSize),
+ zap.Int("engine memory cache size", cfg.MemTableSize),
+ zap.Int("worker concurrency", cfg.WorkerConcurrency))
}
// tryAggressiveMemory lightning memory parameters according memory root's max limitation.
-func tryAggressiveMemory(ctx context.Context, memRoot MemRoot, cfg *lightning.Config) bool {
+func tryAggressiveMemory(ctx context.Context, memRoot MemRoot, cfg *local.BackendConfig) bool {
var defaultMemSize int64
- defaultMemSize = int64(int(cfg.TikvImporter.LocalWriterMemCacheSize) * cfg.TikvImporter.RangeConcurrency)
- defaultMemSize += int64(cfg.TikvImporter.EngineMemCacheSize)
+ defaultMemSize = int64(int(cfg.LocalWriterMemCacheSize) * cfg.WorkerConcurrency / 2)
+ defaultMemSize += int64(cfg.MemTableSize)
if (defaultMemSize + memRoot.CurrentUsage()) > memRoot.MaxMemoryQuota() {
return false
}
logutil.Logger(ctx).Info(LitInfoChgMemSetting,
- zap.Int64("local writer memory cache size", int64(cfg.TikvImporter.LocalWriterMemCacheSize)),
- zap.Int64("engine memory cache size", int64(cfg.TikvImporter.EngineMemCacheSize)),
- zap.Int("range concurrency", cfg.TikvImporter.RangeConcurrency))
+ zap.Int64("local writer memory cache size", cfg.LocalWriterMemCacheSize),
+ zap.Int("engine memory cache size", cfg.MemTableSize),
+ zap.Int("worker concurrency", cfg.WorkerConcurrency))
return true
}
diff --git a/pkg/ddl/ingest/engine.go b/pkg/ddl/ingest/engine.go
index b48b45f59a4c8..7bcdebce101f8 100644
--- a/pkg/ddl/ingest/engine.go
+++ b/pkg/ddl/ingest/engine.go
@@ -23,7 +23,6 @@ import (
"github.com/pingcap/tidb/pkg/lightning/backend"
"github.com/pingcap/tidb/pkg/lightning/backend/kv"
"github.com/pingcap/tidb/pkg/lightning/common"
- "github.com/pingcap/tidb/pkg/lightning/config"
"github.com/pingcap/tidb/pkg/util/generic"
"github.com/pingcap/tidb/pkg/util/logutil"
"go.uber.org/zap"
@@ -55,7 +54,6 @@ type engineInfo struct {
uuid uuid.UUID
cfg *backend.EngineConfig
- litCfg *config.Config
writerCache generic.SyncMap[int, backend.EngineWriter]
memRoot MemRoot
flushLock *sync.RWMutex
@@ -67,7 +65,6 @@ func newEngineInfo(
jobID, indexID int64,
unique bool,
cfg *backend.EngineConfig,
- litCfg *config.Config,
en *backend.OpenedEngine,
uuid uuid.UUID,
memRoot MemRoot,
@@ -78,7 +75,6 @@ func newEngineInfo(
indexID: indexID,
unique: unique,
cfg: cfg,
- litCfg: litCfg,
openedEngine: en,
uuid: uuid,
writerCache: generic.NewSyncMap[int, backend.EngineWriter](4),
diff --git a/pkg/ddl/ingest/engine_mgr.go b/pkg/ddl/ingest/engine_mgr.go
index b856f7496c37f..a36dad357d74e 100644
--- a/pkg/ddl/ingest/engine_mgr.go
+++ b/pkg/ddl/ingest/engine_mgr.go
@@ -79,7 +79,6 @@ func (bc *litBackendCtx) Register(indexIDs []int64, uniques []bool, tbl table.Ta
indexID,
uniques[i],
cfg,
- bc.cfg,
openedEngine,
openedEngine.GetEngineUUID(),
bc.memRoot,
diff --git a/pkg/ddl/ingest/integration_test.go b/pkg/ddl/ingest/integration_test.go
index 6cbc5b4380362..e47e955b9450c 100644
--- a/pkg/ddl/ingest/integration_test.go
+++ b/pkg/ddl/ingest/integration_test.go
@@ -21,12 +21,9 @@ import (
"testing"
"github.com/pingcap/failpoint"
- "github.com/pingcap/tidb/pkg/ddl"
"github.com/pingcap/tidb/pkg/ddl/ingest"
ingesttestutil "github.com/pingcap/tidb/pkg/ddl/ingest/testutil"
"github.com/pingcap/tidb/pkg/ddl/testutil"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
- "github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/testkit"
@@ -90,7 +87,7 @@ func TestIngestError(t *testing.T) {
tk.MustExec("set global tidb_enable_dist_task = 0")
defer ingesttestutil.InjectMockBackendMgr(t, store)()
- tk.MustExec("set @@global.tidb_ddl_reorg_worker_cnt = 1;")
+ tk.MustExec("set @@tidb_ddl_reorg_worker_cnt = 1;")
tk.MustExec("create table t (a int primary key, b int);")
for i := 0; i < 4; i++ {
tk.MustExec(fmt.Sprintf("insert into t values (%d, %d);", i*10000, i*10000))
@@ -158,10 +155,8 @@ func TestAddIndexIngestCancel(t *testing.T) {
tk.MustExec("create table t (a int, b int);")
tk.MustExec("insert into t (a, b) values (1, 1), (2, 2), (3, 3);")
- defHook := dom.DDL().GetHook()
- customHook := newTestCallBack(t, dom)
cancelled := false
- customHook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if cancelled {
return
}
@@ -178,35 +173,15 @@ func TestAddIndexIngestCancel(t *testing.T) {
cancelled = true
}
}
- }
- dom.DDL().SetHook(customHook)
+ })
tk.MustGetErrCode("alter table t add index idx(b);", errno.ErrCancelledDDLJob)
require.True(t, cancelled)
- dom.DDL().SetHook(defHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore")
ok, err := ingest.LitBackCtxMgr.CheckMoreTasksAvailable()
require.NoError(t, err)
require.True(t, ok)
}
-type testCallback struct {
- ddl.Callback
- OnJobRunBeforeExported func(job *model.Job)
-}
-
-func newTestCallBack(t *testing.T, dom *domain.Domain) *testCallback {
- defHookFactory, err := ddl.GetCustomizedHook("default_hook")
- require.NoError(t, err)
- return &testCallback{
- Callback: defHookFactory(dom),
- }
-}
-
-func (c *testCallback) OnJobRunBefore(job *model.Job) {
- if c.OnJobRunBeforeExported != nil {
- c.OnJobRunBeforeExported(job)
- }
-}
-
func TestIngestPartitionRowCount(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
@@ -240,7 +215,7 @@ func TestAddIndexIngestClientError(t *testing.T) {
}
func TestAddIndexCancelOnNoneState(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tkCancel := testkit.NewTestKit(t, store)
defer ingesttestutil.InjectMockBackendMgr(t, store)()
@@ -249,16 +224,14 @@ func TestAddIndexCancelOnNoneState(t *testing.T) {
tk.MustExec(`create table t (c1 int, c2 int, c3 int)`)
tk.MustExec("insert into t values(1, 1, 1);")
- hook := &callback.TestDDLCallback{Do: dom}
first := true
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.SchemaState == model.StateNone && first {
_, err := tkCancel.Exec(fmt.Sprintf("admin cancel ddl jobs %d", job.ID))
assert.NoError(t, err)
first = false
}
- }
- dom.DDL().SetHook(hook.Clone())
+ })
tk.MustGetErrCode("alter table t add index idx1(c1)", errno.ErrCancelledDDLJob)
available, err := ingest.LitBackCtxMgr.CheckMoreTasksAvailable()
require.NoError(t, err)
diff --git a/pkg/ddl/ingest/mock.go b/pkg/ddl/ingest/mock.go
index 4d9261ecfc672..8d5fe8744dccf 100644
--- a/pkg/ddl/ingest/mock.go
+++ b/pkg/ddl/ingest/mock.go
@@ -56,7 +56,8 @@ func (m *MockBackendCtxMgr) CheckMoreTasksAvailable() (bool, error) {
}
// Register implements BackendCtxMgr.Register interface.
-func (m *MockBackendCtxMgr) Register(ctx context.Context, jobID int64, unique bool, etcdClient *clientv3.Client, pdSvcDiscovery pd.ServiceDiscovery, resourceGroupName string) (BackendCtx, error) {
+func (m *MockBackendCtxMgr) Register(ctx context.Context, jobID int64, unique bool, etcdClient *clientv3.Client,
+ pdSvcDiscovery pd.ServiceDiscovery, resourceGroupName string, importConc int) (BackendCtx, error) {
logutil.DDLIngestLogger().Info("mock backend mgr register", zap.Int64("jobID", jobID))
if mockCtx, ok := m.runningJobs[jobID]; ok {
return mockCtx, nil
diff --git a/pkg/ddl/integration_test.go b/pkg/ddl/integration_test.go
index e323154dc1bb9..581e6bad3d5e2 100644
--- a/pkg/ddl/integration_test.go
+++ b/pkg/ddl/integration_test.go
@@ -17,27 +17,22 @@ package ddl_test
import (
"testing"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/stretchr/testify/require"
)
func TestDDLStatementsBackFill(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test;")
needReorg := false
- callback := &callback.TestDDLCallback{
- Do: dom,
- }
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.SchemaState == model.StateWriteReorganization {
needReorg = true
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- dom.DDL().SetHook(callback)
+ })
tk.MustExec("create table t (a int, b char(65));")
tk.MustExec("insert into t values (1, '123');")
testCases := []struct {
diff --git a/pkg/ddl/job_table.go b/pkg/ddl/job_scheduler.go
similarity index 89%
rename from pkg/ddl/job_table.go
rename to pkg/ddl/job_scheduler.go
index 71de6aea62406..a242e45aae41f 100644
--- a/pkg/ddl/job_table.go
+++ b/pkg/ddl/job_scheduler.go
@@ -33,8 +33,8 @@ import (
"github.com/pingcap/kvproto/pkg/kvrpcpb"
"github.com/pingcap/tidb/pkg/ddl/ingest"
"github.com/pingcap/tidb/pkg/ddl/logutil"
+ "github.com/pingcap/tidb/pkg/ddl/serverstate"
sess "github.com/pingcap/tidb/pkg/ddl/session"
- "github.com/pingcap/tidb/pkg/ddl/syncer"
"github.com/pingcap/tidb/pkg/ddl/systable"
"github.com/pingcap/tidb/pkg/ddl/util"
"github.com/pingcap/tidb/pkg/kv"
@@ -58,7 +58,6 @@ var (
// is a new DDL job.
addingDDLJobNotifyKey = "/tidb/ddl/add_ddl_job_general"
dispatchLoopWaitingDuration = 1 * time.Second
- localWorkerWaitingDuration = 10 * time.Millisecond
schedulerLoopRetryInterval = time.Second
)
@@ -77,8 +76,6 @@ func (t jobType) String() string {
return "general"
case jobTypeReorg:
return "reorg"
- case jobTypeLocal:
- return "local"
}
return "unknown job type: " + strconv.Itoa(int(t))
}
@@ -86,7 +83,6 @@ func (t jobType) String() string {
const (
jobTypeGeneral jobType = iota
jobTypeReorg
- jobTypeLocal
)
type ownerListener struct {
@@ -106,12 +102,15 @@ func (l *ownerListener) OnBecomeOwner() {
sysTblMgr: sysTblMgr,
schemaLoader: l.ddl.schemaLoader,
minJobIDRefresher: l.ddl.minJobIDRefresher,
+ unSyncedTracker: newUnSyncedJobTracker(),
+ schemaVerMgr: newSchemaVersionManager(l.ddl.store),
ddlCtx: l.ddl.ddlCtx,
ddlJobNotifyCh: l.ddl.ddlJobNotifyCh,
sessPool: l.ddl.sessPool,
delRangeMgr: l.ddl.delRangeMgr,
}
+ l.ddl.reorgCtx.setOwnerTS(time.Now().Unix())
l.scheduler.start()
}
@@ -132,6 +131,8 @@ type jobScheduler struct {
sysTblMgr systable.Manager
schemaLoader SchemaLoader
minJobIDRefresher *systable.MinJobIDRefresher
+ unSyncedTracker *unSyncedJobTracker
+ schemaVerMgr *schemaVersionManager
// those fields are created or initialized on start
reorgWorkerPool *workerPool
@@ -167,7 +168,7 @@ func (s *jobScheduler) start() {
s.generalDDLWorkerPool = newDDLWorkerPool(pools.NewResourcePool(workerFactory(generalWorker), generalWorkerCnt, generalWorkerCnt, 0), jobTypeGeneral)
s.wg.RunWithLog(s.scheduleLoop)
s.wg.RunWithLog(func() {
- s.schemaSyncer.SyncJobSchemaVerLoop(s.schCtx)
+ s.schemaVerSyncer.SyncJobSchemaVerLoop(s.schCtx)
})
}
@@ -193,7 +194,7 @@ func hasSysDB(job *model.Job) bool {
}
func (s *jobScheduler) processJobDuringUpgrade(sess *sess.Session, job *model.Job) (isRunnable bool, err error) {
- if s.stateSyncer.IsUpgradingState() {
+ if s.serverStateSyncer.IsUpgradingState() {
if job.IsPaused() {
return false, nil
}
@@ -245,21 +246,6 @@ func (s *jobScheduler) processJobDuringUpgrade(sess *sess.Session, job *model.Jo
return true, nil
}
-// startLocalWorkerLoop starts the local worker loop to run the DDL job of v2.
-func (d *ddl) startLocalWorkerLoop() {
- for {
- select {
- case <-d.ctx.Done():
- return
- case jobW, ok := <-d.localJobCh:
- if !ok {
- return
- }
- d.delivery2LocalWorker(d.localWorkerPool, jobW)
- }
- }
-}
-
func (s *jobScheduler) scheduleLoop() {
const retryInterval = 3 * time.Second
for {
@@ -296,8 +282,6 @@ func (s *jobScheduler) schedule() error {
}
ticker := time.NewTicker(dispatchLoopWaitingDuration)
defer ticker.Stop()
- // TODO move waitSchemaSyncedController out of ddlCtx.
- s.clearOnceMap()
s.mustReloadSchemas()
for {
@@ -339,11 +323,11 @@ func (s *jobScheduler) schedule() error {
// TODO make it run in a separate routine.
func (s *jobScheduler) checkAndUpdateClusterState(needUpdate bool) error {
select {
- case _, ok := <-s.stateSyncer.WatchChan():
+ case _, ok := <-s.serverStateSyncer.WatchChan():
if !ok {
- // TODO stateSyncer should only be started when we are the owner, and use
+ // TODO serverStateSyncer should only be started when we are the owner, and use
// the context of scheduler, will refactor it later.
- s.stateSyncer.Rewatch(s.ddlCtx.ctx)
+ s.serverStateSyncer.Rewatch(s.ddlCtx.ctx)
}
default:
if !needUpdate {
@@ -351,17 +335,17 @@ func (s *jobScheduler) checkAndUpdateClusterState(needUpdate bool) error {
}
}
- oldState := s.stateSyncer.IsUpgradingState()
- stateInfo, err := s.stateSyncer.GetGlobalState(s.schCtx)
+ oldState := s.serverStateSyncer.IsUpgradingState()
+ stateInfo, err := s.serverStateSyncer.GetGlobalState(s.schCtx)
if err != nil {
logutil.DDLLogger().Warn("get global state failed", zap.Error(err))
return errors.Trace(err)
}
logutil.DDLLogger().Info("get global state and global state change",
- zap.Bool("oldState", oldState), zap.Bool("currState", s.stateSyncer.IsUpgradingState()))
+ zap.Bool("oldState", oldState), zap.Bool("currState", s.serverStateSyncer.IsUpgradingState()))
ownerOp := owner.OpNone
- if stateInfo.State == syncer.StateUpgrading {
+ if stateInfo.State == serverstate.StateUpgrading {
ownerOp = owner.OpSyncUpgradingState
}
err = s.ownerManager.SetOwnerOpValue(s.schCtx, ownerOp)
@@ -465,51 +449,6 @@ func (s *jobScheduler) mustReloadSchemas() {
}
}
-// delivery2LocalWorker runs the DDL job of v2 in local.
-// send the result to the error channels in the task.
-// delivery2Localworker owns the worker, need to put it back to the pool in this function.
-func (d *ddl) delivery2LocalWorker(pool *workerPool, jobW *JobWrapper) {
- job := jobW.Job
- wk, err := pool.get()
- if err != nil {
- jobW.NotifyError(err)
- return
- }
- for wk == nil {
- select {
- case <-d.ctx.Done():
- return
- case <-time.After(localWorkerWaitingDuration):
- }
- wk, err = pool.get()
- if err != nil {
- jobW.NotifyError(err)
- return
- }
- }
- d.wg.Run(func() {
- metrics.DDLRunningJobCount.WithLabelValues(pool.tp().String()).Inc()
- defer func() {
- metrics.DDLRunningJobCount.WithLabelValues(pool.tp().String()).Dec()
- }()
-
- for i := int64(0); i < variable.GetDDLErrorCountLimit(); i++ {
- err = wk.HandleLocalDDLJob(d.ddlCtx, job)
- // since local the job is not inserted into the ddl job queue, we need to add retry logic here.
- if err == nil || !isRetryableError(err) {
- break
- }
- logutil.DDLLogger().Info("handle local ddl job", zap.Int64("retry times", i), zap.Error(err))
- time.Sleep(time.Second)
- }
- pool.put(wk)
- if err != nil {
- logutil.DDLLogger().Info("handle ddl job failed", zap.Error(err), zap.Stringer("job", job))
- }
- jobW.NotifyError(err)
- })
-}
-
// deliveryJob deliver the job to the worker to run it asynchronously.
// the worker will run the job until it's finished, paused or another owner takes
// over and finished it.
@@ -519,6 +458,7 @@ func (s *jobScheduler) deliveryJob(wk *worker, pool *workerPool, job *model.Job)
jobID, involvedSchemaInfos := job.ID, job.GetInvolvingSchemaInfo()
s.runningJobs.addRunning(jobID, involvedSchemaInfos)
metrics.DDLRunningJobCount.WithLabelValues(pool.tp().String()).Inc()
+ jobCtx := s.getJobRunCtx()
s.wg.Run(func() {
defer func() {
r := recover()
@@ -536,7 +476,7 @@ func (s *jobScheduler) deliveryJob(wk *worker, pool *workerPool, job *model.Job)
pool.put(wk)
}()
for {
- err := s.transitOneJobStepAndWaitSync(wk, job)
+ err := s.transitOneJobStepAndWaitSync(wk, jobCtx, job)
if err != nil {
logutil.DDLLogger().Info("run job failed", zap.Error(err), zap.Stringer("job", job))
} else if job.InFinalState() {
@@ -571,14 +511,30 @@ func (s *jobScheduler) deliveryJob(wk *worker, pool *workerPool, job *model.Job)
})
}
+func (s *jobScheduler) getJobRunCtx() *jobContext {
+ return &jobContext{
+ ctx: s.schCtx,
+ unSyncedJobTracker: s.unSyncedTracker,
+ schemaVersionManager: s.schemaVerMgr,
+ infoCache: s.infoCache,
+ autoidCli: s.autoidCli,
+ store: s.store,
+
+ oldDDLCtx: s.ddlCtx,
+ }
+}
+
// transitOneJobStepAndWaitSync runs one step of the DDL job, persist it and
// waits for other TiDB node to synchronize.
-func (s *jobScheduler) transitOneJobStepAndWaitSync(wk *worker, job *model.Job) error {
+func (s *jobScheduler) transitOneJobStepAndWaitSync(wk *worker, jobCtx *jobContext, job *model.Job) error {
failpoint.InjectCall("beforeRunOneJobStep")
ownerID := s.ownerManager.ID()
// suppose we failed to sync version last time, we need to check and sync it
// before run to maintain the 2-version invariant.
- if !job.NotStarted() && (!s.isSynced(job) || !s.maybeAlreadyRunOnce(job.ID)) {
+ // if owner not change, we need try to sync when it's un-synced.
+ // if owner changed, we need to try sync it if the job is not started by
+ // current owner.
+ if jobCtx.isUnSynced(job.ID) || (job.Started() && !jobCtx.maybeAlreadyRunOnce(job.ID)) {
if variable.EnableMDL.Load() {
version, err := s.sysTblMgr.GetMDLVer(s.schCtx, job.ID)
if err == nil {
@@ -586,9 +542,7 @@ func (s *jobScheduler) transitOneJobStepAndWaitSync(wk *worker, job *model.Job)
if err != nil {
return err
}
- s.setAlreadyRunOnce(job.ID)
s.cleanMDLInfo(job, ownerID)
- return nil
} else if err != systable.ErrNotFound {
wk.jobLogger(job).Warn("check MDL info failed", zap.Error(err))
return err
@@ -599,11 +553,11 @@ func (s *jobScheduler) transitOneJobStepAndWaitSync(wk *worker, job *model.Job)
time.Sleep(time.Second)
return err
}
- s.setAlreadyRunOnce(job.ID)
}
+ jobCtx.setAlreadyRunOnce(job.ID)
}
- schemaVer, err := wk.transitOneJobStep(s.ddlCtx, job)
+ schemaVer, err := wk.transitOneJobStep(s.ddlCtx, jobCtx, job)
if err != nil {
tidblogutil.Logger(wk.logCtx).Info("handle ddl job failed", zap.Error(err), zap.Stringer("job", job))
return err
@@ -625,14 +579,44 @@ func (s *jobScheduler) transitOneJobStepAndWaitSync(wk *worker, job *model.Job)
return err
}
s.cleanMDLInfo(job, ownerID)
- s.synced(job)
+ jobCtx.removeUnSynced(job.ID)
- s.mu.RLock()
- s.mu.hook.OnJobUpdated(job)
- s.mu.RUnlock()
+ failpoint.InjectCall("onJobUpdated", job)
return nil
}
+// cleanMDLInfo cleans metadata lock info.
+func (s *jobScheduler) cleanMDLInfo(job *model.Job, ownerID string) {
+ if !variable.EnableMDL.Load() {
+ return
+ }
+ var sql string
+ if tidbutil.IsSysDB(strings.ToLower(job.SchemaName)) {
+ // DDLs that modify system tables could only happen in upgrade process,
+ // we should not reference 'owner_id'. Otherwise, there is a circular blocking problem.
+ sql = fmt.Sprintf("delete from mysql.tidb_mdl_info where job_id = %d", job.ID)
+ } else {
+ sql = fmt.Sprintf("delete from mysql.tidb_mdl_info where job_id = %d and owner_id = '%s'", job.ID, ownerID)
+ }
+ sctx, _ := s.sessPool.Get()
+ defer s.sessPool.Put(sctx)
+ se := sess.NewSession(sctx)
+ se.GetSessionVars().SetDiskFullOpt(kvrpcpb.DiskFullOpt_AllowedOnAlmostFull)
+ _, err := se.Execute(s.schCtx, sql, "delete-mdl-info")
+ if err != nil {
+ logutil.DDLLogger().Warn("unexpected error when clean mdl info", zap.Int64("job ID", job.ID), zap.Error(err))
+ return
+ }
+ // TODO we need clean it when version of JobStateRollbackDone is synced also.
+ if job.State == model.JobStateSynced && s.etcdCli != nil {
+ path := fmt.Sprintf("%s/%d/", util.DDLAllSchemaVersionsByJob, job.ID)
+ _, err = s.etcdCli.Delete(s.schCtx, path, clientv3.WithPrefix())
+ if err != nil {
+ logutil.DDLLogger().Warn("delete versions failed", zap.Int64("job ID", job.ID), zap.Error(err))
+ }
+ }
+}
+
func (d *ddl) getTableByTxn(r autoid.Requirement, schemaID, tableID int64) (*model.DBInfo, table.Table, error) {
var tbl table.Table
var dbInfo *model.DBInfo
@@ -679,7 +663,7 @@ func insertDDLJobs2Table(ctx context.Context, se *sess.Session, jobWs ...*JobWra
}
fmt.Fprintf(&sql, "(%d, %t, %s, %s, %s, %d, %t)", jobW.ID, jobW.MayNeedReorg(),
strconv.Quote(job2SchemaIDs(jobW.Job)), strconv.Quote(job2TableIDs(jobW.Job)),
- util.WrapKey2String(b), jobW.Type, !jobW.NotStarted())
+ util.WrapKey2String(b), jobW.Type, jobW.Started())
}
se.GetSessionVars().SetDiskFullOpt(kvrpcpb.DiskFullOpt_AllowedOnAlmostFull)
_, err := se.Execute(ctx, sql.String(), "insert_job")
@@ -727,12 +711,6 @@ func job2UniqueIDs(job *model.Job, schema bool) string {
return strconv.FormatInt(job.TableID, 10)
}
-func (w *worker) deleteDDLJob(job *model.Job) error {
- sql := fmt.Sprintf("delete from mysql.tidb_ddl_job where job_id = %d", job.ID)
- _, err := w.sess.Execute(context.Background(), sql, "delete_job")
- return errors.Trace(err)
-}
-
func updateDDLJob2Table(se *sess.Session, job *model.Job, updateRawArgs bool) error {
b, err := job.Encode(updateRawArgs)
if err != nil {
diff --git a/pkg/ddl/job_scheduler_test.go b/pkg/ddl/job_scheduler_test.go
index 1393fe1e9b7d2..4e4f04cb14e32 100644
--- a/pkg/ddl/job_scheduler_test.go
+++ b/pkg/ddl/job_scheduler_test.go
@@ -61,3 +61,11 @@ func TestMustReloadSchemas(t *testing.T) {
sch.mustReloadSchemas()
require.True(t, ctrl.Satisfied())
}
+
+func TestUnSyncedJobTracker(t *testing.T) {
+ jt := newUnSyncedJobTracker()
+ jt.addUnSynced(1)
+ require.True(t, jt.isUnSynced(1))
+ jt.removeUnSynced(1)
+ require.False(t, jt.isUnSynced(1))
+}
diff --git a/pkg/ddl/job_table_test.go b/pkg/ddl/job_scheduler_testkit_test.go
similarity index 96%
rename from pkg/ddl/job_table_test.go
rename to pkg/ddl/job_scheduler_testkit_test.go
index 7a016e7b223b3..bc5d11d3d90bf 100644
--- a/pkg/ddl/job_table_test.go
+++ b/pkg/ddl/job_scheduler_testkit_test.go
@@ -24,7 +24,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/tidb/pkg/ddl"
- "github.com/pingcap/tidb/pkg/ddl/syncer"
+ "github.com/pingcap/tidb/pkg/ddl/serverstate"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/testfailpoint"
@@ -187,7 +187,7 @@ func TestUpgradingRelatedJobState(t *testing.T) {
{"alter table e2 add index idx3(id)", model.JobStateRollbackDone, errors.New("[ddl:8214]Cancelled DDL job")},
}
- testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ddl/mockUpgradingState", `return(true)`)
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ddl/serverstate/mockUpgradingState", `return(true)`)
// TODO this case only checks that when a job cannot be paused, it can still run normally.
// we should add a ut for processJobDuringUpgrade, not this complex integration test.
@@ -202,7 +202,7 @@ func TestUpgradingRelatedJobState(t *testing.T) {
tk2.MustExec(fmt.Sprintf("admin cancel ddl jobs %d", job.ID))
}
if job.State == testCases[num].jobState {
- dom.DDL().StateSyncer().UpdateGlobalState(context.Background(), &syncer.StateInfo{State: syncer.StateUpgrading})
+ dom.DDL().StateSyncer().UpdateGlobalState(context.Background(), &serverstate.StateInfo{State: serverstate.StateUpgrading})
}
})
@@ -214,7 +214,7 @@ func TestUpgradingRelatedJobState(t *testing.T) {
_, err := tk.Exec(tc.sql)
require.Equal(t, tc.err.Error(), err.Error())
}
- dom.DDL().StateSyncer().UpdateGlobalState(context.Background(), &syncer.StateInfo{State: syncer.StateNormalRunning})
+ dom.DDL().StateSyncer().UpdateGlobalState(context.Background(), &serverstate.StateInfo{State: serverstate.StateNormalRunning})
}
}
diff --git a/pkg/ddl/job_submitter.go b/pkg/ddl/job_submitter.go
new file mode 100644
index 0000000000000..c80232ff5dbca
--- /dev/null
+++ b/pkg/ddl/job_submitter.go
@@ -0,0 +1,669 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ddl
+
+import (
+ "context"
+ "fmt"
+ "math"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
+ "github.com/pingcap/kvproto/pkg/kvrpcpb"
+ "github.com/pingcap/tidb/pkg/config"
+ "github.com/pingcap/tidb/pkg/ddl/logutil"
+ sess "github.com/pingcap/tidb/pkg/ddl/session"
+ ddlutil "github.com/pingcap/tidb/pkg/ddl/util"
+ "github.com/pingcap/tidb/pkg/infoschema"
+ "github.com/pingcap/tidb/pkg/kv"
+ "github.com/pingcap/tidb/pkg/meta"
+ "github.com/pingcap/tidb/pkg/metrics"
+ "github.com/pingcap/tidb/pkg/parser/ast"
+ "github.com/pingcap/tidb/pkg/parser/model"
+ "github.com/pingcap/tidb/pkg/parser/terror"
+ "github.com/pingcap/tidb/pkg/sessionctx/variable"
+ "github.com/pingcap/tidb/pkg/util"
+ "github.com/pingcap/tidb/pkg/util/dbterror"
+ "github.com/pingcap/tidb/pkg/util/mathutil"
+ tikv "github.com/tikv/client-go/v2/kv"
+ "github.com/tikv/client-go/v2/oracle"
+ "go.uber.org/zap"
+)
+
+func (d *ddl) limitDDLJobs() {
+ defer util.Recover(metrics.LabelDDL, "limitDDLJobs", nil, true)
+
+ jobWs := make([]*JobWrapper, 0, batchAddingJobs)
+ ch := d.limitJobCh
+ for {
+ select {
+ // the channel is never closed
+ case jobW := <-ch:
+ jobWs = jobWs[:0]
+ failpoint.InjectCall("afterGetJobFromLimitCh", ch)
+ jobLen := len(ch)
+ jobWs = append(jobWs, jobW)
+ for i := 0; i < jobLen; i++ {
+ jobWs = append(jobWs, <-ch)
+ }
+ d.addBatchDDLJobs(jobWs)
+ case <-d.ctx.Done():
+ return
+ }
+ }
+}
+
+// addBatchDDLJobs gets global job IDs and puts the DDL jobs in the DDL queue.
+func (d *ddl) addBatchDDLJobs(jobWs []*JobWrapper) {
+ startTime := time.Now()
+ var (
+ err error
+ newWs []*JobWrapper
+ )
+ // DDLForce2Queue is a flag to tell DDL worker to always push the job to the DDL queue.
+ toTable := !variable.DDLForce2Queue.Load()
+ fastCreate := variable.EnableFastCreateTable.Load()
+ if toTable {
+ if fastCreate {
+ newWs, err = mergeCreateTableJobs(jobWs)
+ if err != nil {
+ logutil.DDLLogger().Warn("failed to merge create table jobs", zap.Error(err))
+ } else {
+ jobWs = newWs
+ }
+ }
+ err = d.addBatchDDLJobs2Table(jobWs)
+ } else {
+ err = d.addBatchDDLJobs2Queue(jobWs)
+ }
+ var jobs string
+ for _, jobW := range jobWs {
+ if err == nil {
+ err = jobW.cacheErr
+ }
+ jobW.NotifyResult(err)
+ jobs += jobW.Job.String() + "; "
+ metrics.DDLWorkerHistogram.WithLabelValues(metrics.WorkerAddDDLJob, jobW.Job.Type.String(),
+ metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
+ }
+ if err != nil {
+ logutil.DDLLogger().Warn("add DDL jobs failed", zap.String("jobs", jobs), zap.Error(err))
+ } else {
+ logutil.DDLLogger().Info("add DDL jobs",
+ zap.Int("batch count", len(jobWs)),
+ zap.String("jobs", jobs),
+ zap.Bool("table", toTable),
+ zap.Bool("fast_create", fastCreate))
+ }
+}
+
+// mergeCreateTableJobs merges CreateTable jobs to CreateTables.
+func mergeCreateTableJobs(jobWs []*JobWrapper) ([]*JobWrapper, error) {
+ if len(jobWs) <= 1 {
+ return jobWs, nil
+ }
+ resJobWs := make([]*JobWrapper, 0, len(jobWs))
+ mergeableJobWs := make(map[string][]*JobWrapper, len(jobWs))
+ for _, jobW := range jobWs {
+ // we don't merge jobs with ID pre-allocated.
+ if jobW.Type != model.ActionCreateTable || jobW.IDAllocated {
+ resJobWs = append(resJobWs, jobW)
+ continue
+ }
+ // ActionCreateTables doesn't support foreign key now.
+ tbInfo, ok := jobW.Args[0].(*model.TableInfo)
+ if !ok || len(tbInfo.ForeignKeys) > 0 {
+ resJobWs = append(resJobWs, jobW)
+ continue
+ }
+ // CreateTables only support tables of same schema now.
+ mergeableJobWs[jobW.Job.SchemaName] = append(mergeableJobWs[jobW.Job.SchemaName], jobW)
+ }
+
+ for schema, jobs := range mergeableJobWs {
+ total := len(jobs)
+ if total <= 1 {
+ resJobWs = append(resJobWs, jobs...)
+ continue
+ }
+ const maxBatchSize = 8
+ batchCount := (total + maxBatchSize - 1) / maxBatchSize
+ start := 0
+ for _, batchSize := range mathutil.Divide2Batches(total, batchCount) {
+ batch := jobs[start : start+batchSize]
+ job, err := mergeCreateTableJobsOfSameSchema(batch)
+ if err != nil {
+ return nil, err
+ }
+ start += batchSize
+ logutil.DDLLogger().Info("merge create table jobs", zap.String("schema", schema),
+ zap.Int("total", total), zap.Int("batch_size", batchSize))
+
+ newJobW := &JobWrapper{
+ Job: job,
+ ResultCh: make([]chan jobSubmitResult, 0, batchSize),
+ }
+ // merge the result channels.
+ for _, j := range batch {
+ newJobW.ResultCh = append(newJobW.ResultCh, j.ResultCh...)
+ }
+ resJobWs = append(resJobWs, newJobW)
+ }
+ }
+ return resJobWs, nil
+}
+
+// buildQueryStringFromJobs takes a slice of Jobs and concatenates their
+// queries into a single query string.
+// Each query is separated by a semicolon and a space.
+// Trailing spaces are removed from each query, and a semicolon is appended
+// if it's not already present.
+func buildQueryStringFromJobs(jobs []*JobWrapper) string {
+ var queryBuilder strings.Builder
+ for i, job := range jobs {
+ q := strings.TrimSpace(job.Query)
+ if !strings.HasSuffix(q, ";") {
+ q += ";"
+ }
+ queryBuilder.WriteString(q)
+
+ if i < len(jobs)-1 {
+ queryBuilder.WriteString(" ")
+ }
+ }
+ return queryBuilder.String()
+}
+
+// mergeCreateTableJobsOfSameSchema combine CreateTableJobs to BatchCreateTableJob.
+func mergeCreateTableJobsOfSameSchema(jobWs []*JobWrapper) (*model.Job, error) {
+ if len(jobWs) == 0 {
+ return nil, errors.Trace(fmt.Errorf("expect non-empty jobs"))
+ }
+
+ var combinedJob *model.Job
+
+ args := make([]*model.TableInfo, 0, len(jobWs))
+ involvingSchemaInfo := make([]model.InvolvingSchemaInfo, 0, len(jobWs))
+ var foreignKeyChecks bool
+
+ // if there is any duplicated table name
+ duplication := make(map[string]struct{})
+ for _, job := range jobWs {
+ if combinedJob == nil {
+ combinedJob = job.Clone()
+ combinedJob.Type = model.ActionCreateTables
+ combinedJob.Args = combinedJob.Args[:0]
+ foreignKeyChecks = job.Args[1].(bool)
+ }
+ // append table job args
+ info, ok := job.Args[0].(*model.TableInfo)
+ if !ok {
+ return nil, errors.Trace(fmt.Errorf("expect model.TableInfo, but got %T", job.Args[0]))
+ }
+ args = append(args, info)
+
+ if _, ok := duplication[info.Name.L]; ok {
+ // return err even if create table if not exists
+ return nil, infoschema.ErrTableExists.FastGenByArgs("can not batch create tables with same name")
+ }
+
+ duplication[info.Name.L] = struct{}{}
+
+ involvingSchemaInfo = append(involvingSchemaInfo,
+ model.InvolvingSchemaInfo{
+ Database: job.SchemaName,
+ Table: info.Name.L,
+ })
+ }
+
+ combinedJob.Args = append(combinedJob.Args, args)
+ combinedJob.Args = append(combinedJob.Args, foreignKeyChecks)
+ combinedJob.InvolvingSchemaInfo = involvingSchemaInfo
+ combinedJob.Query = buildQueryStringFromJobs(jobWs)
+
+ return combinedJob, nil
+}
+
+// addBatchDDLJobs2Table gets global job IDs and puts the DDL jobs in the DDL job table.
+func (d *ddl) addBatchDDLJobs2Table(jobWs []*JobWrapper) error {
+ var err error
+
+ if len(jobWs) == 0 {
+ return nil
+ }
+
+ ctx := kv.WithInternalSourceType(d.ctx, kv.InternalTxnDDL)
+ se, err := d.sessPool.Get()
+ if err != nil {
+ return errors.Trace(err)
+ }
+ defer d.sessPool.Put(se)
+ found, err := d.sysTblMgr.HasFlashbackClusterJob(ctx, d.minJobIDRefresher.GetCurrMinJobID())
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if found {
+ return errors.Errorf("Can't add ddl job, have flashback cluster job")
+ }
+
+ var (
+ startTS = uint64(0)
+ bdrRole = string(ast.BDRRoleNone)
+ )
+
+ err = kv.RunInNewTxn(ctx, d.store, true, func(_ context.Context, txn kv.Transaction) error {
+ t := meta.NewMeta(txn)
+
+ bdrRole, err = t.GetBDRRole()
+ if err != nil {
+ return errors.Trace(err)
+ }
+ startTS = txn.StartTS()
+
+ if variable.DDLForce2Queue.Load() {
+ if err := d.checkFlashbackJobInQueue(t); err != nil {
+ return err
+ }
+ }
+
+ return nil
+ })
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ for _, jobW := range jobWs {
+ job := jobW.Job
+ job.Version = currentVersion
+ job.StartTS = startTS
+ job.BDRRole = bdrRole
+
+ // BDR mode only affects the DDL not from CDC
+ if job.CDCWriteSource == 0 && bdrRole != string(ast.BDRRoleNone) {
+ if job.Type == model.ActionMultiSchemaChange && job.MultiSchemaInfo != nil {
+ for _, subJob := range job.MultiSchemaInfo.SubJobs {
+ if ast.DeniedByBDR(ast.BDRRole(bdrRole), subJob.Type, job) {
+ return dbterror.ErrBDRRestrictedDDL.FastGenByArgs(bdrRole)
+ }
+ }
+ } else if ast.DeniedByBDR(ast.BDRRole(bdrRole), job.Type, job) {
+ return dbterror.ErrBDRRestrictedDDL.FastGenByArgs(bdrRole)
+ }
+ }
+
+ setJobStateToQueueing(job)
+
+ if d.serverStateSyncer.IsUpgradingState() && !hasSysDB(job) {
+ if err = pauseRunningJob(sess.NewSession(se), job, model.AdminCommandBySystem); err != nil {
+ logutil.DDLUpgradingLogger().Warn("pause user DDL by system failed", zap.Stringer("job", job), zap.Error(err))
+ jobW.cacheErr = err
+ continue
+ }
+ logutil.DDLUpgradingLogger().Info("pause user DDL by system successful", zap.Stringer("job", job))
+ }
+ }
+
+ se.GetSessionVars().SetDiskFullOpt(kvrpcpb.DiskFullOpt_AllowedOnAlmostFull)
+ ddlSe := sess.NewSession(se)
+ if err = GenGIDAndInsertJobsWithRetry(ctx, ddlSe, jobWs); err != nil {
+ return errors.Trace(err)
+ }
+ for _, jobW := range jobWs {
+ d.initJobDoneCh(jobW.ID)
+ }
+
+ return nil
+}
+
+func (d *ddl) initJobDoneCh(jobID int64) {
+ d.ddlJobDoneChMap.Store(jobID, make(chan struct{}, 1))
+}
+
+func (d *ddl) addBatchDDLJobs2Queue(jobWs []*JobWrapper) error {
+ ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL)
+ // lock to reduce conflict
+ d.globalIDLock.Lock()
+ defer d.globalIDLock.Unlock()
+ return kv.RunInNewTxn(ctx, d.store, true, func(_ context.Context, txn kv.Transaction) error {
+ t := meta.NewMeta(txn)
+
+ count := getRequiredGIDCount(jobWs)
+ ids, err := t.GenGlobalIDs(count)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ assignGIDsForJobs(jobWs, ids)
+
+ if err := d.checkFlashbackJobInQueue(t); err != nil {
+ return errors.Trace(err)
+ }
+
+ for _, jobW := range jobWs {
+ job := jobW.Job
+ job.Version = currentVersion
+ job.StartTS = txn.StartTS()
+ setJobStateToQueueing(job)
+ if err = buildJobDependence(t, job); err != nil {
+ return errors.Trace(err)
+ }
+ jobListKey := meta.DefaultJobListKey
+ if job.MayNeedReorg() {
+ jobListKey = meta.AddIndexJobListKey
+ }
+ if err = t.EnQueueDDLJob(job, jobListKey); err != nil {
+ return errors.Trace(err)
+ }
+ }
+ failpoint.Inject("mockAddBatchDDLJobsErr", func(val failpoint.Value) {
+ if val.(bool) {
+ failpoint.Return(errors.Errorf("mockAddBatchDDLJobsErr"))
+ }
+ })
+ return nil
+ })
+}
+
+func (*ddl) checkFlashbackJobInQueue(t *meta.Meta) error {
+ jobs, err := t.GetAllDDLJobsInQueue(meta.DefaultJobListKey)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ for _, job := range jobs {
+ if job.Type == model.ActionFlashbackCluster {
+ return errors.Errorf("Can't add ddl job, have flashback cluster job")
+ }
+ }
+ return nil
+}
+
+// GenGIDAndInsertJobsWithRetry generate job related global ID and inserts DDL jobs to the DDL job
+// table with retry. job id allocation and job insertion are in the same transaction,
+// as we want to make sure DDL jobs are inserted in id order, then we can query from
+// a min job ID when scheduling DDL jobs to mitigate https://github.com/pingcap/tidb/issues/52905.
+// so this function has side effect, it will set table/db/job id of 'jobs'.
+func GenGIDAndInsertJobsWithRetry(ctx context.Context, ddlSe *sess.Session, jobWs []*JobWrapper) error {
+ count := getRequiredGIDCount(jobWs)
+ return genGIDAndCallWithRetry(ctx, ddlSe, count, func(ids []int64) error {
+ failpoint.Inject("mockGenGlobalIDFail", func(val failpoint.Value) {
+ if val.(bool) {
+ failpoint.Return(errors.New("gofail genGlobalIDs error"))
+ }
+ })
+ assignGIDsForJobs(jobWs, ids)
+ injectModifyJobArgFailPoint(jobWs)
+ return insertDDLJobs2Table(ctx, ddlSe, jobWs...)
+ })
+}
+
+// getRequiredGIDCount returns the count of required global IDs for the jobs. it's calculated
+// as: the count of jobs + the count of IDs for the jobs which do NOT have pre-allocated ID.
+func getRequiredGIDCount(jobWs []*JobWrapper) int {
+ count := len(jobWs)
+ idCountForTable := func(info *model.TableInfo) int {
+ c := 1
+ if partitionInfo := info.GetPartitionInfo(); partitionInfo != nil {
+ c += len(partitionInfo.Definitions)
+ }
+ return c
+ }
+ for _, jobW := range jobWs {
+ if jobW.IDAllocated {
+ continue
+ }
+ switch jobW.Type {
+ case model.ActionCreateView, model.ActionCreateSequence, model.ActionCreateTable:
+ info := jobW.Args[0].(*model.TableInfo)
+ count += idCountForTable(info)
+ case model.ActionCreateTables:
+ infos := jobW.Args[0].([]*model.TableInfo)
+ for _, info := range infos {
+ count += idCountForTable(info)
+ }
+ case model.ActionCreateSchema:
+ count++
+ }
+ // TODO support other type of jobs
+ }
+ return count
+}
+
+// assignGIDsForJobs should be used with getRequiredGIDCount, and len(ids) must equal
+// what getRequiredGIDCount returns.
+func assignGIDsForJobs(jobWs []*JobWrapper, ids []int64) {
+ idx := 0
+
+ assignIDsForTable := func(info *model.TableInfo) {
+ info.ID = ids[idx]
+ idx++
+ if partitionInfo := info.GetPartitionInfo(); partitionInfo != nil {
+ for i := range partitionInfo.Definitions {
+ partitionInfo.Definitions[i].ID = ids[idx]
+ idx++
+ }
+ }
+ }
+ for _, jobW := range jobWs {
+ switch jobW.Type {
+ case model.ActionCreateView, model.ActionCreateSequence, model.ActionCreateTable:
+ info := jobW.Args[0].(*model.TableInfo)
+ if !jobW.IDAllocated {
+ assignIDsForTable(info)
+ }
+ jobW.TableID = info.ID
+ case model.ActionCreateTables:
+ if !jobW.IDAllocated {
+ infos := jobW.Args[0].([]*model.TableInfo)
+ for _, info := range infos {
+ assignIDsForTable(info)
+ }
+ }
+ case model.ActionCreateSchema:
+ dbInfo := jobW.Args[0].(*model.DBInfo)
+ if !jobW.IDAllocated {
+ dbInfo.ID = ids[idx]
+ idx++
+ }
+ jobW.SchemaID = dbInfo.ID
+ }
+ // TODO support other type of jobs
+ jobW.ID = ids[idx]
+ idx++
+ }
+}
+
+// genGIDAndCallWithRetry generates global IDs and calls the function with retry.
+// generate ID and call function runs in the same transaction.
+func genGIDAndCallWithRetry(ctx context.Context, ddlSe *sess.Session, count int, fn func(ids []int64) error) error {
+ var resErr error
+ for i := uint(0); i < kv.MaxRetryCnt; i++ {
+ resErr = func() (err error) {
+ if err := ddlSe.Begin(ctx); err != nil {
+ return errors.Trace(err)
+ }
+ defer func() {
+ if err != nil {
+ ddlSe.Rollback()
+ }
+ }()
+ txn, err := ddlSe.Txn()
+ if err != nil {
+ return errors.Trace(err)
+ }
+ txn.SetOption(kv.Pessimistic, true)
+ forUpdateTS, err := lockGlobalIDKey(ctx, ddlSe, txn)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ txn.GetSnapshot().SetOption(kv.SnapshotTS, forUpdateTS)
+
+ m := meta.NewMeta(txn)
+ ids, err := m.GenGlobalIDs(count)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if err = fn(ids); err != nil {
+ return errors.Trace(err)
+ }
+ return ddlSe.Commit(ctx)
+ }()
+
+ if resErr != nil && kv.IsTxnRetryableError(resErr) {
+ logutil.DDLLogger().Warn("insert job meet retryable error", zap.Error(resErr))
+ kv.BackOff(i)
+ continue
+ }
+ break
+ }
+ return resErr
+}
+
+// lockGlobalIDKey locks the global ID key in the meta store. it keeps trying if
+// meet write conflict, we cannot have a fixed retry count for this error, see this
+// https://github.com/pingcap/tidb/issues/27197#issuecomment-2216315057.
+// this part is same as how we implement pessimistic + repeatable read isolation
+// level in SQL executor, see doLockKeys.
+// NextGlobalID is a meta key, so we cannot use "select xx for update", if we store
+// it into a table row or using advisory lock, we will depends on a system table
+// that is created by us, cyclic. although we can create a system table without using
+// DDL logic, we will only consider change it when we have data dictionary and keep
+// it this way now.
+// TODO maybe we can unify the lock mechanism with SQL executor in the future, or
+// implement it inside TiKV client-go.
+func lockGlobalIDKey(ctx context.Context, ddlSe *sess.Session, txn kv.Transaction) (uint64, error) {
+ var (
+ iteration uint
+ forUpdateTs = txn.StartTS()
+ ver kv.Version
+ err error
+ )
+ waitTime := ddlSe.GetSessionVars().LockWaitTimeout
+ m := meta.NewMeta(txn)
+ idKey := m.GlobalIDKey()
+ for {
+ lockCtx := tikv.NewLockCtx(forUpdateTs, waitTime, time.Now())
+ err = txn.LockKeys(ctx, lockCtx, idKey)
+ if err == nil || !terror.ErrorEqual(kv.ErrWriteConflict, err) {
+ break
+ }
+ // ErrWriteConflict contains a conflict-commit-ts in most case, but it cannot
+ // be used as forUpdateTs, see comments inside handleAfterPessimisticLockError
+ ver, err = ddlSe.GetStore().CurrentVersion(oracle.GlobalTxnScope)
+ if err != nil {
+ break
+ }
+ forUpdateTs = ver.Ver
+
+ kv.BackOff(iteration)
+ // avoid it keep growing and overflow.
+ iteration = min(iteration+1, math.MaxInt)
+ }
+ return forUpdateTs, err
+}
+
+// TODO this failpoint is only checking how job scheduler handle
+// corrupted job args, we should test it there by UT, not here.
+func injectModifyJobArgFailPoint(jobWs []*JobWrapper) {
+ failpoint.Inject("MockModifyJobArg", func(val failpoint.Value) {
+ if val.(bool) {
+ for _, jobW := range jobWs {
+ job := jobW.Job
+ // Corrupt the DDL job argument.
+ if job.Type == model.ActionMultiSchemaChange {
+ if len(job.MultiSchemaInfo.SubJobs) > 0 && len(job.MultiSchemaInfo.SubJobs[0].Args) > 0 {
+ job.MultiSchemaInfo.SubJobs[0].Args[0] = 1
+ }
+ } else if len(job.Args) > 0 {
+ job.Args[0] = 1
+ }
+ }
+ }
+ })
+}
+
+func setJobStateToQueueing(job *model.Job) {
+ if job.Type == model.ActionMultiSchemaChange && job.MultiSchemaInfo != nil {
+ for _, sub := range job.MultiSchemaInfo.SubJobs {
+ sub.State = model.JobStateQueueing
+ }
+ }
+ job.State = model.JobStateQueueing
+}
+
+// buildJobDependence sets the curjob's dependency-ID.
+// The dependency-job's ID must less than the current job's ID, and we need the largest one in the list.
+func buildJobDependence(t *meta.Meta, curJob *model.Job) error {
+ // Jobs in the same queue are ordered. If we want to find a job's dependency-job, we need to look for
+ // it from the other queue. So if the job is "ActionAddIndex" job, we need find its dependency-job from DefaultJobList.
+ jobListKey := meta.DefaultJobListKey
+ if !curJob.MayNeedReorg() {
+ jobListKey = meta.AddIndexJobListKey
+ }
+ jobs, err := t.GetAllDDLJobsInQueue(jobListKey)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ for _, job := range jobs {
+ if curJob.ID < job.ID {
+ continue
+ }
+ isDependent, err := curJob.IsDependentOn(job)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if isDependent {
+ logutil.DDLLogger().Info("current DDL job depends on other job",
+ zap.Stringer("currentJob", curJob),
+ zap.Stringer("dependentJob", job))
+ curJob.DependencyID = job.ID
+ break
+ }
+ }
+ return nil
+}
+
+func (e *executor) notifyNewJobSubmitted(ch chan struct{}, etcdPath string, jobID int64, jobType string) {
+ // If the workers don't run, we needn't notify workers.
+ // TODO: It does not affect informing the backfill worker.
+ if !config.GetGlobalConfig().Instance.TiDBEnableDDL.Load() {
+ return
+ }
+ if e.ownerManager.IsOwner() {
+ asyncNotify(ch)
+ } else {
+ e.notifyNewJobByEtcd(etcdPath, jobID, jobType)
+ }
+}
+
+func (e *executor) notifyNewJobByEtcd(etcdPath string, jobID int64, jobType string) {
+ if e.etcdCli == nil {
+ return
+ }
+
+ jobIDStr := strconv.FormatInt(jobID, 10)
+ timeStart := time.Now()
+ err := ddlutil.PutKVToEtcd(e.ctx, e.etcdCli, 1, etcdPath, jobIDStr)
+ if err != nil {
+ logutil.DDLLogger().Info("notify handling DDL job failed",
+ zap.String("etcdPath", etcdPath),
+ zap.Int64("jobID", jobID),
+ zap.String("type", jobType),
+ zap.Error(err))
+ }
+ metrics.DDLWorkerHistogram.WithLabelValues(metrics.WorkerNotifyDDLJob, jobType, metrics.RetLabel(err)).Observe(time.Since(timeStart).Seconds())
+}
diff --git a/pkg/ddl/ddl_worker.go b/pkg/ddl/job_worker.go
similarity index 55%
rename from pkg/ddl/ddl_worker.go
rename to pkg/ddl/job_worker.go
index 451b894937d7e..cad1d14d29bc2 100644
--- a/pkg/ddl/ddl_worker.go
+++ b/pkg/ddl/job_worker.go
@@ -17,10 +17,8 @@ package ddl
import (
"context"
"fmt"
- "math"
"math/rand"
"os"
- "strconv"
"strings"
"sync"
"sync/atomic"
@@ -32,11 +30,12 @@ import (
"github.com/pingcap/tidb/pkg/ddl/logutil"
sess "github.com/pingcap/tidb/pkg/ddl/session"
"github.com/pingcap/tidb/pkg/ddl/util"
+ "github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/meta"
+ "github.com/pingcap/tidb/pkg/meta/autoid"
"github.com/pingcap/tidb/pkg/metrics"
"github.com/pingcap/tidb/pkg/parser"
- "github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/terror"
"github.com/pingcap/tidb/pkg/sessionctx"
@@ -49,11 +48,8 @@ import (
"github.com/pingcap/tidb/pkg/util/resourcegrouptag"
"github.com/pingcap/tidb/pkg/util/topsql"
topsqlstate "github.com/pingcap/tidb/pkg/util/topsql/state"
- tikv "github.com/tikv/client-go/v2/kv"
- "github.com/tikv/client-go/v2/oracle"
"github.com/tikv/client-go/v2/tikvrpc"
kvutil "github.com/tikv/client-go/v2/util"
- clientv3 "go.etcd.io/etcd/client/v3"
atomicutil "go.uber.org/atomic"
"go.uber.org/zap"
)
@@ -81,6 +77,27 @@ func SetWaitTimeWhenErrorOccurred(dur time.Duration) {
atomic.StoreInt64(&WaitTimeWhenErrorOccurred, int64(dur))
}
+// jobContext is the context for execution of a DDL job.
+type jobContext struct {
+ // below fields are shared by all DDL jobs
+ ctx context.Context
+ *unSyncedJobTracker
+ *schemaVersionManager
+ infoCache *infoschema.InfoCache
+ autoidCli *autoid.ClientDiscover
+ store kv.Storage
+
+ // TODO reorg part of code couple this struct so much, remove it later.
+ oldDDLCtx *ddlCtx
+}
+
+func (c *jobContext) getAutoIDRequirement() autoid.Requirement {
+ return &asAutoIDRequirement{
+ store: c.store,
+ autoidCli: c.autoidCli,
+ }
+}
+
type workerType byte
const (
@@ -88,9 +105,6 @@ const (
generalWorker workerType = 0
// addIdxWorker is the worker who handles the operation of adding indexes.
addIdxWorker workerType = 1
- // loaclWorker is the worker who handles the operation in local TiDB.
- // currently it only handle CreateTable job of fast create table enabled.
- localWorker workerType = 2
)
// worker is used for handling DDL jobs.
@@ -100,7 +114,7 @@ type worker struct {
tp workerType
addingDDLJobKey string
ddlJobCh chan struct{}
- // for local mode worker, it's ctx of 'ddl', else it's the ctx of 'job scheduler'.
+ // it's the ctx of 'job scheduler'.
ctx context.Context
wg sync.WaitGroup
@@ -113,8 +127,9 @@ type worker struct {
*ddlCtx
}
-// JobContext is the ddl job execution context.
-type JobContext struct {
+// ReorgContext contains context info for reorg job.
+// TODO there is another reorgCtx, merge them.
+type ReorgContext struct {
// below fields are cache for top sql
ddlJobCtx context.Context
cacheSQL string
@@ -126,9 +141,9 @@ type JobContext struct {
cloudStorageURI string
}
-// NewJobContext returns a new ddl job context.
-func NewJobContext() *JobContext {
- return &JobContext{
+// NewReorgContext returns a new ddl job context.
+func NewReorgContext() *ReorgContext {
+ return &ReorgContext{
ddlJobCtx: context.Background(),
cacheSQL: "",
cacheNormalizedSQL: "",
@@ -170,8 +185,6 @@ func (w *worker) typeStr() string {
str = "general"
case addIdxWorker:
str = "add index"
- case localWorker:
- str = "local worker"
default:
str = "unknown"
}
@@ -191,24 +204,6 @@ func (w *worker) Close() {
tidblogutil.Logger(w.logCtx).Info("DDL worker closed", zap.Duration("take time", time.Since(startTime)))
}
-func (e *executor) notifyNewJobByEtcd(etcdPath string, jobID int64, jobType string) {
- if e.etcdCli == nil {
- return
- }
-
- jobIDStr := strconv.FormatInt(jobID, 10)
- timeStart := time.Now()
- err := util.PutKVToEtcd(e.ctx, e.etcdCli, 1, etcdPath, jobIDStr)
- if err != nil {
- logutil.DDLLogger().Info("notify handling DDL job failed",
- zap.String("etcdPath", etcdPath),
- zap.Int64("jobID", jobID),
- zap.String("type", jobType),
- zap.Error(err))
- }
- metrics.DDLWorkerHistogram.WithLabelValues(metrics.WorkerNotifyDDLJob, jobType, metrics.RetLabel(err)).Observe(time.Since(timeStart).Seconds())
-}
-
func asyncNotify(ch chan struct{}) {
select {
case ch <- struct{}{}:
@@ -216,537 +211,6 @@ func asyncNotify(ch chan struct{}) {
}
}
-func (d *ddl) limitDDLJobs(ch chan *JobWrapper, handler func([]*JobWrapper)) {
- defer tidbutil.Recover(metrics.LabelDDL, "limitDDLJobs", nil, true)
-
- jobWs := make([]*JobWrapper, 0, batchAddingJobs)
- for {
- select {
- // the channel is never closed
- case jobW := <-ch:
- jobWs = jobWs[:0]
- jobLen := len(ch)
- jobWs = append(jobWs, jobW)
- for i := 0; i < jobLen; i++ {
- jobWs = append(jobWs, <-ch)
- }
- handler(jobWs)
- case <-d.ctx.Done():
- return
- }
- }
-}
-
-// addBatchDDLJobsV1 gets global job IDs and puts the DDL jobs in the DDL queue.
-func (d *ddl) addBatchDDLJobsV1(jobWs []*JobWrapper) {
- startTime := time.Now()
- var err error
- // DDLForce2Queue is a flag to tell DDL worker to always push the job to the DDL queue.
- toTable := !variable.DDLForce2Queue.Load()
- if toTable {
- err = d.addBatchDDLJobs(jobWs)
- } else {
- err = d.addBatchDDLJobs2Queue(jobWs)
- }
- var jobs string
- for _, jobW := range jobWs {
- if err == nil {
- err = jobW.cacheErr
- }
- jobW.NotifyError(err)
- jobs += jobW.Job.String() + "; "
- metrics.DDLWorkerHistogram.WithLabelValues(metrics.WorkerAddDDLJob, jobW.Job.Type.String(),
- metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
- }
- if err != nil {
- logutil.DDLLogger().Warn("add DDL jobs failed", zap.String("jobs", jobs), zap.Error(err))
- } else {
- logutil.DDLLogger().Info("add DDL jobs",
- zap.Int("batch count", len(jobWs)),
- zap.String("jobs", jobs),
- zap.Bool("table", toTable))
- }
-}
-
-// addBatchLocalDDLJobs gets global job IDs and delivery the DDL jobs to local TiDB
-func (d *ddl) addBatchLocalDDLJobs(jobWs []*JobWrapper) {
- if newJobWs, err := combineBatchCreateTableJobs(jobWs); err == nil {
- jobWs = newJobWs
- }
- err := d.addBatchDDLJobs(jobWs)
- if err != nil {
- for _, jobW := range jobWs {
- jobW.NotifyError(err)
- }
- logutil.DDLLogger().Error("add DDL jobs failed", zap.Bool("local_mode", true), zap.Error(err))
- } else {
- logutil.DDLLogger().Info("add DDL jobs",
- zap.Bool("local_mode", true),
- zap.Int("batch count", len(jobWs)))
- }
-}
-
-// buildJobDependence sets the curjob's dependency-ID.
-// The dependency-job's ID must less than the current job's ID, and we need the largest one in the list.
-func buildJobDependence(t *meta.Meta, curJob *model.Job) error {
- // Jobs in the same queue are ordered. If we want to find a job's dependency-job, we need to look for
- // it from the other queue. So if the job is "ActionAddIndex" job, we need find its dependency-job from DefaultJobList.
- jobListKey := meta.DefaultJobListKey
- if !curJob.MayNeedReorg() {
- jobListKey = meta.AddIndexJobListKey
- }
- jobs, err := t.GetAllDDLJobsInQueue(jobListKey)
- if err != nil {
- return errors.Trace(err)
- }
-
- for _, job := range jobs {
- if curJob.ID < job.ID {
- continue
- }
- isDependent, err := curJob.IsDependentOn(job)
- if err != nil {
- return errors.Trace(err)
- }
- if isDependent {
- logutil.DDLLogger().Info("current DDL job depends on other job",
- zap.Stringer("currentJob", curJob),
- zap.Stringer("dependentJob", job))
- curJob.DependencyID = job.ID
- break
- }
- }
- return nil
-}
-
-func (d *ddl) addBatchDDLJobs2Queue(jobWs []*JobWrapper) error {
- ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL)
- // lock to reduce conflict
- d.globalIDLock.Lock()
- defer d.globalIDLock.Unlock()
- return kv.RunInNewTxn(ctx, d.store, true, func(_ context.Context, txn kv.Transaction) error {
- t := meta.NewMeta(txn)
- ids, err := t.GenGlobalIDs(len(jobWs))
- if err != nil {
- return errors.Trace(err)
- }
-
- if err := d.checkFlashbackJobInQueue(t); err != nil {
- return errors.Trace(err)
- }
-
- for i, jobW := range jobWs {
- job := jobW.Job
- job.Version = currentVersion
- job.StartTS = txn.StartTS()
- job.ID = ids[i]
- setJobStateToQueueing(job)
- if err = buildJobDependence(t, job); err != nil {
- return errors.Trace(err)
- }
- jobListKey := meta.DefaultJobListKey
- if job.MayNeedReorg() {
- jobListKey = meta.AddIndexJobListKey
- }
- if err = t.EnQueueDDLJob(job, jobListKey); err != nil {
- return errors.Trace(err)
- }
- }
- failpoint.Inject("mockAddBatchDDLJobsErr", func(val failpoint.Value) {
- if val.(bool) {
- failpoint.Return(errors.Errorf("mockAddBatchDDLJobsErr"))
- }
- })
- return nil
- })
-}
-
-func (*ddl) checkFlashbackJobInQueue(t *meta.Meta) error {
- jobs, err := t.GetAllDDLJobsInQueue(meta.DefaultJobListKey)
- if err != nil {
- return errors.Trace(err)
- }
- for _, job := range jobs {
- if job.Type == model.ActionFlashbackCluster {
- return errors.Errorf("Can't add ddl job, have flashback cluster job")
- }
- }
- return nil
-}
-
-// TODO this failpoint is only checking how job scheduler handle
-// corrupted job args, we should test it there by UT, not here.
-func injectModifyJobArgFailPoint(jobWs []*JobWrapper) {
- failpoint.Inject("MockModifyJobArg", func(val failpoint.Value) {
- if val.(bool) {
- for _, jobW := range jobWs {
- job := jobW.Job
- // Corrupt the DDL job argument.
- if job.Type == model.ActionMultiSchemaChange {
- if len(job.MultiSchemaInfo.SubJobs) > 0 && len(job.MultiSchemaInfo.SubJobs[0].Args) > 0 {
- job.MultiSchemaInfo.SubJobs[0].Args[0] = 1
- }
- } else if len(job.Args) > 0 {
- job.Args[0] = 1
- }
- }
- }
- })
-}
-
-func setJobStateToQueueing(job *model.Job) {
- if job.Type == model.ActionMultiSchemaChange && job.MultiSchemaInfo != nil {
- for _, sub := range job.MultiSchemaInfo.SubJobs {
- sub.State = model.JobStateQueueing
- }
- }
- job.State = model.JobStateQueueing
-}
-
-// addBatchDDLJobs gets global job IDs and puts the DDL jobs in the DDL job table or local worker.
-func (d *ddl) addBatchDDLJobs(jobWs []*JobWrapper) error {
- var err error
-
- if len(jobWs) == 0 {
- return nil
- }
-
- ctx := kv.WithInternalSourceType(d.ctx, kv.InternalTxnDDL)
- se, err := d.sessPool.Get()
- if err != nil {
- return errors.Trace(err)
- }
- defer d.sessPool.Put(se)
- found, err := d.sysTblMgr.HasFlashbackClusterJob(ctx, d.minJobIDRefresher.GetCurrMinJobID())
- if err != nil {
- return errors.Trace(err)
- }
- if found {
- return errors.Errorf("Can't add ddl job, have flashback cluster job")
- }
-
- var (
- startTS = uint64(0)
- bdrRole = string(ast.BDRRoleNone)
- )
-
- err = kv.RunInNewTxn(ctx, d.store, true, func(_ context.Context, txn kv.Transaction) error {
- t := meta.NewMeta(txn)
-
- bdrRole, err = t.GetBDRRole()
- if err != nil {
- return errors.Trace(err)
- }
- startTS = txn.StartTS()
-
- // for localmode, we still need to check this variable if upgrading below v6.2.
- if variable.DDLForce2Queue.Load() {
- if err := d.checkFlashbackJobInQueue(t); err != nil {
- return err
- }
- }
-
- return nil
- })
- if err != nil {
- return errors.Trace(err)
- }
-
- for _, jobW := range jobWs {
- job := jobW.Job
- job.Version = currentVersion
- job.StartTS = startTS
- job.BDRRole = bdrRole
-
- // BDR mode only affects the DDL not from CDC
- if job.CDCWriteSource == 0 && bdrRole != string(ast.BDRRoleNone) {
- if job.Type == model.ActionMultiSchemaChange && job.MultiSchemaInfo != nil {
- for _, subJob := range job.MultiSchemaInfo.SubJobs {
- if ast.DeniedByBDR(ast.BDRRole(bdrRole), subJob.Type, job) {
- return dbterror.ErrBDRRestrictedDDL.FastGenByArgs(bdrRole)
- }
- }
- } else if ast.DeniedByBDR(ast.BDRRole(bdrRole), job.Type, job) {
- return dbterror.ErrBDRRestrictedDDL.FastGenByArgs(bdrRole)
- }
- }
-
- setJobStateToQueueing(job)
-
- // currently doesn't support pause job in local mode.
- if d.stateSyncer.IsUpgradingState() && !hasSysDB(job) && !job.LocalMode {
- if err = pauseRunningJob(sess.NewSession(se), job, model.AdminCommandBySystem); err != nil {
- logutil.DDLUpgradingLogger().Warn("pause user DDL by system failed", zap.Stringer("job", job), zap.Error(err))
- jobW.cacheErr = err
- continue
- }
- logutil.DDLUpgradingLogger().Info("pause user DDL by system successful", zap.Stringer("job", job))
- }
- }
-
- se.GetSessionVars().SetDiskFullOpt(kvrpcpb.DiskFullOpt_AllowedOnAlmostFull)
- ddlSe := sess.NewSession(se)
- localMode := jobWs[0].Job.LocalMode
- if localMode {
- if err = fillJobRelatedGIDs(ctx, ddlSe, jobWs); err != nil {
- return err
- }
- for _, jobW := range jobWs {
- if _, err := jobW.Encode(true); err != nil {
- return err
- }
- d.localJobCh <- jobW
- }
- return nil
- }
-
- if err = GenGIDAndInsertJobsWithRetry(ctx, ddlSe, jobWs); err != nil {
- return errors.Trace(err)
- }
- for _, jobW := range jobWs {
- d.initJobDoneCh(jobW.ID)
- }
-
- return nil
-}
-
-// GenGIDAndInsertJobsWithRetry generate job related global ID and inserts DDL jobs to the DDL job
-// table with retry. job id allocation and job insertion are in the same transaction,
-// as we want to make sure DDL jobs are inserted in id order, then we can query from
-// a min job ID when scheduling DDL jobs to mitigate https://github.com/pingcap/tidb/issues/52905.
-// so this function has side effect, it will set table/db/job id of 'jobs'.
-func GenGIDAndInsertJobsWithRetry(ctx context.Context, ddlSe *sess.Session, jobWs []*JobWrapper) error {
- count := getRequiredGIDCount(jobWs)
- return genGIDAndCallWithRetry(ctx, ddlSe, count, func(ids []int64) error {
- failpoint.Inject("mockGenGlobalIDFail", func(val failpoint.Value) {
- if val.(bool) {
- failpoint.Return(errors.New("gofail genGlobalIDs error"))
- }
- })
- assignGIDsForJobs(jobWs, ids)
- injectModifyJobArgFailPoint(jobWs)
- return insertDDLJobs2Table(ctx, ddlSe, jobWs...)
- })
-}
-
-// fillJobRelatedGIDs similar to GenGIDAndInsertJobsWithRetry, but only fill job related global IDs.
-func fillJobRelatedGIDs(ctx context.Context, ddlSe *sess.Session, jobWs []*JobWrapper) error {
- var allocatedIDs []int64
- count := getRequiredGIDCount(jobWs)
- if err := genGIDAndCallWithRetry(ctx, ddlSe, count, func(ids []int64) error {
- allocatedIDs = ids
- return nil
- }); err != nil {
- return errors.Trace(err)
- }
-
- assignGIDsForJobs(jobWs, allocatedIDs)
- return nil
-}
-
-// getRequiredGIDCount returns the count of required global IDs for the jobs. it's calculated
-// as: the count of jobs + the count of IDs for the jobs which do NOT have pre-allocated ID.
-func getRequiredGIDCount(jobWs []*JobWrapper) int {
- count := len(jobWs)
- idCountForTable := func(info *model.TableInfo) int {
- c := 1
- if partitionInfo := info.GetPartitionInfo(); partitionInfo != nil {
- c += len(partitionInfo.Definitions)
- }
- return c
- }
- for _, jobW := range jobWs {
- if jobW.IDAllocated {
- continue
- }
- switch jobW.Type {
- case model.ActionCreateView, model.ActionCreateSequence, model.ActionCreateTable:
- info := jobW.Args[0].(*model.TableInfo)
- count += idCountForTable(info)
- case model.ActionCreateTables:
- infos := jobW.Args[0].([]*model.TableInfo)
- for _, info := range infos {
- count += idCountForTable(info)
- }
- case model.ActionCreateSchema:
- count++
- }
- // TODO support other type of jobs
- }
- return count
-}
-
-// assignGIDsForJobs should be used with getRequiredGIDCount, and len(ids) must equal
-// what getRequiredGIDCount returns.
-func assignGIDsForJobs(jobWs []*JobWrapper, ids []int64) {
- idx := 0
-
- assignIDsForTable := func(info *model.TableInfo) {
- info.ID = ids[idx]
- idx++
- if partitionInfo := info.GetPartitionInfo(); partitionInfo != nil {
- for i := range partitionInfo.Definitions {
- partitionInfo.Definitions[i].ID = ids[idx]
- idx++
- }
- }
- }
- for _, jobW := range jobWs {
- switch jobW.Type {
- case model.ActionCreateView, model.ActionCreateSequence, model.ActionCreateTable:
- info := jobW.Args[0].(*model.TableInfo)
- if !jobW.IDAllocated {
- assignIDsForTable(info)
- }
- jobW.TableID = info.ID
- case model.ActionCreateTables:
- if !jobW.IDAllocated {
- infos := jobW.Args[0].([]*model.TableInfo)
- for _, info := range infos {
- assignIDsForTable(info)
- }
- }
- case model.ActionCreateSchema:
- dbInfo := jobW.Args[0].(*model.DBInfo)
- if !jobW.IDAllocated {
- dbInfo.ID = ids[idx]
- idx++
- }
- jobW.SchemaID = dbInfo.ID
- }
- // TODO support other type of jobs
- jobW.ID = ids[idx]
- idx++
- }
-}
-
-// genGIDAndCallWithRetry generates global IDs and calls the function with retry.
-// generate ID and call function runs in the same transaction.
-func genGIDAndCallWithRetry(ctx context.Context, ddlSe *sess.Session, count int, fn func(ids []int64) error) error {
- var resErr error
- for i := uint(0); i < kv.MaxRetryCnt; i++ {
- resErr = func() (err error) {
- if err := ddlSe.Begin(ctx); err != nil {
- return errors.Trace(err)
- }
- defer func() {
- if err != nil {
- ddlSe.Rollback()
- }
- }()
- txn, err := ddlSe.Txn()
- if err != nil {
- return errors.Trace(err)
- }
- txn.SetOption(kv.Pessimistic, true)
- forUpdateTS, err := lockGlobalIDKey(ctx, ddlSe, txn)
- if err != nil {
- return errors.Trace(err)
- }
- txn.GetSnapshot().SetOption(kv.SnapshotTS, forUpdateTS)
-
- m := meta.NewMeta(txn)
- ids, err := m.GenGlobalIDs(count)
- if err != nil {
- return errors.Trace(err)
- }
- if err = fn(ids); err != nil {
- return errors.Trace(err)
- }
- return ddlSe.Commit(ctx)
- }()
-
- if resErr != nil && kv.IsTxnRetryableError(resErr) {
- logutil.DDLLogger().Warn("insert job meet retryable error", zap.Error(resErr))
- kv.BackOff(i)
- continue
- }
- break
- }
- return resErr
-}
-
-// lockGlobalIDKey locks the global ID key in the meta store. it keeps trying if
-// meet write conflict, we cannot have a fixed retry count for this error, see this
-// https://github.com/pingcap/tidb/issues/27197#issuecomment-2216315057.
-// this part is same as how we implement pessimistic + repeatable read isolation
-// level in SQL executor, see doLockKeys.
-// NextGlobalID is a meta key, so we cannot use "select xx for update", if we store
-// it into a table row or using advisory lock, we will depends on a system table
-// that is created by us, cyclic. although we can create a system table without using
-// DDL logic, we will only consider change it when we have data dictionary and keep
-// it this way now.
-// TODO maybe we can unify the lock mechanism with SQL executor in the future, or
-// implement it inside TiKV client-go.
-func lockGlobalIDKey(ctx context.Context, ddlSe *sess.Session, txn kv.Transaction) (uint64, error) {
- var (
- iteration uint
- forUpdateTs = txn.StartTS()
- ver kv.Version
- err error
- )
- waitTime := ddlSe.GetSessionVars().LockWaitTimeout
- m := meta.NewMeta(txn)
- idKey := m.GlobalIDKey()
- for {
- lockCtx := tikv.NewLockCtx(forUpdateTs, waitTime, time.Now())
- err = txn.LockKeys(ctx, lockCtx, idKey)
- if err == nil || !terror.ErrorEqual(kv.ErrWriteConflict, err) {
- break
- }
- // ErrWriteConflict contains a conflict-commit-ts in most case, but it cannot
- // be used as forUpdateTs, see comments inside handleAfterPessimisticLockError
- ver, err = ddlSe.GetStore().CurrentVersion(oracle.GlobalTxnScope)
- if err != nil {
- break
- }
- forUpdateTs = ver.Ver
-
- kv.BackOff(iteration)
- // avoid it keep growing and overflow.
- iteration = min(iteration+1, math.MaxInt)
- }
- return forUpdateTs, err
-}
-
-// combineBatchCreateTableJobs combine batch jobs to another batch jobs.
-// currently it only support combine CreateTable to CreateTables.
-func combineBatchCreateTableJobs(jobWs []*JobWrapper) ([]*JobWrapper, error) {
- if len(jobWs) <= 1 {
- return jobWs, nil
- }
- var schemaName string
- jobs := make([]*model.Job, 0, len(jobWs))
- for i, jobW := range jobWs {
- // we don't merge jobs with ID pre-allocated.
- if jobW.Job.Type != model.ActionCreateTable || jobW.IDAllocated {
- return jobWs, nil
- }
- if i == 0 {
- schemaName = jobW.Job.SchemaName
- } else if jobW.Job.SchemaName != schemaName {
- return jobWs, nil
- }
- jobs = append(jobs, jobW.Job)
- }
-
- job, err := BatchCreateTableWithJobs(jobs)
- if err != nil {
- return jobWs, err
- }
- logutil.DDLLogger().Info("combine jobs to batch create table job", zap.Int("len", len(jobWs)))
-
- newJobW := &JobWrapper{
- Job: job,
- ErrChs: []chan error{},
- }
- // combine the error chans.
- for _, j := range jobWs {
- newJobW.ErrChs = append(newJobW.ErrChs, j.ErrChs...)
- }
- return []*JobWrapper{newJobW}, nil
-}
-
func injectFailPointForGetJob(job *model.Job) {
if job == nil {
return
@@ -826,38 +290,6 @@ func (w *worker) registerMDLInfo(job *model.Job, ver int64) error {
return err
}
-// cleanMDLInfo cleans metadata lock info.
-func (s *jobScheduler) cleanMDLInfo(job *model.Job, ownerID string) {
- if !variable.EnableMDL.Load() {
- return
- }
- var sql string
- if tidbutil.IsSysDB(strings.ToLower(job.SchemaName)) {
- // DDLs that modify system tables could only happen in upgrade process,
- // we should not reference 'owner_id'. Otherwise, there is a circular blocking problem.
- sql = fmt.Sprintf("delete from mysql.tidb_mdl_info where job_id = %d", job.ID)
- } else {
- sql = fmt.Sprintf("delete from mysql.tidb_mdl_info where job_id = %d and owner_id = '%s'", job.ID, ownerID)
- }
- sctx, _ := s.sessPool.Get()
- defer s.sessPool.Put(sctx)
- se := sess.NewSession(sctx)
- se.GetSessionVars().SetDiskFullOpt(kvrpcpb.DiskFullOpt_AllowedOnAlmostFull)
- _, err := se.Execute(s.schCtx, sql, "delete-mdl-info")
- if err != nil {
- logutil.DDLLogger().Warn("unexpected error when clean mdl info", zap.Int64("job ID", job.ID), zap.Error(err))
- return
- }
- // TODO we need clean it when version of JobStateRollbackDone is synced also.
- if job.State == model.JobStateSynced && s.etcdCli != nil {
- path := fmt.Sprintf("%s/%d/", util.DDLAllSchemaVersionsByJob, job.ID)
- _, err = s.etcdCli.Delete(s.schCtx, path, clientv3.WithPrefix())
- if err != nil {
- logutil.DDLLogger().Warn("delete versions failed", zap.Int64("job ID", job.ID), zap.Error(err))
- }
- }
-}
-
// JobNeedGC is called to determine whether delete-ranges need to be generated for the provided job.
//
// NOTICE: BR also uses jobNeedGC to determine whether delete-ranges need to be generated for the provided job.
@@ -925,13 +357,9 @@ func (w *worker) finishDDLJob(t *meta.Meta, job *model.Job) (err error) {
if err != nil {
return errors.Trace(err)
}
- // for local mode job, we didn't insert the job to ddl table now.
- // so no need to delete it.
- if !job.LocalMode {
- err = w.deleteDDLJob(job)
- if err != nil {
- return errors.Trace(err)
- }
+ err = w.deleteDDLJob(job)
+ if err != nil {
+ return errors.Trace(err)
}
job.BinlogInfo.FinishedTS = t.StartTS
@@ -949,6 +377,12 @@ func (w *worker) finishDDLJob(t *meta.Meta, job *model.Job) (err error) {
return errors.Trace(err)
}
+func (w *worker) deleteDDLJob(job *model.Job) error {
+ sql := fmt.Sprintf("delete from mysql.tidb_ddl_job where job_id = %d", job.ID)
+ _, err := w.sess.Execute(context.Background(), sql, "delete_job")
+ return errors.Trace(err)
+}
+
func finishRecoverTable(w *worker, job *model.Job) error {
var (
recoverInfo *RecoverInfo
@@ -985,7 +419,7 @@ func finishRecoverSchema(w *worker, job *model.Job) error {
return nil
}
-func (w *JobContext) setDDLLabelForTopSQL(jobQuery string) {
+func (w *ReorgContext) setDDLLabelForTopSQL(jobQuery string) {
if !topsqlstate.TopSQLEnabled() || jobQuery == "" {
return
}
@@ -1014,7 +448,7 @@ func getDDLRequestSource(jobType model.ActionType) string {
return kv.InternalTxnDDL
}
-func (w *JobContext) setDDLLabelForDiagnosis(jobType model.ActionType) {
+func (w *ReorgContext) setDDLLabelForDiagnosis(jobType model.ActionType) {
if w.tp != "" {
return
}
@@ -1077,7 +511,7 @@ func (w *worker) prepareTxn(job *model.Job) (kv.Transaction, error) {
//
// The first return value is the schema version after running the job. If it's
// non-zero, caller should wait for other nodes to catch up.
-func (w *worker) transitOneJobStep(d *ddlCtx, job *model.Job) (int64, error) {
+func (w *worker) transitOneJobStep(d *ddlCtx, jobCtx *jobContext, job *model.Job) (int64, error) {
var (
err error
)
@@ -1086,12 +520,7 @@ func (w *worker) transitOneJobStep(d *ddlCtx, job *model.Job) (int64, error) {
if err != nil {
return 0, err
}
- var t *meta.Meta
- if variable.EnableFastCreateTable.Load() {
- t = meta.NewMeta(txn, meta.WithUpdateTableName())
- } else {
- t = meta.NewMeta(txn)
- }
+ t := meta.NewMeta(txn)
if job.IsDone() || job.IsRollbackDone() || job.IsCancelled() {
if job.IsDone() {
@@ -1114,26 +543,22 @@ func (w *worker) transitOneJobStep(d *ddlCtx, job *model.Job) (int64, error) {
})
return 0, w.handleJobDone(d, job, t)
}
- d.mu.RLock()
- d.mu.hook.OnJobRunBefore(job)
- d.mu.RUnlock()
+ failpoint.InjectCall("onJobRunBefore", job)
// If running job meets error, we will save this error in job Error and retry
// later if the job is not cancelled.
- schemaVer, updateRawArgs, runJobErr := w.runOneJobStep(d, t, job)
+ schemaVer, updateRawArgs, runJobErr := w.runOneJobStep(jobCtx, t, job)
- d.mu.RLock()
- d.mu.hook.OnJobRunAfter(job)
- d.mu.RUnlock()
+ failpoint.InjectCall("onJobRunAfter", job)
if job.IsCancelled() {
- defer d.unlockSchemaVersion(job.ID)
+ defer jobCtx.unlockSchemaVersion(job.ID)
w.sess.Reset()
return 0, w.handleJobDone(d, job, t)
}
if err = w.checkBeforeCommit(); err != nil {
- d.unlockSchemaVersion(job.ID)
+ jobCtx.unlockSchemaVersion(job.ID)
return 0, err
}
@@ -1153,24 +578,24 @@ func (w *worker) transitOneJobStep(d *ddlCtx, job *model.Job) (int64, error) {
err = w.registerMDLInfo(job, schemaVer)
if err != nil {
w.sess.Rollback()
- d.unlockSchemaVersion(job.ID)
+ jobCtx.unlockSchemaVersion(job.ID)
return 0, err
}
err = w.updateDDLJob(job, updateRawArgs)
if err = w.handleUpdateJobError(t, job, err); err != nil {
w.sess.Rollback()
- d.unlockSchemaVersion(job.ID)
+ jobCtx.unlockSchemaVersion(job.ID)
return 0, err
}
writeBinlog(d.binlogCli, txn, job)
// reset the SQL digest to make topsql work right.
w.sess.GetSessionVars().StmtCtx.ResetSQLDigest(job.Query)
err = w.sess.Commit(w.ctx)
- d.unlockSchemaVersion(job.ID)
+ jobCtx.unlockSchemaVersion(job.ID)
if err != nil {
return 0, err
}
- w.registerSync(job)
+ jobCtx.addUnSynced(job.ID)
// If error is non-retryable, we can ignore the sleep.
if runJobErr != nil && errorIsRetryable(runJobErr, job) {
@@ -1188,7 +613,7 @@ func (w *worker) transitOneJobStep(d *ddlCtx, job *model.Job) (int64, error) {
}
func (w *worker) checkBeforeCommit() error {
- if !w.ddlCtx.isOwner() && w.tp != localWorker {
+ if !w.ddlCtx.isOwner() {
// Since this TiDB instance is not a DDL owner anymore,
// it should not commit any transaction.
w.sess.Rollback()
@@ -1202,54 +627,7 @@ func (w *worker) checkBeforeCommit() error {
return nil
}
-// HandleLocalDDLJob handles local ddl job like fast create table.
-// Compare with normal ddl job:
-// 1. directly insert the job to history job table(incompatible with CDC).
-// 2. no need to wait schema version(only support create table now).
-// 3. no register mdl info(only support create table now).
-func (w *worker) HandleLocalDDLJob(d *ddlCtx, job *model.Job) (err error) {
- txn, err := w.prepareTxn(job)
- if err != nil {
- return err
- }
-
- t := meta.NewMeta(txn, meta.WithUpdateTableName())
- d.mu.RLock()
- d.mu.hook.OnJobRunBefore(job)
- d.mu.RUnlock()
-
- _, _, err = w.runOneJobStep(d, t, job)
- defer d.unlockSchemaVersion(job.ID)
- if err != nil {
- return err
- }
- // no need to rollback for fast create table now.
- if job.IsCancelling() {
- job.State = model.JobStateCancelled
- job.Error = dbterror.ErrCancelledDDLJob
- }
- if job.IsCancelled() {
- w.sess.Reset()
- if err = w.handleJobDone(d, job, t); err != nil {
- return err
- }
- // return job.Error to let caller know the job is cancelled.
- return job.Error
- }
-
- d.mu.RLock()
- d.mu.hook.OnJobRunAfter(job)
- d.mu.RUnlock()
-
- writeBinlog(d.binlogCli, txn, job)
- // reset the SQL digest to make topsql work right.
- w.sess.GetSessionVars().StmtCtx.ResetSQLDigest(job.Query)
-
- job.State = model.JobStateSynced
- return w.handleJobDone(d, job, t)
-}
-
-func (w *JobContext) getResourceGroupTaggerForTopSQL() tikvrpc.ResourceGroupTagger {
+func (w *ReorgContext) getResourceGroupTaggerForTopSQL() tikvrpc.ResourceGroupTagger {
if !topsqlstate.TopSQLEnabled() || w.cacheDigest == nil {
return nil
}
@@ -1262,7 +640,7 @@ func (w *JobContext) getResourceGroupTaggerForTopSQL() tikvrpc.ResourceGroupTagg
return tagger
}
-func (w *JobContext) ddlJobSourceType() string {
+func (w *ReorgContext) ddlJobSourceType() string {
return w.tp
}
@@ -1384,7 +762,7 @@ func (w *worker) processJobPausingRequest(d *ddlCtx, job *model.Job) (isRunnable
// synchronized. So JobStateSynced *step* is added to make sure there is
// waitSchemaChanged to wait for all nodes to catch up JobStateDone.
func (w *worker) runOneJobStep(
- d *ddlCtx,
+ jobCtx *jobContext,
t *meta.Meta,
job *model.Job,
) (ver int64, updateRawArgs bool, err error) {
@@ -1409,14 +787,14 @@ func (w *worker) runOneJobStep(
if job.IsCancelling() {
w.jobLogger(job).Debug("cancel DDL job", zap.String("job", job.String()))
- ver, err = convertJob2RollbackJob(w, d, t, job)
+ ver, err = convertJob2RollbackJob(w, jobCtx, t, job)
// if job is converted to rollback job, the job.Args may be changed for the
// rollback logic, so we let caller persist the new arguments.
updateRawArgs = job.IsRollingback()
return
}
- isRunnable, err := w.processJobPausingRequest(d, job)
+ isRunnable, err := w.processJobPausingRequest(jobCtx.oldDDLCtx, job)
if !isRunnable {
return ver, false, err
}
@@ -1435,128 +813,128 @@ func (w *worker) runOneJobStep(
// change has no effect when retrying it.
switch job.Type {
case model.ActionCreateSchema:
- ver, err = onCreateSchema(d, t, job)
+ ver, err = onCreateSchema(jobCtx, t, job)
case model.ActionModifySchemaCharsetAndCollate:
- ver, err = onModifySchemaCharsetAndCollate(d, t, job)
+ ver, err = onModifySchemaCharsetAndCollate(jobCtx, t, job)
case model.ActionDropSchema:
- ver, err = onDropSchema(d, t, job)
+ ver, err = onDropSchema(jobCtx, t, job)
case model.ActionRecoverSchema:
- ver, err = w.onRecoverSchema(d, t, job)
+ ver, err = w.onRecoverSchema(jobCtx, t, job)
case model.ActionModifySchemaDefaultPlacement:
- ver, err = onModifySchemaDefaultPlacement(d, t, job)
+ ver, err = onModifySchemaDefaultPlacement(jobCtx, t, job)
case model.ActionCreateTable:
- ver, err = onCreateTable(d, t, job)
+ ver, err = onCreateTable(jobCtx, t, job)
case model.ActionCreateTables:
- ver, err = onCreateTables(d, t, job)
+ ver, err = onCreateTables(jobCtx, t, job)
case model.ActionRepairTable:
- ver, err = onRepairTable(d, t, job)
+ ver, err = onRepairTable(jobCtx, t, job)
case model.ActionCreateView:
- ver, err = onCreateView(d, t, job)
+ ver, err = onCreateView(jobCtx, t, job)
case model.ActionDropTable, model.ActionDropView, model.ActionDropSequence:
- ver, err = onDropTableOrView(d, t, job)
+ ver, err = onDropTableOrView(jobCtx, t, job)
case model.ActionDropTablePartition:
- ver, err = w.onDropTablePartition(d, t, job)
+ ver, err = w.onDropTablePartition(jobCtx, t, job)
case model.ActionTruncateTablePartition:
- ver, err = w.onTruncateTablePartition(d, t, job)
+ ver, err = w.onTruncateTablePartition(jobCtx, t, job)
case model.ActionExchangeTablePartition:
- ver, err = w.onExchangeTablePartition(d, t, job)
+ ver, err = w.onExchangeTablePartition(jobCtx, t, job)
case model.ActionAddColumn:
- ver, err = onAddColumn(d, t, job)
+ ver, err = onAddColumn(jobCtx, t, job)
case model.ActionDropColumn:
- ver, err = onDropColumn(d, t, job)
+ ver, err = onDropColumn(jobCtx, t, job)
case model.ActionModifyColumn:
- ver, err = w.onModifyColumn(d, t, job)
+ ver, err = w.onModifyColumn(jobCtx, t, job)
case model.ActionSetDefaultValue:
- ver, err = onSetDefaultValue(d, t, job)
+ ver, err = onSetDefaultValue(jobCtx, t, job)
case model.ActionAddIndex:
- ver, err = w.onCreateIndex(d, t, job, false)
+ ver, err = w.onCreateIndex(jobCtx, t, job, false)
case model.ActionAddPrimaryKey:
- ver, err = w.onCreateIndex(d, t, job, true)
+ ver, err = w.onCreateIndex(jobCtx, t, job, true)
case model.ActionDropIndex, model.ActionDropPrimaryKey:
- ver, err = onDropIndex(d, t, job)
+ ver, err = onDropIndex(jobCtx, t, job)
case model.ActionRenameIndex:
- ver, err = onRenameIndex(d, t, job)
+ ver, err = onRenameIndex(jobCtx, t, job)
case model.ActionAddForeignKey:
- ver, err = w.onCreateForeignKey(d, t, job)
+ ver, err = w.onCreateForeignKey(jobCtx, t, job)
case model.ActionDropForeignKey:
- ver, err = onDropForeignKey(d, t, job)
+ ver, err = onDropForeignKey(jobCtx, t, job)
case model.ActionTruncateTable:
- ver, err = w.onTruncateTable(d, t, job)
+ ver, err = w.onTruncateTable(jobCtx, t, job)
case model.ActionRebaseAutoID:
- ver, err = onRebaseAutoIncrementIDType(d, t, job)
+ ver, err = onRebaseAutoIncrementIDType(jobCtx, t, job)
case model.ActionRebaseAutoRandomBase:
- ver, err = onRebaseAutoRandomType(d, t, job)
+ ver, err = onRebaseAutoRandomType(jobCtx, t, job)
case model.ActionRenameTable:
- ver, err = onRenameTable(d, t, job)
+ ver, err = onRenameTable(jobCtx, t, job)
case model.ActionShardRowID:
- ver, err = w.onShardRowID(d, t, job)
+ ver, err = w.onShardRowID(jobCtx, t, job)
case model.ActionModifyTableComment:
- ver, err = onModifyTableComment(d, t, job)
+ ver, err = onModifyTableComment(jobCtx, t, job)
case model.ActionModifyTableAutoIdCache:
- ver, err = onModifyTableAutoIDCache(d, t, job)
+ ver, err = onModifyTableAutoIDCache(jobCtx, t, job)
case model.ActionAddTablePartition:
- ver, err = w.onAddTablePartition(d, t, job)
+ ver, err = w.onAddTablePartition(jobCtx, t, job)
case model.ActionModifyTableCharsetAndCollate:
- ver, err = onModifyTableCharsetAndCollate(d, t, job)
+ ver, err = onModifyTableCharsetAndCollate(jobCtx, t, job)
case model.ActionRecoverTable:
- ver, err = w.onRecoverTable(d, t, job)
+ ver, err = w.onRecoverTable(jobCtx, t, job)
case model.ActionLockTable:
- ver, err = onLockTables(d, t, job)
+ ver, err = onLockTables(jobCtx, t, job)
case model.ActionUnlockTable:
- ver, err = onUnlockTables(d, t, job)
+ ver, err = onUnlockTables(jobCtx, t, job)
case model.ActionSetTiFlashReplica:
- ver, err = w.onSetTableFlashReplica(d, t, job)
+ ver, err = w.onSetTableFlashReplica(jobCtx, t, job)
case model.ActionUpdateTiFlashReplicaStatus:
- ver, err = onUpdateFlashReplicaStatus(d, t, job)
+ ver, err = onUpdateFlashReplicaStatus(jobCtx, t, job)
case model.ActionCreateSequence:
- ver, err = onCreateSequence(d, t, job)
+ ver, err = onCreateSequence(jobCtx, t, job)
case model.ActionAlterIndexVisibility:
- ver, err = onAlterIndexVisibility(d, t, job)
+ ver, err = onAlterIndexVisibility(jobCtx, t, job)
case model.ActionAlterSequence:
- ver, err = onAlterSequence(d, t, job)
+ ver, err = onAlterSequence(jobCtx, t, job)
case model.ActionRenameTables:
- ver, err = onRenameTables(d, t, job)
+ ver, err = onRenameTables(jobCtx, t, job)
case model.ActionAlterTableAttributes:
- ver, err = onAlterTableAttributes(d, t, job)
+ ver, err = onAlterTableAttributes(jobCtx, t, job)
case model.ActionAlterTablePartitionAttributes:
- ver, err = onAlterTablePartitionAttributes(d, t, job)
+ ver, err = onAlterTablePartitionAttributes(jobCtx, t, job)
case model.ActionCreatePlacementPolicy:
- ver, err = onCreatePlacementPolicy(d, t, job)
+ ver, err = onCreatePlacementPolicy(jobCtx, t, job)
case model.ActionDropPlacementPolicy:
- ver, err = onDropPlacementPolicy(d, t, job)
+ ver, err = onDropPlacementPolicy(jobCtx, t, job)
case model.ActionAlterPlacementPolicy:
- ver, err = onAlterPlacementPolicy(d, t, job)
+ ver, err = onAlterPlacementPolicy(jobCtx, t, job)
case model.ActionAlterTablePartitionPlacement:
- ver, err = onAlterTablePartitionPlacement(d, t, job)
+ ver, err = onAlterTablePartitionPlacement(jobCtx, t, job)
case model.ActionAlterTablePlacement:
- ver, err = onAlterTablePlacement(d, t, job)
+ ver, err = onAlterTablePlacement(jobCtx, t, job)
case model.ActionCreateResourceGroup:
- ver, err = onCreateResourceGroup(w.ctx, d, t, job)
+ ver, err = onCreateResourceGroup(jobCtx, t, job)
case model.ActionAlterResourceGroup:
- ver, err = onAlterResourceGroup(d, t, job)
+ ver, err = onAlterResourceGroup(jobCtx, t, job)
case model.ActionDropResourceGroup:
- ver, err = onDropResourceGroup(d, t, job)
+ ver, err = onDropResourceGroup(jobCtx, t, job)
case model.ActionAlterCacheTable:
- ver, err = onAlterCacheTable(d, t, job)
+ ver, err = onAlterCacheTable(jobCtx, t, job)
case model.ActionAlterNoCacheTable:
- ver, err = onAlterNoCacheTable(d, t, job)
+ ver, err = onAlterNoCacheTable(jobCtx, t, job)
case model.ActionFlashbackCluster:
- ver, err = w.onFlashbackCluster(d, t, job)
+ ver, err = w.onFlashbackCluster(jobCtx, t, job)
case model.ActionMultiSchemaChange:
- ver, err = onMultiSchemaChange(w, d, t, job)
+ ver, err = onMultiSchemaChange(w, jobCtx, t, job)
case model.ActionReorganizePartition, model.ActionRemovePartitioning,
model.ActionAlterTablePartitioning:
- ver, err = w.onReorganizePartition(d, t, job)
+ ver, err = w.onReorganizePartition(jobCtx, t, job)
case model.ActionAlterTTLInfo:
- ver, err = onTTLInfoChange(d, t, job)
+ ver, err = onTTLInfoChange(jobCtx, t, job)
case model.ActionAlterTTLRemove:
- ver, err = onTTLInfoRemove(d, t, job)
+ ver, err = onTTLInfoRemove(jobCtx, t, job)
case model.ActionAddCheckConstraint:
- ver, err = w.onAddCheckConstraint(d, t, job)
+ ver, err = w.onAddCheckConstraint(jobCtx, t, job)
case model.ActionDropCheckConstraint:
- ver, err = onDropCheckConstraint(d, t, job)
+ ver, err = onDropCheckConstraint(jobCtx, t, job)
case model.ActionAlterCheckConstraint:
- ver, err = w.onAlterCheckConstraint(d, t, job)
+ ver, err = w.onAlterCheckConstraint(jobCtx, t, job)
default:
// Invalid job, cancel it.
job.State = model.JobStateCancelled
@@ -1621,7 +999,7 @@ func waitSchemaChanged(ctx context.Context, d *ddlCtx, latestSchemaVersion int64
return nil
}
- err = d.schemaSyncer.OwnerUpdateGlobalVersion(ctx, latestSchemaVersion)
+ err = d.schemaVerSyncer.OwnerUpdateGlobalVersion(ctx, latestSchemaVersion)
if err != nil {
logutil.DDLLogger().Info("update latest schema version failed", zap.Int64("ver", latestSchemaVersion), zap.Error(err))
if variable.EnableMDL.Load() {
diff --git a/pkg/ddl/ddl_worker_test.go b/pkg/ddl/job_worker_test.go
similarity index 99%
rename from pkg/ddl/ddl_worker_test.go
rename to pkg/ddl/job_worker_test.go
index 38dff40a70369..66e3e26227744 100644
--- a/pkg/ddl/ddl_worker_test.go
+++ b/pkg/ddl/job_worker_test.go
@@ -38,7 +38,7 @@ func TestCheckOwner(t *testing.T) {
time.Sleep(testLease)
require.Equal(t, dom.DDL().OwnerManager().IsOwner(), true)
- require.Equal(t, dom.DDL().GetLease(), testLease)
+ require.Equal(t, dom.GetSchemaLease(), testLease)
}
func TestInvalidDDLJob(t *testing.T) {
diff --git a/pkg/ddl/mock.go b/pkg/ddl/mock.go
index 7be8f499fa01e..11e1de3669a88 100644
--- a/pkg/ddl/mock.go
+++ b/pkg/ddl/mock.go
@@ -16,20 +16,12 @@ package ddl
import (
"context"
- "sync"
- "sync/atomic"
- "time"
"github.com/pingcap/errors"
- "github.com/pingcap/failpoint"
- "github.com/pingcap/tidb/pkg/ddl/syncer"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/charset"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/sessionctx"
- "github.com/pingcap/tidb/pkg/sessionctx/variable"
- clientv3 "go.etcd.io/etcd/client/v3"
- atomicutil "go.uber.org/atomic"
)
// SetBatchInsertDeleteRangeSize sets the batch insert/delete range size in the test
@@ -37,179 +29,6 @@ func SetBatchInsertDeleteRangeSize(i int) {
batchInsertDeleteRangeSize = i
}
-var _ syncer.SchemaSyncer = &MockSchemaSyncer{}
-
-const mockCheckVersInterval = 2 * time.Millisecond
-
-// MockSchemaSyncer is a mock schema syncer, it is exported for testing.
-type MockSchemaSyncer struct {
- selfSchemaVersion int64
- mdlSchemaVersions sync.Map
- globalVerCh chan clientv3.WatchResponse
- mockSession chan struct{}
-}
-
-// NewMockSchemaSyncer creates a new mock SchemaSyncer.
-func NewMockSchemaSyncer() syncer.SchemaSyncer {
- return &MockSchemaSyncer{}
-}
-
-// Init implements SchemaSyncer.Init interface.
-func (s *MockSchemaSyncer) Init(_ context.Context) error {
- s.mdlSchemaVersions = sync.Map{}
- s.globalVerCh = make(chan clientv3.WatchResponse, 1)
- s.mockSession = make(chan struct{}, 1)
- return nil
-}
-
-// GlobalVersionCh implements SchemaSyncer.GlobalVersionCh interface.
-func (s *MockSchemaSyncer) GlobalVersionCh() clientv3.WatchChan {
- return s.globalVerCh
-}
-
-// WatchGlobalSchemaVer implements SchemaSyncer.WatchGlobalSchemaVer interface.
-func (*MockSchemaSyncer) WatchGlobalSchemaVer(context.Context) {}
-
-// UpdateSelfVersion implements SchemaSyncer.UpdateSelfVersion interface.
-func (s *MockSchemaSyncer) UpdateSelfVersion(_ context.Context, jobID int64, version int64) error {
- failpoint.Inject("mockUpdateMDLToETCDError", func(val failpoint.Value) {
- if val.(bool) {
- failpoint.Return(errors.New("mock update mdl to etcd error"))
- }
- })
- if variable.EnableMDL.Load() {
- s.mdlSchemaVersions.Store(jobID, version)
- } else {
- atomic.StoreInt64(&s.selfSchemaVersion, version)
- }
- return nil
-}
-
-// Done implements SchemaSyncer.Done interface.
-func (s *MockSchemaSyncer) Done() <-chan struct{} {
- return s.mockSession
-}
-
-// CloseSession mockSession, it is exported for testing.
-func (s *MockSchemaSyncer) CloseSession() {
- close(s.mockSession)
-}
-
-// Restart implements SchemaSyncer.Restart interface.
-func (s *MockSchemaSyncer) Restart(_ context.Context) error {
- s.mockSession = make(chan struct{}, 1)
- return nil
-}
-
-// OwnerUpdateGlobalVersion implements SchemaSyncer.OwnerUpdateGlobalVersion interface.
-func (s *MockSchemaSyncer) OwnerUpdateGlobalVersion(_ context.Context, _ int64) error {
- select {
- case s.globalVerCh <- clientv3.WatchResponse{}:
- default:
- }
- return nil
-}
-
-// OwnerCheckAllVersions implements SchemaSyncer.OwnerCheckAllVersions interface.
-func (s *MockSchemaSyncer) OwnerCheckAllVersions(ctx context.Context, jobID int64, latestVer int64) error {
- ticker := time.NewTicker(mockCheckVersInterval)
- defer ticker.Stop()
-
- failpoint.Inject("mockOwnerCheckAllVersionSlow", func(val failpoint.Value) {
- if v, ok := val.(int); ok && v == int(jobID) {
- time.Sleep(2 * time.Second)
- }
- })
-
- for {
- select {
- case <-ctx.Done():
- failpoint.Inject("checkOwnerCheckAllVersionsWaitTime", func(v failpoint.Value) {
- if v.(bool) {
- panic("shouldn't happen")
- }
- })
- return errors.Trace(ctx.Err())
- case <-ticker.C:
- if variable.EnableMDL.Load() {
- ver, ok := s.mdlSchemaVersions.Load(jobID)
- if ok && ver.(int64) >= latestVer {
- return nil
- }
- } else {
- ver := atomic.LoadInt64(&s.selfSchemaVersion)
- if ver >= latestVer {
- return nil
- }
- }
- }
- }
-}
-
-// SyncJobSchemaVerLoop implements SchemaSyncer.SyncJobSchemaVerLoop interface.
-func (*MockSchemaSyncer) SyncJobSchemaVerLoop(context.Context) {
-}
-
-// Close implements SchemaSyncer.Close interface.
-func (*MockSchemaSyncer) Close() {}
-
-// NewMockStateSyncer creates a new mock StateSyncer.
-func NewMockStateSyncer() syncer.StateSyncer {
- return &MockStateSyncer{}
-}
-
-// clusterState mocks cluster state.
-// We move it from MockStateSyncer to here. Because we want to make it unaffected by ddl close.
-var clusterState *atomicutil.Pointer[syncer.StateInfo]
-
-// MockStateSyncer is a mock state syncer, it is exported for testing.
-type MockStateSyncer struct {
- globalVerCh chan clientv3.WatchResponse
- mockSession chan struct{}
-}
-
-// Init implements StateSyncer.Init interface.
-func (s *MockStateSyncer) Init(context.Context) error {
- s.globalVerCh = make(chan clientv3.WatchResponse, 1)
- s.mockSession = make(chan struct{}, 1)
- state := syncer.NewStateInfo(syncer.StateNormalRunning)
- if clusterState == nil {
- clusterState = atomicutil.NewPointer(state)
- }
- return nil
-}
-
-// UpdateGlobalState implements StateSyncer.UpdateGlobalState interface.
-func (s *MockStateSyncer) UpdateGlobalState(_ context.Context, stateInfo *syncer.StateInfo) error {
- failpoint.Inject("mockUpgradingState", func(val failpoint.Value) {
- if val.(bool) {
- clusterState.Store(stateInfo)
- failpoint.Return(nil)
- }
- })
- s.globalVerCh <- clientv3.WatchResponse{}
- clusterState.Store(stateInfo)
- return nil
-}
-
-// GetGlobalState implements StateSyncer.GetGlobalState interface.
-func (*MockStateSyncer) GetGlobalState(context.Context) (*syncer.StateInfo, error) {
- return clusterState.Load(), nil
-}
-
-// IsUpgradingState implements StateSyncer.IsUpgradingState interface.
-func (*MockStateSyncer) IsUpgradingState() bool {
- return clusterState.Load().State == syncer.StateUpgrading
-}
-
-// WatchChan implements StateSyncer.WatchChan interface.
-func (s *MockStateSyncer) WatchChan() clientv3.WatchChan {
- return s.globalVerCh
-}
-
-// Rewatch implements StateSyncer.Rewatch interface.
-func (*MockStateSyncer) Rewatch(context.Context) {}
-
type mockDelRange struct {
}
diff --git a/pkg/ddl/modify_column.go b/pkg/ddl/modify_column.go
new file mode 100644
index 0000000000000..4978bb73397e1
--- /dev/null
+++ b/pkg/ddl/modify_column.go
@@ -0,0 +1,1314 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ddl
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "strings"
+
+ "github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
+ "github.com/pingcap/tidb/pkg/ddl/logutil"
+ sess "github.com/pingcap/tidb/pkg/ddl/session"
+ "github.com/pingcap/tidb/pkg/errctx"
+ "github.com/pingcap/tidb/pkg/expression"
+ exprctx "github.com/pingcap/tidb/pkg/expression/context"
+ "github.com/pingcap/tidb/pkg/infoschema"
+ "github.com/pingcap/tidb/pkg/kv"
+ "github.com/pingcap/tidb/pkg/meta"
+ "github.com/pingcap/tidb/pkg/meta/autoid"
+ "github.com/pingcap/tidb/pkg/metrics"
+ "github.com/pingcap/tidb/pkg/parser"
+ "github.com/pingcap/tidb/pkg/parser/ast"
+ "github.com/pingcap/tidb/pkg/parser/charset"
+ "github.com/pingcap/tidb/pkg/parser/format"
+ "github.com/pingcap/tidb/pkg/parser/model"
+ "github.com/pingcap/tidb/pkg/parser/mysql"
+ "github.com/pingcap/tidb/pkg/sessionctx"
+ "github.com/pingcap/tidb/pkg/sessionctx/variable"
+ statsutil "github.com/pingcap/tidb/pkg/statistics/handle/util"
+ "github.com/pingcap/tidb/pkg/table"
+ "github.com/pingcap/tidb/pkg/table/tables"
+ "github.com/pingcap/tidb/pkg/types"
+ "github.com/pingcap/tidb/pkg/util"
+ "github.com/pingcap/tidb/pkg/util/dbterror"
+ "go.uber.org/zap"
+)
+
+type modifyingColInfo struct {
+ newCol *model.ColumnInfo
+ oldColName *model.CIStr
+ modifyColumnTp byte
+ updatedAutoRandomBits uint64
+ changingCol *model.ColumnInfo
+ changingIdxs []*model.IndexInfo
+ pos *ast.ColumnPosition
+ removedIdxs []int64
+}
+
+func (w *worker) onModifyColumn(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+ dbInfo, tblInfo, oldCol, modifyInfo, err := getModifyColumnInfo(t, job)
+ if err != nil {
+ return ver, err
+ }
+
+ if job.IsRollingback() {
+ // For those column-type-change jobs which don't reorg the data.
+ if !needChangeColumnData(oldCol, modifyInfo.newCol) {
+ return rollbackModifyColumnJob(jobCtx, t, tblInfo, job, modifyInfo.newCol, oldCol, modifyInfo.modifyColumnTp)
+ }
+ // For those column-type-change jobs which reorg the data.
+ return rollbackModifyColumnJobWithData(jobCtx, t, tblInfo, job, oldCol, modifyInfo)
+ }
+
+ // If we want to rename the column name, we need to check whether it already exists.
+ if modifyInfo.newCol.Name.L != modifyInfo.oldColName.L {
+ c := model.FindColumnInfo(tblInfo.Columns, modifyInfo.newCol.Name.L)
+ if c != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(infoschema.ErrColumnExists.GenWithStackByArgs(modifyInfo.newCol.Name))
+ }
+ }
+
+ failpoint.Inject("uninitializedOffsetAndState", func(val failpoint.Value) {
+ //nolint:forcetypeassert
+ if val.(bool) {
+ if modifyInfo.newCol.State != model.StatePublic {
+ failpoint.Return(ver, errors.New("the column state is wrong"))
+ }
+ }
+ })
+
+ err = checkAndApplyAutoRandomBits(jobCtx, t, dbInfo, tblInfo, oldCol, modifyInfo.newCol, modifyInfo.updatedAutoRandomBits)
+ if err != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+
+ if !needChangeColumnData(oldCol, modifyInfo.newCol) {
+ return w.doModifyColumn(jobCtx, t, job, dbInfo, tblInfo, modifyInfo.newCol, oldCol, modifyInfo.pos)
+ }
+
+ if err = isGeneratedRelatedColumn(tblInfo, modifyInfo.newCol, oldCol); err != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+ if tblInfo.Partition != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs("table is partition table"))
+ }
+
+ changingCol := modifyInfo.changingCol
+ if changingCol == nil {
+ newColName := model.NewCIStr(genChangingColumnUniqueName(tblInfo, oldCol))
+ if mysql.HasPriKeyFlag(oldCol.GetFlag()) {
+ job.State = model.JobStateCancelled
+ msg := "this column has primary key flag"
+ return ver, dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs(msg)
+ }
+
+ changingCol = modifyInfo.newCol.Clone()
+ changingCol.Name = newColName
+ changingCol.ChangeStateInfo = &model.ChangeStateInfo{DependencyColumnOffset: oldCol.Offset}
+
+ originDefVal, err := GetOriginDefaultValueForModifyColumn(newReorgExprCtx(), changingCol, oldCol)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ if err = changingCol.SetOriginDefaultValue(originDefVal); err != nil {
+ return ver, errors.Trace(err)
+ }
+
+ InitAndAddColumnToTable(tblInfo, changingCol)
+ indexesToChange := FindRelatedIndexesToChange(tblInfo, oldCol.Name)
+ for _, info := range indexesToChange {
+ newIdxID := AllocateIndexID(tblInfo)
+ if !info.isTemp {
+ // We create a temp index for each normal index.
+ tmpIdx := info.IndexInfo.Clone()
+ tmpIdxName := genChangingIndexUniqueName(tblInfo, info.IndexInfo)
+ setIdxIDName(tmpIdx, newIdxID, model.NewCIStr(tmpIdxName))
+ SetIdxColNameOffset(tmpIdx.Columns[info.Offset], changingCol)
+ tblInfo.Indices = append(tblInfo.Indices, tmpIdx)
+ } else {
+ // The index is a temp index created by previous modify column job(s).
+ // We can overwrite it to reduce reorg cost, because it will be dropped eventually.
+ tmpIdx := info.IndexInfo
+ oldTempIdxID := tmpIdx.ID
+ setIdxIDName(tmpIdx, newIdxID, tmpIdx.Name /* unchanged */)
+ SetIdxColNameOffset(tmpIdx.Columns[info.Offset], changingCol)
+ modifyInfo.removedIdxs = append(modifyInfo.removedIdxs, oldTempIdxID)
+ }
+ }
+ } else {
+ changingCol = model.FindColumnInfoByID(tblInfo.Columns, modifyInfo.changingCol.ID)
+ if changingCol == nil {
+ logutil.DDLLogger().Error("the changing column has been removed", zap.Error(err))
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(infoschema.ErrColumnNotExists.GenWithStackByArgs(oldCol.Name, tblInfo.Name))
+ }
+ }
+
+ return w.doModifyColumnTypeWithData(jobCtx, t, job, dbInfo, tblInfo, changingCol, oldCol, modifyInfo.newCol.Name, modifyInfo.pos, modifyInfo.removedIdxs)
+}
+
+// rollbackModifyColumnJob rollbacks the job when an error occurs.
+func rollbackModifyColumnJob(jobCtx *jobContext, t *meta.Meta, tblInfo *model.TableInfo, job *model.Job, newCol, oldCol *model.ColumnInfo, modifyColumnTp byte) (ver int64, _ error) {
+ var err error
+ if oldCol.ID == newCol.ID && modifyColumnTp == mysql.TypeNull {
+ // field NotNullFlag flag reset.
+ tblInfo.Columns[oldCol.Offset].SetFlag(oldCol.GetFlag() &^ mysql.NotNullFlag)
+ // field PreventNullInsertFlag flag reset.
+ tblInfo.Columns[oldCol.Offset].SetFlag(oldCol.GetFlag() &^ mysql.PreventNullInsertFlag)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ }
+ job.FinishTableJob(model.JobStateRollbackDone, model.StateNone, ver, tblInfo)
+ // For those column-type-change type which doesn't need reorg data, we should also mock the job args for delete range.
+ job.Args = []any{[]int64{}, []int64{}}
+ return ver, nil
+}
+
+func getModifyColumnInfo(t *meta.Meta, job *model.Job) (*model.DBInfo, *model.TableInfo, *model.ColumnInfo, *modifyingColInfo, error) {
+ modifyInfo := &modifyingColInfo{pos: &ast.ColumnPosition{}}
+ err := job.DecodeArgs(&modifyInfo.newCol, &modifyInfo.oldColName, modifyInfo.pos, &modifyInfo.modifyColumnTp,
+ &modifyInfo.updatedAutoRandomBits, &modifyInfo.changingCol, &modifyInfo.changingIdxs, &modifyInfo.removedIdxs)
+ if err != nil {
+ job.State = model.JobStateCancelled
+ return nil, nil, nil, modifyInfo, errors.Trace(err)
+ }
+
+ dbInfo, err := checkSchemaExistAndCancelNotExistJob(t, job)
+ if err != nil {
+ return nil, nil, nil, modifyInfo, errors.Trace(err)
+ }
+
+ tblInfo, err := GetTableInfoAndCancelFaultJob(t, job, job.SchemaID)
+ if err != nil {
+ return nil, nil, nil, modifyInfo, errors.Trace(err)
+ }
+
+ oldCol := model.FindColumnInfo(tblInfo.Columns, modifyInfo.oldColName.L)
+ if oldCol == nil || oldCol.State != model.StatePublic {
+ job.State = model.JobStateCancelled
+ return nil, nil, nil, modifyInfo, errors.Trace(infoschema.ErrColumnNotExists.GenWithStackByArgs(*(modifyInfo.oldColName), tblInfo.Name))
+ }
+
+ return dbInfo, tblInfo, oldCol, modifyInfo, errors.Trace(err)
+}
+
+// GetOriginDefaultValueForModifyColumn gets the original default value for modifying column.
+// Since column type change is implemented as adding a new column then substituting the old one.
+// Case exists when update-where statement fetch a NULL for not-null column without any default data,
+// it will errors.
+// So we set original default value here to prevent this error. If the oldCol has the original default value, we use it.
+// Otherwise we set the zero value as original default value.
+// Besides, in insert & update records, we have already implement using the casted value of relative column to insert
+// rather than the original default value.
+func GetOriginDefaultValueForModifyColumn(ctx exprctx.BuildContext, changingCol, oldCol *model.ColumnInfo) (any, error) {
+ var err error
+ originDefVal := oldCol.GetOriginDefaultValue()
+ if originDefVal != nil {
+ odv, err := table.CastColumnValue(ctx, types.NewDatum(originDefVal), changingCol, false, false)
+ if err != nil {
+ logutil.DDLLogger().Info("cast origin default value failed", zap.Error(err))
+ }
+ if !odv.IsNull() {
+ if originDefVal, err = odv.ToString(); err != nil {
+ originDefVal = nil
+ logutil.DDLLogger().Info("convert default value to string failed", zap.Error(err))
+ }
+ }
+ }
+ if originDefVal == nil {
+ originDefVal, err = generateOriginDefaultValue(changingCol, nil)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ }
+ return originDefVal, nil
+}
+
+// rollbackModifyColumnJobWithData is used to rollback modify-column job which need to reorg the data.
+func rollbackModifyColumnJobWithData(jobCtx *jobContext, t *meta.Meta, tblInfo *model.TableInfo, job *model.Job, oldCol *model.ColumnInfo, modifyInfo *modifyingColInfo) (ver int64, err error) {
+ // If the not-null change is included, we should clean the flag info in oldCol.
+ if modifyInfo.modifyColumnTp == mysql.TypeNull {
+ // Reset NotNullFlag flag.
+ tblInfo.Columns[oldCol.Offset].SetFlag(oldCol.GetFlag() &^ mysql.NotNullFlag)
+ // Reset PreventNullInsertFlag flag.
+ tblInfo.Columns[oldCol.Offset].SetFlag(oldCol.GetFlag() &^ mysql.PreventNullInsertFlag)
+ }
+ var changingIdxIDs []int64
+ if modifyInfo.changingCol != nil {
+ changingIdxIDs = buildRelatedIndexIDs(tblInfo, modifyInfo.changingCol.ID)
+ // The job is in the middle state. The appended changingCol and changingIndex should
+ // be removed from the tableInfo as well.
+ removeChangingColAndIdxs(tblInfo, modifyInfo.changingCol.ID)
+ }
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, true)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ job.FinishTableJob(model.JobStateRollbackDone, model.StateNone, ver, tblInfo)
+ // Reconstruct the job args to add the temporary index ids into delete range table.
+ job.Args = []any{changingIdxIDs, getPartitionIDs(tblInfo)}
+ return ver, nil
+}
+
+// doModifyColumn updates the column information and reorders all columns. It does not support modifying column data.
+func (w *worker) doModifyColumn(
+ jobCtx *jobContext, t *meta.Meta, job *model.Job, dbInfo *model.DBInfo, tblInfo *model.TableInfo,
+ newCol, oldCol *model.ColumnInfo, pos *ast.ColumnPosition) (ver int64, _ error) {
+ if oldCol.ID != newCol.ID {
+ job.State = model.JobStateRollingback
+ return ver, dbterror.ErrColumnInChange.GenWithStackByArgs(oldCol.Name, newCol.ID)
+ }
+ // Column from null to not null.
+ if !mysql.HasNotNullFlag(oldCol.GetFlag()) && mysql.HasNotNullFlag(newCol.GetFlag()) {
+ noPreventNullFlag := !mysql.HasPreventNullInsertFlag(oldCol.GetFlag())
+
+ // We need to check after the flag is set
+ if !noPreventNullFlag {
+ delayForAsyncCommit()
+ }
+
+ // Introduce the `mysql.PreventNullInsertFlag` flag to prevent users from inserting or updating null values.
+ err := modifyColsFromNull2NotNull(w, dbInfo, tblInfo, []*model.ColumnInfo{oldCol}, newCol, oldCol.GetType() != newCol.GetType())
+ if err != nil {
+ if dbterror.ErrWarnDataTruncated.Equal(err) || dbterror.ErrInvalidUseOfNull.Equal(err) {
+ job.State = model.JobStateRollingback
+ }
+ return ver, err
+ }
+ // The column should get into prevent null status first.
+ if noPreventNullFlag {
+ return updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, true)
+ }
+ }
+
+ if job.MultiSchemaInfo != nil && job.MultiSchemaInfo.Revertible {
+ job.MarkNonRevertible()
+ // Store the mark and enter the next DDL handling loop.
+ return updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, false)
+ }
+
+ if err := adjustTableInfoAfterModifyColumn(tblInfo, newCol, oldCol, pos); err != nil {
+ job.State = model.JobStateRollingback
+ return ver, errors.Trace(err)
+ }
+
+ childTableInfos, err := adjustForeignKeyChildTableInfoAfterModifyColumn(jobCtx.infoCache, t, job, tblInfo, newCol, oldCol)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, true, childTableInfos...)
+ if err != nil {
+ // Modified the type definition of 'null' to 'not null' before this, so rollBack the job when an error occurs.
+ job.State = model.JobStateRollingback
+ return ver, errors.Trace(err)
+ }
+
+ job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tblInfo)
+ // For those column-type-change type which doesn't need reorg data, we should also mock the job args for delete range.
+ job.Args = []any{[]int64{}, []int64{}}
+ return ver, nil
+}
+
+func adjustTableInfoAfterModifyColumn(
+ tblInfo *model.TableInfo, newCol, oldCol *model.ColumnInfo, pos *ast.ColumnPosition) error {
+ // We need the latest column's offset and state. This information can be obtained from the store.
+ newCol.Offset = oldCol.Offset
+ newCol.State = oldCol.State
+ if pos != nil && pos.RelativeColumn != nil && oldCol.Name.L == pos.RelativeColumn.Name.L {
+ // For cases like `modify column b after b`, it should report this error.
+ return errors.Trace(infoschema.ErrColumnNotExists.GenWithStackByArgs(oldCol.Name, tblInfo.Name))
+ }
+ destOffset, err := LocateOffsetToMove(oldCol.Offset, pos, tblInfo)
+ if err != nil {
+ return errors.Trace(infoschema.ErrColumnNotExists.GenWithStackByArgs(oldCol.Name, tblInfo.Name))
+ }
+ tblInfo.Columns[oldCol.Offset] = newCol
+ tblInfo.MoveColumnInfo(oldCol.Offset, destOffset)
+ updateNewIdxColsNameOffset(tblInfo.Indices, oldCol.Name, newCol)
+ updateFKInfoWhenModifyColumn(tblInfo, oldCol.Name, newCol.Name)
+ updateTTLInfoWhenModifyColumn(tblInfo, oldCol.Name, newCol.Name)
+ return nil
+}
+
+func updateFKInfoWhenModifyColumn(tblInfo *model.TableInfo, oldCol, newCol model.CIStr) {
+ if oldCol.L == newCol.L {
+ return
+ }
+ for _, fk := range tblInfo.ForeignKeys {
+ for i := range fk.Cols {
+ if fk.Cols[i].L == oldCol.L {
+ fk.Cols[i] = newCol
+ }
+ }
+ }
+}
+
+func updateTTLInfoWhenModifyColumn(tblInfo *model.TableInfo, oldCol, newCol model.CIStr) {
+ if oldCol.L == newCol.L {
+ return
+ }
+ if tblInfo.TTLInfo != nil {
+ if tblInfo.TTLInfo.ColumnName.L == oldCol.L {
+ tblInfo.TTLInfo.ColumnName = newCol
+ }
+ }
+}
+
+func adjustForeignKeyChildTableInfoAfterModifyColumn(infoCache *infoschema.InfoCache, t *meta.Meta, job *model.Job, tblInfo *model.TableInfo, newCol, oldCol *model.ColumnInfo) ([]schemaIDAndTableInfo, error) {
+ if !variable.EnableForeignKey.Load() || newCol.Name.L == oldCol.Name.L {
+ return nil, nil
+ }
+ is := infoCache.GetLatest()
+ referredFKs := is.GetTableReferredForeignKeys(job.SchemaName, tblInfo.Name.L)
+ if len(referredFKs) == 0 {
+ return nil, nil
+ }
+ fkh := newForeignKeyHelper()
+ fkh.addLoadedTable(job.SchemaName, tblInfo.Name.L, job.SchemaID, tblInfo)
+ for _, referredFK := range referredFKs {
+ info, err := fkh.getTableFromStorage(is, t, referredFK.ChildSchema, referredFK.ChildTable)
+ if err != nil {
+ if infoschema.ErrTableNotExists.Equal(err) || infoschema.ErrDatabaseNotExists.Equal(err) {
+ continue
+ }
+ return nil, err
+ }
+ fkInfo := model.FindFKInfoByName(info.tblInfo.ForeignKeys, referredFK.ChildFKName.L)
+ if fkInfo == nil {
+ continue
+ }
+ for i := range fkInfo.RefCols {
+ if fkInfo.RefCols[i].L == oldCol.Name.L {
+ fkInfo.RefCols[i] = newCol.Name
+ }
+ }
+ }
+ infoList := make([]schemaIDAndTableInfo, 0, len(fkh.loaded))
+ for _, info := range fkh.loaded {
+ if info.tblInfo.ID == tblInfo.ID {
+ continue
+ }
+ infoList = append(infoList, info)
+ }
+ return infoList, nil
+}
+
+func (w *worker) doModifyColumnTypeWithData(
+ jobCtx *jobContext, t *meta.Meta, job *model.Job,
+ dbInfo *model.DBInfo, tblInfo *model.TableInfo, changingCol, oldCol *model.ColumnInfo,
+ colName model.CIStr, pos *ast.ColumnPosition, rmIdxIDs []int64) (ver int64, _ error) {
+ var err error
+ originalState := changingCol.State
+ targetCol := changingCol.Clone()
+ targetCol.Name = colName
+ changingIdxs := buildRelatedIndexInfos(tblInfo, changingCol.ID)
+ switch changingCol.State {
+ case model.StateNone:
+ // Column from null to not null.
+ if !mysql.HasNotNullFlag(oldCol.GetFlag()) && mysql.HasNotNullFlag(changingCol.GetFlag()) {
+ // Introduce the `mysql.PreventNullInsertFlag` flag to prevent users from inserting or updating null values.
+ err := modifyColsFromNull2NotNull(w, dbInfo, tblInfo, []*model.ColumnInfo{oldCol}, targetCol, oldCol.GetType() != changingCol.GetType())
+ if err != nil {
+ if dbterror.ErrWarnDataTruncated.Equal(err) || dbterror.ErrInvalidUseOfNull.Equal(err) {
+ job.State = model.JobStateRollingback
+ }
+ return ver, err
+ }
+ }
+ // none -> delete only
+ updateChangingObjState(changingCol, changingIdxs, model.StateDeleteOnly)
+ failpoint.Inject("mockInsertValueAfterCheckNull", func(val failpoint.Value) {
+ if valStr, ok := val.(string); ok {
+ var sctx sessionctx.Context
+ sctx, err := w.sessPool.Get()
+ if err != nil {
+ failpoint.Return(ver, err)
+ }
+ defer w.sessPool.Put(sctx)
+
+ ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL)
+ //nolint:forcetypeassert
+ _, _, err = sctx.GetRestrictedSQLExecutor().ExecRestrictedSQL(ctx, nil, valStr)
+ if err != nil {
+ job.State = model.JobStateCancelled
+ failpoint.Return(ver, err)
+ }
+ }
+ })
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, originalState != changingCol.State)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ // Make sure job args change after `updateVersionAndTableInfoWithCheck`, otherwise, the job args will
+ // be updated in `updateDDLJob` even if it meets an error in `updateVersionAndTableInfoWithCheck`.
+ job.SchemaState = model.StateDeleteOnly
+ metrics.GetBackfillProgressByLabel(metrics.LblModifyColumn, job.SchemaName, tblInfo.Name.String()).Set(0)
+ job.Args = append(job.Args, changingCol, changingIdxs, rmIdxIDs)
+ case model.StateDeleteOnly:
+ // Column from null to not null.
+ if !mysql.HasNotNullFlag(oldCol.GetFlag()) && mysql.HasNotNullFlag(changingCol.GetFlag()) {
+ // Introduce the `mysql.PreventNullInsertFlag` flag to prevent users from inserting or updating null values.
+ err := modifyColsFromNull2NotNull(w, dbInfo, tblInfo, []*model.ColumnInfo{oldCol}, targetCol, oldCol.GetType() != changingCol.GetType())
+ if err != nil {
+ if dbterror.ErrWarnDataTruncated.Equal(err) || dbterror.ErrInvalidUseOfNull.Equal(err) {
+ job.State = model.JobStateRollingback
+ }
+ return ver, err
+ }
+ }
+ // delete only -> write only
+ updateChangingObjState(changingCol, changingIdxs, model.StateWriteOnly)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != changingCol.State)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ job.SchemaState = model.StateWriteOnly
+ failpoint.InjectCall("afterModifyColumnStateDeleteOnly", job.ID)
+ case model.StateWriteOnly:
+ // write only -> reorganization
+ updateChangingObjState(changingCol, changingIdxs, model.StateWriteReorganization)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != changingCol.State)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+ // Initialize SnapshotVer to 0 for later reorganization check.
+ job.SnapshotVer = 0
+ job.SchemaState = model.StateWriteReorganization
+ case model.StateWriteReorganization:
+ tbl, err := getTable(jobCtx.getAutoIDRequirement(), dbInfo.ID, tblInfo)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+
+ var done bool
+ if job.MultiSchemaInfo != nil {
+ done, ver, err = doReorgWorkForModifyColumnMultiSchema(w, jobCtx, t, job, tbl, oldCol, changingCol, changingIdxs)
+ } else {
+ done, ver, err = doReorgWorkForModifyColumn(w, jobCtx, t, job, tbl, oldCol, changingCol, changingIdxs)
+ }
+ if !done {
+ return ver, err
+ }
+
+ rmIdxIDs = append(buildRelatedIndexIDs(tblInfo, oldCol.ID), rmIdxIDs...)
+
+ err = adjustTableInfoAfterModifyColumnWithData(tblInfo, pos, oldCol, changingCol, colName, changingIdxs)
+ if err != nil {
+ job.State = model.JobStateRollingback
+ return ver, errors.Trace(err)
+ }
+
+ updateChangingObjState(changingCol, changingIdxs, model.StatePublic)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != changingCol.State)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+
+ // Finish this job.
+ job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tblInfo)
+ // Refactor the job args to add the old index ids into delete range table.
+ job.Args = []any{rmIdxIDs, getPartitionIDs(tblInfo)}
+ modifyColumnEvent := statsutil.NewModifyColumnEvent(
+ job.SchemaID,
+ tblInfo,
+ []*model.ColumnInfo{changingCol},
+ )
+ asyncNotifyEvent(jobCtx, modifyColumnEvent)
+ default:
+ err = dbterror.ErrInvalidDDLState.GenWithStackByArgs("column", changingCol.State)
+ }
+
+ return ver, errors.Trace(err)
+}
+
+func doReorgWorkForModifyColumnMultiSchema(w *worker, jobCtx *jobContext, t *meta.Meta, job *model.Job, tbl table.Table,
+ oldCol, changingCol *model.ColumnInfo, changingIdxs []*model.IndexInfo) (done bool, ver int64, err error) {
+ if job.MultiSchemaInfo.Revertible {
+ done, ver, err = doReorgWorkForModifyColumn(w, jobCtx, t, job, tbl, oldCol, changingCol, changingIdxs)
+ if done {
+ // We need another round to wait for all the others sub-jobs to finish.
+ job.MarkNonRevertible()
+ }
+ // We need another round to run the reorg process.
+ return false, ver, err
+ }
+ // Non-revertible means all the sub jobs finished.
+ return true, ver, err
+}
+
+func doReorgWorkForModifyColumn(w *worker, jobCtx *jobContext, t *meta.Meta, job *model.Job, tbl table.Table,
+ oldCol, changingCol *model.ColumnInfo, changingIdxs []*model.IndexInfo) (done bool, ver int64, err error) {
+ job.ReorgMeta.ReorgTp = model.ReorgTypeTxn
+ sctx, err1 := w.sessPool.Get()
+ if err1 != nil {
+ err = errors.Trace(err1)
+ return
+ }
+ defer w.sessPool.Put(sctx)
+ rh := newReorgHandler(sess.NewSession(sctx))
+ dbInfo, err := t.GetDatabase(job.SchemaID)
+ if err != nil {
+ return false, ver, errors.Trace(err)
+ }
+ reorgInfo, err := getReorgInfo(jobCtx.oldDDLCtx.jobContext(job.ID, job.ReorgMeta),
+ jobCtx, rh, job, dbInfo, tbl, BuildElements(changingCol, changingIdxs), false)
+ if err != nil || reorgInfo == nil || reorgInfo.first {
+ // If we run reorg firstly, we should update the job snapshot version
+ // and then run the reorg next time.
+ return false, ver, errors.Trace(err)
+ }
+
+ // Inject a failpoint so that we can pause here and do verification on other components.
+ // With a failpoint-enabled version of TiDB, you can trigger this failpoint by the following command:
+ // enable: curl -X PUT -d "pause" "http://127.0.0.1:10080/fail/github.com/pingcap/tidb/pkg/ddl/mockDelayInModifyColumnTypeWithData".
+ // disable: curl -X DELETE "http://127.0.0.1:10080/fail/github.com/pingcap/tidb/pkg/ddl/mockDelayInModifyColumnTypeWithData"
+ failpoint.Inject("mockDelayInModifyColumnTypeWithData", func() {})
+ err = w.runReorgJob(reorgInfo, tbl.Meta(), func() (addIndexErr error) {
+ defer util.Recover(metrics.LabelDDL, "onModifyColumn",
+ func() {
+ addIndexErr = dbterror.ErrCancelledDDLJob.GenWithStack("modify table `%v` column `%v` panic", tbl.Meta().Name, oldCol.Name)
+ }, false)
+ // Use old column name to generate less confusing error messages.
+ changingColCpy := changingCol.Clone()
+ changingColCpy.Name = oldCol.Name
+ return w.updateCurrentElement(tbl, reorgInfo)
+ })
+ if err != nil {
+ if dbterror.ErrPausedDDLJob.Equal(err) {
+ return false, ver, nil
+ }
+
+ if dbterror.ErrWaitReorgTimeout.Equal(err) {
+ // If timeout, we should return, check for the owner and re-wait job done.
+ return false, ver, nil
+ }
+ if kv.IsTxnRetryableError(err) || dbterror.ErrNotOwner.Equal(err) {
+ return false, ver, errors.Trace(err)
+ }
+ if err1 := rh.RemoveDDLReorgHandle(job, reorgInfo.elements); err1 != nil {
+ logutil.DDLLogger().Warn("run modify column job failed, RemoveDDLReorgHandle failed, can't convert job to rollback",
+ zap.String("job", job.String()), zap.Error(err1))
+ }
+ logutil.DDLLogger().Warn("run modify column job failed, convert job to rollback", zap.Stringer("job", job), zap.Error(err))
+ job.State = model.JobStateRollingback
+ return false, ver, errors.Trace(err)
+ }
+ return true, ver, nil
+}
+
+func adjustTableInfoAfterModifyColumnWithData(tblInfo *model.TableInfo, pos *ast.ColumnPosition,
+ oldCol, changingCol *model.ColumnInfo, newName model.CIStr, changingIdxs []*model.IndexInfo) (err error) {
+ if pos != nil && pos.RelativeColumn != nil && oldCol.Name.L == pos.RelativeColumn.Name.L {
+ // For cases like `modify column b after b`, it should report this error.
+ return errors.Trace(infoschema.ErrColumnNotExists.GenWithStackByArgs(oldCol.Name, tblInfo.Name))
+ }
+ internalColName := changingCol.Name
+ changingCol = replaceOldColumn(tblInfo, oldCol, changingCol, newName)
+ if len(changingIdxs) > 0 {
+ updateNewIdxColsNameOffset(changingIdxs, internalColName, changingCol)
+ indexesToRemove := filterIndexesToRemove(changingIdxs, newName, tblInfo)
+ replaceOldIndexes(tblInfo, indexesToRemove)
+ }
+ if tblInfo.TTLInfo != nil {
+ updateTTLInfoWhenModifyColumn(tblInfo, oldCol.Name, changingCol.Name)
+ }
+ // Move the new column to a correct offset.
+ destOffset, err := LocateOffsetToMove(changingCol.Offset, pos, tblInfo)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ tblInfo.MoveColumnInfo(changingCol.Offset, destOffset)
+ return nil
+}
+
+func checkModifyColumnWithGeneratedColumnsConstraint(allCols []*table.Column, oldColName model.CIStr) error {
+ for _, col := range allCols {
+ if col.GeneratedExpr == nil {
+ continue
+ }
+ dependedColNames := FindColumnNamesInExpr(col.GeneratedExpr.Internal())
+ for _, name := range dependedColNames {
+ if name.Name.L == oldColName.L {
+ if col.Hidden {
+ return dbterror.ErrDependentByFunctionalIndex.GenWithStackByArgs(oldColName.O)
+ }
+ return dbterror.ErrDependentByGeneratedColumn.GenWithStackByArgs(oldColName.O)
+ }
+ }
+ }
+ return nil
+}
+
+// GetModifiableColumnJob returns a DDL job of model.ActionModifyColumn.
+func GetModifiableColumnJob(
+ ctx context.Context,
+ sctx sessionctx.Context,
+ is infoschema.InfoSchema, // WARN: is maybe nil here.
+ ident ast.Ident,
+ originalColName model.CIStr,
+ schema *model.DBInfo,
+ t table.Table,
+ spec *ast.AlterTableSpec,
+) (*model.Job, error) {
+ var err error
+ specNewColumn := spec.NewColumns[0]
+
+ col := table.FindCol(t.Cols(), originalColName.L)
+ if col == nil {
+ return nil, infoschema.ErrColumnNotExists.GenWithStackByArgs(originalColName, ident.Name)
+ }
+ newColName := specNewColumn.Name.Name
+ if newColName.L == model.ExtraHandleName.L {
+ return nil, dbterror.ErrWrongColumnName.GenWithStackByArgs(newColName.L)
+ }
+ errG := checkModifyColumnWithGeneratedColumnsConstraint(t.Cols(), originalColName)
+
+ // If we want to rename the column name, we need to check whether it already exists.
+ if newColName.L != originalColName.L {
+ c := table.FindCol(t.Cols(), newColName.L)
+ if c != nil {
+ return nil, infoschema.ErrColumnExists.GenWithStackByArgs(newColName)
+ }
+
+ // And also check the generated columns dependency, if some generated columns
+ // depend on this column, we can't rename the column name.
+ if errG != nil {
+ return nil, errors.Trace(errG)
+ }
+ }
+
+ // Constraints in the new column means adding new constraints. Errors should thrown,
+ // which will be done by `processColumnOptions` later.
+ if specNewColumn.Tp == nil {
+ // Make sure the column definition is simple field type.
+ return nil, errors.Trace(dbterror.ErrUnsupportedModifyColumn)
+ }
+
+ if err = checkColumnAttributes(specNewColumn.Name.OrigColName(), specNewColumn.Tp); err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ newCol := table.ToColumn(&model.ColumnInfo{
+ ID: col.ID,
+ // We use this PR(https://github.com/pingcap/tidb/pull/6274) as the dividing line to define whether it is a new version or an old version TiDB.
+ // The old version TiDB initializes the column's offset and state here.
+ // The new version TiDB doesn't initialize the column's offset and state, and it will do the initialization in run DDL function.
+ // When we do the rolling upgrade the following may happen:
+ // a new version TiDB builds the DDL job that doesn't be set the column's offset and state,
+ // and the old version TiDB is the DDL owner, it doesn't get offset and state from the store. Then it will encounter errors.
+ // So here we set offset and state to support the rolling upgrade.
+ Offset: col.Offset,
+ State: col.State,
+ OriginDefaultValue: col.OriginDefaultValue,
+ OriginDefaultValueBit: col.OriginDefaultValueBit,
+ FieldType: *specNewColumn.Tp,
+ Name: newColName,
+ Version: col.Version,
+ })
+
+ if err = ProcessColumnCharsetAndCollation(sctx, col, newCol, t.Meta(), specNewColumn, schema); err != nil {
+ return nil, err
+ }
+
+ if err = checkModifyColumnWithForeignKeyConstraint(is, schema.Name.L, t.Meta(), col.ColumnInfo, newCol.ColumnInfo); err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ // Copy index related options to the new spec.
+ indexFlags := col.FieldType.GetFlag() & (mysql.PriKeyFlag | mysql.UniqueKeyFlag | mysql.MultipleKeyFlag)
+ newCol.FieldType.AddFlag(indexFlags)
+ if mysql.HasPriKeyFlag(col.FieldType.GetFlag()) {
+ newCol.FieldType.AddFlag(mysql.NotNullFlag)
+ // TODO: If user explicitly set NULL, we should throw error ErrPrimaryCantHaveNull.
+ }
+
+ if err = ProcessModifyColumnOptions(sctx, newCol, specNewColumn.Options); err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ if err = checkModifyTypes(&col.FieldType, &newCol.FieldType, isColumnWithIndex(col.Name.L, t.Meta().Indices)); err != nil {
+ if strings.Contains(err.Error(), "Unsupported modifying collation") {
+ colErrMsg := "Unsupported modifying collation of column '%s' from '%s' to '%s' when index is defined on it."
+ err = dbterror.ErrUnsupportedModifyCollation.GenWithStack(colErrMsg, col.Name.L, col.GetCollate(), newCol.GetCollate())
+ }
+ return nil, errors.Trace(err)
+ }
+ needChangeColData := needChangeColumnData(col.ColumnInfo, newCol.ColumnInfo)
+ if needChangeColData {
+ if err = isGeneratedRelatedColumn(t.Meta(), newCol.ColumnInfo, col.ColumnInfo); err != nil {
+ return nil, errors.Trace(err)
+ }
+ if t.Meta().Partition != nil {
+ return nil, dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs("table is partition table")
+ }
+ }
+
+ // Check that the column change does not affect the partitioning column
+ // It must keep the same type, int [unsigned], [var]char, date[time]
+ if t.Meta().Partition != nil {
+ pt, ok := t.(table.PartitionedTable)
+ if !ok {
+ // Should never happen!
+ return nil, dbterror.ErrNotAllowedTypeInPartition.GenWithStackByArgs(newCol.Name.O)
+ }
+ isPartitioningColumn := false
+ for _, name := range pt.GetPartitionColumnNames() {
+ if strings.EqualFold(name.L, col.Name.L) {
+ isPartitioningColumn = true
+ break
+ }
+ }
+ if isPartitioningColumn {
+ // TODO: update the partitioning columns with new names if column is renamed
+ // Would be an extension from MySQL which does not support it.
+ if col.Name.L != newCol.Name.L {
+ return nil, dbterror.ErrDependentByPartitionFunctional.GenWithStackByArgs(col.Name.L)
+ }
+ if !isColTypeAllowedAsPartitioningCol(t.Meta().Partition.Type, newCol.FieldType) {
+ return nil, dbterror.ErrNotAllowedTypeInPartition.GenWithStackByArgs(newCol.Name.O)
+ }
+ pi := pt.Meta().GetPartitionInfo()
+ if len(pi.Columns) == 0 {
+ // non COLUMNS partitioning, only checks INTs, not their actual range
+ // There are many edge cases, like when truncating SQL Mode is allowed
+ // which will change the partitioning expression value resulting in a
+ // different partition. Better be safe and not allow decreasing of length.
+ // TODO: Should we allow it in strict mode? Wait for a use case / request.
+ if newCol.FieldType.GetFlen() < col.FieldType.GetFlen() {
+ return nil, dbterror.ErrUnsupportedModifyCollation.GenWithStack("Unsupported modify column, decreasing length of int may result in truncation and change of partition")
+ }
+ }
+ // Basically only allow changes of the length/decimals for the column
+ // Note that enum is not allowed, so elems are not checked
+ // TODO: support partition by ENUM
+ if newCol.FieldType.EvalType() != col.FieldType.EvalType() ||
+ newCol.FieldType.GetFlag() != col.FieldType.GetFlag() ||
+ newCol.FieldType.GetCollate() != col.FieldType.GetCollate() ||
+ newCol.FieldType.GetCharset() != col.FieldType.GetCharset() {
+ return nil, dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs("can't change the partitioning column, since it would require reorganize all partitions")
+ }
+ // Generate a new PartitionInfo and validate it together with the new column definition
+ // Checks if all partition definition values are compatible.
+ // Similar to what buildRangePartitionDefinitions would do in terms of checks.
+
+ tblInfo := pt.Meta()
+ newTblInfo := *tblInfo
+ // Replace col with newCol and see if we can generate a new SHOW CREATE TABLE
+ // and reparse it and build new partition definitions (which will do additional
+ // checks columns vs partition definition values
+ newCols := make([]*model.ColumnInfo, 0, len(newTblInfo.Columns))
+ for _, c := range newTblInfo.Columns {
+ if c.ID == col.ID {
+ newCols = append(newCols, newCol.ColumnInfo)
+ continue
+ }
+ newCols = append(newCols, c)
+ }
+ newTblInfo.Columns = newCols
+
+ var buf bytes.Buffer
+ AppendPartitionInfo(tblInfo.GetPartitionInfo(), &buf, mysql.ModeNone)
+ // The parser supports ALTER TABLE ... PARTITION BY ... even if the ddl code does not yet :)
+ // Ignoring warnings
+ stmt, _, err := parser.New().ParseSQL("ALTER TABLE t " + buf.String())
+ if err != nil {
+ // Should never happen!
+ return nil, dbterror.ErrUnsupportedModifyColumn.GenWithStack("cannot parse generated PartitionInfo")
+ }
+ at, ok := stmt[0].(*ast.AlterTableStmt)
+ if !ok || len(at.Specs) != 1 || at.Specs[0].Partition == nil {
+ return nil, dbterror.ErrUnsupportedModifyColumn.GenWithStack("cannot parse generated PartitionInfo")
+ }
+ pAst := at.Specs[0].Partition
+ _, err = buildPartitionDefinitionsInfo(
+ exprctx.CtxWithHandleTruncateErrLevel(sctx.GetExprCtx(), errctx.LevelError),
+ pAst.Definitions, &newTblInfo, uint64(len(newTblInfo.Partition.Definitions)),
+ )
+ if err != nil {
+ return nil, dbterror.ErrUnsupportedModifyColumn.GenWithStack("New column does not match partition definitions: %s", err.Error())
+ }
+ }
+ }
+
+ // We don't support modifying column from not_auto_increment to auto_increment.
+ if !mysql.HasAutoIncrementFlag(col.GetFlag()) && mysql.HasAutoIncrementFlag(newCol.GetFlag()) {
+ return nil, dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs("can't set auto_increment")
+ }
+ // Not support auto id with default value.
+ if mysql.HasAutoIncrementFlag(newCol.GetFlag()) && newCol.GetDefaultValue() != nil {
+ return nil, dbterror.ErrInvalidDefaultValue.GenWithStackByArgs(newCol.Name)
+ }
+ // Disallow modifying column from auto_increment to not auto_increment if the session variable `AllowRemoveAutoInc` is false.
+ if !sctx.GetSessionVars().AllowRemoveAutoInc && mysql.HasAutoIncrementFlag(col.GetFlag()) && !mysql.HasAutoIncrementFlag(newCol.GetFlag()) {
+ return nil, dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs("can't remove auto_increment without @@tidb_allow_remove_auto_inc enabled")
+ }
+
+ // We support modifying the type definitions of 'null' to 'not null' now.
+ var modifyColumnTp byte
+ if !mysql.HasNotNullFlag(col.GetFlag()) && mysql.HasNotNullFlag(newCol.GetFlag()) {
+ if err = checkForNullValue(ctx, sctx, true, ident.Schema, ident.Name, newCol.ColumnInfo, col.ColumnInfo); err != nil {
+ return nil, errors.Trace(err)
+ }
+ // `modifyColumnTp` indicates that there is a type modification.
+ modifyColumnTp = mysql.TypeNull
+ }
+
+ if err = checkColumnWithIndexConstraint(t.Meta(), col.ColumnInfo, newCol.ColumnInfo); err != nil {
+ return nil, err
+ }
+
+ // As same with MySQL, we don't support modifying the stored status for generated columns.
+ if err = checkModifyGeneratedColumn(sctx, schema.Name, t, col, newCol, specNewColumn, spec.Position); err != nil {
+ return nil, errors.Trace(err)
+ }
+ if errG != nil {
+ // According to issue https://github.com/pingcap/tidb/issues/24321,
+ // changing the type of a column involving generating a column is prohibited.
+ return nil, dbterror.ErrUnsupportedOnGeneratedColumn.GenWithStackByArgs(errG.Error())
+ }
+
+ if t.Meta().TTLInfo != nil {
+ // the column referenced by TTL should be a time type
+ if t.Meta().TTLInfo.ColumnName.L == originalColName.L && !types.IsTypeTime(newCol.ColumnInfo.FieldType.GetType()) {
+ return nil, errors.Trace(dbterror.ErrUnsupportedColumnInTTLConfig.GenWithStackByArgs(newCol.ColumnInfo.Name.O))
+ }
+ }
+
+ var newAutoRandBits uint64
+ if newAutoRandBits, err = checkAutoRandom(t.Meta(), col, specNewColumn); err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ txn, err := sctx.Txn(true)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ bdrRole, err := meta.NewMeta(txn).GetBDRRole()
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ if bdrRole == string(ast.BDRRolePrimary) &&
+ deniedByBDRWhenModifyColumn(newCol.FieldType, col.FieldType, specNewColumn.Options) {
+ return nil, dbterror.ErrBDRRestrictedDDL.FastGenByArgs(bdrRole)
+ }
+
+ job := &model.Job{
+ SchemaID: schema.ID,
+ TableID: t.Meta().ID,
+ SchemaName: schema.Name.L,
+ TableName: t.Meta().Name.L,
+ Type: model.ActionModifyColumn,
+ BinlogInfo: &model.HistoryInfo{},
+ ReorgMeta: NewDDLReorgMeta(sctx),
+ CtxVars: []any{needChangeColData},
+ Args: []any{&newCol.ColumnInfo, originalColName, spec.Position, modifyColumnTp, newAutoRandBits},
+ CDCWriteSource: sctx.GetSessionVars().CDCWriteSource,
+ SQLMode: sctx.GetSessionVars().SQLMode,
+ }
+ return job, nil
+}
+
+func needChangeColumnData(oldCol, newCol *model.ColumnInfo) bool {
+ toUnsigned := mysql.HasUnsignedFlag(newCol.GetFlag())
+ originUnsigned := mysql.HasUnsignedFlag(oldCol.GetFlag())
+ needTruncationOrToggleSign := func() bool {
+ return (newCol.GetFlen() > 0 && (newCol.GetFlen() < oldCol.GetFlen() || newCol.GetDecimal() < oldCol.GetDecimal())) ||
+ (toUnsigned != originUnsigned)
+ }
+ // Ignore the potential max display length represented by integer's flen, use default flen instead.
+ defaultOldColFlen, _ := mysql.GetDefaultFieldLengthAndDecimal(oldCol.GetType())
+ defaultNewColFlen, _ := mysql.GetDefaultFieldLengthAndDecimal(newCol.GetType())
+ needTruncationOrToggleSignForInteger := func() bool {
+ return (defaultNewColFlen > 0 && defaultNewColFlen < defaultOldColFlen) || (toUnsigned != originUnsigned)
+ }
+
+ // Deal with the same type.
+ if oldCol.GetType() == newCol.GetType() {
+ switch oldCol.GetType() {
+ case mysql.TypeNewDecimal:
+ // Since type decimal will encode the precision, frac, negative(signed) and wordBuf into storage together, there is no short
+ // cut to eliminate data reorg change for column type change between decimal.
+ return oldCol.GetFlen() != newCol.GetFlen() || oldCol.GetDecimal() != newCol.GetDecimal() || toUnsigned != originUnsigned
+ case mysql.TypeEnum, mysql.TypeSet:
+ return IsElemsChangedToModifyColumn(oldCol.GetElems(), newCol.GetElems())
+ case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong:
+ return toUnsigned != originUnsigned
+ case mysql.TypeString:
+ // Due to the behavior of padding \x00 at binary type, always change column data when binary length changed
+ if types.IsBinaryStr(&oldCol.FieldType) {
+ return newCol.GetFlen() != oldCol.GetFlen()
+ }
+ }
+
+ return needTruncationOrToggleSign()
+ }
+
+ if ConvertBetweenCharAndVarchar(oldCol.GetType(), newCol.GetType()) {
+ return true
+ }
+
+ // Deal with the different type.
+ switch oldCol.GetType() {
+ case mysql.TypeVarchar, mysql.TypeString, mysql.TypeVarString, mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob:
+ switch newCol.GetType() {
+ case mysql.TypeVarchar, mysql.TypeString, mysql.TypeVarString, mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob:
+ return needTruncationOrToggleSign()
+ }
+ case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong:
+ switch newCol.GetType() {
+ case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong:
+ return needTruncationOrToggleSignForInteger()
+ }
+ // conversion between float and double needs reorganization, see issue #31372
+ }
+
+ return true
+}
+
+// ConvertBetweenCharAndVarchar check whether column converted between char and varchar
+// TODO: it is used for plugins. so change plugin's using and remove it.
+func ConvertBetweenCharAndVarchar(oldCol, newCol byte) bool {
+ return types.ConvertBetweenCharAndVarchar(oldCol, newCol)
+}
+
+// IsElemsChangedToModifyColumn check elems changed
+func IsElemsChangedToModifyColumn(oldElems, newElems []string) bool {
+ if len(newElems) < len(oldElems) {
+ return true
+ }
+ for index, oldElem := range oldElems {
+ newElem := newElems[index]
+ if oldElem != newElem {
+ return true
+ }
+ }
+ return false
+}
+
+// ProcessColumnCharsetAndCollation process column charset and collation
+func ProcessColumnCharsetAndCollation(sctx sessionctx.Context, col *table.Column, newCol *table.Column, meta *model.TableInfo, specNewColumn *ast.ColumnDef, schema *model.DBInfo) error {
+ var chs, coll string
+ var err error
+ // TODO: Remove it when all table versions are greater than or equal to TableInfoVersion1.
+ // If newCol's charset is empty and the table's version less than TableInfoVersion1,
+ // we will not modify the charset of the column. This behavior is not compatible with MySQL.
+ if len(newCol.FieldType.GetCharset()) == 0 && meta.Version < model.TableInfoVersion1 {
+ chs = col.FieldType.GetCharset()
+ coll = col.FieldType.GetCollate()
+ } else {
+ chs, coll, err = getCharsetAndCollateInColumnDef(sctx.GetSessionVars(), specNewColumn)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ chs, coll, err = ResolveCharsetCollation(sctx.GetSessionVars(),
+ ast.CharsetOpt{Chs: chs, Col: coll},
+ ast.CharsetOpt{Chs: meta.Charset, Col: meta.Collate},
+ ast.CharsetOpt{Chs: schema.Charset, Col: schema.Collate},
+ )
+ chs, coll = OverwriteCollationWithBinaryFlag(sctx.GetSessionVars(), specNewColumn, chs, coll)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ }
+
+ if err = setCharsetCollationFlenDecimal(&newCol.FieldType, newCol.Name.O, chs, coll, sctx.GetSessionVars()); err != nil {
+ return errors.Trace(err)
+ }
+ decodeEnumSetBinaryLiteralToUTF8(&newCol.FieldType, chs)
+ return nil
+}
+
+// checkColumnWithIndexConstraint is used to check the related index constraint of the modified column.
+// Index has a max-prefix-length constraint. eg: a varchar(100), index idx(a), modifying column a to a varchar(4000)
+// will cause index idx to break the max-prefix-length constraint.
+func checkColumnWithIndexConstraint(tbInfo *model.TableInfo, originalCol, newCol *model.ColumnInfo) error {
+ columns := make([]*model.ColumnInfo, 0, len(tbInfo.Columns))
+ columns = append(columns, tbInfo.Columns...)
+ // Replace old column with new column.
+ for i, col := range columns {
+ if col.Name.L != originalCol.Name.L {
+ continue
+ }
+ columns[i] = newCol.Clone()
+ columns[i].Name = originalCol.Name
+ break
+ }
+
+ pkIndex := tables.FindPrimaryIndex(tbInfo)
+
+ checkOneIndex := func(indexInfo *model.IndexInfo) (err error) {
+ var modified bool
+ for _, col := range indexInfo.Columns {
+ if col.Name.L == originalCol.Name.L {
+ modified = true
+ break
+ }
+ }
+ if !modified {
+ return
+ }
+ err = checkIndexInModifiableColumns(columns, indexInfo.Columns)
+ if err != nil {
+ return
+ }
+ err = checkIndexPrefixLength(columns, indexInfo.Columns)
+ return
+ }
+
+ // Check primary key first.
+ var err error
+
+ if pkIndex != nil {
+ err = checkOneIndex(pkIndex)
+ if err != nil {
+ return err
+ }
+ }
+
+ // Check secondary indexes.
+ for _, indexInfo := range tbInfo.Indices {
+ if indexInfo.Primary {
+ continue
+ }
+ // the second param should always be set to true, check index length only if it was modified
+ // checkOneIndex needs one param only.
+ err = checkOneIndex(indexInfo)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func checkIndexInModifiableColumns(columns []*model.ColumnInfo, idxColumns []*model.IndexColumn) error {
+ for _, ic := range idxColumns {
+ col := model.FindColumnInfo(columns, ic.Name.L)
+ if col == nil {
+ return dbterror.ErrKeyColumnDoesNotExits.GenWithStack("column does not exist: %s", ic.Name)
+ }
+
+ prefixLength := types.UnspecifiedLength
+ if types.IsTypePrefixable(col.FieldType.GetType()) && col.FieldType.GetFlen() > ic.Length {
+ // When the index column is changed, prefix length is only valid
+ // if the type is still prefixable and larger than old prefix length.
+ prefixLength = ic.Length
+ }
+ if err := checkIndexColumn(nil, col, prefixLength); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// checkModifyTypes checks if the 'origin' type can be modified to 'to' type no matter directly change
+// or change by reorg. It returns error if the two types are incompatible and correlated change are not
+// supported. However, even the two types can be change, if the "origin" type contains primary key, error will be returned.
+func checkModifyTypes(origin *types.FieldType, to *types.FieldType, needRewriteCollationData bool) error {
+ canReorg, err := types.CheckModifyTypeCompatible(origin, to)
+ if err != nil {
+ if !canReorg {
+ return errors.Trace(dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs(err.Error()))
+ }
+ if mysql.HasPriKeyFlag(origin.GetFlag()) {
+ msg := "this column has primary key flag"
+ return dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs(msg)
+ }
+ }
+
+ err = checkModifyCharsetAndCollation(to.GetCharset(), to.GetCollate(), origin.GetCharset(), origin.GetCollate(), needRewriteCollationData)
+
+ if err != nil {
+ if to.GetCharset() == charset.CharsetGBK || origin.GetCharset() == charset.CharsetGBK {
+ return errors.Trace(err)
+ }
+ // column type change can handle the charset change between these two types in the process of the reorg.
+ if dbterror.ErrUnsupportedModifyCharset.Equal(err) && canReorg {
+ return nil
+ }
+ }
+ return errors.Trace(err)
+}
+
+// ProcessModifyColumnOptions process column options.
+func ProcessModifyColumnOptions(ctx sessionctx.Context, col *table.Column, options []*ast.ColumnOption) error {
+ var sb strings.Builder
+ restoreFlags := format.RestoreStringSingleQuotes | format.RestoreKeyWordLowercase | format.RestoreNameBackQuotes |
+ format.RestoreSpacesAroundBinaryOperation | format.RestoreWithoutSchemaName | format.RestoreWithoutSchemaName
+ restoreCtx := format.NewRestoreCtx(restoreFlags, &sb)
+
+ var hasDefaultValue, setOnUpdateNow bool
+ var err error
+ var hasNullFlag bool
+ for _, opt := range options {
+ switch opt.Tp {
+ case ast.ColumnOptionDefaultValue:
+ hasDefaultValue, err = SetDefaultValue(ctx, col, opt)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ case ast.ColumnOptionComment:
+ err := setColumnComment(ctx, col, opt)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ case ast.ColumnOptionNotNull:
+ col.AddFlag(mysql.NotNullFlag)
+ case ast.ColumnOptionNull:
+ hasNullFlag = true
+ col.DelFlag(mysql.NotNullFlag)
+ case ast.ColumnOptionAutoIncrement:
+ col.AddFlag(mysql.AutoIncrementFlag)
+ case ast.ColumnOptionPrimaryKey:
+ return errors.Trace(dbterror.ErrUnsupportedModifyColumn.GenWithStack("can't change column constraint (PRIMARY KEY)"))
+ case ast.ColumnOptionUniqKey:
+ return errors.Trace(dbterror.ErrUnsupportedModifyColumn.GenWithStack("can't change column constraint (UNIQUE KEY)"))
+ case ast.ColumnOptionOnUpdate:
+ // TODO: Support other time functions.
+ if !(col.GetType() == mysql.TypeTimestamp || col.GetType() == mysql.TypeDatetime) {
+ return dbterror.ErrInvalidOnUpdate.GenWithStackByArgs(col.Name)
+ }
+ if !expression.IsValidCurrentTimestampExpr(opt.Expr, &col.FieldType) {
+ return dbterror.ErrInvalidOnUpdate.GenWithStackByArgs(col.Name)
+ }
+ col.AddFlag(mysql.OnUpdateNowFlag)
+ setOnUpdateNow = true
+ case ast.ColumnOptionGenerated:
+ sb.Reset()
+ err = opt.Expr.Restore(restoreCtx)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ col.GeneratedExprString = sb.String()
+ col.GeneratedStored = opt.Stored
+ col.Dependences = make(map[string]struct{})
+ // Only used by checkModifyGeneratedColumn, there is no need to set a ctor for it.
+ col.GeneratedExpr = table.NewClonableExprNode(nil, opt.Expr)
+ for _, colName := range FindColumnNamesInExpr(opt.Expr) {
+ col.Dependences[colName.Name.L] = struct{}{}
+ }
+ case ast.ColumnOptionCollate:
+ col.SetCollate(opt.StrValue)
+ case ast.ColumnOptionReference:
+ return errors.Trace(dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs("can't modify with references"))
+ case ast.ColumnOptionFulltext:
+ return errors.Trace(dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs("can't modify with full text"))
+ case ast.ColumnOptionCheck:
+ return errors.Trace(dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs("can't modify with check"))
+ // Ignore ColumnOptionAutoRandom. It will be handled later.
+ case ast.ColumnOptionAutoRandom:
+ default:
+ return errors.Trace(dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs(fmt.Sprintf("unknown column option type: %d", opt.Tp)))
+ }
+ }
+
+ if err = processAndCheckDefaultValueAndColumn(ctx, col, nil, hasDefaultValue, setOnUpdateNow, hasNullFlag); err != nil {
+ return errors.Trace(err)
+ }
+
+ return nil
+}
+
+func checkAutoRandom(tableInfo *model.TableInfo, originCol *table.Column, specNewColumn *ast.ColumnDef) (uint64, error) {
+ var oldShardBits, oldRangeBits uint64
+ if isClusteredPKColumn(originCol, tableInfo) {
+ oldShardBits = tableInfo.AutoRandomBits
+ oldRangeBits = tableInfo.AutoRandomRangeBits
+ }
+ newShardBits, newRangeBits, err := extractAutoRandomBitsFromColDef(specNewColumn)
+ if err != nil {
+ return 0, errors.Trace(err)
+ }
+ switch {
+ case oldShardBits == newShardBits:
+ case oldShardBits < newShardBits:
+ addingAutoRandom := oldShardBits == 0
+ if addingAutoRandom {
+ convFromAutoInc := mysql.HasAutoIncrementFlag(originCol.GetFlag()) && originCol.IsPKHandleColumn(tableInfo)
+ if !convFromAutoInc {
+ return 0, dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomAlterChangeFromAutoInc)
+ }
+ }
+ if autoid.AutoRandomShardBitsMax < newShardBits {
+ errMsg := fmt.Sprintf(autoid.AutoRandomOverflowErrMsg,
+ autoid.AutoRandomShardBitsMax, newShardBits, specNewColumn.Name.Name.O)
+ return 0, dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(errMsg)
+ }
+ // increasing auto_random shard bits is allowed.
+ case oldShardBits > newShardBits:
+ if newShardBits == 0 {
+ return 0, dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomAlterErrMsg)
+ }
+ return 0, dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomDecreaseBitErrMsg)
+ }
+
+ modifyingAutoRandCol := oldShardBits > 0 || newShardBits > 0
+ if modifyingAutoRandCol {
+ // Disallow changing the column field type.
+ if originCol.GetType() != specNewColumn.Tp.GetType() {
+ return 0, dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomModifyColTypeErrMsg)
+ }
+ if originCol.GetType() != mysql.TypeLonglong {
+ return 0, dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(fmt.Sprintf(autoid.AutoRandomOnNonBigIntColumn, types.TypeStr(originCol.GetType())))
+ }
+ // Disallow changing from auto_random to auto_increment column.
+ if containsColumnOption(specNewColumn, ast.ColumnOptionAutoIncrement) {
+ return 0, dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomIncompatibleWithAutoIncErrMsg)
+ }
+ // Disallow specifying a default value on auto_random column.
+ if containsColumnOption(specNewColumn, ast.ColumnOptionDefaultValue) {
+ return 0, dbterror.ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomIncompatibleWithDefaultValueErrMsg)
+ }
+ }
+ if rangeBitsIsChanged(oldRangeBits, newRangeBits) {
+ return 0, dbterror.ErrInvalidAutoRandom.FastGenByArgs(autoid.AutoRandomUnsupportedAlterRangeBits)
+ }
+ return newShardBits, nil
+}
+
+func isClusteredPKColumn(col *table.Column, tblInfo *model.TableInfo) bool {
+ switch {
+ case tblInfo.PKIsHandle:
+ return mysql.HasPriKeyFlag(col.GetFlag())
+ case tblInfo.IsCommonHandle:
+ pk := tables.FindPrimaryIndex(tblInfo)
+ for _, c := range pk.Columns {
+ if c.Name.L == col.Name.L {
+ return true
+ }
+ }
+ return false
+ default:
+ return false
+ }
+}
+
+func rangeBitsIsChanged(oldBits, newBits uint64) bool {
+ if oldBits == 0 {
+ oldBits = autoid.AutoRandomRangeBitsDefault
+ }
+ if newBits == 0 {
+ newBits = autoid.AutoRandomRangeBitsDefault
+ }
+ return oldBits != newBits
+}
diff --git a/pkg/ddl/modify_column_test.go b/pkg/ddl/modify_column_test.go
index c951bf7c5e75b..450be74cda387 100644
--- a/pkg/ddl/modify_column_test.go
+++ b/pkg/ddl/modify_column_test.go
@@ -24,7 +24,6 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/ddl"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/meta"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/model"
@@ -33,6 +32,7 @@ import (
"github.com/pingcap/tidb/pkg/sessiontxn"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/external"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/util/mock"
"github.com/stretchr/testify/require"
)
@@ -49,7 +49,7 @@ func batchInsert(tk *testkit.TestKit, tbl string, start, end int) {
}
func TestModifyColumnReorgInfo(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
originalTimeout := ddl.ReorgWaitTimeout
ddl.ReorgWaitTimeout = 10 * time.Millisecond
@@ -75,14 +75,13 @@ func TestModifyColumnReorgInfo(t *testing.T) {
tbl := external.GetTableByName(t, tk, "test", "t1")
// Check insert null before job first update.
- hook := &callback.TestDDLCallback{Do: dom}
var checkErr error
var currJob *model.Job
var elements []*meta.Element
ctx := mock.NewContext()
ctx.Store = store
times := 0
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if tbl.Meta().ID != job.TableID || checkErr != nil || job.SchemaState != model.StateWriteReorganization {
return
}
@@ -113,9 +112,8 @@ func TestModifyColumnReorgInfo(t *testing.T) {
indexInfo := tbl.Meta().FindIndexByName("idx2")
elements = []*meta.Element{{ID: indexInfo.ID, TypeKey: meta.IndexElementKey}}
}
- }
+ })
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/MockGetIndexRecordErr", `return("cantDecodeRecordErr")`))
- dom.DDL().SetHook(hook)
err := tk.ExecToErr(sql)
require.EqualError(t, err, "[ddl:8202]Cannot decode index value, because mock can't decode record error")
require.NoError(t, checkErr)
@@ -189,7 +187,7 @@ func TestModifyColumnNullToNotNullWithChangingVal2(t *testing.T) {
}
func TestModifyColumnNullToNotNull(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, 600*time.Millisecond)
+ store := testkit.CreateMockStoreWithSchemaLease(t, 600*time.Millisecond)
tk1 := testkit.NewTestKit(t, store)
tk2 := testkit.NewTestKit(t, store)
@@ -201,19 +199,17 @@ func TestModifyColumnNullToNotNull(t *testing.T) {
tbl := external.GetTableByName(t, tk1, "test", "t1")
// Check insert null before job first update.
- hook := &callback.TestDDLCallback{Do: dom}
tk1.MustExec("delete from t1")
once := sync.Once{}
var checkErr error
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if tbl.Meta().ID != job.TableID {
return
}
once.Do(func() {
checkErr = tk2.ExecToErr("insert into t1 values ()")
})
- }
- dom.DDL().SetHook(hook)
+ })
err := tk1.ExecToErr("alter table t1 change c2 c2 int not null")
require.NoError(t, checkErr)
require.EqualError(t, err, "[ddl:1138]Invalid use of NULL value")
@@ -221,7 +217,7 @@ func TestModifyColumnNullToNotNull(t *testing.T) {
// Check insert error when column has PreventNullInsertFlag.
tk1.MustExec("delete from t1")
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if tbl.Meta().ID != job.TableID {
return
}
@@ -231,8 +227,7 @@ func TestModifyColumnNullToNotNull(t *testing.T) {
}
// now c2 has PreventNullInsertFlag, an error is expected.
checkErr = tk2.ExecToErr("insert into t1 values ()")
- }
- dom.DDL().SetHook(hook)
+ })
tk1.MustExec("alter table t1 change c2 c2 int not null")
require.EqualError(t, checkErr, "[table:1048]Column 'c2' cannot be null")
@@ -244,7 +239,7 @@ func TestModifyColumnNullToNotNull(t *testing.T) {
}
func TestModifyColumnNullToNotNullWithChangingVal(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, 600*time.Millisecond)
+ store := testkit.CreateMockStoreWithSchemaLease(t, 600*time.Millisecond)
tk1 := testkit.NewTestKit(t, store)
tk2 := testkit.NewTestKit(t, store)
@@ -256,19 +251,17 @@ func TestModifyColumnNullToNotNullWithChangingVal(t *testing.T) {
tbl := external.GetTableByName(t, tk1, "test", "t1")
// Check insert null before job first update.
- hook := &callback.TestDDLCallback{Do: dom}
tk1.MustExec("delete from t1")
once := sync.Once{}
var checkErr error
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if tbl.Meta().ID != job.TableID {
return
}
once.Do(func() {
checkErr = tk2.ExecToErr("insert into t1 values ()")
})
- }
- dom.DDL().SetHook(hook)
+ })
err := tk1.ExecToErr("alter table t1 change c2 c2 tinyint not null")
require.NoError(t, checkErr)
require.EqualError(t, err, "[ddl:1265]Data truncated for column 'c2' at row 1")
@@ -276,7 +269,7 @@ func TestModifyColumnNullToNotNullWithChangingVal(t *testing.T) {
// Check insert error when column has PreventNullInsertFlag.
tk1.MustExec("delete from t1")
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if tbl.Meta().ID != job.TableID {
return
}
@@ -286,8 +279,7 @@ func TestModifyColumnNullToNotNullWithChangingVal(t *testing.T) {
}
// now c2 has PreventNullInsertFlag, an error is expected.
checkErr = tk2.ExecToErr("insert into t1 values ()")
- }
- dom.DDL().SetHook(hook)
+ })
tk1.MustExec("alter table t1 change c2 c2 tinyint not null")
require.EqualError(t, checkErr, "[table:1048]Column 'c2' cannot be null")
diff --git a/pkg/ddl/multi_schema_change.go b/pkg/ddl/multi_schema_change.go
index e2186fdea00f3..4edf6b171de44 100644
--- a/pkg/ddl/multi_schema_change.go
+++ b/pkg/ddl/multi_schema_change.go
@@ -15,10 +15,6 @@
package ddl
import (
- "fmt"
-
- "github.com/pingcap/errors"
- ddllogutil "github.com/pingcap/tidb/pkg/ddl/logutil"
"github.com/pingcap/tidb/pkg/meta"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/model"
@@ -26,129 +22,9 @@ import (
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/util/dbterror"
- "github.com/pingcap/tidb/pkg/util/intest"
- "go.uber.org/zap"
)
-func (e *executor) MultiSchemaChange(ctx sessionctx.Context, ti ast.Ident, info *model.MultiSchemaInfo) error {
- subJobs := info.SubJobs
- if len(subJobs) == 0 {
- return nil
- }
- schema, t, err := e.getSchemaAndTableByIdent(ti)
- if err != nil {
- return errors.Trace(err)
- }
-
- logFn := ddllogutil.DDLLogger().Warn
- if intest.InTest {
- logFn = ddllogutil.DDLLogger().Fatal
- }
-
- var involvingSchemaInfo []model.InvolvingSchemaInfo
- for _, j := range subJobs {
- switch j.Type {
- case model.ActionAlterTablePlacement:
- ref, ok := j.Args[0].(*model.PolicyRefInfo)
- if !ok {
- logFn("unexpected type of policy reference info",
- zap.Any("args[0]", j.Args[0]),
- zap.String("type", fmt.Sprintf("%T", j.Args[0])))
- continue
- }
- if ref == nil {
- continue
- }
- involvingSchemaInfo = append(involvingSchemaInfo, model.InvolvingSchemaInfo{
- Policy: ref.Name.L,
- Mode: model.SharedInvolving,
- })
- case model.ActionAddForeignKey:
- ref, ok := j.Args[0].(*model.FKInfo)
- if !ok {
- logFn("unexpected type of foreign key info",
- zap.Any("args[0]", j.Args[0]),
- zap.String("type", fmt.Sprintf("%T", j.Args[0])))
- continue
- }
- involvingSchemaInfo = append(involvingSchemaInfo, model.InvolvingSchemaInfo{
- Database: ref.RefSchema.L,
- Table: ref.RefTable.L,
- Mode: model.SharedInvolving,
- })
- case model.ActionAlterTablePartitionPlacement:
- if len(j.Args) < 2 {
- logFn("unexpected number of arguments for partition placement",
- zap.Int("len(args)", len(j.Args)),
- zap.Any("args", j.Args))
- continue
- }
- ref, ok := j.Args[1].(*model.PolicyRefInfo)
- if !ok {
- logFn("unexpected type of policy reference info",
- zap.Any("args[0]", j.Args[0]),
- zap.String("type", fmt.Sprintf("%T", j.Args[0])))
- continue
- }
- if ref == nil {
- continue
- }
- involvingSchemaInfo = append(involvingSchemaInfo, model.InvolvingSchemaInfo{
- Policy: ref.Name.L,
- Mode: model.SharedInvolving,
- })
- }
- }
-
- if len(involvingSchemaInfo) > 0 {
- involvingSchemaInfo = append(involvingSchemaInfo, model.InvolvingSchemaInfo{
- Database: schema.Name.L,
- Table: t.Meta().Name.L,
- })
- }
-
- job := &model.Job{
- SchemaID: schema.ID,
- TableID: t.Meta().ID,
- SchemaName: schema.Name.L,
- TableName: t.Meta().Name.L,
- Type: model.ActionMultiSchemaChange,
- BinlogInfo: &model.HistoryInfo{},
- Args: nil,
- MultiSchemaInfo: info,
- ReorgMeta: nil,
- CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
- InvolvingSchemaInfo: involvingSchemaInfo,
- SQLMode: ctx.GetSessionVars().SQLMode,
- }
- if containsDistTaskSubJob(subJobs) {
- job.ReorgMeta, err = newReorgMetaFromVariables(job, ctx)
- if err != nil {
- return err
- }
- } else {
- job.ReorgMeta = NewDDLReorgMeta(ctx)
- }
-
- err = checkMultiSchemaInfo(info, t)
- if err != nil {
- return errors.Trace(err)
- }
- mergeAddIndex(info)
- return e.DoDDLJob(ctx, job)
-}
-
-func containsDistTaskSubJob(subJobs []*model.SubJob) bool {
- for _, sub := range subJobs {
- if sub.Type == model.ActionAddIndex ||
- sub.Type == model.ActionAddPrimaryKey {
- return true
- }
- }
- return false
-}
-
-func onMultiSchemaChange(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func onMultiSchemaChange(w *worker, jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
if job.MultiSchemaInfo.Revertible {
// Handle the rolling back job.
if job.IsRollingback() {
@@ -159,7 +35,7 @@ func onMultiSchemaChange(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job) (ve
continue
}
proxyJob := sub.ToProxyJob(job, i)
- ver, _, err = w.runOneJobStep(d, t, &proxyJob)
+ ver, _, err = w.runOneJobStep(jobCtx, t, &proxyJob)
err = handleRollbackException(err, proxyJob.Error)
if err != nil {
return ver, err
@@ -182,7 +58,7 @@ func onMultiSchemaChange(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job) (ve
continue
}
proxyJob := sub.ToProxyJob(job, i)
- ver, _, err = w.runOneJobStep(d, t, &proxyJob)
+ ver, _, err = w.runOneJobStep(jobCtx, t, &proxyJob)
sub.FromProxyJob(&proxyJob, ver)
handleRevertibleException(job, sub, proxyJob.Error)
return ver, err
@@ -208,7 +84,7 @@ func onMultiSchemaChange(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job) (ve
if schemaVersionGenerated {
proxyJob.MultiSchemaInfo.SkipVersion = true
}
- proxyJobVer, _, err := w.runOneJobStep(d, t, &proxyJob)
+ proxyJobVer, _, err := w.runOneJobStep(jobCtx, t, &proxyJob)
if !schemaVersionGenerated && proxyJobVer != 0 {
schemaVersionGenerated = true
ver = proxyJobVer
@@ -228,7 +104,7 @@ func onMultiSchemaChange(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job) (ve
// if we fail on "add column c int", the allocator is rebased to 100
// which cannot be rollback, but it's table-info.AutoIncID is rollback by below call.
// TODO we should also change schema diff of 'ver' if len(actionTypes) > 1.
- return updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ return updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
}
actionTypes = append(actionTypes, sub.Type)
}
@@ -257,7 +133,7 @@ func onMultiSchemaChange(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job) (ve
continue
}
proxyJob := sub.ToProxyJob(job, i)
- ver, _, err = w.runOneJobStep(d, t, &proxyJob)
+ ver, _, err = w.runOneJobStep(jobCtx, t, &proxyJob)
sub.FromProxyJob(&proxyJob, ver)
return ver, err
}
@@ -473,7 +349,6 @@ func mergeAddIndex(info *model.MultiSchemaInfo) {
var indexPartSpecifications [][]*ast.IndexPartSpecification
var indexOption []*ast.IndexOption
var hiddenCols [][]*model.ColumnInfo
- var global []bool
newSubJobs := make([]*model.SubJob, 0, len(info.SubJobs))
for _, subJob := range info.SubJobs {
@@ -483,13 +358,12 @@ func mergeAddIndex(info *model.MultiSchemaInfo) {
indexPartSpecifications = append(indexPartSpecifications, subJob.Args[2].([]*ast.IndexPartSpecification))
indexOption = append(indexOption, subJob.Args[3].(*ast.IndexOption))
hiddenCols = append(hiddenCols, subJob.Args[4].([]*model.ColumnInfo))
- global = append(global, subJob.Args[5].(bool))
} else {
newSubJobs = append(newSubJobs, subJob)
}
}
- mergedSubJob.Args = []any{unique, indexNames, indexPartSpecifications, indexOption, hiddenCols, global}
+ mergedSubJob.Args = []any{unique, indexNames, indexPartSpecifications, indexOption, hiddenCols}
// place the merged add index job at the end of the sub-jobs.
newSubJobs = append(newSubJobs, mergedSubJob)
info.SubJobs = newSubJobs
diff --git a/pkg/ddl/multi_schema_change_test.go b/pkg/ddl/multi_schema_change_test.go
index 7202d9bb99bec..6812af5c41567 100644
--- a/pkg/ddl/multi_schema_change_test.go
+++ b/pkg/ddl/multi_schema_change_test.go
@@ -20,8 +20,6 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/ddl"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
- "github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/model"
@@ -34,14 +32,13 @@ import (
)
func TestMultiSchemaChangeAddColumnsCancelled(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
- originHook := dom.DDL().GetHook()
tk.MustExec("create table t (a int);")
tk.MustExec("insert into t values (1);")
- hook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
+ hook := newCancelJobHook(t, store, func(job *model.Job) bool {
// Cancel job when the column 'c' is in write-reorg.
if job.Type != model.ActionMultiSchemaChange {
return false
@@ -49,10 +46,10 @@ func TestMultiSchemaChangeAddColumnsCancelled(t *testing.T) {
assertMultiSchema(t, job, 3)
return job.MultiSchemaInfo.SubJobs[1].SchemaState == model.StateWriteReorganization
})
- dom.DDL().SetHook(hook)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", hook.OnJobUpdated)
sql := "alter table t add column b int default 2, add column c int default 3, add column d int default 4;"
tk.MustGetErrCode(sql, errno.ErrCancelledDDLJob)
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
hook.MustCancelDone(t)
tk.MustQuery("select * from t;").Check(testkit.Rows("1"))
}
@@ -80,15 +77,14 @@ func TestMultiSchemaChangeAddColumnsParallel(t *testing.T) {
}
func TestMultiSchemaChangeDropColumnsCancelled(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
- originHook := dom.DDL().GetHook()
// Test for cancelling the job in a middle state.
tk.MustExec("create table t (a int default 1, b int default 2, c int default 3, d int default 4);")
tk.MustExec("insert into t values ();")
- hook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
+ hook := newCancelJobHook(t, store, func(job *model.Job) bool {
// Cancel job when the column 'a' is in delete-reorg.
if job.Type != model.ActionMultiSchemaChange {
return false
@@ -96,9 +92,9 @@ func TestMultiSchemaChangeDropColumnsCancelled(t *testing.T) {
assertMultiSchema(t, job, 3)
return job.MultiSchemaInfo.SubJobs[1].SchemaState == model.StateDeleteReorganization
})
- dom.DDL().SetHook(hook)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", hook.OnJobUpdated)
tk.MustExec("alter table t drop column b, drop column a, drop column d;")
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
hook.MustCancelFailed(t)
tk.MustQuery("select * from t;").Check(testkit.Rows("3"))
@@ -106,7 +102,7 @@ func TestMultiSchemaChangeDropColumnsCancelled(t *testing.T) {
tk.MustExec("drop table if exists t;")
tk.MustExec("create table t (a int default 1, b int default 2, c int default 3, d int default 4);")
tk.MustExec("insert into t values ();")
- hook = newCancelJobHook(t, store, dom, func(job *model.Job) bool {
+ hook = newCancelJobHook(t, store, func(job *model.Job) bool {
// Cancel job when the column 'a' is in public.
if job.Type != model.ActionMultiSchemaChange {
return false
@@ -114,24 +110,23 @@ func TestMultiSchemaChangeDropColumnsCancelled(t *testing.T) {
assertMultiSchema(t, job, 3)
return job.MultiSchemaInfo.SubJobs[1].SchemaState == model.StatePublic
})
- dom.DDL().SetHook(hook)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", hook.OnJobUpdated)
tk.MustGetErrCode("alter table t drop column b, drop column a, drop column d;", errno.ErrCancelledDDLJob)
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
hook.MustCancelDone(t)
tk.MustQuery("select * from t;").Check(testkit.Rows("1 2 3 4"))
}
func TestMultiSchemaChangeDropIndexedColumnsCancelled(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
- originHook := dom.DDL().GetHook()
// Test for cancelling the job in a middle state.
tk.MustExec("create table t (a int default 1, b int default 2, c int default 3, d int default 4, " +
"index(a), index(b), index(c), index(d));")
tk.MustExec("insert into t values ();")
- hook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
+ hook := newCancelJobHook(t, store, func(job *model.Job) bool {
// Cancel job when the column 'a' is in delete-reorg.
if job.Type != model.ActionMultiSchemaChange {
return false
@@ -139,9 +134,8 @@ func TestMultiSchemaChangeDropIndexedColumnsCancelled(t *testing.T) {
assertMultiSchema(t, job, 3)
return job.MultiSchemaInfo.SubJobs[1].SchemaState == model.StateDeleteReorganization
})
- dom.DDL().SetHook(hook)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", hook.OnJobUpdated)
tk.MustExec("alter table t drop column b, drop column a, drop column d;")
- dom.DDL().SetHook(originHook)
hook.MustCancelFailed(t)
tk.MustQuery("select * from t;").Check(testkit.Rows("3"))
}
@@ -165,8 +159,7 @@ func TestMultiSchemaChangeDropColumnsParallel(t *testing.T) {
}
func TestMultiSchemaChangeRenameColumns(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
- originHook := dom.DDL().GetHook()
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -209,7 +202,7 @@ func TestMultiSchemaChangeRenameColumns(t *testing.T) {
tk.MustExec("drop table if exists t")
tk.MustExec("create table t (a int default 1, b int default 2)")
tk.MustExec("insert into t values ()")
- hook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
+ hook := newCancelJobHook(t, store, func(job *model.Job) bool {
// Cancel job when the column 'c' is in write-reorg.
if job.Type != model.ActionMultiSchemaChange {
return false
@@ -217,9 +210,9 @@ func TestMultiSchemaChangeRenameColumns(t *testing.T) {
assertMultiSchema(t, job, 2)
return job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateWriteReorganization
})
- dom.DDL().SetHook(hook)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", hook.OnJobUpdated)
tk.MustGetErrCode("alter table t add column c int default 3, rename column b to d;", errno.ErrCancelledDDLJob)
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
tk.MustQuery("select b from t").Check(testkit.Rows("2"))
tk.MustGetErrCode("select d from t", errno.ErrBadField)
@@ -227,24 +220,21 @@ func TestMultiSchemaChangeRenameColumns(t *testing.T) {
tk.MustExec("drop table if exists t")
tk.MustExec("create table t (a int default 1, b int default 2)")
tk.MustExec("insert into t values ()")
- hook1 := &callback.TestDDLCallback{Do: dom}
- hook1.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
assert.Equal(t, model.ActionMultiSchemaChange, job.Type)
if job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateWriteReorganization {
rs, _ := tk.Exec("select b from t")
assert.Equal(t, tk.ResultSetToResult(rs, "").Rows()[0][0], "2")
}
- }
- dom.DDL().SetHook(hook1)
+ })
tk.MustExec("alter table t add column c int default 3, rename column b to d;")
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore")
tk.MustQuery("select d from t").Check(testkit.Rows("2"))
tk.MustGetErrCode("select b from t", errno.ErrBadField)
}
func TestMultiSchemaChangeAlterColumns(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
- originHook := dom.DDL().GetHook()
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -281,7 +271,7 @@ func TestMultiSchemaChangeAlterColumns(t *testing.T) {
// Test cancel job with alter columns
tk.MustExec("drop table if exists t")
tk.MustExec("create table t (a int default 1, b int default 2)")
- hook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
+ hook := newCancelJobHook(t, store, func(job *model.Job) bool {
// Cancel job when the column 'a' is in write-reorg.
if job.Type != model.ActionMultiSchemaChange {
return false
@@ -289,32 +279,28 @@ func TestMultiSchemaChangeAlterColumns(t *testing.T) {
assertMultiSchema(t, job, 2)
return job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateWriteReorganization
})
- dom.DDL().SetHook(hook)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", hook.OnJobUpdated)
tk.MustGetErrCode("alter table t add column c int default 3, alter column b set default 3;", errno.ErrCancelledDDLJob)
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
tk.MustExec("insert into t values ()")
tk.MustQuery("select * from t").Check(testkit.Rows("1 2"))
// Test dml stmts when do alter
tk.MustExec("drop table if exists t")
tk.MustExec("create table t (a int default 1, b int default 2)")
- hook1 := &callback.TestDDLCallback{Do: dom}
- hook1.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
assert.Equal(t, model.ActionMultiSchemaChange, job.Type)
if job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateWriteOnly {
tk2 := testkit.NewTestKit(t, store)
tk2.MustExec("insert into test.t values ()")
}
- }
- dom.DDL().SetHook(hook1)
+ })
tk.MustExec("alter table t add column c int default 3, alter column b set default 3;")
- dom.DDL().SetHook(originHook)
tk.MustQuery("select * from t").Check(testkit.Rows("1 2 3"))
}
func TestMultiSchemaChangeChangeColumns(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
- originHook := dom.DDL().GetHook()
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -353,7 +339,7 @@ func TestMultiSchemaChangeChangeColumns(t *testing.T) {
tk.MustExec("drop table if exists t")
tk.MustExec("create table t (a int default 1, b int default 2)")
tk.MustExec("insert into t values ()")
- hook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
+ hook := newCancelJobHook(t, store, func(job *model.Job) bool {
// Cancel job when the column 'c' is in write-reorg.
if job.Type != model.ActionMultiSchemaChange {
return false
@@ -361,24 +347,23 @@ func TestMultiSchemaChangeChangeColumns(t *testing.T) {
assertMultiSchema(t, job, 2)
return job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateWriteReorganization
})
- dom.DDL().SetHook(hook)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", hook.OnJobUpdated)
tk.MustGetErrCode("alter table t add column c int default 3, change column b d bigint default 4;", errno.ErrCancelledDDLJob)
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
tk.MustQuery("select b from t").Check(testkit.Rows("2"))
tk.MustGetErrCode("select d from t", errno.ErrBadField)
}
func TestMultiSchemaChangeAddIndexesCancelled(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
- originHook := dom.DDL().GetHook()
// Test cancel successfully.
tk.MustExec("drop table if exists t;")
tk.MustExec("create table t (a int, b int, c int);")
tk.MustExec("insert into t values (1, 2, 3);")
- cancelHook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
+ cancelHook := newCancelJobHook(t, store, func(job *model.Job) bool {
// Cancel the job when index 't2' is in write-reorg.
if job.Type != model.ActionMultiSchemaChange {
return false
@@ -386,11 +371,11 @@ func TestMultiSchemaChangeAddIndexesCancelled(t *testing.T) {
assertMultiSchema(t, job, 1)
return job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateWriteReorganization
})
- dom.DDL().SetHook(cancelHook)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", cancelHook.OnJobUpdated)
tk.MustGetErrCode("alter table t "+
"add index t(a, b), add index t1(a), "+
"add index t2(a), add index t3(a, b);", errno.ErrCancelledDDLJob)
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
cancelHook.MustCancelDone(t)
tk.MustQuery("show index from t;").Check(testkit.Rows( /* no index */ ))
tk.MustQuery("select * from t;").Check(testkit.Rows("1 2 3"))
@@ -400,7 +385,7 @@ func TestMultiSchemaChangeAddIndexesCancelled(t *testing.T) {
tk.MustExec("drop table if exists t;")
tk.MustExec("create table t (a int, b int, c int);")
tk.MustExec("insert into t values (1, 2, 3);")
- cancelHook = newCancelJobHook(t, store, dom, func(job *model.Job) bool {
+ cancelHook = newCancelJobHook(t, store, func(job *model.Job) bool {
// Cancel the job when index 't1' is in public.
if job.Type != model.ActionMultiSchemaChange {
return false
@@ -408,33 +393,32 @@ func TestMultiSchemaChangeAddIndexesCancelled(t *testing.T) {
assertMultiSchema(t, job, 1)
return job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StatePublic
})
- dom.DDL().SetHook(cancelHook)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", cancelHook.OnJobUpdated)
tk.MustExec("alter table t add index t(a, b), add index t1(a), " +
"add index t2(a), add index t3(a, b);")
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
cancelHook.MustCancelFailed(t)
tk.MustQuery("select * from t use index(t, t1, t2, t3);").Check(testkit.Rows("1 2 3"))
tk.MustExec("admin check table t;")
}
func TestMultiSchemaChangeDropIndexesCancelled(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test;")
- originHook := dom.DDL().GetHook()
// Test for cancelling the job in a middle state.
tk.MustExec("create table t (a int, b int, index(a), unique index(b), index idx(a, b));")
- hook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
+ hook := newCancelJobHook(t, store, func(job *model.Job) bool {
if job.Type != model.ActionMultiSchemaChange {
return false
}
assertMultiSchema(t, job, 3)
return job.MultiSchemaInfo.SubJobs[1].SchemaState == model.StateDeleteOnly
})
- dom.DDL().SetHook(hook)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", hook.OnJobUpdated)
tk.MustExec("alter table t drop index a, drop index b, drop index idx;")
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
hook.MustCancelFailed(t)
tk.MustGetErrCode("select * from t use index (a);", errno.ErrKeyDoesNotExist)
tk.MustGetErrCode("select * from t use index (b);", errno.ErrKeyDoesNotExist)
@@ -443,16 +427,16 @@ func TestMultiSchemaChangeDropIndexesCancelled(t *testing.T) {
// Test for cancelling the job in none state.
tk.MustExec("drop table if exists t;")
tk.MustExec("create table t (a int, b int, index(a), unique index(b), index idx(a, b));")
- hook = newCancelJobHook(t, store, dom, func(job *model.Job) bool {
+ hook = newCancelJobHook(t, store, func(job *model.Job) bool {
if job.Type != model.ActionMultiSchemaChange {
return false
}
assertMultiSchema(t, job, 3)
return job.MultiSchemaInfo.SubJobs[1].SchemaState == model.StatePublic
})
- dom.DDL().SetHook(hook)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", hook.OnJobUpdated)
tk.MustGetErrCode("alter table t drop index a, drop index b, drop index idx;", errno.ErrCancelledDDLJob)
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
hook.MustCancelDone(t)
tk.MustQuery("select * from t use index (a);").Check(testkit.Rows())
tk.MustQuery("select * from t use index (b);").Check(testkit.Rows())
@@ -478,10 +462,9 @@ func TestMultiSchemaChangeDropIndexesParallel(t *testing.T) {
}
func TestMultiSchemaChangeRenameIndexes(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
- originHook := dom.DDL().GetHook()
// Test rename index.
tk.MustExec("drop table if exists t")
@@ -514,7 +497,7 @@ func TestMultiSchemaChangeRenameIndexes(t *testing.T) {
tk.MustExec("drop table if exists t")
tk.MustExec("create table t (a int default 1, b int default 2, index t(a))")
tk.MustExec("insert into t values ()")
- hook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
+ hook := newCancelJobHook(t, store, func(job *model.Job) bool {
// Cancel job when the column 'c' is in write-reorg.
if job.Type != model.ActionMultiSchemaChange {
return false
@@ -522,33 +505,32 @@ func TestMultiSchemaChangeRenameIndexes(t *testing.T) {
assertMultiSchema(t, job, 2)
return job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateWriteReorganization
})
- dom.DDL().SetHook(hook)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", hook.OnJobUpdated)
tk.MustGetErrCode("alter table t add column c int default 3, rename index t to t1;", errno.ErrCancelledDDLJob)
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
tk.MustQuery("select * from t use index (t);").Check(testkit.Rows("1 2"))
tk.MustGetErrCode("select * from t use index (t1);", errno.ErrKeyDoesNotExist)
}
func TestMultiSchemaChangeModifyColumnsCancelled(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test;")
- originHook := dom.DDL().GetHook()
// Test for cancelling the job in a middle state.
tk.MustExec("create table t (a int, b int, c int, index i1(a), unique index i2(b), index i3(a, b));")
tk.MustExec("insert into t values (1, 2, 3);")
- hook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
+ hook := newCancelJobHook(t, store, func(job *model.Job) bool {
if job.Type != model.ActionMultiSchemaChange {
return false
}
assertMultiSchema(t, job, 3)
return job.MultiSchemaInfo.SubJobs[2].SchemaState == model.StateWriteReorganization
})
- dom.DDL().SetHook(hook)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", hook.OnJobUpdated)
sql := "alter table t modify column a tinyint, modify column b bigint, modify column c char(20);"
tk.MustGetErrCode(sql, errno.ErrCancelledDDLJob)
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
hook.MustCancelDone(t)
tk.MustQuery("select * from t;").Check(testkit.Rows("1 2 3"))
tk.MustQuery("select * from t use index (i1, i2, i3);").Check(testkit.Rows("1 2 3"))
@@ -558,7 +540,7 @@ func TestMultiSchemaChangeModifyColumnsCancelled(t *testing.T) {
}
func TestMultiSchemaChangeAlterIndex(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test;")
@@ -592,10 +574,8 @@ func TestMultiSchemaChangeAlterIndex(t *testing.T) {
tk.MustExec("drop table t;")
tk.MustExec("create table t (a int, b int, index i1(a, b), index i2(b));")
tk.MustExec("insert into t values (1, 2);")
- originHook := dom.DDL().GetHook()
var checked bool
- callback := &callback.TestDDLCallback{Do: dom}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.MultiSchemaInfo == nil {
return
}
@@ -606,11 +586,9 @@ func TestMultiSchemaChangeAlterIndex(t *testing.T) {
assert.NoError(t, err)
assert.NoError(t, rs.Close())
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- dom.DDL().SetHook(callback)
+ })
tk.MustExec("alter table t alter index i1 invisible, modify column a tinyint, alter index i2 invisible;")
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
require.True(t, checked)
tk.MustGetErrCode("select * from t use index (i1);", errno.ErrKeyDoesNotExist)
tk.MustGetErrCode("select * from t use index (i2);", errno.ErrKeyDoesNotExist)
@@ -619,7 +597,7 @@ func TestMultiSchemaChangeAlterIndex(t *testing.T) {
}
func TestMultiSchemaChangeMixCancelled(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test;")
tk.MustExec("set global tidb_enable_dist_task = 0;")
@@ -627,18 +605,17 @@ func TestMultiSchemaChangeMixCancelled(t *testing.T) {
tk.MustExec("create table t (a int, b int, c int, index i1(c), index i2(c));")
tk.MustExec("insert into t values (1, 2, 3);")
- origin := dom.DDL().GetHook()
- cancelHook := newCancelJobHook(t, store, dom, func(job *model.Job) bool {
+ cancelHook := newCancelJobHook(t, store, func(job *model.Job) bool {
return job.MultiSchemaInfo != nil &&
len(job.MultiSchemaInfo.SubJobs) > 8 &&
job.MultiSchemaInfo.SubJobs[8].SchemaState == model.StateWriteReorganization
})
- dom.DDL().SetHook(cancelHook)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", cancelHook.OnJobUpdated)
tk.MustGetErrCode("alter table t add column d int default 4, add index i3(c), "+
"drop column a, drop column if exists z, add column if not exists e int default 5, "+
"drop index i2, add column f int default 6, drop column b, drop index i1, add column if not exists g int;",
errno.ErrCancelledDDLJob)
- dom.DDL().SetHook(origin)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
cancelHook.MustCancelDone(t)
tk.MustQuery("select * from t;").Check(testkit.Rows("1 2 3"))
tk.MustQuery("select * from t use index(i1, i2);").Check(testkit.Rows("1 2 3"))
@@ -646,14 +623,15 @@ func TestMultiSchemaChangeMixCancelled(t *testing.T) {
}
func TestMultiSchemaChangeAdminShowDDLJobs(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 1;")
- originHook := dom.DDL().GetHook()
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ tk.MustExec("create table t (a int, b int, c int)")
+ tk.MustExec("insert into t values (1, 2, 3)")
+
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
assert.Equal(t, model.ActionMultiSchemaChange, job.Type)
if job.MultiSchemaInfo.SubJobs[0].SchemaState == model.StateDeleteOnly {
newTk := testkit.NewTestKit(t, store)
@@ -670,18 +648,13 @@ func TestMultiSchemaChangeAdminShowDDLJobs(t *testing.T) {
assert.True(t, len(rows[1][10].(string)) > 0)
assert.Equal(t, "create table", rows[2][3])
}
- }
-
- tk.MustExec("create table t (a int, b int, c int)")
- tk.MustExec("insert into t values (1, 2, 3)")
+ })
- dom.DDL().SetHook(hook)
tk.MustExec("alter table t add index t(a), add index t1(b)")
- dom.DDL().SetHook(originHook)
}
func TestMultiSchemaChangeWithExpressionIndex(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test;")
tk.MustExec("create table t (a int, b int);")
@@ -692,10 +665,8 @@ func TestMultiSchemaChangeWithExpressionIndex(t *testing.T) {
errno.ErrDupEntry)
tk.MustQuery("select * from t;").Check(testkit.Rows("1 2", "2 1"))
- originHook := dom.DDL().GetHook()
- hook := &callback.TestDDLCallback{Do: dom}
var checkErr error
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if checkErr != nil {
return
}
@@ -709,11 +680,10 @@ func TestMultiSchemaChangeWithExpressionIndex(t *testing.T) {
}
_, checkErr = tk2.Exec("insert into t values (10, 10);")
}
- }
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("alter table t add column c int default 10, add index idx1((a + b)), add unique index idx2((a + b));")
require.NoError(t, checkErr)
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore")
tk.MustExec("drop table if exists t;")
tk.MustExec("create table t (a int, b int);")
@@ -759,7 +729,7 @@ func TestMultiSchemaChangeSchemaVersion(t *testing.T) {
}
func TestMultiSchemaChangeMixedWithUpdate(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test;")
tk.MustExec("create table t (c_1 int, c_2 char(20), c_pos_1 int, c_idx_visible int, c_3 decimal(5, 3), " +
@@ -769,10 +739,8 @@ func TestMultiSchemaChangeMixedWithUpdate(t *testing.T) {
tk.MustExec("insert into t values (100, 'c_2_insert', 101, 12, 2.1, '10:00:00', " +
"'2020-01-01 10:00:00', 'wer', '10:00:00', 2.1, 12, 'qwer', 12, 'asdf');")
- originHook := dom.DDL().GetHook()
- hook := &callback.TestDDLCallback{Do: dom}
var checkErr error
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if checkErr != nil {
return
}
@@ -788,8 +756,7 @@ func TestMultiSchemaChangeMixedWithUpdate(t *testing.T) {
return
}
}
- }
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("alter table t " +
"add index i_add_1(c_add_idx_1), " +
"drop index idx_drop, " +
@@ -806,7 +773,6 @@ func TestMultiSchemaChangeMixedWithUpdate(t *testing.T) {
"alter index idx_visible invisible, " +
"modify column c_3 decimal(10, 2);")
require.NoError(t, checkErr)
- dom.DDL().SetHook(originHook)
}
func TestMultiSchemaChangeBlockedByRowLevelChecksum(t *testing.T) {
@@ -836,8 +802,6 @@ type cancelOnceHook struct {
cancelErr error
pred func(job *model.Job) bool
s sessionctx.Context
-
- callback.TestDDLCallback
}
func (c *cancelOnceHook) OnJobUpdated(job *model.Job) {
@@ -863,15 +827,14 @@ func (c *cancelOnceHook) MustCancelFailed(t *testing.T) {
require.Contains(t, c.cancelErr.Error(), strconv.Itoa(errno.ErrCannotCancelDDLJob))
}
-func newCancelJobHook(t *testing.T, store kv.Storage, dom *domain.Domain,
+func newCancelJobHook(t *testing.T, store kv.Storage,
pred func(job *model.Job) bool) *cancelOnceHook {
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
return &cancelOnceHook{
- store: store,
- pred: pred,
- TestDDLCallback: callback.TestDDLCallback{Do: dom},
- s: tk.Session(),
+ store: store,
+ pred: pred,
+ s: tk.Session(),
}
}
diff --git a/pkg/ddl/mv_index_test.go b/pkg/ddl/mv_index_test.go
index 1891d5bdbc62a..9fe51ece6d5c2 100644
--- a/pkg/ddl/mv_index_test.go
+++ b/pkg/ddl/mv_index_test.go
@@ -19,14 +19,14 @@ import (
"strings"
"testing"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
)
func TestMultiValuedIndexOnlineDDL(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -45,20 +45,17 @@ func TestMultiValuedIndexOnlineDDL(t *testing.T) {
internalTK := testkit.NewTestKit(t, store)
internalTK.MustExec("use test")
- hook := &callback.TestDDLCallback{Do: dom}
n := 100
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
internalTK.MustExec(fmt.Sprintf("insert into t values (%d, '[%d, %d, %d]')", n, n, n+1, n+2))
internalTK.MustExec(fmt.Sprintf("delete from t where pk = %d", n-4))
internalTK.MustExec(fmt.Sprintf("update t set a = '[%d, %d, %d]' where pk = %d", n-3, n-2, n+1000, n-3))
n++
- }
- o := dom.DDL().GetHook()
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("alter table t add index idx((cast(a as signed array)))")
tk.MustExec("admin check table t")
- dom.DDL().SetHook(o)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore")
tk.MustExec("drop table if exists t;")
tk.MustExec("create table t (pk int primary key, a json);")
diff --git a/pkg/ddl/options.go b/pkg/ddl/options.go
index 6340b7ac4058d..8712cda792f1b 100644
--- a/pkg/ddl/options.go
+++ b/pkg/ddl/options.go
@@ -32,7 +32,6 @@ type Options struct {
Store kv.Storage
AutoIDClient *autoid.ClientDiscover
InfoCache *infoschema.InfoCache
- Hook Callback
Lease time.Duration
SchemaLoader SchemaLoader
}
@@ -65,13 +64,6 @@ func WithAutoIDClient(cli *autoid.ClientDiscover) Option {
}
}
-// WithHook specifies the `Callback` of DDL used to notify the outer module when events are triggered
-func WithHook(callback Callback) Option {
- return func(options *Options) {
- options.Hook = callback
- }
-}
-
// WithLease specifies the schema lease duration
func WithLease(lease time.Duration) Option {
return func(options *Options) {
diff --git a/pkg/ddl/options_test.go b/pkg/ddl/options_test.go
index a7a299c1515e7..7463674f1489f 100644
--- a/pkg/ddl/options_test.go
+++ b/pkg/ddl/options_test.go
@@ -32,14 +32,12 @@ func TestOptions(t *testing.T) {
err := client.Close()
require.NoError(t, err)
}()
- callback := &ddl.BaseCallback{}
lease := time.Second * 3
store := &mock.Store{}
infoHandle := infoschema.NewCache(nil, 16)
options := []ddl.Option{
ddl.WithEtcdClient(client),
- ddl.WithHook(callback),
ddl.WithLease(lease),
ddl.WithStore(store),
ddl.WithInfoCache(infoHandle),
@@ -51,7 +49,6 @@ func TestOptions(t *testing.T) {
}
require.Equal(t, client, opt.EtcdCli)
- require.Equal(t, callback, opt.Hook)
require.Equal(t, lease, opt.Lease)
require.Equal(t, store, opt.Store)
require.Equal(t, infoHandle, opt.InfoCache)
diff --git a/pkg/ddl/partition.go b/pkg/ddl/partition.go
index 55ce25396fd1f..2c7582df789e6 100644
--- a/pkg/ddl/partition.go
+++ b/pkg/ddl/partition.go
@@ -45,6 +45,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/parser/opcode"
"github.com/pingcap/tidb/pkg/parser/terror"
+ field_types "github.com/pingcap/tidb/pkg/parser/types"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
statsutil "github.com/pingcap/tidb/pkg/statistics/handle/util"
@@ -61,6 +62,7 @@ import (
"github.com/pingcap/tidb/pkg/util/mathutil"
decoder "github.com/pingcap/tidb/pkg/util/rowDecoder"
"github.com/pingcap/tidb/pkg/util/slice"
+ "github.com/pingcap/tidb/pkg/util/sqlkiller"
"github.com/pingcap/tidb/pkg/util/stringutil"
"github.com/tikv/client-go/v2/tikv"
kvutil "github.com/tikv/client-go/v2/util"
@@ -91,10 +93,10 @@ func checkAddPartition(t *meta.Meta, job *model.Job) (*model.TableInfo, *model.P
}
// TODO: Move this into reorganize partition!
-func (w *worker) onAddTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func (w *worker) onAddTablePartition(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
// Handle the rolling back job
if job.IsRollingback() {
- ver, err := w.onDropTablePartition(d, t, job)
+ ver, err := w.onDropTablePartition(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -133,7 +135,7 @@ func (w *worker) onAddTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (v
// move the adding definition into tableInfo.
updateAddingPartitionInfo(partInfo, tblInfo)
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -186,9 +188,9 @@ func (w *worker) onAddTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (v
if tblInfo.TiFlashReplica != nil && tblInfo.TiFlashReplica.Available {
// For available state, the new added partition should wait it's replica to
// be finished. Otherwise the query to this partition will be blocked.
- needRetry, err := checkPartitionReplica(tblInfo.TiFlashReplica.Count, addingDefinitions, d)
+ needRetry, err := checkPartitionReplica(tblInfo.TiFlashReplica.Count, addingDefinitions, jobCtx)
if err != nil {
- return convertAddTablePartitionJob2RollbackJob(d, t, job, err, tblInfo)
+ return convertAddTablePartitionJob2RollbackJob(jobCtx, t, job, err, tblInfo)
}
if needRetry {
// The new added partition hasn't been replicated.
@@ -217,9 +219,9 @@ func (w *worker) onAddTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (v
// For normal and replica finished table, move the `addingDefinitions` into `Definitions`.
updatePartitionInfo(tblInfo)
- preSplitAndScatter(w.sess.Context, d.store, tblInfo, addingDefinitions)
+ preSplitAndScatter(w.sess.Context, jobCtx.store, tblInfo, addingDefinitions)
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -231,7 +233,7 @@ func (w *worker) onAddTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (v
tblInfo,
partInfo,
)
- asyncNotifyEvent(d, addPartitionEvent)
+ asyncNotifyEvent(jobCtx, addPartitionEvent)
default:
err = dbterror.ErrInvalidDDLState.GenWithStackByArgs("partition", job.SchemaState)
}
@@ -407,7 +409,7 @@ func checkAddPartitionValue(meta *model.TableInfo, part *model.PartitionInfo) er
return nil
}
-func checkPartitionReplica(replicaCount uint64, addingDefinitions []model.PartitionDefinition, d *ddlCtx) (needWait bool, err error) {
+func checkPartitionReplica(replicaCount uint64, addingDefinitions []model.PartitionDefinition, jobCtx *jobContext) (needWait bool, err error) {
failpoint.Inject("mockWaitTiFlashReplica", func(val failpoint.Value) {
if val.(bool) {
failpoint.Return(true, nil)
@@ -420,7 +422,7 @@ func checkPartitionReplica(replicaCount uint64, addingDefinitions []model.Partit
})
ctx := context.Background()
- pdCli := d.store.(tikv.Storage).GetRegionCache().PDClient()
+ pdCli := jobCtx.store.(tikv.Storage).GetRegionCache().PDClient()
stores, err := pdCli.GetAllStores(ctx)
if err != nil {
return needWait, errors.Trace(err)
@@ -621,16 +623,83 @@ func buildTablePartitionInfo(ctx sessionctx.Context, s *ast.PartitionOptions, tb
}
}
- partCols, err := getPartitionColSlices(exprCtx, tbInfo, s)
- if err != nil {
- return errors.Trace(err)
+ if len(s.UpdateIndexes) > 0 {
+ updateIndexes := make([]model.UpdateIndexInfo, 0, len(s.UpdateIndexes))
+ dupCheck := make(map[string]struct{})
+ for _, idxUpdate := range s.UpdateIndexes {
+ idxOffset := -1
+ for i := range tbInfo.Indices {
+ if strings.EqualFold(tbInfo.Indices[i].Name.L, idxUpdate.Name) {
+ idxOffset = i
+ break
+ }
+ }
+ if idxOffset == -1 {
+ if strings.EqualFold("primary", idxUpdate.Name) &&
+ tbInfo.PKIsHandle {
+ return dbterror.ErrUniqueKeyNeedAllFieldsInPf.GenWithStackByArgs("CLUSTERED INDEX")
+ }
+ return dbterror.ErrWrongNameForIndex.GenWithStackByArgs(idxUpdate.Name)
+ }
+ if _, ok := dupCheck[strings.ToLower(idxUpdate.Name)]; ok {
+ return dbterror.ErrWrongNameForIndex.GenWithStackByArgs(idxUpdate.Name)
+ }
+ dupCheck[strings.ToLower(idxUpdate.Name)] = struct{}{}
+ if idxUpdate.Option != nil && idxUpdate.Option.Global {
+ tbInfo.Indices[idxOffset].Global = true
+ } else {
+ tbInfo.Indices[idxOffset].Global = false
+ }
+ updateIndexes = append(updateIndexes, model.UpdateIndexInfo{IndexName: idxUpdate.Name, Global: tbInfo.Indices[idxOffset].Global})
+ tbInfo.Partition.DDLUpdateIndexes = updateIndexes
+ }
}
for _, index := range tbInfo.Indices {
- if index.Unique && !checkUniqueKeyIncludePartKey(partCols, index.Columns) {
- index.Global = ctx.GetSessionVars().EnableGlobalIndex
+ if index.Unique {
+ ck, err := checkPartitionKeysConstraint(pi, index.Columns, tbInfo)
+ if err != nil {
+ return err
+ }
+ if !ck {
+ indexTp := ""
+ if !ctx.GetSessionVars().EnableGlobalIndex {
+ if index.Primary {
+ indexTp = "PRIMARY KEY"
+ if tbInfo.IsCommonHandle {
+ indexTp = "CLUSTERED INDEX"
+ }
+ } else {
+ indexTp = "UNIQUE INDEX"
+ }
+ } else if index.Primary && tbInfo.IsCommonHandle {
+ indexTp = "CLUSTERED INDEX"
+ }
+ if indexTp != "" {
+ return dbterror.ErrUniqueKeyNeedAllFieldsInPf.GenWithStackByArgs(indexTp)
+ }
+ if !index.Global {
+ return dbterror.ErrGlobalIndexNotExplicitlySet.GenWithStackByArgs(index.Name.O)
+ }
+ }
}
}
+ if tbInfo.PKIsHandle {
+ // This case is covers when the Handle is the PK (only ints), since it would not
+ // have an entry in the tblInfo.Indices
+ indexCols := []*model.IndexColumn{{
+ Name: tbInfo.GetPkName(),
+ Length: types.UnspecifiedLength,
+ }}
+ ck, err := checkPartitionKeysConstraint(pi, indexCols, tbInfo)
+ if err != nil {
+ return err
+ }
+ if !ck {
+ return dbterror.ErrUniqueKeyNeedAllFieldsInPf.GenWithStackByArgs("CLUSTERED INDEX")
+ }
+ }
+
return nil
}
@@ -659,6 +728,50 @@ func getPartitionColSlices(sctx expression.BuildContext, tblInfo *model.TableInf
return nil, errors.Errorf("Table partition metadata not correct, neither partition expression or list of partition columns")
}
+func checkColumnsPartitionType(tbInfo *model.TableInfo) error {
+ for _, col := range tbInfo.Partition.Columns {
+ colInfo := tbInfo.FindPublicColumnByName(col.L)
+ if colInfo == nil {
+ return errors.Trace(dbterror.ErrFieldNotFoundPart)
+ }
+ if !isColTypeAllowedAsPartitioningCol(tbInfo.Partition.Type, colInfo.FieldType) {
+ return dbterror.ErrNotAllowedTypeInPartition.GenWithStackByArgs(col.O)
+ }
+ }
+ return nil
+}
+
+func isValidKeyPartitionColType(fieldType types.FieldType) bool {
+ switch fieldType.GetType() {
+ case mysql.TypeBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob, mysql.TypeJSON, mysql.TypeGeometry, mysql.TypeTiDBVectorFloat32:
+ return false
+ default:
+ return true
+ }
+}
+
+func isColTypeAllowedAsPartitioningCol(partType model.PartitionType, fieldType types.FieldType) bool {
+ // For key partition, the permitted partition field types can be all field types except
+ // BLOB, JSON, Geometry
+ if partType == model.PartitionTypeKey {
+ return isValidKeyPartitionColType(fieldType)
+ }
+ // The permitted data types are shown in the following list:
+ // All integer types
+ // DATE and DATETIME
+ // CHAR, VARCHAR, BINARY, and VARBINARY
+ // See https://dev.mysql.com/doc/mysql-partitioning-excerpt/5.7/en/partitioning-columns.html
+ // Note that also TIME is allowed in MySQL. Also see https://bugs.mysql.com/bug.php?id=84362
+ switch fieldType.GetType() {
+ case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong:
+ case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeDuration:
+ case mysql.TypeVarchar, mysql.TypeString:
+ default:
+ return false
+ }
+ return true
+}
+
// getPartitionIntervalFromTable checks if a partitioned table matches a generated INTERVAL partitioned scheme
// will return nil if error occurs, i.e. not an INTERVAL partitioned table
func getPartitionIntervalFromTable(ctx expression.BuildContext, tbInfo *model.TableInfo) *ast.PartitionInterval {
@@ -1030,6 +1143,66 @@ func generatePartitionDefinitionsFromInterval(ctx expression.BuildContext, partO
return nil
}
+func checkAndGetColumnsTypeAndValuesMatch(ctx expression.BuildContext, colTypes []types.FieldType, exprs []ast.ExprNode) ([]types.Datum, error) {
+ // Validate() has already checked len(colNames) = len(exprs)
+ // create table ... partition by range columns (cols)
+ // partition p0 values less than (expr)
+ // check the type of cols[i] and expr is consistent.
+ valDatums := make([]types.Datum, 0, len(colTypes))
+ for i, colExpr := range exprs {
+ if _, ok := colExpr.(*ast.MaxValueExpr); ok {
+ valDatums = append(valDatums, types.NewStringDatum(partitionMaxValue))
+ continue
+ }
+ if d, ok := colExpr.(*ast.DefaultExpr); ok {
+ if d.Name != nil {
+ return nil, dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
+ }
+ continue
+ }
+ colType := colTypes[i]
+ val, err := expression.EvalSimpleAst(ctx, colExpr)
+ if err != nil {
+ return nil, err
+ }
+ // Check val.ConvertTo(colType) doesn't work, so we need this case by case check.
+ vkind := val.Kind()
+ switch colType.GetType() {
+ case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeDuration:
+ switch vkind {
+ case types.KindString, types.KindBytes, types.KindNull:
+ default:
+ return nil, dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
+ }
+ case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong:
+ switch vkind {
+ case types.KindInt64, types.KindUint64, types.KindNull:
+ default:
+ return nil, dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
+ }
+ case mysql.TypeFloat, mysql.TypeDouble:
+ switch vkind {
+ case types.KindFloat32, types.KindFloat64, types.KindNull:
+ default:
+ return nil, dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
+ }
+ case mysql.TypeString, mysql.TypeVarString:
+ switch vkind {
+ case types.KindString, types.KindBytes, types.KindNull, types.KindBinaryLiteral:
+ default:
+ return nil, dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
+ }
+ }
+ evalCtx := ctx.GetEvalCtx()
+ newVal, err := val.ConvertTo(evalCtx.TypeCtx(), &colType)
+ if err != nil {
+ return nil, dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
+ }
+ valDatums = append(valDatums, newVal)
+ }
+ return valDatums, nil
+}
+
func astIntValueExprFromStr(s string, unsigned bool) (ast.ExprNode, error) {
if unsigned {
u, err := strconv.ParseUint(s, 10, 64)
@@ -1200,6 +1373,10 @@ func GeneratePartDefsFromInterval(ctx expression.BuildContext, tp ast.AlterTable
// Last partition!
break
}
+ // The last loop still not reach the max value, return error.
+ if i == mysql.PartitionCountLimit-1 {
+ return errors.Trace(dbterror.ErrTooManyPartitions)
+ }
}
if len(tbInfo.Partition.Definitions)+len(partDefs) > mysql.PartitionCountLimit {
return errors.Trace(dbterror.ErrTooManyPartitions)
@@ -1975,7 +2152,7 @@ func dropLabelRules(ctx context.Context, schemaName, tableName string, partNames
}
// onDropTablePartition deletes old partition meta.
-func (w *worker) onDropTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func (w *worker) onDropTablePartition(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
var partNames []string
partInfo := model.PartitionInfo{}
if err := job.DecodeArgs(&partNames, &partInfo); err != nil {
@@ -2023,7 +2200,7 @@ func (w *worker) onDropTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (
tblInfo.Partition.ClearReorgIntermediateInfo()
}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -2083,17 +2260,17 @@ func (w *worker) onDropTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (
}
job.SchemaState = model.StateDeleteOnly
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != job.SchemaState)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != job.SchemaState)
case model.StateDeleteOnly:
// This state is not a real 'DeleteOnly' state, because tidb does not maintaining the state check in partitionDefinition.
// Insert this state to confirm all servers can not see the old partitions when reorg is running,
// so that no new data will be inserted into old partitions when reorganizing.
job.SchemaState = model.StateDeleteReorganization
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != job.SchemaState)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != job.SchemaState)
case model.StateDeleteReorganization:
oldTblInfo := getTableInfoWithDroppingPartitions(tblInfo)
physicalTableIDs = getPartitionIDsFromDefinitions(tblInfo.Partition.DroppingDefinitions)
- tbl, err := getTable(d.getAutoIDRequirement(), job.SchemaID, oldTblInfo)
+ tbl, err := getTable(jobCtx.getAutoIDRequirement(), job.SchemaID, oldTblInfo)
if err != nil {
return ver, errors.Trace(err)
}
@@ -2116,14 +2293,14 @@ func (w *worker) onDropTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (
}
defer w.sessPool.Put(sctx)
rh := newReorgHandler(sess.NewSession(sctx))
- reorgInfo, err := getReorgInfoFromPartitions(d.jobContext(job.ID, job.ReorgMeta), d, rh, job, dbInfo, pt, physicalTableIDs, elements)
+ reorgInfo, err := getReorgInfoFromPartitions(jobCtx.oldDDLCtx.jobContext(job.ID, job.ReorgMeta), jobCtx, rh, job, dbInfo, pt, physicalTableIDs, elements)
if err != nil || reorgInfo.first {
// If we run reorg firstly, we should update the job snapshot version
// and then run the reorg next time.
return ver, errors.Trace(err)
}
- err = w.runReorgJob(reorgInfo, tbl.Meta(), d.lease, func() (dropIndexErr error) {
+ err = w.runReorgJob(reorgInfo, tbl.Meta(), func() (dropIndexErr error) {
defer tidbutil.Recover(metrics.LabelDDL, "onDropTablePartition",
func() {
dropIndexErr = dbterror.ErrCancelledDDLJob.GenWithStack("drop partition panic")
@@ -2149,7 +2326,7 @@ func (w *worker) onDropTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (
tblInfo.Partition.DroppingDefinitions = nil
// used by ApplyDiff in updateSchemaVersion
job.CtxVars = []any{physicalTableIDs}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -2160,7 +2337,7 @@ func (w *worker) onDropTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (
tblInfo,
&model.PartitionInfo{Definitions: droppedDefs},
)
- asyncNotifyEvent(d, dropPartitionEvent)
+ asyncNotifyEvent(jobCtx, dropPartitionEvent)
// A background job will be created to delete old partition data.
job.Args = []any{physicalTableIDs}
default:
@@ -2187,7 +2364,7 @@ func removeTiFlashAvailablePartitionIDs(tblInfo *model.TableInfo, pids []int64)
}
// onTruncateTablePartition truncates old partition meta.
-func (w *worker) onTruncateTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (int64, error) {
+func (w *worker) onTruncateTablePartition(jobCtx *jobContext, t *meta.Meta, job *model.Job) (int64, error) {
var ver int64
var oldIDs, newIDs []int64
if err := job.DecodeArgs(&oldIDs, &newIDs); err != nil {
@@ -2237,10 +2414,10 @@ func (w *worker) onTruncateTablePartition(d *ddlCtx, t *meta.Meta, job *model.Jo
return ver, err
}
- preSplitAndScatter(w.sess.Context, d.store, tblInfo, newPartitions)
+ preSplitAndScatter(w.sess.Context, jobCtx.store, tblInfo, newPartitions)
job.CtxVars = []any{oldIDs, newIDs}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -2253,7 +2430,7 @@ func (w *worker) onTruncateTablePartition(d *ddlCtx, t *meta.Meta, job *model.Jo
&model.PartitionInfo{Definitions: newPartitions},
&model.PartitionInfo{Definitions: oldPartitions},
)
- asyncNotifyEvent(d, truncatePartitionEvent)
+ asyncNotifyEvent(jobCtx, truncatePartitionEvent)
// A background job will be created to delete old partition data.
job.Args = []any{oldIDs}
@@ -2279,19 +2456,19 @@ func (w *worker) onTruncateTablePartition(d *ddlCtx, t *meta.Meta, job *model.Jo
pi.NewPartitionIDs = newIDs[:]
job.SchemaState = model.StateDeleteOnly
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
case model.StateDeleteOnly:
// This state is not a real 'DeleteOnly' state, because tidb does not maintaining the state check in partitionDefinition.
// Insert this state to confirm all servers can not see the old partitions when reorg is running,
// so that no new data will be inserted into old partitions when reorganizing.
job.SchemaState = model.StateDeleteReorganization
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
case model.StateDeleteReorganization:
// Step2: clear global index rows.
physicalTableIDs := oldIDs
oldTblInfo := getTableInfoWithOriginalPartitions(tblInfo, oldIDs, newIDs)
- tbl, err := getTable(d.getAutoIDRequirement(), job.SchemaID, oldTblInfo)
+ tbl, err := getTable(jobCtx.getAutoIDRequirement(), job.SchemaID, oldTblInfo)
if err != nil {
return ver, errors.Trace(err)
}
@@ -2314,14 +2491,14 @@ func (w *worker) onTruncateTablePartition(d *ddlCtx, t *meta.Meta, job *model.Jo
}
defer w.sessPool.Put(sctx)
rh := newReorgHandler(sess.NewSession(sctx))
- reorgInfo, err := getReorgInfoFromPartitions(d.jobContext(job.ID, job.ReorgMeta), d, rh, job, dbInfo, pt, physicalTableIDs, elements)
+ reorgInfo, err := getReorgInfoFromPartitions(jobCtx.oldDDLCtx.jobContext(job.ID, job.ReorgMeta), jobCtx, rh, job, dbInfo, pt, physicalTableIDs, elements)
if err != nil || reorgInfo.first {
// If we run reorg firstly, we should update the job snapshot version
// and then run the reorg next time.
return ver, errors.Trace(err)
}
- err = w.runReorgJob(reorgInfo, tbl.Meta(), d.lease, func() (dropIndexErr error) {
+ err = w.runReorgJob(reorgInfo, tbl.Meta(), func() (dropIndexErr error) {
defer tidbutil.Recover(metrics.LabelDDL, "onDropTablePartition",
func() {
dropIndexErr = dbterror.ErrCancelledDDLJob.GenWithStack("drop partition panic")
@@ -2376,11 +2553,11 @@ func (w *worker) onTruncateTablePartition(d *ddlCtx, t *meta.Meta, job *model.Jo
tblInfo.Partition.DroppingDefinitions = nil
tblInfo.Partition.NewPartitionIDs = nil
- preSplitAndScatter(w.sess.Context, d.store, tblInfo, newPartitions)
+ preSplitAndScatter(w.sess.Context, jobCtx.store, tblInfo, newPartitions)
// used by ApplyDiff in updateSchemaVersion
job.CtxVars = []any{oldIDs, newIDs}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -2392,7 +2569,7 @@ func (w *worker) onTruncateTablePartition(d *ddlCtx, t *meta.Meta, job *model.Jo
&model.PartitionInfo{Definitions: newPartitions},
&model.PartitionInfo{Definitions: oldPartitions},
)
- asyncNotifyEvent(d, truncatePartitionEvent)
+ asyncNotifyEvent(jobCtx, truncatePartitionEvent)
// A background job will be created to delete old partition data.
job.Args = []any{oldIDs}
default:
@@ -2484,7 +2661,7 @@ func updateTruncatePartitionLabelRules(job *model.Job, t *meta.Meta, oldPartitio
}
// onExchangeTablePartition exchange partition data
-func (w *worker) onExchangeTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func (w *worker) onExchangeTablePartition(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
var (
// defID only for updateSchemaVersion
defID int64
@@ -2517,7 +2694,7 @@ func (w *worker) onExchangeTablePartition(d *ddlCtx, t *meta.Meta, job *model.Jo
}
if job.IsRollingback() {
- return rollbackExchangeTablePartition(d, t, job, nt)
+ return rollbackExchangeTablePartition(jobCtx, t, job, nt)
}
pt, err := getTableInfo(t, ptID, ptSchemaID)
if err != nil {
@@ -2584,7 +2761,7 @@ func (w *worker) onExchangeTablePartition(d *ddlCtx, t *meta.Meta, job *model.Jo
// into the table using the schema version
// before the exchange is made.
job.SchemaState = model.StateWriteOnly
- return updateVersionAndTableInfoWithCheck(d, t, job, nt, true, ptInfo...)
+ return updateVersionAndTableInfoWithCheck(jobCtx, t, job, nt, true, ptInfo...)
}
// From now on, nt (the non-partitioned table) has
// ExchangePartitionInfo set, meaning it is restricted
@@ -2592,9 +2769,7 @@ func (w *worker) onExchangeTablePartition(d *ddlCtx, t *meta.Meta, job *model.Jo
// partition to be exchange with.
// So we need to rollback that change, instead of just cancelling.
- if d.lease > 0 {
- delayForAsyncCommit()
- }
+ delayForAsyncCommit()
if defID != partDef.ID {
// Should never happen, should have been updated above, in previous state!
@@ -2609,11 +2784,11 @@ func (w *worker) onExchangeTablePartition(d *ddlCtx, t *meta.Meta, job *model.Jo
}
if withValidation {
- ntbl, err := getTable(d.getAutoIDRequirement(), job.SchemaID, nt)
+ ntbl, err := getTable(jobCtx.getAutoIDRequirement(), job.SchemaID, nt)
if err != nil {
return ver, errors.Trace(err)
}
- ptbl, err := getTable(d.getAutoIDRequirement(), ptSchemaID, pt)
+ ptbl, err := getTable(jobCtx.getAutoIDRequirement(), ptSchemaID, pt)
if err != nil {
return ver, errors.Trace(err)
}
@@ -2646,7 +2821,7 @@ func (w *worker) onExchangeTablePartition(d *ddlCtx, t *meta.Meta, job *model.Jo
// Recreate non-partition table meta info,
// by first delete it with the old table id
- err = t.DropTableOrView(job.SchemaID, job.SchemaName, nt.ID, nt.Name.L)
+ err = t.DropTableOrView(job.SchemaID, nt.ID)
if err != nil {
return ver, errors.Trace(err)
}
@@ -2663,7 +2838,7 @@ func (w *worker) onExchangeTablePartition(d *ddlCtx, t *meta.Meta, job *model.Jo
return ver, errors.Trace(err)
}
- err = t.CreateTableOrView(job.SchemaID, job.SchemaName, nt)
+ err = t.CreateTableOrView(job.SchemaID, nt)
if err != nil {
return ver, errors.Trace(err)
}
@@ -2755,7 +2930,7 @@ func (w *worker) onExchangeTablePartition(d *ddlCtx, t *meta.Meta, job *model.Jo
job.SchemaState = model.StatePublic
nt.ExchangePartitionInfo = nil
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, nt, true)
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, nt, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -2767,10 +2942,22 @@ func (w *worker) onExchangeTablePartition(d *ddlCtx, t *meta.Meta, job *model.Jo
&model.PartitionInfo{Definitions: []model.PartitionDefinition{originalPartitionDef}},
originalNt,
)
- asyncNotifyEvent(d, exchangePartitionEvent)
+ asyncNotifyEvent(jobCtx, exchangePartitionEvent)
return ver, nil
}
+func getNewGlobal(partInfo *model.PartitionInfo, idx *model.IndexInfo) bool {
+ if len(partInfo.DDLUpdateIndexes) == 0 {
+ return idx.Global
+ }
+ for _, newIdx := range partInfo.DDLUpdateIndexes {
+ if strings.EqualFold(idx.Name.L, newIdx.IndexName) {
+ return newIdx.Global
+ }
+ }
+ return idx.Global
+}
+
func getReorgPartitionInfo(t *meta.Meta, job *model.Job) (*model.TableInfo, []string, *model.PartitionInfo, []model.PartitionDefinition, []model.PartitionDefinition, error) {
schemaID := job.SchemaID
tblInfo, err := GetTableInfoAndCancelFaultJob(t, job, schemaID)
@@ -2874,10 +3061,10 @@ func getReorgPartitionInfo(t *meta.Meta, job *model.Job) (*model.TableInfo, []st
//
// Everything now looks as it should, no memory of old partitions/indexes,
// and no more double writing, since the previous state is only reading the new partitions/indexes.
-func (w *worker) onReorganizePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func (w *worker) onReorganizePartition(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
// Handle the rolling back job
if job.IsRollingback() {
- ver, err := w.onDropTablePartition(d, t, job)
+ ver, err := w.onDropTablePartition(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -2941,28 +3128,56 @@ func (w *worker) onReorganizePartition(d *ddlCtx, t *meta.Meta, job *model.Job)
// All global indexes must be recreated, we cannot update them in-place, since we must have
// both old and new set of partition ids in the unique index at the same time!
for _, index := range tblInfo.Indices {
+ newGlobal := getNewGlobal(partInfo, index)
+ if job.Type == model.ActionRemovePartitioning {
+ // When removing partitioning, set all indexes to 'local' since it will become a non-partitioned table!
+ newGlobal = false
+ }
if !index.Unique {
// for now, only unique index can be global, non-unique indexes are 'local'
+ // TODO: For the future loosen this restriction and allow non-unique global indexes
+ if newGlobal {
+ job.State = model.JobStateCancelled
+ return ver, dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs(fmt.Sprintf("PARTITION BY, index '%v' is not unique, but has Global Index set", index.Name.O))
+ }
continue
}
inAllPartitionColumns, err := checkPartitionKeysConstraint(partInfo, index.Columns, tblInfo)
if err != nil {
return ver, errors.Trace(err)
}
- if index.Global || !inAllPartitionColumns {
+ if !inAllPartitionColumns {
+ // Currently only support Explicit Global indexes.
+ if !newGlobal {
+ job.State = model.JobStateCancelled
+ return ver, dbterror.ErrGlobalIndexNotExplicitlySet.GenWithStackByArgs(index.Name.O)
+ }
// Duplicate the unique indexes with new index ids.
// If previously was Global or will be Global:
// it must be recreated with new index ID
+ // TODO: Could we allow that session in StateWriteReorganization, when StateDeleteReorganization
+ // has started, may not find changes through the global index that sessions in StateDeleteReorganization made?
+ // If so, then we could avoid copying the full Global Index if it has not changed from LOCAL!
+ // It might be possible to use the new, not yet public partitions to access those rows?!
+ // Just that it would not work with explicit partition select SELECT FROM t PARTITION (p,...)
newIndex := index.Clone()
newIndex.State = model.StateDeleteOnly
newIndex.ID = AllocateIndexID(tblInfo)
- if inAllPartitionColumns {
+ newIndex.Global = true
+ tblInfo.Indices = append(tblInfo.Indices, newIndex)
+ } else {
+ if newGlobal {
+ // TODO: For the future loosen this restriction and allow global indexes for unique keys also including all partitioning columns
+ return ver, dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs(fmt.Sprintf("PARTITION BY, index '%v' is unique and contains all partitioning columns, but has Global Index set", index.Name.O))
+ }
+ if index.Global {
+ // Index was previously Global, now it needs to be duplicated and become a local index.
+ newIndex := index.Clone()
+ newIndex.State = model.StateDeleteOnly
+ newIndex.ID = AllocateIndexID(tblInfo)
newIndex.Global = false
- } else {
- // If not including all partitioning columns, make it Global
- newIndex.Global = true
+ tblInfo.Indices = append(tblInfo.Indices, newIndex)
}
- tblInfo.Indices = append(tblInfo.Indices, newIndex)
}
}
// From now on we cannot just cancel the DDL, we must roll back if changesMade!
@@ -2984,7 +3199,7 @@ func (w *worker) onReorganizePartition(d *ddlCtx, t *meta.Meta, job *model.Job)
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
}
- return convertAddTablePartitionJob2RollbackJob(d, t, job, err, tblInfo)
+ return convertAddTablePartitionJob2RollbackJob(jobCtx, t, job, err, tblInfo)
}
if len(bundles) > 0 {
@@ -2993,7 +3208,7 @@ func (w *worker) onReorganizePartition(d *ddlCtx, t *meta.Meta, job *model.Job)
job.State = model.JobStateCancelled
return ver, errors.Wrapf(err, "failed to notify PD the placement rules")
}
- return convertAddTablePartitionJob2RollbackJob(d, t, job, err, tblInfo)
+ return convertAddTablePartitionJob2RollbackJob(jobCtx, t, job, err, tblInfo)
}
changesMade = true
}
@@ -3009,12 +3224,12 @@ func (w *worker) onReorganizePartition(d *ddlCtx, t *meta.Meta, job *model.Job)
job.State = model.JobStateCancelled
return ver, err
}
- return convertAddTablePartitionJob2RollbackJob(d, t, job, err, tblInfo)
+ return convertAddTablePartitionJob2RollbackJob(jobCtx, t, job, err, tblInfo)
}
// Doing the preSplitAndScatter here, since all checks are completed,
// and we will soon start writing to the new partitions.
- if s, ok := d.store.(kv.SplittableStore); ok && s != nil {
+ if s, ok := jobCtx.store.(kv.SplittableStore); ok && s != nil {
// partInfo only contains the AddingPartitions
splitPartitionTableRegion(w.sess.Context, s, tblInfo, partInfo.Definitions, true)
}
@@ -3023,7 +3238,7 @@ func (w *worker) onReorganizePartition(d *ddlCtx, t *meta.Meta, job *model.Job)
metrics.GetBackfillProgressByLabel(metrics.LblReorgPartition, job.SchemaName, tblInfo.Name.String()).Set(0.1 / float64(math.MaxUint64))
job.SchemaState = model.StateDeleteOnly
tblInfo.Partition.DDLState = model.StateDeleteOnly
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -3053,11 +3268,11 @@ func (w *worker) onReorganizePartition(d *ddlCtx, t *meta.Meta, job *model.Job)
// For available state, the new added partition should wait its replica to
// be finished, otherwise the query to this partition will be blocked.
count := tblInfo.TiFlashReplica.Count
- needRetry, err := checkPartitionReplica(count, addingDefinitions, d)
+ needRetry, err := checkPartitionReplica(count, addingDefinitions, jobCtx)
if err != nil {
// need to rollback, since we tried to register the new
// partitions before!
- return convertAddTablePartitionJob2RollbackJob(d, t, job, err, tblInfo)
+ return convertAddTablePartitionJob2RollbackJob(jobCtx, t, job, err, tblInfo)
}
if needRetry {
// The new added partition hasn't been replicated.
@@ -3081,7 +3296,7 @@ func (w *worker) onReorganizePartition(d *ddlCtx, t *meta.Meta, job *model.Job)
}
tblInfo.Partition.DDLState = model.StateWriteOnly
metrics.GetBackfillProgressByLabel(metrics.LblReorgPartition, job.SchemaName, tblInfo.Name.String()).Set(0.2 / float64(math.MaxUint64))
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
job.SchemaState = model.StateWriteOnly
case model.StateWriteOnly:
// Insert this state to confirm all servers can see the new partitions when reorg is running,
@@ -3094,16 +3309,16 @@ func (w *worker) onReorganizePartition(d *ddlCtx, t *meta.Meta, job *model.Job)
}
tblInfo.Partition.DDLState = model.StateWriteReorganization
metrics.GetBackfillProgressByLabel(metrics.LblReorgPartition, job.SchemaName, tblInfo.Name.String()).Set(0.3 / float64(math.MaxUint64))
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
job.SchemaState = model.StateWriteReorganization
case model.StateWriteReorganization:
physicalTableIDs := getPartitionIDsFromDefinitions(tblInfo.Partition.DroppingDefinitions)
- tbl, err2 := getTable(d.getAutoIDRequirement(), job.SchemaID, tblInfo)
+ tbl, err2 := getTable(jobCtx.getAutoIDRequirement(), job.SchemaID, tblInfo)
if err2 != nil {
return ver, errors.Trace(err2)
}
var done bool
- done, ver, err = doPartitionReorgWork(w, d, t, job, tbl, physicalTableIDs)
+ done, ver, err = doPartitionReorgWork(w, jobCtx, t, job, tbl, physicalTableIDs)
if !done {
return ver, err
@@ -3160,7 +3375,7 @@ func (w *worker) onReorganizePartition(d *ddlCtx, t *meta.Meta, job *model.Job)
// since they are a part of the normal Definitions that other nodes with
// the current schema version. So we need to double write for one more schema version
tblInfo.Partition.DDLState = model.StateDeleteReorganization
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
job.SchemaState = model.StateDeleteReorganization
case model.StateDeleteReorganization:
@@ -3213,7 +3428,7 @@ func (w *worker) onReorganizePartition(d *ddlCtx, t *meta.Meta, job *model.Job)
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
}
- err = t.DropTableOrView(job.SchemaID, job.SchemaName, tblInfo.ID, tblInfo.Name.L)
+ err = t.DropTableOrView(job.SchemaID, tblInfo.ID)
if err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
@@ -3236,14 +3451,14 @@ func (w *worker) onReorganizePartition(d *ddlCtx, t *meta.Meta, job *model.Job)
return ver, errors.Trace(err)
}
// TODO: Add failpoint here?
- err = t.CreateTableOrView(job.SchemaID, job.SchemaName, tblInfo)
+ err = t.CreateTableOrView(job.SchemaID, tblInfo)
if err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
}
}
job.CtxVars = []any{physicalTableIDs, newIDs}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
failpoint.Inject("reorgPartWriteReorgSchemaVersionUpdateFail", func(val failpoint.Value) {
if val.(bool) {
err = errors.New("Injected error by reorgPartWriteReorgSchemaVersionUpdateFail")
@@ -3265,7 +3480,7 @@ func (w *worker) onReorganizePartition(d *ddlCtx, t *meta.Meta, job *model.Job)
if err != nil {
return ver, errors.Trace(err)
}
- asyncNotifyEvent(d, event)
+ asyncNotifyEvent(jobCtx, event)
// A background job will be created to delete old partition data.
job.Args = []any{physicalTableIDs}
@@ -3315,7 +3530,7 @@ func newStatsDDLEventForJob(
return event, nil
}
-func doPartitionReorgWork(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job, tbl table.Table, physTblIDs []int64) (done bool, ver int64, err error) {
+func doPartitionReorgWork(w *worker, jobCtx *jobContext, t *meta.Meta, job *model.Job, tbl table.Table, physTblIDs []int64) (done bool, ver int64, err error) {
job.ReorgMeta.ReorgTp = model.ReorgTypeTxn
sctx, err1 := w.sessPool.Get()
if err1 != nil {
@@ -3340,8 +3555,8 @@ func doPartitionReorgWork(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job, tb
if err != nil {
return false, ver, errors.Trace(err)
}
- reorgInfo, err := getReorgInfoFromPartitions(d.jobContext(job.ID, job.ReorgMeta), d, rh, job, dbInfo, partTbl, physTblIDs, elements)
- err = w.runReorgJob(reorgInfo, tbl.Meta(), d.lease, func() (reorgErr error) {
+ reorgInfo, err := getReorgInfoFromPartitions(jobCtx.oldDDLCtx.jobContext(job.ID, job.ReorgMeta), jobCtx, rh, job, dbInfo, partTbl, physTblIDs, elements)
+ err = w.runReorgJob(reorgInfo, tbl.Meta(), func() (reorgErr error) {
defer tidbutil.Recover(metrics.LabelDDL, "doPartitionReorgWork",
func() {
reorgErr = dbterror.ErrCancelledDDLJob.GenWithStack("reorganize partition for table `%v` panic", tbl.Meta().Name)
@@ -3366,7 +3581,7 @@ func doPartitionReorgWork(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job, tb
}
logutil.DDLLogger().Warn("reorg partition job failed, convert job to rollback", zap.Stringer("job", job), zap.Error(err))
// TODO: rollback new global indexes! TODO: How to handle new index ids?
- ver, err = convertAddTablePartitionJob2RollbackJob(d, t, job, err, tbl.Meta())
+ ver, err = convertAddTablePartitionJob2RollbackJob(jobCtx, t, job, err, tbl.Meta())
return false, ver, errors.Trace(err)
}
return true, ver, err
@@ -3383,7 +3598,7 @@ type reorgPartitionWorker struct {
reorgedTbl table.PartitionedTable
}
-func newReorgPartitionWorker(i int, t table.PhysicalTable, decodeColMap map[int64]decoder.Column, reorgInfo *reorgInfo, jc *JobContext) (*reorgPartitionWorker, error) {
+func newReorgPartitionWorker(i int, t table.PhysicalTable, decodeColMap map[int64]decoder.Column, reorgInfo *reorgInfo, jc *ReorgContext) (*reorgPartitionWorker, error) {
bCtx, err := newBackfillCtx(i, reorgInfo, reorgInfo.SchemaName, t, jc, "reorg_partition_rate", false)
if err != nil {
return nil, err
@@ -3627,11 +3842,11 @@ func (w *worker) reorgPartitionDataAndIndex(t table.Table, reorgInfo *reorgInfo)
physTbl = tbl
}
// Get the original start handle and end handle.
- currentVer, err := getValidCurrentVersion(reorgInfo.d.store)
+ currentVer, err := getValidCurrentVersion(reorgInfo.jobCtx.store)
if err != nil {
return errors.Trace(err)
}
- startHandle, endHandle, err := getTableRange(reorgInfo.NewJobContext(), reorgInfo.d, physTbl, currentVer.Ver, reorgInfo.Job.Priority)
+ startHandle, endHandle, err := getTableRange(reorgInfo.NewJobContext(), reorgInfo.jobCtx.store, physTbl, currentVer.Ver, reorgInfo.Job.Priority)
if err != nil {
return errors.Trace(err)
}
@@ -3695,11 +3910,11 @@ func (w *worker) reorgPartitionDataAndIndex(t table.Table, reorgInfo *reorgInfo)
physTbl = tbl
}
// Get the original start handle and end handle.
- currentVer, err := getValidCurrentVersion(reorgInfo.d.store)
+ currentVer, err := getValidCurrentVersion(reorgInfo.jobCtx.store)
if err != nil {
return errors.Trace(err)
}
- startHandle, endHandle, err := getTableRange(reorgInfo.NewJobContext(), reorgInfo.d, physTbl, currentVer.Ver, reorgInfo.Job.Priority)
+ startHandle, endHandle, err := getTableRange(reorgInfo.NewJobContext(), reorgInfo.jobCtx.store, physTbl, currentVer.Ver, reorgInfo.Job.Priority)
if err != nil {
return errors.Trace(err)
}
@@ -4657,3 +4872,156 @@ func generatePartValuesWithTp(partVal types.Datum, tp types.FieldType) (string,
return "", dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
}
+
+func checkPartitionDefinitionConstraints(ctx sessionctx.Context, tbInfo *model.TableInfo) error {
+ var err error
+ if err = checkPartitionNameUnique(tbInfo.Partition); err != nil {
+ return errors.Trace(err)
+ }
+ if err = checkAddPartitionTooManyPartitions(uint64(len(tbInfo.Partition.Definitions))); err != nil {
+ return err
+ }
+ if err = checkAddPartitionOnTemporaryMode(tbInfo); err != nil {
+ return err
+ }
+ if err = checkPartitionColumnsUnique(tbInfo); err != nil {
+ return err
+ }
+
+ switch tbInfo.Partition.Type {
+ case model.PartitionTypeRange:
+ err = checkPartitionByRange(ctx, tbInfo)
+ case model.PartitionTypeHash, model.PartitionTypeKey:
+ err = checkPartitionByHash(ctx, tbInfo)
+ case model.PartitionTypeList:
+ err = checkPartitionByList(ctx, tbInfo)
+ }
+ return errors.Trace(err)
+}
+
+func checkPartitionByHash(ctx sessionctx.Context, tbInfo *model.TableInfo) error {
+ return checkNoHashPartitions(ctx, tbInfo.Partition.Num)
+}
+
+// checkPartitionByRange checks validity of a "BY RANGE" partition.
+func checkPartitionByRange(ctx sessionctx.Context, tbInfo *model.TableInfo) error {
+ failpoint.Inject("CheckPartitionByRangeErr", func() {
+ ctx.GetSessionVars().SQLKiller.SendKillSignal(sqlkiller.QueryMemoryExceeded)
+ panic(ctx.GetSessionVars().SQLKiller.HandleSignal())
+ })
+ pi := tbInfo.Partition
+
+ if len(pi.Columns) == 0 {
+ return checkRangePartitionValue(ctx, tbInfo)
+ }
+
+ return checkRangeColumnsPartitionValue(ctx, tbInfo)
+}
+
+func checkRangeColumnsPartitionValue(ctx sessionctx.Context, tbInfo *model.TableInfo) error {
+ // Range columns partition key supports multiple data types with integer、datetime、string.
+ pi := tbInfo.Partition
+ defs := pi.Definitions
+ if len(defs) < 1 {
+ return ast.ErrPartitionsMustBeDefined.GenWithStackByArgs("RANGE")
+ }
+
+ curr := &defs[0]
+ if len(curr.LessThan) != len(pi.Columns) {
+ return errors.Trace(ast.ErrPartitionColumnList)
+ }
+ var prev *model.PartitionDefinition
+ for i := 1; i < len(defs); i++ {
+ prev, curr = curr, &defs[i]
+ succ, err := checkTwoRangeColumns(ctx, curr, prev, pi, tbInfo)
+ if err != nil {
+ return err
+ }
+ if !succ {
+ return errors.Trace(dbterror.ErrRangeNotIncreasing)
+ }
+ }
+ return nil
+}
+
+func checkTwoRangeColumns(ctx sessionctx.Context, curr, prev *model.PartitionDefinition, pi *model.PartitionInfo, tbInfo *model.TableInfo) (bool, error) {
+ if len(curr.LessThan) != len(pi.Columns) {
+ return false, errors.Trace(ast.ErrPartitionColumnList)
+ }
+ for i := 0; i < len(pi.Columns); i++ {
+ // Special handling for MAXVALUE.
+ if strings.EqualFold(curr.LessThan[i], partitionMaxValue) && !strings.EqualFold(prev.LessThan[i], partitionMaxValue) {
+ // If current is maxvalue, it certainly >= previous.
+ return true, nil
+ }
+ if strings.EqualFold(prev.LessThan[i], partitionMaxValue) {
+ // Current is not maxvalue, and previous is maxvalue.
+ return false, nil
+ }
+
+ // The tuples of column values used to define the partitions are strictly increasing:
+ // PARTITION p0 VALUES LESS THAN (5,10,'ggg')
+ // PARTITION p1 VALUES LESS THAN (10,20,'mmm')
+ // PARTITION p2 VALUES LESS THAN (15,30,'sss')
+ colInfo := findColumnByName(pi.Columns[i].L, tbInfo)
+ cmp, err := parseAndEvalBoolExpr(ctx.GetExprCtx(), curr.LessThan[i], prev.LessThan[i], colInfo, tbInfo)
+ if err != nil {
+ return false, err
+ }
+
+ if cmp > 0 {
+ return true, nil
+ }
+
+ if cmp < 0 {
+ return false, nil
+ }
+ }
+ return false, nil
+}
+
+// equal, return 0
+// greater, return 1
+// less, return -1
+func parseAndEvalBoolExpr(ctx expression.BuildContext, l, r string, colInfo *model.ColumnInfo, tbInfo *model.TableInfo) (int64, error) {
+ lexpr, err := expression.ParseSimpleExpr(ctx, l, expression.WithTableInfo("", tbInfo), expression.WithCastExprTo(&colInfo.FieldType))
+ if err != nil {
+ return 0, err
+ }
+ rexpr, err := expression.ParseSimpleExpr(ctx, r, expression.WithTableInfo("", tbInfo), expression.WithCastExprTo(&colInfo.FieldType))
+ if err != nil {
+ return 0, err
+ }
+
+ e, err := expression.NewFunctionBase(ctx, ast.EQ, field_types.NewFieldType(mysql.TypeLonglong), lexpr, rexpr)
+ if err != nil {
+ return 0, err
+ }
+ e.SetCharsetAndCollation(colInfo.GetCharset(), colInfo.GetCollate())
+ res, _, err1 := e.EvalInt(ctx.GetEvalCtx(), chunk.Row{})
+ if err1 != nil {
+ return 0, err1
+ }
+ if res == 1 {
+ return 0, nil
+ }
+
+ e, err = expression.NewFunctionBase(ctx, ast.GT, field_types.NewFieldType(mysql.TypeLonglong), lexpr, rexpr)
+ if err != nil {
+ return 0, err
+ }
+ e.SetCharsetAndCollation(colInfo.GetCharset(), colInfo.GetCollate())
+ res, _, err1 = e.EvalInt(ctx.GetEvalCtx(), chunk.Row{})
+ if err1 != nil {
+ return 0, err1
+ }
+ if res > 0 {
+ return 1, nil
+ }
+ return -1, nil
+}
+
+// checkPartitionByList checks validity of a "BY LIST" partition.
+func checkPartitionByList(ctx sessionctx.Context, tbInfo *model.TableInfo) error {
+ return checkListPartitionValue(ctx.GetExprCtx(), tbInfo)
+}
diff --git a/pkg/ddl/partition_test.go b/pkg/ddl/partition_test.go
index c221a7be568f3..29a0b7ec8b4ac 100644
--- a/pkg/ddl/partition_test.go
+++ b/pkg/ddl/partition_test.go
@@ -20,12 +20,12 @@ import (
"time"
"github.com/pingcap/tidb/pkg/ddl"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/types"
"github.com/stretchr/testify/require"
)
@@ -177,14 +177,12 @@ func TestReorganizePartitionRollback(t *testing.T) {
defer close(wait)
ddlDone := make(chan error)
defer close(ddlDone)
- hook := &callback.TestDDLCallback{Do: do}
- hook.OnJobRunAfterExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunAfter", func(job *model.Job) {
if job.Type == model.ActionReorganizePartition && job.SchemaState == model.StateWriteReorganization {
<-wait
<-wait
}
- }
- do.DDL().SetHook(hook)
+ })
go func() {
tk2 := testkit.NewTestKit(t, store)
diff --git a/pkg/ddl/placement_policy.go b/pkg/ddl/placement_policy.go
index 2277ab3199fe2..53983b06c7f59 100644
--- a/pkg/ddl/placement_policy.go
+++ b/pkg/ddl/placement_policy.go
@@ -24,11 +24,15 @@ import (
"github.com/pingcap/tidb/pkg/domain/infosync"
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/meta"
+ "github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/model"
+ "github.com/pingcap/tidb/pkg/sessionctx"
+ "github.com/pingcap/tidb/pkg/sessionctx/variable"
+ "github.com/pingcap/tidb/pkg/sessiontxn"
"github.com/pingcap/tidb/pkg/util/dbterror"
)
-func onCreatePlacementPolicy(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onCreatePlacementPolicy(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
policyInfo := &model.PolicyInfo{}
var orReplace bool
if err := job.DecodeArgs(policyInfo, &orReplace); err != nil {
@@ -42,7 +46,7 @@ func onCreatePlacementPolicy(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64
return ver, errors.Trace(err)
}
- existPolicy, err := getPlacementPolicyByName(d, t, policyInfo.Name)
+ existPolicy, err := getPlacementPolicyByName(jobCtx.infoCache, t, policyInfo.Name)
if err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
@@ -62,7 +66,7 @@ func onCreatePlacementPolicy(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64
}
job.SchemaID = replacePolicy.ID
- ver, err = updateSchemaVersion(d, t, job)
+ ver, err = updateSchemaVersion(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -81,7 +85,7 @@ func onCreatePlacementPolicy(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64
}
job.SchemaID = policyInfo.ID
- ver, err = updateSchemaVersion(d, t, job)
+ ver, err = updateSchemaVersion(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -112,13 +116,13 @@ func getPolicyInfo(t *meta.Meta, policyID int64) (*model.PolicyInfo, error) {
return policy, nil
}
-func getPlacementPolicyByName(d *ddlCtx, t *meta.Meta, policyName model.CIStr) (*model.PolicyInfo, error) {
+func getPlacementPolicyByName(infoCache *infoschema.InfoCache, t *meta.Meta, policyName model.CIStr) (*model.PolicyInfo, error) {
currVer, err := t.GetSchemaVersion()
if err != nil {
return nil, err
}
- is := d.infoCache.GetLatest()
+ is := infoCache.GetLatest()
if is != nil && is.SchemaMetaVersion() == currVer {
// Use cached policy.
policy, ok := is.PolicyByName(policyName)
@@ -177,13 +181,13 @@ func checkAllTablePlacementPoliciesExistAndCancelNonExistJob(t *meta.Meta, job *
return nil
}
-func onDropPlacementPolicy(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onDropPlacementPolicy(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
policyInfo, err := checkPlacementPolicyExistAndCancelNonExistJob(t, job, job.SchemaID)
if err != nil {
return ver, errors.Trace(err)
}
- err = checkPlacementPolicyNotInUse(d, t, policyInfo)
+ err = checkPlacementPolicyNotInUse(jobCtx.infoCache, t, policyInfo)
if err != nil {
if dbterror.ErrPlacementPolicyInUse.Equal(err) {
job.State = model.JobStateCancelled
@@ -199,7 +203,7 @@ func onDropPlacementPolicy(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64,
if err != nil {
return ver, errors.Trace(err)
}
- ver, err = updateSchemaVersion(d, t, job)
+ ver, err = updateSchemaVersion(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -212,7 +216,7 @@ func onDropPlacementPolicy(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64,
if err != nil {
return ver, errors.Trace(err)
}
- ver, err = updateSchemaVersion(d, t, job)
+ ver, err = updateSchemaVersion(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -223,7 +227,7 @@ func onDropPlacementPolicy(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64,
if err = t.DropPolicy(policyInfo.ID); err != nil {
return ver, errors.Trace(err)
}
- ver, err = updateSchemaVersion(d, t, job)
+ ver, err = updateSchemaVersion(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -235,7 +239,7 @@ func onDropPlacementPolicy(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64,
return ver, errors.Trace(err)
}
-func onAlterPlacementPolicy(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onAlterPlacementPolicy(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
alterPolicy := &model.PolicyInfo{}
if err := job.DecodeArgs(alterPolicy); err != nil {
job.State = model.JobStateCancelled
@@ -260,7 +264,7 @@ func onAlterPlacementPolicy(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64,
return ver, errors.Trace(err)
}
- ver, err = updateSchemaVersion(d, t, job)
+ ver, err = updateSchemaVersion(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -340,12 +344,12 @@ func updateExistPlacementPolicy(t *meta.Meta, policy *model.PolicyInfo) error {
return nil
}
-func checkPlacementPolicyNotInUse(d *ddlCtx, t *meta.Meta, policy *model.PolicyInfo) error {
+func checkPlacementPolicyNotInUse(infoCache *infoschema.InfoCache, t *meta.Meta, policy *model.PolicyInfo) error {
currVer, err := t.GetSchemaVersion()
if err != nil {
return err
}
- is := d.infoCache.GetLatest()
+ is := infoCache.GetLatest()
if is != nil && is.SchemaMetaVersion() == currVer {
err = CheckPlacementPolicyNotInUseFromInfoSchema(is, policy)
} else {
@@ -489,3 +493,174 @@ func GetRangePlacementPolicyName(ctx context.Context, rangeBundleID string) (str
}
return "", nil
}
+
+func buildPolicyInfo(name model.CIStr, options []*ast.PlacementOption) (*model.PolicyInfo, error) {
+ policyInfo := &model.PolicyInfo{PlacementSettings: &model.PlacementSettings{}}
+ policyInfo.Name = name
+ for _, opt := range options {
+ err := SetDirectPlacementOpt(policyInfo.PlacementSettings, opt.Tp, opt.StrValue, opt.UintValue)
+ if err != nil {
+ return nil, err
+ }
+ }
+ return policyInfo, nil
+}
+
+func removeTablePlacement(tbInfo *model.TableInfo) bool {
+ hasPlacementSettings := false
+ if tbInfo.PlacementPolicyRef != nil {
+ tbInfo.PlacementPolicyRef = nil
+ hasPlacementSettings = true
+ }
+
+ if removePartitionPlacement(tbInfo.Partition) {
+ hasPlacementSettings = true
+ }
+
+ return hasPlacementSettings
+}
+
+func removePartitionPlacement(partInfo *model.PartitionInfo) bool {
+ if partInfo == nil {
+ return false
+ }
+
+ hasPlacementSettings := false
+ for i := range partInfo.Definitions {
+ def := &partInfo.Definitions[i]
+ if def.PlacementPolicyRef != nil {
+ def.PlacementPolicyRef = nil
+ hasPlacementSettings = true
+ }
+ }
+ return hasPlacementSettings
+}
+
+func handleDatabasePlacement(ctx sessionctx.Context, dbInfo *model.DBInfo) error {
+ if dbInfo.PlacementPolicyRef == nil {
+ return nil
+ }
+
+ sessVars := ctx.GetSessionVars()
+ if sessVars.PlacementMode == variable.PlacementModeIgnore {
+ dbInfo.PlacementPolicyRef = nil
+ sessVars.StmtCtx.AppendNote(
+ errors.NewNoStackErrorf("Placement is ignored when TIDB_PLACEMENT_MODE is '%s'", variable.PlacementModeIgnore),
+ )
+ return nil
+ }
+
+ var err error
+ dbInfo.PlacementPolicyRef, err = checkAndNormalizePlacementPolicy(ctx, dbInfo.PlacementPolicyRef)
+ return err
+}
+
+func handleTablePlacement(ctx sessionctx.Context, tbInfo *model.TableInfo) error {
+ sessVars := ctx.GetSessionVars()
+ if sessVars.PlacementMode == variable.PlacementModeIgnore && removeTablePlacement(tbInfo) {
+ sessVars.StmtCtx.AppendNote(
+ errors.NewNoStackErrorf("Placement is ignored when TIDB_PLACEMENT_MODE is '%s'", variable.PlacementModeIgnore),
+ )
+ return nil
+ }
+
+ var err error
+ tbInfo.PlacementPolicyRef, err = checkAndNormalizePlacementPolicy(ctx, tbInfo.PlacementPolicyRef)
+ if err != nil {
+ return err
+ }
+
+ if tbInfo.Partition != nil {
+ for i := range tbInfo.Partition.Definitions {
+ partition := &tbInfo.Partition.Definitions[i]
+ partition.PlacementPolicyRef, err = checkAndNormalizePlacementPolicy(ctx, partition.PlacementPolicyRef)
+ if err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func handlePartitionPlacement(ctx sessionctx.Context, partInfo *model.PartitionInfo) error {
+ sessVars := ctx.GetSessionVars()
+ if sessVars.PlacementMode == variable.PlacementModeIgnore && removePartitionPlacement(partInfo) {
+ sessVars.StmtCtx.AppendNote(
+ errors.NewNoStackErrorf("Placement is ignored when TIDB_PLACEMENT_MODE is '%s'", variable.PlacementModeIgnore),
+ )
+ return nil
+ }
+
+ var err error
+ for i := range partInfo.Definitions {
+ partition := &partInfo.Definitions[i]
+ partition.PlacementPolicyRef, err = checkAndNormalizePlacementPolicy(ctx, partition.PlacementPolicyRef)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func checkAndNormalizePlacementPolicy(ctx sessionctx.Context, placementPolicyRef *model.PolicyRefInfo) (*model.PolicyRefInfo, error) {
+ if placementPolicyRef == nil {
+ return nil, nil
+ }
+
+ if placementPolicyRef.Name.L == defaultPlacementPolicyName {
+ // When policy name is 'default', it means to remove the placement settings
+ return nil, nil
+ }
+
+ policy, ok := sessiontxn.GetTxnManager(ctx).GetTxnInfoSchema().PolicyByName(placementPolicyRef.Name)
+ if !ok {
+ return nil, errors.Trace(infoschema.ErrPlacementPolicyNotExists.GenWithStackByArgs(placementPolicyRef.Name))
+ }
+
+ placementPolicyRef.ID = policy.ID
+ return placementPolicyRef, nil
+}
+
+func checkIgnorePlacementDDL(ctx sessionctx.Context) bool {
+ sessVars := ctx.GetSessionVars()
+ if sessVars.PlacementMode == variable.PlacementModeIgnore {
+ sessVars.StmtCtx.AppendNote(
+ errors.NewNoStackErrorf("Placement is ignored when TIDB_PLACEMENT_MODE is '%s'", variable.PlacementModeIgnore),
+ )
+ return true
+ }
+ return false
+}
+
+// SetDirectPlacementOpt tries to make the PlacementSettings assignments generic for Schema/Table/Partition
+func SetDirectPlacementOpt(placementSettings *model.PlacementSettings, placementOptionType ast.PlacementOptionType, stringVal string, uintVal uint64) error {
+ switch placementOptionType {
+ case ast.PlacementOptionPrimaryRegion:
+ placementSettings.PrimaryRegion = stringVal
+ case ast.PlacementOptionRegions:
+ placementSettings.Regions = stringVal
+ case ast.PlacementOptionFollowerCount:
+ placementSettings.Followers = uintVal
+ case ast.PlacementOptionVoterCount:
+ placementSettings.Voters = uintVal
+ case ast.PlacementOptionLearnerCount:
+ placementSettings.Learners = uintVal
+ case ast.PlacementOptionSchedule:
+ placementSettings.Schedule = stringVal
+ case ast.PlacementOptionConstraints:
+ placementSettings.Constraints = stringVal
+ case ast.PlacementOptionLeaderConstraints:
+ placementSettings.LeaderConstraints = stringVal
+ case ast.PlacementOptionLearnerConstraints:
+ placementSettings.LearnerConstraints = stringVal
+ case ast.PlacementOptionFollowerConstraints:
+ placementSettings.FollowerConstraints = stringVal
+ case ast.PlacementOptionVoterConstraints:
+ placementSettings.VoterConstraints = stringVal
+ case ast.PlacementOptionSurvivalPreferences:
+ placementSettings.SurvivalPreferences = stringVal
+ default:
+ return errors.Trace(errors.New("unknown placement policy option"))
+ }
+ return nil
+}
diff --git a/pkg/ddl/placement_policy_ddl_test.go b/pkg/ddl/placement_policy_ddl_test.go
index 603a41a502828..379e5c62f6c1c 100644
--- a/pkg/ddl/placement_policy_ddl_test.go
+++ b/pkg/ddl/placement_policy_ddl_test.go
@@ -24,6 +24,7 @@ import (
"github.com/pingcap/tidb/pkg/meta"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/sessionctx"
+ "github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/util/dbterror"
"github.com/stretchr/testify/require"
@@ -91,21 +92,21 @@ func TestPlacementPolicyInUse(t *testing.T) {
t1.PlacementPolicyRef = &model.PolicyRefInfo{ID: p1.ID, Name: p1.Name}
testCreateTable(t, sctx, d, db1, t1)
t1.State = model.StatePublic
- db1.Tables = append(db1.Tables, t1)
+ db1.Deprecated.Tables = append(db1.Deprecated.Tables, t1)
t2, err := testTableInfo(store, "t2", 1)
require.NoError(t, err)
t2.PlacementPolicyRef = &model.PolicyRefInfo{ID: p1.ID, Name: p1.Name}
testCreateTable(t, sctx, d, db2, t2)
t2.State = model.StatePublic
- db2.Tables = append(db2.Tables, t2)
+ db2.Deprecated.Tables = append(db2.Deprecated.Tables, t2)
t3, err := testTableInfo(store, "t3", 1)
require.NoError(t, err)
t3.PlacementPolicyRef = &model.PolicyRefInfo{ID: p2.ID, Name: p2.Name}
testCreateTable(t, sctx, d, db1, t3)
t3.State = model.StatePublic
- db1.Tables = append(db1.Tables, t3)
+ db1.Deprecated.Tables = append(db1.Deprecated.Tables, t3)
dbP, err := testSchemaInfo(store, "db_p")
require.NoError(t, err)
@@ -117,9 +118,10 @@ func TestPlacementPolicyInUse(t *testing.T) {
t4.Partition.Definitions[0].PlacementPolicyRef = &model.PolicyRefInfo{ID: p5.ID, Name: p5.Name}
testCreateTable(t, sctx, d, db1, t4)
t4.State = model.StatePublic
- db1.Tables = append(db1.Tables, t4)
+ db1.Deprecated.Tables = append(db1.Deprecated.Tables, t4)
- builder, err := infoschema.NewBuilder(dom, nil, infoschema.NewData()).InitWithDBInfos(
+ builder := infoschema.NewBuilder(dom, nil, infoschema.NewData(), variable.SchemaCacheSize.Load() > 0)
+ err = builder.InitWithDBInfos(
[]*model.DBInfo{db1, db2, dbP},
[]*model.PolicyInfo{p1, p2, p3, p4, p5},
nil,
diff --git a/pkg/ddl/placement_policy_test.go b/pkg/ddl/placement_policy_test.go
index ba9aac2ca6e5b..ddbb7e4a6543d 100644
--- a/pkg/ddl/placement_policy_test.go
+++ b/pkg/ddl/placement_policy_test.go
@@ -26,7 +26,6 @@ import (
"github.com/pingcap/tidb/pkg/ddl"
"github.com/pingcap/tidb/pkg/ddl/placement"
"github.com/pingcap/tidb/pkg/ddl/util"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/domain/infosync"
mysql "github.com/pingcap/tidb/pkg/errno"
@@ -180,15 +179,14 @@ func TestPlacementPolicy(t *testing.T) {
}
func testPlacementPolicy(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
// clearAllBundles(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("drop placement policy if exists x")
- hook := &callback.TestDDLCallback{Do: dom}
var policyID int64
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if policyID != 0 {
return
}
@@ -197,9 +195,7 @@ func testPlacementPolicy(t *testing.T) {
policyID = job.SchemaID
return
}
- }
- hook.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("create placement policy x " +
"LEARNERS=1 " +
diff --git a/pkg/ddl/primary_key_handle_test.go b/pkg/ddl/primary_key_handle_test.go
index 4db5759915e39..6265a8e7154a0 100644
--- a/pkg/ddl/primary_key_handle_test.go
+++ b/pkg/ddl/primary_key_handle_test.go
@@ -38,7 +38,7 @@ import (
func getTableMaxHandle(t *testing.T, d ddl.DDL, tbl table.Table, store kv.Storage) (kv.Handle, bool) {
ver, err := store.CurrentVersion(kv.GlobalTxnScope)
require.NoError(t, err)
- maxHandle, emptyTable, err := d.GetTableMaxHandle(ddl.NewJobContext(), ver.Ver, tbl.(table.PhysicalTable))
+ maxHandle, emptyTable, err := ddl.GetTableMaxHandle(ddl.NewReorgContext(), store, ver.Ver, tbl.(table.PhysicalTable))
require.NoError(t, err)
return maxHandle, emptyTable
}
diff --git a/pkg/ddl/reorg.go b/pkg/ddl/reorg.go
index c5295281ee686..8d2df7ca83e4a 100644
--- a/pkg/ddl/reorg.go
+++ b/pkg/ddl/reorg.go
@@ -65,7 +65,7 @@ type reorgCtx struct {
// If the reorganization job is done, we will use this channel to notify outer.
// TODO: Now we use goroutine to simulate reorganization jobs, later we may
// use a persistent job list.
- doneCh chan error
+ doneCh chan reorgFnResult
// rowCount is used to simulate a job's row count.
rowCount int64
jobState model.JobState
@@ -80,6 +80,13 @@ type reorgCtx struct {
references atomicutil.Int32
}
+// reorgFnResult records the DDL owner TS before executing reorg function, in order to help
+// receiver determine if the result is from reorg function of previous DDL owner in this instance.
+type reorgFnResult struct {
+ ownerTS int64
+ err error
+}
+
func newReorgExprCtx() exprctx.ExprContext {
evalCtx := contextstatic.NewStaticEvalContext(
contextstatic.WithSQLMode(mysql.ModeNone),
@@ -133,9 +140,8 @@ func newReorgSessCtx(store kv.Storage) sessionctx.Context {
return c
}
-const defaultWaitReorgTimeout = 10 * time.Second
-
// ReorgWaitTimeout is the timeout that wait ddl in write reorganization stage.
+// make it a var for testing.
var ReorgWaitTimeout = 5 * time.Second
func (rc *reorgCtx) notifyJobState(state model.JobState) {
@@ -219,11 +225,10 @@ func (rc *reorgCtx) getRowCount() int64 {
func (w *worker) runReorgJob(
reorgInfo *reorgInfo,
tblInfo *model.TableInfo,
- lease time.Duration,
reorgFn func() error,
) error {
job := reorgInfo.Job
- d := reorgInfo.d
+ d := reorgInfo.jobCtx.oldDDLCtx
// This is for tests compatible, because most of the early tests try to build the reorg job manually
// without reorg meta info, which will cause nil pointer in here.
if job.ReorgMeta == nil {
@@ -251,27 +256,29 @@ func (w *worker) runReorgJob(
return dbterror.ErrCancelledDDLJob
}
+ beOwnerTS := w.ddlCtx.reorgCtx.getOwnerTS()
rc = w.newReorgCtx(reorgInfo.Job.ID, reorgInfo.Job.GetRowCount())
w.wg.Add(1)
go func() {
defer w.wg.Done()
- rc.doneCh <- reorgFn()
+ err := reorgFn()
+ rc.doneCh <- reorgFnResult{ownerTS: beOwnerTS, err: err}
}()
}
- waitTimeout := defaultWaitReorgTimeout
- // if lease is 0, we are using a local storage,
- // and we can wait the reorganization to be done here.
- // if lease > 0, we don't need to wait here because
- // we should update some job's progress context and try checking again,
- // so we use a very little timeout here.
- if lease > 0 {
- waitTimeout = ReorgWaitTimeout
- }
-
+ waitTimeout := ReorgWaitTimeout
// wait reorganization job done or timeout
select {
- case err := <-rc.doneCh:
+ case res := <-rc.doneCh:
+ err := res.err
+ curTS := w.ddlCtx.reorgCtx.getOwnerTS()
+ if res.ownerTS != curTS {
+ d.removeReorgCtx(job.ID)
+ logutil.DDLLogger().Warn("owner ts mismatch, return timeout error and retry",
+ zap.Int64("prevTS", res.ownerTS),
+ zap.Int64("curTS", curTS))
+ return dbterror.ErrWaitReorgTimeout
+ }
// Since job is cancelled，we don't care about its partial counts.
if rc.isReorgCanceled() || terror.ErrorEqual(err, dbterror.ErrCancelledDDLJob) {
d.removeReorgCtx(job.ID)
@@ -482,7 +489,7 @@ type reorgInfo struct {
StartKey kv.Key
EndKey kv.Key
- d *ddlCtx
+ jobCtx *jobContext
first bool
mergingTmpIdx bool
// PhysicalTableID is used for partitioned table.
@@ -495,8 +502,8 @@ type reorgInfo struct {
currElement *meta.Element
}
-func (r *reorgInfo) NewJobContext() *JobContext {
- return r.d.jobContext(r.Job.ID, r.Job.ReorgMeta)
+func (r *reorgInfo) NewJobContext() *ReorgContext {
+ return r.jobCtx.oldDDLCtx.jobContext(r.Job.ID, r.Job.ReorgMeta)
}
func (r *reorgInfo) String() string {
@@ -514,7 +521,7 @@ func (r *reorgInfo) String() string {
}
func constructDescTableScanPB(physicalTableID int64, tblInfo *model.TableInfo, handleCols []*model.ColumnInfo) *tipb.Executor {
- tblScan := tables.BuildTableScanFromInfos(tblInfo, handleCols)
+ tblScan := tables.BuildTableScanFromInfos(tblInfo, handleCols, false)
tblScan.TableId = physicalTableID
tblScan.Desc = true
return &tipb.Executor{Tp: tipb.ExecType_TypeTableScan, TblScan: tblScan}
@@ -552,9 +559,9 @@ func getColumnsTypes(columns []*model.ColumnInfo) []*types.FieldType {
}
// buildDescTableScan builds a desc table scan upon tblInfo.
-func (dc *ddlCtx) buildDescTableScan(ctx *JobContext, startTS uint64, tbl table.PhysicalTable,
+func buildDescTableScan(ctx *ReorgContext, store kv.Storage, startTS uint64, tbl table.PhysicalTable,
handleCols []*model.ColumnInfo, limit uint64) (distsql.SelectResult, error) {
- distSQLCtx := newDefaultReorgDistSQLCtx(dc.store.GetClient())
+ distSQLCtx := newDefaultReorgDistSQLCtx(store.GetClient())
dagPB, err := buildDescTableScanDAG(distSQLCtx, tbl, handleCols, limit)
if err != nil {
return nil, errors.Trace(err)
@@ -594,7 +601,7 @@ func (dc *ddlCtx) buildDescTableScan(ctx *JobContext, startTS uint64, tbl table.
}
// GetTableMaxHandle gets the max handle of a PhysicalTable.
-func (dc *ddlCtx) GetTableMaxHandle(ctx *JobContext, startTS uint64, tbl table.PhysicalTable) (maxHandle kv.Handle, emptyTable bool, err error) {
+func GetTableMaxHandle(ctx *ReorgContext, store kv.Storage, startTS uint64, tbl table.PhysicalTable) (maxHandle kv.Handle, emptyTable bool, err error) {
var handleCols []*model.ColumnInfo
var pkIdx *model.IndexInfo
tblInfo := tbl.Meta()
@@ -617,7 +624,7 @@ func (dc *ddlCtx) GetTableMaxHandle(ctx *JobContext, startTS uint64, tbl table.P
}
// build a desc scan of tblInfo, which limit is 1, we can use it to retrieve the last handle of the table.
- result, err := dc.buildDescTableScan(ctx, startTS, tbl, handleCols, 1)
+ result, err := buildDescTableScan(ctx, store, startTS, tbl, handleCols, 1)
if err != nil {
return nil, false, errors.Trace(err)
}
@@ -659,9 +666,9 @@ func buildCommonHandleFromChunkRow(loc *time.Location, tblInfo *model.TableInfo,
}
// getTableRange gets the start and end handle of a table (or partition).
-func getTableRange(ctx *JobContext, d *ddlCtx, tbl table.PhysicalTable, snapshotVer uint64, priority int) (startHandleKey, endHandleKey kv.Key, err error) {
+func getTableRange(ctx *ReorgContext, store kv.Storage, tbl table.PhysicalTable, snapshotVer uint64, priority int) (startHandleKey, endHandleKey kv.Key, err error) {
// Get the start handle of this partition.
- err = iterateSnapshotKeys(ctx, d.store, priority, tbl.RecordPrefix(), snapshotVer, nil, nil,
+ err = iterateSnapshotKeys(ctx, store, priority, tbl.RecordPrefix(), snapshotVer, nil, nil,
func(_ kv.Handle, rowKey kv.Key, _ []byte) (bool, error) {
startHandleKey = rowKey
return false, nil
@@ -669,7 +676,7 @@ func getTableRange(ctx *JobContext, d *ddlCtx, tbl table.PhysicalTable, snapshot
if err != nil {
return startHandleKey, endHandleKey, errors.Trace(err)
}
- maxHandle, isEmptyTable, err := d.GetTableMaxHandle(ctx, snapshotVer, tbl)
+ maxHandle, isEmptyTable, err := GetTableMaxHandle(ctx, store, snapshotVer, tbl)
if err != nil {
return startHandleKey, nil, errors.Trace(err)
}
@@ -702,7 +709,7 @@ func getValidCurrentVersion(store kv.Storage) (ver kv.Version, err error) {
return ver, nil
}
-func getReorgInfo(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job, dbInfo *model.DBInfo,
+func getReorgInfo(ctx *ReorgContext, jobCtx *jobContext, rh *reorgHandler, job *model.Job, dbInfo *model.DBInfo,
tbl table.Table, elements []*meta.Element, mergingTmpIdx bool) (*reorgInfo, error) {
var (
element *meta.Element
@@ -726,10 +733,8 @@ func getReorgInfo(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job,
})
info.first = true
- if d.lease > 0 { // Only delay when it's not in test.
- delayForAsyncCommit()
- }
- ver, err := getValidCurrentVersion(d.store)
+ delayForAsyncCommit()
+ ver, err := getValidCurrentVersion(jobCtx.store)
if err != nil {
return nil, errors.Trace(err)
}
@@ -748,7 +753,7 @@ func getReorgInfo(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job,
start = tablecodec.EncodeIndexSeekKey(pid, firstElemTempID, nil)
end = tablecodec.EncodeIndexSeekKey(pid, lastElemTempID, []byte{255})
} else {
- start, end, err = getTableRange(ctx, d, tb, ver.Ver, job.Priority)
+ start, end, err = getTableRange(ctx, jobCtx.store, tb, ver.Ver, job.Priority)
if err != nil {
return nil, errors.Trace(err)
}
@@ -797,7 +802,7 @@ func getReorgInfo(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job,
}
}
info.Job = job
- info.d = d
+ info.jobCtx = jobCtx
info.StartKey = start
info.EndKey = end
info.PhysicalTableID = pid
@@ -809,7 +814,7 @@ func getReorgInfo(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job,
return &info, nil
}
-func getReorgInfoFromPartitions(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job, dbInfo *model.DBInfo, tbl table.PartitionedTable, partitionIDs []int64, elements []*meta.Element) (*reorgInfo, error) {
+func getReorgInfoFromPartitions(ctx *ReorgContext, jobCtx *jobContext, rh *reorgHandler, job *model.Job, dbInfo *model.DBInfo, tbl table.PartitionedTable, partitionIDs []int64, elements []*meta.Element) (*reorgInfo, error) {
var (
element *meta.Element
start kv.Key
@@ -819,17 +824,15 @@ func getReorgInfoFromPartitions(ctx *JobContext, d *ddlCtx, rh *reorgHandler, jo
)
if job.SnapshotVer == 0 {
info.first = true
- if d.lease > 0 { // Only delay when it's not in test.
- delayForAsyncCommit()
- }
- ver, err := getValidCurrentVersion(d.store)
+ delayForAsyncCommit()
+ ver, err := getValidCurrentVersion(jobCtx.store)
if err != nil {
return nil, errors.Trace(err)
}
pid = partitionIDs[0]
physTbl := tbl.GetPartition(pid)
- start, end, err = getTableRange(ctx, d, physTbl, ver.Ver, job.Priority)
+ start, end, err = getTableRange(ctx, jobCtx.store, physTbl, ver.Ver, job.Priority)
if err != nil {
return nil, errors.Trace(err)
}
@@ -860,7 +863,7 @@ func getReorgInfoFromPartitions(ctx *JobContext, d *ddlCtx, rh *reorgHandler, jo
}
}
info.Job = job
- info.d = d
+ info.jobCtx = jobCtx
info.StartKey = start
info.EndKey = end
info.PhysicalTableID = pid
diff --git a/pkg/ddl/reorg_partition_test.go b/pkg/ddl/reorg_partition_test.go
index a39280c4c1727..1425642551e7d 100644
--- a/pkg/ddl/reorg_partition_test.go
+++ b/pkg/ddl/reorg_partition_test.go
@@ -23,7 +23,6 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/ddl/logutil"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/parser/model"
@@ -141,10 +140,6 @@ func getAllDataForPhysicalTable(t *testing.T, ctx sessionctx.Context, physTable
return all
}
-type TestReorgDDLCallback struct {
- *callback.TestDDLCallback
-}
-
func TestReorgPartitionConcurrent(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
@@ -157,13 +152,8 @@ func TestReorgPartitionConcurrent(t *testing.T) {
` partition p1 values less than (20),` +
` partition pMax values less than (MAXVALUE))`)
tk.MustExec(`insert into t values (1,"1",1), (10,"10",10),(23,"23",32),(34,"34",43),(45,"45",54),(56,"56",65)`)
- dom := domain.GetDomain(tk.Session())
- originHook := dom.DDL().GetHook()
- defer dom.DDL().SetHook(originHook)
syncOnChanged := make(chan bool)
defer close(syncOnChanged)
- hook := &TestReorgDDLCallback{TestDDLCallback: &callback.TestDDLCallback{Do: dom}}
- dom.DDL().SetHook(hook)
testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/afterReorganizePartition", func() {
<-syncOnChanged
// We want to wait here
@@ -174,7 +164,7 @@ func TestReorgPartitionConcurrent(t *testing.T) {
defer close(wait)
currState := model.StateNone
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.Type == model.ActionReorganizePartition &&
(job.SchemaState == model.StateDeleteOnly ||
job.SchemaState == model.StateWriteOnly ||
@@ -185,7 +175,7 @@ func TestReorgPartitionConcurrent(t *testing.T) {
<-wait
<-wait
}
- }
+ })
alterErr := make(chan error, 1)
go backgroundExec(store, schemaName, "alter table t reorganize partition p1 into (partition p1a values less than (15), partition p1b values less than (20))", alterErr)
@@ -341,24 +331,19 @@ func TestReorgPartitionFailConcurrent(t *testing.T) {
` partition p1 values less than (20),` +
` partition pMax values less than (MAXVALUE))`)
tk.MustExec(`insert into t values (1,"1",1), (12,"12",21),(23,"23",32),(34,"34",43),(45,"45",54),(56,"56",65)`)
- dom := domain.GetDomain(tk.Session())
- originHook := dom.DDL().GetHook()
- defer dom.DDL().SetHook(originHook)
- hook := &callback.TestDDLCallback{Do: dom}
- dom.DDL().SetHook(hook)
wait := make(chan bool)
defer close(wait)
// Test insert of duplicate key during copy phase
injected := false
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.Type == model.ActionReorganizePartition && job.SchemaState == model.StateWriteReorganization && !injected {
injected = true
<-wait
<-wait
}
- }
+ })
alterErr := make(chan error, 1)
go backgroundExec(store, schemaName, "alter table t reorganize partition p1 into (partition p1a values less than (15), partition p1b values less than (20))", alterErr)
wait <- true
@@ -389,7 +374,7 @@ func TestReorgPartitionFailConcurrent(t *testing.T) {
// Test reorg of duplicate key
prevState := model.StateNone
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.Type == model.ActionReorganizePartition &&
job.SchemaState == model.StateWriteReorganization &&
job.SnapshotVer == 0 &&
@@ -405,7 +390,7 @@ func TestReorgPartitionFailConcurrent(t *testing.T) {
<-wait
<-wait
}
- }
+ })
go backgroundExec(store, schemaName, "alter table t reorganize partition p1a,p1b into (partition p1a values less than (14), partition p1b values less than (17), partition p1c values less than (20))", alterErr)
wait <- true
infoSchema := sessiontxn.GetTxnManager(tk.Session()).GetTxnInfoSchema()
@@ -483,23 +468,17 @@ func TestReorgPartitionFailInject(t *testing.T) {
` partition pMax values less than (MAXVALUE))`)
tk.MustExec(`insert into t values (1,"1",1), (12,"12",21),(23,"23",32),(34,"34",43),(45,"45",54),(56,"56",65)`)
- dom := domain.GetDomain(tk.Session())
- originHook := dom.DDL().GetHook()
- defer dom.DDL().SetHook(originHook)
- hook := &callback.TestDDLCallback{Do: dom}
- dom.DDL().SetHook(hook)
-
wait := make(chan bool)
defer close(wait)
injected := false
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.Type == model.ActionReorganizePartition && job.SchemaState == model.StateWriteReorganization && !injected {
injected = true
<-wait
<-wait
}
- }
+ })
alterErr := make(chan error, 1)
go backgroundExec(store, schemaName, "alter table t reorganize partition p1 into (partition p1a values less than (15), partition p1b values less than (20))", alterErr)
wait <- true
diff --git a/pkg/ddl/repair_table_test.go b/pkg/ddl/repair_table_test.go
index 0246b3bb6e5ad..bffd508172aca 100644
--- a/pkg/ddl/repair_table_test.go
+++ b/pkg/ddl/repair_table_test.go
@@ -20,13 +20,13 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/parser/terror"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/external"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/util/domainutil"
"github.com/stretchr/testify/require"
)
@@ -39,7 +39,7 @@ func TestRepairTable(t *testing.T) {
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/infoschema/repairFetchCreateTable"))
}()
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, repairTableLease)
+ store := testkit.CreateMockStoreWithSchemaLease(t, repairTableLease)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -101,9 +101,8 @@ func TestRepairTable(t *testing.T) {
// Repaired tableInfo has been filtered by `domain.InfoSchema()`, so get it in repairInfo.
originTableInfo, _ := domainutil.RepairInfo.GetRepairedTableInfoByTableName("test", "origin")
- hook := &callback.TestDDLCallback{Do: dom}
var repairErr error
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.Type != model.ActionRepairTable {
return
}
@@ -123,10 +122,7 @@ func TestRepairTable(t *testing.T) {
if repairErr != nil && terror.ErrorEqual(repairErr, infoschema.ErrTableNotExists) {
repairErr = nil
}
- }
- originalHook := dom.DDL().GetHook()
- defer dom.DDL().SetHook(originalHook)
- dom.DDL().SetHook(hook)
+ })
// Exec the repair statement to override the tableInfo.
tk.MustExec("admin repair table origin CREATE TABLE origin (a int primary key nonclustered auto_increment, b varchar(5), c int);")
diff --git a/pkg/ddl/resource_group.go b/pkg/ddl/resource_group.go
index 39265a46dee6a..4a4d6914e0ded 100644
--- a/pkg/ddl/resource_group.go
+++ b/pkg/ddl/resource_group.go
@@ -16,6 +16,8 @@ package ddl
import (
"context"
+ "math"
+ "slices"
"strings"
"time"
@@ -26,8 +28,10 @@ import (
rg "github.com/pingcap/tidb/pkg/domain/resourcegroup"
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/meta"
+ "github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/util/dbterror"
+ kvutil "github.com/tikv/client-go/v2/util"
"go.uber.org/zap"
)
@@ -38,7 +42,7 @@ const (
alreadyExists = "already exists"
)
-func onCreateResourceGroup(ctx context.Context, d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onCreateResourceGroup(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
groupInfo := &model.ResourceGroupInfo{}
if err := job.DecodeArgs(groupInfo); err != nil {
job.State = model.JobStateCancelled
@@ -63,7 +67,7 @@ func onCreateResourceGroup(ctx context.Context, d *ddlCtx, t *meta.Meta, job *mo
return ver, errors.Trace(err)
}
- ctx, cancel := context.WithTimeout(ctx, defaultInfosyncTimeout)
+ ctx, cancel := context.WithTimeout(jobCtx.ctx, defaultInfosyncTimeout)
defer cancel()
err = infosync.AddResourceGroup(ctx, protoGroup)
if err != nil {
@@ -75,7 +79,7 @@ func onCreateResourceGroup(ctx context.Context, d *ddlCtx, t *meta.Meta, job *mo
}
}
job.SchemaID = groupInfo.ID
- ver, err = updateSchemaVersion(d, t, job)
+ ver, err = updateSchemaVersion(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -87,7 +91,7 @@ func onCreateResourceGroup(ctx context.Context, d *ddlCtx, t *meta.Meta, job *mo
}
}
-func onAlterResourceGroup(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onAlterResourceGroup(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
alterGroupInfo := &model.ResourceGroupInfo{}
if err := job.DecodeArgs(alterGroupInfo); err != nil {
job.State = model.JobStateCancelled
@@ -122,7 +126,7 @@ func onAlterResourceGroup(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _
return ver, errors.Trace(err)
}
- ver, err = updateSchemaVersion(d, t, job)
+ ver, err = updateSchemaVersion(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -142,7 +146,7 @@ func checkResourceGroupExist(t *meta.Meta, job *model.Job, groupID int64) (*mode
return nil, err
}
-func onDropResourceGroup(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onDropResourceGroup(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
groupInfo, err := checkResourceGroupExist(t, job, job.SchemaID)
if err != nil {
return ver, errors.Trace(err)
@@ -161,7 +165,7 @@ func onDropResourceGroup(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _
if err != nil {
return ver, errors.Trace(err)
}
- ver, err = updateSchemaVersion(d, t, job)
+ ver, err = updateSchemaVersion(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -172,3 +176,156 @@ func onDropResourceGroup(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _
}
return ver, errors.Trace(err)
}
+
+func buildResourceGroup(oldGroup *model.ResourceGroupInfo, options []*ast.ResourceGroupOption) (*model.ResourceGroupInfo, error) {
+ groupInfo := &model.ResourceGroupInfo{Name: oldGroup.Name, ID: oldGroup.ID, ResourceGroupSettings: model.NewResourceGroupSettings()}
+ if oldGroup.ResourceGroupSettings != nil {
+ *groupInfo.ResourceGroupSettings = *oldGroup.ResourceGroupSettings
+ }
+ for _, opt := range options {
+ err := SetDirectResourceGroupSettings(groupInfo, opt)
+ if err != nil {
+ return nil, err
+ }
+ }
+ groupInfo.ResourceGroupSettings.Adjust()
+ return groupInfo, nil
+}
+
+// SetDirectResourceGroupSettings tries to set the ResourceGroupSettings.
+func SetDirectResourceGroupSettings(groupInfo *model.ResourceGroupInfo, opt *ast.ResourceGroupOption) error {
+ resourceGroupSettings := groupInfo.ResourceGroupSettings
+ switch opt.Tp {
+ case ast.ResourceRURate:
+ return SetDirectResourceGroupRUSecondOption(resourceGroupSettings, opt.UintValue, opt.BoolValue)
+ case ast.ResourcePriority:
+ resourceGroupSettings.Priority = opt.UintValue
+ case ast.ResourceUnitCPU:
+ resourceGroupSettings.CPULimiter = opt.StrValue
+ case ast.ResourceUnitIOReadBandwidth:
+ resourceGroupSettings.IOReadBandwidth = opt.StrValue
+ case ast.ResourceUnitIOWriteBandwidth:
+ resourceGroupSettings.IOWriteBandwidth = opt.StrValue
+ case ast.ResourceBurstableOpiton:
+ // Some about BurstLimit(b):
+ // - If b == 0, that means the limiter is unlimited capacity. default use in resource controller (burst with a rate within a unlimited capacity).
+ // - If b < 0, that means the limiter is unlimited capacity and fillrate(r) is ignored, can be seen as r == Inf (burst with a inf rate within a unlimited capacity).
+ // - If b > 0, that means the limiter is limited capacity. (current not used).
+ limit := int64(0)
+ if opt.BoolValue {
+ limit = -1
+ }
+ resourceGroupSettings.BurstLimit = limit
+ case ast.ResourceGroupRunaway:
+ if len(opt.RunawayOptionList) == 0 {
+ resourceGroupSettings.Runaway = nil
+ }
+ for _, opt := range opt.RunawayOptionList {
+ if err := SetDirectResourceGroupRunawayOption(resourceGroupSettings, opt); err != nil {
+ return err
+ }
+ }
+ case ast.ResourceGroupBackground:
+ if groupInfo.Name.L != rg.DefaultResourceGroupName {
+ // FIXME: this is a temporary restriction, so we don't add a error-code for it.
+ return errors.New("unsupported operation. Currently, only the default resource group support change background settings")
+ }
+ if len(opt.BackgroundOptions) == 0 {
+ resourceGroupSettings.Background = nil
+ }
+ for _, opt := range opt.BackgroundOptions {
+ if err := SetDirectResourceGroupBackgroundOption(resourceGroupSettings, opt); err != nil {
+ return err
+ }
+ }
+ default:
+ return errors.Trace(errors.New("unknown resource unit type"))
+ }
+ return nil
+}
+
+// SetDirectResourceGroupRUSecondOption tries to set ru second part of the ResourceGroupSettings.
+func SetDirectResourceGroupRUSecondOption(resourceGroupSettings *model.ResourceGroupSettings, intVal uint64, unlimited bool) error {
+ if unlimited {
+ resourceGroupSettings.RURate = uint64(math.MaxInt32)
+ resourceGroupSettings.BurstLimit = -1
+ } else {
+ resourceGroupSettings.RURate = intVal
+ }
+ return nil
+}
+
+// SetDirectResourceGroupRunawayOption tries to set runaway part of the ResourceGroupSettings.
+func SetDirectResourceGroupRunawayOption(resourceGroupSettings *model.ResourceGroupSettings, opt *ast.ResourceGroupRunawayOption) error {
+ if resourceGroupSettings.Runaway == nil {
+ resourceGroupSettings.Runaway = &model.ResourceGroupRunawaySettings{}
+ }
+ settings := resourceGroupSettings.Runaway
+ switch opt.Tp {
+ case ast.RunawayRule:
+ // because execute time won't be too long, we use `time` pkg which does not support to parse unit 'd'.
+ dur, err := time.ParseDuration(opt.RuleOption.ExecElapsed)
+ if err != nil {
+ return err
+ }
+ settings.ExecElapsedTimeMs = uint64(dur.Milliseconds())
+ case ast.RunawayAction:
+ settings.Action = opt.ActionOption.Type
+ case ast.RunawayWatch:
+ settings.WatchType = opt.WatchOption.Type
+ if dur := opt.WatchOption.Duration; len(dur) > 0 {
+ dur, err := time.ParseDuration(dur)
+ if err != nil {
+ return err
+ }
+ settings.WatchDurationMs = dur.Milliseconds()
+ } else {
+ settings.WatchDurationMs = 0
+ }
+ default:
+ return errors.Trace(errors.New("unknown runaway option type"))
+ }
+ return nil
+}
+
+// SetDirectResourceGroupBackgroundOption set background configs of the ResourceGroupSettings.
+func SetDirectResourceGroupBackgroundOption(resourceGroupSettings *model.ResourceGroupSettings, opt *ast.ResourceGroupBackgroundOption) error {
+ if resourceGroupSettings.Background == nil {
+ resourceGroupSettings.Background = &model.ResourceGroupBackgroundSettings{}
+ }
+ switch opt.Type {
+ case ast.BackgroundOptionTaskNames:
+ jobTypes, err := parseBackgroundJobTypes(opt.StrValue)
+ if err != nil {
+ return err
+ }
+ resourceGroupSettings.Background.JobTypes = jobTypes
+ default:
+ return errors.Trace(errors.New("unknown background option type"))
+ }
+ return nil
+}
+
+func parseBackgroundJobTypes(t string) ([]string, error) {
+ if len(t) == 0 {
+ return []string{}, nil
+ }
+
+ segs := strings.Split(t, ",")
+ res := make([]string, 0, len(segs))
+ for _, s := range segs {
+ ty := strings.ToLower(strings.TrimSpace(s))
+ if len(ty) > 0 {
+ if !slices.Contains(kvutil.ExplicitTypeList, ty) {
+ return nil, infoschema.ErrResourceGroupInvalidBackgroundTaskName.GenWithStackByArgs(ty)
+ }
+ res = append(res, ty)
+ }
+ }
+ return res, nil
+}
+
+func checkResourceGroupValidation(groupInfo *model.ResourceGroupInfo) error {
+ _, err := resourcegroup.NewGroupFromOptions(groupInfo.Name.L, groupInfo.ResourceGroupSettings)
+ return err
+}
diff --git a/pkg/ddl/restart_test.go b/pkg/ddl/restart_test.go
index 19aee4f68b5d6..f0cb18bbb4425 100644
--- a/pkg/ddl/restart_test.go
+++ b/pkg/ddl/restart_test.go
@@ -46,7 +46,7 @@ func restartWorkers(t *testing.T, store kv.Storage, d *domain.Domain) {
newDDL, newDDLExecutor := ddl.NewDDL(context.Background(),
ddl.WithStore(d.Store()),
ddl.WithInfoCache(d.InfoCache()),
- ddl.WithLease(d.DDL().GetLease()),
+ ddl.WithLease(d.GetSchemaLease()),
ddl.WithSchemaLoader(d),
)
d.SetDDL(newDDL, newDDLExecutor)
@@ -92,7 +92,7 @@ func testRunInterruptedJob(t *testing.T, store kv.Storage, d *domain.Domain, job
done := make(chan error, 1)
go runInterruptedJob(t, store, d.DDLExecutor(), job, done)
- ticker := time.NewTicker(d.DDL().GetLease())
+ ticker := time.NewTicker(d.GetSchemaLease())
defer ticker.Stop()
for {
select {
@@ -140,7 +140,7 @@ func TestStat(t *testing.T) {
done := make(chan error, 1)
go runInterruptedJob(t, store, dom.DDLExecutor(), job, done)
- ticker := time.NewTicker(dom.DDL().GetLease() * 1)
+ ticker := time.NewTicker(dom.GetSchemaLease() * 1)
defer ticker.Stop()
ver := getDDLSchemaVer(t, dom.DDL())
LOOP:
diff --git a/pkg/ddl/rollingback.go b/pkg/ddl/rollingback.go
index b72b2ee76d392..0676e708a4917 100644
--- a/pkg/ddl/rollingback.go
+++ b/pkg/ddl/rollingback.go
@@ -45,7 +45,7 @@ func UpdateColsNull2NotNull(tblInfo *model.TableInfo, indexInfo *model.IndexInfo
}
func convertAddIdxJob2RollbackJob(
- d *ddlCtx,
+ jobCtx *jobContext,
t *meta.Meta,
job *model.Job,
tblInfo *model.TableInfo,
@@ -84,7 +84,7 @@ func convertAddIdxJob2RollbackJob(
// the second and the third args will be used in onDropIndex.
job.Args = []any{idxNames, ifExists, getPartitionIDs(tblInfo)}
job.SchemaState = model.StateDeleteOnly
- ver, err1 := updateVersionAndTableInfo(d, t, job, tblInfo, originalState != model.StateDeleteOnly)
+ ver, err1 := updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != model.StateDeleteOnly)
if err1 != nil {
return ver, errors.Trace(err1)
}
@@ -99,7 +99,7 @@ func convertAddIdxJob2RollbackJob(
// convertNotReorgAddIdxJob2RollbackJob converts the add index job that are not started workers to rollingbackJob,
// to rollback add index operations. job.SnapshotVer == 0 indicates the workers are not started.
-func convertNotReorgAddIdxJob2RollbackJob(d *ddlCtx, t *meta.Meta, job *model.Job, occuredErr error) (ver int64, err error) {
+func convertNotReorgAddIdxJob2RollbackJob(jobCtx *jobContext, t *meta.Meta, job *model.Job, occuredErr error) (ver int64, err error) {
defer func() {
if ingest.LitBackCtxMgr != nil {
ingest.LitBackCtxMgr.Unregister(job.ID)
@@ -136,20 +136,20 @@ func convertNotReorgAddIdxJob2RollbackJob(d *ddlCtx, t *meta.Meta, job *model.Jo
job.State = model.JobStateCancelled
return ver, dbterror.ErrCancelledDDLJob
}
- return convertAddIdxJob2RollbackJob(d, t, job, tblInfo, indexesInfo, occuredErr)
+ return convertAddIdxJob2RollbackJob(jobCtx, t, job, tblInfo, indexesInfo, occuredErr)
}
// rollingbackModifyColumn change the modifying-column job into rolling back state.
// Since modifying column job has two types: normal-type and reorg-type, we should handle it respectively.
// normal-type has only two states: None -> Public
// reorg-type has five states: None -> Delete-only -> Write-only -> Write-org -> Public
-func rollingbackModifyColumn(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func rollingbackModifyColumn(w *worker, jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
if needNotifyAndStopReorgWorker(job) {
// column type change workers are started. we have to ask them to exit.
w.jobLogger(job).Info("run the cancelling DDL job", zap.String("job", job.String()))
- d.notifyReorgWorkerJobStateChange(job)
+ jobCtx.oldDDLCtx.notifyReorgWorkerJobStateChange(job)
// Give the this kind of ddl one more round to run, the dbterror.ErrCancelledDDLJob should be fetched from the bottom up.
- return w.onModifyColumn(d, t, job)
+ return w.onModifyColumn(jobCtx, t, job)
}
_, tblInfo, oldCol, jp, err := getModifyColumnInfo(t, job)
if err != nil {
@@ -183,7 +183,7 @@ func rollingbackModifyColumn(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job)
return ver, dbterror.ErrCancelledDDLJob
}
-func rollingbackAddColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func rollingbackAddColumn(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
tblInfo, columnInfo, col, _, _, err := checkAddColumn(t, job)
if err != nil {
return ver, errors.Trace(err)
@@ -198,7 +198,7 @@ func rollingbackAddColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, e
job.SchemaState = model.StateDeleteOnly
job.Args = []any{col.Name}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != columnInfo.State)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != columnInfo.State)
if err != nil {
return ver, errors.Trace(err)
}
@@ -207,8 +207,8 @@ func rollingbackAddColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, e
return ver, dbterror.ErrCancelledDDLJob
}
-func rollingbackDropColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
- _, colInfo, idxInfos, _, err := checkDropColumn(d, t, job)
+func rollingbackDropColumn(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
+ _, colInfo, idxInfos, _, err := checkDropColumn(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -239,8 +239,8 @@ func rollingbackDropColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64,
return ver, nil
}
-func rollingbackDropIndex(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
- _, indexInfo, _, err := checkDropIndex(d, t, job)
+func rollingbackDropIndex(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
+ _, indexInfo, _, err := checkDropIndex(jobCtx.infoCache, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -259,15 +259,15 @@ func rollingbackDropIndex(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, e
}
}
-func rollingbackAddIndex(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job, isPK bool) (ver int64, err error) {
+func rollingbackAddIndex(w *worker, jobCtx *jobContext, t *meta.Meta, job *model.Job, isPK bool) (ver int64, err error) {
if needNotifyAndStopReorgWorker(job) {
// add index workers are started. need to ask them to exit.
w.jobLogger(job).Info("run the cancelling DDL job", zap.String("job", job.String()))
- d.notifyReorgWorkerJobStateChange(job)
- ver, err = w.onCreateIndex(d, t, job, isPK)
+ jobCtx.oldDDLCtx.notifyReorgWorkerJobStateChange(job)
+ ver, err = w.onCreateIndex(jobCtx, t, job, isPK)
} else {
// add index's reorg workers are not running, remove the indexInfo in tableInfo.
- ver, err = convertNotReorgAddIdxJob2RollbackJob(d, t, job, dbterror.ErrCancelledDDLJob)
+ ver, err = convertNotReorgAddIdxJob2RollbackJob(jobCtx, t, job, dbterror.ErrCancelledDDLJob)
}
return
}
@@ -287,12 +287,12 @@ func needNotifyAndStopReorgWorker(job *model.Job) bool {
// rollbackExchangeTablePartition will clear the non-partitioned
// table's ExchangePartitionInfo state.
-func rollbackExchangeTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job, tblInfo *model.TableInfo) (ver int64, err error) {
+func rollbackExchangeTablePartition(jobCtx *jobContext, t *meta.Meta, job *model.Job, tblInfo *model.TableInfo) (ver int64, err error) {
tblInfo.ExchangePartitionInfo = nil
job.State = model.JobStateRollbackDone
job.SchemaState = model.StatePublic
if len(tblInfo.Constraints) == 0 {
- return updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ return updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
}
var (
defID int64
@@ -314,11 +314,11 @@ func rollbackExchangeTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job, tbl
schemaID: ptSchemaID,
tblInfo: pt,
})
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true, ptInfo...)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true, ptInfo...)
return ver, errors.Trace(err)
}
-func rollingbackExchangeTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func rollingbackExchangeTablePartition(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
if job.SchemaState == model.StateNone {
// Nothing is changed
job.State = model.JobStateCancelled
@@ -329,11 +329,11 @@ func rollingbackExchangeTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job)
if err != nil {
return ver, errors.Trace(err)
}
- ver, err = rollbackExchangeTablePartition(d, t, job, nt)
+ ver, err = rollbackExchangeTablePartition(jobCtx, t, job, nt)
return ver, errors.Trace(err)
}
-func convertAddTablePartitionJob2RollbackJob(d *ddlCtx, t *meta.Meta, job *model.Job, otherwiseErr error, tblInfo *model.TableInfo) (ver int64, err error) {
+func convertAddTablePartitionJob2RollbackJob(jobCtx *jobContext, t *meta.Meta, job *model.Job, otherwiseErr error, tblInfo *model.TableInfo) (ver int64, err error) {
addingDefinitions := tblInfo.Partition.AddingDefinitions
partNames := make([]string, 0, len(addingDefinitions))
for _, pd := range addingDefinitions {
@@ -352,7 +352,7 @@ func convertAddTablePartitionJob2RollbackJob(d *ddlCtx, t *meta.Meta, job *model
} else {
job.Args = []any{partNames}
}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -360,7 +360,7 @@ func convertAddTablePartitionJob2RollbackJob(d *ddlCtx, t *meta.Meta, job *model
return ver, errors.Trace(otherwiseErr)
}
-func rollingbackAddTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func rollingbackAddTablePartition(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
tblInfo, _, addingDefinitions, err := checkAddPartition(t, job)
if err != nil {
return ver, errors.Trace(err)
@@ -371,7 +371,7 @@ func rollingbackAddTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (ver
return ver, errors.Trace(dbterror.ErrCancelledDDLJob)
}
// addingDefinitions is also in tblInfo, here pass the tblInfo as parameter directly.
- return convertAddTablePartitionJob2RollbackJob(d, t, job, dbterror.ErrCancelledDDLJob, tblInfo)
+ return convertAddTablePartitionJob2RollbackJob(jobCtx, t, job, dbterror.ErrCancelledDDLJob, tblInfo)
}
func rollingbackDropTableOrView(t *meta.Meta, job *model.Job) error {
@@ -447,7 +447,7 @@ func rollingbackTruncateTable(t *meta.Meta, job *model.Job) (ver int64, err erro
return cancelOnlyNotHandledJob(job, model.StateNone)
}
-func rollingbackReorganizePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func rollingbackReorganizePartition(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
if job.SchemaState == model.StateNone {
job.State = model.JobStateCancelled
return ver, dbterror.ErrCancelledDDLJob
@@ -460,7 +460,7 @@ func rollingbackReorganizePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
// addingDefinitions is also in tblInfo, here pass the tblInfo as parameter directly.
// TODO: Test this with reorganize partition p1 into (partition p1 ...)!
- return convertAddTablePartitionJob2RollbackJob(d, t, job, dbterror.ErrCancelledDDLJob, tblInfo)
+ return convertAddTablePartitionJob2RollbackJob(jobCtx, t, job, dbterror.ErrCancelledDDLJob, tblInfo)
}
func pauseReorgWorkers(w *worker, d *ddlCtx, job *model.Job) (err error) {
@@ -472,29 +472,29 @@ func pauseReorgWorkers(w *worker, d *ddlCtx, job *model.Job) (err error) {
return dbterror.ErrPausedDDLJob.GenWithStackByArgs(job.ID)
}
-func convertJob2RollbackJob(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func convertJob2RollbackJob(w *worker, jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
switch job.Type {
case model.ActionAddColumn:
- ver, err = rollingbackAddColumn(d, t, job)
+ ver, err = rollingbackAddColumn(jobCtx, t, job)
case model.ActionAddIndex:
- ver, err = rollingbackAddIndex(w, d, t, job, false)
+ ver, err = rollingbackAddIndex(w, jobCtx, t, job, false)
case model.ActionAddPrimaryKey:
- ver, err = rollingbackAddIndex(w, d, t, job, true)
+ ver, err = rollingbackAddIndex(w, jobCtx, t, job, true)
case model.ActionAddTablePartition:
- ver, err = rollingbackAddTablePartition(d, t, job)
+ ver, err = rollingbackAddTablePartition(jobCtx, t, job)
case model.ActionReorganizePartition, model.ActionRemovePartitioning,
model.ActionAlterTablePartitioning:
- ver, err = rollingbackReorganizePartition(d, t, job)
+ ver, err = rollingbackReorganizePartition(jobCtx, t, job)
case model.ActionDropColumn:
- ver, err = rollingbackDropColumn(d, t, job)
+ ver, err = rollingbackDropColumn(jobCtx, t, job)
case model.ActionDropIndex, model.ActionDropPrimaryKey:
- ver, err = rollingbackDropIndex(d, t, job)
+ ver, err = rollingbackDropIndex(jobCtx, t, job)
case model.ActionDropTable, model.ActionDropView, model.ActionDropSequence:
err = rollingbackDropTableOrView(t, job)
case model.ActionDropTablePartition:
ver, err = rollingbackDropTablePartition(t, job)
case model.ActionExchangeTablePartition:
- ver, err = rollingbackExchangeTablePartition(d, t, job)
+ ver, err = rollingbackExchangeTablePartition(jobCtx, t, job)
case model.ActionDropSchema:
err = rollingbackDropSchema(t, job)
case model.ActionRenameIndex:
@@ -502,7 +502,7 @@ func convertJob2RollbackJob(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job)
case model.ActionTruncateTable:
ver, err = rollingbackTruncateTable(t, job)
case model.ActionModifyColumn:
- ver, err = rollingbackModifyColumn(w, d, t, job)
+ ver, err = rollingbackModifyColumn(w, jobCtx, t, job)
case model.ActionDropForeignKey, model.ActionTruncateTablePartition:
ver, err = cancelOnlyNotHandledJob(job, model.StatePublic)
case model.ActionRebaseAutoID, model.ActionShardRowID, model.ActionAddForeignKey,
@@ -515,11 +515,11 @@ func convertJob2RollbackJob(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job)
case model.ActionMultiSchemaChange:
err = rollingBackMultiSchemaChange(job)
case model.ActionAddCheckConstraint:
- ver, err = rollingBackAddConstraint(d, t, job)
+ ver, err = rollingBackAddConstraint(jobCtx, t, job)
case model.ActionDropCheckConstraint:
ver, err = rollingBackDropConstraint(t, job)
case model.ActionAlterCheckConstraint:
- ver, err = rollingBackAlterConstraint(d, t, job)
+ ver, err = rollingBackAlterConstraint(jobCtx, t, job)
default:
job.State = model.JobStateCancelled
err = dbterror.ErrCancelledDDLJob
@@ -566,7 +566,7 @@ func convertJob2RollbackJob(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job)
return
}
-func rollingBackAddConstraint(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func rollingBackAddConstraint(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
_, tblInfo, constrInfoInMeta, _, err := checkAddCheckConstraint(t, job)
if err != nil {
return ver, errors.Trace(err)
@@ -586,7 +586,7 @@ func rollingBackAddConstraint(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int6
if job.IsRollingback() {
job.State = model.JobStateRollbackDone
}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
return ver, errors.Trace(err)
}
@@ -606,7 +606,7 @@ func rollingBackDropConstraint(t *meta.Meta, job *model.Job) (ver int64, err err
return ver, nil
}
-func rollingBackAlterConstraint(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func rollingBackAlterConstraint(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
_, tblInfo, constraintInfo, enforced, err := checkAlterCheckConstraint(t, job)
if err != nil {
return ver, errors.Trace(err)
@@ -624,6 +624,6 @@ func rollingBackAlterConstraint(d *ddlCtx, t *meta.Meta, job *model.Job) (ver in
if job.IsRollingback() {
job.State = model.JobStateRollbackDone
}
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tblInfo, true)
return ver, errors.Trace(err)
}
diff --git a/pkg/ddl/rollingback_test.go b/pkg/ddl/rollingback_test.go
index bbd78be112372..7e10f00072c4c 100644
--- a/pkg/ddl/rollingback_test.go
+++ b/pkg/ddl/rollingback_test.go
@@ -22,17 +22,17 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/ddl"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/external"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/util/sqlexec"
"github.com/stretchr/testify/require"
)
// TestCancelJobMeetError is used to test canceling ddl job failure when convert ddl job to a rolling back job.
func TestCancelAddIndexJobError(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk1 := testkit.NewTestKit(t, store)
@@ -51,14 +51,12 @@ func TestCancelAddIndexJobError(t *testing.T) {
tbl := external.GetTableByName(t, tk, "test", "t_cancel_add_index") //nolint:typecheck
require.NotNil(t, tbl)
- d := dom.DDL()
- hook := &callback.TestDDLCallback{Do: dom}
var (
checkErr error
jobID atomic.Int64
res sqlexec.RecordSet
)
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.TableID != tbl.Meta().ID {
return
}
@@ -78,9 +76,7 @@ func TestCancelAddIndexJobError(t *testing.T) {
checkErr = err
}
}
- }
- hook.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(hook)
+ })
// This will hang on stateDeleteOnly, and the job will be canceled.
err := tk.ExecToErr("alter table t_cancel_add_index add index idx(a)")
diff --git a/pkg/ddl/sanity_check.go b/pkg/ddl/sanity_check.go
index 53d446191d9f1..9f8086beefedd 100644
--- a/pkg/ddl/sanity_check.go
+++ b/pkg/ddl/sanity_check.go
@@ -109,9 +109,10 @@ func expectedDeleteRangeCnt(ctx delRangeCntCtx, job *model.Job) (int, error) {
case model.ActionAddIndex, model.ActionAddPrimaryKey:
indexID := make([]int64, 1)
ifExists := make([]bool, 1)
+ isGlobal := make([]bool, 0, 1)
var partitionIDs []int64
if err := job.DecodeArgs(&indexID[0], &ifExists[0], &partitionIDs); err != nil {
- if err := job.DecodeArgs(&indexID, &ifExists, &partitionIDs); err != nil {
+ if err := job.DecodeArgs(&indexID, &ifExists, &partitionIDs, &isGlobal); err != nil {
var unique bool
if err := job.DecodeArgs(&unique); err == nil {
// The first argument is bool means nothing need to be added to delete-range table.
@@ -120,11 +121,18 @@ func expectedDeleteRangeCnt(ctx delRangeCntCtx, job *model.Job) (int, error) {
return 0, errors.Trace(err)
}
}
- idxIDNumFactor := len(indexID) // Add temporary index to del-range table.
- if job.State == model.JobStateRollbackDone {
- idxIDNumFactor = 2 * len(indexID) // Add origin index to del-range table.
+ ret := 0
+ for i := 0; i < len(indexID); i++ {
+ num := mathutil.Max(len(partitionIDs), 1) // Add temporary index to del-range table.
+ if len(isGlobal) != 0 && isGlobal[i] {
+ num = 1 // Global index only has one del-range.
+ }
+ if job.State == model.JobStateRollbackDone {
+ num *= 2 // Add origin index to del-range table.
+ }
+ ret += num
}
- return mathutil.Max(len(partitionIDs)*idxIDNumFactor, idxIDNumFactor), nil
+ return ret, nil
case model.ActionDropIndex, model.ActionDropPrimaryKey:
var indexName any
ifNotExists := make([]bool, 1)
diff --git a/pkg/ddl/schema.go b/pkg/ddl/schema.go
index 590b3be5e8c2a..5e8551a43cec1 100644
--- a/pkg/ddl/schema.go
+++ b/pkg/ddl/schema.go
@@ -27,7 +27,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/model"
)
-func onCreateSchema(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onCreateSchema(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
schemaID := job.SchemaID
dbInfo := &model.DBInfo{}
if err := job.DecodeArgs(dbInfo); err != nil {
@@ -39,7 +39,7 @@ func onCreateSchema(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error
dbInfo.ID = schemaID
dbInfo.State = model.StateNone
- err := checkSchemaNotExists(d, schemaID, dbInfo)
+ err := checkSchemaNotExists(jobCtx.infoCache, schemaID, dbInfo)
if err != nil {
if infoschema.ErrDatabaseExists.Equal(err) {
// The database already exists, can't create it, we should cancel this job now.
@@ -48,7 +48,7 @@ func onCreateSchema(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error
return ver, errors.Trace(err)
}
- ver, err = updateSchemaVersion(d, t, job)
+ ver, err = updateSchemaVersion(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -72,8 +72,8 @@ func onCreateSchema(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error
// checkSchemaNotExists checks whether the database already exists.
// see checkTableNotExists for the rationale of why we check using info schema only.
-func checkSchemaNotExists(d *ddlCtx, schemaID int64, dbInfo *model.DBInfo) error {
- is := d.infoCache.GetLatest()
+func checkSchemaNotExists(infoCache *infoschema.InfoCache, schemaID int64, dbInfo *model.DBInfo) error {
+ is := infoCache.GetLatest()
// Check database exists by name.
if is.SchemaExists(dbInfo.Name) {
return infoschema.ErrDatabaseExists.GenWithStackByArgs(dbInfo.Name)
@@ -85,7 +85,7 @@ func checkSchemaNotExists(d *ddlCtx, schemaID int64, dbInfo *model.DBInfo) error
return nil
}
-func onModifySchemaCharsetAndCollate(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onModifySchemaCharsetAndCollate(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
var toCharset, toCollate string
if err := job.DecodeArgs(&toCharset, &toCollate); err != nil {
job.State = model.JobStateCancelled
@@ -108,14 +108,14 @@ func onModifySchemaCharsetAndCollate(d *ddlCtx, t *meta.Meta, job *model.Job) (v
if err = t.UpdateDatabase(dbInfo); err != nil {
return ver, errors.Trace(err)
}
- if ver, err = updateSchemaVersion(d, t, job); err != nil {
+ if ver, err = updateSchemaVersion(jobCtx, t, job); err != nil {
return ver, errors.Trace(err)
}
job.FinishDBJob(model.JobStateDone, model.StatePublic, ver, dbInfo)
return ver, nil
}
-func onModifySchemaDefaultPlacement(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onModifySchemaDefaultPlacement(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
var placementPolicyRef *model.PolicyRefInfo
if err := job.DecodeArgs(&placementPolicyRef); err != nil {
job.State = model.JobStateCancelled
@@ -144,26 +144,26 @@ func onModifySchemaDefaultPlacement(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
if err = t.UpdateDatabase(dbInfo); err != nil {
return ver, errors.Trace(err)
}
- if ver, err = updateSchemaVersion(d, t, job); err != nil {
+ if ver, err = updateSchemaVersion(jobCtx, t, job); err != nil {
return ver, errors.Trace(err)
}
job.FinishDBJob(model.JobStateDone, model.StatePublic, ver, dbInfo)
return ver, nil
}
-func onDropSchema(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onDropSchema(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
dbInfo, err := checkSchemaExistAndCancelNotExistJob(t, job)
if err != nil {
return ver, errors.Trace(err)
}
if dbInfo.State == model.StatePublic {
- err = checkDatabaseHasForeignKeyReferredInOwner(d, t, job)
+ err = checkDatabaseHasForeignKeyReferredInOwner(jobCtx, job)
if err != nil {
return ver, errors.Trace(err)
}
}
- ver, err = updateSchemaVersion(d, t, job)
+ ver, err = updateSchemaVersion(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -210,7 +210,7 @@ func onDropSchema(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error)
if err != nil {
return ver, errors.Trace(err)
}
- if err = t.DropDatabase(dbInfo.ID, dbInfo.Name.L); err != nil {
+ if err = t.DropDatabase(dbInfo.ID); err != nil {
break
}
@@ -227,7 +227,7 @@ func onDropSchema(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error)
return ver, errors.Trace(err)
}
-func (w *worker) onRecoverSchema(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func (w *worker) onRecoverSchema(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
var (
recoverSchemaInfo *RecoverSchemaInfo
recoverSchemaCheckFlag int64
@@ -321,7 +321,7 @@ func (w *worker) onRecoverSchema(d *ddlCtx, t *meta.Meta, job *model.Job) (ver i
schemaInfo.State = model.StatePublic
// use to update InfoSchema
job.SchemaID = schemaInfo.ID
- ver, err = updateSchemaVersion(d, t, job)
+ ver, err = updateSchemaVersion(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
diff --git a/pkg/ddl/schema_test.go b/pkg/ddl/schema_test.go
index 8403e6f3d26dd..39e4ed6d70274 100644
--- a/pkg/ddl/schema_test.go
+++ b/pkg/ddl/schema_test.go
@@ -31,7 +31,6 @@ import (
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/parser/terror"
"github.com/pingcap/tidb/pkg/sessionctx"
- "github.com/pingcap/tidb/pkg/sessiontxn"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/types"
"github.com/stretchr/testify/require"
@@ -240,10 +239,10 @@ func TestSchema(t *testing.T) {
testCheckTableState(t, store, dbInfo, tblInfo1, model.StatePublic)
testCheckJobDone(t, store, tJob1.ID, true)
tbl1 := testGetTable(t, domain, tblInfo1.ID)
- err = sessiontxn.NewTxn(context.Background(), tk.Session())
+ txn, err := newTxn(tk.Session())
require.NoError(t, err)
for i := 1; i <= 100; i++ {
- _, err := tbl1.AddRecord(tk.Session().GetTableCtx(), types.MakeDatums(i, i, i))
+ _, err := tbl1.AddRecord(tk.Session().GetTableCtx(), txn, types.MakeDatums(i, i, i))
require.NoError(t, err)
}
// create table t1 with 1034 records.
@@ -254,10 +253,10 @@ func TestSchema(t *testing.T) {
testCheckTableState(t, store, dbInfo, tblInfo2, model.StatePublic)
testCheckJobDone(t, store, tJob2.ID, true)
tbl2 := testGetTable(t, domain, tblInfo2.ID)
- err = sessiontxn.NewTxn(context.Background(), tk2.Session())
+ txn, err = newTxn(tk.Session())
require.NoError(t, err)
for i := 1; i <= 1034; i++ {
- _, err := tbl2.AddRecord(tk2.Session().GetTableCtx(), types.MakeDatums(i, i, i))
+ _, err := tbl2.AddRecord(tk2.Session().GetTableCtx(), txn, types.MakeDatums(i, i, i))
require.NoError(t, err)
}
tk3 := testkit.NewTestKit(t, store)
diff --git a/pkg/ddl/schema_version.go b/pkg/ddl/schema_version.go
index 96fbdad427d81..a2464aec88c6f 100644
--- a/pkg/ddl/schema_version.go
+++ b/pkg/ddl/schema_version.go
@@ -313,8 +313,8 @@ func SetSchemaDiffForMultiInfos(diff *model.SchemaDiff, multiInfos ...schemaIDAn
}
// updateSchemaVersion increments the schema version by 1 and sets SchemaDiff.
-func updateSchemaVersion(d *ddlCtx, t *meta.Meta, job *model.Job, multiInfos ...schemaIDAndTableInfo) (int64, error) {
- schemaVersion, err := d.setSchemaVersion(job, d.store)
+func updateSchemaVersion(jobCtx *jobContext, t *meta.Meta, job *model.Job, multiInfos ...schemaIDAndTableInfo) (int64, error) {
+ schemaVersion, err := jobCtx.setSchemaVersion(job)
if err != nil {
return 0, errors.Trace(err)
}
@@ -372,7 +372,7 @@ func checkAllVersions(ctx context.Context, d *ddlCtx, job *model.Job, latestSche
})
// OwnerCheckAllVersions returns only when all TiDB schemas are synced(exclude the isolated TiDB).
- err := d.schemaSyncer.OwnerCheckAllVersions(ctx, job.ID, latestSchemaVersion)
+ err := d.schemaVerSyncer.OwnerCheckAllVersions(ctx, job.ID, latestSchemaVersion)
if err != nil {
logutil.DDLLogger().Info("wait latest schema version encounter error", zap.Int64("ver", latestSchemaVersion),
zap.Int64("jobID", job.ID), zap.Duration("take time", time.Since(timeStart)), zap.Error(err))
diff --git a/pkg/ddl/schematracker/BUILD.bazel b/pkg/ddl/schematracker/BUILD.bazel
index 8a721b4a0c524..a364d002f6e28 100644
--- a/pkg/ddl/schematracker/BUILD.bazel
+++ b/pkg/ddl/schematracker/BUILD.bazel
@@ -11,7 +11,8 @@ go_library(
visibility = ["//visibility:public"],
deps = [
"//pkg/ddl",
- "//pkg/ddl/syncer",
+ "//pkg/ddl/schemaver",
+ "//pkg/ddl/serverstate",
"//pkg/ddl/systable",
"//pkg/infoschema",
"//pkg/kv",
diff --git a/pkg/ddl/schematracker/checker.go b/pkg/ddl/schematracker/checker.go
index f8ffeb46919d1..721a9a25f78c4 100644
--- a/pkg/ddl/schematracker/checker.go
+++ b/pkg/ddl/schematracker/checker.go
@@ -20,11 +20,11 @@ import (
"fmt"
"strings"
"sync/atomic"
- "time"
"github.com/ngaut/pools"
"github.com/pingcap/tidb/pkg/ddl"
- "github.com/pingcap/tidb/pkg/ddl/syncer"
+ "github.com/pingcap/tidb/pkg/ddl/schemaver"
+ "github.com/pingcap/tidb/pkg/ddl/serverstate"
"github.com/pingcap/tidb/pkg/ddl/systable"
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/kv"
@@ -38,7 +38,6 @@ import (
"github.com/pingcap/tidb/pkg/statistics/handle"
"github.com/pingcap/tidb/pkg/store/helper"
"github.com/pingcap/tidb/pkg/store/mockstore"
- "github.com/pingcap/tidb/pkg/table"
pumpcli "github.com/pingcap/tidb/pkg/tidb-binlog/pump_client"
)
@@ -490,11 +489,6 @@ func (d *Checker) Start(ctxPool *pools.ResourcePool) error {
return d.realDDL.Start(ctxPool)
}
-// GetLease implements the DDL interface.
-func (d *Checker) GetLease() time.Duration {
- return d.realDDL.GetLease()
-}
-
// Stats implements the DDL interface.
func (d *Checker) Stats(vars *variable.SessionVars) (map[string]any, error) {
return d.realDDL.Stats(vars)
@@ -516,12 +510,12 @@ func (d *Checker) RegisterStatsHandle(h *handle.Handle) {
}
// SchemaSyncer implements the DDL interface.
-func (d *Checker) SchemaSyncer() syncer.SchemaSyncer {
+func (d *Checker) SchemaSyncer() schemaver.Syncer {
return d.realDDL.SchemaSyncer()
}
// StateSyncer implements the DDL interface.
-func (d *Checker) StateSyncer() syncer.StateSyncer {
+func (d *Checker) StateSyncer() serverstate.Syncer {
return d.realDDL.StateSyncer()
}
@@ -535,26 +529,11 @@ func (d *Checker) GetID() string {
return d.realDDL.GetID()
}
-// GetTableMaxHandle implements the DDL interface.
-func (d *Checker) GetTableMaxHandle(ctx *ddl.JobContext, startTS uint64, tbl table.PhysicalTable) (kv.Handle, bool, error) {
- return d.realDDL.GetTableMaxHandle(ctx, startTS, tbl)
-}
-
// SetBinlogClient implements the DDL interface.
func (d *Checker) SetBinlogClient(client *pumpcli.PumpsClient) {
d.realDDL.SetBinlogClient(client)
}
-// GetHook implements the DDL interface.
-func (d *Checker) GetHook() ddl.Callback {
- return d.realDDL.GetHook()
-}
-
-// SetHook implements the DDL interface.
-func (d *Checker) SetHook(h ddl.Callback) {
- d.realDDL.SetHook(h)
-}
-
// DoDDLJob implements the DDL interface.
func (d *Checker) DoDDLJob(ctx sessionctx.Context, job *model.Job) error {
de := d.realExecutor.(ddl.ExecutorForTest)
diff --git a/pkg/ddl/schematracker/dm_tracker.go b/pkg/ddl/schematracker/dm_tracker.go
index d66c66f780e57..5c01159af7092 100644
--- a/pkg/ddl/schematracker/dm_tracker.go
+++ b/pkg/ddl/schematracker/dm_tracker.go
@@ -38,11 +38,14 @@ import (
"github.com/pingcap/tidb/pkg/util/dbterror"
)
-// SchemaTracker is used to track schema changes by DM. It implements DDL interface and by applying DDL, it updates the
-// table structure to keep tracked with upstream changes.
-// It embeds an InfoStore which stores DBInfo and TableInfo. The DBInfo and TableInfo can be treated as immutable, so
-// after reading them by SchemaByName or TableByName, later modifications made by SchemaTracker will not change them.
-// SchemaTracker is not thread-safe.
+// SchemaTracker is used to track schema changes by DM. It implements
+// ddl.Executor interface and by applying DDL, it updates the table structure to
+// keep tracked with upstream changes.
+//
+// It embeds an InfoStore which stores DBInfo and TableInfo. The DBInfo and
+// TableInfo can be treated as immutable, so after reading them by SchemaByName
+// or TableByName, later modifications made by SchemaTracker will not change
+// them. SchemaTracker is not thread-safe.
type SchemaTracker struct {
*InfoStore
}
@@ -427,7 +430,6 @@ func (d *SchemaTracker) createIndex(
indexName,
false,
unique,
- false,
indexPartSpecifications,
indexOption,
model.StatePublic,
@@ -872,7 +874,6 @@ func (d *SchemaTracker) createPrimaryKey(
indexName,
true,
true,
- false,
indexPartSpecifications,
indexOption,
model.StatePublic,
diff --git a/pkg/ddl/schematracker/dm_tracker_test.go b/pkg/ddl/schematracker/dm_tracker_test.go
index b438d59a5d913..ca0ec21499173 100644
--- a/pkg/ddl/schematracker/dm_tracker_test.go
+++ b/pkg/ddl/schematracker/dm_tracker_test.go
@@ -86,6 +86,10 @@ func TestCreateTableLongIndex(t *testing.T) {
tracker := schematracker.NewSchemaTracker(2)
tracker.CreateTestDB(nil)
execCreate(t, tracker, sql)
+ sql2 := "create table test.t2 (c1 int, c2 blob, c3 varchar(64), unique index idx_c2(c2(555555)));"
+ execCreate(t, tracker, sql2)
+ sql3 := "create table test.t3 (c1 int, c2 blob, c3 varchar(64), index idx_c2_c3(c3, c2(555555)));"
+ execCreate(t, tracker, sql3)
}
func execAlter(t *testing.T, tracker schematracker.SchemaTracker, sql string) {
diff --git a/pkg/ddl/syncer/BUILD.bazel b/pkg/ddl/schemaver/BUILD.bazel
similarity index 85%
rename from pkg/ddl/syncer/BUILD.bazel
rename to pkg/ddl/schemaver/BUILD.bazel
index 9f125f09c825a..db28e7939d2b7 100644
--- a/pkg/ddl/syncer/BUILD.bazel
+++ b/pkg/ddl/schemaver/BUILD.bazel
@@ -1,12 +1,12 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
- name = "syncer",
+ name = "schemaver",
srcs = [
- "state_syncer.go",
+ "mem_syncer.go",
"syncer.go",
],
- importpath = "github.com/pingcap/tidb/pkg/ddl/syncer",
+ importpath = "github.com/pingcap/tidb/pkg/ddl/schemaver",
visibility = ["//visibility:public"],
deps = [
"//pkg/ddl/logutil",
@@ -21,22 +21,20 @@ go_library(
"@io_etcd_go_etcd_api_v3//mvccpb",
"@io_etcd_go_etcd_client_v3//:client",
"@io_etcd_go_etcd_client_v3//concurrency",
- "@org_uber_go_atomic//:atomic",
"@org_uber_go_zap//:zap",
],
)
go_test(
- name = "syncer_test",
+ name = "schemaver_test",
timeout = "short",
srcs = [
- "state_syncer_test.go",
"syncer_nokit_test.go",
"syncer_test.go",
],
- embed = [":syncer"],
+ embed = [":schemaver"],
flaky = True,
- shard_count = 6,
+ shard_count = 5,
deps = [
"//pkg/ddl",
"//pkg/ddl/util",
diff --git a/pkg/ddl/schemaver/mem_syncer.go b/pkg/ddl/schemaver/mem_syncer.go
new file mode 100644
index 0000000000000..41d7969de878e
--- /dev/null
+++ b/pkg/ddl/schemaver/mem_syncer.go
@@ -0,0 +1,140 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package schemaver
+
+import (
+ "context"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
+ "github.com/pingcap/tidb/pkg/sessionctx/variable"
+ clientv3 "go.etcd.io/etcd/client/v3"
+)
+
+const checkVersionsInterval = 2 * time.Millisecond
+
+// MemSyncer is in memory schema version syncer, used for uni-store where there is
+// only 1 TiDB instance. it's mainly for test.
+// exported for testing.
+type MemSyncer struct {
+ selfSchemaVersion int64
+ mdlSchemaVersions sync.Map
+ globalVerCh chan clientv3.WatchResponse
+ mockSession chan struct{}
+}
+
+var _ Syncer = &MemSyncer{}
+
+// NewMemSyncer creates a new memory Syncer.
+func NewMemSyncer() Syncer {
+ return &MemSyncer{}
+}
+
+// Init implements Syncer.Init interface.
+func (s *MemSyncer) Init(_ context.Context) error {
+ s.mdlSchemaVersions = sync.Map{}
+ s.globalVerCh = make(chan clientv3.WatchResponse, 1)
+ s.mockSession = make(chan struct{}, 1)
+ return nil
+}
+
+// GlobalVersionCh implements Syncer.GlobalVersionCh interface.
+func (s *MemSyncer) GlobalVersionCh() clientv3.WatchChan {
+ return s.globalVerCh
+}
+
+// WatchGlobalSchemaVer implements Syncer.WatchGlobalSchemaVer interface.
+func (*MemSyncer) WatchGlobalSchemaVer(context.Context) {}
+
+// UpdateSelfVersion implements Syncer.UpdateSelfVersion interface.
+func (s *MemSyncer) UpdateSelfVersion(_ context.Context, jobID int64, version int64) error {
+ failpoint.Inject("mockUpdateMDLToETCDError", func(val failpoint.Value) {
+ if val.(bool) {
+ failpoint.Return(errors.New("mock update mdl to etcd error"))
+ }
+ })
+ if variable.EnableMDL.Load() {
+ s.mdlSchemaVersions.Store(jobID, version)
+ } else {
+ atomic.StoreInt64(&s.selfSchemaVersion, version)
+ }
+ return nil
+}
+
+// Done implements Syncer.Done interface.
+func (s *MemSyncer) Done() <-chan struct{} {
+ return s.mockSession
+}
+
+// CloseSession mockSession, it is exported for testing.
+func (s *MemSyncer) CloseSession() {
+ close(s.mockSession)
+}
+
+// Restart implements Syncer.Restart interface.
+func (s *MemSyncer) Restart(_ context.Context) error {
+ s.mockSession = make(chan struct{}, 1)
+ return nil
+}
+
+// OwnerUpdateGlobalVersion implements Syncer.OwnerUpdateGlobalVersion interface.
+func (s *MemSyncer) OwnerUpdateGlobalVersion(_ context.Context, _ int64) error {
+ select {
+ case s.globalVerCh <- clientv3.WatchResponse{}:
+ default:
+ }
+ return nil
+}
+
+// OwnerCheckAllVersions implements Syncer.OwnerCheckAllVersions interface.
+func (s *MemSyncer) OwnerCheckAllVersions(ctx context.Context, jobID int64, latestVer int64) error {
+ ticker := time.NewTicker(checkVersionsInterval)
+ defer ticker.Stop()
+
+ failpoint.Inject("mockOwnerCheckAllVersionSlow", func(val failpoint.Value) {
+ if v, ok := val.(int); ok && v == int(jobID) {
+ time.Sleep(2 * time.Second)
+ }
+ })
+
+ for {
+ select {
+ case <-ctx.Done():
+ return errors.Trace(ctx.Err())
+ case <-ticker.C:
+ if variable.EnableMDL.Load() {
+ ver, ok := s.mdlSchemaVersions.Load(jobID)
+ if ok && ver.(int64) >= latestVer {
+ return nil
+ }
+ } else {
+ ver := atomic.LoadInt64(&s.selfSchemaVersion)
+ if ver >= latestVer {
+ return nil
+ }
+ }
+ }
+ }
+}
+
+// SyncJobSchemaVerLoop implements Syncer.SyncJobSchemaVerLoop interface.
+func (*MemSyncer) SyncJobSchemaVerLoop(context.Context) {
+}
+
+// Close implements Syncer.Close interface.
+func (*MemSyncer) Close() {}
diff --git a/pkg/ddl/syncer/syncer.go b/pkg/ddl/schemaver/syncer.go
similarity index 81%
rename from pkg/ddl/syncer/syncer.go
rename to pkg/ddl/schemaver/syncer.go
index 818d4747eed06..b1df3c9b50fcb 100644
--- a/pkg/ddl/syncer/syncer.go
+++ b/pkg/ddl/schemaver/syncer.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package syncer
+package schemaver
import (
"context"
@@ -57,58 +57,13 @@ var (
CheckVersFirstWaitTime = 50 * time.Millisecond
)
-// Watcher is responsible for watching the etcd path related operations.
-type Watcher interface {
- // WatchChan returns the chan for watching etcd path.
- WatchChan() clientv3.WatchChan
- // Watch watches the etcd path.
- Watch(ctx context.Context, etcdCli *clientv3.Client, path string)
- // Rewatch rewatches the etcd path.
- Rewatch(ctx context.Context, etcdCli *clientv3.Client, path string)
-}
-
-type watcher struct {
- sync.RWMutex
- wCh clientv3.WatchChan
-}
-
-// WatchChan implements SyncerWatch.WatchChan interface.
-func (w *watcher) WatchChan() clientv3.WatchChan {
- w.RLock()
- defer w.RUnlock()
- return w.wCh
-}
-
-// Watch implements SyncerWatch.Watch interface.
-func (w *watcher) Watch(ctx context.Context, etcdCli *clientv3.Client, path string) {
- w.Lock()
- w.wCh = etcdCli.Watch(ctx, path)
- w.Unlock()
-}
-
-// Rewatch implements SyncerWatch.Rewatch interface.
-func (w *watcher) Rewatch(ctx context.Context, etcdCli *clientv3.Client, path string) {
- startTime := time.Now()
- // Make sure the wCh doesn't receive the information of 'close' before we finish the rewatch.
- w.Lock()
- w.wCh = nil
- w.Unlock()
-
- go func() {
- defer func() {
- metrics.DeploySyncerHistogram.WithLabelValues(metrics.SyncerRewatch, metrics.RetLabel(nil)).Observe(time.Since(startTime).Seconds())
- }()
- wCh := etcdCli.Watch(ctx, path)
-
- w.Lock()
- w.wCh = wCh
- w.Unlock()
- logutil.DDLLogger().Info("syncer rewatch global info finished")
- }()
-}
-
-// SchemaSyncer is used to synchronize schema version between the DDL worker leader and followers through etcd.
-type SchemaSyncer interface {
+// Syncer is used to synchronize schema version between the DDL owner and follower.
+// DDL owner and follower only depends on a subset of the methods of Syncer.
+// DDL owner will use this interface to update the global schema version, and wait
+// all followers to update schema to the target version.
+// followers use it to receive version change events, reload schema and update their
+// version.
+type Syncer interface {
// Init sets the global schema version path to etcd if it isn't exist,
// then watch this path, and initializes the self schema version to etcd.
Init(ctx context.Context) error
@@ -130,7 +85,7 @@ type SchemaSyncer interface {
OwnerCheckAllVersions(ctx context.Context, jobID int64, latestVer int64) error
// SyncJobSchemaVerLoop syncs the schema versions on all TiDB nodes for DDL jobs.
SyncJobSchemaVerLoop(ctx context.Context)
- // Close ends SchemaSyncer.
+ // Close ends Syncer.
Close()
}
@@ -210,11 +165,12 @@ func (v *nodeVersions) getMatchFn() func(map[string]int64) bool {
return v.onceMatchFn
}
-type schemaVersionSyncer struct {
+// etcdSyncer is a Syncer based on etcd. used for TiKV store.
+type etcdSyncer struct {
selfSchemaVerPath string
etcdCli *clientv3.Client
session unsafe.Pointer
- globalVerWatcher watcher
+ globalVerWatcher util.Watcher
ddlID string
mu sync.RWMutex
@@ -222,11 +178,12 @@ type schemaVersionSyncer struct {
jobNodeVerPrefix string
}
-// NewSchemaSyncer creates a new SchemaSyncer.
-func NewSchemaSyncer(etcdCli *clientv3.Client, id string) SchemaSyncer {
- return &schemaVersionSyncer{
+// NewEtcdSyncer creates a new Syncer.
+func NewEtcdSyncer(etcdCli *clientv3.Client, id string) Syncer {
+ return &etcdSyncer{
etcdCli: etcdCli,
selfSchemaVerPath: fmt.Sprintf("%s/%s", util.DDLAllSchemaVersions, id),
+ globalVerWatcher: util.NewWatcher(),
ddlID: id,
jobNodeVersions: make(map[int64]*nodeVersions),
@@ -234,8 +191,8 @@ func NewSchemaSyncer(etcdCli *clientv3.Client, id string) SchemaSyncer {
}
}
-// Init implements SchemaSyncer.Init interface.
-func (s *schemaVersionSyncer) Init(ctx context.Context) error {
+// Init implements Syncer.Init interface.
+func (s *etcdSyncer) Init(ctx context.Context) error {
startTime := time.Now()
var err error
defer func() {
@@ -263,16 +220,16 @@ func (s *schemaVersionSyncer) Init(ctx context.Context) error {
return errors.Trace(err)
}
-func (s *schemaVersionSyncer) loadSession() *concurrency.Session {
+func (s *etcdSyncer) loadSession() *concurrency.Session {
return (*concurrency.Session)(atomic.LoadPointer(&s.session))
}
-func (s *schemaVersionSyncer) storeSession(session *concurrency.Session) {
+func (s *etcdSyncer) storeSession(session *concurrency.Session) {
atomic.StorePointer(&s.session, (unsafe.Pointer)(session))
}
-// Done implements SchemaSyncer.Done interface.
-func (s *schemaVersionSyncer) Done() <-chan struct{} {
+// Done implements Syncer.Done interface.
+func (s *etcdSyncer) Done() <-chan struct{} {
failpoint.Inject("ErrorMockSessionDone", func(val failpoint.Value) {
if val.(bool) {
err := s.loadSession().Close()
@@ -283,8 +240,8 @@ func (s *schemaVersionSyncer) Done() <-chan struct{} {
return s.loadSession().Done()
}
-// Restart implements SchemaSyncer.Restart interface.
-func (s *schemaVersionSyncer) Restart(ctx context.Context) error {
+// Restart implements Syncer.Restart interface.
+func (s *etcdSyncer) Restart(ctx context.Context) error {
startTime := time.Now()
var err error
defer func() {
@@ -307,18 +264,18 @@ func (s *schemaVersionSyncer) Restart(ctx context.Context) error {
return errors.Trace(err)
}
-// GlobalVersionCh implements SchemaSyncer.GlobalVersionCh interface.
-func (s *schemaVersionSyncer) GlobalVersionCh() clientv3.WatchChan {
+// GlobalVersionCh implements Syncer.GlobalVersionCh interface.
+func (s *etcdSyncer) GlobalVersionCh() clientv3.WatchChan {
return s.globalVerWatcher.WatchChan()
}
-// WatchGlobalSchemaVer implements SchemaSyncer.WatchGlobalSchemaVer interface.
-func (s *schemaVersionSyncer) WatchGlobalSchemaVer(ctx context.Context) {
+// WatchGlobalSchemaVer implements Syncer.WatchGlobalSchemaVer interface.
+func (s *etcdSyncer) WatchGlobalSchemaVer(ctx context.Context) {
s.globalVerWatcher.Rewatch(ctx, s.etcdCli, util.DDLGlobalSchemaVersion)
}
-// UpdateSelfVersion implements SchemaSyncer.UpdateSelfVersion interface.
-func (s *schemaVersionSyncer) UpdateSelfVersion(ctx context.Context, jobID int64, version int64) error {
+// UpdateSelfVersion implements Syncer.UpdateSelfVersion interface.
+func (s *etcdSyncer) UpdateSelfVersion(ctx context.Context, jobID int64, version int64) error {
startTime := time.Now()
ver := strconv.FormatInt(version, 10)
var err error
@@ -336,8 +293,8 @@ func (s *schemaVersionSyncer) UpdateSelfVersion(ctx context.Context, jobID int64
return errors.Trace(err)
}
-// OwnerUpdateGlobalVersion implements SchemaSyncer.OwnerUpdateGlobalVersion interface.
-func (s *schemaVersionSyncer) OwnerUpdateGlobalVersion(ctx context.Context, version int64) error {
+// OwnerUpdateGlobalVersion implements Syncer.OwnerUpdateGlobalVersion interface.
+func (s *etcdSyncer) OwnerUpdateGlobalVersion(ctx context.Context, version int64) error {
startTime := time.Now()
ver := strconv.FormatInt(version, 10)
// TODO: If the version is larger than the original global version, we need set the version.
@@ -348,7 +305,7 @@ func (s *schemaVersionSyncer) OwnerUpdateGlobalVersion(ctx context.Context, vers
}
// removeSelfVersionPath remove the self path from etcd.
-func (s *schemaVersionSyncer) removeSelfVersionPath() error {
+func (s *etcdSyncer) removeSelfVersionPath() error {
startTime := time.Now()
var err error
defer func() {
@@ -359,8 +316,8 @@ func (s *schemaVersionSyncer) removeSelfVersionPath() error {
return errors.Trace(err)
}
-// OwnerCheckAllVersions implements SchemaSyncer.OwnerCheckAllVersions interface.
-func (s *schemaVersionSyncer) OwnerCheckAllVersions(ctx context.Context, jobID int64, latestVer int64) error {
+// OwnerCheckAllVersions implements Syncer.OwnerCheckAllVersions interface.
+func (s *etcdSyncer) OwnerCheckAllVersions(ctx context.Context, jobID int64, latestVer int64) error {
startTime := time.Now()
if !variable.EnableMDL.Load() {
time.Sleep(CheckVersFirstWaitTime)
@@ -478,8 +435,8 @@ func (s *schemaVersionSyncer) OwnerCheckAllVersions(ctx context.Context, jobID i
}
}
-// SyncJobSchemaVerLoop implements SchemaSyncer.SyncJobSchemaVerLoop interface.
-func (s *schemaVersionSyncer) SyncJobSchemaVerLoop(ctx context.Context) {
+// SyncJobSchemaVerLoop implements Syncer.SyncJobSchemaVerLoop interface.
+func (s *etcdSyncer) SyncJobSchemaVerLoop(ctx context.Context) {
for {
s.syncJobSchemaVer(ctx)
logutil.DDLLogger().Info("schema version sync loop interrupted, retrying...")
@@ -491,7 +448,7 @@ func (s *schemaVersionSyncer) SyncJobSchemaVerLoop(ctx context.Context) {
}
}
-func (s *schemaVersionSyncer) syncJobSchemaVer(ctx context.Context) {
+func (s *etcdSyncer) syncJobSchemaVer(ctx context.Context) {
resp, err := s.etcdCli.Get(ctx, s.jobNodeVerPrefix, clientv3.WithPrefix())
if err != nil {
logutil.DDLLogger().Info("get all job versions failed", zap.Error(err))
@@ -543,7 +500,7 @@ func (s *schemaVersionSyncer) syncJobSchemaVer(ctx context.Context) {
}
}
-func (s *schemaVersionSyncer) handleJobSchemaVerKV(kv *mvccpb.KeyValue, tp mvccpb.Event_EventType) {
+func (s *etcdSyncer) handleJobSchemaVerKV(kv *mvccpb.KeyValue, tp mvccpb.Event_EventType) {
jobID, tidbID, schemaVer, valid := decodeJobVersionEvent(kv, tp, s.jobNodeVerPrefix)
if !valid {
logutil.DDLLogger().Error("invalid job version kv", zap.Stringer("kv", kv), zap.Stringer("type", tp))
@@ -570,7 +527,7 @@ func (s *schemaVersionSyncer) handleJobSchemaVerKV(kv *mvccpb.KeyValue, tp mvccp
}
}
-func (s *schemaVersionSyncer) jobSchemaVerMatchOrSet(jobID int64, matchFn func(map[string]int64) bool) *nodeVersions {
+func (s *etcdSyncer) jobSchemaVerMatchOrSet(jobID int64, matchFn func(map[string]int64) bool) *nodeVersions {
s.mu.Lock()
defer s.mu.Unlock()
@@ -621,7 +578,7 @@ func isUpdatedLatestVersion(key, val string, latestVer int64, notMatchVerCnt, in
return true
}
-func (s *schemaVersionSyncer) Close() {
+func (s *etcdSyncer) Close() {
err := s.removeSelfVersionPath()
if err != nil {
logutil.DDLLogger().Error("remove self version path failed", zap.Error(err))
diff --git a/pkg/ddl/syncer/syncer_nokit_test.go b/pkg/ddl/schemaver/syncer_nokit_test.go
similarity index 97%
rename from pkg/ddl/syncer/syncer_nokit_test.go
rename to pkg/ddl/schemaver/syncer_nokit_test.go
index 15d0cbf0eac44..21dc98ad7cf2a 100644
--- a/pkg/ddl/syncer/syncer_nokit_test.go
+++ b/pkg/ddl/schemaver/syncer_nokit_test.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package syncer
+package schemaver
import (
"context"
@@ -95,7 +95,7 @@ func TestSyncJobSchemaVerLoop(t *testing.T) {
etcdCli := mockCluster.RandClient()
_, err := etcdCli.Put(ctx, util.DDLAllSchemaVersionsByJob+"/1/aa", "123")
require.NoError(t, err)
- s := NewSchemaSyncer(etcdCli, "1111").(*schemaVersionSyncer)
+ s := NewEtcdSyncer(etcdCli, "1111").(*etcdSyncer)
var wg sync.WaitGroup
wg.Add(1)
go func() {
@@ -150,9 +150,9 @@ func TestSyncJobSchemaVerLoop(t *testing.T) {
require.NoError(t, err)
// job 3 is matched after restart from a compaction error
- require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/syncer/mockCompaction", `1*return(true)`))
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/schemaver/mockCompaction", `1*return(true)`))
t.Cleanup(func() {
- require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/syncer/mockCompaction"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/schemaver/mockCompaction"))
})
notifyCh = make(chan struct{}, 1)
item = s.jobSchemaVerMatchOrSet(3, func(m map[string]int64) bool {
diff --git a/pkg/ddl/syncer/syncer_test.go b/pkg/ddl/schemaver/syncer_test.go
similarity index 96%
rename from pkg/ddl/syncer/syncer_test.go
rename to pkg/ddl/schemaver/syncer_test.go
index 4add1837ce0c5..6f16930c916f2 100644
--- a/pkg/ddl/syncer/syncer_test.go
+++ b/pkg/ddl/schemaver/syncer_test.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package syncer_test
+package schemaver_test
import (
"context"
@@ -24,7 +24,7 @@ import (
"github.com/pingcap/errors"
. "github.com/pingcap/tidb/pkg/ddl"
- "github.com/pingcap/tidb/pkg/ddl/syncer"
+ "github.com/pingcap/tidb/pkg/ddl/schemaver"
util2 "github.com/pingcap/tidb/pkg/ddl/util"
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/parser/terror"
@@ -51,10 +51,10 @@ func TestSyncerSimple(t *testing.T) {
}
integration.BeforeTestExternal(t)
- origin := syncer.CheckVersFirstWaitTime
- syncer.CheckVersFirstWaitTime = 0
+ origin := schemaver.CheckVersFirstWaitTime
+ schemaver.CheckVersFirstWaitTime = 0
defer func() {
- syncer.CheckVersFirstWaitTime = origin
+ schemaver.CheckVersFirstWaitTime = origin
}()
store, err := mockstore.NewMockStore()
@@ -91,7 +91,7 @@ func TestSyncerSimple(t *testing.T) {
defer d.SchemaSyncer().Close()
key := util2.DDLAllSchemaVersions + "/" + d.OwnerManager().ID()
- checkRespKV(t, 1, key, syncer.InitialVersion, resp.Kvs...)
+ checkRespKV(t, 1, key, schemaver.InitialVersion, resp.Kvs...)
ic2 := infoschema.NewCache(nil, 2)
ic2.Insert(infoschema.MockInfoSchemaWithSchemaVer(nil, 0), 0)
diff --git a/pkg/ddl/sequence.go b/pkg/ddl/sequence.go
index 8e5a0c77fad9b..9eff78a2cc0ff 100644
--- a/pkg/ddl/sequence.go
+++ b/pkg/ddl/sequence.go
@@ -27,7 +27,7 @@ import (
"github.com/pingcap/tidb/pkg/util/mathutil"
)
-func onCreateSequence(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onCreateSequence(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
schemaID := job.SchemaID
tbInfo := &model.TableInfo{}
if err := job.DecodeArgs(tbInfo); err != nil {
@@ -37,7 +37,7 @@ func onCreateSequence(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ err
}
tbInfo.State = model.StateNone
- err := checkTableNotExists(d, schemaID, tbInfo.Name.L)
+ err := checkTableNotExists(jobCtx.infoCache, schemaID, tbInfo.Name.L)
if err != nil {
if infoschema.ErrDatabaseNotExists.Equal(err) || infoschema.ErrTableExists.Equal(err) {
job.State = model.JobStateCancelled
@@ -50,7 +50,7 @@ func onCreateSequence(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ err
return ver, errors.Trace(err)
}
- ver, err = updateSchemaVersion(d, t, job)
+ ver, err = updateSchemaVersion(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -75,7 +75,7 @@ func createSequenceWithCheck(t *meta.Meta, job *model.Job, tbInfo *model.TableIn
} else {
sequenceBase = tbInfo.Sequence.Start + 1
}
- return t.CreateSequenceAndSetSeqValue(job.SchemaID, job.SchemaName, tbInfo, sequenceBase)
+ return t.CreateSequenceAndSetSeqValue(job.SchemaID, tbInfo, sequenceBase)
default:
return dbterror.ErrInvalidDDLState.GenWithStackByArgs("sequence", tbInfo.State)
}
@@ -231,7 +231,7 @@ func alterSequenceOptions(sequenceOptions []*ast.SequenceOption, ident ast.Ident
return false, 0, nil
}
-func onAlterSequence(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onAlterSequence(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
schemaID := job.SchemaID
var (
sequenceOpts []*ast.SequenceOption
@@ -279,7 +279,7 @@ func onAlterSequence(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ erro
// Store the sequence info into kv.
// Set shouldUpdateVer always to be true even altering doesn't take effect, since some tools like drainer won't take
// care of SchemaVersion=0.
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
diff --git a/pkg/ddl/serverstate/BUILD.bazel b/pkg/ddl/serverstate/BUILD.bazel
new file mode 100644
index 0000000000000..9c983d8036b30
--- /dev/null
+++ b/pkg/ddl/serverstate/BUILD.bazel
@@ -0,0 +1,45 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+ name = "serverstate",
+ srcs = [
+ "mem_syncer.go",
+ "syncer.go",
+ ],
+ importpath = "github.com/pingcap/tidb/pkg/ddl/serverstate",
+ visibility = ["//visibility:public"],
+ deps = [
+ "//pkg/ddl/logutil",
+ "//pkg/ddl/util",
+ "//pkg/metrics",
+ "//pkg/util",
+ "@com_github_pingcap_errors//:errors",
+ "@com_github_pingcap_failpoint//:failpoint",
+ "@io_etcd_go_etcd_api_v3//mvccpb",
+ "@io_etcd_go_etcd_client_v3//:client",
+ "@io_etcd_go_etcd_client_v3//concurrency",
+ "@org_uber_go_atomic//:atomic",
+ "@org_uber_go_zap//:zap",
+ ],
+)
+
+go_test(
+ name = "serverstate_test",
+ timeout = "short",
+ srcs = ["syncer_test.go"],
+ flaky = True,
+ deps = [
+ ":serverstate",
+ "//pkg/ddl",
+ "//pkg/ddl/schemaver",
+ "//pkg/ddl/util",
+ "//pkg/infoschema",
+ "//pkg/session",
+ "//pkg/sessionctx/variable",
+ "//pkg/store/mockstore",
+ "//pkg/util",
+ "@com_github_stretchr_testify//require",
+ "@io_etcd_go_etcd_api_v3//mvccpb",
+ "@io_etcd_go_etcd_tests_v3//integration",
+ ],
+)
diff --git a/pkg/ddl/serverstate/mem_syncer.go b/pkg/ddl/serverstate/mem_syncer.go
new file mode 100644
index 0000000000000..6489e99e057e1
--- /dev/null
+++ b/pkg/ddl/serverstate/mem_syncer.go
@@ -0,0 +1,81 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package serverstate
+
+import (
+ "context"
+
+ "github.com/pingcap/failpoint"
+ clientv3 "go.etcd.io/etcd/client/v3"
+ atomicutil "go.uber.org/atomic"
+)
+
+// NewMemSyncer creates a new memSyncer.
+func NewMemSyncer() Syncer {
+ return &memSyncer{}
+}
+
+// clusterState mocks cluster state.
+// We move it from memSyncer to here. Because we want to make it unaffected by ddl close.
+var clusterState *atomicutil.Pointer[StateInfo]
+
+// memSyncer is memory based server state syncer used for uni-store.
+// it's mainly for test as there is only 1 instance for uni-store.
+type memSyncer struct {
+ globalVerCh chan clientv3.WatchResponse
+ mockSession chan struct{}
+}
+
+// Init implements Syncer.Init interface.
+func (s *memSyncer) Init(context.Context) error {
+ s.globalVerCh = make(chan clientv3.WatchResponse, 1)
+ s.mockSession = make(chan struct{}, 1)
+ state := NewStateInfo(StateNormalRunning)
+ if clusterState == nil {
+ clusterState = atomicutil.NewPointer(state)
+ }
+ return nil
+}
+
+// UpdateGlobalState implements Syncer.UpdateGlobalState interface.
+func (s *memSyncer) UpdateGlobalState(_ context.Context, stateInfo *StateInfo) error {
+ failpoint.Inject("mockUpgradingState", func(val failpoint.Value) {
+ if val.(bool) {
+ clusterState.Store(stateInfo)
+ failpoint.Return(nil)
+ }
+ })
+ s.globalVerCh <- clientv3.WatchResponse{}
+ clusterState.Store(stateInfo)
+ return nil
+}
+
+// GetGlobalState implements Syncer.GetGlobalState interface.
+func (*memSyncer) GetGlobalState(context.Context) (*StateInfo, error) {
+ return clusterState.Load(), nil
+}
+
+// IsUpgradingState implements Syncer.IsUpgradingState interface.
+func (*memSyncer) IsUpgradingState() bool {
+ return clusterState.Load().State == StateUpgrading
+}
+
+// WatchChan implements Syncer.WatchChan interface.
+func (s *memSyncer) WatchChan() clientv3.WatchChan {
+ return s.globalVerCh
+}
+
+// Rewatch implements Syncer.Rewatch interface.
+func (*memSyncer) Rewatch(context.Context) {}
diff --git a/pkg/ddl/syncer/state_syncer.go b/pkg/ddl/serverstate/syncer.go
similarity index 77%
rename from pkg/ddl/syncer/state_syncer.go
rename to pkg/ddl/serverstate/syncer.go
index d65a7b6585911..f603ea8b0e084 100644
--- a/pkg/ddl/syncer/state_syncer.go
+++ b/pkg/ddl/serverstate/syncer.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package syncer
+package serverstate
import (
"context"
@@ -33,6 +33,7 @@ import (
)
const (
+ keyOpDefaultRetryCnt = 3
// keyOpDefaultTimeout is the default time out for etcd store.
keyOpDefaultTimeout = 1 * time.Second
statePrompt = "global-state-syncer"
@@ -42,8 +43,10 @@ const (
StateNormalRunning = ""
)
-// StateSyncer is used to synchronize schema version between the DDL worker leader and followers through etcd.
-type StateSyncer interface {
+// Syncer is used to synchronize server state.
+// currently there are only 2 states: running/upgrading, and is only used for the
+// 'smooth upgrade' feature.
+type Syncer interface {
// Init sets the global schema version path to etcd if it isn't exist,
// then watch this path, and initializes the self schema version to etcd.
Init(ctx context.Context) error
@@ -82,27 +85,29 @@ func (info *StateInfo) Unmarshal(v []byte) error {
return json.Unmarshal(v, info)
}
-type serverStateSyncer struct {
+// etcdSyncer is a Syncer implementation based on etcd.
+type etcdSyncer struct {
etcdPath string
prompt string
etcdCli *clientv3.Client
session *concurrency.Session
clusterState *atomicutil.Pointer[StateInfo]
- globalStateWatcher watcher
+ globalStateWatcher util.Watcher
}
-// NewStateSyncer creates a new StateSyncer.
-func NewStateSyncer(etcdCli *clientv3.Client, etcdPath string) StateSyncer {
- return &serverStateSyncer{
- etcdCli: etcdCli,
- etcdPath: etcdPath,
- clusterState: atomicutil.NewPointer(NewStateInfo(StateNormalRunning)),
- prompt: statePrompt,
+// NewEtcdSyncer creates a new Syncer.
+func NewEtcdSyncer(etcdCli *clientv3.Client, etcdPath string) Syncer {
+ return &etcdSyncer{
+ etcdCli: etcdCli,
+ etcdPath: etcdPath,
+ clusterState: atomicutil.NewPointer(NewStateInfo(StateNormalRunning)),
+ prompt: statePrompt,
+ globalStateWatcher: util.NewWatcher(),
}
}
-// Init implements StateSyncer.Init interface.
-func (s *serverStateSyncer) Init(ctx context.Context) error {
+// Init implements Syncer.Init interface.
+func (s *etcdSyncer) Init(ctx context.Context) error {
startTime := time.Now()
var err error
defer func() {
@@ -125,22 +130,22 @@ func (s *serverStateSyncer) Init(ctx context.Context) error {
return errors.Trace(err)
}
-// WatchChan implements StateSyncer.WatchChan interface.
-func (s *serverStateSyncer) WatchChan() clientv3.WatchChan {
+// WatchChan implements Syncer.WatchChan interface.
+func (s *etcdSyncer) WatchChan() clientv3.WatchChan {
return s.globalStateWatcher.WatchChan()
}
-// Rewatch implements StateSyncer.Rewatch interface.
-func (s *serverStateSyncer) Rewatch(ctx context.Context) {
+// Rewatch implements Syncer.Rewatch interface.
+func (s *etcdSyncer) Rewatch(ctx context.Context) {
s.globalStateWatcher.Rewatch(ctx, s.etcdCli, s.etcdPath)
}
-// IsUpgradingState implements StateSyncer.IsUpgradingState interface.
-func (s *serverStateSyncer) IsUpgradingState() bool {
+// IsUpgradingState implements Syncer.IsUpgradingState interface.
+func (s *etcdSyncer) IsUpgradingState() bool {
return s.clusterState.Load().State == StateUpgrading
}
-func (*serverStateSyncer) getKeyValue(ctx context.Context, etcdCli *clientv3.Client, key string, retryCnt int, timeout time.Duration, opts ...clientv3.OpOption) ([]*mvccpb.KeyValue, error) {
+func (*etcdSyncer) getKeyValue(ctx context.Context, etcdCli *clientv3.Client, key string, retryCnt int, timeout time.Duration, opts ...clientv3.OpOption) ([]*mvccpb.KeyValue, error) {
var err error
var resp *clientv3.GetResponse
for i := 0; i < retryCnt; i++ {
@@ -167,8 +172,8 @@ func (*serverStateSyncer) getKeyValue(ctx context.Context, etcdCli *clientv3.Cli
return nil, errors.Trace(err)
}
-// GetGlobalState implements StateSyncer.GetGlobalState interface.
-func (s *serverStateSyncer) GetGlobalState(ctx context.Context) (*StateInfo, error) {
+// GetGlobalState implements Syncer.GetGlobalState interface.
+func (s *etcdSyncer) GetGlobalState(ctx context.Context) (*StateInfo, error) {
startTime := time.Now()
kvs, err := s.getKeyValue(ctx, s.etcdCli, s.etcdPath, keyOpDefaultRetryCnt, keyOpDefaultTimeout)
if err != nil {
@@ -192,8 +197,8 @@ func (s *serverStateSyncer) GetGlobalState(ctx context.Context) (*StateInfo, err
return state, nil
}
-// UpdateGlobalState implements StateSyncer.UpdateGlobalState interface.
-func (s *serverStateSyncer) UpdateGlobalState(ctx context.Context, stateInfo *StateInfo) error {
+// UpdateGlobalState implements Syncer.UpdateGlobalState interface.
+func (s *etcdSyncer) UpdateGlobalState(ctx context.Context, stateInfo *StateInfo) error {
startTime := time.Now()
stateStr, err := stateInfo.Marshal()
if err != nil {
diff --git a/pkg/ddl/syncer/state_syncer_test.go b/pkg/ddl/serverstate/syncer_test.go
similarity index 76%
rename from pkg/ddl/syncer/state_syncer_test.go
rename to pkg/ddl/serverstate/syncer_test.go
index 8a6c3952f694e..0c24430ce6bc9 100644
--- a/pkg/ddl/syncer/state_syncer_test.go
+++ b/pkg/ddl/serverstate/syncer_test.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package syncer_test
+package serverstate_test
import (
"context"
@@ -21,7 +21,8 @@ import (
"time"
. "github.com/pingcap/tidb/pkg/ddl"
- "github.com/pingcap/tidb/pkg/ddl/syncer"
+ "github.com/pingcap/tidb/pkg/ddl/schemaver"
+ "github.com/pingcap/tidb/pkg/ddl/serverstate"
util2 "github.com/pingcap/tidb/pkg/ddl/util"
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/session"
@@ -29,9 +30,22 @@ import (
"github.com/pingcap/tidb/pkg/store/mockstore"
"github.com/pingcap/tidb/pkg/util"
"github.com/stretchr/testify/require"
+ "go.etcd.io/etcd/api/v3/mvccpb"
"go.etcd.io/etcd/tests/v3/integration"
)
+func checkRespKV(t *testing.T, kvCount int, key, val string, kvs ...*mvccpb.KeyValue) {
+ require.Len(t, kvs, kvCount)
+
+ if kvCount == 0 {
+ return
+ }
+
+ kv := kvs[0]
+ require.Equal(t, key, string(kv.Key))
+ require.Equal(t, val, string(kv.Value))
+}
+
func TestStateSyncerSimple(t *testing.T) {
variable.EnableMDL.Store(false)
if runtime.GOOS == "windows" {
@@ -39,10 +53,10 @@ func TestStateSyncerSimple(t *testing.T) {
}
integration.BeforeTestExternal(t)
- origin := syncer.CheckVersFirstWaitTime
- syncer.CheckVersFirstWaitTime = 0
+ origin := schemaver.CheckVersFirstWaitTime
+ schemaver.CheckVersFirstWaitTime = 0
defer func() {
- syncer.CheckVersFirstWaitTime = origin
+ schemaver.CheckVersFirstWaitTime = origin
}()
store, err := mockstore.NewMockStore()
@@ -62,7 +76,7 @@ func TestStateSyncerSimple(t *testing.T) {
ctx,
WithEtcdClient(cli),
WithStore(store),
- WithLease(testLease),
+ WithLease(5*time.Millisecond),
WithInfoCache(ic),
WithSchemaLoader(domain),
)
@@ -76,14 +90,14 @@ func TestStateSyncerSimple(t *testing.T) {
// for GetGlobalState
// for the initial state
- stateInfo := &syncer.StateInfo{State: syncer.StateNormalRunning}
+ stateInfo := &serverstate.StateInfo{State: serverstate.StateNormalRunning}
respState, err := d.StateSyncer().GetGlobalState(ctx)
require.Nil(t, err)
require.Equal(t, stateInfo, respState)
require.False(t, d.StateSyncer().IsUpgradingState())
// for watchCh
var checkErr string
- stateInfo.State = syncer.StateUpgrading
+ stateInfo.State = serverstate.StateUpgrading
stateInfoByte, err := stateInfo.Marshal()
require.Nil(t, err)
checkValue := func() {
@@ -94,7 +108,7 @@ func TestStateSyncerSimple(t *testing.T) {
return
}
checkRespKV(t, 1, util2.ServerGlobalState, string(stateInfoByte), resp.Events[0].Kv)
- if stateInfo.State == syncer.StateUpgrading {
+ if stateInfo.State == serverstate.StateUpgrading {
require.False(t, d.StateSyncer().IsUpgradingState())
} else {
require.True(t, d.StateSyncer().IsUpgradingState())
@@ -103,7 +117,7 @@ func TestStateSyncerSimple(t *testing.T) {
respState, err := d.StateSyncer().GetGlobalState(ctx)
require.Nil(t, err)
require.Equal(t, stateInfo, respState)
- if stateInfo.State == syncer.StateUpgrading {
+ if stateInfo.State == serverstate.StateUpgrading {
require.True(t, d.StateSyncer().IsUpgradingState())
} else {
require.False(t, d.StateSyncer().IsUpgradingState())
@@ -117,15 +131,15 @@ func TestStateSyncerSimple(t *testing.T) {
// for update UpdateGlobalState
// for StateUpgrading
wg.Run(checkValue)
- require.NoError(t, d.StateSyncer().UpdateGlobalState(ctx, &syncer.StateInfo{State: syncer.StateUpgrading}))
+ require.NoError(t, d.StateSyncer().UpdateGlobalState(ctx, &serverstate.StateInfo{State: serverstate.StateUpgrading}))
wg.Wait()
require.Equal(t, "", checkErr)
// for StateNormalRunning
- stateInfo.State = syncer.StateNormalRunning
+ stateInfo.State = serverstate.StateNormalRunning
stateInfoByte, err = stateInfo.Marshal()
require.Nil(t, err)
wg.Run(checkValue)
- require.NoError(t, d.StateSyncer().UpdateGlobalState(ctx, &syncer.StateInfo{State: syncer.StateNormalRunning}))
+ require.NoError(t, d.StateSyncer().UpdateGlobalState(ctx, &serverstate.StateInfo{State: serverstate.StateNormalRunning}))
wg.Wait()
require.Equal(t, "", checkErr)
}
diff --git a/pkg/ddl/stat_test.go b/pkg/ddl/stat_test.go
index 562e51e155545..b46350cbe1132 100644
--- a/pkg/ddl/stat_test.go
+++ b/pkg/ddl/stat_test.go
@@ -23,7 +23,6 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/tidb/pkg/ddl"
"github.com/pingcap/tidb/pkg/ddl/util"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/model"
@@ -32,6 +31,7 @@ import (
sessiontypes "github.com/pingcap/tidb/pkg/session/types"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/external"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/types"
"github.com/stretchr/testify/require"
)
@@ -114,7 +114,7 @@ func buildCreateIdxJob(dbInfo *model.DBInfo, tblInfo *model.TableInfo, unique bo
func TestIssue42268(t *testing.T) {
// issue 42268 missing table name in 'admin show ddl' result during drop table
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t_0")
@@ -127,8 +127,7 @@ func TestIssue42268(t *testing.T) {
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if tbl.Meta().ID != job.TableID {
return
}
@@ -139,8 +138,7 @@ func TestIssue42268(t *testing.T) {
tblName := fmt.Sprintf("%s", rs.Rows()[0][2])
require.Equal(t, tblName, "t_0")
}
- }
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("drop table t_0")
}
diff --git a/pkg/ddl/table.go b/pkg/ddl/table.go
index 0603b5b524e78..b32e9e6e83134 100644
--- a/pkg/ddl/table.go
+++ b/pkg/ddl/table.go
@@ -30,7 +30,6 @@ import (
sess "github.com/pingcap/tidb/pkg/ddl/session"
"github.com/pingcap/tidb/pkg/domain/infosync"
"github.com/pingcap/tidb/pkg/infoschema"
- "github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/meta"
"github.com/pingcap/tidb/pkg/meta/autoid"
"github.com/pingcap/tidb/pkg/parser/ast"
@@ -50,251 +49,6 @@ import (
const tiflashCheckTiDBHTTPAPIHalfInterval = 2500 * time.Millisecond
-// DANGER: it is an internal function used by onCreateTable and onCreateTables, for reusing code. Be careful.
-// 1. it expects the argument of job has been deserialized.
-// 2. it won't call updateSchemaVersion, FinishTableJob and asyncNotifyEvent.
-func createTable(d *ddlCtx, t *meta.Meta, job *model.Job, fkCheck bool) (*model.TableInfo, error) {
- schemaID := job.SchemaID
- tbInfo := job.Args[0].(*model.TableInfo)
-
- tbInfo.State = model.StateNone
- var err error
- if variable.EnableFastCreateTable.Load() {
- err = checkTableNotExistsByName(d, t, schemaID, job.SchemaName, tbInfo.Name.L)
- } else {
- err = checkTableNotExists(d, schemaID, tbInfo.Name.L)
- }
- if err != nil {
- if infoschema.ErrDatabaseNotExists.Equal(err) || infoschema.ErrTableExists.Equal(err) {
- job.State = model.JobStateCancelled
- }
- return tbInfo, errors.Trace(err)
- }
-
- err = checkConstraintNamesNotExists(t, schemaID, tbInfo.Constraints)
- if err != nil {
- if infoschema.ErrCheckConstraintDupName.Equal(err) {
- job.State = model.JobStateCancelled
- }
- return tbInfo, errors.Trace(err)
- }
-
- retryable, err := checkTableForeignKeyValidInOwner(d, t, job, tbInfo, fkCheck)
- if err != nil {
- if !retryable {
- job.State = model.JobStateCancelled
- }
- return tbInfo, errors.Trace(err)
- }
- // Allocate foreign key ID.
- for _, fkInfo := range tbInfo.ForeignKeys {
- fkInfo.ID = allocateFKIndexID(tbInfo)
- fkInfo.State = model.StatePublic
- }
- switch tbInfo.State {
- case model.StateNone:
- // none -> public
- tbInfo.State = model.StatePublic
- tbInfo.UpdateTS = t.StartTS
- err = createTableOrViewWithCheck(t, job, schemaID, tbInfo)
- if err != nil {
- return tbInfo, errors.Trace(err)
- }
-
- failpoint.Inject("checkOwnerCheckAllVersionsWaitTime", func(val failpoint.Value) {
- if val.(bool) {
- failpoint.Return(tbInfo, errors.New("mock create table error"))
- }
- })
-
- // build table & partition bundles if any.
- if err = checkAllTablePlacementPoliciesExistAndCancelNonExistJob(t, job, tbInfo); err != nil {
- return tbInfo, errors.Trace(err)
- }
-
- if tbInfo.TiFlashReplica != nil {
- replicaInfo := tbInfo.TiFlashReplica
- if pi := tbInfo.GetPartitionInfo(); pi != nil {
- logutil.DDLLogger().Info("Set TiFlash replica pd rule for partitioned table when creating", zap.Int64("tableID", tbInfo.ID))
- if e := infosync.ConfigureTiFlashPDForPartitions(false, &pi.Definitions, replicaInfo.Count, &replicaInfo.LocationLabels, tbInfo.ID); e != nil {
- job.State = model.JobStateCancelled
- return tbInfo, errors.Trace(e)
- }
- // Partitions that in adding mid-state. They have high priorities, so we should set accordingly pd rules.
- if e := infosync.ConfigureTiFlashPDForPartitions(true, &pi.AddingDefinitions, replicaInfo.Count, &replicaInfo.LocationLabels, tbInfo.ID); e != nil {
- job.State = model.JobStateCancelled
- return tbInfo, errors.Trace(e)
- }
- } else {
- logutil.DDLLogger().Info("Set TiFlash replica pd rule when creating", zap.Int64("tableID", tbInfo.ID))
- if e := infosync.ConfigureTiFlashPDForTable(tbInfo.ID, replicaInfo.Count, &replicaInfo.LocationLabels); e != nil {
- job.State = model.JobStateCancelled
- return tbInfo, errors.Trace(e)
- }
- }
- }
-
- bundles, err := placement.NewFullTableBundles(t, tbInfo)
- if err != nil {
- job.State = model.JobStateCancelled
- return tbInfo, errors.Trace(err)
- }
-
- // Send the placement bundle to PD.
- err = infosync.PutRuleBundlesWithDefaultRetry(context.TODO(), bundles)
- if err != nil {
- job.State = model.JobStateCancelled
- return tbInfo, errors.Wrapf(err, "failed to notify PD the placement rules")
- }
-
- return tbInfo, nil
- default:
- return tbInfo, dbterror.ErrInvalidDDLState.GenWithStackByArgs("table", tbInfo.State)
- }
-}
-
-func onCreateTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
- failpoint.Inject("mockExceedErrorLimit", func(val failpoint.Value) {
- if val.(bool) {
- failpoint.Return(ver, errors.New("mock do job error"))
- }
- })
-
- // just decode, createTable will use it as Args[0]
- tbInfo := &model.TableInfo{}
- fkCheck := false
- if err := job.DecodeArgs(tbInfo, &fkCheck); err != nil {
- // Invalid arguments, cancel this job.
- job.State = model.JobStateCancelled
- return ver, errors.Trace(err)
- }
-
- if len(tbInfo.ForeignKeys) > 0 {
- return createTableWithForeignKeys(d, t, job, tbInfo, fkCheck)
- }
-
- tbInfo, err := createTable(d, t, job, fkCheck)
- if err != nil {
- return ver, errors.Trace(err)
- }
-
- ver, err = updateSchemaVersion(d, t, job)
- if err != nil {
- return ver, errors.Trace(err)
- }
-
- // Finish this job.
- job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tbInfo)
- createTableEvent := statsutil.NewCreateTableEvent(
- job.SchemaID,
- tbInfo,
- )
- asyncNotifyEvent(d, createTableEvent)
- return ver, errors.Trace(err)
-}
-
-func createTableWithForeignKeys(d *ddlCtx, t *meta.Meta, job *model.Job, tbInfo *model.TableInfo, fkCheck bool) (ver int64, err error) {
- switch tbInfo.State {
- case model.StateNone, model.StatePublic:
- // create table in non-public or public state. The function `createTable` will always reset
- // the `tbInfo.State` with `model.StateNone`, so it's fine to just call the `createTable` with
- // public state.
- // when `br` restores table, the state of `tbInfo` will be public.
- tbInfo, err = createTable(d, t, job, fkCheck)
- if err != nil {
- return ver, errors.Trace(err)
- }
- tbInfo.State = model.StateWriteOnly
- ver, err = updateVersionAndTableInfo(d, t, job, tbInfo, true)
- if err != nil {
- return ver, errors.Trace(err)
- }
- job.SchemaState = model.StateWriteOnly
- case model.StateWriteOnly:
- tbInfo.State = model.StatePublic
- ver, err = updateVersionAndTableInfo(d, t, job, tbInfo, true)
- if err != nil {
- return ver, errors.Trace(err)
- }
- job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tbInfo)
- createTableEvent := statsutil.NewCreateTableEvent(
- job.SchemaID,
- tbInfo,
- )
- asyncNotifyEvent(d, createTableEvent)
- return ver, nil
- default:
- return ver, errors.Trace(dbterror.ErrInvalidDDLJob.GenWithStackByArgs("table", tbInfo.State))
- }
- return ver, errors.Trace(err)
-}
-
-func onCreateTables(d *ddlCtx, t *meta.Meta, job *model.Job) (int64, error) {
- var ver int64
-
- var args []*model.TableInfo
- fkCheck := false
- err := job.DecodeArgs(&args, &fkCheck)
- if err != nil {
- // Invalid arguments, cancel this job.
- job.State = model.JobStateCancelled
- return ver, errors.Trace(err)
- }
-
- // We don't construct jobs for every table, but only tableInfo
- // The following loop creates a stub job for every table
- //
- // it clones a stub job from the ActionCreateTables job
- stubJob := job.Clone()
- stubJob.Args = make([]any, 1)
- for i := range args {
- stubJob.TableID = args[i].ID
- stubJob.Args[0] = args[i]
- if args[i].Sequence != nil {
- err := createSequenceWithCheck(t, stubJob, args[i])
- if err != nil {
- job.State = model.JobStateCancelled
- return ver, errors.Trace(err)
- }
- } else {
- tbInfo, err := createTable(d, t, stubJob, fkCheck)
- if err != nil {
- job.State = model.JobStateCancelled
- return ver, errors.Trace(err)
- }
- args[i] = tbInfo
- }
- }
-
- ver, err = updateSchemaVersion(d, t, job)
- if err != nil {
- return ver, errors.Trace(err)
- }
-
- job.State = model.JobStateDone
- job.SchemaState = model.StatePublic
- job.BinlogInfo.SetTableInfos(ver, args)
-
- for i := range args {
- createTableEvent := statsutil.NewCreateTableEvent(
- job.SchemaID,
- args[i],
- )
- asyncNotifyEvent(d, createTableEvent)
- }
-
- return ver, errors.Trace(err)
-}
-
-func createTableOrViewWithCheck(t *meta.Meta, job *model.Job, schemaID int64, tbInfo *model.TableInfo) error {
- err := checkTableInfoValid(tbInfo)
- if err != nil {
- job.State = model.JobStateCancelled
- return errors.Trace(err)
- }
- return t.CreateTableOrView(schemaID, job.SchemaName, tbInfo)
-}
-
func repairTableOrViewWithCheck(t *meta.Meta, job *model.Job, schemaID int64, tbInfo *model.TableInfo) error {
err := checkTableInfoValid(tbInfo)
if err != nil {
@@ -304,70 +58,7 @@ func repairTableOrViewWithCheck(t *meta.Meta, job *model.Job, schemaID int64, tb
return t.UpdateTable(schemaID, tbInfo)
}
-func onCreateView(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
- schemaID := job.SchemaID
- tbInfo := &model.TableInfo{}
- var orReplace bool
- var _placeholder int64 // oldTblInfoID
- if err := job.DecodeArgs(tbInfo, &orReplace, &_placeholder); err != nil {
- // Invalid arguments, cancel this job.
- job.State = model.JobStateCancelled
- return ver, errors.Trace(err)
- }
- tbInfo.State = model.StateNone
-
- oldTableID, err := findTableIDByName(d, t, schemaID, tbInfo.Name.L)
- if infoschema.ErrTableNotExists.Equal(err) {
- err = nil
- }
- failpoint.InjectCall("onDDLCreateView", job)
- if err != nil {
- if infoschema.ErrDatabaseNotExists.Equal(err) {
- job.State = model.JobStateCancelled
- return ver, errors.Trace(err)
- } else if !infoschema.ErrTableExists.Equal(err) {
- return ver, errors.Trace(err)
- }
- if !orReplace {
- job.State = model.JobStateCancelled
- return ver, errors.Trace(err)
- }
- }
- ver, err = updateSchemaVersion(d, t, job)
- if err != nil {
- return ver, errors.Trace(err)
- }
- switch tbInfo.State {
- case model.StateNone:
- // none -> public
- tbInfo.State = model.StatePublic
- tbInfo.UpdateTS = t.StartTS
- if oldTableID > 0 && orReplace {
- err = t.DropTableOrView(schemaID, job.SchemaName, oldTableID, tbInfo.Name.L)
- if err != nil {
- job.State = model.JobStateCancelled
- return ver, errors.Trace(err)
- }
- err = t.GetAutoIDAccessors(schemaID, oldTableID).Del()
- if err != nil {
- job.State = model.JobStateCancelled
- return ver, errors.Trace(err)
- }
- }
- err = createTableOrViewWithCheck(t, job, schemaID, tbInfo)
- if err != nil {
- job.State = model.JobStateCancelled
- return ver, errors.Trace(err)
- }
- // Finish this job.
- job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tbInfo)
- return ver, nil
- default:
- return ver, dbterror.ErrInvalidDDLState.GenWithStackByArgs("table", tbInfo.State)
- }
-}
-
-func onDropTableOrView(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onDropTableOrView(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
tblInfo, err := checkTableExistAndCancelNonExistJob(t, job, job.SchemaID)
if err != nil {
return ver, errors.Trace(err)
@@ -378,20 +69,20 @@ func onDropTableOrView(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ er
case model.StatePublic:
// public -> write only
if job.Type == model.ActionDropTable {
- err = checkDropTableHasForeignKeyReferredInOwner(d, t, job)
+ err = checkDropTableHasForeignKeyReferredInOwner(jobCtx.infoCache, job)
if err != nil {
return ver, err
}
}
tblInfo.State = model.StateWriteOnly
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != tblInfo.State)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != tblInfo.State)
if err != nil {
return ver, errors.Trace(err)
}
case model.StateWriteOnly:
// write only -> delete only
tblInfo.State = model.StateDeleteOnly
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != tblInfo.State)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != tblInfo.State)
if err != nil {
return ver, errors.Trace(err)
}
@@ -401,16 +92,16 @@ func onDropTableOrView(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ er
ruleIDs := append(getPartitionRuleIDs(job.SchemaName, tblInfo), fmt.Sprintf(label.TableIDFormat, label.IDPrefix, job.SchemaName, tblInfo.Name.L))
job.CtxVars = []any{oldIDs}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, originalState != tblInfo.State)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, originalState != tblInfo.State)
if err != nil {
return ver, errors.Trace(err)
}
if tblInfo.IsSequence() {
- if err = t.DropSequence(job.SchemaID, job.SchemaName, job.TableID, job.TableName); err != nil {
+ if err = t.DropSequence(job.SchemaID, job.TableID); err != nil {
return ver, errors.Trace(err)
}
} else {
- if err = t.DropTableOrView(job.SchemaID, job.SchemaName, job.TableID, job.TableName); err != nil {
+ if err = t.DropTableOrView(job.SchemaID, job.TableID); err != nil {
return ver, errors.Trace(err)
}
if err = t.GetAutoIDAccessors(job.SchemaID, job.TableID).Del(); err != nil {
@@ -435,7 +126,7 @@ func onDropTableOrView(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ er
job.SchemaID,
tblInfo,
)
- asyncNotifyEvent(d, dropTableEvent)
+ asyncNotifyEvent(jobCtx, dropTableEvent)
}
default:
return ver, errors.Trace(dbterror.ErrInvalidDDLState.GenWithStackByArgs("table", tblInfo.State))
@@ -444,7 +135,7 @@ func onDropTableOrView(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ er
return ver, errors.Trace(err)
}
-func (w *worker) onRecoverTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func (w *worker) onRecoverTable(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
var (
recoverInfo *RecoverInfo
recoverTableCheckFlag int64
@@ -469,7 +160,7 @@ func (w *worker) onRecoverTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver in
return ver, errors.Trace(err)
}
- err = checkTableNotExists(d, schemaID, tblInfo.Name.L)
+ err = checkTableNotExists(jobCtx.infoCache, schemaID, tblInfo.Name.L)
if err != nil {
if infoschema.ErrDatabaseNotExists.Equal(err) || infoschema.ErrTableExists.Equal(err) {
job.State = model.JobStateCancelled
@@ -536,7 +227,7 @@ func (w *worker) onRecoverTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver in
tableInfo := tblInfo.Clone()
tableInfo.State = model.StatePublic
tableInfo.UpdateTS = t.StartTS
- ver, err = updateVersionAndTableInfo(d, t, job, tableInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tableInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -576,7 +267,7 @@ func (w *worker) recoverTable(t *meta.Meta, job *model.Job, recoverInfo *Recover
tableInfo := recoverInfo.TableInfo.Clone()
tableInfo.State = model.StatePublic
tableInfo.UpdateTS = t.StartTS
- err = t.CreateTableAndSetAutoID(recoverInfo.SchemaID, recoverInfo.OldSchemaName, tableInfo, recoverInfo.AutoIDs)
+ err = t.CreateTableAndSetAutoID(recoverInfo.SchemaID, tableInfo, recoverInfo.AutoIDs)
if err != nil {
return ver, errors.Trace(err)
}
@@ -732,7 +423,7 @@ func getTableInfo(t *meta.Meta, tableID, schemaID int64) (*model.TableInfo, erro
// onTruncateTable delete old table meta, and creates a new table identical to old table except for table ID.
// As all the old data is encoded with old table ID, it can not be accessed anymore.
// A background job will be created to delete old data.
-func (w *worker) onTruncateTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func (w *worker) onTruncateTable(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
schemaID := job.SchemaID
tableID := job.TableID
var newTableID int64
@@ -753,11 +444,11 @@ func (w *worker) onTruncateTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver i
}
// Copy the old tableInfo for later usage.
oldTblInfo := tblInfo.Clone()
- err = checkTruncateTableHasForeignKeyReferredInOwner(d, t, job, tblInfo, fkCheck)
+ err = checkTruncateTableHasForeignKeyReferredInOwner(jobCtx.infoCache, job, tblInfo, fkCheck)
if err != nil {
return ver, err
}
- err = t.DropTableOrView(schemaID, job.SchemaName, tblInfo.ID, tblInfo.Name.L)
+ err = t.DropTableOrView(schemaID, tblInfo.ID)
if err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
@@ -854,7 +545,7 @@ func (w *worker) onTruncateTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver i
return 0, errors.Wrapf(err, "failed to notify PD the placement rules")
}
- err = t.CreateTableOrView(schemaID, job.SchemaName, tblInfo)
+ err = t.CreateTableOrView(schemaID, tblInfo)
if err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
@@ -870,9 +561,9 @@ func (w *worker) onTruncateTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver i
if pi := tblInfo.GetPartitionInfo(); pi != nil {
partitions = tblInfo.GetPartitionInfo().Definitions
}
- preSplitAndScatter(w.sess.Context, d.store, tblInfo, partitions)
+ preSplitAndScatter(w.sess.Context, jobCtx.store, tblInfo, partitions)
- ver, err = updateSchemaVersion(d, t, job)
+ ver, err = updateSchemaVersion(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -882,21 +573,21 @@ func (w *worker) onTruncateTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver i
tblInfo,
oldTblInfo,
)
- asyncNotifyEvent(d, truncateTableEvent)
+ asyncNotifyEvent(jobCtx, truncateTableEvent)
startKey := tablecodec.EncodeTablePrefix(tableID)
job.Args = []any{startKey, oldPartitionIDs}
return ver, nil
}
-func onRebaseAutoIncrementIDType(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
- return onRebaseAutoID(d, d.store, t, job, autoid.AutoIncrementType)
+func onRebaseAutoIncrementIDType(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+ return onRebaseAutoID(jobCtx, t, job, autoid.AutoIncrementType)
}
-func onRebaseAutoRandomType(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
- return onRebaseAutoID(d, d.store, t, job, autoid.AutoRandomType)
+func onRebaseAutoRandomType(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+ return onRebaseAutoID(jobCtx, t, job, autoid.AutoRandomType)
}
-func onRebaseAutoID(d *ddlCtx, _ kv.Storage, t *meta.Meta, job *model.Job, tp autoid.AllocatorType) (ver int64, _ error) {
+func onRebaseAutoID(jobCtx *jobContext, t *meta.Meta, job *model.Job, tp autoid.AllocatorType) (ver int64, _ error) {
schemaID := job.SchemaID
var (
newBase int64
@@ -919,7 +610,7 @@ func onRebaseAutoID(d *ddlCtx, _ kv.Storage, t *meta.Meta, job *model.Job, tp au
return ver, errors.Trace(err)
}
- tbl, err := getTable(d.getAutoIDRequirement(), schemaID, tblInfo)
+ tbl, err := getTable(jobCtx.getAutoIDRequirement(), schemaID, tblInfo)
if err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
@@ -957,7 +648,7 @@ func onRebaseAutoID(d *ddlCtx, _ kv.Storage, t *meta.Meta, job *model.Job, tp au
return ver, errors.Trace(err)
}
}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -965,7 +656,7 @@ func onRebaseAutoID(d *ddlCtx, _ kv.Storage, t *meta.Meta, job *model.Job, tp au
return ver, nil
}
-func onModifyTableAutoIDCache(d *ddlCtx, t *meta.Meta, job *model.Job) (int64, error) {
+func onModifyTableAutoIDCache(jobCtx *jobContext, t *meta.Meta, job *model.Job) (int64, error) {
var cache int64
if err := job.DecodeArgs(&cache); err != nil {
job.State = model.JobStateCancelled
@@ -978,7 +669,7 @@ func onModifyTableAutoIDCache(d *ddlCtx, t *meta.Meta, job *model.Job) (int64, e
}
tblInfo.AutoIdCache = cache
- ver, err := updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err := updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -986,7 +677,7 @@ func onModifyTableAutoIDCache(d *ddlCtx, t *meta.Meta, job *model.Job) (int64, e
return ver, nil
}
-func (w *worker) onShardRowID(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func (w *worker) onShardRowID(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
var shardRowIDBits uint64
err := job.DecodeArgs(&shardRowIDBits)
if err != nil {
@@ -1001,7 +692,7 @@ func (w *worker) onShardRowID(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int6
if shardRowIDBits < tblInfo.ShardRowIDBits {
tblInfo.ShardRowIDBits = shardRowIDBits
} else {
- tbl, err := getTable(d.getAutoIDRequirement(), job.SchemaID, tblInfo)
+ tbl, err := getTable(jobCtx.getAutoIDRequirement(), job.SchemaID, tblInfo)
if err != nil {
return ver, errors.Trace(err)
}
@@ -1014,7 +705,7 @@ func (w *worker) onShardRowID(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int6
// MaxShardRowIDBits use to check the overflow of auto ID.
tblInfo.MaxShardRowIDBits = shardRowIDBits
}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
@@ -1043,7 +734,7 @@ func verifyNoOverflowShardBits(s *sess.Pool, tbl table.Table, shardRowIDBits uin
return nil
}
-func onRenameTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onRenameTable(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
var oldSchemaID int64
var oldSchemaName model.CIStr
var tableName model.CIStr
@@ -1054,10 +745,10 @@ func onRenameTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error)
}
if job.SchemaState == model.StatePublic {
- return finishJobRenameTable(d, t, job)
+ return finishJobRenameTable(jobCtx, t, job)
}
newSchemaID := job.SchemaID
- err := checkTableNotExists(d, newSchemaID, tableName.L)
+ err := checkTableNotExists(jobCtx.infoCache, newSchemaID, tableName.L)
if err != nil {
if infoschema.ErrDatabaseNotExists.Equal(err) || infoschema.ErrTableExists.Equal(err) {
job.State = model.JobStateCancelled
@@ -1075,11 +766,11 @@ func onRenameTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error)
return ver, errors.Trace(err)
}
fkh := newForeignKeyHelper()
- err = adjustForeignKeyChildTableInfoAfterRenameTable(d, t, job, &fkh, tblInfo, oldSchemaName, oldTableName, tableName, newSchemaID)
+ err = adjustForeignKeyChildTableInfoAfterRenameTable(jobCtx.infoCache, t, job, &fkh, tblInfo, oldSchemaName, oldTableName, tableName, newSchemaID)
if err != nil {
return ver, errors.Trace(err)
}
- ver, err = updateSchemaVersion(d, t, job, fkh.getLoadedTables()...)
+ ver, err = updateSchemaVersion(jobCtx, t, job, fkh.getLoadedTables()...)
if err != nil {
return ver, errors.Trace(err)
}
@@ -1087,7 +778,7 @@ func onRenameTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error)
return ver, nil
}
-func onRenameTables(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onRenameTables(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
oldSchemaIDs := []int64{}
newSchemaIDs := []int64{}
tableNames := []*model.CIStr{}
@@ -1100,7 +791,7 @@ func onRenameTables(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error
}
if job.SchemaState == model.StatePublic {
- return finishJobRenameTables(d, t, job, tableNames, tableIDs, newSchemaIDs)
+ return finishJobRenameTables(jobCtx, t, job, tableNames, tableIDs, newSchemaIDs)
}
var err error
@@ -1116,13 +807,13 @@ func onRenameTables(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error
if err != nil {
return ver, errors.Trace(err)
}
- err = adjustForeignKeyChildTableInfoAfterRenameTable(d, t, job, &fkh, tblInfo, *oldSchemaNames[i], *oldTableNames[i], *tableNames[i], newSchemaIDs[i])
+ err = adjustForeignKeyChildTableInfoAfterRenameTable(jobCtx.infoCache, t, job, &fkh, tblInfo, *oldSchemaNames[i], *oldTableNames[i], *tableNames[i], newSchemaIDs[i])
if err != nil {
return ver, errors.Trace(err)
}
}
- ver, err = updateSchemaVersion(d, t, job, fkh.getLoadedTables()...)
+ ver, err = updateSchemaVersion(jobCtx, t, job, fkh.getLoadedTables()...)
if err != nil {
return ver, errors.Trace(err)
}
@@ -1131,7 +822,7 @@ func onRenameTables(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error
}
func checkAndRenameTables(t *meta.Meta, job *model.Job, tblInfo *model.TableInfo, oldSchemaID, newSchemaID int64, oldSchemaName, tableName *model.CIStr) (ver int64, _ error) {
- err := t.DropTableOrView(oldSchemaID, oldSchemaName.L, tblInfo.ID, tblInfo.Name.L)
+ err := t.DropTableOrView(oldSchemaID, tblInfo.ID)
if err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
@@ -1166,7 +857,7 @@ func checkAndRenameTables(t *meta.Meta, job *model.Job, tblInfo *model.TableInfo
}
tblInfo.Name = *tableName
- err = t.CreateTableOrView(newSchemaID, job.SchemaName, tblInfo)
+ err = t.CreateTableOrView(newSchemaID, tblInfo)
if err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
@@ -1181,14 +872,11 @@ func checkAndRenameTables(t *meta.Meta, job *model.Job, tblInfo *model.TableInfo
return ver, nil
}
-func adjustForeignKeyChildTableInfoAfterRenameTable(d *ddlCtx, t *meta.Meta, job *model.Job, fkh *foreignKeyHelper, tblInfo *model.TableInfo, oldSchemaName, oldTableName, newTableName model.CIStr, newSchemaID int64) error {
+func adjustForeignKeyChildTableInfoAfterRenameTable(infoCache *infoschema.InfoCache, t *meta.Meta, job *model.Job, fkh *foreignKeyHelper, tblInfo *model.TableInfo, oldSchemaName, oldTableName, newTableName model.CIStr, newSchemaID int64) error {
if !variable.EnableForeignKey.Load() || newTableName.L == oldTableName.L {
return nil
}
- is, err := getAndCheckLatestInfoSchema(d, t)
- if err != nil {
- return err
- }
+ is := infoCache.GetLatest()
newDB, ok := is.SchemaByID(newSchemaID)
if !ok {
job.State = model.JobStateCancelled
@@ -1215,7 +903,7 @@ func adjustForeignKeyChildTableInfoAfterRenameTable(d *ddlCtx, t *meta.Meta, job
childFKInfo.RefTable = newTableName
}
for _, info := range fkh.loaded {
- err = updateTable(t, info.schemaID, info.tblInfo)
+ err := updateTable(t, info.schemaID, info.tblInfo)
if err != nil {
return err
}
@@ -1233,7 +921,7 @@ func adjustForeignKeyChildTableInfoAfterRenameTable(d *ddlCtx, t *meta.Meta, job
// there may be DMLs that use the old schema.
// - TiCDC cannot handle the DMLs that use the old schema, because
// the commit TS of the DMLs are greater than the job state updating TS.
-func finishJobRenameTable(d *ddlCtx, t *meta.Meta, job *model.Job) (int64, error) {
+func finishJobRenameTable(jobCtx *jobContext, t *meta.Meta, job *model.Job) (int64, error) {
tblInfo, err := getTableInfo(t, job.TableID, job.SchemaID)
if err != nil {
job.State = model.JobStateCancelled
@@ -1248,7 +936,7 @@ func finishJobRenameTable(d *ddlCtx, t *meta.Meta, job *model.Job) (int64, error
if err != nil {
return 0, errors.Trace(err)
}
- ver, err := updateSchemaVersion(d, t, job)
+ ver, err := updateSchemaVersion(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -1257,7 +945,7 @@ func finishJobRenameTable(d *ddlCtx, t *meta.Meta, job *model.Job) (int64, error
return ver, nil
}
-func finishJobRenameTables(d *ddlCtx, t *meta.Meta, job *model.Job,
+func finishJobRenameTables(jobCtx *jobContext, t *meta.Meta, job *model.Job,
tableNames []*model.CIStr, tableIDs, newSchemaIDs []int64) (int64, error) {
tblSchemaIDs := make(map[int64]int64, len(tableIDs))
for i := range tableIDs {
@@ -1283,7 +971,7 @@ func finishJobRenameTables(d *ddlCtx, t *meta.Meta, job *model.Job,
if err != nil {
return 0, errors.Trace(err)
}
- ver, err := updateSchemaVersion(d, t, job)
+ ver, err := updateSchemaVersion(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -1292,7 +980,7 @@ func finishJobRenameTables(d *ddlCtx, t *meta.Meta, job *model.Job,
return ver, nil
}
-func onModifyTableComment(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onModifyTableComment(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
var comment string
if err := job.DecodeArgs(&comment); err != nil {
job.State = model.JobStateCancelled
@@ -1310,7 +998,7 @@ func onModifyTableComment(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _
}
tblInfo.Comment = comment
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -1318,7 +1006,7 @@ func onModifyTableComment(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _
return ver, nil
}
-func onModifyTableCharsetAndCollate(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onModifyTableCharsetAndCollate(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
var toCharset, toCollate string
var needsOverwriteCols bool
if err := job.DecodeArgs(&toCharset, &toCollate, &needsOverwriteCols); err != nil {
@@ -1364,7 +1052,7 @@ func onModifyTableCharsetAndCollate(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
}
}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -1372,7 +1060,7 @@ func onModifyTableCharsetAndCollate(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
return ver, nil
}
-func (w *worker) onSetTableFlashReplica(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func (w *worker) onSetTableFlashReplica(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
var replicaInfo ast.TiFlashReplicaSpec
if err := job.DecodeArgs(&replicaInfo); err != nil {
job.State = model.JobStateCancelled
@@ -1434,7 +1122,7 @@ func (w *worker) onSetTableFlashReplica(d *ddlCtx, t *meta.Meta, job *model.Job)
tblInfo.TiFlashReplica = nil
}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -1452,7 +1140,7 @@ func (w *worker) checkTiFlashReplicaCount(replicaCount uint64) error {
return checkTiFlashReplicaCount(ctx, replicaCount)
}
-func onUpdateFlashReplicaStatus(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onUpdateFlashReplicaStatus(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
var available bool
var physicalID int64
if err := job.DecodeArgs(&available, &physicalID); err != nil {
@@ -1504,7 +1192,7 @@ func onUpdateFlashReplicaStatus(d *ddlCtx, t *meta.Meta, job *model.Job) (ver in
if tblInfo.TiFlashReplica.Available {
logutil.DDLLogger().Info("TiFlash replica available", zap.Int64("tableID", tblInfo.ID))
}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -1525,24 +1213,11 @@ func onUpdateFlashReplicaStatus(d *ddlCtx, t *meta.Meta, job *model.Job) (ver in
// jobs in the order of job id. During syncing J1, B should have synced the schema
// with the latest schema version, so when B runs J2, below check will see the table
// T already exists, and J2 will fail.
-func checkTableNotExists(d *ddlCtx, schemaID int64, tableName string) error {
- is := d.infoCache.GetLatest()
+func checkTableNotExists(infoCache *infoschema.InfoCache, schemaID int64, tableName string) error {
+ is := infoCache.GetLatest()
return checkTableNotExistsFromInfoSchema(is, schemaID, tableName)
}
-func checkTableNotExistsByName(d *ddlCtx, t *meta.Meta, schemaID int64, schemaName, tableName string) error {
- // Try to use memory schema info to check first.
- currVer, err := t.GetSchemaVersion()
- if err != nil {
- return err
- }
- is := d.infoCache.GetLatest()
- if is != nil && is.SchemaMetaVersion() == currVer {
- return checkTableNotExistsFromInfoSchema(is, schemaID, tableName)
- }
- return t.CheckTableNameNotExists(t.TableNameKey(schemaName, tableName))
-}
-
func checkConstraintNamesNotExists(t *meta.Meta, schemaID int64, constraints []*model.ConstraintInfo) error {
if len(constraints) == 0 {
return nil
@@ -1591,50 +1266,8 @@ func checkTableNotExistsFromInfoSchema(is infoschema.InfoSchema, schemaID int64,
return nil
}
-func findTableIDByName(d *ddlCtx, t *meta.Meta, schemaID int64, tableName string) (int64, error) {
- // Try to use memory schema info to check first.
- currVer, err := t.GetSchemaVersion()
- if err != nil {
- return 0, err
- }
- is := d.infoCache.GetLatest()
- if is != nil && is.SchemaMetaVersion() == currVer {
- return findTableIDFromInfoSchema(is, schemaID, tableName)
- }
-
- return findTableIDFromStore(t, schemaID, tableName)
-}
-
-func findTableIDFromInfoSchema(is infoschema.InfoSchema, schemaID int64, tableName string) (int64, error) {
- schema, ok := is.SchemaByID(schemaID)
- if !ok {
- return 0, infoschema.ErrDatabaseNotExists.GenWithStackByArgs("")
- }
- tbl, err := is.TableByName(context.Background(), schema.Name, model.NewCIStr(tableName))
- if err != nil {
- return 0, err
- }
- return tbl.Meta().ID, nil
-}
-
-func findTableIDFromStore(t *meta.Meta, schemaID int64, tableName string) (int64, error) {
- tbls, err := t.ListSimpleTables(schemaID)
- if err != nil {
- if meta.ErrDBNotExists.Equal(err) {
- return 0, infoschema.ErrDatabaseNotExists.GenWithStackByArgs("")
- }
- return 0, errors.Trace(err)
- }
- for _, tbl := range tbls {
- if tbl.Name.L == tableName {
- return tbl.ID, nil
- }
- }
- return 0, infoschema.ErrTableNotExists.FastGenByArgs(tableName)
-}
-
// updateVersionAndTableInfoWithCheck checks table info validate and updates the schema version and the table information
-func updateVersionAndTableInfoWithCheck(d *ddlCtx, t *meta.Meta, job *model.Job, tblInfo *model.TableInfo, shouldUpdateVer bool, multiInfos ...schemaIDAndTableInfo) (
+func updateVersionAndTableInfoWithCheck(jobCtx *jobContext, t *meta.Meta, job *model.Job, tblInfo *model.TableInfo, shouldUpdateVer bool, multiInfos ...schemaIDAndTableInfo) (
ver int64, err error) {
err = checkTableInfoValid(tblInfo)
if err != nil {
@@ -1648,11 +1281,11 @@ func updateVersionAndTableInfoWithCheck(d *ddlCtx, t *meta.Meta, job *model.Job,
return ver, errors.Trace(err)
}
}
- return updateVersionAndTableInfo(d, t, job, tblInfo, shouldUpdateVer, multiInfos...)
+ return updateVersionAndTableInfo(jobCtx, t, job, tblInfo, shouldUpdateVer, multiInfos...)
}
// updateVersionAndTableInfo updates the schema version and the table information.
-func updateVersionAndTableInfo(d *ddlCtx, t *meta.Meta, job *model.Job, tblInfo *model.TableInfo, shouldUpdateVer bool, multiInfos ...schemaIDAndTableInfo) (
+func updateVersionAndTableInfo(jobCtx *jobContext, t *meta.Meta, job *model.Job, tblInfo *model.TableInfo, shouldUpdateVer bool, multiInfos ...schemaIDAndTableInfo) (
ver int64, err error) {
failpoint.Inject("mockUpdateVersionAndTableInfoErr", func(val failpoint.Value) {
switch val.(int) {
@@ -1667,7 +1300,7 @@ func updateVersionAndTableInfo(d *ddlCtx, t *meta.Meta, job *model.Job, tblInfo
}
})
if shouldUpdateVer && (job.MultiSchemaInfo == nil || !job.MultiSchemaInfo.SkipVersion) {
- ver, err = updateSchemaVersion(d, t, job, multiInfos...)
+ ver, err = updateSchemaVersion(jobCtx, t, job, multiInfos...)
if err != nil {
return 0, errors.Trace(err)
}
@@ -1698,7 +1331,7 @@ type schemaIDAndTableInfo struct {
tblInfo *model.TableInfo
}
-func onRepairTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onRepairTable(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) {
schemaID := job.SchemaID
tblInfo := &model.TableInfo{}
@@ -1719,7 +1352,7 @@ func onRepairTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error)
// When in repair mode, the repaired table in a server is not access to user,
// the table after repairing will be removed from repair list. Other server left
// behind alive may need to restart to get the latest schema version.
- ver, err = updateSchemaVersion(d, t, job)
+ ver, err = updateSchemaVersion(jobCtx, t, job)
if err != nil {
return ver, errors.Trace(err)
}
@@ -1740,7 +1373,7 @@ func onRepairTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error)
}
}
-func onAlterTableAttributes(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func onAlterTableAttributes(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
rule := label.NewRule()
err = job.DecodeArgs(rule)
if err != nil {
@@ -1763,7 +1396,7 @@ func onAlterTableAttributes(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64,
job.State = model.JobStateCancelled
return 0, errors.Wrapf(err, "failed to notify PD the label rules")
}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -1772,7 +1405,7 @@ func onAlterTableAttributes(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64,
return ver, nil
}
-func onAlterTablePartitionAttributes(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func onAlterTablePartitionAttributes(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
var partitionID int64
rule := label.NewRule()
err = job.DecodeArgs(&partitionID, rule)
@@ -1801,7 +1434,7 @@ func onAlterTablePartitionAttributes(d *ddlCtx, t *meta.Meta, job *model.Job) (v
job.State = model.JobStateCancelled
return 0, errors.Wrapf(err, "failed to notify PD the label rules")
}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -1810,7 +1443,7 @@ func onAlterTablePartitionAttributes(d *ddlCtx, t *meta.Meta, job *model.Job) (v
return ver, nil
}
-func onAlterTablePartitionPlacement(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func onAlterTablePartitionPlacement(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
var partitionID int64
policyRefInfo := &model.PolicyRefInfo{}
err = job.DecodeArgs(&partitionID, &policyRefInfo)
@@ -1842,7 +1475,7 @@ func onAlterTablePartitionPlacement(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
return 0, errors.Trace(table.ErrUnknownPartition.GenWithStackByArgs("drop?", tblInfo.Name.O))
}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -1875,7 +1508,7 @@ func onAlterTablePartitionPlacement(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
return ver, nil
}
-func onAlterTablePlacement(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func onAlterTablePlacement(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
policyRefInfo := &model.PolicyRefInfo{}
err = job.DecodeArgs(&policyRefInfo)
if err != nil {
@@ -1894,7 +1527,7 @@ func onAlterTablePlacement(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64,
oldTableEnablesPlacement := tblInfo.PlacementPolicyRef != nil
tblInfo.PlacementPolicyRef = policyRefInfo
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -1965,7 +1598,7 @@ func updateLabelRules(job *model.Job, tblInfo *model.TableInfo, oldRules map[str
return infosync.UpdateLabelRules(context.TODO(), patch)
}
-func onAlterCacheTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func onAlterCacheTable(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
tbInfo, err := GetTableInfoAndCancelFaultJob(t, job, job.SchemaID)
if err != nil {
return 0, errors.Trace(err)
@@ -1988,14 +1621,14 @@ func onAlterCacheTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err
case model.TableCacheStatusDisable:
// disable -> switching
tbInfo.TableCacheStatusType = model.TableCacheStatusSwitching
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tbInfo, true)
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tbInfo, true)
if err != nil {
return ver, err
}
case model.TableCacheStatusSwitching:
// switching -> enable
tbInfo.TableCacheStatusType = model.TableCacheStatusEnable
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tbInfo, true)
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tbInfo, true)
if err != nil {
return ver, err
}
@@ -2008,7 +1641,7 @@ func onAlterCacheTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err
return ver, err
}
-func onAlterNoCacheTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func onAlterNoCacheTable(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
tbInfo, err := GetTableInfoAndCancelFaultJob(t, job, job.SchemaID)
if err != nil {
return 0, errors.Trace(err)
@@ -2023,14 +1656,14 @@ func onAlterNoCacheTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, er
case model.TableCacheStatusEnable:
// enable -> switching
tbInfo.TableCacheStatusType = model.TableCacheStatusSwitching
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tbInfo, true)
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tbInfo, true)
if err != nil {
return ver, err
}
case model.TableCacheStatusSwitching:
// switching -> disable
tbInfo.TableCacheStatusType = model.TableCacheStatusDisable
- ver, err = updateVersionAndTableInfoWithCheck(d, t, job, tbInfo, true)
+ ver, err = updateVersionAndTableInfoWithCheck(jobCtx, t, job, tbInfo, true)
if err != nil {
return ver, err
}
diff --git a/pkg/ddl/table_lock.go b/pkg/ddl/table_lock.go
index ba2648d227e36..65f722732e200 100644
--- a/pkg/ddl/table_lock.go
+++ b/pkg/ddl/table_lock.go
@@ -22,7 +22,7 @@ import (
"github.com/pingcap/tidb/pkg/util/dbterror"
)
-func onLockTables(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func onLockTables(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
arg := &LockTablesArg{}
if err := job.DecodeArgs(arg); err != nil {
// Invalid arguments, cancel this job.
@@ -32,7 +32,7 @@ func onLockTables(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error
// Unlock table first.
if arg.IndexOfUnlock < len(arg.UnlockTables) {
- return unlockTables(d, t, job, arg)
+ return unlockTables(jobCtx, t, job, arg)
}
// Check table locked by other, this can be only checked at the first time.
@@ -75,13 +75,13 @@ func onLockTables(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error
// none -> pre_lock
tbInfo.Lock.State = model.TableLockStatePreLock
tbInfo.Lock.TS = t.StartTS
- ver, err = updateVersionAndTableInfo(d, t, job, tbInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tbInfo, true)
// If the state of the lock is public, it means the lock is a read lock and already locked by other session,
// so this request of lock table doesn't need pre-lock state, just update the TS and table info is ok.
case model.TableLockStatePreLock, model.TableLockStatePublic:
tbInfo.Lock.State = model.TableLockStatePublic
tbInfo.Lock.TS = t.StartTS
- ver, err = updateVersionAndTableInfo(d, t, job, tbInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tbInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -167,7 +167,7 @@ func checkTableLocked(tbInfo *model.TableInfo, lockTp model.TableLockType, sessi
}
// unlockTables uses unlock a batch of table lock one by one.
-func unlockTables(d *ddlCtx, t *meta.Meta, job *model.Job, arg *LockTablesArg) (ver int64, err error) {
+func unlockTables(jobCtx *jobContext, t *meta.Meta, job *model.Job, arg *LockTablesArg) (ver int64, err error) {
if arg.IndexOfUnlock >= len(arg.UnlockTables) {
return ver, nil
}
@@ -186,7 +186,7 @@ func unlockTables(d *ddlCtx, t *meta.Meta, job *model.Job, arg *LockTablesArg) (
needUpdateTableInfo := unlockTable(tbInfo, arg)
if needUpdateTableInfo {
- ver, err = updateVersionAndTableInfo(d, t, job, tbInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tbInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -222,7 +222,7 @@ func unlockTable(tbInfo *model.TableInfo, arg *LockTablesArg) (needUpdateTableIn
return true
}
-func onUnlockTables(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func onUnlockTables(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
arg := &LockTablesArg{}
if err := job.DecodeArgs(arg); err != nil {
// Invalid arguments, cancel this job.
@@ -230,7 +230,7 @@ func onUnlockTables(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err err
return ver, errors.Trace(err)
}
- ver, err = unlockTables(d, t, job, arg)
+ ver, err = unlockTables(jobCtx, t, job, arg)
if arg.IndexOfUnlock == len(arg.UnlockTables) {
job.FinishTableJob(model.JobStateDone, model.StateNone, ver, nil)
}
diff --git a/pkg/ddl/table_modify_test.go b/pkg/ddl/table_modify_test.go
index 1b69aed2d66e1..03f14eddbe53b 100644
--- a/pkg/ddl/table_modify_test.go
+++ b/pkg/ddl/table_modify_test.go
@@ -20,8 +20,6 @@ import (
"time"
"github.com/pingcap/tidb/pkg/ddl"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
- "github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/model"
@@ -29,6 +27,7 @@ import (
sessiontypes "github.com/pingcap/tidb/pkg/session/types"
"github.com/pingcap/tidb/pkg/sessiontxn"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/util"
"github.com/stretchr/testify/require"
)
@@ -88,7 +87,7 @@ func TestLockTableReadOnly(t *testing.T) {
// TestConcurrentLockTables test concurrent lock/unlock tables.
func TestConcurrentLockTables(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, tableModifyLease)
+ store := testkit.CreateMockStoreWithSchemaLease(t, tableModifyLease)
tk1 := testkit.NewTestKit(t, store)
tk2 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
@@ -98,7 +97,7 @@ func TestConcurrentLockTables(t *testing.T) {
// Test concurrent lock tables read.
sql1 := "lock tables t1 read"
sql2 := "lock tables t1 read"
- testParallelExecSQL(t, store, dom, sql1, sql2, tk1.Session(), tk2.Session(), func(t *testing.T, err1, err2 error) {
+ testParallelExecSQL(t, store, sql1, sql2, tk1.Session(), tk2.Session(), func(t *testing.T, err1, err2 error) {
require.NoError(t, err1)
require.NoError(t, err2)
})
@@ -108,7 +107,7 @@ func TestConcurrentLockTables(t *testing.T) {
// Test concurrent lock tables write.
sql1 = "lock tables t1 write"
sql2 = "lock tables t1 write"
- testParallelExecSQL(t, store, dom, sql1, sql2, tk1.Session(), tk2.Session(), func(t *testing.T, err1, err2 error) {
+ testParallelExecSQL(t, store, sql1, sql2, tk1.Session(), tk2.Session(), func(t *testing.T, err1, err2 error) {
require.NoError(t, err1)
require.True(t, terror.ErrorEqual(err2, infoschema.ErrTableLocked))
})
@@ -118,7 +117,7 @@ func TestConcurrentLockTables(t *testing.T) {
// Test concurrent lock tables write local.
sql1 = "lock tables t1 write local"
sql2 = "lock tables t1 write local"
- testParallelExecSQL(t, store, dom, sql1, sql2, tk1.Session(), tk2.Session(), func(t *testing.T, err1, err2 error) {
+ testParallelExecSQL(t, store, sql1, sql2, tk1.Session(), tk2.Session(), func(t *testing.T, err1, err2 error) {
require.NoError(t, err1)
require.True(t, terror.ErrorEqual(err2, infoschema.ErrTableLocked))
})
@@ -127,10 +126,9 @@ func TestConcurrentLockTables(t *testing.T) {
tk2.MustExec("unlock tables")
}
-func testParallelExecSQL(t *testing.T, store kv.Storage, dom *domain.Domain, sql1, sql2 string, se1, se2 sessiontypes.Session, f func(t *testing.T, err1, err2 error)) {
- callback := &callback.TestDDLCallback{}
+func testParallelExecSQL(t *testing.T, store kv.Storage, sql1, sql2 string, se1, se2 sessiontypes.Session, f func(t *testing.T, err1, err2 error)) {
times := 0
- callback.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if times != 0 {
return
}
@@ -148,11 +146,8 @@ func testParallelExecSQL(t *testing.T, store kv.Storage, dom *domain.Domain, sql
time.Sleep(5 * time.Millisecond)
}
times++
- }
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- d.SetHook(callback)
+ })
+ defer testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore")
var wg util.WaitGroupWrapper
var err1 error
diff --git a/pkg/ddl/table_test.go b/pkg/ddl/table_test.go
index 2f514bfcf30a4..7b0d4268b877a 100644
--- a/pkg/ddl/table_test.go
+++ b/pkg/ddl/table_test.go
@@ -24,7 +24,6 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/tidb/pkg/ddl"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/meta"
@@ -33,7 +32,6 @@ import (
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/sessionctx"
- "github.com/pingcap/tidb/pkg/sessiontxn"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/testfailpoint"
@@ -222,11 +220,12 @@ func TestTable(t *testing.T) {
[]any{newTblInfo}, ctx, de, store)
ctx = testkit.NewTestKit(t, store).Session()
- require.NoError(t, sessiontxn.NewTxn(context.Background(), ctx))
+ txn, err := newTxn(ctx)
+ require.NoError(t, err)
count := 2000
tbl := testGetTable(t, domain, tblInfo.ID)
for i := 1; i <= count; i++ {
- _, err := tbl.AddRecord(ctx.GetTableCtx(), types.MakeDatums(i, i, i))
+ _, err := tbl.AddRecord(ctx.GetTableCtx(), txn, types.MakeDatums(i, i, i))
require.NoError(t, err)
}
require.NoError(t, ctx.CommitTxn(context.Background()))
@@ -600,10 +599,9 @@ func TestAlterTTL(t *testing.T) {
}
func TestRenameTableIntermediateState(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk2 := testkit.NewTestKit(t, store)
- originHook := dom.DDL().GetHook()
tk.MustExec("create database db1;")
tk.MustExec("create database db2;")
tk.MustExec("create table db1.t(a int);")
@@ -622,10 +620,9 @@ func TestRenameTableIntermediateState(t *testing.T) {
var finishedJobID int64
for _, tc := range testCases {
- hook := &callback.TestDDLCallback{Do: dom}
runInsert := false
var jobID int64 = 0
- fn := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.ID <= finishedJobID {
// The job has been done, OnJobUpdated may be invoked later asynchronously.
// We should skip the done job.
@@ -646,9 +643,7 @@ func TestRenameTableIntermediateState(t *testing.T) {
runInsert = true
jobID = job.ID
}
- }
- hook.OnJobUpdatedExported.Store(&fn)
- dom.DDL().SetHook(hook)
+ })
tk.MustExec(tc.renameSQL)
result := tk.MustQuery(fmt.Sprintf("select * from %s;", tc.finalDB))
if len(tc.errMsg) > 0 {
@@ -659,7 +654,6 @@ func TestRenameTableIntermediateState(t *testing.T) {
tk.MustExec(fmt.Sprintf("delete from %s;", tc.finalDB))
finishedJobID = jobID
}
- dom.DDL().SetHook(originHook)
}
func TestCreateSameTableOrDBOnOwnerChange(t *testing.T) {
diff --git a/pkg/ddl/tests/adminpause/BUILD.bazel b/pkg/ddl/tests/adminpause/BUILD.bazel
index 3650c5b1c36c2..c11bac8d47f86 100644
--- a/pkg/ddl/tests/adminpause/BUILD.bazel
+++ b/pkg/ddl/tests/adminpause/BUILD.bazel
@@ -34,7 +34,6 @@ go_test(
"//pkg/config",
"//pkg/ddl",
"//pkg/ddl/testutil",
- "//pkg/ddl/util/callback",
"//pkg/domain",
"//pkg/errno",
"//pkg/parser/model",
diff --git a/pkg/ddl/tests/adminpause/global.go b/pkg/ddl/tests/adminpause/global.go
index c75a2ee4d1afc..2d6bd9aa79c0d 100644
--- a/pkg/ddl/tests/adminpause/global.go
+++ b/pkg/ddl/tests/adminpause/global.go
@@ -35,8 +35,8 @@ func prepareDomain(t *testing.T) (*domain.Domain, *testkit.TestKit, *testkit.Tes
adminCommandKit := testkit.NewTestKit(t, store)
ddlctrl.ReorgWaitTimeout = 10 * time.Millisecond
- stmtKit.MustExec("set @@global.tidb_ddl_reorg_batch_size = 2")
- stmtKit.MustExec("set @@global.tidb_ddl_reorg_worker_cnt = 1")
+ stmtKit.MustExec("set @@tidb_ddl_reorg_batch_size = 2")
+ stmtKit.MustExec("set @@tidb_ddl_reorg_worker_cnt = 1")
stmtKit = testkit.NewTestKit(t, store)
stmtKit.MustExec("use test")
diff --git a/pkg/ddl/tests/adminpause/pause_cancel_test.go b/pkg/ddl/tests/adminpause/pause_cancel_test.go
index 7f7f642c68913..3a4c33c3d3fd9 100644
--- a/pkg/ddl/tests/adminpause/pause_cancel_test.go
+++ b/pkg/ddl/tests/adminpause/pause_cancel_test.go
@@ -22,7 +22,6 @@ import (
"testing"
testddlutil "github.com/pingcap/tidb/pkg/ddl/testutil"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/parser/model"
@@ -127,11 +126,7 @@ func pauseAndCancelStmt(t *testing.T, stmtKit *testkit.TestKit, adminCommandKit
stmtKit.MustExec(prepareStmt)
}
- var hook = &callback.TestDDLCallback{Do: dom}
- originalHook := dom.DDL().GetHook()
-
- hook.OnJobRunBeforeExported = pauseFunc
- dom.DDL().SetHook(hook.Clone())
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", pauseFunc)
isPaused.Store(false)
isCancelled.Store(false)
@@ -151,7 +146,7 @@ func pauseAndCancelStmt(t *testing.T, stmtKit *testkit.TestKit, adminCommandKit
}
// Release the hook, so that we could run the `rollbackStmts` successfully.
- dom.DDL().SetHook(originalHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore")
testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/beforeRefreshJob")
for _, rollbackStmt := range stmtCase.rollbackStmts {
diff --git a/pkg/ddl/tests/adminpause/pause_negative_test.go b/pkg/ddl/tests/adminpause/pause_negative_test.go
index f283d96d2c2ba..eb1383778ba1f 100644
--- a/pkg/ddl/tests/adminpause/pause_negative_test.go
+++ b/pkg/ddl/tests/adminpause/pause_negative_test.go
@@ -23,17 +23,17 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/ddl"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/util/sqlexec"
"github.com/stretchr/testify/require"
"go.uber.org/atomic"
)
func TestPauseOnWriteConflict(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, dbTestLease)
+ store := testkit.CreateMockStoreWithSchemaLease(t, dbTestLease)
tk1 := testkit.NewTestKit(t, store)
tk2 := testkit.NewTestKit(t, store)
@@ -44,16 +44,11 @@ func TestPauseOnWriteConflict(t *testing.T) {
var pauseErr error
var pauseRS []sqlexec.RecordSet
- hook := &callback.TestDDLCallback{Do: dom}
- d := dom.DDL()
- originalHook := d.GetHook()
- defer d.SetHook(originalHook)
-
var adminMutex sync.RWMutex
jobID := atomic.NewInt64(0)
// Test when pause cannot be retried and adding index succeeds.
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
adminMutex.Lock()
if job.Type == model.ActionAddIndex && job.State == model.JobStateRunning &&
job.SchemaState == model.StateWriteReorganization {
@@ -69,14 +64,13 @@ func TestPauseOnWriteConflict(t *testing.T) {
pauseRS, pauseErr = tk2.Session().Execute(context.Background(), stmt)
}
adminMutex.Unlock()
- }
- d.SetHook(hook.Clone())
+ })
tk1.MustExec("alter table t add index (id)")
require.EqualError(t, pauseErr, "mock failed admin command on ddl jobs")
var cancelRS []sqlexec.RecordSet
var cancelErr error
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
adminMutex.Lock()
if job.Type == model.ActionAddIndex && job.State == model.JobStateRunning &&
job.SchemaState == model.StateWriteReorganization {
@@ -89,8 +83,7 @@ func TestPauseOnWriteConflict(t *testing.T) {
cancelRS, cancelErr = tk2.Session().Execute(context.Background(), stmt)
}
adminMutex.Unlock()
- }
- d.SetHook(hook.Clone())
+ })
tk1.MustGetErrCode("alter table t add index (id)", errno.ErrCancelledDDLJob)
require.NoError(t, pauseErr)
@@ -102,7 +95,7 @@ func TestPauseOnWriteConflict(t *testing.T) {
}
func TestPauseFailedOnCommit(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, dbTestLease)
+ store := testkit.CreateMockStoreWithSchemaLease(t, dbTestLease)
tk1 := testkit.NewTestKit(t, store)
tk2 := testkit.NewTestKit(t, store)
@@ -110,18 +103,13 @@ func TestPauseFailedOnCommit(t *testing.T) {
tk1.MustExec("use test")
tk1.MustExec("create table t(id int)")
- d := dom.DDL()
-
jobID := atomic.NewInt64(0)
var pauseErr error
var jobErrs []error
var adminMutex sync.RWMutex
- hook := &callback.TestDDLCallback{Do: dom}
- originalHook := d.GetHook()
- defer d.SetHook(originalHook)
// Test when pause cannot be retried and adding index succeeds.
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
adminMutex.Lock()
if job.Type == model.ActionAddIndex && job.State == model.JobStateRunning &&
job.SchemaState == model.StateWriteReorganization {
@@ -134,8 +122,7 @@ func TestPauseFailedOnCommit(t *testing.T) {
jobErrs, pauseErr = ddl.PauseJobs(tk2.Session(), []int64{jobID.Load()})
}
adminMutex.Unlock()
- }
- d.SetHook(hook.Clone())
+ })
tk1.MustExec("alter table t add index (id)")
require.EqualError(t, pauseErr, "mock commit failed on admin command on ddl jobs")
diff --git a/pkg/ddl/tests/adminpause/pause_resume_test.go b/pkg/ddl/tests/adminpause/pause_resume_test.go
index 323d85238d9aa..ba3a6d26175ef 100644
--- a/pkg/ddl/tests/adminpause/pause_resume_test.go
+++ b/pkg/ddl/tests/adminpause/pause_resume_test.go
@@ -24,7 +24,6 @@ import (
"github.com/pingcap/failpoint"
testddlutil "github.com/pingcap/tidb/pkg/ddl/testutil"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/parser/model"
@@ -158,12 +157,7 @@ func pauseResumeAndCancel(t *testing.T, stmtKit *testkit.TestKit, adminCommandKi
stmtKit.MustExec(prepareStmt)
}
- var hook = &callback.TestDDLCallback{Do: dom}
- originalHook := dom.DDL().GetHook()
-
- hook.OnJobRunBeforeExported = pauseFunc
- var rf = resumeFunc
- hook.OnJobUpdatedExported.Store(&rf)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", resumeFunc)
Logger.Debug("pauseResumeAndCancel: statement execute", zap.String("DDL Statement", stmtCase.stmt))
if stmtCase.isJobPausable {
@@ -171,7 +165,7 @@ func pauseResumeAndCancel(t *testing.T, stmtKit *testkit.TestKit, adminCommandKi
testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/beforeLoadAndDeliverJobs", func() {
cancelFunc()
})
- dom.DDL().SetHook(hook.Clone())
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", pauseFunc)
stmtKit.MustGetErrCode(stmtCase.stmt, errno.ErrCancelledDDLJob)
testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/beforeLoadAndDeliverJobs")
@@ -179,7 +173,7 @@ func pauseResumeAndCancel(t *testing.T, stmtKit *testkit.TestKit, adminCommandKi
verifyCancelResult(t, adminCommandKit)
} else {
- dom.DDL().SetHook(hook.Clone())
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", pauseFunc)
stmtKit.MustExec(stmtCase.stmt)
Logger.Info("pauseResumeAndCancel: statement execution should finish successfully.")
@@ -193,7 +187,7 @@ func pauseResumeAndCancel(t *testing.T, stmtKit *testkit.TestKit, adminCommandKi
verifyPauseResult(t, adminCommandKit)
verifyResumeResult(t, adminCommandKit)
} else {
- dom.DDL().SetHook(hook.Clone())
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", pauseFunc)
stmtKit.MustExec(stmtCase.stmt)
require.False(t, isPaused)
@@ -202,7 +196,8 @@ func pauseResumeAndCancel(t *testing.T, stmtKit *testkit.TestKit, adminCommandKi
}
// Should not affect the 'stmtCase.rollbackStmts'
- dom.DDL().SetHook(originalHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore")
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
// Statement in `stmtCase` will be finished successfully all the way, need to roll it back.
for _, rollbackStmt := range stmtCase.rollbackStmts {
diff --git a/pkg/ddl/tests/fail/BUILD.bazel b/pkg/ddl/tests/fail/BUILD.bazel
index 7a35750149f79..ffd6263510607 100644
--- a/pkg/ddl/tests/fail/BUILD.bazel
+++ b/pkg/ddl/tests/fail/BUILD.bazel
@@ -14,6 +14,7 @@ go_test(
"//pkg/config",
"//pkg/ddl",
"//pkg/ddl/schematracker",
+ "//pkg/ddl/schemaver",
"//pkg/ddl/testutil",
"//pkg/ddl/util",
"//pkg/domain",
diff --git a/pkg/ddl/tests/fail/fail_db_test.go b/pkg/ddl/tests/fail/fail_db_test.go
index a8c4b3959ce42..a8dc18190451b 100644
--- a/pkg/ddl/tests/fail/fail_db_test.go
+++ b/pkg/ddl/tests/fail/fail_db_test.go
@@ -25,6 +25,7 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/ddl"
"github.com/pingcap/tidb/pkg/ddl/schematracker"
+ "github.com/pingcap/tidb/pkg/ddl/schemaver"
"github.com/pingcap/tidb/pkg/ddl/testutil"
ddlutil "github.com/pingcap/tidb/pkg/ddl/util"
"github.com/pingcap/tidb/pkg/domain"
@@ -48,6 +49,10 @@ type failedSuite struct {
}
func createFailDBSuite(t *testing.T) (s *failedSuite) {
+ return createFailDBSuiteWithLease(t, 200*time.Millisecond)
+}
+
+func createFailDBSuiteWithLease(t *testing.T, lease time.Duration) (s *failedSuite) {
s = new(failedSuite)
var err error
s.store, err = mockstore.NewMockStore(
@@ -57,7 +62,7 @@ func createFailDBSuite(t *testing.T) (s *failedSuite) {
}),
)
require.NoError(t, err)
- session.SetSchemaLease(200 * time.Millisecond)
+ session.SetSchemaLease(lease)
s.dom, err = session.BootstrapSession(s.store)
require.NoError(t, err)
@@ -220,7 +225,7 @@ func TestAddIndexFailed(t *testing.T) {
// TestFailSchemaSyncer test when the schema syncer is done,
// should prohibit DML executing until the syncer is restartd by loadSchemaInLoop.
func TestFailSchemaSyncer(t *testing.T) {
- s := createFailDBSuite(t)
+ s := createFailDBSuiteWithLease(t, 10*time.Second)
tk := testkit.NewTestKit(t, s.store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
@@ -232,7 +237,7 @@ func TestFailSchemaSyncer(t *testing.T) {
domain.SchemaOutOfDateRetryTimes.Store(originalRetryTimes)
}()
require.True(t, s.dom.SchemaValidator.IsStarted())
- mockSyncer, ok := s.dom.DDL().SchemaSyncer().(*ddl.MockSchemaSyncer)
+ mockSyncer, ok := s.dom.DDL().SchemaSyncer().(*schemaver.MemSyncer)
require.True(t, ok)
// make reload failed.
diff --git a/pkg/ddl/tests/fastcreatetable/BUILD.bazel b/pkg/ddl/tests/fastcreatetable/BUILD.bazel
index 85bdc4df8ad74..88429b3a0829d 100644
--- a/pkg/ddl/tests/fastcreatetable/BUILD.bazel
+++ b/pkg/ddl/tests/fastcreatetable/BUILD.bazel
@@ -8,12 +8,17 @@ go_test(
"main_test.go",
],
flaky = True,
+ shard_count = 3,
deps = [
"//pkg/config",
"//pkg/ddl",
+ "//pkg/parser/model",
"//pkg/server",
"//pkg/testkit",
+ "//pkg/testkit/testfailpoint",
"//pkg/testkit/testsetup",
+ "//pkg/util",
+ "@com_github_stretchr_testify//require",
"@org_uber_go_goleak//:goleak",
],
)
diff --git a/pkg/ddl/tests/fastcreatetable/fastcreatetable_test.go b/pkg/ddl/tests/fastcreatetable/fastcreatetable_test.go
index df47fb3dda5fc..1b20945b6e9ac 100644
--- a/pkg/ddl/tests/fastcreatetable/fastcreatetable_test.go
+++ b/pkg/ddl/tests/fastcreatetable/fastcreatetable_test.go
@@ -16,9 +16,15 @@ package fastcreatetable
import (
"testing"
+ "time"
+ "github.com/pingcap/tidb/pkg/ddl"
+ "github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/server"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
+ "github.com/pingcap/tidb/pkg/util"
+ "github.com/stretchr/testify/require"
)
func TestSwitchFastCreateTable(t *testing.T) {
@@ -87,3 +93,55 @@ func TestDDL(t *testing.T) {
tk.MustExec("create table db.tb1(id int)")
tk.MustExec("create table db.tb2(id int)")
}
+
+func TestMergedJob(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ var wg util.WaitGroupWrapper
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("set global tidb_enable_fast_create_table=ON")
+
+ startSchedule := make(chan struct{})
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/beforeLoadAndDeliverJobs", func() {
+ <-startSchedule
+ })
+
+ // this job will be run first.
+ wg.Run(func() {
+ tk1 := testkit.NewTestKit(t, store)
+ tk1.MustExec("use test")
+ tk1.MustExec("create table t(a int)")
+ })
+ require.Eventually(t, func() bool {
+ gotJobs, err := ddl.GetAllDDLJobs(tk.Session())
+ require.NoError(t, err)
+ return len(gotJobs) == 1
+ }, 10*time.Second, 100*time.Millisecond)
+
+ // below 2 jobs are merged into 1, they will fail together.
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/afterGetJobFromLimitCh", func(ch chan *ddl.JobWrapper) {
+ require.Eventually(t, func() bool {
+ return len(ch) == 1
+ }, 10*time.Second, 100*time.Millisecond)
+ })
+ wg.Run(func() {
+ tk1 := testkit.NewTestKit(t, store)
+ tk1.MustExec("use test")
+ tk1.MustExecToErr("create table t(a int)")
+ })
+ wg.Run(func() {
+ tk1 := testkit.NewTestKit(t, store)
+ tk1.MustExec("use test")
+ tk1.MustExecToErr("create table t1(a int)")
+ })
+ require.Eventually(t, func() bool {
+ gotJobs, err := ddl.GetAllDDLJobs(tk.Session())
+ require.NoError(t, err)
+ return len(gotJobs) == 2 && gotJobs[1].Type == model.ActionCreateTables
+ }, 10*time.Second, 100*time.Millisecond)
+
+ // start to run the jobs
+ close(startSchedule)
+ wg.Wait()
+}
diff --git a/pkg/ddl/tests/fk/foreign_key_test.go b/pkg/ddl/tests/fk/foreign_key_test.go
index 108c298f23a88..0a7f56d5f5f74 100644
--- a/pkg/ddl/tests/fk/foreign_key_test.go
+++ b/pkg/ddl/tests/fk/foreign_key_test.go
@@ -960,7 +960,7 @@ func getTableInfo(t *testing.T, dom *domain.Domain, db, tb string) *model.TableI
is := dom.InfoSchema()
tbl, err := is.TableByName(context.Background(), model.NewCIStr(db), model.NewCIStr(tb))
require.NoError(t, err)
- _, exist := is.TableByID(tbl.Meta().ID)
+ _, exist := is.TableByID(context.Background(), tbl.Meta().ID)
require.True(t, exist)
return tbl.Meta()
}
diff --git a/pkg/ddl/tests/indexmerge/BUILD.bazel b/pkg/ddl/tests/indexmerge/BUILD.bazel
index fe7c4a1464af3..ede375edfed6e 100644
--- a/pkg/ddl/tests/indexmerge/BUILD.bazel
+++ b/pkg/ddl/tests/indexmerge/BUILD.bazel
@@ -15,7 +15,6 @@ go_test(
"//pkg/ddl",
"//pkg/ddl/ingest",
"//pkg/ddl/testutil",
- "//pkg/ddl/util/callback",
"//pkg/domain",
"//pkg/errno",
"//pkg/kv",
diff --git a/pkg/ddl/tests/indexmerge/merge_test.go b/pkg/ddl/tests/indexmerge/merge_test.go
index 712578aea4a80..1504c50fb27b1 100644
--- a/pkg/ddl/tests/indexmerge/merge_test.go
+++ b/pkg/ddl/tests/indexmerge/merge_test.go
@@ -23,7 +23,6 @@ import (
"github.com/pingcap/tidb/pkg/ddl"
"github.com/pingcap/tidb/pkg/ddl/ingest"
"github.com/pingcap/tidb/pkg/ddl/testutil"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/model"
@@ -48,11 +47,7 @@ func TestAddIndexMergeProcess(t *testing.T) {
var checkErr error
var runDML, backfillDone bool
- originHook := dom.DDL().GetHook()
- callback := &callback.TestDDLCallback{
- Do: dom,
- }
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if !runDML && job.Type == model.ActionAddIndex && job.SchemaState == model.StateWriteReorganization {
idx := testutil.FindIdxInfo(dom, "test", "t", "idx")
if idx == nil || idx.BackfillState != model.BackfillStateRunning {
@@ -67,11 +62,9 @@ func TestAddIndexMergeProcess(t *testing.T) {
// Write record 7 to the temporary index.
_, checkErr = tk2.Exec("insert into t values (7, 8, 9);")
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- dom.DDL().SetHook(callback)
+ })
tk.MustExec("alter table t add index idx(c1);")
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
require.True(t, backfillDone)
require.True(t, runDML)
require.NoError(t, checkErr)
@@ -81,7 +74,7 @@ func TestAddIndexMergeProcess(t *testing.T) {
}
func TestAddPrimaryKeyMergeProcess(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, 0)
+ store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, time.Second)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk2 := testkit.NewTestKit(t, store)
@@ -136,11 +129,7 @@ func TestAddIndexMergeVersionIndexValue(t *testing.T) {
var checkErr error
var runDML bool
var tblID, idxID int64
- originHook := dom.DDL().GetHook()
- callback := &callback.TestDDLCallback{
- Do: dom,
- }
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if !runDML && job.Type == model.ActionAddIndex && job.SchemaState == model.StateWriteReorganization {
idx := testutil.FindIdxInfo(dom, "test", "t", "idx")
if idx == nil || idx.BackfillState != model.BackfillStateReadyToMerge {
@@ -151,11 +140,9 @@ func TestAddIndexMergeVersionIndexValue(t *testing.T) {
idxID = idx.ID
_, checkErr = tk2.Exec("insert into t values (1);")
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- dom.DDL().SetHook(callback)
+ })
tk.MustExec("alter table t add unique index idx(c1);")
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
require.True(t, runDML)
require.NoError(t, checkErr)
tk.MustExec("admin check table t;")
@@ -191,11 +178,7 @@ func TestAddIndexMergeIndexUntouchedValue(t *testing.T) {
var checkErrs []error
var runInsert bool
var runUpdate bool
- originHook := dom.DDL().GetHook()
- callback := &callback.TestDDLCallback{
- Do: dom,
- }
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.Type != model.ActionAddIndex || job.SchemaState != model.StateWriteReorganization {
return
}
@@ -223,11 +206,9 @@ func TestAddIndexMergeIndexUntouchedValue(t *testing.T) {
_, err = tk2.Exec("commit;")
checkErrs = append(checkErrs, err)
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- dom.DDL().SetHook(callback)
+ })
tk.MustExec("alter table t add index idx(c);")
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
require.True(t, runUpdate)
for _, err := range checkErrs {
require.NoError(t, err)
@@ -247,7 +228,7 @@ func TestAddIndexMergeIndexUntouchedValue(t *testing.T) {
//
// [kv:1062]Duplicate entry '1' for key 't.idx'
func TestCreateUniqueIndexKeyExist(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -261,11 +242,7 @@ func TestCreateUniqueIndexKeyExist(t *testing.T) {
// If waitReorg timeout, the worker may enter writeReorg more than 2 times.
reorgTime := 0
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- callback := &callback.TestDDLCallback{}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if t.Failed() {
return
}
@@ -300,16 +277,14 @@ func TestCreateUniqueIndexKeyExist(t *testing.T) {
assert.NoError(t, err)
// (1, 7), (2, 7), (5, 7), (8, 8), (10, 10), (0, 9)
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(callback)
+ })
tk.MustExec("alter table t add unique index idx((a*b+1))")
tk.MustExec("admin check table t")
tk.MustQuery("select * from t order by a, b").Check(testkit.Rows("0 9", "1 7", "2 7", "5 7", "8 8", "10 10"))
}
func TestAddIndexMergeIndexUpdateOnDeleteOnly(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk2 := testkit.NewTestKit(t, store)
@@ -329,11 +304,7 @@ func TestAddIndexMergeIndexUpdateOnDeleteOnly(t *testing.T) {
tk.MustExec("set @@global.tidb_txn_assertion_level = 'STRICT';")
var checkErrs []error
- originHook := dom.DDL().GetHook()
- callback := &callback.TestDDLCallback{
- Do: dom,
- }
- onJobUpdatedBefore := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.SchemaState == model.StateDeleteOnly {
for _, sql := range updateSQLs {
_, err := tk2.Exec(sql)
@@ -342,11 +313,9 @@ func TestAddIndexMergeIndexUpdateOnDeleteOnly(t *testing.T) {
}
}
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedBefore)
- dom.DDL().SetHook(callback)
+ })
tk.MustExec("alter table t add index idx(b);")
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
for _, err := range checkErrs {
require.NoError(t, err)
}
@@ -354,7 +323,7 @@ func TestAddIndexMergeIndexUpdateOnDeleteOnly(t *testing.T) {
}
func TestAddIndexMergeDeleteUniqueOnWriteOnly(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -364,11 +333,7 @@ func TestAddIndexMergeDeleteUniqueOnWriteOnly(t *testing.T) {
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- callback := &callback.TestDDLCallback{}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if t.Failed() {
return
}
@@ -383,9 +348,7 @@ func TestAddIndexMergeDeleteUniqueOnWriteOnly(t *testing.T) {
_, err = tk1.Exec("delete from t where b = 7;")
assert.NoError(t, err)
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(callback)
+ })
tk.MustExec("alter table t add unique index idx(a);")
tk.MustExec("admin check table t;")
}
@@ -413,7 +376,7 @@ func TestAddIndexMergeDeleteNullUnique(t *testing.T) {
}
func TestAddIndexMergeDoubleDelete(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -422,11 +385,7 @@ func TestAddIndexMergeDoubleDelete(t *testing.T) {
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- callback := &callback.TestDDLCallback{}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if t.Failed() {
return
}
@@ -435,9 +394,7 @@ func TestAddIndexMergeDoubleDelete(t *testing.T) {
_, err := tk1.Exec("insert into t values (1, 1);")
assert.NoError(t, err)
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(callback)
+ })
ddl.MockDMLExecution = func() {
_, err := tk1.Exec("delete from t where id = 1;")
@@ -473,12 +430,9 @@ func TestAddIndexMergeConflictWithPessimistic(t *testing.T) {
tk.MustExec("set @@global.tidb_ddl_enable_fast_reorg = 1;")
tk.MustExec("set @@global.tidb_enable_metadata_lock = 0;")
- originHook := dom.DDL().GetHook()
- callback := &callback.TestDDLCallback{Do: dom}
-
runPessimisticTxn := false
afterPessDML := make(chan struct{}, 1)
- callback.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if t.Failed() {
return
}
@@ -502,8 +456,7 @@ func TestAddIndexMergeConflictWithPessimistic(t *testing.T) {
assert.NoError(t, err)
afterPessDML <- struct{}{}
}
- }
- dom.DDL().SetHook(callback)
+ })
afterCommit := make(chan struct{}, 1)
go func() {
tk.MustExec("alter table t add index idx(a);")
@@ -519,13 +472,13 @@ func TestAddIndexMergeConflictWithPessimistic(t *testing.T) {
<-afterPessDML
tk2.MustExec("rollback;")
<-afterCommit
- dom.DDL().SetHook(originHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore")
tk.MustExec("admin check table t;")
tk.MustQuery("select * from t;").Check(testkit.Rows("1 2"))
}
func TestAddIndexMergeInsertOnMerging(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -534,11 +487,7 @@ func TestAddIndexMergeInsertOnMerging(t *testing.T) {
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- callback := &callback.TestDDLCallback{}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if t.Failed() {
return
}
@@ -553,9 +502,7 @@ func TestAddIndexMergeInsertOnMerging(t *testing.T) {
_, err = tk1.Exec("delete from t where b = 7")
assert.NoError(t, err)
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(callback)
+ })
ddl.MockDMLExecutionStateMerging = func() {
_, err := tk1.Exec("insert into t values (5, 8);")
@@ -602,7 +549,7 @@ func TestAddIndexMergeReplaceOnMerging(t *testing.T) {
}
func TestAddIndexMergeInsertToDeletedTempIndex(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -612,11 +559,7 @@ func TestAddIndexMergeInsertToDeletedTempIndex(t *testing.T) {
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- callback := &callback.TestDDLCallback{}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if t.Failed() {
return
}
@@ -636,9 +579,7 @@ func TestAddIndexMergeInsertToDeletedTempIndex(t *testing.T) {
_, err = tk1.Exec("insert into t values (5, 8);")
assert.Error(t, err)
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(callback)
+ })
tk.MustExec("alter table t add unique index idx(a);")
tk.MustExec("admin check table t;")
@@ -646,7 +587,7 @@ func TestAddIndexMergeInsertToDeletedTempIndex(t *testing.T) {
}
func TestAddIndexMergeReplaceDelete(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -655,11 +596,7 @@ func TestAddIndexMergeReplaceDelete(t *testing.T) {
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- callback := &callback.TestDDLCallback{}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if t.Failed() {
return
}
@@ -668,9 +605,7 @@ func TestAddIndexMergeReplaceDelete(t *testing.T) {
_, err := tk1.Exec("insert into t values (1, 1);")
assert.NoError(t, err)
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(callback)
+ })
ddl.MockDMLExecutionMerging = func() {
_, err := tk1.Exec("replace into t values (2, 1);")
@@ -686,7 +621,7 @@ func TestAddIndexMergeReplaceDelete(t *testing.T) {
}
func TestAddIndexMergeDeleteDifferentHandle(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -696,12 +631,8 @@ func TestAddIndexMergeDeleteDifferentHandle(t *testing.T) {
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- callback := &callback.TestDDLCallback{}
runDML := false
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if t.Failed() || runDML {
return
}
@@ -716,9 +647,7 @@ func TestAddIndexMergeDeleteDifferentHandle(t *testing.T) {
assert.NoError(t, err)
runDML = true
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(callback)
+ })
ddl.MockDMLExecution = func() {
// It is too late to remove the duplicated index value.
@@ -733,7 +662,7 @@ func TestAddIndexMergeDeleteDifferentHandle(t *testing.T) {
}
func TestAddIndexDecodeTempIndexCommonHandle(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -743,12 +672,8 @@ func TestAddIndexDecodeTempIndexCommonHandle(t *testing.T) {
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- callback := &callback.TestDDLCallback{}
runDML := false
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if t.Failed() || runDML {
return
}
@@ -763,9 +688,7 @@ func TestAddIndexDecodeTempIndexCommonHandle(t *testing.T) {
assert.NoError(t, err)
runDML = true
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(callback)
+ })
tk.MustExec("alter table t add unique index idx(c);")
tk.MustExec("admin check table t;")
@@ -773,7 +696,7 @@ func TestAddIndexDecodeTempIndexCommonHandle(t *testing.T) {
}
func TestAddIndexInsertIgnoreOnBackfill(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -782,12 +705,8 @@ func TestAddIndexInsertIgnoreOnBackfill(t *testing.T) {
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- callback := &callback.TestDDLCallback{}
runDML := false
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if t.Failed() || runDML {
return
}
@@ -801,9 +720,7 @@ func TestAddIndexInsertIgnoreOnBackfill(t *testing.T) {
assert.NoError(t, err)
runDML = true
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(callback)
+ })
tk.MustExec("alter table t add unique index idx(b);")
tk.MustExec("admin check table t;")
@@ -811,7 +728,7 @@ func TestAddIndexInsertIgnoreOnBackfill(t *testing.T) {
}
func TestAddIndexMultipleDelete(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -821,11 +738,7 @@ func TestAddIndexMultipleDelete(t *testing.T) {
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- callback := &callback.TestDDLCallback{}
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if t.Failed() {
return
}
@@ -837,9 +750,7 @@ func TestAddIndexMultipleDelete(t *testing.T) {
_, err := tk1.Exec("delete from t where id in (2, 3);")
assert.NoError(t, err)
}
- }
- callback.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- d.SetHook(callback)
+ })
ddl.MockDMLExecution = func() {
_, err := tk1.Exec("delete from t where id = 1;")
@@ -853,7 +764,7 @@ func TestAddIndexMultipleDelete(t *testing.T) {
}
func TestAddIndexDuplicateAndWriteConflict(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -863,12 +774,8 @@ func TestAddIndexDuplicateAndWriteConflict(t *testing.T) {
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- callback := &callback.TestDDLCallback{}
var runCancel bool
- callback.OnJobRunAfterExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunAfter", func(job *model.Job) {
if t.Failed() || runCancel {
return
}
@@ -882,8 +789,7 @@ func TestAddIndexDuplicateAndWriteConflict(t *testing.T) {
assert.NoError(t, err)
runCancel = true
}
- }
- d.SetHook(callback)
+ })
tk.MustGetErrCode("alter table t add unique index idx(b);", errno.ErrCancelledDDLJob)
tk.MustExec("admin check table t;")
@@ -891,7 +797,7 @@ func TestAddIndexDuplicateAndWriteConflict(t *testing.T) {
}
func TestAddIndexUpdateUntouchedValues(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -901,12 +807,8 @@ func TestAddIndexUpdateUntouchedValues(t *testing.T) {
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
- d := dom.DDL()
- originalCallback := d.GetHook()
- defer d.SetHook(originalCallback)
- callback := &callback.TestDDLCallback{}
var runDML bool
- callback.OnJobRunAfterExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunAfter", func(job *model.Job) {
if t.Failed() || runDML {
return
}
@@ -923,8 +825,7 @@ func TestAddIndexUpdateUntouchedValues(t *testing.T) {
assert.NoError(t, err)
runDML = true
}
- }
- d.SetHook(callback)
+ })
tk.MustGetErrCode("alter table t add unique index idx(b);", errno.ErrDupEntry)
tk.MustExec("admin check table t;")
diff --git a/pkg/ddl/tests/metadatalock/BUILD.bazel b/pkg/ddl/tests/metadatalock/BUILD.bazel
index 0314bcf91fd6d..bc8fc8a444520 100644
--- a/pkg/ddl/tests/metadatalock/BUILD.bazel
+++ b/pkg/ddl/tests/metadatalock/BUILD.bazel
@@ -16,8 +16,8 @@ go_test(
"//pkg/errno",
"//pkg/server",
"//pkg/testkit",
+ "//pkg/testkit/testfailpoint",
"//pkg/testkit/testsetup",
- "@com_github_pingcap_failpoint//:failpoint",
"@com_github_stretchr_testify//require",
"@org_uber_go_goleak//:goleak",
],
diff --git a/pkg/ddl/tests/metadatalock/mdl_test.go b/pkg/ddl/tests/metadatalock/mdl_test.go
index 4f5bbc7d622ed..262cb745e7a63 100644
--- a/pkg/ddl/tests/metadatalock/mdl_test.go
+++ b/pkg/ddl/tests/metadatalock/mdl_test.go
@@ -20,11 +20,11 @@ import (
"testing"
"time"
- "github.com/pingcap/failpoint"
ingesttestutil "github.com/pingcap/tidb/pkg/ddl/ingest/testutil"
mysql "github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/server"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/stretchr/testify/require"
)
@@ -1360,10 +1360,7 @@ func TestMDLUpdateEtcdFail(t *testing.T) {
tk.MustExec("use test")
tk.MustExec("create table t(a int);")
- require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/mockUpdateMDLToETCDError", `3*return(true)`))
- defer func() {
- require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/mockUpdateMDLToETCDError"))
- }()
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ddl/schemaver/mockUpdateMDLToETCDError", `3*return(true)`)
tk.MustExec("alter table test.t add column c int")
}
diff --git a/pkg/ddl/tests/partition/BUILD.bazel b/pkg/ddl/tests/partition/BUILD.bazel
index 4470dc519eeb9..714e27cca252a 100644
--- a/pkg/ddl/tests/partition/BUILD.bazel
+++ b/pkg/ddl/tests/partition/BUILD.bazel
@@ -14,7 +14,6 @@ go_test(
"//pkg/ddl",
"//pkg/ddl/logutil",
"//pkg/ddl/testutil",
- "//pkg/ddl/util/callback",
"//pkg/domain",
"//pkg/errno",
"//pkg/kv",
@@ -32,6 +31,7 @@ go_test(
"//pkg/tablecodec",
"//pkg/testkit",
"//pkg/testkit/external",
+ "//pkg/testkit/testfailpoint",
"//pkg/testkit/testsetup",
"//pkg/types",
"//pkg/util/codec",
diff --git a/pkg/ddl/tests/partition/db_partition_test.go b/pkg/ddl/tests/partition/db_partition_test.go
index f2ceb62b8250f..bc189f6e0c742 100644
--- a/pkg/ddl/tests/partition/db_partition_test.go
+++ b/pkg/ddl/tests/partition/db_partition_test.go
@@ -30,7 +30,6 @@ import (
"github.com/pingcap/tidb/pkg/ddl"
"github.com/pingcap/tidb/pkg/ddl/logutil"
"github.com/pingcap/tidb/pkg/ddl/testutil"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/kv"
@@ -48,6 +47,7 @@ import (
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/external"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/codec"
"github.com/pingcap/tidb/pkg/util/dbterror"
@@ -1280,8 +1280,8 @@ func TestDropPartitionWithGlobalIndex(t *testing.T) {
pid := tt.Meta().Partition.Definitions[1].ID
tk.MustExec(`INSERT INTO test_global VALUES (1, 1, 1), (2, 2, 2), (11, 3, 3), (12, 4, 4)`)
- tk.MustExec("Alter Table test_global Add Unique Index idx_b (b);")
- tk.MustExec("Alter Table test_global Add Unique Index idx_c (c);")
+ tk.MustExec("Alter Table test_global Add Unique Index idx_b (b) GLOBAL")
+ tk.MustExec("Alter Table test_global Add Unique Index idx_c (c) global")
tk.MustExec("alter table test_global drop partition p2;")
result := tk.MustQuery("select * from test_global;")
@@ -1317,8 +1317,8 @@ func TestDropMultiPartitionWithGlobalIndex(t *testing.T) {
tt := external.GetTableByName(t, tk, "test", "test_global")
pid := tt.Meta().Partition.Definitions[1].ID
- tk.MustExec("Alter Table test_global Add Unique Index idx_b (b);")
- tk.MustExec("Alter Table test_global Add Unique Index idx_c (c);")
+ tk.MustExec("Alter Table test_global Add Unique Index idx_b (b) global")
+ tk.MustExec("Alter Table test_global Add Unique Index idx_c (c) global")
tk.MustExec(`INSERT INTO test_global VALUES (1, 1, 1), (2, 2, 2), (11, 3, 3), (12, 4, 4), (21, 21, 21), (29, 29, 29)`)
tk.MustExec("alter table test_global drop partition p1, p2;")
@@ -1338,7 +1338,7 @@ func TestDropMultiPartitionWithGlobalIndex(t *testing.T) {
}
func TestGlobalIndexInsertInDropPartition(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("set tidb_enable_global_index=true")
@@ -1352,19 +1352,17 @@ func TestGlobalIndexInsertInDropPartition(t *testing.T) {
partition p2 values less than (20),
partition p3 values less than (30)
);`)
- tk.MustExec("alter table test_global add unique index idx_b (b);")
+ tk.MustExec("alter table test_global add unique index idx_b (b) global")
tk.MustExec("insert into test_global values (1, 1, 1), (8, 8, 8), (11, 11, 11), (12, 12, 12);")
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
assert.Equal(t, model.ActionDropTablePartition, job.Type)
if job.SchemaState == model.StateDeleteOnly {
tk2 := testkit.NewTestKit(t, store)
tk2.MustExec("use test")
tk2.MustExec("insert into test_global values (9, 9, 9)")
}
- }
- dom.DDL().SetHook(hook)
+ })
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
@@ -1375,7 +1373,7 @@ func TestGlobalIndexInsertInDropPartition(t *testing.T) {
}
func TestGlobalIndexUpdateInDropPartition(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("set tidb_enable_global_index=true")
@@ -1389,19 +1387,17 @@ func TestGlobalIndexUpdateInDropPartition(t *testing.T) {
partition p2 values less than (20),
partition p3 values less than (30)
);`)
- tk.MustExec("alter table test_global add unique index idx_b (b);")
+ tk.MustExec("alter table test_global add unique index idx_b (b) global")
tk.MustExec("insert into test_global values (1, 1, 1), (8, 8, 8), (11, 11, 11), (12, 12, 12);")
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
assert.Equal(t, model.ActionDropTablePartition, job.Type)
if job.SchemaState == model.StateDeleteOnly {
tk2 := testkit.NewTestKit(t, store)
tk2.MustExec("use test")
tk2.MustExec("update test_global set a = 2 where a = 11")
}
- }
- dom.DDL().SetHook(hook)
+ })
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
@@ -1428,8 +1424,8 @@ func TestTruncatePartitionWithGlobalIndex(t *testing.T) {
tt := external.GetTableByName(t, tk, "test", "test_global")
pid := tt.Meta().Partition.Definitions[1].ID
- tk.MustExec("Alter Table test_global Add Unique Index idx_b (b);")
- tk.MustExec("Alter Table test_global Add Unique Index idx_c (c);")
+ tk.MustExec("Alter Table test_global Add Unique Index idx_b (b) global")
+ tk.MustExec("Alter Table test_global Add Unique Index idx_c (c) global")
tk.MustExec(`INSERT INTO test_global VALUES (1, 1, 1), (2, 2, 2), (11, 3, 3), (12, 4, 4), (15, 15, 15)`)
tk2 := testkit.NewTestKit(t, store)
@@ -1496,7 +1492,7 @@ func TestTruncatePartitionWithGlobalIndex(t *testing.T) {
}
func TestGlobalIndexUpdateInTruncatePartition(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("set tidb_enable_global_index=true")
@@ -1512,15 +1508,11 @@ func TestGlobalIndexUpdateInTruncatePartition(t *testing.T) {
partition p2 values less than (20),
partition p3 values less than (30)
);`)
- tk.MustExec("alter table test_global add unique index idx_b (b);")
+ tk.MustExec("alter table test_global add unique index idx_b (b) global")
tk.MustExec("insert into test_global values (1, 1, 1), (8, 8, 8), (11, 11, 11), (12, 12, 12);")
tk.MustExec("analyze table test_global")
- originalHook := dom.DDL().GetHook()
- defer dom.DDL().SetHook(originalHook)
-
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
assert.Equal(t, model.ActionTruncateTablePartition, job.Type)
if job.SchemaState == model.StateDeleteOnly {
tk1 := testkit.NewTestKit(t, store)
@@ -1528,15 +1520,14 @@ func TestGlobalIndexUpdateInTruncatePartition(t *testing.T) {
err := tk1.ExecToErr("update test_global set a = 2 where a = 11")
assert.NotNil(t, err)
}
- }
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("alter table test_global truncate partition p1")
tk.MustQuery("select * from test_global use index(idx_b) order by a").Check(testkit.Rows("11 11 11", "12 12 12"))
}
func TestGlobalIndexUpdateInTruncatePartition4Hash(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("set tidb_enable_global_index=true")
@@ -1548,16 +1539,12 @@ func TestGlobalIndexUpdateInTruncatePartition4Hash(t *testing.T) {
tk.MustExec("drop table if exists test_global")
tk.MustExec(`create table test_global ( a int, b int, c int)
partition by hash(a) partitions 4;`)
- tk.MustExec("alter table test_global add unique index idx_b (b);")
+ tk.MustExec("alter table test_global add unique index idx_b (b) global")
tk.MustExec("insert into test_global values (1, 1, 1), (8, 8, 8), (11, 11, 11), (12, 12, 12);")
tk.MustExec("analyze table test_global")
- originalHook := dom.DDL().GetHook()
- defer dom.DDL().SetHook(originalHook)
-
var err error
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
assert.Equal(t, model.ActionTruncateTablePartition, job.Type)
if job.SchemaState == model.StateDeleteOnly {
tk1 := testkit.NewTestKit(t, store)
@@ -1565,14 +1552,13 @@ func TestGlobalIndexUpdateInTruncatePartition4Hash(t *testing.T) {
err = tk1.ExecToErr("update test_global set a = 1 where a = 12")
assert.NotNil(t, err)
}
- }
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("alter table test_global truncate partition p1")
}
func TestGlobalIndexReaderAndIndexLookUpInTruncatePartition(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("set tidb_enable_global_index=true")
@@ -1586,12 +1572,11 @@ func TestGlobalIndexReaderAndIndexLookUpInTruncatePartition(t *testing.T) {
partition p2 values less than (20),
partition p3 values less than (30)
);`)
- tk.MustExec("alter table test_global add unique index idx_b (b);")
+ tk.MustExec("alter table test_global add unique index idx_b (b) global")
tk.MustExec("insert into test_global values (1, 1, 1), (8, 8, 8), (11, 11, 11), (12, 12, 12);")
tk.MustExec("analyze table test_global")
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
assert.Equal(t, model.ActionTruncateTablePartition, job.Type)
if job.SchemaState == model.StateDeleteOnly {
tk1 := testkit.NewTestKit(t, store)
@@ -1602,14 +1587,13 @@ func TestGlobalIndexReaderAndIndexLookUpInTruncatePartition(t *testing.T) {
tk1.MustQuery("select * from test_global use index(idx_b) order by a").Check(testkit.Rows("11 11 11", "12 12 12"))
tk1.MustQuery("select * from test_global use index(idx_b) order by b").Check(testkit.Rows("11 11 11", "12 12 12"))
}
- }
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("alter table test_global truncate partition p1")
}
func TestGlobalIndexInsertInTruncatePartition(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("set tidb_enable_global_index=true")
@@ -1625,13 +1609,12 @@ func TestGlobalIndexInsertInTruncatePartition(t *testing.T) {
partition p2 values less than (20),
partition p3 values less than (30)
);`)
- tk.MustExec("alter table test_global add unique index idx_b (b);")
+ tk.MustExec("alter table test_global add unique index idx_b (b) global")
tk.MustExec("insert into test_global values (1, 1, 1), (8, 8, 8), (11, 11, 11), (12, 12, 12);")
tk.MustExec("analyze table test_global")
var err error
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
assert.Equal(t, model.ActionTruncateTablePartition, job.Type)
if job.SchemaState == model.StateDeleteOnly {
tk1 := testkit.NewTestKit(t, store)
@@ -1639,14 +1622,13 @@ func TestGlobalIndexInsertInTruncatePartition(t *testing.T) {
err = tk1.ExecToErr("insert into test_global values(2, 2, 2)")
assert.NotNil(t, err)
}
- }
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("alter table test_global truncate partition p1")
}
func TestGlobalIndexReaderInDropPartition(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("set tidb_enable_global_index=true")
@@ -1660,12 +1642,11 @@ func TestGlobalIndexReaderInDropPartition(t *testing.T) {
partition p2 values less than (20),
partition p3 values less than (30)
);`)
- tk.MustExec("alter table test_global add unique index idx_b (b);")
+ tk.MustExec("alter table test_global add unique index idx_b (b) global")
tk.MustExec("insert into test_global values (1, 1, 1), (8, 8, 8), (11, 11, 11), (12, 12, 12);")
var indexScanResult *testkit.Result
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
assert.Equal(t, model.ActionDropTablePartition, job.Type)
if job.SchemaState == model.StateDeleteOnly {
tk1 := testkit.NewTestKit(t, store)
@@ -1673,8 +1654,7 @@ func TestGlobalIndexReaderInDropPartition(t *testing.T) {
indexScanResult = tk1.MustQuery("select b from test_global use index(idx_b)").Sort()
}
- }
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("alter table test_global drop partition p1")
@@ -1682,7 +1662,7 @@ func TestGlobalIndexReaderInDropPartition(t *testing.T) {
}
func TestGlobalIndexLookUpInDropPartition(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("set tidb_enable_global_index=true")
@@ -1696,12 +1676,11 @@ func TestGlobalIndexLookUpInDropPartition(t *testing.T) {
partition p2 values less than (20),
partition p3 values less than (30)
);`)
- tk.MustExec("alter table test_global add unique index idx_b (b);")
+ tk.MustExec("alter table test_global add unique index idx_b (b) global")
tk.MustExec("insert into test_global values (1, 1, 1), (8, 8, 8), (11, 11, 11), (12, 12, 12);")
var indexLookupResult *testkit.Result
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
assert.Equal(t, model.ActionDropTablePartition, job.Type)
if job.SchemaState == model.StateDeleteOnly {
tk1 := testkit.NewTestKit(t, store)
@@ -1709,8 +1688,7 @@ func TestGlobalIndexLookUpInDropPartition(t *testing.T) {
tk1.MustExec("analyze table test_global")
indexLookupResult = tk1.MustQuery("select * from test_global use index(idx_b)").Sort()
}
- }
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("alter table test_global drop partition p1")
@@ -1738,7 +1716,7 @@ partition p2 values less than (10))`)
rs = tk.MustQuery("show table p index uidx regions").Rows()
require.Equal(t, len(rs), 3)
- tk.MustExec("alter table p add unique idx(id)")
+ tk.MustExec("alter table p add unique idx(id) global")
rs = tk.MustQuery("show table p regions").Rows()
require.Equal(t, len(rs), 4)
rs = tk.MustQuery("show table p index idx regions").Rows()
@@ -1790,6 +1768,9 @@ func TestAlterTableExchangePartition(t *testing.T) {
tk.MustExec("ALTER TABLE e3 EXCHANGE PARTITION p1 WITH TABLE e2;")
tk.MustQuery("select * from e3 partition(p0)").Check(testkit.Rows())
+ tk.MustQuery("select * from e3 partition(p1)").Check(testkit.Rows())
+ tk.MustQuery("select * from e3 partition(p2)").Check(testkit.Rows())
+ tk.MustQuery("select * from e3 partition(p3)").Check(testkit.Rows())
tk.MustQuery("select * from e2").Check(testkit.Rows("1", "5"))
// validation test for hash partition
@@ -2079,7 +2060,7 @@ func TestExchangePartitionMultiTable(t *testing.T) {
}
func TestExchangePartitionHook(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
// why use tkCancel, not tk.
tkCancel := testkit.NewTestKit(t, store)
@@ -2099,16 +2080,12 @@ func TestExchangePartitionHook(t *testing.T) {
tk.MustExec(`insert into pt values (0), (4), (7)`)
tk.MustExec("insert into nt values (1)")
- hook := &callback.TestDDLCallback{Do: dom}
- dom.DDL().SetHook(hook)
-
- hookFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.Type == model.ActionExchangeTablePartition && job.SchemaState != model.StateNone {
tkCancel.MustExec("use test")
tkCancel.MustGetErrCode("insert into nt values (5)", errno.ErrRowDoesNotMatchGivenPartitionSet)
}
- }
- hook.OnJobUpdatedExported.Store(&hookFunc)
+ })
tk.MustExec("alter table pt exchange partition p0 with table nt")
tk.MustQuery("select * from pt partition(p0)").Check(testkit.Rows("1"))
@@ -2755,25 +2732,19 @@ func TestTruncatePartitionMultipleTimes(t *testing.T) {
tk.MustExec(`create table test.t (a int primary key) partition by range (a) (
partition p0 values less than (10),
partition p1 values less than (maxvalue));`)
- dom := domain.GetDomain(tk.Session())
- originHook := dom.DDL().GetHook()
- defer dom.DDL().SetHook(originHook)
- hook := &callback.TestDDLCallback{}
- dom.DDL().SetHook(hook)
injected := false
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.Type == model.ActionTruncateTablePartition && job.SnapshotVer == 0 && !injected {
injected = true
time.Sleep(30 * time.Millisecond)
}
- }
+ })
var errCount atomic.Int32
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
if job.Type == model.ActionTruncateTablePartition && job.Error != nil {
errCount.Add(1)
}
- }
- hook.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
+ })
done1 := make(chan error, 1)
go backgroundExec(store, "test", "alter table test.t truncate partition p0;", done1)
done2 := make(chan error, 1)
@@ -2944,7 +2915,7 @@ func TestReorgPartitionTiFlash(t *testing.T) {
}
func TestIssue40135Ver2(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -2957,11 +2928,10 @@ func TestIssue40135Ver2(t *testing.T) {
tk.MustExec("CREATE TABLE t40135 ( a int DEFAULT NULL, b varchar(32) DEFAULT 'md', index(a)) PARTITION BY HASH (a) PARTITIONS 6")
tk.MustExec("insert into t40135 values (1, 'md'), (2, 'ma'), (3, 'md'), (4, 'ma'), (5, 'md'), (6, 'ma')")
one := true
- hook := &callback.TestDDLCallback{Do: dom}
var checkErr error
var wg sync.WaitGroup
wg.Add(1)
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.SchemaState == model.StateDeleteOnly {
tk3.MustExec("delete from t40135 where a = 1")
}
@@ -2972,8 +2942,7 @@ func TestIssue40135Ver2(t *testing.T) {
wg.Done()
}()
}
- }
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("alter table t40135 modify column a bigint NULL DEFAULT '6243108' FIRST")
wg.Wait()
require.ErrorContains(t, checkErr, "[ddl:8200]Unsupported modify column: table is partition table")
@@ -3492,10 +3461,10 @@ func TestReorgPartitionGlobalIndex(t *testing.T) {
require.Equal(t, 0, len(tt.Meta().Indices))
pid1 := tt.Meta().Partition.Definitions[1].ID
- tk.MustExec("Alter Table t Add Unique Index idx_b (b)")
+ tk.MustExec("Alter Table t Add Unique Index idx_b (b) global")
tk.MustExec(`INSERT INTO t VALUES (4, 5, 6), (6, 4, 5), (11, 6, 4), (12, 7, 7)`)
tk.MustExec(`INSERT INTO t VALUES (8, 8, 9), (9, 9, 8), (18, 18, 17), (19, 17, 18)`)
- tk.MustExec("Alter Table t Add Unique Index idx_c (c)")
+ tk.MustExec("Alter Table t Add Unique Index idx_c (c) global")
tk.MustExec("alter table t reorganize partition p2 into (partition p2 values less than (15), partition p3 values less than (20), partition pMax values less than (maxvalue))")
result := tk.MustQuery("select * from t")
@@ -3530,7 +3499,7 @@ func TestReorgPartitionGlobalIndex(t *testing.T) {
require.False(t, idxInfo.Global)
require.True(t, idxInfo.Unique)
// This should replace the unique index with a global index
- tk.MustExec(`alter table t partition by range (a) (partition p1 values less than (10), partition p2 values less than (20))`)
+ tk.MustExec(`alter table t partition by range (a) (partition p1 values less than (10), partition p2 values less than (20)) update indexes (idx_b global, idx_c global)`)
tt = external.GetTableByName(t, tk, "test", "t")
require.Equal(t, 2, len(tt.Meta().Indices))
idxInfo = tt.Meta().FindIndexByName("idx_b")
@@ -3539,7 +3508,7 @@ func TestReorgPartitionGlobalIndex(t *testing.T) {
idxInfo = tt.Meta().FindIndexByName("idx_c")
require.True(t, idxInfo.Global)
require.True(t, idxInfo.Unique)
- tk.MustExec(`alter table t partition by hash (b) partitions 3`)
+ tk.MustExec(`alter table t partition by hash (b) partitions 3 update indexes (idx_b local)`)
tt = external.GetTableByName(t, tk, "test", "t")
require.Equal(t, 2, len(tt.Meta().Indices))
idxInfo = tt.Meta().FindIndexByName("idx_b")
@@ -3549,7 +3518,7 @@ func TestReorgPartitionGlobalIndex(t *testing.T) {
require.True(t, idxInfo.Global)
require.True(t, idxInfo.Unique)
tk.MustExec(`alter table t remove partitioning`)
- tk.MustExec(`alter table t partition by range columns (c) (partition p0 values less than (10), partition pMax values less than (maxvalue))`)
+ tk.MustExec(`alter table t partition by range columns (c) (partition p0 values less than (10), partition pMax values less than (maxvalue)) update indexes (idx_b global, idx_c local)`)
tt = external.GetTableByName(t, tk, "test", "t")
require.Equal(t, 2, len(tt.Meta().Indices))
idxInfo = tt.Meta().FindIndexByName("idx_b")
@@ -3568,7 +3537,7 @@ func TestRemovePartitioningGlobalIndex(t *testing.T) {
defer func() {
tk.MustExec("set tidb_enable_global_index=default")
}()
- tk.MustExec(`create table t (a int unsigned not null, b varchar(255), unique key idx_b(b), unique key idx_a(a)) partition by key(b) partitions 3`)
+ tk.MustExec(`create table t (a int unsigned not null, b varchar(255), unique key idx_b(b), unique key idx_a(a) global) partition by key(b) partitions 3`)
tk.MustExec(`create table t2 (a int unsigned not null, b varchar(255), unique key idx_b(b), unique key idx_a(a))`)
tk.MustExec(`INSERT INTO t VALUES (1,1),(2,2),(3,"Hello, World!")`)
tk.MustExec(`INSERT INTO t2 select * from t`)
@@ -3600,7 +3569,7 @@ func TestRemovePartitioningGlobalIndex(t *testing.T) {
tk.MustQuery(`select a from t order by a`).Check(testkit.Rows("1", "2", "3"))
tk.MustQuery(`select b from t order by b`).Check(testkit.Rows("1", "2", "Hello, World!"))
- tk.MustExec(`alter table t partition by key(b) partitions 3`)
+ tk.MustExec(`alter table t partition by key(b) partitions 3 update indexes (idx_a global)`)
tt = external.GetTableByName(t, tk, "test", "t")
require.Equal(t, 2, len(tt.Meta().Indices))
idxInfo = tt.Meta().FindIndexByName("idx_a")
@@ -3639,13 +3608,13 @@ func TestPrimaryGlobalIndex(t *testing.T) {
// Clustered table where PKIsHandle and listed in tableInfo.Indices
tk.MustExec(`create table t (a varchar(255), b varchar(255), primary key (a) clustered)`)
tk.MustContainErrMsg(`alter table t drop primary key`, "Unsupported drop primary key when the table is using clustered index")
- tk.MustContainErrMsg(`alter table t partition by key(b) partitions 3`, `A CLUSTERED INDEX must include all columns in the table's partitioning function`)
+ tk.MustContainErrMsg(`alter table t partition by key(b) partitions 3`, `[ddl:1503]A CLUSTERED INDEX must include all columns in the table's partitioning function`)
checkGlobalAndPK(t, tk, "t", 1, false, true, false)
tk.MustExec(`drop table t`)
// Clustered table where IsCommonHandle and listed in tableInfo.Indices
tk.MustExec(`create table t (a varchar(255), b varchar(255), c int, primary key (a,c) clustered)`)
checkGlobalAndPK(t, tk, "t", 1, false, true, false)
- tk.MustContainErrMsg(`alter table t partition by key(b) partitions 3`, `A CLUSTERED INDEX must include all columns in the table's partitioning function`)
+ tk.MustContainErrMsg(`alter table t partition by key(b) partitions 3`, `[ddl:1503]A CLUSTERED INDEX must include all columns in the table's partitioning function`)
checkGlobalAndPK(t, tk, "t", 1, false, true, false)
tk.MustExec(`drop table t`)
// It can be clustered if the PK contains all the partitioning columns.
@@ -3657,20 +3626,20 @@ func TestPrimaryGlobalIndex(t *testing.T) {
tk.MustExec(`drop table t`)
// NONCLUSTERED tables can have PK as global index.
- tk.MustExec(`create table t (a int primary key nonclustered, b varchar(255)) partition by key(b) partitions 3`)
+ tk.MustExec(`create table t (a int primary key nonclustered global, b varchar(255)) partition by key(b) partitions 3`)
checkGlobalAndPK(t, tk, "t", 1, false, false, true)
tk.MustExec(`alter table t drop primary key`)
- tk.MustExec(`alter table t add primary key (a)`)
+ tk.MustExec(`alter table t add primary key (a) global`)
checkGlobalAndPK(t, tk, "t", 1, false, false, true)
tk.MustExec(`drop table t`)
tk.MustExec(`create table t (a int primary key nonclustered, b varchar(255))`)
checkGlobalAndPK(t, tk, "t", 1, false, false, false)
- tk.MustExec(`alter table t partition by key(b) partitions 3`)
+ tk.MustExec("alter table t partition by key(b) partitions 3 update indexes (`primary` global)")
checkGlobalAndPK(t, tk, "t", 1, false, false, true)
tk.MustExec(`alter table t drop primary key`)
- tk.MustExec(`alter table t add primary key (a)`)
+ tk.MustExec(`alter table t add primary key (a) global`)
checkGlobalAndPK(t, tk, "t", 1, false, false, true)
- tk.MustExec(`alter table t partition by hash(a) partitions 3`)
+ tk.MustExec("alter table t partition by hash(a) partitions 3 update indexes (`primary` local)")
checkGlobalAndPK(t, tk, "t", 1, false, false, false)
tk.MustExec(`alter table t drop primary key`)
tk.MustExec(`alter table t add primary key (a)`)
@@ -3678,10 +3647,10 @@ func TestPrimaryGlobalIndex(t *testing.T) {
tk.MustExec(`drop table t`)
tk.MustExec(`create table t (a varchar(255), b varchar(255), primary key (a) nonclustered)`)
checkGlobalAndPK(t, tk, "t", 1, false, false, false)
- tk.MustExec(`alter table t partition by key(b) partitions 3`)
+ tk.MustExec("alter table t partition by key(b) partitions 3 update indexes (`primary` global)")
checkGlobalAndPK(t, tk, "t", 1, false, false, true)
tk.MustExec(`alter table t drop primary key`)
- tk.MustExec(`alter table t add primary key (a)`)
+ tk.MustExec(`alter table t add primary key (a) global`)
checkGlobalAndPK(t, tk, "t", 1, false, false, true)
tk.MustExec(`drop table t`)
tk.MustExec(`create table t (a varchar(255), b varchar(255), primary key (a, b) nonclustered)`)
@@ -3690,7 +3659,7 @@ func TestPrimaryGlobalIndex(t *testing.T) {
tk.MustExec(`alter table t partition by key(b) partitions 3`)
checkGlobalAndPK(t, tk, "t", 1, false, false, false)
tk.MustExec(`alter table t drop primary key`)
- tk.MustExec(`alter table t add primary key (a)`)
+ tk.MustExec(`alter table t add primary key (a) global`)
}
func TestPrimaryNoGlobalIndex(t *testing.T) {
@@ -3711,17 +3680,17 @@ func TestPrimaryNoGlobalIndex(t *testing.T) {
// Clustered table where PKIsHandle, but the primary key is not listed in tableInfo.Indices
tk.MustExec(`create table t (a int primary key clustered, b varchar(255))`)
checkGlobalAndPK(t, tk, "t", 0, true, false, false)
- tk.MustContainErrMsg(`alter table t partition by key(b) partitions 3`, `A PRIMARY KEY must include all columns in the table's partitioning function`)
+ tk.MustContainErrMsg(`alter table t partition by key(b) partitions 3`, `A CLUSTERED INDEX must include all columns in the table's partitioning function`)
tk.MustExec(`drop table t`)
// Clustered table where PKIsHandle and listed in tableInfo.Indices
tk.MustExec(`create table t (a varchar(255), b varchar(255), primary key (a) clustered)`)
- tk.MustContainErrMsg(`alter table t partition by key(b) partitions 3`, `A PRIMARY KEY must include all columns in the table's partitioning function`)
+ tk.MustContainErrMsg(`alter table t partition by key(b) partitions 3`, `A CLUSTERED INDEX must include all columns in the table's partitioning function`)
checkGlobalAndPK(t, tk, "t", 1, false, true, false)
tk.MustExec(`drop table t`)
// Clustered table where IsCommonHandle and listed in tableInfo.Indices
tk.MustExec(`create table t (a varchar(255), b varchar(255), c int, primary key (a,c) clustered)`)
checkGlobalAndPK(t, tk, "t", 1, false, true, false)
- tk.MustContainErrMsg(`alter table t partition by key(b) partitions 3`, `A PRIMARY KEY must include all columns in the table's partitioning function`)
+ tk.MustContainErrMsg(`alter table t partition by key(b) partitions 3`, `A CLUSTERED INDEX must include all columns in the table's partitioning function`)
checkGlobalAndPK(t, tk, "t", 1, false, true, false)
tk.MustExec(`drop table t`)
// It can be clustered if the PK contains all the partitioning columns.
diff --git a/pkg/ddl/tests/resourcegroup/BUILD.bazel b/pkg/ddl/tests/resourcegroup/BUILD.bazel
index c29f275b5b85d..647c6e100598c 100644
--- a/pkg/ddl/tests/resourcegroup/BUILD.bazel
+++ b/pkg/ddl/tests/resourcegroup/BUILD.bazel
@@ -6,17 +6,18 @@ go_test(
srcs = ["resource_group_test.go"],
flaky = True,
race = "on",
- shard_count = 5,
+ shard_count = 6,
deps = [
"//pkg/ddl/resourcegroup",
- "//pkg/ddl/util/callback",
"//pkg/domain",
"//pkg/domain/infosync",
"//pkg/errno",
"//pkg/parser/auth",
"//pkg/parser/model",
+ "//pkg/server",
"//pkg/sessionctx",
"//pkg/testkit",
+ "//pkg/testkit/testfailpoint",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_kvproto//pkg/resource_manager",
"@com_github_stretchr_testify//require",
diff --git a/pkg/ddl/tests/resourcegroup/resource_group_test.go b/pkg/ddl/tests/resourcegroup/resource_group_test.go
index d0bc799add24b..653bed499cce7 100644
--- a/pkg/ddl/tests/resourcegroup/resource_group_test.go
+++ b/pkg/ddl/tests/resourcegroup/resource_group_test.go
@@ -17,6 +17,7 @@ package resourcegrouptest_test
import (
"context"
"fmt"
+ "math"
"sync/atomic"
"testing"
"time"
@@ -24,34 +25,32 @@ import (
"github.com/pingcap/failpoint"
rmpb "github.com/pingcap/kvproto/pkg/resource_manager"
"github.com/pingcap/tidb/pkg/ddl/resourcegroup"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/domain/infosync"
mysql "github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/parser/auth"
"github.com/pingcap/tidb/pkg/parser/model"
+ "github.com/pingcap/tidb/pkg/server"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/stretchr/testify/require"
)
func TestResourceGroupBasic(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
re := require.New(t)
- hook := &callback.TestDDLCallback{Do: dom}
var groupID atomic.Int64
- onJobUpdatedExportedFunc := func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
// job.SchemaID will be assigned when the group is created.
if (job.SchemaName == "x" || job.SchemaName == "y") && job.Type == model.ActionCreateResourceGroup && job.SchemaID != 0 {
groupID.Store(job.SchemaID)
return
}
- }
- hook.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
- dom.DDL().SetHook(hook)
+ })
tk.MustExec("set global tidb_enable_resource_control = 'off'")
tk.MustGetErrCode("create user usr1 resource group rg1", mysql.ErrResourceGroupSupportDisabled)
@@ -92,11 +91,13 @@ func TestResourceGroupBasic(t *testing.T) {
tk.MustExec("set global tidb_enable_resource_control = off")
tk.MustGetErrCode("alter resource group x RU_PER_SEC=2000 ", mysql.ErrResourceGroupSupportDisabled)
+ tk.MustGetErrCode("alter resource group x RU_PER_SEC=unlimited ", mysql.ErrResourceGroupSupportDisabled)
tk.MustGetErrCode("drop resource group x ", mysql.ErrResourceGroupSupportDisabled)
tk.MustExec("set global tidb_enable_resource_control = DEFAULT")
tk.MustGetErrCode("create resource group x RU_PER_SEC=1000 ", mysql.ErrResourceGroupExists)
+ tk.MustGetErrCode("create resource group x RU_PER_SEC=UNLIMITED ", mysql.ErrResourceGroupExists)
tk.MustExec("alter resource group x RU_PER_SEC=2000 BURSTABLE QUERY_LIMIT=(EXEC_ELAPSED='15s' ACTION DRYRUN WATCH SIMILAR DURATION '10m0s')")
g = testResourceGroupNameFromIS(t, tk.Session(), "x")
@@ -107,16 +108,27 @@ func TestResourceGroupBasic(t *testing.T) {
re.Equal(model.WatchSimilar, g.Runaway.WatchType)
re.Equal(int64(time.Minute*10/time.Millisecond), g.Runaway.WatchDurationMs)
- tk.MustExec("alter resource group x QUERY_LIMIT=(EXEC_ELAPSED='20s' ACTION DRYRUN WATCH SIMILAR)")
+ tk.MustExec("alter resource group x QUERY_LIMIT=(EXEC_ELAPSED='20s' ACTION DRYRUN WATCH SIMILAR) BURSTABLE=FALSE")
g = testResourceGroupNameFromIS(t, tk.Session(), "x")
re.Equal(uint64(2000), g.RURate)
- re.Equal(int64(-1), g.BurstLimit)
+ re.Equal(int64(2000), g.BurstLimit)
re.Equal(uint64(time.Second*20/time.Millisecond), g.Runaway.ExecElapsedTimeMs)
re.Equal(model.RunawayActionDryRun, g.Runaway.Action)
re.Equal(model.WatchSimilar, g.Runaway.WatchType)
re.Equal(int64(0), g.Runaway.WatchDurationMs)
- tk.MustQuery("select * from information_schema.resource_groups where name = 'x'").Check(testkit.Rows("x 2000 MEDIUM YES EXEC_ELAPSED='20s', ACTION=DRYRUN, WATCH=SIMILAR DURATION=UNLIMITED "))
+ tk.MustQuery("select * from information_schema.resource_groups where name = 'x'").Check(testkit.Rows("x 2000 MEDIUM NO EXEC_ELAPSED='20s', ACTION=DRYRUN, WATCH=SIMILAR DURATION=UNLIMITED "))
+
+ tk.MustExec("alter resource group x RU_PER_SEC= unlimited QUERY_LIMIT=(EXEC_ELAPSED='15s' ACTION DRYRUN WATCH SIMILAR DURATION '10m0s')")
+ g = testResourceGroupNameFromIS(t, tk.Session(), "x")
+ re.Equal(uint64(math.MaxInt32), g.RURate)
+ re.Equal(int64(-1), g.BurstLimit)
+ re.Equal(uint64(time.Second*15/time.Millisecond), g.Runaway.ExecElapsedTimeMs)
+ re.Equal(model.RunawayActionDryRun, g.Runaway.Action)
+ re.Equal(model.WatchSimilar, g.Runaway.WatchType)
+ re.Equal(int64(time.Minute*10/time.Millisecond), g.Runaway.WatchDurationMs)
+
+ tk.MustQuery("select * from information_schema.resource_groups where name = 'x'").Check(testkit.Rows("x UNLIMITED MEDIUM YES EXEC_ELAPSED='15s', ACTION=DRYRUN, WATCH=SIMILAR DURATION='10m0s' "))
tk.MustExec("drop resource group x")
g = testResourceGroupNameFromIS(t, tk.Session(), "x")
@@ -310,7 +322,6 @@ func TestResourceGroupRunaway(t *testing.T) {
maxWaitDuration := time.Second * 5
tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, original_sql, match_type from mysql.tidb_runaway_queries", nil,
testkit.Rows("rg1 select /*+ resource_group(rg1) */ * from t identify"), maxWaitDuration, tryInterval)
- // require.Len(t, tk.MustQuery("select SQL_NO_CACHE resource_group_name, original_sql, time from mysql.tidb_runaway_queries").Rows(), 0)
tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, original_sql, time from mysql.tidb_runaway_queries", nil,
nil, maxWaitDuration, tryInterval)
tk.MustExec("alter resource group rg1 RU_PER_SEC=1000 QUERY_LIMIT=(EXEC_ELAPSED='100ms' ACTION=COOLDOWN)")
@@ -363,6 +374,43 @@ func TestResourceGroupRunaway(t *testing.T) {
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/store/copr/sleepCoprAfterReq"))
}
+func TestResourceGroupRunawayExceedTiDBSide(t *testing.T) {
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/domain/FastRunawayGC", `return(true)`))
+ defer func() {
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/domain/FastRunawayGC"))
+ }()
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ sv := server.CreateMockServer(t, store)
+ sv.SetDomain(dom)
+ defer sv.Close()
+
+ conn1 := server.CreateMockConn(t, sv)
+ tk := testkit.NewTestKitWithSession(t, store, conn1.Context().Session)
+
+ go dom.ExpensiveQueryHandle().SetSessionManager(sv).Run()
+ tk.MustExec("set global tidb_enable_resource_control='on'")
+ require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "localhost"}, nil, nil, nil))
+
+ tk.MustExec("use test")
+ tk.MustExec("create table t(a int)")
+ tk.MustExec("insert into t values(1)")
+ tk.MustExec("create resource group rg1 RU_PER_SEC=1000 QUERY_LIMIT=(EXEC_ELAPSED='50ms' ACTION=KILL)")
+
+ require.Eventually(t, func() bool {
+ return dom.RunawayManager().IsSyncerInitialized()
+ }, 20*time.Second, 300*time.Millisecond)
+
+ err := tk.QueryToErr("select /*+ resource_group(rg1) */ sleep(0.5) from t")
+ require.ErrorContains(t, err, "[executor:8253]Query execution was interrupted, identified as runaway query")
+
+ tryInterval := time.Millisecond * 100
+ maxWaitDuration := time.Second * 5
+ tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, original_sql, match_type from mysql.tidb_runaway_queries", nil,
+ testkit.Rows("rg1 select /*+ resource_group(rg1) */ sleep(0.5) from t identify"), maxWaitDuration, tryInterval)
+ tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, original_sql, time from mysql.tidb_runaway_queries", nil,
+ nil, maxWaitDuration, tryInterval)
+}
+
func TestAlreadyExistsDefaultResourceGroup(t *testing.T) {
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/domain/infosync/managerAlreadyCreateSomeGroups", `return(true)`))
defer func() {
diff --git a/pkg/ddl/tests/serial/BUILD.bazel b/pkg/ddl/tests/serial/BUILD.bazel
index 7cf13791302f5..e8c736ec88ebf 100644
--- a/pkg/ddl/tests/serial/BUILD.bazel
+++ b/pkg/ddl/tests/serial/BUILD.bazel
@@ -13,7 +13,6 @@ go_test(
"//pkg/config",
"//pkg/ddl",
"//pkg/ddl/util",
- "//pkg/ddl/util/callback",
"//pkg/domain",
"//pkg/domain/infosync",
"//pkg/errno",
@@ -32,6 +31,7 @@ go_test(
"//pkg/tablecodec",
"//pkg/testkit",
"//pkg/testkit/external",
+ "//pkg/testkit/testfailpoint",
"//pkg/testkit/testsetup",
"//pkg/util/dbterror",
"//pkg/util/dbterror/plannererrors",
diff --git a/pkg/ddl/tests/serial/serial_test.go b/pkg/ddl/tests/serial/serial_test.go
index 3edfbd6355f68..a548e1bc74bbc 100644
--- a/pkg/ddl/tests/serial/serial_test.go
+++ b/pkg/ddl/tests/serial/serial_test.go
@@ -28,7 +28,6 @@ import (
"github.com/pingcap/tidb/pkg/config"
"github.com/pingcap/tidb/pkg/ddl"
"github.com/pingcap/tidb/pkg/ddl/util"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/infoschema"
@@ -45,6 +44,7 @@ import (
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/external"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/util/dbterror"
"github.com/pingcap/tidb/pkg/util/dbterror/plannererrors"
"github.com/pingcap/tidb/pkg/util/gcutil"
@@ -54,7 +54,7 @@ import (
// GetMaxRowID is used for test.
func GetMaxRowID(store kv.Storage, priority int, t table.Table, startHandle, endHandle kv.Key) (kv.Key, error) {
- return ddl.GetRangeEndKey(ddl.NewJobContext(), store, priority, t.RecordPrefix(), startHandle, endHandle)
+ return ddl.GetRangeEndKey(ddl.NewReorgContext(), store, priority, t.RecordPrefix(), startHandle, endHandle)
}
func TestIssue23872(t *testing.T) {
@@ -410,7 +410,7 @@ func TestCreateTableWithLikeAtTemporaryMode(t *testing.T) {
require.Equal(t, plannererrors.ErrOptOnTemporaryTable.GenWithStackByArgs("placement").Error(), err.Error())
}
-func createMockStoreAndDomain(t *testing.T) (store kv.Storage, dom *domain.Domain) {
+func createMockStore(t *testing.T) (store kv.Storage) {
session.SetSchemaLease(200 * time.Millisecond)
session.DisableStats4Test()
ddl.SetWaitTimeWhenErrorOccurred(1 * time.Microsecond)
@@ -418,7 +418,7 @@ func createMockStoreAndDomain(t *testing.T) (store kv.Storage, dom *domain.Domai
var err error
store, err = mockstore.NewMockStore()
require.NoError(t, err)
- dom, err = session.BootstrapSession(store)
+ dom, err := session.BootstrapSession(store)
require.NoError(t, err)
t.Cleanup(func() {
dom.Close()
@@ -429,7 +429,7 @@ func createMockStoreAndDomain(t *testing.T) (store kv.Storage, dom *domain.Domai
// TestCancelAddIndex1 tests canceling ddl job when the add index worker is not started.
func TestCancelAddIndexPanic(t *testing.T) {
- store, dom := createMockStoreAndDomain(t)
+ store := createMockStore(t)
tk := testkit.NewTestKit(t, store)
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/errorMockPanic", `return(true)`))
defer func() {
@@ -448,13 +448,11 @@ func TestCancelAddIndexPanic(t *testing.T) {
oldReorgWaitTimeout := ddl.ReorgWaitTimeout
ddl.ReorgWaitTimeout = 50 * time.Millisecond
defer func() { ddl.ReorgWaitTimeout = oldReorgWaitTimeout }()
- hook := &callback.TestDDLCallback{Do: dom}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.Type == model.ActionAddIndex && job.State == model.JobStateRunning && job.SchemaState == model.StateWriteReorganization && job.SnapshotVer != 0 {
tkCancel.MustQuery(fmt.Sprintf("admin cancel ddl jobs %d", job.ID))
}
- }
- dom.DDL().SetHook(hook)
+ })
rs, err := tk.Exec("alter table t add index idx_c2(c2)")
if rs != nil {
require.NoError(t, rs.Close())
@@ -466,7 +464,7 @@ func TestCancelAddIndexPanic(t *testing.T) {
}
func TestRecoverTableWithTTL(t *testing.T) {
- store, _ := createMockStoreAndDomain(t)
+ store := createMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("create database if not exists test_recover")
tk.MustExec("use test_recover")
@@ -530,7 +528,7 @@ func TestRecoverTableWithTTL(t *testing.T) {
}
func TestRecoverTableByJobID(t *testing.T) {
- store, _ := createMockStoreAndDomain(t)
+ store := createMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("create database if not exists test_recover")
tk.MustExec("use test_recover")
@@ -646,7 +644,7 @@ func TestRecoverTableByJobID(t *testing.T) {
}
func TestRecoverTableByJobIDFail(t *testing.T) {
- store, dom := createMockStoreAndDomain(t)
+ store := createMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("create database if not exists test_recover")
tk.MustExec("use test_recover")
@@ -687,14 +685,12 @@ func TestRecoverTableByJobIDFail(t *testing.T) {
tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop))
// set hook
- hook := &callback.TestDDLCallback{}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.Type == model.ActionRecoverTable {
require.NoError(t, failpoint.Enable("tikvclient/mockCommitError", `return(true)`))
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/mockRecoverTableCommitErr", `return(true)`))
}
- }
- dom.DDL().SetHook(hook)
+ })
// do recover table.
tk.MustExec(fmt.Sprintf("recover table by job %d", jobID))
@@ -714,7 +710,7 @@ func TestRecoverTableByJobIDFail(t *testing.T) {
}
func TestRecoverTableByTableNameFail(t *testing.T) {
- store, dom := createMockStoreAndDomain(t)
+ store := createMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("create database if not exists test_recover")
tk.MustExec("use test_recover")
@@ -746,14 +742,12 @@ func TestRecoverTableByTableNameFail(t *testing.T) {
tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop))
// set hook
- hook := &callback.TestDDLCallback{}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.Type == model.ActionRecoverTable {
require.NoError(t, failpoint.Enable("tikvclient/mockCommitError", `return(true)`))
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/mockRecoverTableCommitErr", `return(true)`))
}
- }
- dom.DDL().SetHook(hook)
+ })
// do recover table.
tk.MustExec("recover table t_recover")
@@ -773,7 +767,7 @@ func TestRecoverTableByTableNameFail(t *testing.T) {
}
func TestCancelJobByErrorCountLimit(t *testing.T) {
- store, _ := createMockStoreAndDomain(t)
+ store := createMockStore(t)
tk := testkit.NewTestKit(t, store)
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/mockExceedErrorLimit", `return(true)`))
defer func() {
@@ -814,14 +808,13 @@ func TestTruncateTableUpdateSchemaVersionErr(t *testing.T) {
}
func TestCanceledJobTakeTime(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("create table t_cjtt(a int)")
- hook := &callback.TestDDLCallback{}
once := sync.Once{}
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
once.Do(func() {
ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL)
err := kv.RunInNewTxn(ctx, store, false, func(ctx context.Context, txn kv.Transaction) error {
@@ -830,12 +823,11 @@ func TestCanceledJobTakeTime(t *testing.T) {
if err != nil {
return err
}
- return m.DropTableOrView(job.SchemaID, job.SchemaName, job.TableID, job.TableName)
+ return m.DropTableOrView(job.SchemaID, job.TableID)
})
require.NoError(t, err)
})
- }
- dom.DDL().SetHook(hook)
+ })
originalWT := ddl.GetWaitTimeWhenErrorOccurred()
ddl.SetWaitTimeWhenErrorOccurred(1 * time.Second)
diff --git a/pkg/ddl/tests/tiflash/BUILD.bazel b/pkg/ddl/tests/tiflash/BUILD.bazel
index 3827bfd8d7140..e3d7289f33f94 100644
--- a/pkg/ddl/tests/tiflash/BUILD.bazel
+++ b/pkg/ddl/tests/tiflash/BUILD.bazel
@@ -28,6 +28,7 @@ go_test(
"//pkg/tablecodec",
"//pkg/testkit",
"//pkg/testkit/external",
+ "//pkg/testkit/testfailpoint",
"//pkg/testkit/testsetup",
"//pkg/types",
"//pkg/util",
diff --git a/pkg/ddl/tests/tiflash/ddl_tiflash_test.go b/pkg/ddl/tests/tiflash/ddl_tiflash_test.go
index 3aca7f8fc3224..a5578277cf6c2 100644
--- a/pkg/ddl/tests/tiflash/ddl_tiflash_test.go
+++ b/pkg/ddl/tests/tiflash/ddl_tiflash_test.go
@@ -45,6 +45,7 @@ import (
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/external"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/sqlkiller"
@@ -95,7 +96,6 @@ func createTiFlashContext(t *testing.T) (*tiflashContext, func()) {
)
require.NoError(t, err)
- session.SetSchemaLease(0)
session.DisableStats4Test()
s.dom, err = session.BootstrapSession(s.store)
infosync.SetMockTiFlash(s.tiflash)
@@ -1386,28 +1386,6 @@ func TestTiFlashAvailableAfterDownOneStore(t *testing.T) {
CheckTableAvailable(s.dom, t, 1, []string{})
}
-// TestDLLCallback copied from ddl.TestDDLCallback, but smaller
-type TestDDLCallback struct {
- *ddl.BaseCallback
- // We recommended to pass the domain parameter to the test ddl callback, it will ensure
- // domain to reload schema before your ddl stepping into the next state change.
- Do ddl.SchemaLoader
-
- // Only need this for now
- OnJobRunBeforeExported func(*model.Job)
-}
-
-// OnJobRunBefore is used to run the user customized logic of `onJobRunBefore` first.
-func (tc *TestDDLCallback) OnJobRunBefore(job *model.Job) {
- logutil.DDLLogger().Info("on job run before", zap.String("job", job.String()))
- if tc.OnJobRunBeforeExported != nil {
- tc.OnJobRunBeforeExported(job)
- return
- }
-
- tc.BaseCallback.OnJobRunBefore(job)
-}
-
func TestTiFlashReorgPartition(t *testing.T) {
s, teardown := createTiFlashContext(t)
defer teardown()
@@ -1430,14 +1408,9 @@ func TestTiFlashReorgPartition(t *testing.T) {
require.True(t, ok)
// Note that the mock TiFlash does not have any data or regions, so the wait for regions being available will fail
- dom := domain.GetDomain(tk.Session())
- originHook := dom.DDL().GetHook()
- defer dom.DDL().SetHook(originHook)
- hook := &TestDDLCallback{Do: dom}
- dom.DDL().SetHook(hook)
done := false
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if !done && job.Type == model.ActionReorganizePartition && job.SchemaState == model.StateDeleteOnly {
// Let it fail once (to check that code path) then increase the count to skip retry
if job.ErrorCount > 0 {
@@ -1445,11 +1418,11 @@ func TestTiFlashReorgPartition(t *testing.T) {
done = true
}
}
- }
+ })
tk.MustContainErrMsg(`alter table ddltiflash reorganize partition p0 into (partition p0 values less than (500000), partition p500k values less than (1000000))`, "[ddl] add partition wait for tiflash replica to complete")
done = false
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if !done && job.Type == model.ActionReorganizePartition && job.SchemaState == model.StateDeleteOnly {
// Let it fail once (to check that code path) then mock the regions into the partitions
if job.ErrorCount > 0 {
@@ -1479,7 +1452,7 @@ func TestTiFlashReorgPartition(t *testing.T) {
done = true
}
}
- }
+ })
tk.MustExec(`alter table ddltiflash reorganize partition p0 into (partition p0 values less than (500000), partition p500k values less than (1000000))`)
tk.MustExec(`admin check table ddltiflash`)
_, ok = s.tiflash.GetPlacementRule(ruleName)
diff --git a/pkg/ddl/testutil/BUILD.bazel b/pkg/ddl/testutil/BUILD.bazel
index 4cfa751c86c17..cee0cfee17fcc 100644
--- a/pkg/ddl/testutil/BUILD.bazel
+++ b/pkg/ddl/testutil/BUILD.bazel
@@ -2,11 +2,15 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "testutil",
- srcs = ["testutil.go"],
+ srcs = [
+ "operator.go",
+ "testutil.go",
+ ],
importpath = "github.com/pingcap/tidb/pkg/ddl/testutil",
visibility = ["//visibility:public"],
deps = [
"//pkg/ddl/logutil",
+ "//pkg/disttask/operator",
"//pkg/domain",
"//pkg/kv",
"//pkg/parser/model",
@@ -18,6 +22,7 @@ go_library(
"//pkg/types",
"@com_github_pingcap_errors//:errors",
"@com_github_stretchr_testify//require",
+ "@org_golang_x_sync//errgroup",
"@org_uber_go_zap//:zap",
],
)
diff --git a/pkg/ddl/testutil/operator.go b/pkg/ddl/testutil/operator.go
new file mode 100644
index 0000000000000..90ba745bc387d
--- /dev/null
+++ b/pkg/ddl/testutil/operator.go
@@ -0,0 +1,107 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testutil
+
+import (
+ "github.com/pingcap/tidb/pkg/disttask/operator"
+ "golang.org/x/sync/errgroup"
+)
+
+// OperatorTestSource is used for dist task operator test.
+type OperatorTestSource[T any] struct {
+ errGroup errgroup.Group
+ ch chan T
+ toBeSent []T
+}
+
+// NewOperatorTestSource creates a new OperatorTestSource.
+func NewOperatorTestSource[T any](toBeSent ...T) *OperatorTestSource[T] {
+ return &OperatorTestSource[T]{
+ ch: make(chan T),
+ toBeSent: toBeSent,
+ }
+}
+
+// SetSink implements disttask/operator.Operator.
+func (s *OperatorTestSource[T]) SetSink(sink operator.DataChannel[T]) {
+ s.ch = sink.Channel()
+}
+
+// Open implements disttask/operator.Operator.
+func (s *OperatorTestSource[T]) Open() error {
+ s.errGroup.Go(func() error {
+ for _, data := range s.toBeSent {
+ s.ch <- data
+ }
+ close(s.ch)
+ return nil
+ })
+ return nil
+}
+
+// Close implements disttask/operator.Operator.
+func (s *OperatorTestSource[T]) Close() error {
+ return s.errGroup.Wait()
+}
+
+// String implements disttask/operator.Operator.
+func (*OperatorTestSource[T]) String() string {
+ return "testSource"
+}
+
+// OperatorTestSink is used for dist task operator test.
+type OperatorTestSink[T any] struct {
+ errGroup errgroup.Group
+ ch chan T
+ collected []T
+}
+
+// NewOperatorTestSink creates a new OperatorTestSink.
+func NewOperatorTestSink[T any]() *OperatorTestSink[T] {
+ return &OperatorTestSink[T]{
+ ch: make(chan T),
+ }
+}
+
+// Open implements disttask/operator.Operator.
+func (s *OperatorTestSink[T]) Open() error {
+ s.errGroup.Go(func() error {
+ for data := range s.ch {
+ s.collected = append(s.collected, data)
+ }
+ return nil
+ })
+ return nil
+}
+
+// Close implements disttask/operator.Operator.
+func (s *OperatorTestSink[T]) Close() error {
+ return s.errGroup.Wait()
+}
+
+// SetSource implements disttask/operator.Operator.
+func (s *OperatorTestSink[T]) SetSource(dataCh operator.DataChannel[T]) {
+ s.ch = dataCh.Channel()
+}
+
+// String implements disttask/operator.Operator.
+func (*OperatorTestSink[T]) String() string {
+ return "testSink"
+}
+
+// Collect the result from OperatorTestSink.
+func (s *OperatorTestSink[T]) Collect() []T {
+ return s.collected
+}
diff --git a/pkg/ddl/tiflash_replica_test.go b/pkg/ddl/tiflash_replica_test.go
index 815ab8f00791d..bb122027fdf05 100644
--- a/pkg/ddl/tiflash_replica_test.go
+++ b/pkg/ddl/tiflash_replica_test.go
@@ -26,7 +26,6 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/config"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/kv"
@@ -38,6 +37,7 @@ import (
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/external"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/dbterror"
"github.com/stretchr/testify/require"
@@ -303,16 +303,15 @@ func TestSkipSchemaChecker(t *testing.T) {
// TestCreateTableWithLike2 tests create table with like when refer table have non-public column/index.
func TestCreateTableWithLike2(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, tiflashReplicaLease)
+ store := testkit.CreateMockStoreWithSchemaLease(t, tiflashReplicaLease)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("create table t1 (a int, b int, c int, index idx1(c));")
tbl1 := external.GetTableByName(t, tk, "test", "t1")
doneCh := make(chan error, 2)
- hook := &callback.TestDDLCallback{Do: dom}
var onceChecker sync.Map
- hook.OnJobRunBeforeExported = func(job *model.Job) {
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore", func(job *model.Job) {
if job.Type != model.ActionAddColumn && job.Type != model.ActionDropColumn &&
job.Type != model.ActionAddIndex && job.Type != model.ActionDropIndex {
return
@@ -329,10 +328,7 @@ func TestCreateTableWithLike2(t *testing.T) {
onceChecker.Store(job.ID, true)
go backgroundExec(store, "test", "create table t2 like t1", doneCh)
}
- }
- originalHook := dom.DDL().GetHook()
- defer dom.DDL().SetHook(originalHook)
- dom.DDL().SetHook(hook)
+ })
// create table when refer table add column
tk.MustExec("alter table t1 add column d int")
@@ -378,7 +374,7 @@ func TestCreateTableWithLike2(t *testing.T) {
require.NoError(t, err)
}()
- dom.DDL().SetHook(originalHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobRunBefore")
tk.MustExec("drop table if exists t1,t2;")
tk.MustExec("create table t1 (a int) partition by hash(a) partitions 2;")
tk.MustExec("alter table t1 set tiflash replica 3 location labels 'a','b';")
diff --git a/pkg/ddl/ttl.go b/pkg/ddl/ttl.go
index a21b4e801f56e..1a616448b5ee6 100644
--- a/pkg/ddl/ttl.go
+++ b/pkg/ddl/ttl.go
@@ -34,14 +34,14 @@ import (
// DefaultTTLJobInterval is the default value for ttl job interval.
const DefaultTTLJobInterval = "1h"
-func onTTLInfoRemove(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func onTTLInfoRemove(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
tblInfo, err := GetTableInfoAndCancelFaultJob(t, job, job.SchemaID)
if err != nil {
return ver, errors.Trace(err)
}
tblInfo.TTLInfo = nil
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
@@ -49,7 +49,7 @@ func onTTLInfoRemove(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err er
return ver, nil
}
-func onTTLInfoChange(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
+func onTTLInfoChange(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, err error) {
// at least one for them is not nil
var ttlInfo *model.TTLInfo
var ttlInfoEnable *bool
@@ -90,7 +90,7 @@ func onTTLInfoChange(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err er
tblInfo.TTLInfo.JobInterval = *ttlInfoJobInterval
}
- ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true)
+ ver, err = updateVersionAndTableInfo(jobCtx, t, job, tblInfo, true)
if err != nil {
return ver, errors.Trace(err)
}
diff --git a/pkg/ddl/util/BUILD.bazel b/pkg/ddl/util/BUILD.bazel
index e03ba28d779a7..dee960f99a863 100644
--- a/pkg/ddl/util/BUILD.bazel
+++ b/pkg/ddl/util/BUILD.bazel
@@ -6,12 +6,15 @@ go_library(
"dead_table_lock_checker.go",
"mock.go",
"util.go",
+ "watcher.go",
],
importpath = "github.com/pingcap/tidb/pkg/ddl/util",
visibility = ["//visibility:public"],
deps = [
"//pkg/ddl/logutil",
+ "//pkg/infoschema/context",
"//pkg/kv",
+ "//pkg/metrics",
"//pkg/parser/model",
"//pkg/parser/terror",
"//pkg/sessionctx",
diff --git a/pkg/ddl/util/callback/BUILD.bazel b/pkg/ddl/util/callback/BUILD.bazel
deleted file mode 100644
index 4ae863b270d46..0000000000000
--- a/pkg/ddl/util/callback/BUILD.bazel
+++ /dev/null
@@ -1,23 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
-
-go_library(
- name = "callback",
- srcs = ["callback.go"],
- importpath = "github.com/pingcap/tidb/pkg/ddl/util/callback",
- visibility = ["//visibility:public"],
- deps = [
- "//pkg/ddl",
- "//pkg/ddl/logutil",
- "//pkg/parser/model",
- "@org_uber_go_zap//:zap",
- ],
-)
-
-go_test(
- name = "callback_test",
- timeout = "short",
- srcs = ["callback_test.go"],
- embed = [":callback"],
- flaky = True,
- deps = ["//pkg/ddl"],
-)
diff --git a/pkg/ddl/util/callback/callback.go b/pkg/ddl/util/callback/callback.go
deleted file mode 100644
index addc26e53799e..0000000000000
--- a/pkg/ddl/util/callback/callback.go
+++ /dev/null
@@ -1,77 +0,0 @@
-// Copyright 2023 PingCAP, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package callback
-
-import (
- "sync/atomic"
-
- "github.com/pingcap/tidb/pkg/ddl"
- "github.com/pingcap/tidb/pkg/ddl/logutil"
- "github.com/pingcap/tidb/pkg/parser/model"
- "go.uber.org/zap"
-)
-
-// TestDDLCallback is used to customize user callback themselves.
-type TestDDLCallback struct {
- *ddl.BaseCallback
- // We recommended to pass the domain parameter to the test ddl callback, it will ensure
- // domain to reload schema before your ddl stepping into the next state change.
- Do ddl.SchemaLoader
-
- OnJobRunBeforeExported func(*model.Job)
- OnJobRunAfterExported func(*model.Job)
- OnJobUpdatedExported atomic.Pointer[func(*model.Job)]
-}
-
-// OnJobRunBefore is used to run the user customized logic of `onJobRunBefore` first.
-func (tc *TestDDLCallback) OnJobRunBefore(job *model.Job) {
- logutil.DDLLogger().Info("on job run before", zap.String("job", job.String()))
- if tc.OnJobRunBeforeExported != nil {
- tc.OnJobRunBeforeExported(job)
- return
- }
-
- tc.BaseCallback.OnJobRunBefore(job)
-}
-
-// OnJobRunAfter is used to run the user customized logic of `OnJobRunAfter` first.
-func (tc *TestDDLCallback) OnJobRunAfter(job *model.Job) {
- logutil.DDLLogger().Info("on job run after", zap.String("job", job.String()))
- if tc.OnJobRunAfterExported != nil {
- tc.OnJobRunAfterExported(job)
- return
- }
-
- tc.BaseCallback.OnJobRunAfter(job)
-}
-
-// OnJobUpdated is used to run the user customized logic of `OnJobUpdated` first.
-func (tc *TestDDLCallback) OnJobUpdated(job *model.Job) {
- logutil.DDLLogger().Info("on job updated", zap.String("job", job.String()))
- if onJobUpdatedExportedFunc := tc.OnJobUpdatedExported.Load(); onJobUpdatedExportedFunc != nil {
- (*onJobUpdatedExportedFunc)(job)
- return
- }
- if job.State == model.JobStateSynced {
- return
- }
-
- tc.BaseCallback.OnJobUpdated(job)
-}
-
-// Clone copies the callback and take its reference
-func (tc *TestDDLCallback) Clone() *TestDDLCallback {
- return &*tc
-}
diff --git a/pkg/ddl/util/dead_table_lock_checker.go b/pkg/ddl/util/dead_table_lock_checker.go
index f0236e164ca6c..24ea730d3a014 100644
--- a/pkg/ddl/util/dead_table_lock_checker.go
+++ b/pkg/ddl/util/dead_table_lock_checker.go
@@ -21,6 +21,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/tidb/pkg/ddl/logutil"
+ infoschema "github.com/pingcap/tidb/pkg/infoschema/context"
"github.com/pingcap/tidb/pkg/parser/model"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
@@ -73,7 +74,7 @@ func (d *DeadTableLockChecker) getAliveServers(ctx context.Context) (map[string]
}
// GetDeadLockedTables gets dead locked tables.
-func (d *DeadTableLockChecker) GetDeadLockedTables(ctx context.Context, schemas []*model.DBInfo) (map[model.SessionInfo][]model.TableLockTpInfo, error) {
+func (d *DeadTableLockChecker) GetDeadLockedTables(ctx context.Context, is infoschema.MetaOnlyInfoSchema) (map[model.SessionInfo][]model.TableLockTpInfo, error) {
if d.etcdCli == nil {
return nil, nil
}
@@ -82,13 +83,17 @@ func (d *DeadTableLockChecker) GetDeadLockedTables(ctx context.Context, schemas
return nil, err
}
deadLockTables := make(map[model.SessionInfo][]model.TableLockTpInfo)
- for _, schema := range schemas {
+ for _, schema := range is.AllSchemas() {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
- for _, tbl := range schema.Tables {
+ tblInfos, err := is.SchemaTableInfos(ctx, schema.Name)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ for _, tbl := range tblInfos {
if tbl.Lock == nil {
continue
}
diff --git a/pkg/ddl/util/watcher.go b/pkg/ddl/util/watcher.go
new file mode 100644
index 0000000000000..f8d92a320c5de
--- /dev/null
+++ b/pkg/ddl/util/watcher.go
@@ -0,0 +1,80 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+import (
+ "context"
+ "sync"
+ "time"
+
+ "github.com/pingcap/tidb/pkg/ddl/logutil"
+ "github.com/pingcap/tidb/pkg/metrics"
+ clientv3 "go.etcd.io/etcd/client/v3"
+)
+
+// Watcher is responsible for watching the etcd path related operations.
+type Watcher interface {
+ // WatchChan returns the chan for watching etcd path.
+ WatchChan() clientv3.WatchChan
+ // Watch watches the etcd path.
+ Watch(ctx context.Context, etcdCli *clientv3.Client, path string)
+ // Rewatch rewatches the etcd path.
+ Rewatch(ctx context.Context, etcdCli *clientv3.Client, path string)
+}
+
+type watcher struct {
+ wCh clientv3.WatchChan
+ sync.RWMutex
+}
+
+// NewWatcher creates a new watcher.
+func NewWatcher() Watcher {
+ return &watcher{}
+}
+
+// WatchChan implements SyncerWatch.WatchChan interface.
+func (w *watcher) WatchChan() clientv3.WatchChan {
+ w.RLock()
+ defer w.RUnlock()
+ return w.wCh
+}
+
+// Watch implements SyncerWatch.Watch interface.
+func (w *watcher) Watch(ctx context.Context, etcdCli *clientv3.Client, path string) {
+ w.Lock()
+ w.wCh = etcdCli.Watch(ctx, path)
+ w.Unlock()
+}
+
+// Rewatch implements SyncerWatch.Rewatch interface.
+func (w *watcher) Rewatch(ctx context.Context, etcdCli *clientv3.Client, path string) {
+ startTime := time.Now()
+ // Make sure the wCh doesn't receive the information of 'close' before we finish the rewatch.
+ w.Lock()
+ w.wCh = nil
+ w.Unlock()
+
+ go func() {
+ defer func() {
+ metrics.DeploySyncerHistogram.WithLabelValues(metrics.SyncerRewatch, metrics.RetLabel(nil)).Observe(time.Since(startTime).Seconds())
+ }()
+ wCh := etcdCli.Watch(ctx, path)
+
+ w.Lock()
+ w.wCh = wCh
+ w.Unlock()
+ logutil.DDLLogger().Info("syncer rewatch global info finished")
+ }()
+}
diff --git a/pkg/distsql/BUILD.bazel b/pkg/distsql/BUILD.bazel
index 7caa213b87619..fe5073763908d 100644
--- a/pkg/distsql/BUILD.bazel
+++ b/pkg/distsql/BUILD.bazel
@@ -48,7 +48,6 @@ go_library(
"@com_github_tikv_client_go_v2//tikvrpc/interceptor",
"@com_github_tikv_client_go_v2//util",
"@org_golang_google_grpc//metadata",
- "@org_golang_x_exp//maps",
"@org_uber_go_zap//:zap",
],
)
diff --git a/pkg/distsql/request_builder.go b/pkg/distsql/request_builder.go
index f5058361c860c..9f0b8b24a2a18 100644
--- a/pkg/distsql/request_builder.go
+++ b/pkg/distsql/request_builder.go
@@ -47,6 +47,9 @@ type RequestBuilder struct {
kv.Request
is infoschema.MetaOnlyInfoSchema
err error
+
+ // When SetDAGRequest is called, builder will also this field.
+ dag *tipb.DAGRequest
}
// Build builds a "kv.Request".
@@ -75,6 +78,29 @@ func (builder *RequestBuilder) Build() (*kv.Request, error) {
if builder.Request.KeyRanges == nil {
builder.Request.KeyRanges = kv.NewNonPartitionedKeyRanges(nil)
}
+
+ if dag := builder.dag; dag != nil {
+ if execCnt := len(dag.Executors); execCnt == 1 {
+ oldConcurrency := builder.Request.Concurrency
+ // select * from t order by id
+ if builder.Request.KeepOrder {
+ // When the DAG is just simple scan and keep order, set concurrency to 2.
+ // If a lot data are returned to client, mysql protocol is the bottleneck so concurrency 2 is enough.
+ // If very few data are returned to client, the speed is not optimal but good enough.
+ switch dag.Executors[0].Tp {
+ case tipb.ExecType_TypeTableScan, tipb.ExecType_TypeIndexScan, tipb.ExecType_TypePartitionTableScan:
+ builder.Request.Concurrency = 2
+ failpoint.Inject("testRateLimitActionMockConsumeAndAssert", func(val failpoint.Value) {
+ if val.(bool) {
+ // When the concurrency is too small, test case tests/realtikvtest/sessiontest.TestCoprocessorOOMAction can't trigger OOM condition
+ builder.Request.Concurrency = oldConcurrency
+ }
+ })
+ }
+ }
+ }
+ }
+
return &builder.Request, builder.err
}
@@ -154,17 +180,18 @@ func (builder *RequestBuilder) SetDAGRequest(dag *tipb.DAGRequest) *RequestBuild
builder.Request.Tp = kv.ReqTypeDAG
builder.Request.Cacheable = true
builder.Request.Data, builder.err = dag.Marshal()
- }
- if execCnt := len(dag.Executors); execCnt != 0 && dag.Executors[execCnt-1].GetLimit() != nil {
- limit := dag.Executors[execCnt-1].GetLimit()
- builder.Request.LimitSize = limit.GetLimit()
- // When the DAG is just simple scan and small limit, set concurrency to 1 would be sufficient.
- if execCnt == 2 {
- if limit.Limit < estimatedRegionRowCount {
- if kr := builder.Request.KeyRanges; kr != nil {
- builder.Request.Concurrency = kr.PartitionNum()
- } else {
- builder.Request.Concurrency = 1
+ builder.dag = dag
+ if execCnt := len(dag.Executors); execCnt != 0 && dag.Executors[execCnt-1].GetLimit() != nil {
+ limit := dag.Executors[execCnt-1].GetLimit()
+ builder.Request.LimitSize = limit.GetLimit()
+ // When the DAG is just simple scan and small limit, set concurrency to 1 would be sufficient.
+ if execCnt == 2 {
+ if limit.Limit < estimatedRegionRowCount {
+ if kr := builder.Request.KeyRanges; kr != nil {
+ builder.Request.Concurrency = kr.PartitionNum()
+ } else {
+ builder.Request.Concurrency = 1
+ }
}
}
}
diff --git a/pkg/distsql/select_result.go b/pkg/distsql/select_result.go
index 5d485d4cf1483..26369c7b1285e 100644
--- a/pkg/distsql/select_result.go
+++ b/pkg/distsql/select_result.go
@@ -19,6 +19,7 @@ import (
"container/heap"
"context"
"fmt"
+ "maps"
"strconv"
"sync/atomic"
"time"
@@ -46,7 +47,6 @@ import (
"github.com/tikv/client-go/v2/tikv"
clientutil "github.com/tikv/client-go/v2/util"
"go.uber.org/zap"
- "golang.org/x/exp/maps"
)
var (
diff --git a/pkg/disttask/framework/storage/BUILD.bazel b/pkg/disttask/framework/storage/BUILD.bazel
index f9b0edf64bc11..d1dc457934de7 100644
--- a/pkg/disttask/framework/storage/BUILD.bazel
+++ b/pkg/disttask/framework/storage/BUILD.bazel
@@ -17,13 +17,13 @@ go_library(
"//pkg/kv",
"//pkg/sessionctx",
"//pkg/sessionctx/variable",
+ "//pkg/util",
"//pkg/util/chunk",
"//pkg/util/cpu",
"//pkg/util/logutil",
"//pkg/util/sqlescape",
"//pkg/util/sqlexec",
"@com_github_docker_go_units//:go-units",
- "@com_github_ngaut_pools//:pools",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_tikv_client_go_v2//util",
diff --git a/pkg/disttask/framework/storage/task_table.go b/pkg/disttask/framework/storage/task_table.go
index 1fb3e1b0746cd..d2c6f77e232c3 100644
--- a/pkg/disttask/framework/storage/task_table.go
+++ b/pkg/disttask/framework/storage/task_table.go
@@ -21,16 +21,16 @@ import (
"sync/atomic"
"github.com/docker/go-units"
- "github.com/ngaut/pools"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/disttask/framework/proto"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
+ "github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/chunk"
"github.com/pingcap/tidb/pkg/util/sqlexec"
- "github.com/tikv/client-go/v2/util"
+ clitutil "github.com/tikv/client-go/v2/util"
)
const (
@@ -99,12 +99,7 @@ type TaskHandle interface {
// TaskManager is the manager of task and subtask.
type TaskManager struct {
- sePool sessionPool
-}
-
-type sessionPool interface {
- Get() (pools.Resource, error)
- Put(resource pools.Resource)
+ sePool util.SessionPool
}
var _ SessionExecutor = &TaskManager{}
@@ -117,7 +112,7 @@ var (
)
// NewTaskManager creates a new task manager.
-func NewTaskManager(sePool sessionPool) *TaskManager {
+func NewTaskManager(sePool util.SessionPool) *TaskManager {
return &TaskManager{
sePool: sePool,
}
@@ -149,7 +144,7 @@ func (mgr *TaskManager) WithNewSession(fn func(se sessionctx.Context) error) err
// WithNewTxn executes the fn in a new transaction.
func (mgr *TaskManager) WithNewTxn(ctx context.Context, fn func(se sessionctx.Context) error) error {
- ctx = util.WithInternalSourceType(ctx, kv.InternalDistTask)
+ ctx = clitutil.WithInternalSourceType(ctx, kv.InternalDistTask)
return mgr.WithNewSession(func(se sessionctx.Context) (err error) {
_, err = sqlexec.ExecSQL(ctx, se.GetSQLExecutor(), "begin")
if err != nil {
diff --git a/pkg/domain/BUILD.bazel b/pkg/domain/BUILD.bazel
index 57c3ee876f8ff..d369511566fd9 100644
--- a/pkg/domain/BUILD.bazel
+++ b/pkg/domain/BUILD.bazel
@@ -53,6 +53,7 @@ go_library(
"//pkg/parser/model",
"//pkg/parser/mysql",
"//pkg/parser/terror",
+ "//pkg/planner/core/metrics",
"//pkg/privilege/privileges",
"//pkg/sessionctx",
"//pkg/sessionctx/sessionstates",
@@ -60,6 +61,8 @@ go_library(
"//pkg/sessionctx/variable",
"//pkg/statistics",
"//pkg/statistics/handle",
+ "//pkg/statistics/handle/autoanalyze",
+ "//pkg/statistics/handle/initstats",
"//pkg/statistics/handle/logutil",
"//pkg/statistics/handle/util",
"//pkg/store/helper",
@@ -81,6 +84,7 @@ go_library(
"//pkg/util/globalconn",
"//pkg/util/intest",
"//pkg/util/logutil",
+ "//pkg/util/mathutil",
"//pkg/util/memory",
"//pkg/util/memoryusagealarm",
"//pkg/util/printer",
@@ -129,12 +133,11 @@ go_test(
"ru_stats_test.go",
"schema_checker_test.go",
"schema_validator_test.go",
- "session_pool_test.go",
"topn_slow_query_test.go",
],
embed = [":domain"],
flaky = True,
- shard_count = 29,
+ shard_count = 28,
deps = [
"//pkg/config",
"//pkg/ddl",
@@ -151,7 +154,6 @@ go_test(
"//pkg/parser/terror",
"//pkg/server",
"//pkg/session",
- "//pkg/sessionctx",
"//pkg/sessionctx/variable",
"//pkg/store/mockstore",
"//pkg/testkit",
diff --git a/pkg/domain/db_test.go b/pkg/domain/db_test.go
index 983e526745c93..60d5003374ce4 100644
--- a/pkg/domain/db_test.go
+++ b/pkg/domain/db_test.go
@@ -119,9 +119,9 @@ func TestAbnormalSessionPool(t *testing.T) {
info.SetSessionManager(svr)
pool := domain.SysSessionPool()
- failpoint.Enable("github.com/pingcap/tidb/pkg/domain/mockSessionPoolReturnError", "return")
+ failpoint.Enable("github.com/pingcap/tidb/pkg/util/mockSessionPoolReturnError", "return")
se, err := pool.Get()
require.Error(t, err)
- failpoint.Disable("github.com/pingcap/tidb/pkg/domain/mockSessionPoolReturnError")
+ failpoint.Disable("github.com/pingcap/tidb/pkg/util/mockSessionPoolReturnError")
require.Equal(t, svr.InternalSessionExists(se), false)
}
diff --git a/pkg/domain/domain.go b/pkg/domain/domain.go
index 49866feb5b0fe..239bab3ff1be8 100644
--- a/pkg/domain/domain.go
+++ b/pkg/domain/domain.go
@@ -62,13 +62,17 @@ import (
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/parser/terror"
+ metrics2 "github.com/pingcap/tidb/pkg/planner/core/metrics"
"github.com/pingcap/tidb/pkg/privilege/privileges"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessionctx/sessionstates"
"github.com/pingcap/tidb/pkg/sessionctx/sysproctrack"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/statistics/handle"
+ "github.com/pingcap/tidb/pkg/statistics/handle/autoanalyze"
+ "github.com/pingcap/tidb/pkg/statistics/handle/initstats"
statslogutil "github.com/pingcap/tidb/pkg/statistics/handle/logutil"
+ handleutil "github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/store/helper"
"github.com/pingcap/tidb/pkg/ttl/ttlworker"
"github.com/pingcap/tidb/pkg/types"
@@ -83,6 +87,7 @@ import (
"github.com/pingcap/tidb/pkg/util/globalconn"
"github.com/pingcap/tidb/pkg/util/intest"
"github.com/pingcap/tidb/pkg/util/logutil"
+ "github.com/pingcap/tidb/pkg/util/mathutil"
"github.com/pingcap/tidb/pkg/util/memory"
"github.com/pingcap/tidb/pkg/util/memoryusagealarm"
"github.com/pingcap/tidb/pkg/util/replayer"
@@ -147,7 +152,8 @@ type Domain struct {
globalCfgSyncer *globalconfigsync.GlobalConfigSyncer
m syncutil.Mutex
SchemaValidator SchemaValidator
- sysSessionPool *sessionPool
+ schemaLease time.Duration
+ sysSessionPool util.SessionPool
exit chan struct{}
// `etcdClient` must be used when keyspace is not set, or when the logic to each etcd path needs to be separated by keyspace.
etcdClient *clientv3.Client
@@ -166,7 +172,12 @@ type Domain struct {
// TODO: use Run for each process in future pr
wg *util.WaitGroupEnhancedWrapper
statsUpdating atomicutil.Int32
- cancelFns struct {
+ // this is the parent context of DDL, and also used by other loops such as closestReplicaReadCheckLoop.
+ // there are other top level contexts in the domain, such as the ones used in
+ // InitDistTaskLoop and loadStatsWorker, domain only stores the cancelFns of them.
+ // TODO unify top level context.
+ ctx context.Context
+ cancelFns struct {
mu sync.Mutex
fns []context.CancelFunc
}
@@ -199,11 +210,6 @@ type Domain struct {
mdlCheckTableInfo *mdlCheckTableInfo
- analyzeMu struct {
- sync.Mutex
- sctxs map[sessionctx.Context]bool
- }
-
mdlCheckCh chan struct{}
stopAutoAnalyze atomicutil.Bool
minJobIDRefresher *systable.MinJobIDRefresher
@@ -273,7 +279,7 @@ func (do *Domain) EtcdClient() *clientv3.Client {
// 3. currentSchemaVersion(before loading)
// 4. the changed table IDs if it is not full load
// 5. an error if any
-func (do *Domain) loadInfoSchema(startTS uint64) (infoschema.InfoSchema, bool, int64, *transaction.RelatedSchemaChange, error) {
+func (do *Domain) loadInfoSchema(startTS uint64, isSnapshot bool) (infoschema.InfoSchema, bool, int64, *transaction.RelatedSchemaChange, error) {
beginTime := time.Now()
defer func() {
infoschema_metrics.LoadSchemaDurationTotal.Observe(time.Since(beginTime).Seconds())
@@ -294,7 +300,6 @@ func (do *Domain) loadInfoSchema(startTS uint64) (infoschema.InfoSchema, bool, i
schemaTs = 0
}
- enableV2 := variable.SchemaCacheSize.Load() > 0
if is := do.infoCache.GetByVersion(neededSchemaVersion); is != nil {
isV2, raw := infoschema.IsV2(is)
if isV2 {
@@ -310,18 +315,17 @@ func (do *Domain) loadInfoSchema(startTS uint64) (infoschema.InfoSchema, bool, i
// the insert method check if schemaTs is zero
do.infoCache.Insert(is, schemaTs)
- if enableV2 == isV2 {
- return is, true, 0, nil, nil
- }
+ return is, true, 0, nil, nil
}
- var isV1V2Switch bool
+ var oldIsV2 bool
+ enableV2 := variable.SchemaCacheSize.Load() > 0
currentSchemaVersion := int64(0)
if oldInfoSchema := do.infoCache.GetLatest(); oldInfoSchema != nil {
currentSchemaVersion = oldInfoSchema.SchemaMetaVersion()
- isV2, _ := infoschema.IsV2(oldInfoSchema)
- isV1V2Switch = enableV2 != isV2
+ oldIsV2, _ = infoschema.IsV2(oldInfoSchema)
}
+ useV2, isV1V2Switch := shouldUseV2(enableV2, oldIsV2, isSnapshot)
// TODO: tryLoadSchemaDiffs has potential risks of failure. And it becomes worse in history reading cases.
// It is only kept because there is no alternative diff/partial loading solution.
@@ -331,8 +335,8 @@ func (do *Domain) loadInfoSchema(startTS uint64) (infoschema.InfoSchema, bool, i
// 3. There are less 100 diffs.
// 4. No regenerated schema diff.
startTime := time.Now()
- if currentSchemaVersion != 0 && neededSchemaVersion > currentSchemaVersion && neededSchemaVersion-currentSchemaVersion < LoadSchemaDiffVersionGapThreshold {
- is, relatedChanges, diffTypes, err := do.tryLoadSchemaDiffs(m, currentSchemaVersion, neededSchemaVersion, startTS)
+ if !isV1V2Switch && currentSchemaVersion != 0 && neededSchemaVersion > currentSchemaVersion && neededSchemaVersion-currentSchemaVersion < LoadSchemaDiffVersionGapThreshold {
+ is, relatedChanges, diffTypes, err := do.tryLoadSchemaDiffs(useV2, m, currentSchemaVersion, neededSchemaVersion, startTS)
if err == nil {
infoschema_metrics.LoadSchemaDurationLoadDiff.Observe(time.Since(startTime).Seconds())
isV2, _ := infoschema.IsV2(is)
@@ -368,11 +372,12 @@ func (do *Domain) loadInfoSchema(startTS uint64) (infoschema.InfoSchema, bool, i
}
infoschema_metrics.LoadSchemaDurationLoadAll.Observe(time.Since(startTime).Seconds())
- newISBuilder, err := infoschema.NewBuilder(do, do.sysFacHack, do.infoCache.Data).InitWithDBInfos(schemas, policies, resourceGroups, neededSchemaVersion)
+ builder := infoschema.NewBuilder(do, do.sysFacHack, do.infoCache.Data, useV2)
+ err = builder.InitWithDBInfos(schemas, policies, resourceGroups, neededSchemaVersion)
if err != nil {
return nil, false, currentSchemaVersion, nil, err
}
- is := newISBuilder.Build(startTS)
+ is := builder.Build(startTS)
isV2, _ := infoschema.IsV2(is)
logutil.BgLogger().Info("full load InfoSchema success",
zap.Bool("isV2", isV2),
@@ -458,20 +463,22 @@ func (do *Domain) fetchAllSchemasWithTables(m *meta.Meta) ([]*model.DBInfo, erro
const fetchSchemaConcurrency = 1
func (*Domain) splitForConcurrentFetch(schemas []*model.DBInfo) [][]*model.DBInfo {
- groupSize := (len(schemas) + fetchSchemaConcurrency - 1) / fetchSchemaConcurrency
- if variable.SchemaCacheSize.Load() > 0 && len(schemas) > 1000 {
+ groupCnt := fetchSchemaConcurrency
+ schemaCnt := len(schemas)
+ if variable.SchemaCacheSize.Load() > 0 && schemaCnt > 1000 {
// TODO: Temporary solution to speed up when too many databases, will refactor it later.
- groupSize = 8
+ groupCnt = 8
}
- splitted := make([][]*model.DBInfo, 0, fetchSchemaConcurrency)
- schemaCnt := len(schemas)
- for i := 0; i < schemaCnt; i += groupSize {
- end := i + groupSize
- if end > schemaCnt {
- end = schemaCnt
- }
- splitted = append(splitted, schemas[i:end])
+
+ splitted := make([][]*model.DBInfo, 0, groupCnt)
+ groupSizes := mathutil.Divide2Batches(schemaCnt, groupCnt)
+
+ start := 0
+ for _, groupSize := range groupSizes {
+ splitted = append(splitted, schemas[start:start+groupSize])
+ start += groupSize
}
+
return splitted
}
@@ -484,7 +491,7 @@ func (*Domain) fetchSchemasWithTables(schemas []*model.DBInfo, m *meta.Meta, don
var tables []*model.TableInfo
var err error
if variable.SchemaCacheSize.Load() > 0 && !infoschema.IsSpecialDB(di.Name.L) {
- name2ID, specialTableInfos, err := meta.GetAllNameToIDAndSpecialAttributeInfo(m, di.ID)
+ name2ID, specialTableInfos, err := meta.GetAllNameToIDAndTheMustLoadedTableInfo(m, di.ID)
if err != nil {
done <- err
return
@@ -504,7 +511,7 @@ func (*Domain) fetchSchemasWithTables(schemas []*model.DBInfo, m *meta.Meta, don
infoschema.ConvertOldVersionUTF8ToUTF8MB4IfNeed(tbInfo)
}
}
- di.Tables = make([]*model.TableInfo, 0, len(tables))
+ diTables := make([]*model.TableInfo, 0, len(tables))
for _, tbl := range tables {
if tbl.State != model.StatePublic {
// schema is not public, can't be used outside.
@@ -522,17 +529,28 @@ func (*Domain) fetchSchemasWithTables(schemas []*model.DBInfo, m *meta.Meta, don
// haven't been deleted from the repair table list.
// Since the repairment is done and table is visible, we should load it.
}
- di.Tables = append(di.Tables, tbl)
+ diTables = append(diTables, tbl)
}
+ di.Deprecated.Tables = diTables
}
done <- nil
}
+// shouldUseV2 decides whether to use infoschema v2.
+// When loading snapshot, infoschema should keep the same as before to avoid v1/v2 switch.
+// Otherwise, it is decided by enabledV2.
+func shouldUseV2(enableV2 bool, oldIsV2 bool, isSnapshot bool) (useV2 bool, isV1V2Switch bool) {
+ if isSnapshot {
+ return oldIsV2, false
+ }
+ return enableV2, enableV2 != oldIsV2
+}
+
// tryLoadSchemaDiffs tries to only load latest schema changes.
// Return true if the schema is loaded successfully.
// Return false if the schema can not be loaded by schema diff, then we need to do full load.
// The second returned value is the delta updated table and partition IDs.
-func (do *Domain) tryLoadSchemaDiffs(m *meta.Meta, usedVersion, newVersion int64, startTS uint64) (infoschema.InfoSchema, *transaction.RelatedSchemaChange, []string, error) {
+func (do *Domain) tryLoadSchemaDiffs(useV2 bool, m *meta.Meta, usedVersion, newVersion int64, startTS uint64) (infoschema.InfoSchema, *transaction.RelatedSchemaChange, []string, error) {
var diffs []*model.SchemaDiff
for usedVersion < newVersion {
usedVersion++
@@ -567,17 +585,18 @@ func (do *Domain) tryLoadSchemaDiffs(m *meta.Meta, usedVersion, newVersion int64
}
})
- builder, err := infoschema.NewBuilder(do, do.sysFacHack, do.infoCache.Data).InitWithOldInfoSchema(do.infoCache.GetLatest())
+ builder := infoschema.NewBuilder(do, do.sysFacHack, do.infoCache.Data, useV2)
+ err := builder.InitWithOldInfoSchema(do.infoCache.GetLatest())
if err != nil {
return nil, nil, nil, errors.Trace(err)
}
+
builder.WithStore(do.store).SetDeltaUpdateBundles()
phyTblIDs := make([]int64, 0, len(diffs))
actions := make([]uint64, 0, len(diffs))
diffTypes := make([]string, 0, len(diffs))
for _, diff := range diffs {
if diff.RegenerateSchemaMap {
- do.infoCache.Data = infoschema.NewData()
return nil, nil, nil, errors.Errorf("Meets a schema diff with RegenerateSchemaMap flag")
}
ids, err := builder.ApplyDiff(m, diff)
@@ -620,7 +639,7 @@ func (do *Domain) GetSnapshotInfoSchema(snapshotTS uint64) (infoschema.InfoSchem
if is := do.infoCache.GetBySnapshotTS(snapshotTS); is != nil {
return is, nil
}
- is, _, _, _, err := do.loadInfoSchema(snapshotTS)
+ is, _, _, _, err := do.loadInfoSchema(snapshotTS, true)
infoschema_metrics.LoadSchemaCounterSnapshot.Inc()
return is, err
}
@@ -721,12 +740,12 @@ func (do *Domain) Reload() error {
}
version := ver.Ver
- is, hitCache, oldSchemaVersion, changes, err := do.loadInfoSchema(version)
+ is, hitCache, oldSchemaVersion, changes, err := do.loadInfoSchema(version, false)
if err != nil {
if version = getFlashbackStartTSFromErrorMsg(err); version != 0 {
// use the latest available version to create domain
version--
- is, hitCache, oldSchemaVersion, changes, err = do.loadInfoSchema(version)
+ is, hitCache, oldSchemaVersion, changes, err = do.loadInfoSchema(version, false)
}
}
if err != nil {
@@ -757,7 +776,7 @@ func (do *Domain) Reload() error {
// lease renew, so it must be executed despite it is cache or not
do.SchemaValidator.Update(version, oldSchemaVersion, is.SchemaMetaVersion(), changes)
- lease := do.DDL().GetLease()
+ lease := do.GetSchemaLease()
sub := time.Since(startTime)
// Reload interval is lease / 2, if load schema time elapses more than this interval,
// some query maybe responded by ErrInfoSchemaExpired error.
@@ -899,6 +918,24 @@ func (do *Domain) topologySyncerKeeper() {
}
}
+// CheckAutoAnalyzeWindows checks the auto analyze windows and kill the auto analyze process if it is not in the window.
+func (do *Domain) CheckAutoAnalyzeWindows() {
+ se, err := do.sysSessionPool.Get()
+
+ if err != nil {
+ logutil.BgLogger().Warn("get system session failed", zap.Error(err))
+ return
+ }
+ // Make sure the session is new.
+ sctx := se.(sessionctx.Context)
+ defer do.sysSessionPool.Put(se)
+ if !autoanalyze.CheckAutoAnalyzeWindow(sctx) {
+ for _, id := range handleutil.GlobalAutoAnalyzeProcessList.All() {
+ do.SysProcTracker().KillSysProcess(id)
+ }
+ }
+}
+
func (do *Domain) refreshMDLCheckTableInfo() {
se, err := do.sysSessionPool.Get()
@@ -1018,11 +1055,11 @@ func (do *Domain) mdlCheckLoop() {
}
}
-func (do *Domain) loadSchemaInLoop(ctx context.Context, lease time.Duration) {
+func (do *Domain) loadSchemaInLoop(ctx context.Context) {
defer util.Recover(metrics.LabelDomain, "loadSchemaInLoop", nil, true)
// Lease renewal can run at any frequency.
// Use lease/2 here as recommend by paper.
- ticker := time.NewTicker(lease / 2)
+ ticker := time.NewTicker(do.schemaLease / 2)
defer func() {
ticker.Stop()
logutil.BgLogger().Info("loadSchemaInLoop exited.")
@@ -1203,13 +1240,27 @@ func (do *Domain) Close() {
const resourceIdleTimeout = 3 * time.Minute // resources in the ResourcePool will be recycled after idleTimeout
// NewDomain creates a new domain. Should not create multiple domains for the same store.
-func NewDomain(store kv.Storage, ddlLease time.Duration, statsLease time.Duration, dumpFileGcLease time.Duration, factory pools.Factory) *Domain {
+func NewDomain(store kv.Storage, schemaLease time.Duration, statsLease time.Duration, dumpFileGcLease time.Duration, factory pools.Factory) *Domain {
+ intest.Assert(schemaLease > 0, "schema lease should be a positive duration")
capacity := 200 // capacity of the sysSessionPool size
do := &Domain{
- store: store,
- exit: make(chan struct{}),
- sysSessionPool: newSessionPool(capacity, factory),
+ store: store,
+ exit: make(chan struct{}),
+ sysSessionPool: util.NewSessionPool(
+ capacity, factory,
+ func(r pools.Resource) {
+ _, ok := r.(sessionctx.Context)
+ intest.Assert(ok)
+ infosync.StoreInternalSession(r)
+ },
+ func(r pools.Resource) {
+ _, ok := r.(sessionctx.Context)
+ intest.Assert(ok)
+ infosync.DeleteInternalSession(r)
+ },
+ ),
statsLease: statsLease,
+ schemaLease: schemaLease,
slowQuery: newTopNSlowQueries(config.GetGlobalConfig().InMemSlowQueryTopNNum, time.Hour*24*7, config.GetGlobalConfig().InMemSlowQueryRecentNum),
dumpFileGcChecker: &dumpFileGcChecker{gcLease: dumpFileGcLease, paths: []string{replayer.GetPlanReplayerDirName(), GetOptimizerTraceDirName(), GetExtractTaskDirName()}},
mdlCheckTableInfo: &mdlCheckTableInfo{
@@ -1223,7 +1274,7 @@ func NewDomain(store kv.Storage, ddlLease time.Duration, statsLease time.Duratio
do.infoCache = infoschema.NewCache(do, int(variable.SchemaVersionCacheLimit.Load()))
do.stopAutoAnalyze.Store(false)
do.wg = util.NewWaitGroupEnhancedWrapper("domain", do.exit, config.GetGlobalConfig().TiDBEnableExitCheck)
- do.SchemaValidator = NewSchemaValidator(ddlLease, do)
+ do.SchemaValidator = NewSchemaValidator(schemaLease, do)
do.expensiveQueryHandle = expensivequery.NewExpensiveQueryHandle(do.exit)
do.memoryUsageAlarmHandle = memoryusagealarm.NewMemoryUsageAlarmHandle(do.exit)
do.serverMemoryLimitHandle = servermemorylimit.NewServerMemoryLimitHandle(do.exit)
@@ -1260,9 +1311,9 @@ func newEtcdCli(addrs []string, ebd kv.EtcdBackend) (*clientv3.Client, error) {
return cli, err
}
-// Init initializes a domain.
+// Init initializes a domain. after return, session can be used to do DMLs but not
+// DDLs which can be used after domain Start.
func (do *Domain) Init(
- ddlLease time.Duration,
sysExecutorFactory func(*Domain) (pools.Resource, error),
ddlInjector func(ddl.DDL, ddl.Executor, *infoschema.InfoCache) *schematracker.Checker,
) error {
@@ -1294,27 +1345,11 @@ func (do *Domain) Init(
}
}
- // TODO: Here we create new sessions with sysFac in DDL,
- // which will use `do` as Domain instead of call `domap.Get`.
- // That's because `domap.Get` requires a lock, but before
- // we initialize Domain finish, we can't require that again.
- // After we remove the lazy logic of creating Domain, we
- // can simplify code here.
- sysFac := func() (pools.Resource, error) {
- return sysExecutorFactory(do)
- }
- sysCtxPool := pools.NewResourcePool(sysFac, 512, 512, resourceIdleTimeout)
-
ctx, cancelFunc := context.WithCancel(context.Background())
+ do.ctx = ctx
do.cancelFns.mu.Lock()
do.cancelFns.fns = append(do.cancelFns.fns, cancelFunc)
do.cancelFns.mu.Unlock()
- var callback ddl.Callback
- newCallbackFunc, err := ddl.GetCustomizedHook("default_hook")
- if err != nil {
- return errors.Trace(err)
- }
- callback = newCallbackFunc(do)
d := do.ddl
eBak := do.ddlExecutor
do.ddl, do.ddlExecutor = ddl.NewDDL(
@@ -1323,8 +1358,7 @@ func (do *Domain) Init(
ddl.WithStore(do.store),
ddl.WithAutoIDClient(do.autoidClient),
ddl.WithInfoCache(do.infoCache),
- ddl.WithHook(callback),
- ddl.WithLease(ddlLease),
+ ddl.WithLease(do.schemaLease),
ddl.WithSchemaLoader(do),
)
@@ -1344,6 +1378,7 @@ func (do *Domain) Init(
// step 1: prepare the info/schema syncer which domain reload needed.
pdCli, pdHTTPCli := do.GetPDClient(), do.GetPDHTTPClient()
skipRegisterToDashboard := config.GetGlobalConfig().SkipRegisterToDashboard
+ var err error
do.info, err = infosync.GlobalInfoSyncerInit(ctx, do.ddl.GetID(), do.ServerID,
do.etcdClient, do.unprefixedEtcdCli, pdCli, pdHTTPCli,
do.Store().GetCodec(), skipRegisterToDashboard)
@@ -1371,9 +1406,6 @@ func (do *Domain) Init(
}
do.isLostConnectionToPD.Store(0)
}
-
- do.wg.Add(1)
- go do.serverIDKeeper()
} else {
// set serverID for standalone deployment to enable 'KILL'.
atomic.StoreUint64(&do.serverID, serverIDForStandalone)
@@ -1395,33 +1427,49 @@ func (do *Domain) Init(
return err
}
- // step 4: start the ddl after the domain reload, avoiding some internal sql running before infoSchema construction.
- err = do.ddl.Start(sysCtxPool)
- if err != nil {
- return err
- }
- do.minJobIDRefresher = do.ddl.GetMinJobIDRefresher()
-
- // TODO there are many place set ddlLease to 0, remove them completely, we want
- // UT and even local uni-store to run similar code path as normal.
- if ddlLease == 0 {
- ddlLease = time.Second
- }
-
sub := time.Since(startReloadTime)
// The reload(in step 2) operation takes more than ddlLease and a new reload operation was not performed,
// the next query will respond by ErrInfoSchemaExpired error. So we do a new reload to update schemaValidator.latestSchemaExpire.
- if sub > (ddlLease / 2) {
+ if sub > (do.schemaLease / 2) {
logutil.BgLogger().Warn("loading schema and starting ddl take a long time, we do a new reload", zap.Duration("take time", sub))
err = do.Reload()
if err != nil {
return err
}
}
+ return nil
+}
+
+// Start starts the domain. After start, DDLs can be executed using session, see
+// Init also.
+func (do *Domain) Start() error {
+ gCfg := config.GetGlobalConfig()
+ if gCfg.EnableGlobalKill && do.etcdClient != nil {
+ do.wg.Add(1)
+ go do.serverIDKeeper()
+ }
+
+ // TODO: Here we create new sessions with sysFac in DDL,
+ // which will use `do` as Domain instead of call `domap.Get`.
+ // That's because `domap.Get` requires a lock, but before
+ // we initialize Domain finish, we can't require that again.
+ // After we remove the lazy logic of creating Domain, we
+ // can simplify code here.
+ sysFac := func() (pools.Resource, error) {
+ return do.sysExecutorFactory(do)
+ }
+ sysCtxPool := pools.NewResourcePool(sysFac, 512, 512, resourceIdleTimeout)
+
+ // start the ddl after the domain reload, avoiding some internal sql running before infoSchema construction.
+ err := do.ddl.Start(sysCtxPool)
+ if err != nil {
+ return err
+ }
+ do.minJobIDRefresher = do.ddl.GetMinJobIDRefresher()
// Local store needs to get the change information for every DDL state in each session.
do.wg.Run(func() {
- do.loadSchemaInLoop(ctx, ddlLease)
+ do.loadSchemaInLoop(do.ctx)
}, "loadSchemaInLoop")
do.wg.Run(do.mdlCheckLoop, "mdlCheckLoop")
do.wg.Run(do.topNSlowQueryLoop, "topNSlowQueryLoop")
@@ -1429,16 +1477,18 @@ func (do *Domain) Init(
do.wg.Run(do.globalConfigSyncerKeeper, "globalConfigSyncerKeeper")
do.wg.Run(do.runawayStartLoop, "runawayStartLoop")
do.wg.Run(do.requestUnitsWriterLoop, "requestUnitsWriterLoop")
+ skipRegisterToDashboard := gCfg.SkipRegisterToDashboard
if !skipRegisterToDashboard {
do.wg.Run(do.topologySyncerKeeper, "topologySyncerKeeper")
}
+ pdCli := do.GetPDClient()
if pdCli != nil {
do.wg.Run(func() {
- do.closestReplicaReadCheckLoop(ctx, pdCli)
+ do.closestReplicaReadCheckLoop(do.ctx, pdCli)
}, "closestReplicaReadCheckLoop")
}
- err = do.initLogBackup(ctx, pdCli)
+ err = do.initLogBackup(do.ctx, pdCli)
if err != nil {
return err
}
@@ -1446,6 +1496,11 @@ func (do *Domain) Init(
return nil
}
+// GetSchemaLease return the schema lease.
+func (do *Domain) GetSchemaLease() time.Duration {
+ return do.schemaLease
+}
+
// InitInfo4Test init infosync for distributed execution test.
func (do *Domain) InitInfo4Test() {
infosync.MockGlobalServerInfoManagerEntry.Add(do.ddl.GetID(), do.ServerID)
@@ -1699,84 +1754,8 @@ func (do *Domain) distTaskFrameworkLoop(ctx context.Context, taskManager *storag
}
}
-type sessionPool struct {
- resources chan pools.Resource
- factory pools.Factory
- mu struct {
- sync.RWMutex
- closed bool
- }
-}
-
-func newSessionPool(capacity int, factory pools.Factory) *sessionPool {
- return &sessionPool{
- resources: make(chan pools.Resource, capacity),
- factory: factory,
- }
-}
-
-func (p *sessionPool) Get() (resource pools.Resource, err error) {
- var ok bool
- select {
- case resource, ok = <-p.resources:
- if !ok {
- err = errors.New("session pool closed")
- }
- default:
- resource, err = p.factory()
- }
-
- // Put the internal session to the map of SessionManager
- failpoint.Inject("mockSessionPoolReturnError", func() {
- err = errors.New("mockSessionPoolReturnError")
- })
-
- if nil == err {
- _, ok = resource.(sessionctx.Context)
- intest.Assert(ok)
- infosync.StoreInternalSession(resource)
- }
-
- return
-}
-
-func (p *sessionPool) Put(resource pools.Resource) {
- _, ok := resource.(sessionctx.Context)
- intest.Assert(ok)
-
- p.mu.RLock()
- defer p.mu.RUnlock()
- // Delete the internal session to the map of SessionManager
- infosync.DeleteInternalSession(resource)
- if p.mu.closed {
- resource.Close()
- return
- }
-
- select {
- case p.resources <- resource:
- default:
- resource.Close()
- }
-}
-
-func (p *sessionPool) Close() {
- p.mu.Lock()
- if p.mu.closed {
- p.mu.Unlock()
- return
- }
- p.mu.closed = true
- close(p.resources)
- p.mu.Unlock()
-
- for r := range p.resources {
- r.Close()
- }
-}
-
// SysSessionPool returns the system session pool.
-func (do *Domain) SysSessionPool() *sessionPool {
+func (do *Domain) SysSessionPool() util.SessionPool {
return do.sysSessionPool
}
@@ -2124,7 +2103,7 @@ func (do *Domain) SetupDumpFileGCChecker(ctx sessionctx.Context) {
// SetupExtractHandle setups extract handler
func (do *Domain) SetupExtractHandle(sctxs []sessionctx.Context) {
- do.extractTaskHandle = NewExtractHandler(sctxs)
+ do.extractTaskHandle = newExtractHandler(do.ctx, sctxs)
}
var planReplayerHandleLease atomic.Uint64
@@ -2269,7 +2248,7 @@ func (do *Domain) StatsHandle() *handle.Handle {
// CreateStatsHandle is used only for test.
func (do *Domain) CreateStatsHandle(ctx, initStatsCtx sessionctx.Context) error {
- h, err := handle.NewHandle(ctx, initStatsCtx, do.statsLease, do.sysSessionPool, &do.sysProcesses, do.GetAutoAnalyzeProcID)
+ h, err := handle.NewHandle(ctx, initStatsCtx, do.statsLease, do.sysSessionPool, &do.sysProcesses, do.NextConnID, do.ReleaseConnID)
if err != nil {
return err
}
@@ -2292,46 +2271,6 @@ func (do *Domain) SetStatsUpdating(val bool) {
}
}
-// ReleaseAnalyzeExec returned extra exec for Analyze
-func (do *Domain) ReleaseAnalyzeExec(sctxs []sessionctx.Context) {
- do.analyzeMu.Lock()
- defer do.analyzeMu.Unlock()
- for _, ctx := range sctxs {
- do.analyzeMu.sctxs[ctx] = false
- }
-}
-
-// FetchAnalyzeExec get needed exec for analyze
-func (do *Domain) FetchAnalyzeExec(need int) []sessionctx.Context {
- if need < 1 {
- return nil
- }
- count := 0
- r := make([]sessionctx.Context, 0)
- do.analyzeMu.Lock()
- defer do.analyzeMu.Unlock()
- for sctx, used := range do.analyzeMu.sctxs {
- if used {
- continue
- }
- r = append(r, sctx)
- do.analyzeMu.sctxs[sctx] = true
- count++
- if count >= need {
- break
- }
- }
- return r
-}
-
-// SetupAnalyzeExec setups exec for Analyze Executor
-func (do *Domain) SetupAnalyzeExec(ctxs []sessionctx.Context) {
- do.analyzeMu.sctxs = make(map[sessionctx.Context]bool)
- for _, ctx := range ctxs {
- do.analyzeMu.sctxs[ctx] = false
- }
-}
-
// LoadAndUpdateStatsLoop loads and updates stats info.
func (do *Domain) LoadAndUpdateStatsLoop(ctxs []sessionctx.Context, initStatsCtx sessionctx.Context) error {
if err := do.UpdateTableStatsLoop(ctxs[0], initStatsCtx); err != nil {
@@ -2346,7 +2285,7 @@ func (do *Domain) LoadAndUpdateStatsLoop(ctxs []sessionctx.Context, initStatsCtx
// It should be called only once in BootstrapSession.
func (do *Domain) UpdateTableStatsLoop(ctx, initStatsCtx sessionctx.Context) error {
ctx.GetSessionVars().InRestrictedSQL = true
- statsHandle, err := handle.NewHandle(ctx, initStatsCtx, do.statsLease, do.sysSessionPool, &do.sysProcesses, do.GetAutoAnalyzeProcID)
+ statsHandle, err := handle.NewHandle(ctx, initStatsCtx, do.statsLease, do.sysSessionPool, &do.sysProcesses, do.NextConnID, do.ReleaseConnID)
if err != nil {
return err
}
@@ -2466,12 +2405,14 @@ func (do *Domain) initStats(ctx context.Context) {
}()
t := time.Now()
liteInitStats := config.GetGlobalConfig().Performance.LiteInitStats
+ initstats.InitStatsPercentage.Store(0)
var err error
if liteInitStats {
err = statsHandle.InitStatsLite(ctx, do.InfoSchema())
} else {
err = statsHandle.InitStats(ctx, do.InfoSchema())
}
+ initstats.InitStatsPercentage.Store(100)
if err != nil {
logutil.BgLogger().Error("init stats info failed", zap.Bool("lite", liteInitStats), zap.Duration("take time", time.Since(t)), zap.Error(err))
} else {
@@ -2613,10 +2554,11 @@ func (do *Domain) updateStatsWorker(_ sessionctx.Context, owner owner.Manager) {
if !owner.IsOwner() {
continue
}
- err := statsHandle.GCStats(do.InfoSchema(), do.DDL().GetLease())
+ err := statsHandle.GCStats(do.InfoSchema(), do.GetSchemaLease())
if err != nil {
logutil.BgLogger().Debug("GC stats failed", zap.Error(err))
}
+ do.CheckAutoAnalyzeWindows()
case <-dumpColStatsUsageTicker.C:
err := statsHandle.DumpColStatsUsageToKV()
if err != nil {
@@ -2769,12 +2711,11 @@ func (do *Domain) NotifyUpdatePrivilege() error {
}
// update locally
- sysSessionPool := do.SysSessionPool()
- ctx, err := sysSessionPool.Get()
+ ctx, err := do.sysSessionPool.Get()
if err != nil {
return err
}
- defer sysSessionPool.Put(ctx)
+ defer do.sysSessionPool.Put(ctx)
return do.PrivilegeHandle().Update(ctx.(sessionctx.Context))
}
@@ -2839,12 +2780,6 @@ func (do *Domain) ReleaseConnID(connID uint64) {
do.connIDAllocator.Release(connID)
}
-// GetAutoAnalyzeProcID returns processID for auto analyze
-// TODO: support IDs for concurrent auto-analyze
-func (do *Domain) GetAutoAnalyzeProcID() uint64 {
- return do.connIDAllocator.GetReservedConnID(reservedConnAnalyze)
-}
-
const (
serverIDEtcdPath = "/tidb/server_id"
refreshServerIDRetryCnt = 3
@@ -2853,9 +2788,6 @@ const (
retrieveServerIDSessionTimeout = 10 * time.Second
acquire32BitsServerIDRetryCnt = 3
-
- // reservedConnXXX must be within [0, globalconn.ReservedCount)
- reservedConnAnalyze = 0
)
var (
@@ -3160,9 +3092,13 @@ func (do *Domain) StopAutoAnalyze() {
}
// InitInstancePlanCache initializes the instance level plan cache for this Domain.
-func (do *Domain) InitInstancePlanCache(softMemLimit, hardMemLimit int64) {
- do.instancePlanCache = NewInstancePlanCache(softMemLimit, hardMemLimit)
+func (do *Domain) InitInstancePlanCache() {
+ softLimit := variable.InstancePlanCacheTargetMemSize.Load()
+ hardLimit := variable.InstancePlanCacheMaxMemSize.Load()
+ do.instancePlanCache = NewInstancePlanCache(softLimit, hardLimit)
+ // use a separate goroutine to avoid the eviction blocking other operations.
do.wg.Run(do.planCacheEvictTrigger, "planCacheEvictTrigger")
+ do.wg.Run(do.planCacheMetricsAndVars, "planCacheMetricsAndVars")
}
// GetInstancePlanCache returns the instance level plan cache in this Domain.
@@ -3170,10 +3106,41 @@ func (do *Domain) GetInstancePlanCache() sessionctx.InstancePlanCache {
return do.instancePlanCache
}
+// planCacheMetricsAndVars updates metrics and variables for Instance Plan Cache periodically.
+func (do *Domain) planCacheMetricsAndVars() {
+ defer util.Recover(metrics.LabelDomain, "planCacheMetricsAndVars", nil, false)
+ ticker := time.NewTicker(time.Second * 15) // 15s by default
+ defer func() {
+ ticker.Stop()
+ logutil.BgLogger().Info("planCacheMetricsAndVars exited.")
+ }()
+
+ for {
+ select {
+ case <-ticker.C:
+ // update limits
+ softLimit := variable.InstancePlanCacheTargetMemSize.Load()
+ hardLimit := variable.InstancePlanCacheMaxMemSize.Load()
+ curSoft, curHard := do.instancePlanCache.GetLimits()
+ if curSoft != softLimit || curHard != hardLimit {
+ do.instancePlanCache.SetLimits(softLimit, hardLimit)
+ }
+
+ // update the metrics
+ size := do.instancePlanCache.Size()
+ memUsage := do.instancePlanCache.MemUsage()
+ metrics2.GetPlanCacheInstanceNumCounter(true).Set(float64(size))
+ metrics2.GetPlanCacheInstanceMemoryUsage(true).Set(float64(memUsage))
+ case <-do.exit:
+ return
+ }
+ }
+}
+
// planCacheEvictTrigger triggers the plan cache eviction periodically.
func (do *Domain) planCacheEvictTrigger() {
defer util.Recover(metrics.LabelDomain, "planCacheEvictTrigger", nil, false)
- ticker := time.NewTicker(time.Second * 5) // 5s by default
+ ticker := time.NewTicker(time.Second * 30) // 30s by default
defer func() {
ticker.Stop()
logutil.BgLogger().Info("planCacheEvictTrigger exited.")
@@ -3182,8 +3149,14 @@ func (do *Domain) planCacheEvictTrigger() {
for {
select {
case <-ticker.C:
- do.instancePlanCache.Evict()
- // TODO: update the metrics
+ // trigger the eviction
+ begin := time.Now()
+ detailInfo, numEvicted := do.instancePlanCache.Evict()
+ metrics2.GetPlanCacheInstanceEvict().Set(float64(numEvicted))
+ logutil.BgLogger().Info("instance plan eviction",
+ zap.String("detail", detailInfo),
+ zap.Int64("num_evicted", int64(numEvicted)),
+ zap.Duration("time_spent", time.Since(begin)))
case <-do.exit:
return
}
diff --git a/pkg/domain/domain_test.go b/pkg/domain/domain_test.go
index bd064e26f34a1..db521e24a7951 100644
--- a/pkg/domain/domain_test.go
+++ b/pkg/domain/domain_test.go
@@ -93,7 +93,8 @@ func TestInfo(t *testing.T) {
ddl.DisableTiFlashPoll(dom.ddl)
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/domain/MockReplaceDDL", `return(true)`))
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/NoDDLDispatchLoop", `return(true)`))
- require.NoError(t, dom.Init(ddlLease, sysMockFactory, nil))
+ require.NoError(t, dom.Init(sysMockFactory, nil))
+ require.NoError(t, dom.Start())
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/NoDDLDispatchLoop"))
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/domain/MockReplaceDDL"))
@@ -120,9 +121,9 @@ func TestInfo(t *testing.T) {
require.Equalf(t, info.ID, infos[ddlID].ID, "server one info %v, info %v", infos[ddlID], info)
// Test the scene where syncer.Done() gets the information.
- require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/syncer/ErrorMockSessionDone", `return(true)`))
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/schemaver/ErrorMockSessionDone", `return(true)`))
<-dom.ddl.SchemaSyncer().Done()
- require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/syncer/ErrorMockSessionDone"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/schemaver/ErrorMockSessionDone"))
time.Sleep(15 * time.Millisecond)
syncerStarted := false
for i := 0; i < 1000; i++ {
diff --git a/pkg/domain/extract.go b/pkg/domain/extract.go
index fdb79e657ed13..7bef58b6939ab 100644
--- a/pkg/domain/extract.go
+++ b/pkg/domain/extract.go
@@ -75,10 +75,10 @@ type ExtractHandle struct {
worker *extractWorker
}
-// NewExtractHandler new extract handler
-func NewExtractHandler(sctxs []sessionctx.Context) *ExtractHandle {
+// newExtractHandler new extract handler
+func newExtractHandler(ctx context.Context, sctxs []sessionctx.Context) *ExtractHandle {
h := &ExtractHandle{}
- h.worker = newExtractWorker(sctxs[0], false)
+ h.worker = newExtractWorker(ctx, sctxs[0], false)
return h
}
@@ -121,8 +121,13 @@ func NewExtractPlanTask(begin, end time.Time) *ExtractTask {
}
}
-func newExtractWorker(sctx sessionctx.Context, isBackgroundWorker bool) *extractWorker {
+func newExtractWorker(
+ ctx context.Context,
+ sctx sessionctx.Context,
+ isBackgroundWorker bool,
+) *extractWorker {
return &extractWorker{
+ ctx: ctx,
sctx: sctx,
isBackgroundWorker: isBackgroundWorker,
}
diff --git a/pkg/domain/historical_stats.go b/pkg/domain/historical_stats.go
index 9b4dd016d2711..e865cfc0f0048 100644
--- a/pkg/domain/historical_stats.go
+++ b/pkg/domain/historical_stats.go
@@ -15,6 +15,8 @@
package domain
import (
+ "context"
+
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
domain_metrics "github.com/pingcap/tidb/pkg/domain/metrics"
@@ -64,7 +66,7 @@ func (w *HistoricalStatsWorker) DumpHistoricalStats(tableID int64, statsHandle *
is := GetDomain(sctx).InfoSchema()
isPartition := false
var tblInfo *model.TableInfo
- tbl, existed := is.TableByID(tableID)
+ tbl, existed := is.TableByID(context.Background(), tableID)
if !existed {
tbl, db, p := is.FindTableByPartitionID(tableID)
if !(tbl != nil && db != nil && p != nil) {
diff --git a/pkg/domain/infosync/label_manager.go b/pkg/domain/infosync/label_manager.go
index 711a0a651ee0b..fb842bbdf72c6 100644
--- a/pkg/domain/infosync/label_manager.go
+++ b/pkg/domain/infosync/label_manager.go
@@ -60,8 +60,8 @@ func (lm *PDLabelManager) GetAllLabelRules(ctx context.Context) ([]*label.Rule,
}
// GetLabelRules implements GetLabelRules
-func (lm *PDLabelManager) GetLabelRules(ctx context.Context, _ []string) (map[string]*label.Rule, error) {
- labelRules, err := lm.pdHTTPCli.GetAllRegionLabelRules(ctx)
+func (lm *PDLabelManager) GetLabelRules(ctx context.Context, ruleIDs []string) (map[string]*label.Rule, error) {
+ labelRules, err := lm.pdHTTPCli.GetRegionLabelRulesByIDs(ctx, ruleIDs)
if err != nil {
return nil, err
}
diff --git a/pkg/domain/plan_replayer_dump.go b/pkg/domain/plan_replayer_dump.go
index 1a5b95e4eb0c2..29601e6d7f667 100644
--- a/pkg/domain/plan_replayer_dump.go
+++ b/pkg/domain/plan_replayer_dump.go
@@ -440,16 +440,17 @@ func dumpMeta(zw *zip.Writer) error {
return nil
}
-func dumpTiFlashReplica(ctx sessionctx.Context, zw *zip.Writer, pairs map[tableNamePair]struct{}) error {
+func dumpTiFlashReplica(sctx sessionctx.Context, zw *zip.Writer, pairs map[tableNamePair]struct{}) error {
bf, err := zw.Create(PlanReplayerTiFlashReplicasFile)
if err != nil {
return errors.AddStack(err)
}
- is := GetDomain(ctx).InfoSchema()
+ is := GetDomain(sctx).InfoSchema()
+ ctx := infoschema.WithRefillOption(context.Background(), false)
for pair := range pairs {
dbName := model.NewCIStr(pair.DBName)
tableName := model.NewCIStr(pair.TableName)
- t, err := is.TableByName(context.Background(), dbName, tableName)
+ t, err := is.TableByName(ctx, dbName, tableName)
if err != nil {
logutil.BgLogger().Warn("failed to find table info", zap.Error(err),
zap.String("dbName", dbName.L), zap.String("tableName", tableName.L))
@@ -496,11 +497,12 @@ func dumpSchemaMeta(zw *zip.Writer, tables map[tableNamePair]struct{}) error {
func dumpStatsMemStatus(zw *zip.Writer, pairs map[tableNamePair]struct{}, do *Domain) error {
statsHandle := do.StatsHandle()
is := do.InfoSchema()
+ ctx := infoschema.WithRefillOption(context.Background(), false)
for pair := range pairs {
if pair.IsView {
continue
}
- tbl, err := is.TableByName(context.Background(), model.NewCIStr(pair.DBName), model.NewCIStr(pair.TableName))
+ tbl, err := is.TableByName(ctx, model.NewCIStr(pair.DBName), model.NewCIStr(pair.TableName))
if err != nil {
return err
}
diff --git a/pkg/domain/resourcegroup/runaway.go b/pkg/domain/resourcegroup/runaway.go
index f19adec9568ea..b7260bd2725c0 100644
--- a/pkg/domain/resourcegroup/runaway.go
+++ b/pkg/domain/resourcegroup/runaway.go
@@ -16,6 +16,7 @@ package resourcegroup
import (
"context"
+ "fmt"
"strings"
"sync"
"sync/atomic"
@@ -53,27 +54,6 @@ const (
// NullTime is a zero time.Time.
var NullTime time.Time
-// RunawayMatchType is used to indicate whether query was interrupted by runaway identification or quarantine watch.
-type RunawayMatchType uint
-
-const (
- // RunawayMatchTypeWatch shows quarantine watch.
- RunawayMatchTypeWatch RunawayMatchType = iota
- // RunawayMatchTypeIdentify shows identification.
- RunawayMatchTypeIdentify
-)
-
-func (t RunawayMatchType) String() string {
- switch t {
- case RunawayMatchTypeWatch:
- return "watch"
- case RunawayMatchTypeIdentify:
- return "identify"
- default:
- panic("unknown type")
- }
-}
-
// RunawayRecord is used to save records which will be insert into mysql.tidb_runaway_queries.
type RunawayRecord struct {
ResourceGroupName string
@@ -399,7 +379,7 @@ func (rm *RunawayManager) getWatchFromWatchList(key string) *QuarantineRecord {
return nil
}
-func (rm *RunawayManager) markRunaway(resourceGroupName, originalSQL, planDigest string, action string, matchType RunawayMatchType, now *time.Time) {
+func (rm *RunawayManager) markRunaway(resourceGroupName, originalSQL, planDigest, action, matchType string, now *time.Time) {
source := rm.serverID
if !rm.syncerInitialized.Load() {
rm.logOnce.Do(func() {
@@ -411,7 +391,7 @@ func (rm *RunawayManager) markRunaway(resourceGroupName, originalSQL, planDigest
case rm.runawayQueriesChan <- &RunawayRecord{
ResourceGroupName: resourceGroupName,
Time: *now,
- Match: matchType.String(),
+ Match: matchType,
Action: action,
SQLText: originalSQL,
PlanDigest: planDigest,
@@ -470,26 +450,31 @@ type RunawayChecker struct {
planDigest string
deadline time.Time
- setting *rmpb.RunawaySettings
+ // From the group runaway settings, which will be applied when a query lacks a specified watch rule.
+ settings *rmpb.RunawaySettings
- markedByRule atomic.Bool
+ // markedByRule is set to true when the query matches the group runaway settings.
+ markedByRule atomic.Bool
+ // markedByWatch is set to true when the query matches the specified watch rules.
markedByWatch bool
watchAction rmpb.RunawayAction
}
-func newRunawayChecker(manager *RunawayManager, resourceGroupName string, setting *rmpb.RunawaySettings, originalSQL, sqlDigest, planDigest string, startTime time.Time) *RunawayChecker {
+func newRunawayChecker(
+ manager *RunawayManager,
+ resourceGroupName string, settings *rmpb.RunawaySettings,
+ originalSQL, sqlDigest, planDigest string, startTime time.Time,
+) *RunawayChecker {
c := &RunawayChecker{
manager: manager,
resourceGroupName: resourceGroupName,
originalSQL: originalSQL,
sqlDigest: sqlDigest,
planDigest: planDigest,
- setting: setting,
- markedByRule: atomic.Bool{},
- markedByWatch: false,
+ settings: settings,
}
- if setting != nil {
- c.deadline = startTime.Add(time.Duration(setting.Rule.ExecElapsedTimeMs) * time.Millisecond)
+ if settings != nil {
+ c.deadline = startTime.Add(time.Duration(settings.Rule.ExecElapsedTimeMs) * time.Millisecond)
}
return c
}
@@ -499,50 +484,96 @@ func (r *RunawayChecker) BeforeExecutor() error {
if r == nil {
return nil
}
+ // Check if the query matches any specified watch rules.
for _, convict := range r.getConvictIdentifiers() {
watched, action := r.manager.examineWatchList(r.resourceGroupName, convict)
- if watched {
- if action == rmpb.RunawayAction_NoneAction && r.setting != nil {
- action = r.setting.Action
- }
- r.markedByWatch = true
- now := time.Now()
- r.watchAction = action
- r.markRunaway(RunawayMatchTypeWatch, action, &now)
- // If no match action, it will do nothing.
- switch action {
- case rmpb.RunawayAction_Kill:
- return exeerrors.ErrResourceGroupQueryRunawayQuarantine
- case rmpb.RunawayAction_CoolDown:
- // This action should be done in BeforeCopRequest.
- return nil
- case rmpb.RunawayAction_DryRun:
- return nil
- default:
- }
+ if !watched {
+ continue
+ }
+ // Use the group runaway settings if none are provided.
+ if action == rmpb.RunawayAction_NoneAction && r.settings != nil {
+ action = r.settings.Action
+ }
+ // Mark it if this is the first time being watched.
+ r.markRunawayByWatch(action)
+ // Take action if needed.
+ switch action {
+ case rmpb.RunawayAction_Kill:
+ // Return an error to interrupt the query.
+ return exeerrors.ErrResourceGroupQueryRunawayQuarantine
+ case rmpb.RunawayAction_CoolDown:
+ // This action will be handled in `BeforeCopRequest`.
+ return nil
+ case rmpb.RunawayAction_DryRun:
+ // Noop.
+ return nil
+ default:
+ // Continue to examine other convicts.
}
}
return nil
}
+// CheckAction is used to check current action of the query.
+// It's safe to call this method concurrently.
+func (r *RunawayChecker) CheckAction() rmpb.RunawayAction {
+ if r == nil {
+ return rmpb.RunawayAction_NoneAction
+ }
+ if r.markedByWatch {
+ return r.watchAction
+ }
+ if r.markedByRule.Load() {
+ return r.settings.Action
+ }
+ return rmpb.RunawayAction_NoneAction
+}
+
+// CheckRuleKillAction checks whether the query should be killed according to the group settings.
+func (r *RunawayChecker) CheckRuleKillAction() bool {
+ // If the group settings are not available and it's not marked by watch, skip this part.
+ if r.settings == nil && !r.markedByWatch {
+ return false
+ }
+ // If the group settings are available and it's not marked by rule, check the execution time.
+ if r.settings != nil && !r.markedByRule.Load() {
+ now := time.Now()
+ until := r.deadline.Sub(now)
+ if until > 0 {
+ return false
+ }
+ r.markRunawayByIdentify(r.settings.Action, &now)
+ return r.settings.Action == rmpb.RunawayAction_Kill
+ }
+ return false
+}
+
+// Rule returns the rule of the runaway checker.
+func (r *RunawayChecker) Rule() string {
+ var execElapsedTime time.Duration
+ if r.settings != nil {
+ execElapsedTime = time.Duration(r.settings.Rule.ExecElapsedTimeMs) * time.Millisecond
+ }
+ return fmt.Sprintf("execElapsedTime:%s", execElapsedTime)
+}
+
// BeforeCopRequest checks runaway and modifies the request if necessary before sending coprocessor request.
func (r *RunawayChecker) BeforeCopRequest(req *tikvrpc.Request) error {
- if r.setting == nil && !r.markedByWatch {
+ // If the group settings are not available and it's not marked by watch, skip this part.
+ if r.settings == nil && !r.markedByWatch {
return nil
}
- marked := r.markedByRule.Load()
- if !marked {
- // note: now we don't check whether query is in watch list again.
- if r.markedByWatch {
- if r.watchAction == rmpb.RunawayAction_CoolDown {
- req.ResourceControlContext.OverridePriority = 1 // set priority to lowest
- }
- }
-
+ // If it's marked by watch and the action is cooldown, override the priority,
+ if r.markedByWatch && r.watchAction == rmpb.RunawayAction_CoolDown {
+ req.ResourceControlContext.OverridePriority = 1 // set priority to lowest
+ }
+ // If group settings are available and the query is not marked by a rule,
+ // verify if it matches any rules in the settings.
+ if r.settings != nil && !r.markedByRule.Load() {
now := time.Now()
until := r.deadline.Sub(now)
if until > 0 {
- if r.setting.Action == rmpb.RunawayAction_Kill {
+ if r.settings.Action == rmpb.RunawayAction_Kill {
// if the execution time is close to the threshold, set a timeout
if until < tikv.ReadTimeoutMedium {
req.Context.MaxExecutionDurationMs = uint64(until.Milliseconds())
@@ -551,39 +582,30 @@ func (r *RunawayChecker) BeforeCopRequest(req *tikvrpc.Request) error {
return nil
}
// execution time exceeds the threshold, mark the query as runaway
- if r.markedByRule.CompareAndSwap(false, true) {
- r.markRunaway(RunawayMatchTypeIdentify, r.setting.Action, &now)
- if !r.markedByWatch {
- r.markQuarantine(&now)
- }
+ r.markRunawayByIdentify(r.settings.Action, &now)
+ // Take action if needed.
+ switch r.settings.Action {
+ case rmpb.RunawayAction_Kill:
+ return exeerrors.ErrResourceGroupQueryRunawayInterrupted
+ case rmpb.RunawayAction_CoolDown:
+ req.ResourceControlContext.OverridePriority = 1 // set priority to lowest
+ return nil
+ default:
+ return nil
}
}
- switch r.setting.Action {
- case rmpb.RunawayAction_Kill:
- return exeerrors.ErrResourceGroupQueryRunawayInterrupted
- case rmpb.RunawayAction_CoolDown:
- req.ResourceControlContext.OverridePriority = 1 // set priority to lowest
- return nil
- case rmpb.RunawayAction_DryRun:
- return nil
- default:
- return nil
- }
+ return nil
}
// CheckCopRespError checks TiKV error after receiving coprocessor response.
func (r *RunawayChecker) CheckCopRespError(err error) error {
- if err == nil || r.setting == nil || r.setting.Action != rmpb.RunawayAction_Kill {
+ if err == nil || r.settings == nil || r.settings.Action != rmpb.RunawayAction_Kill {
return err
}
if strings.HasPrefix(err.Error(), "Coprocessor task terminated due to exceeding the deadline") {
if !r.markedByRule.Load() {
now := time.Now()
- if r.deadline.Before(now) && r.markedByRule.CompareAndSwap(false, true) {
- r.markRunaway(RunawayMatchTypeIdentify, r.setting.Action, &now)
- if !r.markedByWatch {
- r.markQuarantine(&now)
- }
+ if r.deadline.Before(now) && r.markRunawayByIdentify(r.settings.Action, &now) {
return exeerrors.ErrResourceGroupQueryRunawayInterrupted
}
}
@@ -596,25 +618,43 @@ func (r *RunawayChecker) CheckCopRespError(err error) error {
}
func (r *RunawayChecker) markQuarantine(now *time.Time) {
- if r.setting.Watch == nil {
+ if r.settings == nil || r.settings.Watch == nil {
return
}
- ttl := time.Duration(r.setting.Watch.LastingDurationMs) * time.Millisecond
+ ttl := time.Duration(r.settings.Watch.LastingDurationMs) * time.Millisecond
+
+ r.manager.markQuarantine(r.resourceGroupName, r.getSettingConvictIdentifier(), r.settings.Watch.Type, r.settings.Action, ttl, now)
+}
+
+func (r *RunawayChecker) markRunawayByIdentify(action rmpb.RunawayAction, now *time.Time) bool {
+ swapped := r.markedByRule.CompareAndSwap(false, true)
+ if swapped {
+ r.markRunaway("identify", action, now)
+ if !r.markedByWatch {
+ r.markQuarantine(now)
+ }
+ }
+ return swapped
+}
- r.manager.markQuarantine(r.resourceGroupName, r.getSettingConvictIdentifier(), r.setting.Watch.Type, r.setting.Action, ttl, now)
+func (r *RunawayChecker) markRunawayByWatch(action rmpb.RunawayAction) {
+ r.markedByWatch = true
+ r.watchAction = action
+ now := time.Now()
+ r.markRunaway("watch", action, &now)
}
-func (r *RunawayChecker) markRunaway(matchType RunawayMatchType, action rmpb.RunawayAction, now *time.Time) {
- actionStr := strings.ToLower(rmpb.RunawayAction_name[int32(action)])
- metrics.RunawayCheckerCounter.WithLabelValues(r.resourceGroupName, matchType.String(), actionStr).Inc()
+func (r *RunawayChecker) markRunaway(matchType string, action rmpb.RunawayAction, now *time.Time) {
+ actionStr := strings.ToLower(action.String())
+ metrics.RunawayCheckerCounter.WithLabelValues(r.resourceGroupName, matchType, actionStr).Inc()
r.manager.markRunaway(r.resourceGroupName, r.originalSQL, r.planDigest, actionStr, matchType, now)
}
func (r *RunawayChecker) getSettingConvictIdentifier() string {
- if r.setting.Watch == nil {
+ if r == nil || r.settings == nil || r.settings.Watch == nil {
return ""
}
- switch r.setting.Watch.Type {
+ switch r.settings.Watch.Type {
case rmpb.RunawayWatchType_Plan:
return r.planDigest
case rmpb.RunawayWatchType_Similar:
diff --git a/pkg/domain/ru_stats.go b/pkg/domain/ru_stats.go
index 68a308ac3bfc9..1dd64e969b68b 100644
--- a/pkg/domain/ru_stats.go
+++ b/pkg/domain/ru_stats.go
@@ -25,6 +25,7 @@ import (
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/meta"
"github.com/pingcap/tidb/pkg/parser/model"
+ "github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/intest"
"github.com/pingcap/tidb/pkg/util/logutil"
pd "github.com/tikv/pd/client"
@@ -46,7 +47,7 @@ type RUStatsWriter struct {
RMClient pd.ResourceManagerClient
InfoCache *infoschema.InfoCache
store kv.Storage
- sessPool *sessionPool
+ sessPool util.SessionPool
// current time, cache it here to make unit test easier.
StartTime time.Time
}
diff --git a/pkg/domain/runaway.go b/pkg/domain/runaway.go
index 37cc6061ca12b..ae5d50724a047 100644
--- a/pkg/domain/runaway.go
+++ b/pkg/domain/runaway.go
@@ -216,7 +216,7 @@ func (do *Domain) GetRunawayWatchList() []*resourcegroup.QuarantineRecord {
return do.runawayManager.GetWatchList()
}
-// TryToUpdateRunawayWatch is used to to update watch list including
+// TryToUpdateRunawayWatch is used to update watch list including
// creation and deletion by manual trigger.
func (do *Domain) TryToUpdateRunawayWatch() error {
return do.updateNewAndDoneWatch()
@@ -440,7 +440,7 @@ func (do *Domain) handleRemoveStaleRunawayWatch(record *resourcegroup.Quarantine
return err
}
-func execRestrictedSQL(sessPool *sessionPool, sql string, params []any) ([]chunk.Row, error) {
+func execRestrictedSQL(sessPool util.SessionPool, sql string, params []any) ([]chunk.Row, error) {
se, err := sessPool.Get()
defer func() {
sessPool.Put(se)
@@ -484,11 +484,11 @@ func (do *Domain) initResourceGroupsController(ctx context.Context, pdClient pd.
type runawaySyncer struct {
newWatchReader *SystemTableReader
deletionWatchReader *SystemTableReader
- sysSessionPool *sessionPool
+ sysSessionPool util.SessionPool
mu sync.Mutex
}
-func newRunawaySyncer(sysSessionPool *sessionPool) *runawaySyncer {
+func newRunawaySyncer(sysSessionPool util.SessionPool) *runawaySyncer {
return &runawaySyncer{
sysSessionPool: sysSessionPool,
newWatchReader: &SystemTableReader{
diff --git a/pkg/domain/schema_validator.go b/pkg/domain/schema_validator.go
index 726ce050e9c0b..d33b548af6f0f 100644
--- a/pkg/domain/schema_validator.go
+++ b/pkg/domain/schema_validator.go
@@ -22,6 +22,7 @@ import (
"github.com/pingcap/tidb/pkg/metrics"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
+ "github.com/pingcap/tidb/pkg/util/intest"
"github.com/pingcap/tidb/pkg/util/logutil"
"github.com/tikv/client-go/v2/oracle"
"github.com/tikv/client-go/v2/txnkv/transaction"
@@ -78,6 +79,7 @@ type schemaValidator struct {
// NewSchemaValidator returns a SchemaValidator structure.
func NewSchemaValidator(lease time.Duration, do *Domain) SchemaValidator {
+ intest.Assert(lease > 0, "lease should be greater than 0")
return &schemaValidator{
isStarted: true,
lease: lease,
@@ -237,9 +239,6 @@ func (s *schemaValidator) Check(txnTS uint64, schemaVer int64, relatedPhysicalTa
zap.Int64("schemaVer", schemaVer))
return nil, ResultFail
}
- if s.lease == 0 {
- return nil, ResultSucc
- }
// Schema changed, result decided by whether related tables change.
if schemaVer < s.latestSchemaVer {
diff --git a/pkg/domain/sysvar_cache.go b/pkg/domain/sysvar_cache.go
index 0adce1e4bb926..6294aa8260080 100644
--- a/pkg/domain/sysvar_cache.go
+++ b/pkg/domain/sysvar_cache.go
@@ -64,9 +64,7 @@ func (do *Domain) GetSessionCache() (map[string]string, error) {
do.sysVarCache.RLock()
defer do.sysVarCache.RUnlock()
// Perform a deep copy since this will be assigned directly to the session
- newMap := make(map[string]string, len(do.sysVarCache.session))
- maps.Copy(newMap, do.sysVarCache.session)
- return newMap, nil
+ return maps.Clone(do.sysVarCache.session), nil
}
// GetGlobalVar gets an individual global var from the sysvar cache.
@@ -107,12 +105,11 @@ func (do *Domain) rebuildSysVarCache(ctx sessionctx.Context) error {
newSessionCache := make(map[string]string)
newGlobalCache := make(map[string]string)
if ctx == nil {
- sysSessionPool := do.SysSessionPool()
- res, err := sysSessionPool.Get()
+ res, err := do.sysSessionPool.Get()
if err != nil {
return err
}
- defer sysSessionPool.Put(res)
+ defer do.sysSessionPool.Put(res)
ctx = res.(sessionctx.Context)
}
// Only one rebuild can be in progress at a time, this prevents a lost update race
diff --git a/pkg/errno/errcode.go b/pkg/errno/errcode.go
index d6581b10fe3cf..5d56f339273ff 100644
--- a/pkg/errno/errcode.go
+++ b/pkg/errno/errcode.go
@@ -1140,6 +1140,8 @@ const (
ErrPausedDDLJob = 8262
ErrBDRRestrictedDDL = 8263
+ ErrGlobalIndexNotExplicitlySet = 8264
+
// Resource group errors.
ErrResourceGroupExists = 8248
ErrResourceGroupNotExists = 8249
diff --git a/pkg/errno/errname.go b/pkg/errno/errname.go
index 4c6cb61004b00..aadcbb3dfcbca 100644
--- a/pkg/errno/errname.go
+++ b/pkg/errno/errname.go
@@ -1171,4 +1171,6 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{
ErrCannotResumeDDLJob: mysql.Message("Job [%v] can't be resumed: %s", nil),
ErrPausedDDLJob: mysql.Message("Job [%v] has already been paused", nil),
ErrBDRRestrictedDDL: mysql.Message("The operation is not allowed while the bdr role of this cluster is set to %s.", nil),
+
+ ErrGlobalIndexNotExplicitlySet: mysql.Message("Global Index is needed for index '%-.192s', since the unique index is not including all partitioning columns, and GLOBAL is not given as IndexOption", nil),
}
diff --git a/pkg/executor/BUILD.bazel b/pkg/executor/BUILD.bazel
index b6a11412a5d39..886c06820877b 100644
--- a/pkg/executor/BUILD.bazel
+++ b/pkg/executor/BUILD.bazel
@@ -154,6 +154,7 @@ go_library(
"//pkg/planner/context",
"//pkg/planner/core",
"//pkg/planner/core/base",
+ "//pkg/planner/core/operator/logicalop",
"//pkg/planner/util",
"//pkg/planner/util/coreusage",
"//pkg/planner/util/fixcontrol",
@@ -410,6 +411,7 @@ go_test(
"//pkg/planner",
"//pkg/planner/core",
"//pkg/planner/core/base",
+ "//pkg/planner/core/operator/logicalop",
"//pkg/planner/property",
"//pkg/planner/util",
"//pkg/planner/util/coretestsdk",
@@ -474,6 +476,7 @@ go_test(
"@com_github_pingcap_kvproto//pkg/brpb",
"@com_github_pingcap_kvproto//pkg/diagnosticspb",
"@com_github_pingcap_kvproto//pkg/encryptionpb",
+ "@com_github_pingcap_kvproto//pkg/errorpb",
"@com_github_pingcap_kvproto//pkg/kvrpcpb",
"@com_github_pingcap_kvproto//pkg/metapb",
"@com_github_pingcap_log//:log",
@@ -487,6 +490,7 @@ go_test(
"@com_github_tikv_client_go_v2//testutils",
"@com_github_tikv_client_go_v2//tikv",
"@com_github_tikv_client_go_v2//tikvrpc",
+ "@com_github_tikv_client_go_v2//tikvrpc/interceptor",
"@com_github_tikv_client_go_v2//util",
"@com_github_tikv_pd_client//http",
"@org_golang_google_grpc//:grpc",
diff --git a/pkg/executor/adapter.go b/pkg/executor/adapter.go
index 4b5329bf8487c..2a452e4c21604 100644
--- a/pkg/executor/adapter.go
+++ b/pkg/executor/adapter.go
@@ -49,6 +49,7 @@ import (
"github.com/pingcap/tidb/pkg/planner"
plannercore "github.com/pingcap/tidb/pkg/planner/core"
"github.com/pingcap/tidb/pkg/planner/core/base"
+ "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop"
"github.com/pingcap/tidb/pkg/plugin"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessionctx/sessionstates"
@@ -852,7 +853,7 @@ func isNoResultPlan(p base.Plan) bool {
// the Projection has two expressions and two columns in the schema, but we should
// not return the result of the two expressions.
switch raw := p.(type) {
- case *plannercore.LogicalProjection:
+ case *logicalop.LogicalProjection:
if raw.CalculateNoDelay {
return true
}
@@ -984,6 +985,8 @@ func (a *ExecStmt) handleNoDelayExecutor(ctx context.Context, e exec.Executor) (
// Check if "tidb_snapshot" is set for the write executors.
// In history read mode, we can not do write operations.
+ // TODO: it's better to use a.ReadOnly to check if the statement is a write statement
+ // instead of listing executor types here.
switch e.(type) {
case *DeleteExec, *InsertExec, *UpdateExec, *ReplaceExec, *LoadDataExec, *DDLExec, *ImportIntoExec:
snapshotTS := sctx.GetSessionVars().SnapshotTS
diff --git a/pkg/executor/admin.go b/pkg/executor/admin.go
index f38b8bb42eba7..0fc1baa912261 100644
--- a/pkg/executor/admin.go
+++ b/pkg/executor/admin.go
@@ -163,7 +163,7 @@ func (e *CheckIndexRangeExec) constructIndexScanPB() *tipb.Executor {
idxExec := &tipb.IndexScan{
TableId: e.table.ID,
IndexId: e.index.ID,
- Columns: util.ColumnsToProto(e.cols, e.table.PKIsHandle, true),
+ Columns: util.ColumnsToProto(e.cols, e.table.PKIsHandle, true, false),
}
return &tipb.Executor{Tp: tipb.ExecType_TypeIndexScan, IdxScan: idxExec}
}
@@ -228,7 +228,7 @@ func (e *RecoverIndexExec) Open(ctx context.Context) error {
}
func (e *RecoverIndexExec) constructTableScanPB(tblInfo *model.TableInfo, colInfos []*model.ColumnInfo) (*tipb.Executor, error) {
- tblScan := tables.BuildTableScanFromInfos(tblInfo, colInfos)
+ tblScan := tables.BuildTableScanFromInfos(tblInfo, colInfos, false)
tblScan.TableId = e.physicalID
err := tables.SetPBColumnsDefaultValue(e.Ctx().GetExprCtx(), tblScan.Columns, colInfos)
return &tipb.Executor{Tp: tipb.ExecType_TypeTableScan, TblScan: tblScan}, err
@@ -513,8 +513,6 @@ func (e *RecoverIndexExec) backfillIndexInTxn(ctx context.Context, txn kv.Transa
return result, err
}
- // Constrains is already checked.
- e.Ctx().GetSessionVars().StmtCtx.BatchCheck = true
for _, row := range rows {
if row.skip {
continue
@@ -526,7 +524,11 @@ func (e *RecoverIndexExec) backfillIndexInTxn(ctx context.Context, txn kv.Transa
return result, err
}
- _, err = e.index.Create(e.Ctx().GetTableCtx(), txn, row.idxVals, row.handle, row.rsData, table.WithIgnoreAssertion)
+ _, err = e.index.Create(e.Ctx().GetTableCtx(), txn, row.idxVals, row.handle, row.rsData,
+ table.WithIgnoreAssertion,
+ // Constrains have already been checked.
+ table.DupKeyCheckSkip,
+ )
if err != nil {
return result, err
}
@@ -878,7 +880,7 @@ func (e *CleanupIndexExec) constructIndexScanPB() *tipb.Executor {
idxExec := &tipb.IndexScan{
TableId: e.physicalID,
IndexId: e.index.Meta().ID,
- Columns: util.ColumnsToProto(e.columns, e.table.Meta().PKIsHandle, true),
+ Columns: util.ColumnsToProto(e.columns, e.table.Meta().PKIsHandle, true, false),
PrimaryColumnIds: tables.TryGetCommonPkColumnIds(e.table.Meta()),
}
return &tipb.Executor{Tp: tipb.ExecType_TypeIndexScan, IdxScan: idxExec}
diff --git a/pkg/executor/aggfuncs/aggfuncs.go b/pkg/executor/aggfuncs/aggfuncs.go
index 586666554b9e6..bf3bbfc8e56fd 100644
--- a/pkg/executor/aggfuncs/aggfuncs.go
+++ b/pkg/executor/aggfuncs/aggfuncs.go
@@ -73,6 +73,7 @@ var (
_ AggFunc = (*maxMin4Float64)(nil)
_ AggFunc = (*maxMin4Decimal)(nil)
_ AggFunc = (*maxMin4String)(nil)
+ _ AggFunc = (*maxMin4Time)(nil)
_ AggFunc = (*maxMin4Duration)(nil)
_ AggFunc = (*maxMin4JSON)(nil)
_ AggFunc = (*maxMin4VectorFloat32)(nil)
diff --git a/pkg/executor/aggfuncs/func_count.go b/pkg/executor/aggfuncs/func_count.go
index 3f111b9f2b187..3d5db1181e5ca 100644
--- a/pkg/executor/aggfuncs/func_count.go
+++ b/pkg/executor/aggfuncs/func_count.go
@@ -58,8 +58,8 @@ func (e *baseCount) DeserializePartialResult(src *chunk.Chunk) ([]PartialResult,
func (e *baseCount) deserializeForSpill(helper *deserializeHelper) (PartialResult, int64) {
pr, memDelta := e.AllocPartialResult()
- result := *(*partialResult4Count)(pr)
- success := helper.deserializePartialResult4Count(&result)
+ result := (*partialResult4Count)(pr)
+ success := helper.deserializePartialResult4Count(result)
if !success {
return nil, 0
}
diff --git a/pkg/executor/aggfuncs/func_max_min.go b/pkg/executor/aggfuncs/func_max_min.go
index 2eb975051545e..96375f0c8b544 100644
--- a/pkg/executor/aggfuncs/func_max_min.go
+++ b/pkg/executor/aggfuncs/func_max_min.go
@@ -1294,6 +1294,26 @@ func (e *maxMin4Time) MergePartialResult(_ AggFuncUpdateContext, src, dst Partia
return 0, nil
}
+func (e *maxMin4Time) SerializePartialResult(partialResult PartialResult, chk *chunk.Chunk, spillHelper *SerializeHelper) {
+ pr := (*partialResult4MaxMinTime)(partialResult)
+ resBuf := spillHelper.serializePartialResult4MaxMinTime(*pr)
+ chk.AppendBytes(e.ordinal, resBuf)
+}
+
+func (e *maxMin4Time) DeserializePartialResult(src *chunk.Chunk) ([]PartialResult, int64) {
+ return deserializePartialResultCommon(src, e.ordinal, e.deserializeForSpill)
+}
+
+func (e *maxMin4Time) deserializeForSpill(helper *deserializeHelper) (PartialResult, int64) {
+ pr, memDelta := e.AllocPartialResult()
+ result := (*partialResult4MaxMinTime)(pr)
+ success := helper.deserializePartialResult4MaxMinTime(result)
+ if !success {
+ return nil, 0
+ }
+ return pr, memDelta
+}
+
type maxMin4TimeSliding struct {
maxMin4Time
windowInfo
diff --git a/pkg/executor/aggregate/agg_hash_executor.go b/pkg/executor/aggregate/agg_hash_executor.go
index 9b05153015803..a24e4d0a394b4 100644
--- a/pkg/executor/aggregate/agg_hash_executor.go
+++ b/pkg/executor/aggregate/agg_hash_executor.go
@@ -397,10 +397,15 @@ func (e *HashAggExec) initForParallelExec(ctx sessionctx.Context) error {
for i := 0; i < baseRetTypeNum; i++ {
spillChunkFieldTypes[i] = types.NewFieldType(mysql.TypeVarString)
}
+
+ var err error
spillChunkFieldTypes[baseRetTypeNum] = types.NewFieldType(mysql.TypeString)
- e.spillHelper = newSpillHelper(e.memTracker, e.PartialAggFuncs, func() *chunk.Chunk {
+ e.spillHelper, err = newSpillHelper(e.memTracker, e.PartialAggFuncs, e.FinalAggFuncs, func() *chunk.Chunk {
return chunk.New(spillChunkFieldTypes, e.InitCap(), e.MaxChunkSize())
}, spillChunkFieldTypes)
+ if err != nil {
+ return err
+ }
if isTrackerEnabled && isParallelHashAggSpillEnabled {
if e.diskTracker != nil {
diff --git a/pkg/executor/aggregate/agg_spill.go b/pkg/executor/aggregate/agg_spill.go
index 187b4d2c8e690..c383fa3ce7c6e 100644
--- a/pkg/executor/aggregate/agg_spill.go
+++ b/pkg/executor/aggregate/agg_spill.go
@@ -18,6 +18,7 @@ import (
"sync"
"sync/atomic"
+ "github.com/pingcap/errors"
"github.com/pingcap/tidb/pkg/executor/aggfuncs"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/types"
@@ -66,6 +67,8 @@ type parallelHashAggSpillHelper struct {
// They only be used for restoring data that are spilled to disk in partial stage.
aggFuncsForRestoring []aggfuncs.AggFunc
+ finalWorkerAggFuncs []aggfuncs.AggFunc
+
getNewSpillChunkFunc func() *chunk.Chunk
spillChunkFieldTypes []*types.FieldType
}
@@ -73,8 +76,13 @@ type parallelHashAggSpillHelper struct {
func newSpillHelper(
tracker *memory.Tracker,
aggFuncsForRestoring []aggfuncs.AggFunc,
+ finalWorkerAggFuncs []aggfuncs.AggFunc,
getNewSpillChunkFunc func() *chunk.Chunk,
- spillChunkFieldTypes []*types.FieldType) *parallelHashAggSpillHelper {
+ spillChunkFieldTypes []*types.FieldType) (*parallelHashAggSpillHelper, error) {
+ if len(aggFuncsForRestoring) != len(finalWorkerAggFuncs) {
+ return nil, errors.NewNoStackError("len(aggFuncsForRestoring) != len(finalWorkerAggFuncs)")
+ }
+
mu := new(sync.Mutex)
helper := ¶llelHashAggSpillHelper{
lock: struct {
@@ -97,11 +105,12 @@ func newSpillHelper(
memTracker: tracker,
hasError: atomic.Bool{},
aggFuncsForRestoring: aggFuncsForRestoring,
+ finalWorkerAggFuncs: finalWorkerAggFuncs,
getNewSpillChunkFunc: getNewSpillChunkFunc,
spillChunkFieldTypes: spillChunkFieldTypes,
}
- return helper
+ return helper, nil
}
func (p *parallelHashAggSpillHelper) close() {
@@ -294,7 +303,7 @@ func (p *parallelHashAggSpillHelper) processRow(context *processRowContext) (tot
exprCtx := context.ctx.GetExprCtx()
// The key has appeared before, merge results.
for aggPos := 0; aggPos < context.aggFuncNum; aggPos++ {
- memDelta, err := p.aggFuncsForRestoring[aggPos].MergePartialResult(exprCtx.GetEvalCtx(), context.partialResultsRestored[aggPos][context.rowPos], prs[aggPos])
+ memDelta, err := p.finalWorkerAggFuncs[aggPos].MergePartialResult(exprCtx.GetEvalCtx(), context.partialResultsRestored[aggPos][context.rowPos], prs[aggPos])
if err != nil {
return totalMemDelta, 0, err
}
diff --git a/pkg/executor/aggregate/agg_spill_test.go b/pkg/executor/aggregate/agg_spill_test.go
index 5d0f16c67df3c..9504a7fa89304 100644
--- a/pkg/executor/aggregate/agg_spill_test.go
+++ b/pkg/executor/aggregate/agg_spill_test.go
@@ -18,7 +18,7 @@ import (
"context"
"fmt"
"math/rand"
- "strconv"
+ "sort"
"sync"
"sync/atomic"
"testing"
@@ -46,49 +46,6 @@ import (
const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
-type resultsContainer struct {
- rows []chunk.Row
-}
-
-func (r *resultsContainer) add(chk *chunk.Chunk) {
- iter := chunk.NewIterator4Chunk(chk.CopyConstruct())
- for row := iter.Begin(); row != iter.End(); row = iter.Next() {
- r.rows = append(r.rows, row)
- }
-}
-
-func (r *resultsContainer) check(expectRes map[string]float64) bool {
- if len(r.rows) != len(expectRes) {
- return false
- }
-
- cols := getColumns()
- retFields := []*types.FieldType{cols[0].RetType, cols[1].RetType}
- for _, row := range r.rows {
- key := ""
- for i, field := range retFields {
- d := row.GetDatum(i, field)
- resStr, err := d.ToString()
- if err != nil {
- panic("Fail to convert to string")
- }
-
- if i == 0 {
- key = resStr
- } else {
- expectVal, ok := expectRes[key]
- if !ok {
- return false
- }
- if resStr != strconv.Itoa(int(expectVal)) {
- return false
- }
- }
- }
- }
- return true
-}
-
func getRandString() string {
b := make([]byte, 5)
for i := range b {
@@ -108,9 +65,9 @@ func generateData(rowNum int, ndv int) ([]string, []float64) {
// Generate data
for i := 0; i < rowNum; i++ {
- key := keys[i%ndv]
+ key := keys[rand.Intn(ndv)]
col0Data = append(col0Data, key)
- col1Data = append(col1Data, 1) // Always 1
+ col1Data = append(col1Data, float64(rand.Intn(10000000)))
}
// Shuffle data
@@ -146,25 +103,78 @@ func buildMockDataSource(opt testutil.MockDataSourceParameters, col0Data []strin
return mockDatasource
}
-func generateResult(col0 []string) map[string]float64 {
- result := make(map[string]float64, 0)
- length := len(col0)
+func generateCMPFunc(fieldTypes []*types.FieldType) func(chunk.Row, chunk.Row) int {
+ cmpFuncs := make([]chunk.CompareFunc, 0, len(fieldTypes))
+ for _, colType := range fieldTypes {
+ cmpFuncs = append(cmpFuncs, chunk.GetCompareFunc(colType))
+ }
+
+ cmp := func(rowI, rowJ chunk.Row) int {
+ for i, cmpFunc := range cmpFuncs {
+ cmp := cmpFunc(rowI, i, rowJ, i)
+ if cmp != 0 {
+ return cmp
+ }
+ }
+ return 0
+ }
+
+ return cmp
+}
+
+func sortRows(rows []chunk.Row, fieldTypes []*types.FieldType) []chunk.Row {
+ cmp := generateCMPFunc(fieldTypes)
+
+ sort.Slice(rows, func(i, j int) bool {
+ return cmp(rows[i], rows[j]) < 0
+ })
+ return rows
+}
+
+func generateResult(t *testing.T, ctx *mock.Context, dataSource *testutil.MockDataSource) []chunk.Row {
+ aggExec := buildHashAggExecutor(t, ctx, dataSource)
+ dataSource.PrepareChunks()
+ tmpCtx := context.Background()
+ resultRows := make([]chunk.Row, 0)
+ aggExec.Open(tmpCtx)
+ for {
+ chk := exec.NewFirstChunk(aggExec)
+ err := aggExec.Next(tmpCtx, chk)
+ require.Equal(t, nil, err)
+ if chk.NumRows() == 0 {
+ break
+ }
- for i := 0; i < length; i++ {
- _, ok := result[col0[i]]
- if ok {
- result[col0[i]]++
- } else {
- result[col0[i]] = 1
+ rowNum := chk.NumRows()
+ for i := 0; i < rowNum; i++ {
+ resultRows = append(resultRows, chk.GetRow(i))
}
}
- return result
+ aggExec.Close()
+
+ require.False(t, aggExec.IsSpillTriggeredForTest())
+ return sortRows(resultRows, getRetTypes())
+}
+
+func getRetTypes() []*types.FieldType {
+ return []*types.FieldType{
+ types.NewFieldType(mysql.TypeVarString),
+ types.NewFieldType(mysql.TypeDouble),
+ types.NewFieldType(mysql.TypeLonglong),
+ types.NewFieldType(mysql.TypeDouble),
+ types.NewFieldType(mysql.TypeDouble),
+ types.NewFieldType(mysql.TypeDouble),
+ }
}
func getColumns() []*expression.Column {
return []*expression.Column{
{Index: 0, RetType: types.NewFieldType(mysql.TypeVarString)},
{Index: 1, RetType: types.NewFieldType(mysql.TypeDouble)},
+ {Index: 1, RetType: types.NewFieldType(mysql.TypeDouble)},
+ {Index: 1, RetType: types.NewFieldType(mysql.TypeDouble)},
+ {Index: 1, RetType: types.NewFieldType(mysql.TypeDouble)},
+ {Index: 1, RetType: types.NewFieldType(mysql.TypeDouble)},
}
}
@@ -191,22 +201,50 @@ func buildHashAggExecutor(t *testing.T, ctx sessionctx.Context, child exec.Execu
schema := expression.NewSchema(childCols...)
groupItems := []expression.Expression{childCols[0]}
- aggFirstRow, err := aggregation.NewAggFuncDesc(ctx.GetExprCtx(), ast.AggFuncFirstRow, []expression.Expression{childCols[0]}, false)
+ var err error
+ var aggFirstRow *aggregation.AggFuncDesc
+ var aggSum *aggregation.AggFuncDesc
+ var aggCount *aggregation.AggFuncDesc
+ var aggAvg *aggregation.AggFuncDesc
+ var aggMin *aggregation.AggFuncDesc
+ var aggMax *aggregation.AggFuncDesc
+ aggFirstRow, err = aggregation.NewAggFuncDesc(ctx.GetExprCtx(), ast.AggFuncFirstRow, []expression.Expression{childCols[0]}, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ aggSum, err = aggregation.NewAggFuncDesc(ctx.GetExprCtx(), ast.AggFuncSum, []expression.Expression{childCols[1]}, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ aggCount, err = aggregation.NewAggFuncDesc(ctx.GetExprCtx(), ast.AggFuncCount, []expression.Expression{childCols[1]}, false)
if err != nil {
t.Fatal(err)
}
- aggFunc, err := aggregation.NewAggFuncDesc(ctx.GetExprCtx(), ast.AggFuncSum, []expression.Expression{childCols[1]}, false)
+ aggAvg, err = aggregation.NewAggFuncDesc(ctx.GetExprCtx(), ast.AggFuncAvg, []expression.Expression{childCols[1]}, false)
if err != nil {
t.Fatal(err)
}
- aggFuncs := []*aggregation.AggFuncDesc{aggFirstRow, aggFunc}
+ aggMin, err = aggregation.NewAggFuncDesc(ctx.GetExprCtx(), ast.AggFuncMin, []expression.Expression{childCols[1]}, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ aggMax, err = aggregation.NewAggFuncDesc(ctx.GetExprCtx(), ast.AggFuncMax, []expression.Expression{childCols[1]}, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ aggFuncs := []*aggregation.AggFuncDesc{aggFirstRow, aggSum, aggCount, aggAvg, aggMin, aggMax}
aggExec := &aggregate.HashAggExec{
BaseExecutor: exec.NewBaseExecutor(ctx, schema, 0, child),
Sc: ctx.GetSessionVars().StmtCtx,
PartialAggFuncs: make([]aggfuncs.AggFunc, 0, len(aggFuncs)),
+ FinalAggFuncs: make([]aggfuncs.AggFunc, 0, len(aggFuncs)),
GroupByItems: groupItems,
IsUnparallelExec: false,
}
@@ -215,6 +253,10 @@ func buildHashAggExecutor(t *testing.T, ctx sessionctx.Context, child exec.Execu
for i, aggDesc := range aggFuncs {
ordinal := []int{partialOrdinal}
partialOrdinal++
+ if aggDesc.Name == ast.AggFuncAvg {
+ ordinal = append(ordinal, partialOrdinal+1)
+ partialOrdinal++
+ }
partialAggDesc, finalDesc := aggDesc.Split(ordinal)
partialAggFunc := aggfuncs.Build(ctx.GetExprCtx(), partialAggDesc, i)
finalAggFunc := aggfuncs.Build(ctx.GetExprCtx(), finalDesc, i)
@@ -237,30 +279,48 @@ func initCtx(ctx *mock.Context, newRootExceedAction *testutil.MockActionOnExceed
ctx.GetSessionVars().MemTracker.SetActionOnExceed(newRootExceedAction)
}
-// select t0, sum(t1) from t group by t0;
-func executeCorrecResultTest(t *testing.T, ctx *mock.Context, aggExec *aggregate.HashAggExec, dataSource *testutil.MockDataSource, result map[string]float64) {
+func checkResult(expectResult []chunk.Row, actualResult []chunk.Row, retTypes []*types.FieldType) bool {
+ if len(expectResult) != len(actualResult) {
+ return false
+ }
+
+ rowNum := len(expectResult)
+ for i := 0; i < rowNum; i++ {
+ if expectResult[i].ToString(retTypes) != actualResult[i].ToString(retTypes) {
+ return false
+ }
+ }
+
+ return true
+}
+
+func executeCorrecResultTest(t *testing.T, ctx *mock.Context, aggExec *aggregate.HashAggExec, dataSource *testutil.MockDataSource, expectResult []chunk.Row) {
if aggExec == nil {
aggExec = buildHashAggExecutor(t, ctx, dataSource)
}
dataSource.PrepareChunks()
tmpCtx := context.Background()
- chk := exec.NewFirstChunk(aggExec)
- resContainer := resultsContainer{}
+ resultRows := make([]chunk.Row, 0)
aggExec.Open(tmpCtx)
-
for {
+ chk := exec.NewFirstChunk(aggExec)
err := aggExec.Next(tmpCtx, chk)
require.Equal(t, nil, err)
if chk.NumRows() == 0 {
break
}
- resContainer.add(chk)
- chk.Reset()
+
+ rowNum := chk.NumRows()
+ for i := 0; i < rowNum; i++ {
+ resultRows = append(resultRows, chk.GetRow(i))
+ }
}
aggExec.Close()
require.True(t, aggExec.IsSpillTriggeredForTest())
- require.True(t, resContainer.check(result))
+ retTypes := getRetTypes()
+ resultRows = sortRows(resultRows, retTypes)
+ require.True(t, checkResult(expectResult, resultRows, retTypes))
}
func fallBackActionTest(t *testing.T) {
@@ -283,15 +343,12 @@ func fallBackActionTest(t *testing.T) {
dataSource.PrepareChunks()
tmpCtx := context.Background()
chk := exec.NewFirstChunk(aggExec)
- resContainer := resultsContainer{}
aggExec.Open(tmpCtx)
-
for {
aggExec.Next(tmpCtx, chk)
if chk.NumRows() == 0 {
break
}
- resContainer.add(chk)
chk.Reset()
}
aggExec.Close()
@@ -340,21 +397,26 @@ func randomFailTest(t *testing.T, ctx *mock.Context, aggExec *aggregate.HashAggE
})
}
+// sql: select col0, sum(col1), count(col1), avg(col1), min(col1), max(col1) from t group by t.col0;
func TestGetCorrectResult(t *testing.T) {
newRootExceedAction := new(testutil.MockActionOnExceed)
- hardLimitBytesNum := int64(6000000)
ctx := mock.NewContext()
- initCtx(ctx, newRootExceedAction, hardLimitBytesNum, 256)
+ initCtx(ctx, newRootExceedAction, -1, 1024)
rowNum := 100000
ndv := 50000
- col1, col2 := generateData(rowNum, ndv)
- result := generateResult(col1)
+ col0, col1 := generateData(rowNum, ndv)
opt := getMockDataSourceParameters(ctx)
- dataSource := buildMockDataSource(opt, col1, col2)
+ dataSource := buildMockDataSource(opt, col0, col1)
+ result := generateResult(t, ctx, dataSource)
+
+ err := failpoint.Enable("github.com/pingcap/tidb/pkg/executor/aggregate/slowSomePartialWorkers", `return(true)`)
+ require.NoError(t, err)
+ defer require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/executor/aggregate/slowSomePartialWorkers"))
- failpoint.Enable("github.com/pingcap/tidb/pkg/executor/aggregate/slowSomePartialWorkers", `return(true)`)
+ hardLimitBytesNum := int64(6000000)
+ initCtx(ctx, newRootExceedAction, hardLimitBytesNum, 256)
finished := atomic.Bool{}
wg := sync.WaitGroup{}
@@ -380,7 +442,6 @@ func TestGetCorrectResult(t *testing.T) {
finished.Store(true)
wg.Wait()
- require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/executor/aggregate/slowSomePartialWorkers"))
}
func TestFallBackAction(t *testing.T) {
diff --git a/pkg/executor/analyze.go b/pkg/executor/analyze.go
index 14d4296d370ba..5fd61796dd07c 100644
--- a/pkg/executor/analyze.go
+++ b/pkg/executor/analyze.go
@@ -22,7 +22,6 @@ import (
"net"
"strconv"
"strings"
- "time"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
@@ -41,12 +40,10 @@ import (
"github.com/pingcap/tidb/pkg/statistics/handle"
statstypes "github.com/pingcap/tidb/pkg/statistics/handle/types"
handleutil "github.com/pingcap/tidb/pkg/statistics/handle/util"
- "github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/chunk"
"github.com/pingcap/tidb/pkg/util/logutil"
"github.com/pingcap/tidb/pkg/util/sqlescape"
- "github.com/pingcap/tidb/pkg/util/sqlexec"
"github.com/pingcap/tipb/go-tipb"
"github.com/tiancaiamao/gp"
"go.uber.org/zap"
@@ -103,16 +100,16 @@ func (e *AnalyzeExec) Next(ctx context.Context, _ *chunk.Chunk) error {
}
// Get the min number of goroutines for parallel execution.
- concurrency, err := getBuildStatsConcurrency(e.Ctx())
+ buildStatsConcurrency, err := getBuildStatsConcurrency(e.Ctx())
if err != nil {
return err
}
- concurrency = min(len(tasks), concurrency)
+ buildStatsConcurrency = min(len(tasks), buildStatsConcurrency)
// Start workers with channel to collect results.
- taskCh := make(chan *analyzeTask, concurrency)
+ taskCh := make(chan *analyzeTask, buildStatsConcurrency)
resultsCh := make(chan *statistics.AnalyzeResults, 1)
- for i := 0; i < concurrency; i++ {
+ for i := 0; i < buildStatsConcurrency; i++ {
e.wg.Run(func() { e.analyzeWorker(taskCh, resultsCh) })
}
pruneMode := variable.PartitionPruneMode(sessionVars.PartitionPruneMode.Load())
@@ -121,7 +118,7 @@ func (e *AnalyzeExec) Next(ctx context.Context, _ *chunk.Chunk) error {
globalStatsMap := make(map[globalStatsKey]statstypes.GlobalStatsInfo)
g, gctx := errgroup.WithContext(ctx)
g.Go(func() error {
- return e.handleResultsError(ctx, concurrency, needGlobalStats, globalStatsMap, resultsCh, len(tasks))
+ return e.handleResultsError(buildStatsConcurrency, needGlobalStats, globalStatsMap, resultsCh, len(tasks))
})
for _, task := range tasks {
prepareV2AnalyzeJobInfo(task.colExec)
@@ -129,7 +126,9 @@ func (e *AnalyzeExec) Next(ctx context.Context, _ *chunk.Chunk) error {
}
failpoint.Inject("mockKillPendingAnalyzeJob", func() {
dom := domain.GetDomain(e.Ctx())
- dom.SysProcTracker().KillSysProcess(dom.GetAutoAnalyzeProcID())
+ for _, id := range handleutil.GlobalAutoAnalyzeProcessList.All() {
+ dom.SysProcTracker().KillSysProcess(id)
+ }
})
TASKLOOP:
for _, task := range tasks {
@@ -157,11 +156,13 @@ TASKLOOP:
failpoint.Inject("mockKillFinishedAnalyzeJob", func() {
dom := domain.GetDomain(e.Ctx())
- dom.SysProcTracker().KillSysProcess(dom.GetAutoAnalyzeProcID())
+ for _, id := range handleutil.GlobalAutoAnalyzeProcessList.All() {
+ dom.SysProcTracker().KillSysProcess(id)
+ }
})
// If we enabled dynamic prune mode, then we need to generate global stats here for partition tables.
if needGlobalStats {
- err = e.handleGlobalStats(globalStatsMap)
+ err = e.handleGlobalStats(statsHandle, globalStatsMap)
if err != nil {
return err
}
@@ -242,7 +243,7 @@ func filterAndCollectTasks(tasks []*analyzeTask, statsHandle *handle.Handle, is
skippedTables = append(skippedTables, fmt.Sprintf("%s.%s partition (%s)", schema.Name, tbl.Meta().Name.O, def.Name.O))
}
} else {
- tbl, ok := is.TableByID(physicalTableID)
+ tbl, ok := is.TableByID(context.Background(), physicalTableID)
if !ok {
logutil.BgLogger().Warn("Unknown table ID in analyze task", zap.Int64("tid", physicalTableID))
} else {
@@ -367,8 +368,7 @@ func recordHistoricalStats(sctx sessionctx.Context, tableID int64) error {
// handleResultsError will handle the error fetch from resultsCh and record it in log
func (e *AnalyzeExec) handleResultsError(
- ctx context.Context,
- concurrency int,
+ buildStatsConcurrency int,
needGlobalStats bool,
globalStatsMap globalStatsMap,
resultsCh <-chan *statistics.AnalyzeResults,
@@ -384,91 +384,47 @@ func (e *AnalyzeExec) handleResultsError(
}
}
}()
- partitionStatsConcurrency := e.Ctx().GetSessionVars().AnalyzePartitionConcurrency
- // the concurrency of handleResultsError cannot be more than partitionStatsConcurrency
- partitionStatsConcurrency = min(taskNum, partitionStatsConcurrency)
- // If partitionStatsConcurrency > 1, we will try to demand extra session from Domain to save Analyze results in concurrency.
- // If there is no extra session we can use, we will save analyze results in single-thread.
- if partitionStatsConcurrency > 1 {
- dom := domain.GetDomain(e.Ctx())
- subSctxs := dom.FetchAnalyzeExec(partitionStatsConcurrency)
- if len(subSctxs) > 0 {
- defer func() {
- dom.ReleaseAnalyzeExec(subSctxs)
- }()
- internalCtx := kv.WithInternalSourceType(ctx, kv.InternalTxnStats)
- err := e.handleResultsErrorWithConcurrency(internalCtx, concurrency, needGlobalStats, subSctxs, globalStatsMap, resultsCh)
- return err
- }
- }
+ saveStatsConcurrency := e.Ctx().GetSessionVars().AnalyzePartitionConcurrency
+ // The buildStatsConcurrency of saving partition-level stats should not exceed the total number of tasks.
+ saveStatsConcurrency = min(taskNum, saveStatsConcurrency)
+ if saveStatsConcurrency > 1 {
+ logutil.BgLogger().Info("save analyze results concurrently",
+ zap.Int("buildStatsConcurrency", buildStatsConcurrency),
+ zap.Int("saveStatsConcurrency", saveStatsConcurrency),
+ )
+ return e.handleResultsErrorWithConcurrency(buildStatsConcurrency, saveStatsConcurrency, needGlobalStats, globalStatsMap, resultsCh)
+ }
+ logutil.BgLogger().Info("save analyze results in single-thread",
+ zap.Int("buildStatsConcurrency", buildStatsConcurrency),
+ zap.Int("saveStatsConcurrency", saveStatsConcurrency),
+ )
failpoint.Inject("handleResultsErrorSingleThreadPanic", nil)
- tableIDs := map[int64]struct{}{}
-
- // save analyze results in single-thread.
- statsHandle := domain.GetDomain(e.Ctx()).StatsHandle()
- panicCnt := 0
- for panicCnt < concurrency {
- results, ok := <-resultsCh
- if !ok {
- break
- }
- if results.Err != nil {
- err = results.Err
- if isAnalyzeWorkerPanic(err) {
- panicCnt++
- } else {
- logutil.Logger(ctx).Error("analyze failed", zap.Error(err))
- }
- finishJobWithLog(e.Ctx(), results.Job, err)
- continue
- }
- handleGlobalStats(needGlobalStats, globalStatsMap, results)
- tableIDs[results.TableID.GetStatisticsID()] = struct{}{}
-
- if err1 := statsHandle.SaveTableStatsToStorage(results, e.Ctx().GetSessionVars().EnableAnalyzeSnapshot, handleutil.StatsMetaHistorySourceAnalyze); err1 != nil {
- tableID := results.TableID.TableID
- err = err1
- logutil.Logger(ctx).Error("save table stats to storage failed", zap.Error(err), zap.Int64("tableID", tableID))
- finishJobWithLog(e.Ctx(), results.Job, err)
- } else {
- finishJobWithLog(e.Ctx(), results.Job, nil)
- }
- if err := e.Ctx().GetSessionVars().SQLKiller.HandleSignal(); err != nil {
- finishJobWithLog(e.Ctx(), results.Job, err)
- results.DestroyAndPutToPool()
- return err
- }
- results.DestroyAndPutToPool()
- }
- // Dump stats to historical storage.
- for tableID := range tableIDs {
- if err := recordHistoricalStats(e.Ctx(), tableID); err != nil {
- logutil.BgLogger().Error("record historical stats failed", zap.Error(err))
- }
- }
-
- return err
+ return e.handleResultsErrorWithConcurrency(buildStatsConcurrency, saveStatsConcurrency, needGlobalStats, globalStatsMap, resultsCh)
}
-func (e *AnalyzeExec) handleResultsErrorWithConcurrency(ctx context.Context, statsConcurrency int, needGlobalStats bool,
- subSctxs []sessionctx.Context,
- globalStatsMap globalStatsMap, resultsCh <-chan *statistics.AnalyzeResults) error {
- partitionStatsConcurrency := len(subSctxs)
-
+func (e *AnalyzeExec) handleResultsErrorWithConcurrency(
+ buildStatsConcurrency int,
+ saveStatsConcurrency int,
+ needGlobalStats bool,
+ globalStatsMap globalStatsMap,
+ resultsCh <-chan *statistics.AnalyzeResults,
+) error {
+ statsHandle := domain.GetDomain(e.Ctx()).StatsHandle()
wg := util.NewWaitGroupPool(e.gp)
- saveResultsCh := make(chan *statistics.AnalyzeResults, partitionStatsConcurrency)
- errCh := make(chan error, partitionStatsConcurrency)
- for i := 0; i < partitionStatsConcurrency; i++ {
- worker := newAnalyzeSaveStatsWorker(saveResultsCh, subSctxs[i], errCh, &e.Ctx().GetSessionVars().SQLKiller)
+ saveResultsCh := make(chan *statistics.AnalyzeResults, saveStatsConcurrency)
+ errCh := make(chan error, saveStatsConcurrency)
+ for i := 0; i < saveStatsConcurrency; i++ {
+ worker := newAnalyzeSaveStatsWorker(saveResultsCh, errCh, &e.Ctx().GetSessionVars().SQLKiller)
ctx1 := kv.WithInternalSourceType(context.Background(), kv.InternalTxnStats)
wg.Run(func() {
- worker.run(ctx1, e.Ctx().GetSessionVars().EnableAnalyzeSnapshot)
+ worker.run(ctx1, statsHandle, e.Ctx().GetSessionVars().EnableAnalyzeSnapshot)
})
}
tableIDs := map[int64]struct{}{}
panicCnt := 0
var err error
- for panicCnt < statsConcurrency {
+ // Only if all the analyze workers exit can we close the saveResultsCh.
+ for panicCnt < buildStatsConcurrency {
if err := e.Ctx().GetSessionVars().SQLKiller.HandleSignal(); err != nil {
close(saveResultsCh)
return err
@@ -482,9 +438,9 @@ func (e *AnalyzeExec) handleResultsErrorWithConcurrency(ctx context.Context, sta
if isAnalyzeWorkerPanic(err) {
panicCnt++
} else {
- logutil.Logger(ctx).Error("analyze failed", zap.Error(err))
+ logutil.BgLogger().Error("receive error when saving analyze results", zap.Error(err))
}
- finishJobWithLog(e.Ctx(), results.Job, err)
+ finishJobWithLog(statsHandle, results.Job, err)
continue
}
handleGlobalStats(needGlobalStats, globalStatsMap, results)
@@ -512,6 +468,7 @@ func (e *AnalyzeExec) handleResultsErrorWithConcurrency(ctx context.Context, sta
func (e *AnalyzeExec) analyzeWorker(taskCh <-chan *analyzeTask, resultsCh chan<- *statistics.AnalyzeResults) {
var task *analyzeTask
+ statsHandle := domain.GetDomain(e.Ctx()).StatsHandle()
defer func() {
if r := recover(); r != nil {
logutil.BgLogger().Error("analyze worker panicked", zap.Any("recover", r), zap.Stack("stack"))
@@ -535,7 +492,7 @@ func (e *AnalyzeExec) analyzeWorker(taskCh <-chan *analyzeTask, resultsCh chan<-
break
}
failpoint.Inject("handleAnalyzeWorkerPanic", nil)
- StartAnalyzeJob(e.Ctx(), task.job)
+ statsHandle.StartAnalyzeJob(task.job)
switch task.taskType {
case colTask:
select {
@@ -590,147 +547,8 @@ func AddNewAnalyzeJob(ctx sessionctx.Context, job *statistics.AnalyzeJob) {
}
}
-// StartAnalyzeJob marks the state of the analyze job as running and sets the start time.
-func StartAnalyzeJob(sctx sessionctx.Context, job *statistics.AnalyzeJob) {
- if job == nil || job.ID == nil {
- return
- }
- job.StartTime = time.Now()
- job.Progress.SetLastDumpTime(job.StartTime)
- exec := sctx.GetRestrictedSQLExecutor()
- ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnStats)
- const sql = "UPDATE mysql.analyze_jobs SET start_time = CONVERT_TZ(%?, '+00:00', @@TIME_ZONE), state = %? WHERE id = %?"
- _, _, err := exec.ExecRestrictedSQL(ctx, []sqlexec.OptionFuncAlias{sqlexec.ExecOptionUseSessionPool}, sql, job.StartTime.UTC().Format(types.TimeFormat), statistics.AnalyzeRunning, *job.ID)
- if err != nil {
- logutil.BgLogger().Warn("failed to update analyze job", zap.String("update", fmt.Sprintf("%s->%s", statistics.AnalyzePending, statistics.AnalyzeRunning)), zap.Error(err))
- }
- failpoint.Inject("DebugAnalyzeJobOperations", func(val failpoint.Value) {
- if val.(bool) {
- logutil.BgLogger().Info("StartAnalyzeJob",
- zap.Time("start_time", job.StartTime),
- zap.Uint64("job id", *job.ID),
- )
- }
- })
-}
-
-// UpdateAnalyzeJob updates count of the processed rows when increment reaches a threshold.
-func UpdateAnalyzeJob(sctx sessionctx.Context, job *statistics.AnalyzeJob, rowCount int64) {
- if job == nil || job.ID == nil {
- return
- }
- delta := job.Progress.Update(rowCount)
- if delta == 0 {
- return
- }
- exec := sctx.GetRestrictedSQLExecutor()
- ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnStats)
- const sql = "UPDATE mysql.analyze_jobs SET processed_rows = processed_rows + %? WHERE id = %?"
- _, _, err := exec.ExecRestrictedSQL(ctx, []sqlexec.OptionFuncAlias{sqlexec.ExecOptionUseSessionPool}, sql, delta, *job.ID)
- if err != nil {
- logutil.BgLogger().Warn("failed to update analyze job", zap.String("update", fmt.Sprintf("process %v rows", delta)), zap.Error(err))
- }
- failpoint.Inject("DebugAnalyzeJobOperations", func(val failpoint.Value) {
- if val.(bool) {
- logutil.BgLogger().Info("UpdateAnalyzeJob",
- zap.Int64("increase processed_rows", delta),
- zap.Uint64("job id", *job.ID),
- )
- }
- })
-}
-
-// FinishAnalyzeMergeJob finishes analyze merge job
-func FinishAnalyzeMergeJob(sctx sessionctx.Context, job *statistics.AnalyzeJob, analyzeErr error) {
- if job == nil || job.ID == nil {
- return
- }
-
- job.EndTime = time.Now()
- var sql string
- var args []any
- if analyzeErr != nil {
- failReason := analyzeErr.Error()
- const textMaxLength = 65535
- if len(failReason) > textMaxLength {
- failReason = failReason[:textMaxLength]
- }
- sql = "UPDATE mysql.analyze_jobs SET end_time = CONVERT_TZ(%?, '+00:00', @@TIME_ZONE), state = %?, fail_reason = %?, process_id = NULL WHERE id = %?"
- args = []any{job.EndTime.UTC().Format(types.TimeFormat), statistics.AnalyzeFailed, failReason, *job.ID}
- } else {
- sql = "UPDATE mysql.analyze_jobs SET end_time = CONVERT_TZ(%?, '+00:00', @@TIME_ZONE), state = %?, process_id = NULL WHERE id = %?"
- args = []any{job.EndTime.UTC().Format(types.TimeFormat), statistics.AnalyzeFinished, *job.ID}
- }
- exec := sctx.GetRestrictedSQLExecutor()
- ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnStats)
- _, _, err := exec.ExecRestrictedSQL(ctx, []sqlexec.OptionFuncAlias{sqlexec.ExecOptionUseSessionPool}, sql, args...)
- if err != nil {
- var state string
- if analyzeErr != nil {
- state = statistics.AnalyzeFailed
- } else {
- state = statistics.AnalyzeFinished
- }
- logutil.BgLogger().Warn("failed to update analyze job", zap.String("update", fmt.Sprintf("%s->%s", statistics.AnalyzeRunning, state)), zap.Error(err))
- }
- failpoint.Inject("DebugAnalyzeJobOperations", func(val failpoint.Value) {
- if val.(bool) {
- logutil.BgLogger().Info("FinishAnalyzeMergeJob",
- zap.Time("end_time", job.EndTime),
- zap.Uint64("job id", *job.ID),
- )
- }
- })
-}
-
-// FinishAnalyzeJob updates the state of the analyze job to finished/failed according to `meetError` and sets the end time.
-func FinishAnalyzeJob(sctx sessionctx.Context, job *statistics.AnalyzeJob, analyzeErr error) {
- if job == nil || job.ID == nil {
- return
- }
- job.EndTime = time.Now()
- var sql string
- var args []any
- // process_id is used to see which process is running the analyze job and kill the analyze job. After the analyze job
- // is finished(or failed), process_id is useless and we set it to NULL to avoid `kill tidb process_id` wrongly.
- if analyzeErr != nil {
- failReason := analyzeErr.Error()
- const textMaxLength = 65535
- if len(failReason) > textMaxLength {
- failReason = failReason[:textMaxLength]
- }
- sql = "UPDATE mysql.analyze_jobs SET processed_rows = processed_rows + %?, end_time = CONVERT_TZ(%?, '+00:00', @@TIME_ZONE), state = %?, fail_reason = %?, process_id = NULL WHERE id = %?"
- args = []any{job.Progress.GetDeltaCount(), job.EndTime.UTC().Format(types.TimeFormat), statistics.AnalyzeFailed, failReason, *job.ID}
- } else {
- sql = "UPDATE mysql.analyze_jobs SET processed_rows = processed_rows + %?, end_time = CONVERT_TZ(%?, '+00:00', @@TIME_ZONE), state = %?, process_id = NULL WHERE id = %?"
- args = []any{job.Progress.GetDeltaCount(), job.EndTime.UTC().Format(types.TimeFormat), statistics.AnalyzeFinished, *job.ID}
- }
- exec := sctx.GetRestrictedSQLExecutor()
- ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnStats)
- _, _, err := exec.ExecRestrictedSQL(ctx, []sqlexec.OptionFuncAlias{sqlexec.ExecOptionUseSessionPool}, sql, args...)
- if err != nil {
- var state string
- if analyzeErr != nil {
- state = statistics.AnalyzeFailed
- } else {
- state = statistics.AnalyzeFinished
- }
- logutil.BgLogger().Warn("failed to update analyze job", zap.String("update", fmt.Sprintf("%s->%s", statistics.AnalyzeRunning, state)), zap.Error(err))
- }
- failpoint.Inject("DebugAnalyzeJobOperations", func(val failpoint.Value) {
- if val.(bool) {
- logutil.BgLogger().Info("FinishAnalyzeJob",
- zap.Int64("increase processed_rows", job.Progress.GetDeltaCount()),
- zap.Time("end_time", job.EndTime),
- zap.Uint64("job id", *job.ID),
- zap.Error(analyzeErr),
- )
- }
- })
-}
-
-func finishJobWithLog(sctx sessionctx.Context, job *statistics.AnalyzeJob, analyzeErr error) {
- FinishAnalyzeJob(sctx, job, analyzeErr)
+func finishJobWithLog(statsHandle *handle.Handle, job *statistics.AnalyzeJob, analyzeErr error) {
+ statsHandle.FinishAnalyzeJob(job, analyzeErr, statistics.TableAnalysisJob)
if job != nil {
var state string
if analyzeErr != nil {
diff --git a/pkg/executor/analyze_col.go b/pkg/executor/analyze_col.go
index c1d1dc46770d2..78a5190b8d21b 100644
--- a/pkg/executor/analyze_col.go
+++ b/pkg/executor/analyze_col.go
@@ -32,6 +32,7 @@ import (
"github.com/pingcap/tidb/pkg/planner/core"
plannerutil "github.com/pingcap/tidb/pkg/planner/util"
"github.com/pingcap/tidb/pkg/statistics"
+ handleutil "github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util"
@@ -173,10 +174,13 @@ func (e *AnalyzeColumnsExec) buildStats(ranges []*ranger.Range, needExtStats boo
CMSketch: statistics.NewCMSketch(int32(e.opts[ast.AnalyzeOptCMSketchDepth]), int32(e.opts[ast.AnalyzeOptCMSketchWidth])),
}
}
+ statsHandle := domain.GetDomain(e.ctx).StatsHandle()
for {
failpoint.Inject("mockKillRunningV1AnalyzeJob", func() {
dom := domain.GetDomain(e.ctx)
- dom.SysProcTracker().KillSysProcess(dom.GetAutoAnalyzeProcID())
+ for _, id := range handleutil.GlobalAutoAnalyzeProcessList.All() {
+ dom.SysProcTracker().KillSysProcess(id)
+ }
})
if err := e.ctx.GetSessionVars().SQLKiller.HandleSignal(); err != nil {
return nil, nil, nil, nil, nil, err
@@ -225,7 +229,7 @@ func (e *AnalyzeColumnsExec) buildStats(ranges []*ranger.Range, needExtStats boo
rowCount = respSample.Count + respSample.NullCount
collectors[i].MergeSampleCollector(sc, respSample)
}
- UpdateAnalyzeJob(e.ctx, e.job, rowCount)
+ statsHandle.UpdateAnalyzeJobProgress(e.job, rowCount)
}
timeZone := e.ctx.GetSessionVars().Location()
if hasPkHist(e.handleCols) {
diff --git a/pkg/executor/analyze_col_v2.go b/pkg/executor/analyze_col_v2.go
index 479cefe37917b..6eefe269fc865 100644
--- a/pkg/executor/analyze_col_v2.go
+++ b/pkg/executor/analyze_col_v2.go
@@ -30,6 +30,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/statistics"
+ handleutil "github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/types"
@@ -472,6 +473,7 @@ func (e *AnalyzeColumnsExecV2) handleNDVForSpecialIndexes(indexInfos []*model.In
results: make(map[int64]*statistics.AnalyzeResults, len(indexInfos)),
}
var err error
+ statsHandle := domain.GetDomain(e.ctx).StatsHandle()
for panicCnt < statsConcurrncy {
results, ok := <-resultsCh
if !ok {
@@ -479,13 +481,13 @@ func (e *AnalyzeColumnsExecV2) handleNDVForSpecialIndexes(indexInfos []*model.In
}
if results.Err != nil {
err = results.Err
- FinishAnalyzeJob(e.ctx, results.Job, err)
+ statsHandle.FinishAnalyzeJob(results.Job, err, statistics.TableAnalysisJob)
if isAnalyzeWorkerPanic(err) {
panicCnt++
}
continue
}
- FinishAnalyzeJob(e.ctx, results.Job, nil)
+ statsHandle.FinishAnalyzeJob(results.Job, nil, statistics.TableAnalysisJob)
totalResult.results[results.Ars[0].Hist[0].ID] = results
}
if err != nil {
@@ -497,6 +499,7 @@ func (e *AnalyzeColumnsExecV2) handleNDVForSpecialIndexes(indexInfos []*model.In
// subIndexWorker receive the task for each index and return the result for them.
func (e *AnalyzeColumnsExecV2) subIndexWorkerForNDV(taskCh chan *analyzeTask, resultsCh chan *statistics.AnalyzeResults) {
var task *analyzeTask
+ statsHandle := domain.GetDomain(e.ctx).StatsHandle()
defer func() {
if r := recover(); r != nil {
logutil.BgLogger().Error("analyze worker panicked", zap.Any("recover", r), zap.Stack("stack"))
@@ -513,7 +516,7 @@ func (e *AnalyzeColumnsExecV2) subIndexWorkerForNDV(taskCh chan *analyzeTask, re
if !ok {
break
}
- StartAnalyzeJob(e.ctx, task.job)
+ statsHandle.StartAnalyzeJob(task.job)
if task.taskType != idxTask {
resultsCh <- &statistics.AnalyzeResults{
Err: errors.Errorf("incorrect analyze type"),
@@ -627,6 +630,7 @@ func (e *AnalyzeColumnsExecV2) subMergeWorker(resultCh chan<- *samplingMergeResu
for i := 0; i < l; i++ {
retCollector.Base().FMSketches = append(retCollector.Base().FMSketches, statistics.NewFMSketch(statistics.MaxSketchSize))
}
+ statsHandle := domain.GetDomain(e.ctx).StatsHandle()
for {
data, ok := <-taskCh
if !ok {
@@ -648,7 +652,7 @@ func (e *AnalyzeColumnsExecV2) subMergeWorker(resultCh chan<- *samplingMergeResu
// Update processed rows.
subCollector := statistics.NewRowSampleCollector(int(e.analyzePB.ColReq.SampleSize), e.analyzePB.ColReq.GetSampleRate(), l)
subCollector.Base().FromProto(colResp.RowCollector, e.memTracker)
- UpdateAnalyzeJob(e.ctx, e.job, subCollector.Base().Count)
+ statsHandle.UpdateAnalyzeJobProgress(e.job, subCollector.Base().Count)
// Print collect log.
oldRetCollectorSize := retCollector.Base().MemSize
@@ -854,7 +858,9 @@ func readDataAndSendTask(ctx sessionctx.Context, handler *tableResultHandler, me
for {
failpoint.Inject("mockKillRunningV2AnalyzeJob", func() {
dom := domain.GetDomain(ctx)
- dom.SysProcTracker().KillSysProcess(dom.GetAutoAnalyzeProcID())
+ for _, id := range handleutil.GlobalAutoAnalyzeProcessList.All() {
+ dom.SysProcTracker().KillSysProcess(id)
+ }
})
if err := ctx.GetSessionVars().SQLKiller.HandleSignal(); err != nil {
return err
diff --git a/pkg/executor/analyze_global_stats.go b/pkg/executor/analyze_global_stats.go
index 29801ca35fda8..f9a055072c17e 100644
--- a/pkg/executor/analyze_global_stats.go
+++ b/pkg/executor/analyze_global_stats.go
@@ -15,11 +15,13 @@
package executor
import (
+ "context"
"fmt"
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/statistics"
+ "github.com/pingcap/tidb/pkg/statistics/handle"
statstypes "github.com/pingcap/tidb/pkg/statistics/handle/types"
"github.com/pingcap/tidb/pkg/util/logutil"
"go.uber.org/zap"
@@ -35,13 +37,12 @@ type globalStatsKey struct {
// The meaning of value in map is some additional information needed to build global-level stats.
type globalStatsMap map[globalStatsKey]statstypes.GlobalStatsInfo
-func (e *AnalyzeExec) handleGlobalStats(globalStatsMap globalStatsMap) error {
+func (e *AnalyzeExec) handleGlobalStats(statsHandle *handle.Handle, globalStatsMap globalStatsMap) error {
globalStatsTableIDs := make(map[int64]struct{}, len(globalStatsMap))
for globalStatsID := range globalStatsMap {
globalStatsTableIDs[globalStatsID.tableID] = struct{}{}
}
- statsHandle := domain.GetDomain(e.Ctx()).StatsHandle()
tableIDs := make(map[int64]struct{}, len(globalStatsTableIDs))
for tableID := range globalStatsTableIDs {
tableIDs[tableID] = struct{}{}
@@ -55,7 +56,7 @@ func (e *AnalyzeExec) handleGlobalStats(globalStatsMap globalStatsMap) error {
continue
}
AddNewAnalyzeJob(e.Ctx(), job)
- StartAnalyzeJob(e.Ctx(), job)
+ statsHandle.StartAnalyzeJob(job)
mergeStatsErr := func() error {
globalOpts := e.opts
@@ -76,7 +77,7 @@ func (e *AnalyzeExec) handleGlobalStats(globalStatsMap globalStatsMap) error {
}
return err
}()
- FinishAnalyzeMergeJob(e.Ctx(), job, mergeStatsErr)
+ statsHandle.FinishAnalyzeJob(job, mergeStatsErr, statistics.GlobalStatsMergeJob)
}
}
@@ -93,7 +94,7 @@ func (e *AnalyzeExec) handleGlobalStats(globalStatsMap globalStatsMap) error {
func (e *AnalyzeExec) newAnalyzeHandleGlobalStatsJob(key globalStatsKey) *statistics.AnalyzeJob {
dom := domain.GetDomain(e.Ctx())
is := dom.InfoSchema()
- table, ok := is.TableByID(key.tableID)
+ table, ok := is.TableByID(context.Background(), key.tableID)
if !ok {
return nil
}
diff --git a/pkg/executor/analyze_idx.go b/pkg/executor/analyze_idx.go
index a7c0a4b86fb71..1c26be7782d4f 100644
--- a/pkg/executor/analyze_idx.go
+++ b/pkg/executor/analyze_idx.go
@@ -29,6 +29,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/statistics"
+ handleutil "github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/logutil"
"github.com/pingcap/tidb/pkg/util/ranger"
@@ -199,7 +200,9 @@ func (e *AnalyzeIndexExec) buildStatsFromResult(result distsql.SelectResult, nee
for {
failpoint.Inject("mockKillRunningAnalyzeIndexJob", func() {
dom := domain.GetDomain(e.ctx)
- dom.SysProcTracker().KillSysProcess(dom.GetAutoAnalyzeProcID())
+ for _, id := range handleutil.GlobalAutoAnalyzeProcessList.All() {
+ dom.SysProcTracker().KillSysProcess(id)
+ }
})
if err := e.ctx.GetSessionVars().SQLKiller.HandleSignal(); err != nil {
return nil, nil, nil, nil, err
@@ -316,7 +319,8 @@ func updateIndexResult(
needCMS := cms != nil
respHist := statistics.HistogramFromProto(resp.Hist)
if job != nil {
- UpdateAnalyzeJob(ctx, job, int64(respHist.TotalRowCount()))
+ statsHandle := domain.GetDomain(ctx).StatsHandle()
+ statsHandle.UpdateAnalyzeJobProgress(job, int64(respHist.TotalRowCount()))
}
hist, err = statistics.MergeHistograms(ctx.GetSessionVars().StmtCtx, hist, respHist, numBuckets, statsVer)
if err != nil {
diff --git a/pkg/executor/analyze_test.go b/pkg/executor/analyze_test.go
index 8c424a33e6679..fbbf1bcc0d930 100644
--- a/pkg/executor/analyze_test.go
+++ b/pkg/executor/analyze_test.go
@@ -63,7 +63,6 @@ func TestAnalyzeIndexExtractTopN(t *testing.T) {
}()
var dom *domain.Domain
session.DisableStats4Test()
- session.SetSchemaLease(0)
dom, err = session.BootstrapSession(store)
require.NoError(t, err)
defer dom.Close()
diff --git a/pkg/executor/analyze_worker.go b/pkg/executor/analyze_worker.go
index d92df2a1da266..e5fab8af9539c 100644
--- a/pkg/executor/analyze_worker.go
+++ b/pkg/executor/analyze_worker.go
@@ -17,9 +17,8 @@ package executor
import (
"context"
- "github.com/pingcap/tidb/pkg/domain"
- "github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/statistics"
+ "github.com/pingcap/tidb/pkg/statistics/handle"
"github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/util/logutil"
"github.com/pingcap/tidb/pkg/util/sqlkiller"
@@ -28,26 +27,23 @@ import (
type analyzeSaveStatsWorker struct {
resultsCh <-chan *statistics.AnalyzeResults
- sctx sessionctx.Context
errCh chan<- error
killer *sqlkiller.SQLKiller
}
func newAnalyzeSaveStatsWorker(
resultsCh <-chan *statistics.AnalyzeResults,
- sctx sessionctx.Context,
errCh chan<- error,
killer *sqlkiller.SQLKiller) *analyzeSaveStatsWorker {
worker := &analyzeSaveStatsWorker{
resultsCh: resultsCh,
- sctx: sctx,
errCh: errCh,
killer: killer,
}
return worker
}
-func (worker *analyzeSaveStatsWorker) run(ctx context.Context, analyzeSnapshot bool) {
+func (worker *analyzeSaveStatsWorker) run(ctx context.Context, statsHandle *handle.Handle, analyzeSnapshot bool) {
defer func() {
if r := recover(); r != nil {
logutil.BgLogger().Error("analyze save stats worker panicked", zap.Any("recover", r), zap.Stack("stack"))
@@ -56,17 +52,18 @@ func (worker *analyzeSaveStatsWorker) run(ctx context.Context, analyzeSnapshot b
}()
for results := range worker.resultsCh {
if err := worker.killer.HandleSignal(); err != nil {
+ finishJobWithLog(statsHandle, results.Job, err)
+ results.DestroyAndPutToPool()
worker.errCh <- err
return
}
- statsHandle := domain.GetDomain(worker.sctx).StatsHandle()
err := statsHandle.SaveTableStatsToStorage(results, analyzeSnapshot, util.StatsMetaHistorySourceAnalyze)
if err != nil {
logutil.Logger(ctx).Error("save table stats to storage failed", zap.Error(err))
- finishJobWithLog(worker.sctx, results.Job, err)
+ finishJobWithLog(statsHandle, results.Job, err)
worker.errCh <- err
} else {
- finishJobWithLog(worker.sctx, results.Job, nil)
+ finishJobWithLog(statsHandle, results.Job, nil)
}
results.DestroyAndPutToPool()
if err != nil {
diff --git a/pkg/executor/batch_checker.go b/pkg/executor/batch_checker.go
index 5bec8e3837558..a3aed84315672 100644
--- a/pkg/executor/batch_checker.go
+++ b/pkg/executor/batch_checker.go
@@ -280,7 +280,7 @@ func getOldRow(ctx context.Context, sctx sessionctx.Context, txn kv.Transaction,
}
cols := t.WritableCols()
- oldRow, oldRowMap, err := tables.DecodeRawRowData(sctx, t.Meta(), handle, cols, oldValue)
+ oldRow, oldRowMap, err := tables.DecodeRawRowData(sctx.GetExprCtx(), t.Meta(), handle, cols, oldValue)
if err != nil {
return nil, err
}
diff --git a/pkg/executor/benchmark_test.go b/pkg/executor/benchmark_test.go
index 5d15670d25038..dfe65f0954bb3 100644
--- a/pkg/executor/benchmark_test.go
+++ b/pkg/executor/benchmark_test.go
@@ -40,6 +40,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/planner/core"
"github.com/pingcap/tidb/pkg/planner/core/base"
+ "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop"
"github.com/pingcap/tidb/pkg/planner/property"
"github.com/pingcap/tidb/pkg/planner/util"
"github.com/pingcap/tidb/pkg/sessionctx"
@@ -278,7 +279,7 @@ func BenchmarkAggDistinct(b *testing.B) {
}
}
-func buildWindowExecutor(ctx sessionctx.Context, windowFunc string, funcs int, frame *core.WindowFrame, srcExec exec.Executor, schema *expression.Schema, partitionBy []*expression.Column, concurrency int, dataSourceSorted bool) exec.Executor {
+func buildWindowExecutor(ctx sessionctx.Context, windowFunc string, funcs int, frame *logicalop.WindowFrame, srcExec exec.Executor, schema *expression.Schema, partitionBy []*expression.Column, concurrency int, dataSourceSorted bool) exec.Executor {
src := testutil.BuildMockDataPhysicalPlan(ctx, srcExec)
win := new(core.PhysicalWindow)
win.WindowFuncDescs = make([]*aggregation.WindowFuncDesc, 0)
@@ -476,8 +477,8 @@ func baseBenchmarkWindowFunctionsWithFrame(b *testing.B, pipelined int) {
ast.AggFuncBitXor,
}
numFuncs := []int{1, 5}
- frames := []*core.WindowFrame{
- {Type: ast.Rows, Start: &core.FrameBound{UnBounded: true}, End: &core.FrameBound{Type: ast.CurrentRow}},
+ frames := []*logicalop.WindowFrame{
+ {Type: ast.Rows, Start: &logicalop.FrameBound{UnBounded: true}, End: &logicalop.FrameBound{Type: ast.CurrentRow}},
}
sortTypes := []bool{false, true}
concs := []int{1, 2, 3, 4, 5, 6}
@@ -513,7 +514,7 @@ func BenchmarkWindowFunctionsWithFrame(b *testing.B) {
func baseBenchmarkWindowFunctionsAggWindowProcessorAboutFrame(b *testing.B, pipelined int) {
b.ReportAllocs()
windowFunc := ast.AggFuncMax
- frame := &core.WindowFrame{Type: ast.Rows, Start: &core.FrameBound{UnBounded: true}, End: &core.FrameBound{UnBounded: true}}
+ frame := &logicalop.WindowFrame{Type: ast.Rows, Start: &logicalop.FrameBound{UnBounded: true}, End: &logicalop.FrameBound{UnBounded: true}}
cas := testutil.DefaultWindowTestCase()
cas.Rows = 10000
cas.Ndv = 10
@@ -552,10 +553,10 @@ func baseBenchmarkWindowFunctionsWithSlidingWindow(b *testing.B, frameType ast.F
}
row := 100000
ndv := 100
- frame := &core.WindowFrame{
+ frame := &logicalop.WindowFrame{
Type: frameType,
- Start: &core.FrameBound{Type: ast.Preceding, Num: 10},
- End: &core.FrameBound{Type: ast.Following, Num: 10},
+ Start: &logicalop.FrameBound{Type: ast.Preceding, Num: 10},
+ End: &logicalop.FrameBound{Type: ast.Following, Num: 10},
}
for _, windowFunc := range windowFuncs {
cas := testutil.DefaultWindowTestCase()
@@ -585,7 +586,7 @@ type hashJoinTestCase struct {
concurrency int
ctx sessionctx.Context
keyIdx []int
- joinType core.JoinType
+ joinType logicalop.JoinType
disk bool
useOuterToBuild bool
rawData string
@@ -606,7 +607,7 @@ func (tc hashJoinTestCase) String() string {
tc.rows, tc.cols, tc.concurrency, tc.keyIdx, tc.disk)
}
-func defaultHashJoinTestCase(cols []*types.FieldType, joinType core.JoinType, useOuterToBuild bool) *hashJoinTestCase {
+func defaultHashJoinTestCase(cols []*types.FieldType, joinType logicalop.JoinType, useOuterToBuild bool) *hashJoinTestCase {
ctx := mock.NewContext()
ctx.GetSessionVars().InitChunkSize = variable.DefInitChunkSize
ctx.GetSessionVars().MaxChunkSize = variable.DefMaxChunkSize
@@ -620,10 +621,10 @@ func defaultHashJoinTestCase(cols []*types.FieldType, joinType core.JoinType, us
return tc
}
-func prepareResolveIndices(joinSchema, lSchema, rSchema *expression.Schema, joinType core.JoinType) *expression.Schema {
+func prepareResolveIndices(joinSchema, lSchema, rSchema *expression.Schema, joinType logicalop.JoinType) *expression.Schema {
colsNeedResolving := joinSchema.Len()
// The last output column of this two join is the generated column to indicate whether the row is matched or not.
- if joinType == core.LeftOuterSemiJoin || joinType == core.AntiLeftOuterSemiJoin {
+ if joinType == logicalop.LeftOuterSemiJoin || joinType == logicalop.AntiLeftOuterSemiJoin {
colsNeedResolving--
}
mergedSchema := expression.MergeSchema(lSchema, rSchema)
@@ -686,7 +687,7 @@ func prepare4HashJoinV2(testCase *hashJoinTestCase, innerExec, outerExec exec.Ex
// todo: need systematic way to protect.
// physical join should resolveIndices to get right schema column index.
// otherwise, markChildrenUsedColsForTest will fail below.
- joinSchema = prepareResolveIndices(joinSchema, innerExec.Schema(), outerExec.Schema(), core.InnerJoin)
+ joinSchema = prepareResolveIndices(joinSchema, innerExec.Schema(), outerExec.Schema(), logicalop.InnerJoin)
joinKeysColIdx := make([]int, 0, len(testCase.keyIdx))
joinKeysColIdx = append(joinKeysColIdx, testCase.keyIdx...)
@@ -775,7 +776,7 @@ func prepare4HashJoin(testCase *hashJoinTestCase, innerExec, outerExec exec.Exec
// todo: need systematic way to protect.
// physical join should resolveIndices to get right schema column index.
// otherwise, markChildrenUsedColsForTest will fail below.
- joinSchema = prepareResolveIndices(joinSchema, innerExec.Schema(), outerExec.Schema(), core.InnerJoin)
+ joinSchema = prepareResolveIndices(joinSchema, innerExec.Schema(), outerExec.Schema(), logicalop.InnerJoin)
joinKeysColIdx := make([]int, 0, len(testCase.keyIdx))
joinKeysColIdx = append(joinKeysColIdx, testCase.keyIdx...)
diff --git a/pkg/executor/bind.go b/pkg/executor/bind.go
index fef5c9d7ad538..7cfbf32c39647 100644
--- a/pkg/executor/bind.go
+++ b/pkg/executor/bind.go
@@ -22,7 +22,6 @@ import (
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/executor/internal/exec"
"github.com/pingcap/tidb/pkg/parser"
- "github.com/pingcap/tidb/pkg/parser/ast"
plannercore "github.com/pingcap/tidb/pkg/planner/core"
"github.com/pingcap/tidb/pkg/util/chunk"
)
@@ -31,18 +30,9 @@ import (
type SQLBindExec struct {
exec.BaseExecutor
- sqlBindOp plannercore.SQLBindOpType
- normdOrigSQL string
- bindSQL string
- charset string
- collation string
- db string
- isGlobal bool
- bindAst ast.StmtNode
- newStatus string
- source string // by manual or from history, only in create stmt
- sqlDigest string
- planDigest string
+ isGlobal bool
+ sqlBindOp plannercore.SQLBindOpType
+ details []*plannercore.SQLBindOpDetail
}
// Next implements the Executor Next interface.
@@ -74,33 +64,43 @@ func (e *SQLBindExec) Next(_ context.Context, req *chunk.Chunk) error {
}
func (e *SQLBindExec) dropSQLBind() error {
+ if len(e.details) != 1 {
+ return errors.New("SQLBindExec: dropSQLBind should only have one SQLBindOpDetail")
+ }
if !e.isGlobal {
handle := e.Ctx().Value(bindinfo.SessionBindInfoKeyType).(bindinfo.SessionBindingHandle)
- err := handle.DropSessionBinding(e.sqlDigest)
+ err := handle.DropSessionBinding([]string{e.details[0].SQLDigest})
return err
}
- affectedRows, err := domain.GetDomain(e.Ctx()).BindHandle().DropGlobalBinding(e.sqlDigest)
+ affectedRows, err := domain.GetDomain(e.Ctx()).BindHandle().DropGlobalBinding([]string{e.details[0].SQLDigest})
e.Ctx().GetSessionVars().StmtCtx.AddAffectedRows(affectedRows)
return err
}
func (e *SQLBindExec) dropSQLBindByDigest() error {
- if e.sqlDigest == "" {
- return errors.New("sql digest is empty")
+ sqlDigests := make([]string, 0, len(e.details))
+ for _, detail := range e.details {
+ if detail.SQLDigest == "" {
+ return errors.New("SQLBindExec: dropSQLBindByDigest shouldn't contain empty SQLDigest")
+ }
+ sqlDigests = append(sqlDigests, detail.SQLDigest)
}
if !e.isGlobal {
handle := e.Ctx().Value(bindinfo.SessionBindInfoKeyType).(bindinfo.SessionBindingHandle)
- err := handle.DropSessionBinding(e.sqlDigest)
+ err := handle.DropSessionBinding(sqlDigests)
return err
}
- affectedRows, err := domain.GetDomain(e.Ctx()).BindHandle().DropGlobalBinding(e.sqlDigest)
+ affectedRows, err := domain.GetDomain(e.Ctx()).BindHandle().DropGlobalBinding(sqlDigests)
e.Ctx().GetSessionVars().StmtCtx.AddAffectedRows(affectedRows)
return err
}
func (e *SQLBindExec) setBindingStatus() error {
- _, sqlDigest := parser.NormalizeDigestForBinding(e.normdOrigSQL)
- ok, err := domain.GetDomain(e.Ctx()).BindHandle().SetGlobalBindingStatus(e.newStatus, sqlDigest.String())
+ if len(e.details) != 1 {
+ return errors.New("SQLBindExec: setBindingStatus should only have one SQLBindOpDetail")
+ }
+ _, sqlDigest := parser.NormalizeDigestForBinding(e.details[0].NormdOrigSQL)
+ ok, err := domain.GetDomain(e.Ctx()).BindHandle().SetGlobalBindingStatus(e.details[0].NewStatus, sqlDigest.String())
if err == nil && !ok {
warningMess := errors.NewNoStackError("There are no bindings can be set the status. Please check the SQL text")
e.Ctx().GetSessionVars().StmtCtx.AppendWarning(warningMess)
@@ -109,7 +109,13 @@ func (e *SQLBindExec) setBindingStatus() error {
}
func (e *SQLBindExec) setBindingStatusByDigest() error {
- ok, err := domain.GetDomain(e.Ctx()).BindHandle().SetGlobalBindingStatus(e.newStatus, e.sqlDigest)
+ if len(e.details) != 1 {
+ return errors.New("SQLBindExec: setBindingStatusByDigest should only have one SQLBindOpDetail")
+ }
+ ok, err := domain.GetDomain(e.Ctx()).BindHandle().SetGlobalBindingStatus(
+ e.details[0].NewStatus,
+ e.details[0].SQLDigest,
+ )
if err == nil && !ok {
warningMess := errors.NewNoStackError("There are no bindings can be set the status. Please check the SQL text")
e.Ctx().GetSessionVars().StmtCtx.AppendWarning(warningMess)
@@ -129,22 +135,27 @@ func (e *SQLBindExec) createSQLBind() error {
e.Ctx().GetSessionVars().StmtCtx = saveStmtCtx
}()
- binding := bindinfo.Binding{
- OriginalSQL: e.normdOrigSQL,
- Db: e.db,
- BindSQL: e.bindSQL,
- Charset: e.charset,
- Collation: e.collation,
- Status: bindinfo.Enabled,
- Source: e.source,
- SQLDigest: e.sqlDigest,
- PlanDigest: e.planDigest,
+ bindings := make([]*bindinfo.Binding, 0, len(e.details))
+ for _, detail := range e.details {
+ binding := bindinfo.Binding{
+ OriginalSQL: detail.NormdOrigSQL,
+ Db: detail.Db,
+ BindSQL: detail.BindSQL,
+ Charset: detail.Charset,
+ Collation: detail.Collation,
+ Status: bindinfo.Enabled,
+ Source: detail.Source,
+ SQLDigest: detail.SQLDigest,
+ PlanDigest: detail.PlanDigest,
+ }
+ bindings = append(bindings, &binding)
}
+
if !e.isGlobal {
handle := e.Ctx().Value(bindinfo.SessionBindInfoKeyType).(bindinfo.SessionBindingHandle)
- return handle.CreateSessionBinding(e.Ctx(), binding)
+ return handle.CreateSessionBinding(e.Ctx(), bindings)
}
- return domain.GetDomain(e.Ctx()).BindHandle().CreateGlobalBinding(e.Ctx(), binding)
+ return domain.GetDomain(e.Ctx()).BindHandle().CreateGlobalBinding(e.Ctx(), bindings)
}
func (e *SQLBindExec) flushBindings() error {
diff --git a/pkg/executor/brie.go b/pkg/executor/brie.go
index 130f627bc6609..dd019e30b67b3 100644
--- a/pkg/executor/brie.go
+++ b/pkg/executor/brie.go
@@ -753,6 +753,10 @@ func (gs *tidbGlue) UseOneShotSession(_ kv.Storage, _ bool, fn func(se glue.Sess
return fn(glueSession)
}
+func (*tidbGlue) GetClient() glue.GlueClient {
+ return glue.ClientSql
+}
+
type tidbGlueSession struct {
// the session context of the brie task's subtask, such as `CREATE TABLE`.
se sessionctx.Context
diff --git a/pkg/executor/builder.go b/pkg/executor/builder.go
index d54046b2d0186..469682d1a4a10 100644
--- a/pkg/executor/builder.go
+++ b/pkg/executor/builder.go
@@ -62,6 +62,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/terror"
plannercore "github.com/pingcap/tidb/pkg/planner/core"
"github.com/pingcap/tidb/pkg/planner/core/base"
+ "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop"
plannerutil "github.com/pingcap/tidb/pkg/planner/util"
"github.com/pingcap/tidb/pkg/planner/util/coreusage"
"github.com/pingcap/tidb/pkg/sessionctx"
@@ -787,7 +788,7 @@ func (b *executorBuilder) buildSelectLock(v *plannercore.PhysicalLock) exec.Exec
// filter out temporary tables because they do not store any record in tikv and should not write any lock
is := e.Ctx().GetInfoSchema().(infoschema.InfoSchema)
for tblID := range e.tblID2Handle {
- tblInfo, ok := is.TableByID(tblID)
+ tblInfo, ok := is.TableByID(context.Background(), tblID)
if !ok {
b.err = errors.Errorf("Can not get table %d", tblID)
}
@@ -998,6 +999,7 @@ func (b *executorBuilder) buildInsert(v *plannercore.Insert) exec.Executor {
insert := &InsertExec{
InsertValues: ivs,
OnDuplicate: append(v.OnDuplicate, v.GenCols.OnDuplicates...),
+ IgnoreErr: v.IgnoreErr,
}
return insert
}
@@ -1005,7 +1007,7 @@ func (b *executorBuilder) buildInsert(v *plannercore.Insert) exec.Executor {
func (b *executorBuilder) buildImportInto(v *plannercore.ImportInto) exec.Executor {
// see planBuilder.buildImportInto for detail why we use the latest schema here.
latestIS := b.ctx.GetDomainInfoSchema().(infoschema.InfoSchema)
- tbl, ok := latestIS.TableByID(v.Table.TableInfo.ID)
+ tbl, ok := latestIS.TableByID(context.Background(), v.Table.TableInfo.ID)
if !ok {
b.err = errors.Errorf("Can not get table %d", v.Table.TableInfo.ID)
return nil
@@ -1038,7 +1040,7 @@ func (b *executorBuilder) buildImportInto(v *plannercore.ImportInto) exec.Execut
}
func (b *executorBuilder) buildLoadData(v *plannercore.LoadData) exec.Executor {
- tbl, ok := b.is.TableByID(v.Table.TableInfo.ID)
+ tbl, ok := b.is.TableByID(context.Background(), v.Table.TableInfo.ID)
if !ok {
b.err = errors.Errorf("Can not get table %d", v.Table.TableInfo.ID)
return nil
@@ -1426,7 +1428,7 @@ func (b *executorBuilder) buildMergeJoin(v *plannercore.PhysicalMergeJoin) exec.
defaultValues := v.DefaultValues
if defaultValues == nil {
- if v.JoinType == plannercore.RightOuterJoin {
+ if v.JoinType == logicalop.RightOuterJoin {
defaultValues = make([]types.Datum, leftExec.Schema().Len())
} else {
defaultValues = make([]types.Datum, rightExec.Schema().Len())
@@ -1434,7 +1436,7 @@ func (b *executorBuilder) buildMergeJoin(v *plannercore.PhysicalMergeJoin) exec.
}
colsFromChildren := v.Schema().Columns
- if v.JoinType == plannercore.LeftOuterSemiJoin || v.JoinType == plannercore.AntiLeftOuterSemiJoin {
+ if v.JoinType == logicalop.LeftOuterSemiJoin || v.JoinType == logicalop.AntiLeftOuterSemiJoin {
colsFromChildren = colsFromChildren[:len(colsFromChildren)-1]
}
@@ -1445,7 +1447,7 @@ func (b *executorBuilder) buildMergeJoin(v *plannercore.PhysicalMergeJoin) exec.
Joiner: join.NewJoiner(
b.ctx,
v.JoinType,
- v.JoinType == plannercore.RightOuterJoin,
+ v.JoinType == logicalop.RightOuterJoin,
defaultValues,
v.OtherConditions,
exec.RetTypes(leftExec),
@@ -1468,7 +1470,7 @@ func (b *executorBuilder) buildMergeJoin(v *plannercore.PhysicalMergeJoin) exec.
Filters: v.RightConditions,
}
- if v.JoinType == plannercore.RightOuterJoin {
+ if v.JoinType == logicalop.RightOuterJoin {
e.InnerTable = leftTable
e.OuterTable = rightTable
} else {
@@ -1601,7 +1603,7 @@ func (b *executorBuilder) buildHashJoinV2(v *plannercore.PhysicalHashJoin) exec.
}
colsFromChildren := v.Schema().Columns
- if v.JoinType == plannercore.LeftOuterSemiJoin || v.JoinType == plannercore.AntiLeftOuterSemiJoin {
+ if v.JoinType == logicalop.LeftOuterSemiJoin || v.JoinType == logicalop.AntiLeftOuterSemiJoin {
// the matched column is added inside join
colsFromChildren = colsFromChildren[:len(colsFromChildren)-1]
}
@@ -1770,7 +1772,7 @@ func (b *executorBuilder) buildHashJoin(v *plannercore.PhysicalHashJoin) exec.Ex
}
isNAJoin := len(v.LeftNAJoinKeys) > 0
colsFromChildren := v.Schema().Columns
- if v.JoinType == plannercore.LeftOuterSemiJoin || v.JoinType == plannercore.AntiLeftOuterSemiJoin {
+ if v.JoinType == logicalop.LeftOuterSemiJoin || v.JoinType == logicalop.AntiLeftOuterSemiJoin {
colsFromChildren = colsFromChildren[:len(colsFromChildren)-1]
}
childrenUsedSchema := markChildrenUsedCols(colsFromChildren, v.Children()[0].Schema(), v.Children()[1].Schema())
@@ -2330,7 +2332,7 @@ func (b *executorBuilder) buildMemTable(v *plannercore.PhysicalMemTable) exec.Ex
retriever: &hugeMemTableRetriever{
table: v.Table,
columns: v.Columns,
- extractor: v.Extractor.(*plannercore.ColumnsTableExtractor),
+ extractor: v.Extractor.(*plannercore.InfoSchemaColumnsExtractor),
viewSchemaMap: make(map[int64]*expression.Schema),
viewOutputNamesMap: make(map[int64]types.NameSlice),
},
@@ -2379,7 +2381,7 @@ func (b *executorBuilder) buildMemTable(v *plannercore.PhysicalMemTable) exec.Ex
}
}
}
- tb, _ := b.is.TableByID(v.Table.ID)
+ tb, _ := b.is.TableByID(context.Background(), v.Table.ID)
return &TableScanExec{
BaseExecutor: exec.NewBaseExecutor(b.ctx, v.Schema(), v.ID()),
t: tb,
@@ -2462,7 +2464,7 @@ func (b *executorBuilder) buildApply(v *plannercore.PhysicalApply) exec.Executor
OuterExec: outerExec,
OuterFilter: outerFilter,
InnerFilter: innerFilter,
- Outer: v.JoinType != plannercore.InnerJoin,
+ Outer: v.JoinType != logicalop.InnerJoin,
Joiner: tupleJoiner,
OuterSchema: v.OuterSchema,
Sctx: b.ctx,
@@ -2503,7 +2505,7 @@ func (b *executorBuilder) buildApply(v *plannercore.PhysicalApply) exec.Executor
outerExec: outerExec,
outerFilter: outerFilter,
innerFilter: innerFilters,
- outer: v.JoinType != plannercore.InnerJoin,
+ outer: v.JoinType != logicalop.InnerJoin,
joiners: joiners,
corCols: corCols,
concurrency: v.Concurrency,
@@ -2603,7 +2605,7 @@ func (b *executorBuilder) buildUpdate(v *plannercore.Update) exec.Executor {
tblID2table := make(map[int64]table.Table, len(v.TblColPosInfos))
multiUpdateOnSameTable := make(map[int64]bool)
for _, info := range v.TblColPosInfos {
- tbl, _ := b.is.TableByID(info.TblID)
+ tbl, _ := b.is.TableByID(context.Background(), info.TblID)
if _, ok := tblID2table[info.TblID]; ok {
multiUpdateOnSameTable[info.TblID] = true
}
@@ -2649,6 +2651,7 @@ func (b *executorBuilder) buildUpdate(v *plannercore.Update) exec.Executor {
tblID2table: tblID2table,
tblColPosInfos: v.TblColPosInfos,
assignFlag: assignFlag,
+ IgnoreError: v.IgnoreError,
}
updateExec.fkChecks, b.err = buildTblID2FKCheckExecs(b.ctx, tblID2table, v.FKChecks)
if b.err != nil {
@@ -2683,7 +2686,7 @@ func (b *executorBuilder) buildDelete(v *plannercore.Delete) exec.Executor {
b.inDeleteStmt = true
tblID2table := make(map[int64]table.Table, len(v.TblColPosInfos))
for _, info := range v.TblColPosInfos {
- tblID2table[info.TblID], _ = b.is.TableByID(info.TblID)
+ tblID2table[info.TblID], _ = b.is.TableByID(context.Background(), info.TblID)
}
if b.err = b.updateForUpdateTS(); b.err != nil {
@@ -2878,7 +2881,7 @@ func (b *executorBuilder) buildAnalyzeSamplingPushdown(
SampleSize: int64(opts[ast.AnalyzeOptNumSamples]),
SampleRate: sampleRate,
SketchSize: statistics.MaxSketchSize,
- ColumnsInfo: util.ColumnsToProto(task.ColsInfo, task.TblInfo.PKIsHandle, false),
+ ColumnsInfo: util.ColumnsToProto(task.ColsInfo, task.TblInfo.PKIsHandle, false, false),
ColumnGroups: colGroups,
}
if task.TblInfo != nil {
@@ -3008,7 +3011,7 @@ func (b *executorBuilder) buildAnalyzeColumnsPushdown(
BucketSize: int64(opts[ast.AnalyzeOptNumBuckets]),
SampleSize: MaxRegionSampleSize,
SketchSize: statistics.MaxSketchSize,
- ColumnsInfo: util.ColumnsToProto(cols, task.HandleCols != nil && task.HandleCols.IsInt(), false),
+ ColumnsInfo: util.ColumnsToProto(cols, task.HandleCols != nil && task.HandleCols.IsInt(), false, false),
CmsketchDepth: &depth,
CmsketchWidth: &width,
}
@@ -3291,7 +3294,7 @@ func (b *executorBuilder) buildIndexLookUpJoin(v *plannercore.PhysicalIndexJoin)
Finished: &atomic.Value{},
}
colsFromChildren := v.Schema().Columns
- if v.JoinType == plannercore.LeftOuterSemiJoin || v.JoinType == plannercore.AntiLeftOuterSemiJoin {
+ if v.JoinType == logicalop.LeftOuterSemiJoin || v.JoinType == logicalop.AntiLeftOuterSemiJoin {
colsFromChildren = colsFromChildren[:len(colsFromChildren)-1]
}
childrenUsedSchema := markChildrenUsedCols(colsFromChildren, v.Children()[0].Schema(), v.Children()[1].Schema())
@@ -3417,7 +3420,7 @@ func (b *executorBuilder) buildIndexLookUpMergeJoin(v *plannercore.PhysicalIndex
LastColHelper: v.CompareFilters,
}
colsFromChildren := v.Schema().Columns
- if v.JoinType == plannercore.LeftOuterSemiJoin || v.JoinType == plannercore.AntiLeftOuterSemiJoin {
+ if v.JoinType == logicalop.LeftOuterSemiJoin || v.JoinType == logicalop.AntiLeftOuterSemiJoin {
colsFromChildren = colsFromChildren[:len(colsFromChildren)-1]
}
childrenUsedSchema := markChildrenUsedCols(colsFromChildren, v.Children()[0].Schema(), v.Children()[1].Schema())
@@ -3464,7 +3467,7 @@ func buildNoRangeTableReader(b *executorBuilder, v *plannercore.PhysicalTableRea
return nil, err
}
- tbl, _ := b.is.TableByID(ts.Table.ID)
+ tbl, _ := b.is.TableByID(context.Background(), ts.Table.ID)
isPartition, physicalTableID := ts.IsPartition()
if isPartition {
pt := tbl.(table.PartitionedTable)
@@ -3573,7 +3576,7 @@ func (b *executorBuilder) buildMPPGather(v *plannercore.PhysicalTableReader) exe
if hasVirtualCol {
gather.virtualColumnIndex, gather.virtualColumnRetFieldTypes = buildVirtualColumnInfo(gather.Schema(), gather.columns)
}
- tbl, _ := b.is.TableByID(ts.Table.ID)
+ tbl, _ := b.is.TableByID(context.Background(), ts.Table.ID)
isPartition, physicalTableID := ts.IsPartition()
if isPartition {
// Only for static pruning partition table.
@@ -3648,7 +3651,7 @@ func (b *executorBuilder) buildTableReader(v *plannercore.PhysicalTableReader) e
return ret
}
- tmp, _ := b.is.TableByID(ts.Table.ID)
+ tmp, _ := b.is.TableByID(context.Background(), ts.Table.ID)
tbl := tmp.(table.PartitionedTable)
partitions, err := partitionPruning(b.ctx, tbl, v.PlanPartInfo)
if err != nil {
@@ -3800,7 +3803,7 @@ func buildNoRangeIndexReader(b *executorBuilder, v *plannercore.PhysicalIndexRea
return nil, err
}
is := v.IndexPlans[0].(*plannercore.PhysicalIndexScan)
- tbl, _ := b.is.TableByID(is.Table.ID)
+ tbl, _ := b.is.TableByID(context.Background(), is.Table.ID)
isPartition, physicalTableID := is.IsPartition()
if isPartition {
pt := tbl.(table.PartitionedTable)
@@ -3890,7 +3893,7 @@ func (b *executorBuilder) buildIndexReader(v *plannercore.PhysicalIndexReader) e
return ret
}
- tmp, _ := b.is.TableByID(is.Table.ID)
+ tmp, _ := b.is.TableByID(context.Background(), is.Table.ID)
tbl := tmp.(table.PartitionedTable)
partitions, err := partitionPruning(b.ctx, tbl, v.PlanPartInfo)
if err != nil {
@@ -3910,7 +3913,7 @@ func buildTableReq(b *executorBuilder, schemaLen int, plans []base.PhysicalPlan)
tableReq.OutputOffsets = append(tableReq.OutputOffsets, uint32(i))
}
ts := plans[0].(*plannercore.PhysicalTableScan)
- tbl, _ := b.is.TableByID(ts.Table.ID)
+ tbl, _ := b.is.TableByID(context.Background(), ts.Table.ID)
isPartition, physicalTableID := ts.IsPartition()
if isPartition {
pt := tbl.(table.PartitionedTable)
@@ -4081,7 +4084,7 @@ func (b *executorBuilder) buildIndexLookUpReader(v *plannercore.PhysicalIndexLoo
return ret
}
- tmp, _ := b.is.TableByID(is.Table.ID)
+ tmp, _ := b.is.TableByID(context.Background(), is.Table.ID)
tbl := tmp.(table.PartitionedTable)
partitions, err := partitionPruning(b.ctx, tbl, v.PlanPartInfo)
if err != nil {
@@ -4237,7 +4240,7 @@ func (b *executorBuilder) buildIndexMergeReader(v *plannercore.PhysicalIndexMerg
return ret
}
- tmp, _ := b.is.TableByID(ts.Table.ID)
+ tmp, _ := b.is.TableByID(context.Background(), ts.Table.ID)
partitions, err := partitionPruning(b.ctx, tmp.(table.PartitionedTable), v.PlanPartInfo)
if err != nil {
b.err = err
@@ -4383,7 +4386,7 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte
handles, _ := dedupHandles(lookUpContents)
return builder.buildTableReaderFromHandles(ctx, e, handles, canReorderHandles)
}
- tbl, _ := builder.is.TableByID(tbInfo.ID)
+ tbl, _ := builder.is.TableByID(ctx, tbInfo.ID)
pt := tbl.(table.PartitionedTable)
usedPartitionList, err := builder.partitionPruning(pt, v.PlanPartInfo)
if err != nil {
@@ -4655,7 +4658,7 @@ func (builder *dataReaderBuilder) buildIndexReaderForIndexJoin(ctx context.Conte
return e, nil
}
- tbl, _ := builder.executorBuilder.is.TableByID(tbInfo.ID)
+ tbl, _ := builder.executorBuilder.is.TableByID(ctx, tbInfo.ID)
usedPartition, canPrune, contentPos, err := builder.prunePartitionForInnerExecutor(tbl, v.PlanPartInfo, lookUpContents)
if err != nil {
return nil, err
@@ -4718,7 +4721,7 @@ func (builder *dataReaderBuilder) buildIndexLookUpReaderForIndexJoin(ctx context
return e, err
}
- tbl, _ := builder.executorBuilder.is.TableByID(tbInfo.ID)
+ tbl, _ := builder.executorBuilder.is.TableByID(ctx, tbInfo.ID)
usedPartition, canPrune, contentPos, err := builder.prunePartitionForInnerExecutor(tbl, v.PlanPartInfo, lookUpContents)
if err != nil {
return nil, err
@@ -4956,16 +4959,21 @@ func (b *executorBuilder) buildWindow(v *plannercore.PhysicalWindow) exec.Execut
BaseExecutor: base,
groupChecker: vecgroupchecker.NewVecGroupChecker(b.ctx.GetExprCtx().GetEvalCtx(), b.ctx.GetSessionVars().EnableVectorizedExpression, groupByItems),
numWindowFuncs: len(v.WindowFuncDescs),
+ windowFuncs: windowFuncs,
+ partialResults: partialResults,
+ }
+ exec.slidingWindowFuncs = make([]aggfuncs.SlidingWindowAggFunc, len(exec.windowFuncs))
+ for i, windowFunc := range exec.windowFuncs {
+ if slidingWindowAggFunc, ok := windowFunc.(aggfuncs.SlidingWindowAggFunc); ok {
+ exec.slidingWindowFuncs[i] = slidingWindowAggFunc
+ }
}
-
- exec.windowFuncs = windowFuncs
- exec.partialResults = partialResults
if v.Frame == nil {
- exec.start = &plannercore.FrameBound{
+ exec.start = &logicalop.FrameBound{
Type: ast.Preceding,
UnBounded: true,
}
- exec.end = &plannercore.FrameBound{
+ exec.end = &logicalop.FrameBound{
Type: ast.Following,
UnBounded: true,
}
@@ -5121,18 +5129,9 @@ func (b *executorBuilder) buildSQLBindExec(v *plannercore.SQLBindPlan) exec.Exec
e := &SQLBindExec{
BaseExecutor: base,
- sqlBindOp: v.SQLBindOp,
- normdOrigSQL: v.NormdOrigSQL,
- bindSQL: v.BindSQL,
- charset: v.Charset,
- collation: v.Collation,
- db: v.Db,
isGlobal: v.IsGlobal,
- bindAst: v.BindStmt,
- newStatus: v.NewStatus,
- source: v.Source,
- sqlDigest: v.SQLDigest,
- planDigest: v.PlanDigest,
+ sqlBindOp: v.SQLBindOp,
+ details: v.Details,
}
return e
}
@@ -5590,7 +5589,7 @@ func (b *executorBuilder) validCanReadTemporaryTable(tbl *model.TableInfo) error
}
func (b *executorBuilder) getCacheTable(tblInfo *model.TableInfo, startTS uint64) kv.MemBuffer {
- tbl, ok := b.is.TableByID(tblInfo.ID)
+ tbl, ok := b.is.TableByID(context.Background(), tblInfo.ID)
if !ok {
b.err = errors.Trace(infoschema.ErrTableNotExists.GenWithStackByArgs(b.ctx.GetSessionVars().CurrentDB, tblInfo.Name))
return nil
diff --git a/pkg/executor/coprocessor.go b/pkg/executor/coprocessor.go
index 8487e547affb6..218d16627bd0f 100644
--- a/pkg/executor/coprocessor.go
+++ b/pkg/executor/coprocessor.go
@@ -256,7 +256,7 @@ func (h *CoprocessorDAGHandler) buildStreamResponse(chunk *tipb.Chunk) *coproces
func (*CoprocessorDAGHandler) buildErrorResponse(err error) *coprocessor.Response {
return &coprocessor.Response{
- OtherError: err.Error(),
+ OtherError: errors.ErrorStack(err),
}
}
diff --git a/pkg/executor/ddl.go b/pkg/executor/ddl.go
index 70cbfd5472e04..bf935d2f77bb6 100644
--- a/pkg/executor/ddl.go
+++ b/pkg/executor/ddl.go
@@ -408,7 +408,7 @@ func (e *DDLExec) executeRecoverTable(s *ast.RecoverTableStmt) error {
return err
}
// Check the table ID was not exists.
- tbl, ok := dom.InfoSchema().TableByID(tblInfo.ID)
+ tbl, ok := dom.InfoSchema().TableByID(context.Background(), tblInfo.ID)
if ok {
return infoschema.ErrTableExists.GenWithStack("Table '%-.192s' already been recover to '%-.192s', can't be recover repeatedly", s.Table.Name.O, tbl.Meta().Name.O)
}
@@ -463,14 +463,16 @@ func (e *DDLExec) getRecoverTableByJobID(s *ast.RecoverTableStmt, dom *domain.Do
return nil, nil, err
}
// Get table meta from snapshot infoSchema.
- table, ok := snapInfo.TableByID(job.TableID)
+ table, ok := snapInfo.TableByID(ctx, job.TableID)
if !ok {
return nil, nil, infoschema.ErrTableNotExists.GenWithStackByArgs(
fmt.Sprintf("(Schema ID %d)", job.SchemaID),
fmt.Sprintf("(Table ID %d)", job.TableID),
)
}
- return job, table.Meta(), nil
+ // Return the cloned meta here, since meta will be modified later.
+ // This may corrupt the infocache.
+ return job, table.Meta().Clone(), nil
}
// GetDropOrTruncateTableInfoFromJobs gets the dropped/truncated table information from DDL jobs,
@@ -563,7 +565,7 @@ func (e *DDLExec) executeFlashbackTable(s *ast.FlashBackTableStmt) error {
}
// Check the table ID was not exists.
is := domain.GetDomain(e.Ctx()).InfoSchema()
- tbl, ok := is.TableByID(tblInfo.ID)
+ tbl, ok := is.TableByID(context.Background(), tblInfo.ID)
if ok {
return infoschema.ErrTableExists.GenWithStack("Table '%-.192s' already been flashback to '%-.192s', can't be flashback repeatedly", s.Table.Name.O, tbl.Meta().Name.O)
}
diff --git a/pkg/executor/delete.go b/pkg/executor/delete.go
index 40b441c624dfc..ede6be8faa853 100644
--- a/pkg/executor/delete.go
+++ b/pkg/executor/delete.go
@@ -251,7 +251,12 @@ func (e *DeleteExec) removeRowsInTblRowMap(tblRowMap tableRowMapType) error {
}
func (e *DeleteExec) removeRow(ctx sessionctx.Context, t table.Table, h kv.Handle, data []types.Datum) error {
- err := t.RemoveRecord(ctx.GetTableCtx(), h, data)
+ txn, err := e.Ctx().Txn(true)
+ if err != nil {
+ return err
+ }
+
+ err = t.RemoveRecord(ctx.GetTableCtx(), txn, h, data)
if err != nil {
return err
}
diff --git a/pkg/executor/detach.go b/pkg/executor/detach.go
index 8baf207bf1f6c..727503c60e24a 100644
--- a/pkg/executor/detach.go
+++ b/pkg/executor/detach.go
@@ -16,6 +16,7 @@ package executor
import (
"github.com/pingcap/tidb/pkg/executor/internal/exec"
+ "github.com/pingcap/tidb/pkg/expression"
"github.com/pingcap/tidb/pkg/expression/contextsession"
)
@@ -86,6 +87,24 @@ func (iluCtx indexLookUpExecutorContext) Detach() indexLookUpExecutorContext {
return iluCtx
}
+func (pCtx projectionExecutorContext) Detach() projectionExecutorContext {
+ newCtx := pCtx
+ if ctx, ok := pCtx.evalCtx.(*contextsession.SessionEvalContext); ok {
+ newCtx.evalCtx = ctx.IntoStatic()
+ }
+
+ return newCtx
+}
+
+func (sCtx selectionExecutorContext) Detach() selectionExecutorContext {
+ newCtx := sCtx
+ if ctx, ok := sCtx.evalCtx.(*contextsession.SessionEvalContext); ok {
+ newCtx.evalCtx = ctx.IntoStatic()
+ }
+
+ return newCtx
+}
+
// Detach detaches the current executor from the session context.
func (e *TableReaderExecutor) Detach() (exec.Executor, bool) {
newExec := new(TableReaderExecutor)
@@ -115,3 +134,39 @@ func (e *IndexLookUpExecutor) Detach() (exec.Executor, bool) {
return newExec, true
}
+
+// Detach detaches the current executor from the session context.
+func (e *ProjectionExec) Detach() (exec.Executor, bool) {
+ // check whether the `Projection` requires any optional property
+ // Now, no optional property is copied, so if it requires any optional property, it should return false.
+ // TODO: some optional property can be detached. If they are implemented in the future, this check needs to be changed.
+ if !e.evaluatorSuit.RequiredOptionalEvalProps().IsEmpty() {
+ return nil, false
+ }
+
+ newExec := new(ProjectionExec)
+ *newExec = *e
+
+ newExec.projectionExecutorContext = newExec.projectionExecutorContext.Detach()
+
+ return newExec, true
+}
+
+// Detach detaches the current executor from the session context.
+func (e *SelectionExec) Detach() (exec.Executor, bool) {
+ // check whether the `Selection` requires any optional property
+ // Now, no optional property is copied, so if it requires any optional property, it should return false.
+ // TODO: some optional property can be detached. If they are implemented in the future, this check needs to be changed.
+ for _, expr := range e.filters {
+ if !expression.GetOptionalEvalPropsForExpr(expr).IsEmpty() {
+ return nil, false
+ }
+ }
+
+ newExec := new(SelectionExec)
+ *newExec = *e
+
+ newExec.selectionExecutorContext = newExec.selectionExecutorContext.Detach()
+
+ return newExec, true
+}
diff --git a/pkg/executor/detach_integration_test.go b/pkg/executor/detach_integration_test.go
index 7a8de51af47db..c4dd85f7ee790 100644
--- a/pkg/executor/detach_integration_test.go
+++ b/pkg/executor/detach_integration_test.go
@@ -231,3 +231,116 @@ func TestDetachIndexReaderAndIndexLookUp(t *testing.T) {
}
}
}
+
+func TestDetachSelection(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ tk.MustExec("use test")
+ tk.Session().GetSessionVars().SetStatusFlag(mysql.ServerStatusCursorExists, true)
+ tk.MustExec("create table t (a int, b int, c int, key idx_a_b (a,b), key idx_b (b))")
+ for i := 0; i < 10000; i++ {
+ tk.MustExec("insert into t values (?, ?, ?)", i, i, i)
+ }
+
+ tk.MustHavePlan("select a, b from t where c > 100 and c < 200", "Selection")
+ rs, err := tk.Exec("select a, b from t where c > ? and c < ?", 100, 200)
+ require.NoError(t, err)
+ drs, ok, err := rs.(sqlexec.DetachableRecordSet).TryDetach()
+ require.NoError(t, err)
+ require.True(t, ok)
+
+ chk := drs.NewChunk(nil)
+ expectedSelect := 101
+ for {
+ err = drs.Next(context.Background(), chk)
+ require.NoError(t, err)
+
+ if chk.NumRows() == 0 {
+ break
+ }
+ for i := 0; i < chk.NumRows(); i++ {
+ require.Equal(t, int64(expectedSelect), chk.GetRow(i).GetInt64(0))
+ require.Equal(t, int64(expectedSelect), chk.GetRow(i).GetInt64(1))
+ expectedSelect++
+ }
+ }
+
+ // Selection with optional property is not allowed
+ tk.MustExec("set @a = 1")
+ tk.MustHavePlan("select a from t where a + @a > 100 and a < 200", "Selection")
+ rs, err = tk.Exec("select a from t where a + @a > ? and a < ?", 100, 200)
+ require.NoError(t, err)
+ drs, ok, _ = rs.(sqlexec.DetachableRecordSet).TryDetach()
+ require.False(t, ok)
+ require.Nil(t, drs)
+ require.NoError(t, rs.Close())
+}
+
+func TestDetachProjection(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ tk.MustExec("use test")
+ tk.Session().GetSessionVars().SetStatusFlag(mysql.ServerStatusCursorExists, true)
+ tk.MustExec("create table t (a int, b int, c int, key idx_a_b (a,b), key idx_b (b))")
+ for i := 0; i < 10000; i++ {
+ tk.MustExec("insert into t values (?, ?, ?)", i, i, i)
+ }
+
+ tk.MustHavePlan("select a + b from t where a > 100 and a < 200", "Projection")
+ rs, err := tk.Exec("select a + b from t where a > ? and a < ?", 100, 200)
+ require.NoError(t, err)
+ drs, ok, err := rs.(sqlexec.DetachableRecordSet).TryDetach()
+ require.NoError(t, err)
+ require.True(t, ok)
+
+ chk := drs.NewChunk(nil)
+ expectedSelect := 101
+ for {
+ err = drs.Next(context.Background(), chk)
+ require.NoError(t, err)
+
+ if chk.NumRows() == 0 {
+ break
+ }
+ for i := 0; i < chk.NumRows(); i++ {
+ require.Equal(t, int64(2*expectedSelect), chk.GetRow(i).GetInt64(0))
+ expectedSelect++
+ }
+ }
+
+ // Projection with optional property is not allowed
+ tk.MustExec("set @a = 1")
+ tk.MustHavePlan("select a + @a from t where a > 100 and a < 200", "Projection")
+ rs, err = tk.Exec("select a + @a from t where a > ? and a < ?", 100, 200)
+ require.NoError(t, err)
+ drs, ok, _ = rs.(sqlexec.DetachableRecordSet).TryDetach()
+ require.False(t, ok)
+ require.Nil(t, drs)
+ require.NoError(t, rs.Close())
+
+ // Projection with Selection is also allowed
+ tk.MustHavePlan("select a + b from t where c > 100 and c < 200", "Projection")
+ tk.MustHavePlan("select a + b from t where c > 100 and c < 200", "Selection")
+ rs, err = tk.Exec("select a + b from t where c > ? and c < ?", 100, 200)
+ require.NoError(t, err)
+ drs, ok, err = rs.(sqlexec.DetachableRecordSet).TryDetach()
+ require.NoError(t, err)
+ require.True(t, ok)
+
+ chk = drs.NewChunk(nil)
+ expectedSelect = 101
+ for {
+ err = drs.Next(context.Background(), chk)
+ require.NoError(t, err)
+
+ if chk.NumRows() == 0 {
+ break
+ }
+ for i := 0; i < chk.NumRows(); i++ {
+ require.Equal(t, int64(2*expectedSelect), chk.GetRow(i).GetInt64(0))
+ expectedSelect++
+ }
+ }
+}
diff --git a/pkg/executor/executor.go b/pkg/executor/executor.go
index 104c06e6982f1..fad66b9050f9a 100644
--- a/pkg/executor/executor.go
+++ b/pkg/executor/executor.go
@@ -55,6 +55,7 @@ import (
planctx "github.com/pingcap/tidb/pkg/planner/context"
plannercore "github.com/pingcap/tidb/pkg/planner/core"
"github.com/pingcap/tidb/pkg/planner/core/base"
+ "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop"
plannerutil "github.com/pingcap/tidb/pkg/planner/util"
"github.com/pingcap/tidb/pkg/planner/util/fixcontrol"
"github.com/pingcap/tidb/pkg/privilege"
@@ -824,7 +825,7 @@ func getSchemaName(is infoschema.InfoSchema, id int64) string {
func getTableName(is infoschema.InfoSchema, id int64) string {
var tableName string
- table, ok := is.TableByID(id)
+ table, ok := is.TableByID(context.Background(), id)
if ok {
tableName = table.Meta().Name.O
return tableName
@@ -1150,8 +1151,8 @@ func (e *SelectLockExec) Next(ctx context.Context, req *chunk.Chunk) error {
if err != nil {
return err
}
- // If there's no handle or it's not a `SELECT FOR UPDATE` statement.
- if len(e.tblID2Handle) == 0 || (!plannercore.IsSelectForUpdateLockType(e.Lock.LockType)) {
+ // If there's no handle or it's not a `SELECT FOR UPDATE` or `SELECT FOR SHARE` statement.
+ if len(e.tblID2Handle) == 0 || (!logicalop.IsSupportedSelectLockType(e.Lock.LockType)) {
return nil
}
@@ -1184,7 +1185,7 @@ func (e *SelectLockExec) Next(ctx context.Context, req *chunk.Chunk) error {
return nil
}
lockWaitTime := e.Ctx().GetSessionVars().LockWaitTimeout
- if e.Lock.LockType == ast.SelectLockForUpdateNoWait {
+ if e.Lock.LockType == ast.SelectLockForUpdateNoWait || e.Lock.LockType == ast.SelectLockForShareNoWait {
lockWaitTime = tikvstore.LockNoWait
} else if e.Lock.LockType == ast.SelectLockForUpdateWaitN {
lockWaitTime = int64(e.Lock.WaitSec) * 1000
diff --git a/pkg/executor/executor_required_rows_test.go b/pkg/executor/executor_required_rows_test.go
index 734f67859636e..a81dda7e1512a 100644
--- a/pkg/executor/executor_required_rows_test.go
+++ b/pkg/executor/executor_required_rows_test.go
@@ -31,6 +31,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/mysql"
plannercore "github.com/pingcap/tidb/pkg/planner/core"
+ "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop"
"github.com/pingcap/tidb/pkg/planner/util"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
@@ -697,8 +698,8 @@ func TestMergeJoinRequiredRows(t *testing.T) {
panic("not support")
}
}
- joinTypes := []plannercore.JoinType{plannercore.RightOuterJoin, plannercore.LeftOuterJoin,
- plannercore.LeftOuterSemiJoin, plannercore.AntiLeftOuterSemiJoin}
+ joinTypes := []logicalop.JoinType{logicalop.RightOuterJoin, logicalop.LeftOuterJoin,
+ logicalop.LeftOuterSemiJoin, logicalop.AntiLeftOuterSemiJoin}
for _, joinType := range joinTypes {
ctx := defaultCtx()
required := make([]int, 100)
@@ -720,8 +721,8 @@ func TestMergeJoinRequiredRows(t *testing.T) {
}
}
-func buildMergeJoinExec(ctx sessionctx.Context, joinType plannercore.JoinType, innerSrc, outerSrc exec.Executor) exec.Executor {
- if joinType == plannercore.RightOuterJoin {
+func buildMergeJoinExec(ctx sessionctx.Context, joinType logicalop.JoinType, innerSrc, outerSrc exec.Executor) exec.Executor {
+ if joinType == logicalop.RightOuterJoin {
innerSrc, outerSrc = outerSrc, innerSrc
}
diff --git a/pkg/executor/historical_stats_test.go b/pkg/executor/historical_stats_test.go
index e5b659f2672f2..c448eb628c10f 100644
--- a/pkg/executor/historical_stats_test.go
+++ b/pkg/executor/historical_stats_test.go
@@ -75,6 +75,7 @@ func TestRecordHistoryStatsAfterAnalyze(t *testing.T) {
// 3. dump current stats json
dumpJSONTable, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil, true)
require.NoError(t, err)
+ dumpJSONTable.Sort()
jsOrigin, _ := json.Marshal(dumpJSONTable)
// 4. get the historical stats json
@@ -89,6 +90,7 @@ func TestRecordHistoryStatsAfterAnalyze(t *testing.T) {
}
jsonTbl, err := storage.BlocksToJSONTable(data)
require.NoError(t, err)
+ jsonTbl.Sort()
jsCur, err := json.Marshal(jsonTbl)
require.NoError(t, err)
// 5. historical stats must be equal to the current stats
diff --git a/pkg/executor/import_into.go b/pkg/executor/import_into.go
index 7d9a4f92efd95..c41d708d705cb 100644
--- a/pkg/executor/import_into.go
+++ b/pkg/executor/import_into.go
@@ -151,7 +151,7 @@ func (e *ImportIntoExec) fillJobInfo(ctx context.Context, jobID int64, req *chun
}); err != nil {
return err
}
- fillOneImportJobInfo(info, req, unknownImportedRowCount)
+ FillOneImportJobInfo(info, req, unknownImportedRowCount)
return nil
}
diff --git a/pkg/executor/importer/BUILD.bazel b/pkg/executor/importer/BUILD.bazel
index 4ae10013e1088..bfec2f2411196 100644
--- a/pkg/executor/importer/BUILD.bazel
+++ b/pkg/executor/importer/BUILD.bazel
@@ -44,6 +44,7 @@ go_library(
"//pkg/parser/model",
"//pkg/parser/mysql",
"//pkg/parser/terror",
+ "//pkg/planner/context",
"//pkg/planner/core",
"//pkg/planner/util",
"//pkg/sessionctx",
diff --git a/pkg/executor/importer/import.go b/pkg/executor/importer/import.go
index e95eb79faf97d..be3dfb5f434e6 100644
--- a/pkg/executor/importer/import.go
+++ b/pkg/executor/importer/import.go
@@ -45,6 +45,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/parser/terror"
+ planctx "github.com/pingcap/tidb/pkg/planner/context"
plannercore "github.com/pingcap/tidb/pkg/planner/core"
plannerutil "github.com/pingcap/tidb/pkg/planner/util"
"github.com/pingcap/tidb/pkg/sessionctx"
@@ -1291,17 +1292,17 @@ func (p *Plan) IsGlobalSort() bool {
// CreateColAssignExprs creates the column assignment expressions using session context.
// RewriteAstExpr will write ast node in place(due to xxNode.Accept), but it doesn't change node content,
// so we sync it.
-func (e *LoadDataController) CreateColAssignExprs(sctx sessionctx.Context) ([]expression.Expression, []contextutil.SQLWarn, error) {
+func (e *LoadDataController) CreateColAssignExprs(planCtx planctx.PlanContext) ([]expression.Expression, []contextutil.SQLWarn, error) {
e.colAssignMu.Lock()
defer e.colAssignMu.Unlock()
res := make([]expression.Expression, 0, len(e.ColumnAssignments))
allWarnings := []contextutil.SQLWarn{}
for _, assign := range e.ColumnAssignments {
- newExpr, err := plannerutil.RewriteAstExprWithPlanCtx(sctx.GetPlanCtx(), assign.Expr, nil, nil, false)
+ newExpr, err := plannerutil.RewriteAstExprWithPlanCtx(planCtx, assign.Expr, nil, nil, false)
// col assign expr warnings is static, we should generate it for each row processed.
// so we save it and clear it here.
- allWarnings = append(allWarnings, sctx.GetSessionVars().StmtCtx.GetWarnings()...)
- sctx.GetSessionVars().StmtCtx.SetWarnings(nil)
+ allWarnings = append(allWarnings, planCtx.GetSessionVars().StmtCtx.GetWarnings()...)
+ planCtx.GetSessionVars().StmtCtx.SetWarnings(nil)
if err != nil {
return nil, nil, err
}
diff --git a/pkg/executor/importer/kv_encode.go b/pkg/executor/importer/kv_encode.go
index a7ea8d89b59e0..6a6d66af42671 100644
--- a/pkg/executor/importer/kv_encode.go
+++ b/pkg/executor/importer/kv_encode.go
@@ -27,7 +27,6 @@ import (
"github.com/pingcap/tidb/pkg/meta/autoid"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/mysql" //nolint: goimports
- "github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/chunk"
@@ -64,8 +63,8 @@ func NewTableKVEncoder(
return nil, err
}
// we need a non-nil TxnCtx to avoid panic when evaluating set clause
- baseKVEncoder.SessionCtx.Vars.TxnCtx = new(variable.TransactionContext)
- colAssignExprs, _, err := ti.CreateColAssignExprs(baseKVEncoder.SessionCtx)
+ baseKVEncoder.SessionCtx.SetTxnCtxNotNil()
+ colAssignExprs, _, err := ti.CreateColAssignExprs(baseKVEncoder.SessionCtx.GetPlanCtx())
if err != nil {
return nil, err
}
@@ -95,24 +94,20 @@ func (en *tableKVEncoder) Encode(row []types.Datum, rowID int64) (*kv.Pairs, err
}
func (en *tableKVEncoder) GetColumnSize() map[int64]int64 {
- sessionVars := en.SessionCtx.GetSessionVars()
- sessionVars.TxnCtxMu.Lock()
- defer sessionVars.TxnCtxMu.Unlock()
- return sessionVars.TxnCtx.TableDeltaMap[en.TableMeta().ID].ColSize
+ return en.SessionCtx.GetColumnSize(en.TableMeta().ID)
}
// todo merge with code in load_data.go
func (en *tableKVEncoder) parserData2TableData(parserData []types.Datum, rowID int64) ([]types.Datum, error) {
row := make([]types.Datum, 0, len(en.insertColumns))
- sessionVars := en.SessionCtx.GetSessionVars()
setVar := func(name string, col *types.Datum) {
// User variable names are not case-sensitive
// https://dev.mysql.com/doc/refman/8.0/en/user-variables.html
name = strings.ToLower(name)
if col == nil || col.IsNull() {
- sessionVars.UnsetUserVar(name)
+ en.SessionCtx.UnsetUserVar(name)
} else {
- sessionVars.SetUserVarVal(name, *col)
+ en.SessionCtx.SetUserVarVal(name, *col)
}
}
@@ -166,7 +161,7 @@ func (en *tableKVEncoder) getRow(vals []types.Datum, rowID int64) ([]types.Datum
row := make([]types.Datum, len(en.Columns))
hasValue := make([]bool, len(en.Columns))
for i := 0; i < len(en.insertColumns); i++ {
- casted, err := table.CastValue(en.SessionCtx, vals[i], en.insertColumns[i].ToInfo(), false, false)
+ casted, err := table.CastColumnValue(en.SessionCtx.GetExprCtx(), vals[i], en.insertColumns[i].ToInfo(), false, false)
if err != nil {
return nil, err
}
diff --git a/pkg/executor/infoschema_cluster_table_test.go b/pkg/executor/infoschema_cluster_table_test.go
index 89cbfc62aecb4..cb63dfed7d6a6 100644
--- a/pkg/executor/infoschema_cluster_table_test.go
+++ b/pkg/executor/infoschema_cluster_table_test.go
@@ -348,7 +348,7 @@ func TestTikvRegionStatus(t *testing.T) {
"1 test test_t2 1 p_a 1 p1",
))
- tk.MustExec("alter table test_t2 add unique p_b (b);")
+ tk.MustExec("alter table test_t2 add unique p_b (b) global")
tk.MustQuery("select REGION_ID, DB_NAME, TABLE_NAME, IS_INDEX, INDEX_NAME, IS_PARTITION, PARTITION_NAME from information_schema.TIKV_REGION_STATUS where DB_NAME = 'test' and TABLE_NAME = 'test_t2' order by IS_INDEX, IS_PARTITION desc, PARTITION_NAME").Check(testkit.Rows(
"1 test test_t2 0 1 p0",
"1 test test_t2 0 1 p1",
diff --git a/pkg/executor/infoschema_reader.go b/pkg/executor/infoschema_reader.go
index 0356f6e5e7cd1..2e29e9aba12d0 100644
--- a/pkg/executor/infoschema_reader.go
+++ b/pkg/executor/infoschema_reader.go
@@ -47,7 +47,6 @@ import (
"github.com/pingcap/tidb/pkg/parser/charset"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
- "github.com/pingcap/tidb/pkg/parser/terror"
plannercore "github.com/pingcap/tidb/pkg/planner/core"
"github.com/pingcap/tidb/pkg/planner/core/base"
"github.com/pingcap/tidb/pkg/privilege"
@@ -87,6 +86,8 @@ import (
"go.uber.org/zap"
)
+var lowerPrimaryKeyName = strings.ToLower(mysql.PrimaryKeyName)
+
type memtableRetriever struct {
dummyCloser
table *model.TableInfo
@@ -113,32 +114,37 @@ func (e *memtableRetriever) retrieve(ctx context.Context, sctx sessionctx.Contex
if !e.initialized {
is := sctx.GetInfoSchema().(infoschema.InfoSchema)
e.is = is
- dbs := is.AllSchemaNames()
- slices.SortFunc(dbs, func(a, b model.CIStr) int {
- return strings.Compare(a.L, b.L)
- })
+
+ var getAllSchemas = func() []model.CIStr {
+ dbs := is.AllSchemaNames()
+ slices.SortFunc(dbs, func(a, b model.CIStr) int {
+ return strings.Compare(a.L, b.L)
+ })
+ return dbs
+ }
+
var err error
switch e.table.Name.O {
case infoschema.TableSchemata:
- e.setDataFromSchemata(sctx, dbs)
+ err = e.setDataFromSchemata(sctx)
case infoschema.TableStatistics:
- err = e.setDataForStatistics(ctx, sctx, dbs)
+ err = e.setDataForStatistics(ctx, sctx)
case infoschema.TableTables:
- err = e.setDataFromTables(ctx, sctx, dbs)
+ err = e.setDataFromTables(ctx, sctx)
case infoschema.TableReferConst:
- err = e.setDataFromReferConst(ctx, sctx, dbs)
+ err = e.setDataFromReferConst(ctx, sctx)
case infoschema.TableSequences:
- err = e.setDataFromSequences(ctx, sctx, dbs)
+ err = e.setDataFromSequences(ctx, sctx)
case infoschema.TablePartitions:
- err = e.setDataFromPartitions(ctx, sctx, dbs)
+ err = e.setDataFromPartitions(ctx, sctx)
case infoschema.TableClusterInfo:
err = e.dataForTiDBClusterInfo(sctx)
case infoschema.TableAnalyzeStatus:
err = e.setDataForAnalyzeStatus(ctx, sctx)
case infoschema.TableTiDBIndexes:
- err = e.setDataFromIndexes(ctx, sctx, dbs)
+ err = e.setDataFromIndexes(ctx, sctx)
case infoschema.TableViews:
- err = e.setDataFromViews(ctx, sctx, dbs)
+ err = e.setDataFromViews(ctx, sctx)
case infoschema.TableEngines:
e.setDataFromEngines()
case infoschema.TableCharacterSets:
@@ -146,7 +152,7 @@ func (e *memtableRetriever) retrieve(ctx context.Context, sctx sessionctx.Contex
case infoschema.TableCollations:
e.setDataFromCollations()
case infoschema.TableKeyColumn:
- err = e.setDataFromKeyColumnUsage(ctx, sctx, dbs)
+ err = e.setDataFromKeyColumnUsage(ctx, sctx)
case infoschema.TableMetricTables:
e.setDataForMetricTables()
case infoschema.TableProfiling:
@@ -164,12 +170,13 @@ func (e *memtableRetriever) retrieve(ctx context.Context, sctx sessionctx.Contex
case infoschema.TableTiDBHotRegions:
err = e.setDataForTiDBHotRegions(ctx, sctx)
case infoschema.TableConstraints:
- err = e.setDataFromTableConstraints(ctx, sctx, dbs)
+ err = e.setDataFromTableConstraints(ctx, sctx)
case infoschema.TableSessionVar:
e.rows, err = infoschema.GetDataFromSessionVariables(ctx, sctx)
case infoschema.TableTiDBServersInfo:
err = e.setDataForServersInfo(sctx)
case infoschema.TableTiFlashReplica:
+ dbs := getAllSchemas()
err = e.dataForTableTiFlashReplica(ctx, sctx, dbs)
case infoschema.TableTiKVStoreStatus:
err = e.dataForTiKVStoreStatus(ctx, sctx)
@@ -202,15 +209,15 @@ func (e *memtableRetriever) retrieve(ctx context.Context, sctx sessionctx.Contex
case infoschema.TableRunawayWatches:
err = e.setDataFromRunawayWatches(sctx)
case infoschema.TableCheckConstraints:
- err = e.setDataFromCheckConstraints(ctx, sctx, dbs)
+ err = e.setDataFromCheckConstraints(ctx, sctx)
case infoschema.TableTiDBCheckConstraints:
- err = e.setDataFromTiDBCheckConstraints(ctx, sctx, dbs)
+ err = e.setDataFromTiDBCheckConstraints(ctx, sctx)
case infoschema.TableKeywords:
err = e.setDataFromKeywords()
case infoschema.TableTiDBIndexUsage:
- err = e.setDataFromIndexUsage(ctx, sctx, dbs)
+ err = e.setDataFromIndexUsage(ctx, sctx)
case infoschema.ClusterTableTiDBIndexUsage:
- err = e.setDataForClusterIndexUsage(ctx, sctx, dbs)
+ err = e.setDataFromClusterIndexUsage(ctx, sctx)
}
if err != nil {
return nil, err
@@ -236,13 +243,16 @@ func (e *memtableRetriever) retrieve(ctx context.Context, sctx sessionctx.Contex
return adjustColumns(ret, e.columns, e.table), nil
}
-func getAutoIncrementID(ctx context.Context, sctx sessionctx.Context, schema model.CIStr, tblInfo *model.TableInfo) (int64, error) {
- is := sctx.GetInfoSchema().(infoschema.InfoSchema)
- tbl, err := is.TableByName(ctx, schema, tblInfo.Name)
- if err != nil {
- return 0, err
+func getAutoIncrementID(
+ is infoschema.InfoSchema,
+ sctx sessionctx.Context,
+ tblInfo *model.TableInfo,
+) int64 {
+ tbl, ok := is.TableByID(context.Background(), tblInfo.ID)
+ if !ok {
+ return 0
}
- return tbl.Allocators(sctx.GetTableCtx()).Get(autoid.AutoIncrementType).Base() + 1, nil
+ return tbl.Allocators(sctx.GetTableCtx()).Get(autoid.AutoIncrementType).Base() + 1
}
func hasPriv(ctx sessionctx.Context, priv mysql.PrivilegeType) bool {
@@ -336,18 +346,19 @@ func (e *memtableRetriever) setDataForUserAttributes(ctx context.Context, sctx s
return nil
}
-func (e *memtableRetriever) setDataFromSchemata(ctx sessionctx.Context, schemas []model.CIStr) {
+func (e *memtableRetriever) setDataFromSchemata(ctx sessionctx.Context) error {
checker := privilege.GetPrivilegeManager(ctx)
- extractor, ok := e.extractor.(*plannercore.InfoSchemaTablesExtractor)
- if ok && extractor.SkipRequest {
- return
+ ex, ok := e.extractor.(*plannercore.InfoSchemaSchemataExtractor)
+ if !ok {
+ return errors.Errorf("wrong extractor type: %T, expected InfoSchemaSchemataExtractor", e.extractor)
+ }
+ if ex.SkipRequest {
+ return nil
}
+ schemas := ex.ListSchemas(e.is)
rows := make([][]types.Datum, 0, len(schemas))
for _, schemaName := range schemas {
- if ok && extractor.Filter("schema_name", schemaName.L) {
- continue
- }
schema, _ := e.is.SchemaByName(schemaName)
charset := mysql.DefaultCharset
collation := mysql.DefaultCollationName
@@ -378,36 +389,33 @@ func (e *memtableRetriever) setDataFromSchemata(ctx sessionctx.Context, schemas
rows = append(rows, record)
}
e.rows = rows
+ return nil
}
-func (e *memtableRetriever) setDataForStatistics(ctx context.Context, sctx sessionctx.Context, schemas []model.CIStr) error {
+func (e *memtableRetriever) setDataForStatistics(ctx context.Context, sctx sessionctx.Context) error {
checker := privilege.GetPrivilegeManager(sctx)
- extractor, ok := e.extractor.(*plannercore.InfoSchemaTablesExtractor)
- if ok && extractor.SkipRequest {
+ ex, ok := e.extractor.(*plannercore.InfoSchemaStatisticsExtractor)
+ if !ok {
+ return errors.Errorf("wrong extractor type: %T, expected InfoSchemaStatisticsExtractor", e.extractor)
+ }
+ if ex.SkipRequest {
return nil
}
- for _, schema := range schemas {
- if ok && extractor.Filter("table_schema", schema.L) {
+ schemas, tables, err := ex.ListSchemasAndTables(ctx, e.is)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ for i, table := range tables {
+ schema := schemas[i]
+ if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, table.Name.L, "", mysql.AllPrivMask) {
continue
}
- tables, err := e.is.SchemaTableInfos(ctx, schema)
- if err != nil {
- return errors.Trace(err)
- }
- for _, table := range tables {
- if ok && extractor.Filter("table_name", table.Name.L) {
- continue
- }
- if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, table.Name.L, "", mysql.AllPrivMask) {
- continue
- }
- e.setDataForStatisticsInTable(schema, table, extractor)
- }
+ e.setDataForStatisticsInTable(schema, table, ex)
}
return nil
}
-func (e *memtableRetriever) setDataForStatisticsInTable(schema model.CIStr, table *model.TableInfo, extractor *plannercore.InfoSchemaTablesExtractor) {
+func (e *memtableRetriever) setDataForStatisticsInTable(schema model.CIStr, table *model.TableInfo, extractor *plannercore.InfoSchemaStatisticsExtractor) {
var rows [][]types.Datum
if table.PKIsHandle {
if !extractor.Filter("index_name", "primary") {
@@ -497,57 +505,53 @@ func (e *memtableRetriever) setDataForStatisticsInTable(schema model.CIStr, tabl
e.rows = append(e.rows, rows...)
}
-func (e *memtableRetriever) setDataFromReferConst(ctx context.Context, sctx sessionctx.Context, schemas []model.CIStr) error {
+func (e *memtableRetriever) setDataFromReferConst(ctx context.Context, sctx sessionctx.Context) error {
checker := privilege.GetPrivilegeManager(sctx)
var rows [][]types.Datum
- extractor, ok := e.extractor.(*plannercore.InfoSchemaTablesExtractor)
- if ok && extractor.SkipRequest {
+ ex, ok := e.extractor.(*plannercore.InfoSchemaReferConstExtractor)
+ if !ok {
+ return errors.Errorf("wrong extractor type: %T, expected InfoSchemaReferConstExtractor", e.extractor)
+ }
+ if ex.SkipRequest {
return nil
}
- for _, schema := range schemas {
- if ok && extractor.Filter("constraint_schema", schema.L) {
+ schemas, tables, err := ex.ListSchemasAndTables(ctx, e.is)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ for i, table := range tables {
+ schema := schemas[i]
+ if !table.IsBaseTable() {
continue
}
- tables, err := e.is.SchemaTableInfos(ctx, schema)
- if err != nil {
- return errors.Trace(err)
+ if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, table.Name.L, "", mysql.AllPrivMask) {
+ continue
}
- for _, table := range tables {
- if ok && extractor.Filter("table_name", table.Name.L) {
+ for _, fk := range table.ForeignKeys {
+ if ok && ex.Filter("constraint_name", fk.Name.L) {
continue
}
- if !table.IsBaseTable() {
- continue
+ updateRule, deleteRule := "NO ACTION", "NO ACTION"
+ if model.ReferOptionType(fk.OnUpdate) != 0 {
+ updateRule = model.ReferOptionType(fk.OnUpdate).String()
}
- if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, table.Name.L, "", mysql.AllPrivMask) {
- continue
- }
- for _, fk := range table.ForeignKeys {
- if ok && extractor.Filter("constraint_name", fk.Name.L) {
- continue
- }
- updateRule, deleteRule := "NO ACTION", "NO ACTION"
- if model.ReferOptionType(fk.OnUpdate) != 0 {
- updateRule = model.ReferOptionType(fk.OnUpdate).String()
- }
- if model.ReferOptionType(fk.OnDelete) != 0 {
- deleteRule = model.ReferOptionType(fk.OnDelete).String()
- }
- record := types.MakeDatums(
- infoschema.CatalogVal, // CONSTRAINT_CATALOG
- schema.O, // CONSTRAINT_SCHEMA
- fk.Name.O, // CONSTRAINT_NAME
- infoschema.CatalogVal, // UNIQUE_CONSTRAINT_CATALOG
- schema.O, // UNIQUE_CONSTRAINT_SCHEMA
- "PRIMARY", // UNIQUE_CONSTRAINT_NAME
- "NONE", // MATCH_OPTION
- updateRule, // UPDATE_RULE
- deleteRule, // DELETE_RULE
- table.Name.O, // TABLE_NAME
- fk.RefTable.O, // REFERENCED_TABLE_NAME
- )
- rows = append(rows, record)
+ if model.ReferOptionType(fk.OnDelete) != 0 {
+ deleteRule = model.ReferOptionType(fk.OnDelete).String()
}
+ record := types.MakeDatums(
+ infoschema.CatalogVal, // CONSTRAINT_CATALOG
+ schema.O, // CONSTRAINT_SCHEMA
+ fk.Name.O, // CONSTRAINT_NAME
+ infoschema.CatalogVal, // UNIQUE_CONSTRAINT_CATALOG
+ schema.O, // UNIQUE_CONSTRAINT_SCHEMA
+ "PRIMARY", // UNIQUE_CONSTRAINT_NAME
+ "NONE", // MATCH_OPTION
+ updateRule, // UPDATE_RULE
+ deleteRule, // DELETE_RULE
+ table.Name.O, // TABLE_NAME
+ fk.RefTable.O, // REFERENCED_TABLE_NAME
+ )
+ rows = append(rows, record)
}
}
e.rows = rows
@@ -564,153 +568,160 @@ func (e *memtableRetriever) updateStatsCacheIfNeed() bool {
return false
}
-func (e *memtableRetriever) setDataFromTables(ctx context.Context, sctx sessionctx.Context, schemas []model.CIStr) error {
+func (e *memtableRetriever) setDataFromOneTable(
+ sctx sessionctx.Context,
+ loc *time.Location,
+ checker privilege.Manager,
+ schema model.CIStr,
+ table *model.TableInfo,
+ rows [][]types.Datum,
+ useStatsCache bool,
+) ([][]types.Datum, error) {
+ collation := table.Collate
+ if collation == "" {
+ collation = mysql.DefaultCollationName
+ }
+ createTime := types.NewTime(types.FromGoTime(table.GetUpdateTime().In(loc)), mysql.TypeDatetime, types.DefaultFsp)
+
+ createOptions := ""
+
+ if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, table.Name.L, "", mysql.AllPrivMask) {
+ return rows, nil
+ }
+ pkType := "NONCLUSTERED"
+ if !table.IsView() {
+ if table.GetPartitionInfo() != nil {
+ createOptions = "partitioned"
+ } else if table.TableCacheStatusType == model.TableCacheStatusEnable {
+ createOptions = "cached=on"
+ }
+ var autoIncID any
+ hasAutoIncID, _ := infoschema.HasAutoIncrementColumn(table)
+ if hasAutoIncID {
+ autoIncID = getAutoIncrementID(e.is, sctx, table)
+ }
+ tableType := "BASE TABLE"
+ if util.IsSystemView(schema.L) {
+ tableType = "SYSTEM VIEW"
+ }
+ if table.IsSequence() {
+ tableType = "SEQUENCE"
+ }
+ if table.HasClusteredIndex() {
+ pkType = "CLUSTERED"
+ }
+ shardingInfo := infoschema.GetShardingInfo(schema, table)
+ var policyName any
+ if table.PlacementPolicyRef != nil {
+ policyName = table.PlacementPolicyRef.Name.O
+ }
+
+ var rowCount, avgRowLength, dataLength, indexLength uint64
+ if useStatsCache {
+ if table.GetPartitionInfo() == nil {
+ err := cache.TableRowStatsCache.UpdateByID(sctx, table.ID)
+ if err != nil {
+ return rows, err
+ }
+ } else {
+ // needs to update all partitions for partition table.
+ for _, pi := range table.GetPartitionInfo().Definitions {
+ err := cache.TableRowStatsCache.UpdateByID(sctx, pi.ID)
+ if err != nil {
+ return rows, err
+ }
+ }
+ }
+ rowCount, avgRowLength, dataLength, indexLength = cache.TableRowStatsCache.EstimateDataLength(table)
+ }
+
+ record := types.MakeDatums(
+ infoschema.CatalogVal, // TABLE_CATALOG
+ schema.O, // TABLE_SCHEMA
+ table.Name.O, // TABLE_NAME
+ tableType, // TABLE_TYPE
+ "InnoDB", // ENGINE
+ uint64(10), // VERSION
+ "Compact", // ROW_FORMAT
+ rowCount, // TABLE_ROWS
+ avgRowLength, // AVG_ROW_LENGTH
+ dataLength, // DATA_LENGTH
+ uint64(0), // MAX_DATA_LENGTH
+ indexLength, // INDEX_LENGTH
+ uint64(0), // DATA_FREE
+ autoIncID, // AUTO_INCREMENT
+ createTime, // CREATE_TIME
+ nil, // UPDATE_TIME
+ nil, // CHECK_TIME
+ collation, // TABLE_COLLATION
+ nil, // CHECKSUM
+ createOptions, // CREATE_OPTIONS
+ table.Comment, // TABLE_COMMENT
+ table.ID, // TIDB_TABLE_ID
+ shardingInfo, // TIDB_ROW_ID_SHARDING_INFO
+ pkType, // TIDB_PK_TYPE
+ policyName, // TIDB_PLACEMENT_POLICY_NAME
+ )
+ rows = append(rows, record)
+ } else {
+ record := types.MakeDatums(
+ infoschema.CatalogVal, // TABLE_CATALOG
+ schema.O, // TABLE_SCHEMA
+ table.Name.O, // TABLE_NAME
+ "VIEW", // TABLE_TYPE
+ nil, // ENGINE
+ nil, // VERSION
+ nil, // ROW_FORMAT
+ nil, // TABLE_ROWS
+ nil, // AVG_ROW_LENGTH
+ nil, // DATA_LENGTH
+ nil, // MAX_DATA_LENGTH
+ nil, // INDEX_LENGTH
+ nil, // DATA_FREE
+ nil, // AUTO_INCREMENT
+ createTime, // CREATE_TIME
+ nil, // UPDATE_TIME
+ nil, // CHECK_TIME
+ nil, // TABLE_COLLATION
+ nil, // CHECKSUM
+ nil, // CREATE_OPTIONS
+ "VIEW", // TABLE_COMMENT
+ table.ID, // TIDB_TABLE_ID
+ nil, // TIDB_ROW_ID_SHARDING_INFO
+ pkType, // TIDB_PK_TYPE
+ nil, // TIDB_PLACEMENT_POLICY_NAME
+ )
+ rows = append(rows, record)
+ }
+ return rows, nil
+}
+
+func (e *memtableRetriever) setDataFromTables(ctx context.Context, sctx sessionctx.Context) error {
useStatsCache := e.updateStatsCacheIfNeed()
checker := privilege.GetPrivilegeManager(sctx)
var rows [][]types.Datum
- createTimeTp := mysql.TypeDatetime
loc := sctx.GetSessionVars().TimeZone
if loc == nil {
loc = time.Local
}
- extractor, ok := e.extractor.(*plannercore.InfoSchemaTablesExtractor)
- if ok && extractor.SkipRequest {
+ ex, ok := e.extractor.(*plannercore.InfoSchemaTablesExtractor)
+ if !ok {
+ return errors.Errorf("wrong extractor type: %T, expected InfoSchemaTablesExtractor", e.extractor)
+ }
+ if ex.SkipRequest {
return nil
}
- for _, schema := range schemas {
- if ok && extractor.Filter("table_schema", schema.L) {
- continue
- }
- tables, err := e.is.SchemaTableInfos(ctx, schema)
+
+ schemas, tables, err := ex.ListSchemasAndTables(ctx, e.is)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ for i, table := range tables {
+ rows, err = e.setDataFromOneTable(sctx, loc, checker, schemas[i], table, rows, useStatsCache)
if err != nil {
return errors.Trace(err)
}
- for _, table := range tables {
- if ok && extractor.Filter("table_name", table.Name.L) {
- continue
- }
- collation := table.Collate
- if collation == "" {
- collation = mysql.DefaultCollationName
- }
- createTime := types.NewTime(types.FromGoTime(table.GetUpdateTime().In(loc)), createTimeTp, types.DefaultFsp)
-
- createOptions := ""
-
- if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, table.Name.L, "", mysql.AllPrivMask) {
- continue
- }
- pkType := "NONCLUSTERED"
- if !table.IsView() {
- if table.GetPartitionInfo() != nil {
- createOptions = "partitioned"
- } else if table.TableCacheStatusType == model.TableCacheStatusEnable {
- createOptions = "cached=on"
- }
- var err error
- var autoIncID any
- hasAutoIncID, _ := infoschema.HasAutoIncrementColumn(table)
- if hasAutoIncID {
- autoIncID, err = getAutoIncrementID(ctx, sctx, schema, table)
- if err != nil {
- return err
- }
- }
- tableType := "BASE TABLE"
- if util.IsSystemView(schema.L) {
- tableType = "SYSTEM VIEW"
- }
- if table.IsSequence() {
- tableType = "SEQUENCE"
- }
- if table.HasClusteredIndex() {
- pkType = "CLUSTERED"
- }
- shardingInfo := infoschema.GetShardingInfo(schema, table)
- var policyName any
- if table.PlacementPolicyRef != nil {
- policyName = table.PlacementPolicyRef.Name.O
- }
-
- var rowCount, avgRowLength, dataLength, indexLength uint64
- if useStatsCache {
- if table.GetPartitionInfo() == nil {
- err := cache.TableRowStatsCache.UpdateByID(sctx, table.ID)
- if err != nil {
- return err
- }
- } else {
- // needs to update all partitions for partition table.
- for _, pi := range table.GetPartitionInfo().Definitions {
- err := cache.TableRowStatsCache.UpdateByID(sctx, pi.ID)
- if err != nil {
- return err
- }
- }
- }
- rowCount, avgRowLength, dataLength, indexLength = cache.TableRowStatsCache.EstimateDataLength(table)
- }
-
- record := types.MakeDatums(
- infoschema.CatalogVal, // TABLE_CATALOG
- schema.O, // TABLE_SCHEMA
- table.Name.O, // TABLE_NAME
- tableType, // TABLE_TYPE
- "InnoDB", // ENGINE
- uint64(10), // VERSION
- "Compact", // ROW_FORMAT
- rowCount, // TABLE_ROWS
- avgRowLength, // AVG_ROW_LENGTH
- dataLength, // DATA_LENGTH
- uint64(0), // MAX_DATA_LENGTH
- indexLength, // INDEX_LENGTH
- uint64(0), // DATA_FREE
- autoIncID, // AUTO_INCREMENT
- createTime, // CREATE_TIME
- nil, // UPDATE_TIME
- nil, // CHECK_TIME
- collation, // TABLE_COLLATION
- nil, // CHECKSUM
- createOptions, // CREATE_OPTIONS
- table.Comment, // TABLE_COMMENT
- table.ID, // TIDB_TABLE_ID
- shardingInfo, // TIDB_ROW_ID_SHARDING_INFO
- pkType, // TIDB_PK_TYPE
- policyName, // TIDB_PLACEMENT_POLICY_NAME
- )
- rows = append(rows, record)
- } else {
- record := types.MakeDatums(
- infoschema.CatalogVal, // TABLE_CATALOG
- schema.O, // TABLE_SCHEMA
- table.Name.O, // TABLE_NAME
- "VIEW", // TABLE_TYPE
- nil, // ENGINE
- nil, // VERSION
- nil, // ROW_FORMAT
- nil, // TABLE_ROWS
- nil, // AVG_ROW_LENGTH
- nil, // DATA_LENGTH
- nil, // MAX_DATA_LENGTH
- nil, // INDEX_LENGTH
- nil, // DATA_FREE
- nil, // AUTO_INCREMENT
- createTime, // CREATE_TIME
- nil, // UPDATE_TIME
- nil, // CHECK_TIME
- nil, // TABLE_COLLATION
- nil, // CHECKSUM
- nil, // CREATE_OPTIONS
- "VIEW", // TABLE_COMMENT
- table.ID, // TIDB_TABLE_ID
- nil, // TIDB_ROW_ID_SHARDING_INFO
- pkType, // TIDB_PK_TYPE
- nil, // TIDB_PLACEMENT_POLICY_NAME
- )
- rows = append(rows, record)
- }
- }
}
e.rows = rows
return nil
@@ -718,17 +729,17 @@ func (e *memtableRetriever) setDataFromTables(ctx context.Context, sctx sessionc
// Data for inforation_schema.CHECK_CONSTRAINTS
// This is standards (ISO/IEC 9075-11) compliant and is compatible with the implementation in MySQL as well.
-func (e *memtableRetriever) setDataFromCheckConstraints(ctx context.Context, sctx sessionctx.Context, schemas []model.CIStr) error {
+func (e *memtableRetriever) setDataFromCheckConstraints(ctx context.Context, sctx sessionctx.Context) error {
var rows [][]types.Datum
checker := privilege.GetPrivilegeManager(sctx)
- extractor, ok := e.extractor.(*plannercore.InfoSchemaTablesExtractor)
- if ok && extractor.SkipRequest {
+ ex, ok := e.extractor.(*plannercore.InfoSchemaCheckConstraintsExtractor)
+ if !ok {
+ return errors.Errorf("wrong extractor type: %T, expected InfoSchemaCheckConstraintsExtractor", e.extractor)
+ }
+ if ex.SkipRequest {
return nil
}
- for _, schema := range schemas {
- if ok && extractor.Filter("constraint_schema", schema.L) {
- continue
- }
+ for _, schema := range ex.ListSchemas(e.is) {
tables, err := e.is.SchemaTableInfos(ctx, schema)
if err != nil {
return errors.Trace(err)
@@ -742,7 +753,7 @@ func (e *memtableRetriever) setDataFromCheckConstraints(ctx context.Context, sct
if constraint.State != model.StatePublic {
continue
}
- if ok && extractor.Filter("constraint_name", constraint.Name.L) {
+ if ok && ex.Filter("constraint_name", constraint.Name.L) {
continue
}
record := types.MakeDatums(
@@ -762,43 +773,42 @@ func (e *memtableRetriever) setDataFromCheckConstraints(ctx context.Context, sct
// Data for inforation_schema.TIDB_CHECK_CONSTRAINTS
// This has non-standard TiDB specific extensions.
-func (e *memtableRetriever) setDataFromTiDBCheckConstraints(ctx context.Context, sctx sessionctx.Context, schemas []model.CIStr) error {
+func (e *memtableRetriever) setDataFromTiDBCheckConstraints(ctx context.Context, sctx sessionctx.Context) error {
var rows [][]types.Datum
checker := privilege.GetPrivilegeManager(sctx)
- extractor, ok := e.extractor.(*plannercore.InfoSchemaTablesExtractor)
- if ok && extractor.SkipRequest {
+ ex, ok := e.extractor.(*plannercore.InfoSchemaTiDBCheckConstraintsExtractor)
+ if !ok {
+ return errors.Errorf("wrong extractor type: %T, expected InfoSchemaTiDBCheckConstraintsExtractor", e.extractor)
+ }
+ if ex.SkipRequest {
return nil
}
- for _, schema := range schemas {
- if ok && extractor.Filter("constraint_schema", schema.L) {
- continue
- }
- tables, err := e.is.SchemaTableInfos(ctx, schema)
- if err != nil {
- return errors.Trace(err)
- }
- for _, table := range tables {
- if len(table.Constraints) > 0 {
- if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, table.Name.L, "", mysql.SelectPriv) {
+ schemas, tables, err := ex.ListSchemasAndTables(ctx, e.is)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ for i, table := range tables {
+ schema := schemas[i]
+ if len(table.Constraints) > 0 {
+ if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, table.Name.L, "", mysql.SelectPriv) {
+ continue
+ }
+ for _, constraint := range table.Constraints {
+ if constraint.State != model.StatePublic {
continue
}
- for _, constraint := range table.Constraints {
- if constraint.State != model.StatePublic {
- continue
- }
- if ok && extractor.Filter("constraint_name", constraint.Name.L) {
- continue
- }
- record := types.MakeDatums(
- infoschema.CatalogVal, // CONSTRAINT_CATALOG
- schema.O, // CONSTRAINT_SCHEMA
- constraint.Name.O, // CONSTRAINT_NAME
- fmt.Sprintf("(%s)", constraint.ExprString), // CHECK_CLAUSE
- table.Name.O, // TABLE_NAME
- table.ID, // TABLE_ID
- )
- rows = append(rows, record)
+ if ok && ex.Filter("constraint_name", constraint.Name.L) {
+ continue
}
+ record := types.MakeDatums(
+ infoschema.CatalogVal, // CONSTRAINT_CATALOG
+ schema.O, // CONSTRAINT_SCHEMA
+ constraint.Name.O, // CONSTRAINT_NAME
+ fmt.Sprintf("(%s)", constraint.ExprString), // CHECK_CLAUSE
+ table.Name.O, // TABLE_NAME
+ table.ID, // TABLE_ID
+ )
+ rows = append(rows, record)
}
}
}
@@ -808,13 +818,13 @@ func (e *memtableRetriever) setDataFromTiDBCheckConstraints(ctx context.Context,
type hugeMemTableRetriever struct {
dummyCloser
- extractor *plannercore.ColumnsTableExtractor
+ extractor *plannercore.InfoSchemaColumnsExtractor
table *model.TableInfo
columns []*model.ColumnInfo
retrieved bool
initialized bool
rows [][]types.Datum
- dbs []*model.DBInfo
+ dbs []model.CIStr
curTables []*model.TableInfo
dbsIdx int
tblIdx int
@@ -827,15 +837,16 @@ type hugeMemTableRetriever struct {
// retrieve implements the infoschemaRetriever interface
func (e *hugeMemTableRetriever) retrieve(ctx context.Context, sctx sessionctx.Context) ([][]types.Datum, error) {
+ if e.extractor.SkipRequest {
+ e.retrieved = true
+ }
if e.retrieved {
return nil, nil
}
if !e.initialized {
e.is = sessiontxn.GetTxnManager(sctx).GetTxnInfoSchema()
- dbs := e.is.AllSchemas()
- slices.SortFunc(dbs, model.LessDBInfo)
- e.dbs = dbs
+ e.dbs = e.extractor.ListSchemas(e.is)
e.initialized = true
e.rows = make([][]types.Datum, 0, 1024)
e.batch = 1024
@@ -843,7 +854,7 @@ func (e *hugeMemTableRetriever) retrieve(ctx context.Context, sctx sessionctx.Co
var err error
if e.table.Name.O == infoschema.TableColumns {
- err = e.setDataForColumns(ctx, sctx, e.extractor)
+ err = e.setDataForColumns(ctx, sctx)
}
if err != nil {
return nil, err
@@ -853,14 +864,14 @@ func (e *hugeMemTableRetriever) retrieve(ctx context.Context, sctx sessionctx.Co
return adjustColumns(e.rows, e.columns, e.table), nil
}
-func (e *hugeMemTableRetriever) setDataForColumns(ctx context.Context, sctx sessionctx.Context, extractor *plannercore.ColumnsTableExtractor) error {
+func (e *hugeMemTableRetriever) setDataForColumns(ctx context.Context, sctx sessionctx.Context) error {
checker := privilege.GetPrivilegeManager(sctx)
e.rows = e.rows[:0]
for ; e.dbsIdx < len(e.dbs); e.dbsIdx++ {
schema := e.dbs[e.dbsIdx]
var table *model.TableInfo
if len(e.curTables) == 0 {
- tables, err := e.is.SchemaTableInfos(ctx, schema.Name)
+ tables, err := e.extractor.ListTables(ctx, schema, e.is)
if err != nil {
return errors.Trace(err)
}
@@ -869,7 +880,7 @@ func (e *hugeMemTableRetriever) setDataForColumns(ctx context.Context, sctx sess
for e.tblIdx < len(e.curTables) {
table = e.curTables[e.tblIdx]
e.tblIdx++
- if e.setDataForColumnsWithOneTable(ctx, sctx, extractor, schema, table, checker) {
+ if e.setDataForColumnsWithOneTable(ctx, sctx, schema, table, checker) {
return nil
}
}
@@ -882,15 +893,14 @@ func (e *hugeMemTableRetriever) setDataForColumns(ctx context.Context, sctx sess
func (e *hugeMemTableRetriever) setDataForColumnsWithOneTable(
ctx context.Context,
sctx sessionctx.Context,
- extractor *plannercore.ColumnsTableExtractor,
- schema *model.DBInfo,
+ schema model.CIStr,
table *model.TableInfo,
checker privilege.Manager) bool {
hasPrivs := false
var priv mysql.PrivilegeType
if checker != nil {
for _, p := range mysql.AllColumnPrivs {
- if checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.Name.L, table.Name.L, "", p) {
+ if checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, table.Name.L, "", p) {
hasPrivs = true
priv |= p
}
@@ -900,11 +910,16 @@ func (e *hugeMemTableRetriever) setDataForColumnsWithOneTable(
}
}
- e.dataForColumnsInTable(ctx, sctx, schema, table, priv, extractor)
+ e.dataForColumnsInTable(ctx, sctx, schema, table, priv)
return len(e.rows) >= e.batch
}
-func (e *hugeMemTableRetriever) dataForColumnsInTable(ctx context.Context, sctx sessionctx.Context, schema *model.DBInfo, tbl *model.TableInfo, priv mysql.PrivilegeType, extractor *plannercore.ColumnsTableExtractor) {
+func (e *hugeMemTableRetriever) dataForColumnsInTable(
+ ctx context.Context,
+ sctx sessionctx.Context,
+ schema model.CIStr,
+ tbl *model.TableInfo,
+ priv mysql.PrivilegeType) {
if tbl.IsView() {
e.viewMu.Lock()
_, ok := e.viewSchemaMap[tbl.ID]
@@ -916,7 +931,7 @@ func (e *hugeMemTableRetriever) dataForColumnsInTable(ctx context.Context, sctx
is := sessiontxn.GetTxnManager(s).GetTxnInfoSchema()
planBuilder, _ := plannercore.NewPlanBuilder().Init(s.GetPlanCtx(), is, hint.NewQBHintHandler(nil))
var err error
- viewLogicalPlan, err = planBuilder.BuildDataSourceFromView(ctx, schema.Name, tbl, nil, nil)
+ viewLogicalPlan, err = planBuilder.BuildDataSourceFromView(ctx, schema, tbl, nil, nil)
return errors.Trace(err)
}); err != nil {
sctx.GetSessionVars().StmtCtx.AppendWarning(err)
@@ -929,42 +944,8 @@ func (e *hugeMemTableRetriever) dataForColumnsInTable(ctx context.Context, sctx
e.viewMu.Unlock()
}
- var tableSchemaRegexp, tableNameRegexp, columnsRegexp []collate.WildcardPattern
- var tableSchemaFilterEnable,
- tableNameFilterEnable, columnsFilterEnable bool
- if !extractor.SkipRequest {
- tableSchemaFilterEnable = extractor.TableSchema.Count() > 0
- tableNameFilterEnable = extractor.TableName.Count() > 0
- columnsFilterEnable = extractor.ColumnName.Count() > 0
- if len(extractor.TableSchemaPatterns) > 0 {
- tableSchemaRegexp = make([]collate.WildcardPattern, len(extractor.TableSchemaPatterns))
- for i, pattern := range extractor.TableSchemaPatterns {
- tableSchemaRegexp[i] = collate.GetCollatorByID(collate.CollationName2ID(mysql.UTF8MB4DefaultCollation)).Pattern()
- tableSchemaRegexp[i].Compile(pattern, byte('\\'))
- }
- }
- if len(extractor.TableNamePatterns) > 0 {
- tableNameRegexp = make([]collate.WildcardPattern, len(extractor.TableNamePatterns))
- for i, pattern := range extractor.TableNamePatterns {
- tableNameRegexp[i] = collate.GetCollatorByID(collate.CollationName2ID(mysql.UTF8MB4DefaultCollation)).Pattern()
- tableNameRegexp[i].Compile(pattern, byte('\\'))
- }
- }
- if len(extractor.ColumnNamePatterns) > 0 {
- columnsRegexp = make([]collate.WildcardPattern, len(extractor.ColumnNamePatterns))
- for i, pattern := range extractor.ColumnNamePatterns {
- columnsRegexp[i] = collate.GetCollatorByID(collate.CollationName2ID(mysql.UTF8MB4DefaultCollation)).Pattern()
- columnsRegexp[i].Compile(pattern, byte('\\'))
- }
- }
- }
- i := 0
-ForColumnsTag:
- for _, col := range tbl.Columns {
- if col.Hidden {
- continue
- }
- i++
+ cols, ordinalPos := e.extractor.ListColumns(tbl)
+ for i, col := range cols {
ft := &(col.FieldType)
if tbl.IsView() {
e.viewMu.RLock()
@@ -978,32 +959,6 @@ ForColumnsTag:
}
e.viewMu.RUnlock()
}
- if !extractor.SkipRequest {
- if tableSchemaFilterEnable && !extractor.TableSchema.Exist(schema.Name.L) {
- continue
- }
- if tableNameFilterEnable && !extractor.TableName.Exist(tbl.Name.L) {
- continue
- }
- if columnsFilterEnable && !extractor.ColumnName.Exist(col.Name.L) {
- continue
- }
- for _, re := range tableSchemaRegexp {
- if !re.DoMatch(schema.Name.L) {
- continue ForColumnsTag
- }
- }
- for _, re := range tableNameRegexp {
- if !re.DoMatch(tbl.Name.L) {
- continue ForColumnsTag
- }
- }
- for _, re := range columnsRegexp {
- if !re.DoMatch(col.Name.L) {
- continue ForColumnsTag
- }
- }
- }
var charMaxLen, charOctLen, numericPrecision, numericScale, datetimePrecision any
colLen, decimal := ft.GetFlen(), ft.GetDecimal()
@@ -1080,10 +1035,10 @@ ForColumnsTag:
}
record := types.MakeDatums(
infoschema.CatalogVal, // TABLE_CATALOG
- schema.Name.O, // TABLE_SCHEMA
+ schema.O, // TABLE_SCHEMA
tbl.Name.O, // TABLE_NAME
col.Name.O, // COLUMN_NAME
- i, // ORDINAL_POSITION
+ ordinalPos[i], // ORDINAL_POSITION
columnDefault, // COLUMN_DEFAULT
columnDesc.Null, // IS_NULLABLE
types.TypeToStr(colType, ft.GetCharset()), // DATA_TYPE
@@ -1113,185 +1068,181 @@ func calcCharOctLength(lenInChar int, cs string) int {
return lenInBytes
}
-func (e *memtableRetriever) setDataFromPartitions(ctx context.Context, sctx sessionctx.Context, schemas []model.CIStr) error {
+func (e *memtableRetriever) setDataFromPartitions(ctx context.Context, sctx sessionctx.Context) error {
useStatsCache := e.updateStatsCacheIfNeed()
checker := privilege.GetPrivilegeManager(sctx)
var rows [][]types.Datum
createTimeTp := mysql.TypeDatetime
- extractor, ok := e.extractor.(*plannercore.InfoSchemaTablesExtractor)
- if ok && extractor.SkipRequest {
+ ex, ok := e.extractor.(*plannercore.InfoSchemaPartitionsExtractor)
+ if !ok {
+ return errors.Errorf("wrong extractor type: %T, expected InfoSchemaPartitionsExtractor", e.extractor)
+ }
+ if ex.SkipRequest {
return nil
}
-
- for _, schema := range schemas {
- if ok && extractor.Filter("table_schema", schema.L) {
+ schemas, tables, err := ex.ListSchemasAndTables(ctx, e.is)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ for i, table := range tables {
+ schema := schemas[i]
+ if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, table.Name.L, "", mysql.SelectPriv) {
continue
}
- tables, err := e.is.SchemaTableInfos(ctx, schema)
- if err != nil {
- return errors.Trace(err)
+ createTime := types.NewTime(types.FromGoTime(table.GetUpdateTime()), createTimeTp, types.DefaultFsp)
+
+ var rowCount, dataLength, indexLength uint64
+ if useStatsCache {
+ if table.GetPartitionInfo() == nil {
+ err := cache.TableRowStatsCache.UpdateByID(sctx, table.ID)
+ if err != nil {
+ return err
+ }
+ } else {
+ // needs to update needed partitions for partition table.
+ for _, pi := range table.GetPartitionInfo().Definitions {
+ err := cache.TableRowStatsCache.UpdateByID(sctx, pi.ID)
+ if err != nil {
+ return err
+ }
+ }
+ }
}
- for _, table := range tables {
- if ok && extractor.Filter("table_name", table.Name.L) {
- continue
+ if table.GetPartitionInfo() == nil {
+ rowCount = cache.TableRowStatsCache.GetTableRows(table.ID)
+ dataLength, indexLength = cache.TableRowStatsCache.GetDataAndIndexLength(table, table.ID, rowCount)
+ avgRowLength := uint64(0)
+ if rowCount != 0 {
+ avgRowLength = dataLength / rowCount
}
- if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, table.Name.L, "", mysql.SelectPriv) {
+ // If there are any condition on the `PARTITION_NAME` in the extractor, this record should be ignored
+ if len(ex.ColPredicates["partition_name"]) > 0 {
continue
}
- createTime := types.NewTime(types.FromGoTime(table.GetUpdateTime()), createTimeTp, types.DefaultFsp)
+ record := types.MakeDatums(
+ infoschema.CatalogVal, // TABLE_CATALOG
+ schema.O, // TABLE_SCHEMA
+ table.Name.O, // TABLE_NAME
+ nil, // PARTITION_NAME
+ nil, // SUBPARTITION_NAME
+ nil, // PARTITION_ORDINAL_POSITION
+ nil, // SUBPARTITION_ORDINAL_POSITION
+ nil, // PARTITION_METHOD
+ nil, // SUBPARTITION_METHOD
+ nil, // PARTITION_EXPRESSION
+ nil, // SUBPARTITION_EXPRESSION
+ nil, // PARTITION_DESCRIPTION
+ rowCount, // TABLE_ROWS
+ avgRowLength, // AVG_ROW_LENGTH
+ dataLength, // DATA_LENGTH
+ nil, // MAX_DATA_LENGTH
+ indexLength, // INDEX_LENGTH
+ nil, // DATA_FREE
+ createTime, // CREATE_TIME
+ nil, // UPDATE_TIME
+ nil, // CHECK_TIME
+ nil, // CHECKSUM
+ nil, // PARTITION_COMMENT
+ nil, // NODEGROUP
+ nil, // TABLESPACE_NAME
+ nil, // TIDB_PARTITION_ID
+ nil, // TIDB_PLACEMENT_POLICY_NAME
+ )
+ rows = append(rows, record)
+ } else {
+ for i, pi := range table.GetPartitionInfo().Definitions {
+ if ex.Filter("partition_name", pi.Name.L) {
+ continue
+ }
+ rowCount = cache.TableRowStatsCache.GetTableRows(pi.ID)
+ dataLength, indexLength = cache.TableRowStatsCache.GetDataAndIndexLength(table, pi.ID, rowCount)
+ avgRowLength := uint64(0)
+ if rowCount != 0 {
+ avgRowLength = dataLength / rowCount
+ }
+
+ var partitionDesc string
+ if table.Partition.Type == model.PartitionTypeRange {
+ partitionDesc = strings.Join(pi.LessThan, ",")
+ } else if table.Partition.Type == model.PartitionTypeList {
+ if len(pi.InValues) > 0 {
+ buf := bytes.NewBuffer(nil)
+ for i, vs := range pi.InValues {
+ if i > 0 {
+ buf.WriteString(",")
+ }
+ if len(vs) != 1 {
+ buf.WriteString("(")
+ }
+ buf.WriteString(strings.Join(vs, ","))
+ if len(vs) != 1 {
+ buf.WriteString(")")
+ }
+ }
+ partitionDesc = buf.String()
+ }
+ }
- var rowCount, dataLength, indexLength uint64
- if useStatsCache {
- if table.GetPartitionInfo() == nil {
- err := cache.TableRowStatsCache.UpdateByID(sctx, table.ID)
- if err != nil {
- return err
+ partitionMethod := table.Partition.Type.String()
+ partitionExpr := table.Partition.Expr
+ if len(table.Partition.Columns) > 0 {
+ switch table.Partition.Type {
+ case model.PartitionTypeRange:
+ partitionMethod = "RANGE COLUMNS"
+ case model.PartitionTypeList:
+ partitionMethod = "LIST COLUMNS"
+ case model.PartitionTypeKey:
+ partitionMethod = "KEY"
+ default:
+ return errors.Errorf("Inconsistent partition type, have type %v, but with COLUMNS > 0 (%d)", table.Partition.Type, len(table.Partition.Columns))
}
- } else {
- // needs to update needed partitions for partition table.
- for _, pi := range table.GetPartitionInfo().Definitions {
- if ok && extractor.Filter("partition_name", pi.Name.L) {
- continue
- }
- err := cache.TableRowStatsCache.UpdateByID(sctx, pi.ID)
- if err != nil {
- return err
+ buf := bytes.NewBuffer(nil)
+ for i, col := range table.Partition.Columns {
+ if i > 0 {
+ buf.WriteString(",")
}
+ buf.WriteString("`")
+ buf.WriteString(col.String())
+ buf.WriteString("`")
}
+ partitionExpr = buf.String()
}
- }
- if table.GetPartitionInfo() == nil {
- rowCount = cache.TableRowStatsCache.GetTableRows(table.ID)
- dataLength, indexLength = cache.TableRowStatsCache.GetDataAndIndexLength(table, table.ID, rowCount)
- avgRowLength := uint64(0)
- if rowCount != 0 {
- avgRowLength = dataLength / rowCount
+
+ var policyName any
+ if pi.PlacementPolicyRef != nil {
+ policyName = pi.PlacementPolicyRef.Name.O
}
record := types.MakeDatums(
infoschema.CatalogVal, // TABLE_CATALOG
schema.O, // TABLE_SCHEMA
table.Name.O, // TABLE_NAME
- nil, // PARTITION_NAME
+ pi.Name.O, // PARTITION_NAME
nil, // SUBPARTITION_NAME
- nil, // PARTITION_ORDINAL_POSITION
+ i+1, // PARTITION_ORDINAL_POSITION
nil, // SUBPARTITION_ORDINAL_POSITION
- nil, // PARTITION_METHOD
+ partitionMethod, // PARTITION_METHOD
nil, // SUBPARTITION_METHOD
- nil, // PARTITION_EXPRESSION
+ partitionExpr, // PARTITION_EXPRESSION
nil, // SUBPARTITION_EXPRESSION
- nil, // PARTITION_DESCRIPTION
+ partitionDesc, // PARTITION_DESCRIPTION
rowCount, // TABLE_ROWS
avgRowLength, // AVG_ROW_LENGTH
dataLength, // DATA_LENGTH
- nil, // MAX_DATA_LENGTH
+ uint64(0), // MAX_DATA_LENGTH
indexLength, // INDEX_LENGTH
- nil, // DATA_FREE
+ uint64(0), // DATA_FREE
createTime, // CREATE_TIME
nil, // UPDATE_TIME
nil, // CHECK_TIME
nil, // CHECKSUM
- nil, // PARTITION_COMMENT
+ pi.Comment, // PARTITION_COMMENT
nil, // NODEGROUP
nil, // TABLESPACE_NAME
- nil, // TIDB_PARTITION_ID
- nil, // TIDB_PLACEMENT_POLICY_NAME
+ pi.ID, // TIDB_PARTITION_ID
+ policyName, // TIDB_PLACEMENT_POLICY_NAME
)
rows = append(rows, record)
- } else {
- for i, pi := range table.GetPartitionInfo().Definitions {
- if ok && extractor.Filter("partition_name", pi.Name.L) {
- continue
- }
- rowCount = cache.TableRowStatsCache.GetTableRows(pi.ID)
- dataLength, indexLength = cache.TableRowStatsCache.GetDataAndIndexLength(table, pi.ID, rowCount)
- avgRowLength := uint64(0)
- if rowCount != 0 {
- avgRowLength = dataLength / rowCount
- }
-
- var partitionDesc string
- if table.Partition.Type == model.PartitionTypeRange {
- partitionDesc = strings.Join(pi.LessThan, ",")
- } else if table.Partition.Type == model.PartitionTypeList {
- if len(pi.InValues) > 0 {
- buf := bytes.NewBuffer(nil)
- for i, vs := range pi.InValues {
- if i > 0 {
- buf.WriteString(",")
- }
- if len(vs) != 1 {
- buf.WriteString("(")
- }
- buf.WriteString(strings.Join(vs, ","))
- if len(vs) != 1 {
- buf.WriteString(")")
- }
- }
- partitionDesc = buf.String()
- }
- }
-
- partitionMethod := table.Partition.Type.String()
- partitionExpr := table.Partition.Expr
- if len(table.Partition.Columns) > 0 {
- switch table.Partition.Type {
- case model.PartitionTypeRange:
- partitionMethod = "RANGE COLUMNS"
- case model.PartitionTypeList:
- partitionMethod = "LIST COLUMNS"
- case model.PartitionTypeKey:
- partitionMethod = "KEY"
- default:
- return errors.Errorf("Inconsistent partition type, have type %v, but with COLUMNS > 0 (%d)", table.Partition.Type, len(table.Partition.Columns))
- }
- buf := bytes.NewBuffer(nil)
- for i, col := range table.Partition.Columns {
- if i > 0 {
- buf.WriteString(",")
- }
- buf.WriteString("`")
- buf.WriteString(col.String())
- buf.WriteString("`")
- }
- partitionExpr = buf.String()
- }
-
- var policyName any
- if pi.PlacementPolicyRef != nil {
- policyName = pi.PlacementPolicyRef.Name.O
- }
- record := types.MakeDatums(
- infoschema.CatalogVal, // TABLE_CATALOG
- schema.O, // TABLE_SCHEMA
- table.Name.O, // TABLE_NAME
- pi.Name.O, // PARTITION_NAME
- nil, // SUBPARTITION_NAME
- i+1, // PARTITION_ORDINAL_POSITION
- nil, // SUBPARTITION_ORDINAL_POSITION
- partitionMethod, // PARTITION_METHOD
- nil, // SUBPARTITION_METHOD
- partitionExpr, // PARTITION_EXPRESSION
- nil, // SUBPARTITION_EXPRESSION
- partitionDesc, // PARTITION_DESCRIPTION
- rowCount, // TABLE_ROWS
- avgRowLength, // AVG_ROW_LENGTH
- dataLength, // DATA_LENGTH
- uint64(0), // MAX_DATA_LENGTH
- indexLength, // INDEX_LENGTH
- uint64(0), // DATA_FREE
- createTime, // CREATE_TIME
- nil, // UPDATE_TIME
- nil, // CHECK_TIME
- nil, // CHECKSUM
- pi.Comment, // PARTITION_COMMENT
- nil, // NODEGROUP
- nil, // TABLESPACE_NAME
- pi.ID, // TIDB_PARTITION_ID
- policyName, // TIDB_PLACEMENT_POLICY_NAME
- )
- rows = append(rows, record)
- }
}
}
}
@@ -1299,155 +1250,160 @@ func (e *memtableRetriever) setDataFromPartitions(ctx context.Context, sctx sess
return nil
}
-func (e *memtableRetriever) setDataFromIndexes(ctx context.Context, sctx sessionctx.Context, schemas []model.CIStr) error {
- checker := privilege.GetPrivilegeManager(sctx)
- extractor, ok := e.extractor.(*plannercore.InfoSchemaTablesExtractor)
- if ok && extractor.SkipRequest {
+func (e *memtableRetriever) setDataFromIndexes(ctx context.Context, sctx sessionctx.Context) error {
+ ex, ok := e.extractor.(*plannercore.InfoSchemaIndexesExtractor)
+ if !ok {
+ return errors.Errorf("wrong extractor type: %T, expected InfoSchemaIndexesExtractor", e.extractor)
+ }
+ if ex.SkipRequest {
return nil
}
+
+ schemas, tables, err := ex.ListSchemasAndTables(ctx, e.is)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
var rows [][]types.Datum
- for _, schema := range schemas {
- if ok && extractor.Filter("table_schema", schema.L) {
- continue
- }
- tables, err := e.is.SchemaTableInfos(ctx, schema)
+ for i, table := range tables {
+ rows, err = e.setDataFromIndex(sctx, schemas[i], table, rows)
if err != nil {
return errors.Trace(err)
}
- for _, tb := range tables {
- if ok && extractor.Filter("table_name", tb.Name.L) {
- continue
- }
- if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, tb.Name.L, "", mysql.AllPrivMask) {
- continue
- }
-
- if tb.PKIsHandle {
- var pkCol *model.ColumnInfo
- for _, col := range tb.Cols() {
- if mysql.HasPriKeyFlag(col.GetFlag()) {
- pkCol = col
- break
- }
- }
- record := types.MakeDatums(
- schema.O, // TABLE_SCHEMA
- tb.Name.O, // TABLE_NAME
- 0, // NON_UNIQUE
- "PRIMARY", // KEY_NAME
- 1, // SEQ_IN_INDEX
- pkCol.Name.O, // COLUMN_NAME
- nil, // SUB_PART
- "", // INDEX_COMMENT
- nil, // Expression
- 0, // INDEX_ID
- "YES", // IS_VISIBLE
- "YES", // CLUSTERED
- 0, // IS_GLOBAL
- )
- rows = append(rows, record)
- }
- for _, idxInfo := range tb.Indices {
- if idxInfo.State != model.StatePublic {
- continue
- }
- isClustered := "NO"
- if tb.IsCommonHandle && idxInfo.Primary {
- isClustered = "YES"
- }
- for i, col := range idxInfo.Columns {
- nonUniq := 1
- if idxInfo.Unique {
- nonUniq = 0
- }
- var subPart any
- if col.Length != types.UnspecifiedLength {
- subPart = col.Length
- }
- colName := col.Name.O
- var expression any
- expression = nil
- tblCol := tb.Columns[col.Offset]
- if tblCol.Hidden {
- colName = "NULL"
- expression = tblCol.GeneratedExprString
- }
- visible := "YES"
- if idxInfo.Invisible {
- visible = "NO"
- }
- record := types.MakeDatums(
- schema.O, // TABLE_SCHEMA
- tb.Name.O, // TABLE_NAME
- nonUniq, // NON_UNIQUE
- idxInfo.Name.O, // KEY_NAME
- i+1, // SEQ_IN_INDEX
- colName, // COLUMN_NAME
- subPart, // SUB_PART
- idxInfo.Comment, // INDEX_COMMENT
- expression, // Expression
- idxInfo.ID, // INDEX_ID
- visible, // IS_VISIBLE
- isClustered, // CLUSTERED
- idxInfo.Global, // IS_GLOBAL
- )
- rows = append(rows, record)
- }
- }
- }
}
e.rows = rows
return nil
}
-func (e *memtableRetriever) setDataFromViews(ctx context.Context, sctx sessionctx.Context, schemas []model.CIStr) error {
+func (*memtableRetriever) setDataFromIndex(
+ sctx sessionctx.Context,
+ schema model.CIStr,
+ tb *model.TableInfo,
+ rows [][]types.Datum) ([][]types.Datum, error) {
checker := privilege.GetPrivilegeManager(sctx)
- extractor, ok := e.extractor.(*plannercore.InfoSchemaTablesExtractor)
- if ok && extractor.SkipRequest {
- return nil
+ if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, tb.Name.L, "", mysql.AllPrivMask) {
+ return rows, nil
}
- var rows [][]types.Datum
- for _, schema := range schemas {
- if ok && extractor.Filter("table_schema", schema.L) {
+
+ if tb.PKIsHandle {
+ var pkCol *model.ColumnInfo
+ for _, col := range tb.Cols() {
+ if mysql.HasPriKeyFlag(col.GetFlag()) {
+ pkCol = col
+ break
+ }
+ }
+ record := types.MakeDatums(
+ schema.O, // TABLE_SCHEMA
+ tb.Name.O, // TABLE_NAME
+ 0, // NON_UNIQUE
+ "PRIMARY", // KEY_NAME
+ 1, // SEQ_IN_INDEX
+ pkCol.Name.O, // COLUMN_NAME
+ nil, // SUB_PART
+ "", // INDEX_COMMENT
+ nil, // Expression
+ 0, // INDEX_ID
+ "YES", // IS_VISIBLE
+ "YES", // CLUSTERED
+ 0, // IS_GLOBAL
+ )
+ rows = append(rows, record)
+ }
+ for _, idxInfo := range tb.Indices {
+ if idxInfo.State != model.StatePublic {
continue
}
- tables, err := e.is.SchemaTableInfos(ctx, schema)
- if err != nil {
- return errors.Trace(err)
+ isClustered := "NO"
+ if tb.IsCommonHandle && idxInfo.Primary {
+ isClustered = "YES"
}
- for _, table := range tables {
- if ok && extractor.Filter("table_name", table.Name.L) {
- continue
+ for i, col := range idxInfo.Columns {
+ nonUniq := 1
+ if idxInfo.Unique {
+ nonUniq = 0
}
- if !table.IsView() {
- continue
- }
- collation := table.Collate
- charset := table.Charset
- if collation == "" {
- collation = mysql.DefaultCollationName
+ var subPart any
+ if col.Length != types.UnspecifiedLength {
+ subPart = col.Length
}
- if charset == "" {
- charset = mysql.DefaultCharset
+ colName := col.Name.O
+ var expression any
+ expression = nil
+ tblCol := tb.Columns[col.Offset]
+ if tblCol.Hidden {
+ colName = "NULL"
+ expression = tblCol.GeneratedExprString
}
- if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, table.Name.L, "", mysql.AllPrivMask) {
- continue
+ visible := "YES"
+ if idxInfo.Invisible {
+ visible = "NO"
}
record := types.MakeDatums(
- infoschema.CatalogVal, // TABLE_CATALOG
- schema.O, // TABLE_SCHEMA
- table.Name.O, // TABLE_NAME
- table.View.SelectStmt, // VIEW_DEFINITION
- table.View.CheckOption.String(), // CHECK_OPTION
- "NO", // IS_UPDATABLE
- table.View.Definer.String(), // DEFINER
- table.View.Security.String(), // SECURITY_TYPE
- charset, // CHARACTER_SET_CLIENT
- collation, // COLLATION_CONNECTION
+ schema.O, // TABLE_SCHEMA
+ tb.Name.O, // TABLE_NAME
+ nonUniq, // NON_UNIQUE
+ idxInfo.Name.O, // KEY_NAME
+ i+1, // SEQ_IN_INDEX
+ colName, // COLUMN_NAME
+ subPart, // SUB_PART
+ idxInfo.Comment, // INDEX_COMMENT
+ expression, // Expression
+ idxInfo.ID, // INDEX_ID
+ visible, // IS_VISIBLE
+ isClustered, // CLUSTERED
+ idxInfo.Global, // IS_GLOBAL
)
rows = append(rows, record)
}
}
+ return rows, nil
+}
+
+func (e *memtableRetriever) setDataFromViews(ctx context.Context, sctx sessionctx.Context) error {
+ checker := privilege.GetPrivilegeManager(sctx)
+ ex, ok := e.extractor.(*plannercore.InfoSchemaViewsExtractor)
+ if !ok {
+ return errors.Errorf("wrong extractor type: %T, expected InfoSchemaIndexesExtractor", e.extractor)
+ }
+ if ex.SkipRequest {
+ return nil
+ }
+ schemas, tables, err := ex.ListSchemasAndTables(ctx, e.is)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ rows := make([][]types.Datum, 0, len(tables))
+ for i, table := range tables {
+ schema := schemas[i]
+ if !table.IsView() {
+ continue
+ }
+ collation := table.Collate
+ charset := table.Charset
+ if collation == "" {
+ collation = mysql.DefaultCollationName
+ }
+ if charset == "" {
+ charset = mysql.DefaultCharset
+ }
+ if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, table.Name.L, "", mysql.AllPrivMask) {
+ continue
+ }
+ record := types.MakeDatums(
+ infoschema.CatalogVal, // TABLE_CATALOG
+ schema.O, // TABLE_SCHEMA
+ table.Name.O, // TABLE_NAME
+ table.View.SelectStmt, // VIEW_DEFINITION
+ table.View.CheckOption.String(), // CHECK_OPTION
+ "NO", // IS_UPDATABLE
+ table.View.Definer.String(), // DEFINER
+ table.View.Security.String(), // SECURITY_TYPE
+ charset, // CHARACTER_SET_CLIENT
+ collation, // COLLATION_CONNECTION
+ )
+ rows = append(rows, record)
+ }
e.rows = rows
return nil
}
@@ -1701,31 +1657,30 @@ func (e *memtableRetriever) dataForTiDBClusterInfo(ctx sessionctx.Context) error
return nil
}
-func (e *memtableRetriever) setDataFromKeyColumnUsage(ctx context.Context, sctx sessionctx.Context, schemas []model.CIStr) error {
+func (e *memtableRetriever) setDataFromKeyColumnUsage(ctx context.Context, sctx sessionctx.Context) error {
checker := privilege.GetPrivilegeManager(sctx)
- rows := make([][]types.Datum, 0, len(schemas)) // The capacity is not accurate, but it is not a big problem.
- extractor, ok := e.extractor.(*plannercore.InfoSchemaTablesExtractor)
- if ok && extractor.SkipRequest {
+ ex, ok := e.extractor.(*plannercore.InfoSchemaKeyColumnUsageExtractor)
+ if !ok {
+ return errors.Errorf("wrong extractor type: %T, expected InfoSchemaIndexesExtractor", e.extractor)
+ }
+ if ex.SkipRequest {
return nil
}
- for _, schema := range schemas {
- if ok && extractor.Filter("constraint_schema", schema.L) {
+ schemas, tables, err := ex.ListSchemasAndTables(ctx, e.is)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ rows := make([][]types.Datum, 0, len(tables))
+ for i, table := range tables {
+ schema := schemas[i]
+ if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, table.Name.L, "", mysql.AllPrivMask) {
continue
}
- tables, err := e.is.SchemaTableInfos(ctx, schema)
- if err != nil {
- return errors.Trace(err)
- }
- for _, table := range tables {
- if ok && extractor.Filter("table_name", table.Name.L) {
- continue
- }
- if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, table.Name.L, "", mysql.AllPrivMask) {
- continue
- }
- rs := keyColumnUsageInTable(schema, table)
- rows = append(rows, rs...)
+ if ex.Filter("constraint_schema", schema.O) {
+ continue
}
+ rs := keyColumnUsageInTable(schema, table, ex)
+ rows = append(rows, rs...)
}
e.rows = rows
return nil
@@ -1793,27 +1748,29 @@ func (e *memtableRetriever) setDataForMetricTables() {
e.rows = rows
}
-func keyColumnUsageInTable(schema model.CIStr, table *model.TableInfo) [][]types.Datum {
+func keyColumnUsageInTable(schema model.CIStr, table *model.TableInfo, extractor *plannercore.InfoSchemaKeyColumnUsageExtractor) [][]types.Datum {
var rows [][]types.Datum
if table.PKIsHandle {
- for _, col := range table.Columns {
- if mysql.HasPriKeyFlag(col.GetFlag()) {
- record := types.MakeDatums(
- infoschema.CatalogVal, // CONSTRAINT_CATALOG
- schema.O, // CONSTRAINT_SCHEMA
- infoschema.PrimaryConstraint, // CONSTRAINT_NAME
- infoschema.CatalogVal, // TABLE_CATALOG
- schema.O, // TABLE_SCHEMA
- table.Name.O, // TABLE_NAME
- col.Name.O, // COLUMN_NAME
- 1, // ORDINAL_POSITION
- 1, // POSITION_IN_UNIQUE_CONSTRAINT
- nil, // REFERENCED_TABLE_SCHEMA
- nil, // REFERENCED_TABLE_NAME
- nil, // REFERENCED_COLUMN_NAME
- )
- rows = append(rows, record)
- break
+ if extractor == nil || !extractor.Filter("constraint_name", lowerPrimaryKeyName) {
+ for _, col := range table.Columns {
+ if mysql.HasPriKeyFlag(col.GetFlag()) {
+ record := types.MakeDatums(
+ infoschema.CatalogVal, // CONSTRAINT_CATALOG
+ schema.O, // CONSTRAINT_SCHEMA
+ infoschema.PrimaryConstraint, // CONSTRAINT_NAME
+ infoschema.CatalogVal, // TABLE_CATALOG
+ schema.O, // TABLE_SCHEMA
+ table.Name.O, // TABLE_NAME
+ col.Name.O, // COLUMN_NAME
+ 1, // ORDINAL_POSITION
+ 1, // POSITION_IN_UNIQUE_CONSTRAINT
+ nil, // REFERENCED_TABLE_SCHEMA
+ nil, // REFERENCED_TABLE_NAME
+ nil, // REFERENCED_COLUMN_NAME
+ )
+ rows = append(rows, record)
+ break
+ }
}
}
}
@@ -1823,14 +1780,22 @@ func keyColumnUsageInTable(schema model.CIStr, table *model.TableInfo) [][]types
}
for _, index := range table.Indices {
var idxName string
+ var filterIdxName string
if index.Primary {
- idxName = infoschema.PrimaryConstraint
+ idxName = mysql.PrimaryKeyName
+ filterIdxName = lowerPrimaryKeyName
} else if index.Unique {
idxName = index.Name.O
+ filterIdxName = index.Name.L
} else {
// Only handle unique/primary key
continue
}
+
+ if extractor != nil && extractor.Filter("constraint_name", filterIdxName) {
+ continue
+ }
+
for i, key := range index.Columns {
col := nameToCol[key.Name.L]
if col.Hidden {
@@ -1854,6 +1819,10 @@ func keyColumnUsageInTable(schema model.CIStr, table *model.TableInfo) [][]types
}
}
for _, fk := range table.ForeignKeys {
+ if extractor != nil && extractor.Filter("constraint_name", fk.Name.L) {
+ continue
+ }
+
for i, key := range fk.Cols {
fkRefCol := ""
if len(fk.RefCols) > i {
@@ -1880,21 +1849,6 @@ func keyColumnUsageInTable(schema model.CIStr, table *model.TableInfo) [][]types
return rows
}
-func ensureSchemaTables(ctx context.Context, is infoschema.InfoSchema, schemaNames []model.CIStr) []*model.DBInfo {
- // For infoschema v2, Tables of DBInfo could be missing.
- res := make([]*model.DBInfo, 0, len(schemaNames))
- for _, dbName := range schemaNames {
- dbInfoRaw, _ := is.SchemaByName(dbName)
- dbInfo := dbInfoRaw.Clone()
- dbInfo.Tables = dbInfo.Tables[:0]
- var err error
- dbInfo.Tables, err = is.SchemaTableInfos(ctx, dbName)
- terror.Log(err)
- res = append(res, dbInfo)
- }
- return res
-}
-
func (e *memtableRetriever) setDataForTiKVRegionStatus(ctx context.Context, sctx sessionctx.Context) (err error) {
checker := privilege.GetPrivilegeManager(sctx)
var extractorTableIDs []int64
@@ -1936,9 +1890,7 @@ func (e *memtableRetriever) setDataForTiKVRegionStatus(ctx context.Context, sctx
return err
}
}
- schemaNames := is.AllSchemaNames()
- schemas := ensureSchemaTables(ctx, is, schemaNames)
- tableInfos := tikvHelper.GetRegionsTableInfo(allRegionsInfo, schemas)
+ tableInfos := tikvHelper.GetRegionsTableInfo(allRegionsInfo, is, nil)
for i := range allRegionsInfo.Regions {
regionTableList := tableInfos[allRegionsInfo.Regions[i].ID]
if len(regionTableList) == 0 {
@@ -1960,7 +1912,7 @@ func (e *memtableRetriever) setDataForTiKVRegionStatus(ctx context.Context, sctx
}
func (e *memtableRetriever) getRegionsInfoForTable(ctx context.Context, h *helper.Helper, is infoschema.InfoSchema, tableID int64) (*pd.RegionsInfo, error) {
- tbl, _ := is.TableByID(tableID)
+ tbl, _ := is.TableByID(ctx, tableID)
if tbl == nil {
return nil, infoschema.ErrTableExists.GenWithStackByArgs(tableID)
}
@@ -2065,13 +2017,13 @@ func (e *memtableRetriever) setDataForTiDBHotRegions(ctx context.Context, sctx s
Store: tikvStore,
RegionCache: tikvStore.GetRegionCache(),
}
- schemas := tikvHelper.FilterMemDBs(sessiontxn.GetTxnManager(sctx).GetTxnInfoSchema().AllSchemas())
- metrics, err := tikvHelper.ScrapeHotInfo(ctx, helper.HotRead, schemas)
+ is := sessiontxn.GetTxnManager(sctx).GetTxnInfoSchema()
+ metrics, err := tikvHelper.ScrapeHotInfo(ctx, helper.HotRead, is, tikvHelper.FilterMemDBs)
if err != nil {
return err
}
e.setDataForHotRegionByMetrics(metrics, "read")
- metrics, err = tikvHelper.ScrapeHotInfo(ctx, helper.HotWrite, schemas)
+ metrics, err = tikvHelper.ScrapeHotInfo(ctx, helper.HotWrite, is, nil)
if err != nil {
return err
}
@@ -2109,30 +2061,31 @@ func (e *memtableRetriever) setDataForHotRegionByMetrics(metrics []helper.HotTab
}
// setDataFromTableConstraints constructs data for table information_schema.constraints.See https://dev.mysql.com/doc/refman/5.7/en/table-constraints-table.html
-func (e *memtableRetriever) setDataFromTableConstraints(ctx context.Context, sctx sessionctx.Context, schemas []model.CIStr) error {
+func (e *memtableRetriever) setDataFromTableConstraints(ctx context.Context, sctx sessionctx.Context) error {
checker := privilege.GetPrivilegeManager(sctx)
- extractor, ok := e.extractor.(*plannercore.InfoSchemaTablesExtractor)
- if ok && extractor.SkipRequest {
+ ex, ok := e.extractor.(*plannercore.InfoSchemaTableConstraintsExtractor)
+ if !ok {
+ return errors.Errorf("wrong extractor type: %T, expected InfoSchemaIndexesExtractor", e.extractor)
+ }
+ if ex.SkipRequest {
return nil
}
- var rows [][]types.Datum
- for _, schema := range schemas {
- if ok && extractor.Filter("constraint_schema", schema.L) {
+ schemas, tables, err := ex.ListSchemasAndTables(ctx, e.is)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ rows := make([][]types.Datum, 0, len(tables))
+ for i, tbl := range tables {
+ schema := schemas[i]
+ if ex.Filter("constraint_schema", schema.L) {
continue
}
- tables, err := e.is.SchemaTableInfos(ctx, schema)
- if err != nil {
- return errors.Trace(err)
+ if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, tbl.Name.L, "", mysql.AllPrivMask) {
+ continue
}
- for _, tbl := range tables {
- if ok && extractor.Filter("table_name", tbl.Name.L) {
- continue
- }
- if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, tbl.Name.L, "", mysql.AllPrivMask) {
- continue
- }
- if tbl.PKIsHandle {
+ if tbl.PKIsHandle {
+ if !ex.Filter("constraint_name", lowerPrimaryKeyName) {
record := types.MakeDatums(
infoschema.CatalogVal, // CONSTRAINT_CATALOG
schema.O, // CONSTRAINT_SCHEMA
@@ -2143,41 +2096,50 @@ func (e *memtableRetriever) setDataFromTableConstraints(ctx context.Context, sct
)
rows = append(rows, record)
}
+ }
- for _, idx := range tbl.Indices {
- var cname, ctype string
- if idx.Primary {
- cname = mysql.PrimaryKeyName
- ctype = infoschema.PrimaryKeyType
- } else if idx.Unique {
- cname = idx.Name.O
- ctype = infoschema.UniqueKeyType
- } else {
- // The index has no constriant.
- continue
- }
- record := types.MakeDatums(
- infoschema.CatalogVal, // CONSTRAINT_CATALOG
- schema.O, // CONSTRAINT_SCHEMA
- cname, // CONSTRAINT_NAME
- schema.O, // TABLE_SCHEMA
- tbl.Name.O, // TABLE_NAME
- ctype, // CONSTRAINT_TYPE
- )
- rows = append(rows, record)
+ for _, idx := range tbl.Indices {
+ var cname, ctype string
+ var filterName string
+ if idx.Primary {
+ cname = mysql.PrimaryKeyName
+ filterName = lowerPrimaryKeyName
+ ctype = infoschema.PrimaryKeyType
+ } else if idx.Unique {
+ cname = idx.Name.O
+ filterName = idx.Name.L
+ ctype = infoschema.UniqueKeyType
+ } else {
+ // The index has no constriant.
+ continue
}
- // TiDB includes foreign key information for compatibility but foreign keys are not yet enforced.
- for _, fk := range tbl.ForeignKeys {
- record := types.MakeDatums(
- infoschema.CatalogVal, // CONSTRAINT_CATALOG
- schema.O, // CONSTRAINT_SCHEMA
- fk.Name.O, // CONSTRAINT_NAME
- schema.O, // TABLE_SCHEMA
- tbl.Name.O, // TABLE_NAME
- infoschema.ForeignKeyType, // CONSTRAINT_TYPE
- )
- rows = append(rows, record)
+ if ex.Filter("constraint_name", filterName) {
+ continue
+ }
+ record := types.MakeDatums(
+ infoschema.CatalogVal, // CONSTRAINT_CATALOG
+ schema.O, // CONSTRAINT_SCHEMA
+ cname, // CONSTRAINT_NAME
+ schema.O, // TABLE_SCHEMA
+ tbl.Name.O, // TABLE_NAME
+ ctype, // CONSTRAINT_TYPE
+ )
+ rows = append(rows, record)
+ }
+ // TiDB includes foreign key information for compatibility but foreign keys are not yet enforced.
+ for _, fk := range tbl.ForeignKeys {
+ if ex.Filter("constraint_name", fk.Name.L) {
+ continue
}
+ record := types.MakeDatums(
+ infoschema.CatalogVal, // CONSTRAINT_CATALOG
+ schema.O, // CONSTRAINT_SCHEMA
+ fk.Name.O, // CONSTRAINT_NAME
+ schema.O, // TABLE_SCHEMA
+ tbl.Name.O, // TABLE_NAME
+ infoschema.ForeignKeyType, // CONSTRAINT_TYPE
+ )
+ rows = append(rows, record)
}
}
e.rows = rows
@@ -2567,46 +2529,42 @@ func (e *memtableRetriever) setDataForServersInfo(ctx sessionctx.Context) error
return nil
}
-func (e *memtableRetriever) setDataFromSequences(ctx context.Context, sctx sessionctx.Context, schemas []model.CIStr) error {
+func (e *memtableRetriever) setDataFromSequences(ctx context.Context, sctx sessionctx.Context) error {
checker := privilege.GetPrivilegeManager(sctx)
- extractor, ok := e.extractor.(*plannercore.InfoSchemaTablesExtractor)
- if ok && extractor.SkipRequest {
+ extractor, ok := e.extractor.(*plannercore.InfoSchemaSequenceExtractor)
+ if !ok {
+ return errors.Errorf("wrong extractor type: %T, expected InfoSchemaSequenceExtractor", e.extractor)
+ }
+ if extractor.SkipRequest {
return nil
}
- var rows [][]types.Datum
- for _, schema := range schemas {
- if ok && extractor.Filter("sequence_schema", schema.L) {
+ schemas, tables, err := extractor.ListSchemasAndTables(ctx, e.is)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ rows := make([][]types.Datum, 0, len(tables))
+ for i, table := range tables {
+ schema := schemas[i]
+ if !table.IsSequence() {
continue
}
- tables, err := e.is.SchemaTableInfos(ctx, schema)
- if err != nil {
- return errors.Trace(err)
- }
- for _, table := range tables {
- if ok && extractor.Filter("sequence_name", table.Name.L) {
- continue
- }
- if !table.IsSequence() {
- continue
- }
- if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, table.Name.L, "", mysql.AllPrivMask) {
- continue
- }
- record := types.MakeDatums(
- infoschema.CatalogVal, // TABLE_CATALOG
- schema.O, // SEQUENCE_SCHEMA
- table.Name.O, // SEQUENCE_NAME
- table.Sequence.Cache, // Cache
- table.Sequence.CacheValue, // CACHE_VALUE
- table.Sequence.Cycle, // CYCLE
- table.Sequence.Increment, // INCREMENT
- table.Sequence.MaxValue, // MAXVALUE
- table.Sequence.MinValue, // MINVALUE
- table.Sequence.Start, // START
- table.Sequence.Comment, // COMMENT
- )
- rows = append(rows, record)
+ if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.L, table.Name.L, "", mysql.AllPrivMask) {
+ continue
}
+ record := types.MakeDatums(
+ infoschema.CatalogVal, // TABLE_CATALOG
+ schema.O, // SEQUENCE_SCHEMA
+ table.Name.O, // SEQUENCE_NAME
+ table.Sequence.Cache, // Cache
+ table.Sequence.CacheValue, // CACHE_VALUE
+ table.Sequence.Cycle, // CYCLE
+ table.Sequence.Increment, // INCREMENT
+ table.Sequence.MaxValue, // MAXVALUE
+ table.Sequence.MinValue, // MINVALUE
+ table.Sequence.Start, // START
+ table.Sequence.Comment, // COMMENT
+ )
+ rows = append(rows, record)
}
e.rows = rows
return nil
@@ -2622,7 +2580,7 @@ func (e *memtableRetriever) dataForTableTiFlashReplica(_ context.Context, sctx s
rs := e.is.ListTablesWithSpecialAttribute(infoschema.TiFlashAttribute)
for _, schema := range rs {
for _, tbl := range schema.TableInfos {
- if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.DBName, tbl.Name.L, "", mysql.AllPrivMask) {
+ if checker != nil && !checker.RequestVerification(sctx.GetSessionVars().ActiveRoles, schema.DBName.L, tbl.Name.L, "", mysql.AllPrivMask) {
continue
}
var progress float64
@@ -2645,7 +2603,7 @@ func (e *memtableRetriever) dataForTableTiFlashReplica(_ context.Context, sctx s
progressString := types.TruncateFloatToString(progress, 2)
progress, _ = strconv.ParseFloat(progressString, 64)
record := types.MakeDatums(
- schema.DBName, // TABLE_SCHEMA
+ schema.DBName.O, // TABLE_SCHEMA
tbl.Name.O, // TABLE_NAME
tbl.ID, // TABLE_ID
int64(tbl.TiFlashReplica.Count), // REPLICA_COUNT
@@ -3604,10 +3562,10 @@ func (e *memtableRetriever) setDataFromRunawayWatches(sctx sessionctx.Context) e
watch.ResourceGroupName,
watch.StartTime.UTC().Format(time.DateTime),
watch.EndTime.UTC().Format(time.DateTime),
- rmpb.RunawayWatchType_name[int32(watch.Watch)],
+ watch.Watch.String(),
watch.WatchText,
watch.Source,
- rmpb.RunawayAction_name[int32(action)],
+ action.String(),
)
if watch.EndTime.Equal(resourcegroup.NullTime) {
row[3].SetString("UNLIMITED", mysql.DefaultCollationName)
@@ -3636,7 +3594,8 @@ func (e *memtableRetriever) setDataFromResourceGroups() error {
burstable := burstdisableStr
priority := model.PriorityValueToName(uint64(group.Priority))
fillrate := unlimitedFillRate
- isDefaultInReservedSetting := group.Name == resourcegroup.DefaultResourceGroupName && group.RUSettings.RU.Settings.FillRate == math.MaxInt32
+ // RU_PER_SEC = unlimited like the default group settings.
+ isDefaultInReservedSetting := group.RUSettings.RU.Settings.FillRate == math.MaxInt32
if !isDefaultInReservedSetting {
fillrate = strconv.FormatUint(group.RUSettings.RU.Settings.FillRate, 10)
}
@@ -3714,38 +3673,34 @@ func (e *memtableRetriever) setDataFromKeywords() error {
return nil
}
-func (e *memtableRetriever) setDataFromIndexUsage(ctx context.Context, sctx sessionctx.Context, schemas []model.CIStr) error {
+func (e *memtableRetriever) setDataFromIndexUsage(ctx context.Context, sctx sessionctx.Context) error {
dom := domain.GetDomain(sctx)
rows := make([][]types.Datum, 0, 100)
checker := privilege.GetPrivilegeManager(sctx)
- extractor, ok := e.extractor.(*plannercore.InfoSchemaTablesExtractor)
- if ok && extractor.SkipRequest {
+ extractor, ok := e.extractor.(*plannercore.InfoSchemaIndexUsageExtractor)
+ if !ok {
+ return errors.Errorf("wrong extractor type: %T, expected InfoSchemaIndexUsageExtractor", e.extractor)
+ }
+ if extractor.SkipRequest {
return nil
}
+ schemas := extractor.ListSchemas(e.is)
for _, schema := range schemas {
- if ok && extractor.Filter("table_schema", schema.L) {
- continue
- }
- tables, err := dom.InfoSchema().SchemaTableInfos(ctx, schema)
+ tbls, err := extractor.ListTables(ctx, schema, e.is)
if err != nil {
return errors.Trace(err)
}
- for _, tbl := range tables {
- if ok && extractor.Filter("table_name", tbl.Name.L) {
- continue
- }
- allowed := checker == nil || checker.RequestVerification(
+
+ for _, tbl := range tbls {
+ if checker != nil && !checker.RequestVerification(
sctx.GetSessionVars().ActiveRoles,
- schema.L, tbl.Name.L, "", mysql.AllPrivMask)
- if !allowed {
+ schema.L, tbl.Name.L, "", mysql.AllPrivMask) {
continue
}
- for _, idx := range tbl.Indices {
- if ok && extractor.Filter("index_name", idx.Name.L) {
- continue
- }
+ idxs := extractor.ListIndexes(tbl)
+ for _, idx := range idxs {
row := make([]types.Datum, 0, 14)
usage := dom.StatsHandle().GetIndexUsage(tbl.ID, idx.ID)
row = append(row, types.NewStringDatum(schema.O))
@@ -3773,8 +3728,8 @@ func (e *memtableRetriever) setDataFromIndexUsage(ctx context.Context, sctx sess
return nil
}
-func (e *memtableRetriever) setDataForClusterIndexUsage(ctx context.Context, sctx sessionctx.Context, schemas []model.CIStr) error {
- err := e.setDataFromIndexUsage(ctx, sctx, schemas)
+func (e *memtableRetriever) setDataFromClusterIndexUsage(ctx context.Context, sctx sessionctx.Context) error {
+ err := e.setDataFromIndexUsage(ctx, sctx)
if err != nil {
return errors.Trace(err)
}
diff --git a/pkg/executor/infoschema_reader_internal_test.go b/pkg/executor/infoschema_reader_internal_test.go
index 2bb33bca1050b..8f4cbfa28215f 100644
--- a/pkg/executor/infoschema_reader_internal_test.go
+++ b/pkg/executor/infoschema_reader_internal_test.go
@@ -21,6 +21,7 @@ import (
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
+ plannercore "github.com/pingcap/tidb/pkg/planner/core"
"github.com/pingcap/tidb/pkg/types"
"github.com/stretchr/testify/require"
)
@@ -71,12 +72,9 @@ func TestSetDataFromCheckConstraints(t *testing.T) {
},
}
mockIs := infoschema.MockInfoSchema(tblInfos)
- mt := memtableRetriever{is: mockIs}
+ mt := memtableRetriever{is: mockIs, extractor: &plannercore.InfoSchemaCheckConstraintsExtractor{}}
sctx := defaultCtx()
- dbs := []model.CIStr{
- model.NewCIStr("test"),
- }
- err := mt.setDataFromCheckConstraints(context.Background(), sctx, dbs)
+ err := mt.setDataFromCheckConstraints(context.Background(), sctx)
require.NoError(t, err)
require.Equal(t, 1, len(mt.rows)) // 1 row
@@ -136,10 +134,8 @@ func TestSetDataFromTiDBCheckConstraints(t *testing.T) {
}
mockIs := infoschema.MockInfoSchema(tblInfos)
mt.is = mockIs
- dbs := []model.CIStr{
- model.NewCIStr("test"),
- }
- err := mt.setDataFromTiDBCheckConstraints(context.Background(), sctx, dbs)
+ mt.extractor = &plannercore.InfoSchemaTiDBCheckConstraintsExtractor{}
+ err := mt.setDataFromTiDBCheckConstraints(context.Background(), sctx)
require.NoError(t, err)
require.Equal(t, 1, len(mt.rows)) // 1 row
diff --git a/pkg/executor/infoschema_reader_test.go b/pkg/executor/infoschema_reader_test.go
index 00ee5977ae7ef..bd0928d146069 100644
--- a/pkg/executor/infoschema_reader_test.go
+++ b/pkg/executor/infoschema_reader_test.go
@@ -489,6 +489,124 @@ func TestTiFlashSystemTableWithTiFlashV640(t *testing.T) {
tk.MustQuery("show warnings").Check(testkit.Rows())
}
+func TestColumnTable(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table tbl1(col_1 int primary key, col_2 int, col_4 int);")
+ tk.MustExec("create table tbl2(col_1 int primary key, col_2 int, col_3 int);")
+ tk.MustExec("create view view1 as select min(col_1), col_2, max(col_4) as max4 from tbl1 group by col_2;")
+
+ tk.MustQuery("select TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME from information_schema.columns where TABLE_SCHEMA = 'test';").Check(
+ testkit.RowsWithSep("|",
+ "test|tbl1|col_1",
+ "test|tbl1|col_2",
+ "test|tbl1|col_4",
+ "test|tbl2|col_1",
+ "test|tbl2|col_2",
+ "test|tbl2|col_3",
+ "test|view1|min(col_1)",
+ "test|view1|col_2",
+ "test|view1|max4"))
+ tk.MustQuery(`select TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME from information_schema.columns
+ where TABLE_NAME = 'view1' or TABLE_NAME = 'tbl1'`).Check(
+ testkit.RowsWithSep("|",
+ "test|tbl1|col_1",
+ "test|tbl1|col_2",
+ "test|tbl1|col_4",
+ "test|view1|min(col_1)",
+ "test|view1|col_2",
+ "test|view1|max4"))
+ tk.MustQuery("select TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME from information_schema.columns where COLUMN_NAME = \"col_2\";").Check(
+ testkit.RowsWithSep("|",
+ "test|tbl1|col_2",
+ "test|tbl2|col_2",
+ "test|view1|col_2"))
+ tk.MustQuery(`select TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME from information_schema.columns
+ where TABLE_SCHEMA = 'test' and TABLE_NAME = 'tbl2';`).Check(
+ testkit.RowsWithSep("|",
+ "test|tbl2|col_1",
+ "test|tbl2|col_2",
+ "test|tbl2|col_3"))
+ tk.MustQuery(`select TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME from information_schema.columns
+ where TABLE_SCHEMA = 'test' and COLUMN_NAME = 'col_4'`).Check(
+ testkit.RowsWithSep("|",
+ "test|tbl1|col_4"))
+ tk.MustQuery(`select TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME from information_schema.columns
+ where TABLE_NAME = 'view1' and COLUMN_NAME like 'm%%';`).Check(
+ testkit.RowsWithSep("|",
+ "test|view1|min(col_1)",
+ "test|view1|max4"))
+ tk.MustQuery(`select TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME from information_schema.columns
+ where TABLE_SCHEMA = 'test' and TABLE_NAME = 'tbl1' and COLUMN_NAME = 'col_2';`).Check(
+ testkit.RowsWithSep("|",
+ "test|tbl1|col_2"))
+ tk.MustQuery(`select count(*) from information_schema.columns;`).Check(
+ testkit.RowsWithSep("|", "4923"))
+}
+
+func TestIndexUsageTable(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table idt1(col_1 int primary key, col_2 int, index idx_1(col_1), index idx_2(col_2), index idx_3(col_1, col_2));")
+ tk.MustExec("create table idt2(col_1 int primary key, col_2 int, index idx_1(col_1), index idx_2(col_2), index idx_4(col_2, col_1));")
+
+ tk.MustQuery(`select TABLE_SCHEMA, TABLE_NAME, INDEX_NAME from information_schema.tidb_index_usage
+ where TABLE_SCHEMA = 'test';`).Check(
+ testkit.RowsWithSep("|",
+ "test|idt1|idx_1",
+ "test|idt1|idx_2",
+ "test|idt1|idx_3",
+ "test|idt2|idx_1",
+ "test|idt2|idx_2",
+ "test|idt2|idx_4"))
+ tk.MustQuery(`select TABLE_SCHEMA, TABLE_NAME, INDEX_NAME from information_schema.tidb_index_usage where TABLE_NAME = 'idt1'`).Check(
+ testkit.RowsWithSep("|",
+ "test|idt1|idx_1",
+ "test|idt1|idx_2",
+ "test|idt1|idx_3"))
+ tk.MustQuery("select TABLE_SCHEMA, TABLE_NAME, INDEX_NAME from information_schema.tidb_index_usage where INDEX_NAME = 'IDX_3'").Check(
+ testkit.RowsWithSep("|",
+ "test|idt1|idx_3"))
+ tk.MustQuery(`select TABLE_SCHEMA, TABLE_NAME, INDEX_NAME from information_schema.tidb_index_usage
+ where TABLE_SCHEMA = 'test' and TABLE_NAME = 'idt1';`).Check(
+ testkit.RowsWithSep("|",
+ "test|idt1|idx_1",
+ "test|idt1|idx_2",
+ "test|idt1|idx_3"))
+ tk.MustQuery(`select TABLE_SCHEMA, TABLE_NAME, INDEX_NAME from information_schema.tidb_index_usage
+ where TABLE_SCHEMA = 'test' and INDEX_NAME = 'idx_2';`).Check(
+ testkit.RowsWithSep("|",
+ "test|idt1|idx_2",
+ "test|idt2|idx_2"))
+ tk.MustQuery(`select TABLE_SCHEMA, TABLE_NAME, INDEX_NAME from information_schema.tidb_index_usage
+ where TABLE_NAME = 'idt1' and INDEX_NAME = 'idx_1';`).Check(
+ testkit.RowsWithSep("|",
+ "test|idt1|idx_1"))
+ tk.MustQuery(`select TABLE_SCHEMA, TABLE_NAME, INDEX_NAME from information_schema.tidb_index_usage
+ where TABLE_SCHEMA = 'test' and TABLE_NAME = 'idt2' and INDEX_NAME = 'idx_4';`).Check(
+ testkit.RowsWithSep("|",
+ "test|idt2|idx_4"))
+ tk.MustQuery(`select count(*) from information_schema.tidb_index_usage;`).Check(
+ testkit.RowsWithSep("|", "72"))
+
+ tk.MustQuery(`select TABLE_SCHEMA, TABLE_NAME, INDEX_NAME from information_schema.tidb_index_usage
+ where TABLE_SCHEMA = 'test1';`).Check(testkit.Rows())
+ tk.MustQuery(`select TABLE_SCHEMA, TABLE_NAME, INDEX_NAME from information_schema.tidb_index_usage
+ where TABLE_NAME = 'idt3';`).Check(testkit.Rows())
+ tk.MustQuery(`select TABLE_SCHEMA, TABLE_NAME, INDEX_NAME from information_schema.tidb_index_usage
+ where INDEX_NAME = 'IDX_5';`).Check(testkit.Rows())
+ tk.MustQuery(`select TABLE_SCHEMA, TABLE_NAME, INDEX_NAME from information_schema.tidb_index_usage
+ where TABLE_SCHEMA = 'test' and TABLE_NAME = 'idt0';`).Check(testkit.Rows())
+ tk.MustQuery(`select TABLE_SCHEMA, TABLE_NAME, INDEX_NAME from information_schema.tidb_index_usage
+ where TABLE_SCHEMA = 'test1' and INDEX_NAME = 'idx_2';`).Check(testkit.Rows())
+ tk.MustQuery(`select TABLE_SCHEMA, TABLE_NAME, INDEX_NAME from information_schema.tidb_index_usage
+ where TABLE_NAME = 'idt2' and INDEX_NAME = 'idx_3';`).Check(testkit.Rows())
+ tk.MustQuery(`select TABLE_SCHEMA, TABLE_NAME, INDEX_NAME from information_schema.tidb_index_usage
+ where TABLE_SCHEMA = 'test' and TABLE_NAME = 'idt1' and INDEX_NAME = 'idx_4';`).Check(testkit.Rows())
+}
+
// https://github.com/pingcap/tidb/issues/32459.
func TestJoinSystemTableContainsView(t *testing.T) {
store := testkit.CreateMockStore(t)
@@ -500,7 +618,7 @@ func TestJoinSystemTableContainsView(t *testing.T) {
// This is used by grafana when TiDB is specified as the data source.
// See https://github.com/grafana/grafana/blob/e86b6662a187c77656f72bef3b0022bf5ced8b98/public/app/plugins/datasource/mysql/meta_query.ts#L31
for i := 0; i < 10; i++ {
- tk.MustQuery(`
+ tk.MustQueryWithContext(context.Background(), `
SELECT
table_name as table_name,
( SELECT
@@ -627,3 +745,152 @@ func TestReferencedTableSchemaWithForeignKey(t *testing.T) {
WHERE table_name = 't2' AND table_schema = 'test2';`).Check(testkit.Rows(
"id id t1 test2 test"))
}
+
+func TestSameTableNameInTwoSchemas(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("create database test1;")
+ tk.MustExec("create database test2;")
+ tk.MustExec("create table test1.t (a int);")
+ tk.MustExec("create table test2.t (a int);")
+
+ rs := tk.MustQuery("select tidb_table_id from information_schema.tables where table_name = 't' and table_schema = 'test1';").Rows()
+ t1ID, err := strconv.Atoi(rs[0][0].(string))
+ require.NoError(t, err)
+ rs = tk.MustQuery("select tidb_table_id from information_schema.tables where table_name = 't' and table_schema = 'test2';").Rows()
+ t2ID, err := strconv.Atoi(rs[0][0].(string))
+ require.NoError(t, err)
+
+ tk.MustQuery(fmt.Sprintf("select table_schema, table_name, tidb_table_id from information_schema.tables where tidb_table_id = %d;", t1ID)).
+ Check(testkit.Rows(fmt.Sprintf("test1 t %d", t1ID)))
+ tk.MustQuery(fmt.Sprintf("select table_schema, table_name, tidb_table_id from information_schema.tables where tidb_table_id = %d;", t2ID)).
+ Check(testkit.Rows(fmt.Sprintf("test2 t %d", t2ID)))
+
+ tk.MustQuery(fmt.Sprintf("select table_schema, table_name, tidb_table_id from information_schema.tables where table_name = 't' and tidb_table_id = %d;", t1ID)).
+ Check(testkit.Rows(fmt.Sprintf("test1 t %d", t1ID)))
+ tk.MustQuery(fmt.Sprintf("select table_schema, table_name, tidb_table_id from information_schema.tables where table_schema = 'test1' and tidb_table_id = %d;", t1ID)).
+ Check(testkit.Rows(fmt.Sprintf("test1 t %d", t1ID)))
+ tk.MustQuery(fmt.Sprintf("select table_schema, table_name, tidb_table_id from information_schema.tables where table_name = 'unknown' and tidb_table_id = %d;", t1ID)).
+ Check(testkit.Rows())
+ tk.MustQuery(fmt.Sprintf("select table_schema, table_name, tidb_table_id from information_schema.tables where table_schema = 'unknown' and tidb_table_id = %d;", t1ID)).
+ Check(testkit.Rows())
+}
+
+func TestInfoSchemaConditionWorks(t *testing.T) {
+ // this test creates table in different schema with different index name, and check
+ // the condition in the following columns whether work as expected.
+ //
+ // - "table_schema"
+ // - "constraint_schema"
+ // - "table_name"
+ // - "constraint_name"
+ // - "partition_name"
+ // - "schema_name"
+ // - "index_name"
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ for db := 0; db < 2; db++ {
+ for table := 0; table < 2; table++ {
+ tk.MustExec(fmt.Sprintf("create database if not exists Db%d;", db))
+ tk.MustExec(fmt.Sprintf(`create table Db%d.Table%d (id int primary key, data0 varchar(255), data1 varchar(255))
+ partition by range (id) (
+ partition p0 values less than (10),
+ partition p1 values less than (20)
+ );`, db, table))
+ for index := 0; index < 2; index++ {
+ tk.MustExec(fmt.Sprintf("create unique index Idx%d on Db%d.Table%d (id, data%d);", index, db, table, index))
+ }
+ }
+ }
+
+ testColumns := map[string]string{
+ "table_schema": "db",
+ "constraint_schema": "db",
+ "table_name": "table",
+ "constraint_name": "idx",
+ "partition_name": "p",
+ "schema_name": "db",
+ "index_name": "idx",
+ }
+ testTables := []string{}
+ for _, row := range tk.MustQuery("show tables in information_schema").Rows() {
+ tableName := row[0].(string)
+ // exclude some tables which cannot run without TiKV.
+ if strings.HasPrefix(tableName, "CLUSTER_") ||
+ strings.HasPrefix(tableName, "INSPECTION_") ||
+ strings.HasPrefix(tableName, "METRICS_") ||
+ strings.HasPrefix(tableName, "TIFLASH_") ||
+ strings.HasPrefix(tableName, "TIKV_") ||
+ strings.HasPrefix(tableName, "USER_") ||
+ tableName == "TABLE_STORAGE_STATS" ||
+ strings.Contains(tableName, "REGION") {
+ continue
+ }
+ testTables = append(testTables, row[0].(string))
+ }
+ for _, table := range testTables {
+ rs, err := tk.Exec(fmt.Sprintf("select * from information_schema.%s", table))
+ require.NoError(t, err)
+ cols := rs.Fields()
+
+ chk := rs.NewChunk(nil)
+ rowCount := 0
+ for {
+ err := rs.Next(context.Background(), chk)
+ require.NoError(t, err)
+ if chk.NumRows() == 0 {
+ break
+ }
+ rowCount += chk.NumRows()
+ }
+ if rowCount == 0 {
+ // TODO: find a way to test the table without any rows by adding some rows to them.
+ continue
+ }
+ for i := 0; i < len(cols); i++ {
+ colName := cols[i].Column.Name.L
+ if valPrefix, ok := testColumns[colName]; ok {
+ for j := 0; j < 2; j++ {
+ sql := fmt.Sprintf("select * from information_schema.%s where %s = '%s%d';",
+ table, colName, valPrefix, j)
+ rows := tk.MustQuery(sql).Rows()
+ rowCountWithCondition := len(rows)
+ require.Less(t, rowCountWithCondition, rowCount, "%s has no effect on %s. SQL: %s", colName, table, sql)
+
+ // check the condition works as expected
+ for _, row := range rows {
+ require.Equal(t, fmt.Sprintf("%s%d", valPrefix, j), strings.ToLower(row[i].(string)),
+ "%s has no effect on %s. SQL: %s", colName, table, sql)
+ }
+ }
+ }
+ }
+ }
+
+ // Test the PRIMARY constraint filter
+ rows := tk.MustQuery("select constraint_name, table_schema from information_schema.table_constraints where constraint_name = 'PRIMARY' and table_schema = 'db0';").Rows()
+ require.Equal(t, 2, len(rows))
+ for _, row := range rows {
+ require.Equal(t, "PRIMARY", row[0].(string))
+ require.Equal(t, "Db0", row[1].(string))
+ }
+ rows = tk.MustQuery("select constraint_name, table_schema from information_schema.key_column_usage where constraint_name = 'PRIMARY' and table_schema = 'db1';").Rows()
+ require.Equal(t, 2, len(rows))
+ for _, row := range rows {
+ require.Equal(t, "PRIMARY", row[0].(string))
+ require.Equal(t, "Db1", row[1].(string))
+ }
+
+ // Test the `partition_name` filter
+ tk.MustExec("create database if not exists db_no_partition;")
+ tk.MustExec("create table db_no_partition.t_no_partition (id int primary key, data0 varchar(255), data1 varchar(255));")
+ tk.MustExec(`create table db_no_partition.t_partition (id int primary key, data0 varchar(255), data1 varchar(255))
+ partition by range (id) (
+ partition p0 values less than (10),
+ partition p1 values less than (20)
+ );`)
+ rows = tk.MustQuery("select * from information_schema.partitions where table_schema = 'db_no_partition' and partition_name is NULL;").Rows()
+ require.Equal(t, 1, len(rows))
+ rows = tk.MustQuery("select * from information_schema.partitions where table_schema = 'db_no_partition' and (partition_name is NULL or partition_name = 'p0');").Rows()
+ require.Equal(t, 2, len(rows))
+}
diff --git a/pkg/executor/insert.go b/pkg/executor/insert.go
index 0878dabf8be1d..a2b5174b075af 100644
--- a/pkg/executor/insert.go
+++ b/pkg/executor/insert.go
@@ -22,7 +22,6 @@ import (
"time"
"github.com/pingcap/errors"
- "github.com/pingcap/tidb/pkg/errctx"
"github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/executor/internal/exec"
"github.com/pingcap/tidb/pkg/expression"
@@ -31,6 +30,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/parser/terror"
+ "github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/table/tables"
"github.com/pingcap/tidb/pkg/tablecodec"
@@ -47,6 +47,7 @@ import (
type InsertExec struct {
*InsertValues
OnDuplicate []*expression.Assignment
+ IgnoreErr bool
evalBuffer4Dup chunk.MutRow
curInsertVals chunk.MutRow
row4Update []types.Datum
@@ -66,7 +67,6 @@ func (e *InsertExec) exec(ctx context.Context, rows [][]types.Datum) error {
// If tidb_batch_insert is ON and not in a transaction, we could use BatchInsert mode.
sessVars := e.Ctx().GetSessionVars()
defer sessVars.CleanBuffers()
- ignoreErr := sessVars.StmtCtx.ErrGroupLevel(errctx.ErrGroupDupKey) != errctx.LevelError
txn, err := e.Ctx().Txn(true)
if err != nil {
@@ -92,13 +92,14 @@ func (e *InsertExec) exec(ctx context.Context, rows [][]types.Datum) error {
if err != nil {
return err
}
- } else if ignoreErr {
+ } else if e.IgnoreErr {
err := e.batchCheckAndInsert(ctx, rows, e.addRecord, false)
if err != nil {
return err
}
} else {
start := time.Now()
+ dupKeyCheck := optimizeDupKeyCheckForNormalInsert(sessVars, txn)
for i, row := range rows {
var err error
sizeHintStep := int(sessVars.ShardAllocateStep)
@@ -108,9 +109,9 @@ func (e *InsertExec) exec(ctx context.Context, rows [][]types.Datum) error {
if sizeHint > remain {
sizeHint = remain
}
- err = e.addRecordWithAutoIDHint(ctx, row, sizeHint)
+ err = e.addRecordWithAutoIDHint(ctx, row, sizeHint, dupKeyCheck)
} else {
- err = e.addRecord(ctx, row)
+ err = e.addRecord(ctx, row, dupKeyCheck)
}
if err != nil {
return err
@@ -193,7 +194,7 @@ func (e *InsertValues) prefetchDataCache(ctx context.Context, txn kv.Transaction
}
// updateDupRow updates a duplicate row to a new row.
-func (e *InsertExec) updateDupRow(ctx context.Context, idxInBatch int, txn kv.Transaction, row toBeCheckedRow, handle kv.Handle, _ []*expression.Assignment) error {
+func (e *InsertExec) updateDupRow(ctx context.Context, idxInBatch int, txn kv.Transaction, row toBeCheckedRow, handle kv.Handle, _ []*expression.Assignment, dupKeyCheck table.DupKeyCheckMode) error {
oldRow, err := getOldRow(ctx, e.Ctx(), txn, row.t, handle, e.GenExprs)
if err != nil {
return err
@@ -204,7 +205,7 @@ func (e *InsertExec) updateDupRow(ctx context.Context, idxInBatch int, txn kv.Tr
extraCols = e.Ctx().GetSessionVars().CurrInsertBatchExtraCols[idxInBatch]
}
- err = e.doDupRowUpdate(ctx, handle, oldRow, row.row, extraCols, e.OnDuplicate, idxInBatch)
+ err = e.doDupRowUpdate(ctx, handle, oldRow, row.row, extraCols, e.OnDuplicate, idxInBatch, dupKeyCheck)
if kv.ErrKeyExists.Equal(err) || table.ErrCheckConstraintViolated.Equal(err) {
ec := e.Ctx().GetSessionVars().StmtCtx.ErrCtx()
return ec.HandleErrorWithAlias(kv.ErrKeyExists, err, err)
@@ -236,6 +237,18 @@ func (e *InsertExec) batchUpdateDupRows(ctx context.Context, newRows [][]types.D
e.stats.Prefetch += time.Since(prefetchStart)
}
+ // Use `optimizeDupKeyCheckForUpdate` to determine the update operation when the row meets the conflict in
+ // `INSERT ... ON DUPLICATE KEY UPDATE` statement.
+ // Though it is in an insert statement, `ON DUP KEY UPDATE` follows the dup-key check behavior of update.
+ // For example, it will ignore variable `tidb_constraint_check_in_place`, see the test case:
+ // https://github.com/pingcap/tidb/blob/3117d3fae50bbb5dabcde7b9589f92bfbbda5dc6/pkg/executor/test/writetest/write_test.go#L419-L426
+ updateDupKeyCheck := optimizeDupKeyCheckForUpdate(txn, e.IgnoreErr)
+ // Do not use `updateDupKeyCheck` for `AddRecord` because it is not optimized for insert.
+ // It seems that we can just use `DupKeyCheckSkip` here because all constraints are checked.
+ // But we still use `optimizeDupKeyCheckForNormalInsert` to make the refactor same behavior with the original code.
+ // TODO: just use `DupKeyCheckSkip` here.
+ addRecordDupKeyCheck := optimizeDupKeyCheckForNormalInsert(e.Ctx().GetSessionVars(), txn)
+
for i, r := range toBeCheckedRows {
if r.handleKey != nil {
handle, err := tablecodec.DecodeRowKey(r.handleKey.newKey)
@@ -243,7 +256,7 @@ func (e *InsertExec) batchUpdateDupRows(ctx context.Context, newRows [][]types.D
return err
}
- err = e.updateDupRow(ctx, i, txn, r, handle, e.OnDuplicate)
+ err = e.updateDupRow(ctx, i, txn, r, handle, e.OnDuplicate, updateDupKeyCheck)
if err == nil {
continue
}
@@ -260,7 +273,7 @@ func (e *InsertExec) batchUpdateDupRows(ctx context.Context, newRows [][]types.D
if handle == nil {
continue
}
- err = e.updateDupRow(ctx, i, txn, r, handle, e.OnDuplicate)
+ err = e.updateDupRow(ctx, i, txn, r, handle, e.OnDuplicate, updateDupKeyCheck)
if err != nil {
if kv.IsErrNotFound(err) {
// Data index inconsistent? A unique key provide the handle information, but the
@@ -282,7 +295,7 @@ func (e *InsertExec) batchUpdateDupRows(ctx context.Context, newRows [][]types.D
// and key-values should be filled back to dupOldRowValues for the further row check,
// due to there may be duplicate keys inside the insert statement.
if newRows[i] != nil {
- err := e.addRecord(ctx, newRows[i])
+ err := e.addRecord(ctx, newRows[i], addRecordDupKeyCheck)
if err != nil {
return err
}
@@ -294,6 +307,41 @@ func (e *InsertExec) batchUpdateDupRows(ctx context.Context, newRows [][]types.D
return nil
}
+// optimizeDupKeyCheckForNormalInsert trys to optimize the DupKeyCheckMode for an insert statement according to the
+// transaction and system variables.
+// If the DupKeyCheckMode of the current statement can be optimized, it will return `DupKeyCheckLazy` to avoid the
+// redundant requests to TiKV, otherwise, `DupKeyCheckInPlace` will be returned.
+// This method only works for "normal" insert statements, that means the options like "IGNORE" and "ON DUPLICATE KEY"
+// in a statement are not considerate, and callers should handle the above cases by themselves.
+func optimizeDupKeyCheckForNormalInsert(vars *variable.SessionVars, txn kv.Transaction) table.DupKeyCheckMode {
+ if !vars.ConstraintCheckInPlace || txn.IsPessimistic() || txn.IsPipelined() {
+ // We can just check duplicated key lazily without keys in storage for the below cases:
+ // - `txn.Pipelined()` is true.
+ // It means the user is using `@@tidb_dml_type="bulk"` to insert rows in bulk mode.
+ // DupKeyCheckLazy should be used to improve the performance.
+ // - The current transaction is pessimistic. The duplicate key check can be postponed to the lock stage.
+ // - The current transaction is optimistic but `tidb_constraint_check_in_place` is set to false.
+ return table.DupKeyCheckLazy
+ }
+ return table.DupKeyCheckInPlace
+}
+
+// getPessimisticLazyCheckMode returns the lazy check mode for pessimistic txn.
+// The returned `PessimisticLazyDupKeyCheckMode` only takes effect for pessimistic txn with `DupKeyCheckLazy`;
+// otherwise, this option will be ignored.
+func getPessimisticLazyCheckMode(vars *variable.SessionVars) table.PessimisticLazyDupKeyCheckMode {
+ if !vars.ConstraintCheckInPlacePessimistic && vars.InTxn() && !vars.InRestrictedSQL && vars.ConnectionID > 0 {
+ // We can postpone the duplicated key check to the prewrite stage when both of the following conditions are met:
+ // - `tidb_constraint_check_in_place_pessimistic='OFF'`.
+ // - The current transaction should be an explicit transaction because an autocommit txn cannot get
+ // any benefits from checking the duplicated key in the prewrite stage.
+ // - The current connection is a user connection, and we always check duplicated key in place for
+ // internal connections.
+ return table.DupKeyCheckInPrewrite
+ }
+ return table.DupKeyCheckInAcquireLock
+}
+
// Next implements the Executor Next interface.
func (e *InsertExec) Next(ctx context.Context, req *chunk.Chunk) error {
req.Reset()
@@ -379,7 +427,7 @@ func (e *InsertExec) initEvalBuffer4Dup() {
// doDupRowUpdate updates the duplicate row.
func (e *InsertExec) doDupRowUpdate(ctx context.Context, handle kv.Handle, oldRow []types.Datum, newRow []types.Datum,
- extraCols []types.Datum, cols []*expression.Assignment, idxInBatch int) error {
+ extraCols []types.Datum, cols []*expression.Assignment, idxInBatch int, dupKeyMode table.DupKeyCheckMode) error {
assignFlag := make([]bool, len(e.Table.WritableCols()))
// See http://dev.mysql.com/doc/refman/5.7/en/miscellaneous-functions.html#function_values
e.curInsertVals.SetDatums(newRow...)
@@ -426,7 +474,7 @@ func (e *InsertExec) doDupRowUpdate(ctx context.Context, handle kv.Handle, oldRo
}
newData := e.row4Update[:len(oldRow)]
- _, err := updateRecord(ctx, e.Ctx(), handle, oldRow, newData, assignFlag, e.Table, true, e.memTracker, e.fkChecks, e.fkCascades)
+ _, err := updateRecord(ctx, e.Ctx(), handle, oldRow, newData, assignFlag, e.Table, true, e.memTracker, e.fkChecks, e.fkCascades, dupKeyMode)
if err != nil {
return err
}
@@ -440,7 +488,7 @@ func (e *InsertExec) setMessage() {
if e.SelectExec != nil || numRecords > 1 {
numWarnings := stmtCtx.WarningCount()
var numDuplicates uint64
- if stmtCtx.ErrGroupLevel(errctx.ErrGroupDupKey) != errctx.LevelError {
+ if e.IgnoreErr {
// if ignoreErr
numDuplicates = numRecords - stmtCtx.CopiedRows()
} else {
diff --git a/pkg/executor/insert_common.go b/pkg/executor/insert_common.go
index f1353406bb287..27d6918a468a9 100644
--- a/pkg/executor/insert_common.go
+++ b/pkg/executor/insert_common.go
@@ -694,7 +694,7 @@ func (e *InsertValues) fillRow(ctx context.Context, row []types.Datum, hasValue
// Handle exchange partition
tbl := e.Table.Meta()
if tbl.ExchangePartitionInfo != nil && tbl.GetPartitionInfo() == nil {
- if err := checkRowForExchangePartition(e.Ctx().GetTableCtx(), row, tbl); err != nil {
+ if err := checkRowForExchangePartition(e.Ctx(), row, tbl); err != nil {
return nil, err
}
}
@@ -1190,11 +1190,9 @@ func (e *InsertValues) handleDuplicateKey(ctx context.Context, txn kv.Transactio
// All duplicate rows will be ignored and appended as duplicate warnings.
func (e *InsertValues) batchCheckAndInsert(
ctx context.Context, rows [][]types.Datum,
- addRecord func(ctx context.Context, row []types.Datum) error,
+ addRecord func(ctx context.Context, row []types.Datum, dupKeyCheck table.DupKeyCheckMode) error,
replace bool,
) error {
- // all the rows will be checked, so it is safe to set BatchCheck = true
- e.Ctx().GetSessionVars().StmtCtx.BatchCheck = true
defer tracing.StartRegion(ctx, "InsertValues.batchCheckAndInsert").End()
start := time.Now()
// Get keys need to be checked.
@@ -1307,7 +1305,8 @@ func (e *InsertValues) batchCheckAndInsert(
// it should be added to values map for the further row check.
// There may be duplicate keys inside the insert statement.
e.Ctx().GetSessionVars().StmtCtx.AddCopiedRows(1)
- err = addRecord(ctx, rows[i])
+ // all the rows have been checked, so it is safe to use DupKeyCheckSkip
+ err = addRecord(ctx, rows[i], table.DupKeyCheckSkip)
if err != nil {
// throw warning when violate check constraint
if table.ErrCheckConstraintViolated.Equal(err) {
@@ -1368,9 +1367,9 @@ func (e *InsertValues) removeRow(
}
if ph, ok := handle.(kv.PartitionHandle); ok {
- err = e.Table.(table.PartitionedTable).GetPartition(ph.PartitionID).RemoveRecord(e.Ctx().GetTableCtx(), ph.Handle, oldRow)
+ err = e.Table.(table.PartitionedTable).GetPartition(ph.PartitionID).RemoveRecord(e.Ctx().GetTableCtx(), txn, ph.Handle, oldRow)
} else {
- err = r.t.RemoveRecord(e.Ctx().GetTableCtx(), handle, oldRow)
+ err = r.t.RemoveRecord(e.Ctx().GetTableCtx(), txn, handle, oldRow)
}
if err != nil {
return false, err
@@ -1405,23 +1404,24 @@ func (e *InsertValues) equalDatumsAsBinary(a []types.Datum, b []types.Datum) (bo
return true, nil
}
-func (e *InsertValues) addRecord(ctx context.Context, row []types.Datum) error {
- return e.addRecordWithAutoIDHint(ctx, row, 0)
+func (e *InsertValues) addRecord(ctx context.Context, row []types.Datum, dupKeyCheck table.DupKeyCheckMode) error {
+ return e.addRecordWithAutoIDHint(ctx, row, 0, dupKeyCheck)
}
func (e *InsertValues) addRecordWithAutoIDHint(
- ctx context.Context, row []types.Datum, reserveAutoIDCount int,
+ ctx context.Context, row []types.Datum, reserveAutoIDCount int, dupKeyCheck table.DupKeyCheckMode,
) (err error) {
vars := e.Ctx().GetSessionVars()
- if !vars.ConstraintCheckInPlace {
- vars.PresumeKeyNotExists = true
+ txn, err := e.Ctx().Txn(true)
+ if err != nil {
+ return err
}
+ pessimisticLazyCheck := getPessimisticLazyCheckMode(vars)
if reserveAutoIDCount > 0 {
- _, err = e.Table.AddRecord(e.Ctx().GetTableCtx(), row, table.WithCtx(ctx), table.WithReserveAutoIDHint(reserveAutoIDCount))
+ _, err = e.Table.AddRecord(e.Ctx().GetTableCtx(), txn, row, table.WithCtx(ctx), table.WithReserveAutoIDHint(reserveAutoIDCount), dupKeyCheck, pessimisticLazyCheck)
} else {
- _, err = e.Table.AddRecord(e.Ctx().GetTableCtx(), row, table.WithCtx(ctx))
+ _, err = e.Table.AddRecord(e.Ctx().GetTableCtx(), txn, row, table.WithCtx(ctx), dupKeyCheck, pessimisticLazyCheck)
}
- vars.PresumeKeyNotExists = false
if err != nil {
return err
}
@@ -1429,7 +1429,7 @@ func (e *InsertValues) addRecordWithAutoIDHint(
if e.lastInsertID != 0 {
vars.SetLastInsertID(e.lastInsertID)
}
- if !vars.StmtCtx.BatchCheck {
+ if dupKeyCheck != table.DupKeyCheckSkip {
for _, fkc := range e.fkChecks {
err = fkc.insertRowNeedToCheck(vars.StmtCtx, row)
if err != nil {
diff --git a/pkg/executor/internal/exec/indexusage_test.go b/pkg/executor/internal/exec/indexusage_test.go
index 65f544763f98f..4377d39f25cb7 100644
--- a/pkg/executor/internal/exec/indexusage_test.go
+++ b/pkg/executor/internal/exec/indexusage_test.go
@@ -363,7 +363,7 @@ func TestIndexUsageReporterWithGlobalIndex(t *testing.T) {
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("set tidb_enable_global_index='on'")
- tk.MustExec(`create table t (pk int primary key, id_1 int, unique key idx_1(id_1))
+ tk.MustExec(`create table t (pk int primary key, id_1 int, unique key idx_1(id_1) global)
partition by range (pk) (
partition p0 values less than (10),
partition p1 values less than (20),
diff --git a/pkg/executor/internal/querywatch/query_watch.go b/pkg/executor/internal/querywatch/query_watch.go
index 8fbc2b72be5f4..07b19104399e5 100644
--- a/pkg/executor/internal/querywatch/query_watch.go
+++ b/pkg/executor/internal/querywatch/query_watch.go
@@ -132,7 +132,7 @@ func fromQueryWatchOptionList(ctx context.Context, sctx, newSctx sessionctx.Cont
// validateWatchRecord follows several designs:
// 1. If no resource group is set, the default resource group is used
-// 2. If no action is specified, the action of the resource group is used. If no, an error message is displayed.
+// 2. If no action is specified, the action of the resource group is used. If no, an error message displayed.
func validateWatchRecord(record *resourcegroup.QuarantineRecord, client *rmclient.ResourceGroupsController) error {
if len(record.ResourceGroupName) == 0 {
record.ResourceGroupName = resourcegroup.DefaultResourceGroupName
diff --git a/pkg/executor/internal/testutil/BUILD.bazel b/pkg/executor/internal/testutil/BUILD.bazel
index 40d5d3d0a12de..4f1b73b227dd0 100644
--- a/pkg/executor/internal/testutil/BUILD.bazel
+++ b/pkg/executor/internal/testutil/BUILD.bazel
@@ -16,8 +16,8 @@ go_library(
"//pkg/expression",
"//pkg/parser/ast",
"//pkg/parser/mysql",
- "//pkg/planner/core",
"//pkg/planner/core/base",
+ "//pkg/planner/core/operator/logicalop",
"//pkg/planner/property",
"//pkg/sessionctx",
"//pkg/sessionctx/variable",
diff --git a/pkg/executor/internal/testutil/window.go b/pkg/executor/internal/testutil/window.go
index d80d4560648ec..cf3dfe85b8801 100644
--- a/pkg/executor/internal/testutil/window.go
+++ b/pkg/executor/internal/testutil/window.go
@@ -21,7 +21,7 @@ import (
"github.com/pingcap/tidb/pkg/expression"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/mysql"
- "github.com/pingcap/tidb/pkg/planner/core"
+ "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/types"
@@ -31,7 +31,7 @@ import (
// WindowTestCase has a fixed schema (col Double, partitionBy LongLong, rawData VarString(16), col LongLong).
type WindowTestCase struct {
Ctx sessionctx.Context
- Frame *core.WindowFrame
+ Frame *logicalop.WindowFrame
WindowFunc string
RawDataSmall string
Columns []*expression.Column
diff --git a/pkg/executor/join/BUILD.bazel b/pkg/executor/join/BUILD.bazel
index d034744556ed2..e5280312ad640 100644
--- a/pkg/executor/join/BUILD.bazel
+++ b/pkg/executor/join/BUILD.bazel
@@ -31,6 +31,7 @@ go_library(
"//pkg/parser/mysql",
"//pkg/parser/terror",
"//pkg/planner/core",
+ "//pkg/planner/core/operator/logicalop",
"//pkg/sessionctx",
"//pkg/sessionctx/stmtctx",
"//pkg/sessionctx/variable",
@@ -79,7 +80,7 @@ go_test(
],
embed = [":join"],
flaky = True,
- shard_count = 47,
+ shard_count = 49,
deps = [
"//pkg/config",
"//pkg/domain",
@@ -87,7 +88,8 @@ go_test(
"//pkg/expression",
"//pkg/parser/ast",
"//pkg/parser/mysql",
- "//pkg/planner/core",
+ "//pkg/planner/core/operator/logicalop",
+ "//pkg/session",
"//pkg/sessionctx",
"//pkg/sessionctx/variable",
"//pkg/testkit",
diff --git a/pkg/executor/join/base_join_probe.go b/pkg/executor/join/base_join_probe.go
index b7eb65f0212b3..6ef5704733956 100644
--- a/pkg/executor/join/base_join_probe.go
+++ b/pkg/executor/join/base_join_probe.go
@@ -23,7 +23,7 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/expression"
"github.com/pingcap/tidb/pkg/parser/mysql"
- "github.com/pingcap/tidb/pkg/planner/core"
+ "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/chunk"
"github.com/pingcap/tidb/pkg/util/codec"
@@ -66,6 +66,10 @@ type ProbeV2 interface {
NeedScanRowTable() bool
// InitForScanRowTable do some pre-work before ScanRowTable, it must be called before ScanRowTable
InitForScanRowTable()
+ // Return probe collsion
+ GetProbeCollision() uint64
+ // Reset probe collsion
+ ResetProbeCollision()
}
type offsetAndLength struct {
@@ -130,6 +134,16 @@ type baseJoinProbe struct {
tmpChk *chunk.Chunk
rowIndexInfos []*matchedRowInfo
selected []bool
+
+ probeCollision uint64
+}
+
+func (j *baseJoinProbe) GetProbeCollision() uint64 {
+ return j.probeCollision
+}
+
+func (j *baseJoinProbe) ResetProbeCollision() {
+ j.probeCollision = 0
}
func (j *baseJoinProbe) IsCurrentChunkProbeDone() bool {
@@ -484,7 +498,7 @@ func isKeyMatched(keyMode keyMode, serializedKey []byte, rowStart unsafe.Pointer
}
// NewJoinProbe create a join probe used for hash join v2
-func NewJoinProbe(ctx *HashJoinCtxV2, workID uint, joinType core.JoinType, keyIndex []int, joinedColumnTypes, probeKeyTypes []*types.FieldType, rightAsBuildSide bool) ProbeV2 {
+func NewJoinProbe(ctx *HashJoinCtxV2, workID uint, joinType logicalop.JoinType, keyIndex []int, joinedColumnTypes, probeKeyTypes []*types.FieldType, rightAsBuildSide bool) ProbeV2 {
base := baseJoinProbe{
ctx: ctx,
workID: workID,
@@ -526,11 +540,11 @@ func NewJoinProbe(ctx *HashJoinCtxV2, workID uint, joinType core.JoinType, keyIn
base.rowIndexInfos = make([]*matchedRowInfo, 0, chunk.InitialCapacity)
}
switch joinType {
- case core.InnerJoin:
+ case logicalop.InnerJoin:
return &innerJoinProbe{base}
- case core.LeftOuterJoin:
+ case logicalop.LeftOuterJoin:
return newOuterJoinProbe(base, !rightAsBuildSide, rightAsBuildSide)
- case core.RightOuterJoin:
+ case logicalop.RightOuterJoin:
return newOuterJoinProbe(base, rightAsBuildSide, rightAsBuildSide)
default:
panic("unsupported join type")
diff --git a/pkg/executor/join/hash_join_base.go b/pkg/executor/join/hash_join_base.go
index 7947b96653e6c..573f7614b99b1 100644
--- a/pkg/executor/join/hash_join_base.go
+++ b/pkg/executor/join/hash_join_base.go
@@ -25,7 +25,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/executor/internal/exec"
- plannercore "github.com/pingcap/tidb/pkg/planner/core"
+ "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/chunk"
@@ -55,7 +55,7 @@ type hashJoinCtxBase struct {
finished atomic.Bool
IsNullEQ []bool
buildFinished chan error
- JoinType plannercore.JoinType
+ JoinType logicalop.JoinType
IsNullAware bool
memTracker *memory.Tracker // track memory usage.
diskTracker *disk.Tracker // track disk usage.
diff --git a/pkg/executor/join/hash_join_v1.go b/pkg/executor/join/hash_join_v1.go
index 649eac1eb466b..981c652e5a5a3 100644
--- a/pkg/executor/join/hash_join_v1.go
+++ b/pkg/executor/join/hash_join_v1.go
@@ -30,7 +30,7 @@ import (
"github.com/pingcap/tidb/pkg/executor/unionexec"
"github.com/pingcap/tidb/pkg/expression"
"github.com/pingcap/tidb/pkg/parser/terror"
- plannercore "github.com/pingcap/tidb/pkg/planner/core"
+ "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/types"
@@ -216,7 +216,7 @@ func (e *HashJoinV1Exec) fetchAndProbeHashTable(ctx context.Context) {
defer trace.StartRegion(ctx, "HashJoinProbeSideFetcher").End()
e.ProbeSideTupleFetcher.fetchProbeSideChunks(ctx, e.MaxChunkSize(), func() bool {
return e.ProbeSideTupleFetcher.RowContainer.Len() == uint64(0)
- }, e.ProbeSideTupleFetcher.JoinType == plannercore.InnerJoin || e.ProbeSideTupleFetcher.JoinType == plannercore.SemiJoin,
+ }, e.ProbeSideTupleFetcher.JoinType == logicalop.InnerJoin || e.ProbeSideTupleFetcher.JoinType == logicalop.SemiJoin,
false, e.ProbeSideTupleFetcher.IsOuterJoin, &e.ProbeSideTupleFetcher.hashJoinCtxBase)
}, e.ProbeSideTupleFetcher.handleProbeSideFetcherPanic)
@@ -737,8 +737,8 @@ func (w *ProbeWorkerV1) joinNAASJMatchProbeSideRow2Chunk(probeKey uint64, probeK
// For NA-AntiLeftOuterSemiJoin, we couldn't match null-bucket first, because once y set has a same key x and null
// key, we should return the result as left side row appended with a scalar value 0 which is from same key matching failure.
func (w *ProbeWorkerV1) joinNAAJMatchProbeSideRow2Chunk(probeKey uint64, probeKeyNullBits *bitmap.ConcurrentBitmap, probeSideRow chunk.Row, hCtx *HashContext, joinResult *hashjoinWorkerResult) (bool, int64, *hashjoinWorkerResult) {
- naAntiSemiJoin := w.HashJoinCtx.JoinType == plannercore.AntiSemiJoin && w.HashJoinCtx.IsNullAware
- naAntiLeftOuterSemiJoin := w.HashJoinCtx.JoinType == plannercore.AntiLeftOuterSemiJoin && w.HashJoinCtx.IsNullAware
+ naAntiSemiJoin := w.HashJoinCtx.JoinType == logicalop.AntiSemiJoin && w.HashJoinCtx.IsNullAware
+ naAntiLeftOuterSemiJoin := w.HashJoinCtx.JoinType == logicalop.AntiLeftOuterSemiJoin && w.HashJoinCtx.IsNullAware
if naAntiSemiJoin {
return w.joinNAASJMatchProbeSideRow2Chunk(probeKey, probeKeyNullBits, probeSideRow, hCtx, joinResult)
}
diff --git a/pkg/executor/join/hash_join_v2.go b/pkg/executor/join/hash_join_v2.go
index b77ea183442cb..e510e82c528f4 100644
--- a/pkg/executor/join/hash_join_v2.go
+++ b/pkg/executor/join/hash_join_v2.go
@@ -28,7 +28,7 @@ import (
"github.com/pingcap/tidb/pkg/executor/internal/exec"
"github.com/pingcap/tidb/pkg/expression"
"github.com/pingcap/tidb/pkg/parser/mysql"
- plannercore "github.com/pingcap/tidb/pkg/planner/core"
+ "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/channel"
@@ -44,16 +44,12 @@ var (
enableHashJoinV2 = atomic.Bool{}
)
-func init() {
- enableHashJoinV2.Store(true)
-}
-
// IsHashJoinV2Enabled return true if hash join v2 is enabled
func IsHashJoinV2Enabled() bool {
// sizeOfUintptr should always equal to sizeOfUnsafePointer, because according to golang's doc,
// a Pointer can be converted to an uintptr. Add this check here in case in the future go runtime
// change this
- return enableHashJoinV2.Load() && sizeOfUintptr >= sizeOfUnsafePointer
+ return !heapObjectsCanMove() && enableHashJoinV2.Load() && sizeOfUintptr >= sizeOfUnsafePointer
}
// SetEnableHashJoinV2 enable/disable hash join v2
@@ -339,10 +335,10 @@ func (e *HashJoinV2Exec) Open(ctx context.Context) error {
}
func (fetcher *ProbeSideTupleFetcherV2) shouldLimitProbeFetchSize() bool {
- if fetcher.JoinType == plannercore.LeftOuterJoin && fetcher.RightAsBuildSide {
+ if fetcher.JoinType == logicalop.LeftOuterJoin && fetcher.RightAsBuildSide {
return true
}
- if fetcher.JoinType == plannercore.RightOuterJoin && !fetcher.RightAsBuildSide {
+ if fetcher.JoinType == logicalop.RightOuterJoin && !fetcher.RightAsBuildSide {
return true
}
return false
@@ -378,13 +374,13 @@ func (w *BuildWorkerV2) splitPartitionAndAppendToRowTable(typeCtx types.Context,
func (e *HashJoinV2Exec) canSkipProbeIfHashTableIsEmpty() bool {
switch e.JoinType {
- case plannercore.InnerJoin:
+ case logicalop.InnerJoin:
return true
- case plannercore.LeftOuterJoin:
+ case logicalop.LeftOuterJoin:
return !e.RightAsBuildSide
- case plannercore.RightOuterJoin:
+ case logicalop.RightOuterJoin:
return e.RightAsBuildSide
- case plannercore.SemiJoin:
+ case logicalop.SemiJoin:
return e.RightAsBuildSide
default:
return false
@@ -401,6 +397,7 @@ func (e *HashJoinV2Exec) initializeForProbe() {
for i := uint(0); i < e.Concurrency; i++ {
e.ProbeWorkers[i].initializeForProbe(e.ProbeSideTupleFetcher.probeChkResourceCh, e.ProbeSideTupleFetcher.probeResultChs[i], e)
+ e.ProbeWorkers[i].JoinProbe.ResetProbeCollision()
}
}
@@ -443,6 +440,11 @@ func (e *HashJoinV2Exec) handleJoinWorkerPanic(r any) {
func (e *HashJoinV2Exec) waitJoinWorkersAndCloseResultChan() {
e.workerWg.Wait()
+ if e.stats != nil {
+ for _, prober := range e.ProbeWorkers {
+ e.stats.hashStat.probeCollision += int64(prober.JoinProbe.GetProbeCollision())
+ }
+ }
if e.ProbeWorkers[0] != nil && e.ProbeWorkers[0].JoinProbe.NeedScanRowTable() {
for i := uint(0); i < e.Concurrency; i++ {
var workerID = i
@@ -863,12 +865,20 @@ func (*hashJoinRuntimeStatsV2) Tp() int {
func (e *hashJoinRuntimeStatsV2) String() string {
buf := bytes.NewBuffer(make([]byte, 0, 128))
if e.fetchAndBuildHashTable > 0 {
- buf.WriteString("build_hash_table:{total:")
+ buf.WriteString("build_hash_table:{concurrency:")
+ buf.WriteString(strconv.Itoa(e.concurrent))
+ buf.WriteString(", total:")
buf.WriteString(execdetails.FormatDuration(e.fetchAndBuildHashTable))
buf.WriteString(", fetch:")
buf.WriteString(execdetails.FormatDuration(time.Duration(int64(e.fetchAndBuildHashTable) - e.maxBuildHashTable - e.maxPartitionData)))
+ buf.WriteString(", partition:")
+ buf.WriteString(execdetails.FormatDuration(time.Duration(e.partitionData)))
+ buf.WriteString(", max partition:")
+ buf.WriteString(execdetails.FormatDuration(time.Duration(e.maxPartitionData)))
buf.WriteString(", build:")
buf.WriteString(execdetails.FormatDuration(time.Duration(e.buildHashTable)))
+ buf.WriteString(", max build:")
+ buf.WriteString(execdetails.FormatDuration(time.Duration(e.maxBuildHashTable)))
buf.WriteString("}")
}
if e.probe > 0 {
diff --git a/pkg/executor/join/index_lookup_hash_join.go b/pkg/executor/join/index_lookup_hash_join.go
index b3151def67437..fd368ef77911d 100644
--- a/pkg/executor/join/index_lookup_hash_join.go
+++ b/pkg/executor/join/index_lookup_hash_join.go
@@ -146,7 +146,7 @@ func (e *IndexNestedLoopHashJoin) Open(ctx context.Context) error {
return nil
}
-func (e *IndexNestedLoopHashJoin) startWorkers(ctx context.Context) {
+func (e *IndexNestedLoopHashJoin) startWorkers(ctx context.Context, initBatchSize int) {
concurrency := e.Ctx().GetSessionVars().IndexLookupJoinConcurrency()
if e.stats != nil {
e.stats.concurrency = concurrency
@@ -164,7 +164,7 @@ func (e *IndexNestedLoopHashJoin) startWorkers(ctx context.Context) {
}
e.joinChkResourceCh = make([]chan *chunk.Chunk, concurrency)
e.WorkerWg.Add(1)
- ow := e.newOuterWorker(innerCh)
+ ow := e.newOuterWorker(innerCh, initBatchSize)
go util.WithRecovery(func() { ow.run(e.ctxWithCancel) }, e.finishJoinWorkers)
for i := 0; i < concurrency; i++ {
@@ -221,7 +221,7 @@ func (e *IndexNestedLoopHashJoin) wait4JoinWorkers() {
// Next implements the IndexNestedLoopHashJoin Executor interface.
func (e *IndexNestedLoopHashJoin) Next(ctx context.Context, req *chunk.Chunk) error {
if !e.prepared {
- e.startWorkers(ctx)
+ e.startWorkers(ctx, req.RequiredRows())
e.prepared = true
}
req.Reset()
@@ -411,14 +411,16 @@ func (*indexHashJoinOuterWorker) pushToChan(ctx context.Context, task *indexHash
return false
}
-func (e *IndexNestedLoopHashJoin) newOuterWorker(innerCh chan *indexHashJoinTask) *indexHashJoinOuterWorker {
+func (e *IndexNestedLoopHashJoin) newOuterWorker(innerCh chan *indexHashJoinTask, initBatchSize int) *indexHashJoinOuterWorker {
+ maxBatchSize := e.Ctx().GetSessionVars().IndexJoinBatchSize
+ batchSize := min(initBatchSize, maxBatchSize)
ow := &indexHashJoinOuterWorker{
outerWorker: outerWorker{
OuterCtx: e.OuterCtx,
ctx: e.Ctx(),
executor: e.Children(0),
- batchSize: 32,
- maxBatchSize: e.Ctx().GetSessionVars().IndexJoinBatchSize,
+ batchSize: batchSize,
+ maxBatchSize: maxBatchSize,
parentMemTracker: e.memTracker,
lookup: &e.IndexLookUpJoin,
},
@@ -563,6 +565,7 @@ func (iw *indexHashJoinInnerWorker) getNewJoinResult(ctx context.Context) (*inde
select {
case joinResult.chk, ok = <-iw.joinChkResourceCh:
case <-ctx.Done():
+ joinResult.err = ctx.Err()
return joinResult, false
}
return joinResult, ok
@@ -783,7 +786,10 @@ func (iw *indexHashJoinInnerWorker) joinMatchedInnerRow2Chunk(ctx context.Contex
select {
case iw.resultCh <- joinResult:
case <-ctx.Done():
+ joinResult.err = ctx.Err()
+ return false, joinResult
}
+ failpoint.InjectCall("joinMatchedInnerRow2Chunk")
joinResult, ok = iw.getNewJoinResult(ctx)
if !ok {
return false, joinResult
diff --git a/pkg/executor/join/index_lookup_join.go b/pkg/executor/join/index_lookup_join.go
index e289ae7ab44b3..e35aed79908a8 100644
--- a/pkg/executor/join/index_lookup_join.go
+++ b/pkg/executor/join/index_lookup_join.go
@@ -186,7 +186,7 @@ func (e *IndexLookUpJoin) Open(ctx context.Context) error {
return nil
}
-func (e *IndexLookUpJoin) startWorkers(ctx context.Context) {
+func (e *IndexLookUpJoin) startWorkers(ctx context.Context, initBatchSize int) {
concurrency := e.Ctx().GetSessionVars().IndexLookupJoinConcurrency()
if e.stats != nil {
e.stats.concurrency = concurrency
@@ -197,7 +197,7 @@ func (e *IndexLookUpJoin) startWorkers(ctx context.Context) {
e.cancelFunc = cancelFunc
innerCh := make(chan *lookUpJoinTask, concurrency)
e.WorkerWg.Add(1)
- go e.newOuterWorker(resultCh, innerCh).run(workerCtx, e.WorkerWg)
+ go e.newOuterWorker(resultCh, innerCh, initBatchSize).run(workerCtx, e.WorkerWg)
for i := 0; i < concurrency; i++ {
innerWorker := e.newInnerWorker(innerCh)
e.WorkerWg.Add(1)
@@ -205,15 +205,17 @@ func (e *IndexLookUpJoin) startWorkers(ctx context.Context) {
}
}
-func (e *IndexLookUpJoin) newOuterWorker(resultCh, innerCh chan *lookUpJoinTask) *outerWorker {
+func (e *IndexLookUpJoin) newOuterWorker(resultCh, innerCh chan *lookUpJoinTask, initBatchSize int) *outerWorker {
+ maxBatchSize := e.Ctx().GetSessionVars().IndexJoinBatchSize
+ batchSize := min(initBatchSize, maxBatchSize)
ow := &outerWorker{
OuterCtx: e.OuterCtx,
ctx: e.Ctx(),
executor: e.Children(0),
resultCh: resultCh,
innerCh: innerCh,
- batchSize: 32,
- maxBatchSize: e.Ctx().GetSessionVars().IndexJoinBatchSize,
+ batchSize: batchSize,
+ maxBatchSize: maxBatchSize,
parentMemTracker: e.memTracker,
lookup: e,
}
@@ -273,7 +275,7 @@ func (e *IndexLookUpJoin) newInnerWorker(taskCh chan *lookUpJoinTask) *innerWork
// Next implements the Executor interface.
func (e *IndexLookUpJoin) Next(ctx context.Context, req *chunk.Chunk) error {
if !e.prepared {
- e.startWorkers(ctx)
+ e.startWorkers(ctx, req.RequiredRows())
e.prepared = true
}
if e.IsOuterJoin {
diff --git a/pkg/executor/join/index_lookup_join_test.go b/pkg/executor/join/index_lookup_join_test.go
index d9c6025bf9ecd..571d0967388c6 100644
--- a/pkg/executor/join/index_lookup_join_test.go
+++ b/pkg/executor/join/index_lookup_join_test.go
@@ -18,10 +18,12 @@ import (
"context"
"fmt"
"math/rand"
+ "runtime"
"strings"
"testing"
"github.com/pingcap/failpoint"
+ "github.com/pingcap/tidb/pkg/session"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/stretchr/testify/require"
)
@@ -134,3 +136,41 @@ func TestIssue45716(t *testing.T) {
require.Error(t, err)
tk.MustContainErrMsg(err.Error(), "test inlNewInnerPanic")
}
+
+func TestIssue54688(t *testing.T) {
+ val := runtime.GOMAXPROCS(1)
+ defer func() {
+ runtime.GOMAXPROCS(val)
+ }()
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test;")
+ tk.MustExec("drop table if exists t, s;")
+ tk.MustExec("create table t(a int, index(a));")
+ tk.MustExec("create table s(a int, index(a));")
+ tk.MustExec("insert into t values(1), (2), (3), (4), (5), (6), (7), (8), (9), (10), (11), (12), (13), (14), (15), (16);")
+ tk.MustExec("insert into s values(1), (2), (3), (4), (5), (6), (7), (8), (9), (10), (11), (12), (13), (14), (15), (16);")
+ tk.MustExec("insert into s select * from s")
+ tk.MustExec("insert into s select * from s")
+ tk.MustExec("insert into s select * from s")
+ tk.MustExec("insert into s select * from s")
+ tk.MustExec("insert into s select * from s")
+ tk.MustExec("insert into s select * from s")
+ tk.MustExec("insert into s select * from s")
+ tk.MustExec("insert into s select * from s")
+ tk.MustExec("set @@tidb_index_lookup_join_concurrency=1;")
+ tk.MustExec("set @@tidb_index_join_batch_size=1000000;")
+
+ for i := 0; i <= 100; i++ {
+ rs, err := tk.Exec("select /*+ INL_HASH_JOIN(s) */ * from t join s on t.a=s.a")
+ require.NoError(t, err)
+ context, cancel := context.WithCancel(context.Background())
+ require.NoError(t, failpoint.EnableCall("github.com/pingcap/tidb/pkg/executor/join/joinMatchedInnerRow2Chunk",
+ func() {
+ cancel()
+ },
+ ))
+ _, _ = session.GetRows4Test(context, nil, rs)
+ rs.Close()
+ }
+}
diff --git a/pkg/executor/join/index_lookup_merge_join.go b/pkg/executor/join/index_lookup_merge_join.go
index a1b19c8c41283..c181eb04548d7 100644
--- a/pkg/executor/join/index_lookup_merge_join.go
+++ b/pkg/executor/join/index_lookup_merge_join.go
@@ -443,11 +443,6 @@ func (imw *innerMergeWorker) handleTask(ctx context.Context, task *lookUpMergeJo
}
}
task.memTracker.Consume(int64(cap(task.outerOrderIdx)))
- failpoint.Inject("IndexMergeJoinMockOOM", func(val failpoint.Value) {
- if val.(bool) {
- panic("OOM test index merge join doesn't hang here.")
- }
- })
// NeedOuterSort means the outer side property items can't guarantee the order of join keys.
// Because the necessary condition of merge join is both outer and inner keep order of join keys.
// In this case, we need sort the outer side.
diff --git a/pkg/executor/join/index_lookup_merge_join_test.go b/pkg/executor/join/index_lookup_merge_join_test.go
index 2557a6032e08f..d025a0b76dbcc 100644
--- a/pkg/executor/join/index_lookup_merge_join_test.go
+++ b/pkg/executor/join/index_lookup_merge_join_test.go
@@ -22,25 +22,6 @@ import (
"github.com/stretchr/testify/require"
)
-func TestIndexLookupMergeJoinHang(t *testing.T) {
- require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/executor/join/IndexMergeJoinMockOOM", `return(true)`))
- defer func() {
- require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/executor/join/IndexMergeJoinMockOOM"))
- }()
- store := testkit.CreateMockStore(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("use test")
- tk.MustExec("drop table if exists t1, t2")
- tk.MustExec("create table t1 (a int,b int,index idx(a))")
- tk.MustExec("create table t2 (a int,b int,index idx(a))")
- tk.MustExec("insert into t1 values (1,1),(2,2),(3,3),(2000,2000)")
- tk.MustExec("insert into t2 values (1,1),(2,2),(3,3),(2000,2000)")
- // Do not hang in index merge join when OOM occurs.
- err := tk.QueryToErr("select /*+ INL_MERGE_JOIN(t1, t2) */ * from t1, t2 where t1.a = t2.a")
- require.Error(t, err)
- require.Equal(t, "OOM test index merge join doesn't hang here.", err.Error())
-}
-
func TestIssue18068(t *testing.T) {
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/executor/join/testIssue18068", `return(true)`))
defer func() {
@@ -64,3 +45,18 @@ func TestIssue18068(t *testing.T) {
tk.MustExec("select /*+ inl_merge_join(s)*/ 1 from t join s on t.a = s.a limit 1")
tk.MustExec("select /*+ inl_merge_join(s)*/ 1 from t join s on t.a = s.a limit 1")
}
+
+func TestIssue54064(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists A, B")
+ tk.MustExec("create table A\n(id int primary key nonclustered auto_increment,\nx varchar(32) not null,\ny char(5) not null,\nz varchar(25) not null,\nkey idx_sub_tsk(z,x,y)\n)")
+ tk.MustExec("create table B\n( y char(5) not null,\nz varchar(25) not null,\nx varchar(32) not null,\nprimary key(z, x, y) nonclustered\n)\n")
+ tk.MustExec("insert into A (y, z, x) values\n('CN000', '123', 'RW '),\n('CN000', '456', '123');")
+ tk.MustExec("insert into B values\n('CN000', '123', 'RW '),\n('CN000', '456', '123');")
+ tk.MustQuery("select /*+ inl_merge_join(a, b) */\na.*\nfrom a join b on a.y=b.y and a.z=b.z and a.x = b.x\nwhere a.y='CN000' order by 1,2;").Check(
+ testkit.Rows("1 RW CN000 123", "2 123 CN000 456"))
+ res := tk.MustQuery("explain format='brief' select /*+ inl_merge_join(a, b) */\na.*\nfrom a join b on a.y=b.y and a.z=b.z and a.x = b.x\nwhere a.y='CN000' order by 1,2;")
+ require.NotRegexp(t, "IndexMergeJoin_.*", res.Rows()[0][0])
+}
diff --git a/pkg/executor/join/inner_join_probe.go b/pkg/executor/join/inner_join_probe.go
index f46d9510e2d65..a8b3e95abddee 100644
--- a/pkg/executor/join/inner_join_probe.go
+++ b/pkg/executor/join/inner_join_probe.go
@@ -15,7 +15,6 @@
package join
import (
- "sync/atomic"
"unsafe"
"github.com/pingcap/tidb/pkg/expression"
@@ -54,9 +53,7 @@ func (j *innerJoinProbe) Probe(joinResult *hashjoinWorkerResult, sqlKiller *sqlk
j.matchedRowsForCurrentProbeRow++
remainCap--
} else {
- if j.ctx.stats != nil {
- atomic.AddInt64(&j.ctx.stats.hashStat.probeCollision, 1)
- }
+ j.probeCollision++
}
j.matchedRowsHeaders[j.currentProbeRow] = getNextRowAddress(candidateRow)
} else {
diff --git a/pkg/executor/join/inner_join_probe_test.go b/pkg/executor/join/inner_join_probe_test.go
index ddb43a1662453..dea29dce5c248 100644
--- a/pkg/executor/join/inner_join_probe_test.go
+++ b/pkg/executor/join/inner_join_probe_test.go
@@ -24,7 +24,7 @@ import (
"github.com/pingcap/tidb/pkg/expression"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/mysql"
- plannercore "github.com/pingcap/tidb/pkg/planner/core"
+ "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/chunk"
@@ -199,7 +199,7 @@ func checkChunksEqual(t *testing.T, expectedChunks []*chunk.Chunk, resultChunks
func testJoinProbe(t *testing.T, withSel bool, leftKeyIndex []int, rightKeyIndex []int, leftKeyTypes []*types.FieldType, rightKeyTypes []*types.FieldType,
leftTypes []*types.FieldType, rightTypes []*types.FieldType, rightAsBuildSide bool, leftUsed []int, rightUsed []int,
leftUsedByOtherCondition []int, rightUsedByOtherCondition []int, leftFilter expression.CNFExprs, rightFilter expression.CNFExprs,
- otherCondition expression.CNFExprs, partitionNumber int, joinType plannercore.JoinType, inputRowNumber int) {
+ otherCondition expression.CNFExprs, partitionNumber int, joinType logicalop.JoinType, inputRowNumber int) {
buildKeyIndex, probeKeyIndex := leftKeyIndex, rightKeyIndex
buildKeyTypes, probeKeyTypes := leftKeyTypes, rightKeyTypes
buildTypes, probeTypes := leftTypes, rightTypes
@@ -214,29 +214,29 @@ func testJoinProbe(t *testing.T, withSel bool, leftKeyIndex []int, rightKeyIndex
buildUsed = rightUsed
buildUsedByOtherCondition = rightUsedByOtherCondition
buildFilter, probeFilter = rightFilter, leftFilter
- if joinType == plannercore.RightOuterJoin {
+ if joinType == logicalop.RightOuterJoin {
needUsedFlag = true
}
} else {
switch joinType {
- case plannercore.LeftOuterJoin, plannercore.SemiJoin, plannercore.AntiSemiJoin:
+ case logicalop.LeftOuterJoin, logicalop.SemiJoin, logicalop.AntiSemiJoin:
needUsedFlag = true
- case plannercore.LeftOuterSemiJoin, plannercore.AntiLeftOuterSemiJoin:
+ case logicalop.LeftOuterSemiJoin, logicalop.AntiLeftOuterSemiJoin:
require.NoError(t, errors.New("left semi/anti join does not support use left as build side"))
}
}
switch joinType {
- case plannercore.InnerJoin:
+ case logicalop.InnerJoin:
require.Equal(t, 0, len(leftFilter), "inner join does not support left filter")
require.Equal(t, 0, len(rightFilter), "inner join does not support right filter")
- case plannercore.LeftOuterJoin:
+ case logicalop.LeftOuterJoin:
require.Equal(t, 0, len(rightFilter), "left outer join does not support right filter")
- case plannercore.RightOuterJoin:
+ case logicalop.RightOuterJoin:
require.Equal(t, 0, len(leftFilter), "right outer join does not support left filter")
- case plannercore.SemiJoin, plannercore.AntiSemiJoin:
+ case logicalop.SemiJoin, logicalop.AntiSemiJoin:
require.Equal(t, 0, len(leftFilter), "semi/anti join does not support left filter")
require.Equal(t, 0, len(rightFilter), "semi/anti join does not support right filter")
- case plannercore.LeftOuterSemiJoin, plannercore.AntiLeftOuterSemiJoin:
+ case logicalop.LeftOuterSemiJoin, logicalop.AntiLeftOuterSemiJoin:
require.Equal(t, 0, len(rightFilter), "left outer semi/anti join does not support right filter")
}
joinedTypes := make([]*types.FieldType, 0, len(leftTypes)+len(rightTypes))
@@ -245,13 +245,13 @@ func testJoinProbe(t *testing.T, withSel bool, leftKeyIndex []int, rightKeyIndex
resultTypes := make([]*types.FieldType, 0, len(leftUsed)+len(rightUsed))
for _, colIndex := range leftUsed {
resultTypes = append(resultTypes, leftTypes[colIndex].Clone())
- if joinType == plannercore.RightOuterJoin {
+ if joinType == logicalop.RightOuterJoin {
resultTypes[len(resultTypes)-1].DelFlag(mysql.NotNullFlag)
}
}
for _, colIndex := range rightUsed {
resultTypes = append(resultTypes, rightTypes[colIndex].Clone())
- if joinType == plannercore.LeftOuterJoin {
+ if joinType == logicalop.LeftOuterJoin {
resultTypes[len(resultTypes)-1].DelFlag(mysql.NotNullFlag)
}
}
@@ -390,15 +390,15 @@ func testJoinProbe(t *testing.T, withSel bool, leftKeyIndex []int, rightKeyIndex
checkVirtualRows(t, resultChunks)
switch joinType {
- case plannercore.InnerJoin:
+ case logicalop.InnerJoin:
expectedChunks := genInnerJoinResult(t, hashJoinCtx.SessCtx, leftChunks, rightChunks, leftKeyIndex, rightKeyIndex, leftTypes, rightTypes,
leftKeyTypes, rightKeyTypes, leftUsed, rightUsed, otherCondition, resultTypes)
checkChunksEqual(t, expectedChunks, resultChunks, resultTypes)
- case plannercore.LeftOuterJoin:
+ case logicalop.LeftOuterJoin:
expectedChunks := genLeftOuterJoinResult(t, hashJoinCtx.SessCtx, leftFilter, leftChunks, rightChunks, leftKeyIndex, rightKeyIndex, leftTypes,
rightTypes, leftKeyTypes, rightKeyTypes, leftUsed, rightUsed, otherCondition, resultTypes)
checkChunksEqual(t, expectedChunks, resultChunks, resultTypes)
- case plannercore.RightOuterJoin:
+ case logicalop.RightOuterJoin:
expectedChunks := genRightOuterJoinResult(t, hashJoinCtx.SessCtx, rightFilter, leftChunks, rightChunks, leftKeyIndex, rightKeyIndex, leftTypes,
rightTypes, leftKeyTypes, rightKeyTypes, leftUsed, rightUsed, otherCondition, resultTypes)
checkChunksEqual(t, expectedChunks, resultChunks, resultTypes)
@@ -461,9 +461,9 @@ func TestInnerJoinProbeBasic(t *testing.T) {
// inner join does not have left/right Filter
for _, rightAsBuild := range rightAsBuildSide {
testJoinProbe(t, false, tc.leftKeyIndex, tc.rightKeyIndex, tc.leftKeyTypes, tc.rightKeyTypes, tc.leftTypes, tc.rightTypes, rightAsBuild, tc.leftUsed,
- tc.rightUsed, tc.leftUsedByOtherCondition, tc.rightUsedByOtherCondition, nil, nil, tc.otherCondition, partitionNumber, plannercore.InnerJoin, 200)
+ tc.rightUsed, tc.leftUsedByOtherCondition, tc.rightUsedByOtherCondition, nil, nil, tc.otherCondition, partitionNumber, logicalop.InnerJoin, 200)
testJoinProbe(t, false, tc.leftKeyIndex, tc.rightKeyIndex, toNullableTypes(tc.leftKeyTypes), toNullableTypes(tc.rightKeyTypes),
- toNullableTypes(tc.leftTypes), toNullableTypes(tc.rightTypes), rightAsBuild, tc.leftUsed, tc.rightUsed, tc.leftUsedByOtherCondition, tc.rightUsedByOtherCondition, nil, nil, tc.otherCondition, partitionNumber, plannercore.InnerJoin, 200)
+ toNullableTypes(tc.leftTypes), toNullableTypes(tc.rightTypes), rightAsBuild, tc.leftUsed, tc.rightUsed, tc.leftUsedByOtherCondition, tc.rightUsedByOtherCondition, nil, nil, tc.otherCondition, partitionNumber, logicalop.InnerJoin, 200)
}
}
}
@@ -522,8 +522,8 @@ func TestInnerJoinProbeAllJoinKeys(t *testing.T) {
for _, rightAsBuild := range rightAsBuildSide {
lKeyTypes := []*types.FieldType{lTypes[i]}
rKeyTypes := []*types.FieldType{rTypes[i]}
- testJoinProbe(t, false, []int{i}, []int{i}, lKeyTypes, rKeyTypes, lTypes, rTypes, rightAsBuild, lUsed, rUsed, nil, nil, nil, nil, nil, partitionNumber, plannercore.InnerJoin, 100)
- testJoinProbe(t, false, []int{i}, []int{i}, toNullableTypes(lKeyTypes), toNullableTypes(rKeyTypes), nullableLTypes, nullableRTypes, rightAsBuild, lUsed, rUsed, nil, nil, nil, nil, nil, partitionNumber, plannercore.InnerJoin, 100)
+ testJoinProbe(t, false, []int{i}, []int{i}, lKeyTypes, rKeyTypes, lTypes, rTypes, rightAsBuild, lUsed, rUsed, nil, nil, nil, nil, nil, partitionNumber, logicalop.InnerJoin, 100)
+ testJoinProbe(t, false, []int{i}, []int{i}, toNullableTypes(lKeyTypes), toNullableTypes(rKeyTypes), nullableLTypes, nullableRTypes, rightAsBuild, lUsed, rUsed, nil, nil, nil, nil, nil, partitionNumber, logicalop.InnerJoin, 100)
}
}
// composed key
@@ -531,29 +531,29 @@ func TestInnerJoinProbeAllJoinKeys(t *testing.T) {
for _, rightAsBuild := range rightAsBuildSide {
lKeyTypes := []*types.FieldType{intTp, uintTp}
rKeyTypes := []*types.FieldType{intTp, uintTp}
- testJoinProbe(t, false, []int{1, 2}, []int{1, 2}, lKeyTypes, rKeyTypes, lTypes, rTypes, rightAsBuild, lUsed, rUsed, nil, nil, nil, nil, nil, partitionNumber, plannercore.InnerJoin, 100)
- testJoinProbe(t, false, []int{1, 2}, []int{1, 2}, toNullableTypes(lKeyTypes), toNullableTypes(rKeyTypes), nullableLTypes, nullableRTypes, rightAsBuild, lUsed, rUsed, nil, nil, nil, nil, nil, partitionNumber, plannercore.InnerJoin, 100)
+ testJoinProbe(t, false, []int{1, 2}, []int{1, 2}, lKeyTypes, rKeyTypes, lTypes, rTypes, rightAsBuild, lUsed, rUsed, nil, nil, nil, nil, nil, partitionNumber, logicalop.InnerJoin, 100)
+ testJoinProbe(t, false, []int{1, 2}, []int{1, 2}, toNullableTypes(lKeyTypes), toNullableTypes(rKeyTypes), nullableLTypes, nullableRTypes, rightAsBuild, lUsed, rUsed, nil, nil, nil, nil, nil, partitionNumber, logicalop.InnerJoin, 100)
}
// variable size, inlined
for _, rightAsBuild := range rightAsBuildSide {
lKeyTypes := []*types.FieldType{intTp, binaryStringTp}
rKeyTypes := []*types.FieldType{intTp, binaryStringTp}
- testJoinProbe(t, false, []int{1, 17}, []int{1, 17}, lKeyTypes, rKeyTypes, lTypes, rTypes, rightAsBuild, lUsed, rUsed, nil, nil, nil, nil, nil, partitionNumber, plannercore.InnerJoin, 100)
- testJoinProbe(t, false, []int{1, 17}, []int{1, 17}, toNullableTypes(lKeyTypes), toNullableTypes(rKeyTypes), nullableLTypes, nullableRTypes, rightAsBuild, lUsed, rUsed, nil, nil, nil, nil, nil, partitionNumber, plannercore.InnerJoin, 100)
+ testJoinProbe(t, false, []int{1, 17}, []int{1, 17}, lKeyTypes, rKeyTypes, lTypes, rTypes, rightAsBuild, lUsed, rUsed, nil, nil, nil, nil, nil, partitionNumber, logicalop.InnerJoin, 100)
+ testJoinProbe(t, false, []int{1, 17}, []int{1, 17}, toNullableTypes(lKeyTypes), toNullableTypes(rKeyTypes), nullableLTypes, nullableRTypes, rightAsBuild, lUsed, rUsed, nil, nil, nil, nil, nil, partitionNumber, logicalop.InnerJoin, 100)
}
// fixed size, not inlined
for _, rightAsBuild := range rightAsBuildSide {
lKeyTypes := []*types.FieldType{intTp, datetimeTp}
rKeyTypes := []*types.FieldType{intTp, datetimeTp}
- testJoinProbe(t, false, []int{1, 13}, []int{1, 13}, lKeyTypes, rKeyTypes, lTypes, rTypes, rightAsBuild, lUsed, rUsed, nil, nil, nil, nil, nil, partitionNumber, plannercore.InnerJoin, 100)
- testJoinProbe(t, false, []int{1, 13}, []int{1, 13}, toNullableTypes(lKeyTypes), toNullableTypes(rKeyTypes), nullableLTypes, nullableRTypes, rightAsBuild, lUsed, rUsed, nil, nil, nil, nil, nil, partitionNumber, plannercore.InnerJoin, 100)
+ testJoinProbe(t, false, []int{1, 13}, []int{1, 13}, lKeyTypes, rKeyTypes, lTypes, rTypes, rightAsBuild, lUsed, rUsed, nil, nil, nil, nil, nil, partitionNumber, logicalop.InnerJoin, 100)
+ testJoinProbe(t, false, []int{1, 13}, []int{1, 13}, toNullableTypes(lKeyTypes), toNullableTypes(rKeyTypes), nullableLTypes, nullableRTypes, rightAsBuild, lUsed, rUsed, nil, nil, nil, nil, nil, partitionNumber, logicalop.InnerJoin, 100)
}
// variable size, not inlined
for _, rightAsBuild := range rightAsBuildSide {
lKeyTypes := []*types.FieldType{intTp, decimalTp}
rKeyTypes := []*types.FieldType{intTp, decimalTp}
- testJoinProbe(t, false, []int{1, 14}, []int{1, 14}, lKeyTypes, rKeyTypes, lTypes, rTypes, rightAsBuild, lUsed, rUsed, nil, nil, nil, nil, nil, partitionNumber, plannercore.InnerJoin, 100)
- testJoinProbe(t, false, []int{1, 14}, []int{1, 14}, toNullableTypes(lKeyTypes), toNullableTypes(rKeyTypes), nullableLTypes, nullableRTypes, rightAsBuild, lUsed, rUsed, nil, nil, nil, nil, nil, partitionNumber, plannercore.InnerJoin, 100)
+ testJoinProbe(t, false, []int{1, 14}, []int{1, 14}, lKeyTypes, rKeyTypes, lTypes, rTypes, rightAsBuild, lUsed, rUsed, nil, nil, nil, nil, nil, partitionNumber, logicalop.InnerJoin, 100)
+ testJoinProbe(t, false, []int{1, 14}, []int{1, 14}, toNullableTypes(lKeyTypes), toNullableTypes(rKeyTypes), nullableLTypes, nullableRTypes, rightAsBuild, lUsed, rUsed, nil, nil, nil, nil, nil, partitionNumber, logicalop.InnerJoin, 100)
}
}
@@ -581,9 +581,9 @@ func TestInnerJoinProbeOtherCondition(t *testing.T) {
partitionNumber := 4
for _, rightAsBuild := range rightAsBuildSide {
- testJoinProbe(t, false, []int{0}, []int{0}, []*types.FieldType{intTp}, []*types.FieldType{intTp}, lTypes, rTypes, rightAsBuild, []int{1, 2, 4}, []int{0}, []int{1}, []int{3}, nil, nil, otherCondition, partitionNumber, plannercore.InnerJoin, 200)
- testJoinProbe(t, false, []int{0}, []int{0}, []*types.FieldType{intTp}, []*types.FieldType{intTp}, lTypes, rTypes, rightAsBuild, []int{}, []int{}, []int{1}, []int{3}, nil, nil, otherCondition, partitionNumber, plannercore.InnerJoin, 200)
- testJoinProbe(t, false, []int{0}, []int{0}, []*types.FieldType{nullableIntTp}, []*types.FieldType{nullableIntTp}, toNullableTypes(lTypes), toNullableTypes(rTypes), rightAsBuild, []int{1, 2, 4}, []int{0}, []int{1}, []int{3}, nil, nil, otherCondition, partitionNumber, plannercore.InnerJoin, 200)
+ testJoinProbe(t, false, []int{0}, []int{0}, []*types.FieldType{intTp}, []*types.FieldType{intTp}, lTypes, rTypes, rightAsBuild, []int{1, 2, 4}, []int{0}, []int{1}, []int{3}, nil, nil, otherCondition, partitionNumber, logicalop.InnerJoin, 200)
+ testJoinProbe(t, false, []int{0}, []int{0}, []*types.FieldType{intTp}, []*types.FieldType{intTp}, lTypes, rTypes, rightAsBuild, []int{}, []int{}, []int{1}, []int{3}, nil, nil, otherCondition, partitionNumber, logicalop.InnerJoin, 200)
+ testJoinProbe(t, false, []int{0}, []int{0}, []*types.FieldType{nullableIntTp}, []*types.FieldType{nullableIntTp}, toNullableTypes(lTypes), toNullableTypes(rTypes), rightAsBuild, []int{1, 2, 4}, []int{0}, []int{1}, []int{3}, nil, nil, otherCondition, partitionNumber, logicalop.InnerJoin, 200)
}
}
@@ -616,9 +616,9 @@ func TestInnerJoinProbeWithSel(t *testing.T) {
for _, rightAsBuild := range rightAsBuildSide {
for _, oc := range otherConditions {
- testJoinProbe(t, true, []int{0}, []int{0}, []*types.FieldType{intTp}, []*types.FieldType{intTp}, lTypes, rTypes, rightAsBuild, []int{1, 2, 4}, []int{0}, []int{1}, []int{3}, nil, nil, oc, partitionNumber, plannercore.InnerJoin, 500)
- testJoinProbe(t, true, []int{0}, []int{0}, []*types.FieldType{intTp}, []*types.FieldType{intTp}, lTypes, rTypes, rightAsBuild, []int{}, []int{}, []int{1}, []int{3}, nil, nil, oc, partitionNumber, plannercore.InnerJoin, 500)
- testJoinProbe(t, true, []int{0}, []int{0}, []*types.FieldType{nullableIntTp}, []*types.FieldType{nullableIntTp}, toNullableTypes(lTypes), toNullableTypes(rTypes), rightAsBuild, []int{1, 2, 4}, []int{0}, []int{1}, []int{3}, nil, nil, oc, partitionNumber, plannercore.InnerJoin, 500)
+ testJoinProbe(t, true, []int{0}, []int{0}, []*types.FieldType{intTp}, []*types.FieldType{intTp}, lTypes, rTypes, rightAsBuild, []int{1, 2, 4}, []int{0}, []int{1}, []int{3}, nil, nil, oc, partitionNumber, logicalop.InnerJoin, 500)
+ testJoinProbe(t, true, []int{0}, []int{0}, []*types.FieldType{intTp}, []*types.FieldType{intTp}, lTypes, rTypes, rightAsBuild, []int{}, []int{}, []int{1}, []int{3}, nil, nil, oc, partitionNumber, logicalop.InnerJoin, 500)
+ testJoinProbe(t, true, []int{0}, []int{0}, []*types.FieldType{nullableIntTp}, []*types.FieldType{nullableIntTp}, toNullableTypes(lTypes), toNullableTypes(rTypes), rightAsBuild, []int{1, 2, 4}, []int{0}, []int{1}, []int{3}, nil, nil, oc, partitionNumber, logicalop.InnerJoin, 500)
}
}
}
diff --git a/pkg/executor/join/join_row_table.go b/pkg/executor/join/join_row_table.go
index bfa9b015eefa1..5c1fc669b695f 100644
--- a/pkg/executor/join/join_row_table.go
+++ b/pkg/executor/join/join_row_table.go
@@ -38,6 +38,9 @@ const sizeOfUintptr = int(unsafe.Sizeof(uintptr(0)))
var fakeAddrPlaceHolder = []byte{0, 0, 0, 0, 0, 0, 0, 0}
+//go:linkname heapObjectsCanMove runtime.heapObjectsCanMove
+func heapObjectsCanMove() bool
+
type rowTableSegment struct {
/*
The row storage used in hash join, the layout is
@@ -108,7 +111,7 @@ func setNextRowAddress(rowStart unsafe.Pointer, nextRowAddress unsafe.Pointer) {
// Save unsafe.Pointer into current Row header. Generally speaking it is unsafe or even illegal in go
// since after save the value of unsafe.Pointer into row header, it has no pointer semantics the value
// in the row header may become invalid after GC. It is ok to save unsafe.Pointer so far because
- // 1. Go has a non-moving GC(https://tip.golang.org/doc/gc-guide), which means if the object is in heap, the address will not be changed after GC
+ // 1. the check of heapObjectsCanMove makes sure that if the object is in heap, the address will not be changed after GC
// 2. `rowStart` only points to a valid address in `rawData`. `rawData` is a slice in `rowTableSegment`, and it will be used by multiple goroutines,
// and its size will be runtime expanded, this kind of slice will always be allocated in heap
*(*unsafe.Pointer)(rowStart) = nextRowAddress
@@ -232,11 +235,10 @@ func (rt *rowTable) getValidJoinKeyPos(rowIndex int) int {
}
type keyProp struct {
- canBeInlined bool
- keyLength int
- isStringRelatedType bool
- isKeyInteger bool
- isKeyUnsigned bool
+ canBeInlined bool
+ keyLength int
+ isKeyInteger bool
+ isKeyUnsigned bool
}
func getKeyProp(tp *types.FieldType) *keyProp {
@@ -251,31 +253,31 @@ func getKeyProp(tp *types.FieldType) *keyProp {
// duration type is always signed
isKeyUnsigned = false
}
- return &keyProp{canBeInlined: true, keyLength: chunk.GetFixedLen(tp), isStringRelatedType: false, isKeyInteger: true, isKeyUnsigned: isKeyUnsigned}
+ return &keyProp{canBeInlined: true, keyLength: chunk.GetFixedLen(tp), isKeyInteger: true, isKeyUnsigned: isKeyUnsigned}
case mysql.TypeVarchar, mysql.TypeVarString, mysql.TypeString, mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob:
collator := collate.GetCollator(tp.GetCollate())
- return &keyProp{canBeInlined: collate.CanUseRawMemAsKey(collator), keyLength: chunk.VarElemLen, isStringRelatedType: true, isKeyInteger: false, isKeyUnsigned: false}
+ return &keyProp{canBeInlined: collate.CanUseRawMemAsKey(collator), keyLength: chunk.VarElemLen, isKeyInteger: false, isKeyUnsigned: false}
case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeTimestamp:
// date related type will use uint64 as serialized key
- return &keyProp{canBeInlined: false, keyLength: int(serialization.Uint64Len), isStringRelatedType: false, isKeyInteger: true, isKeyUnsigned: true}
+ return &keyProp{canBeInlined: false, keyLength: int(serialization.Uint64Len), isKeyInteger: true, isKeyUnsigned: true}
case mysql.TypeFloat:
// float will use float64 as serialized key
- return &keyProp{canBeInlined: false, keyLength: int(serialization.Float64Len), isStringRelatedType: false, isKeyInteger: false, isKeyUnsigned: false}
+ return &keyProp{canBeInlined: false, keyLength: int(serialization.Float64Len), isKeyInteger: false, isKeyUnsigned: false}
case mysql.TypeNewDecimal:
// Although decimal is fixed length, but its key is not fixed length
- return &keyProp{canBeInlined: false, keyLength: chunk.VarElemLen, isStringRelatedType: false, isKeyInteger: false, isKeyUnsigned: false}
+ return &keyProp{canBeInlined: false, keyLength: chunk.VarElemLen, isKeyInteger: false, isKeyUnsigned: false}
case mysql.TypeEnum:
if mysql.HasEnumSetAsIntFlag(tp.GetFlag()) {
// enum int type is always unsigned
- return &keyProp{canBeInlined: false, keyLength: int(serialization.Uint64Len), isStringRelatedType: false, isKeyInteger: true, isKeyUnsigned: true}
+ return &keyProp{canBeInlined: false, keyLength: int(serialization.Uint64Len), isKeyInteger: true, isKeyUnsigned: true}
}
- return &keyProp{canBeInlined: false, keyLength: chunk.VarElemLen, isStringRelatedType: false, isKeyInteger: false, isKeyUnsigned: false}
+ return &keyProp{canBeInlined: false, keyLength: chunk.VarElemLen, isKeyInteger: false, isKeyUnsigned: false}
case mysql.TypeBit:
// bit type is always unsigned
- return &keyProp{canBeInlined: false, keyLength: int(serialization.Uint64Len), isStringRelatedType: false, isKeyInteger: true, isKeyUnsigned: true}
+ return &keyProp{canBeInlined: false, keyLength: int(serialization.Uint64Len), isKeyInteger: true, isKeyUnsigned: true}
default:
keyLength := chunk.GetFixedLen(tp)
- return &keyProp{canBeInlined: false, keyLength: keyLength, isStringRelatedType: false, isKeyInteger: false, isKeyUnsigned: false}
+ return &keyProp{canBeInlined: false, keyLength: keyLength, isKeyInteger: false, isKeyUnsigned: false}
}
}
@@ -322,6 +324,7 @@ func newTableMeta(buildKeyIndex []int, buildTypes, buildKeyTypes, probeKeyTypes
meta.serializeModes = make([]codec.SerializeMode, 0, len(buildKeyIndex))
isAllKeyInteger := true
hasFixedSizeKeyColumn := false
+ varLengthKeyNumber := 0
for index, keyIndex := range buildKeyIndex {
keyType := buildKeyTypes[index]
prop := getKeyProp(keyType)
@@ -330,6 +333,7 @@ func newTableMeta(buildKeyIndex []int, buildTypes, buildKeyTypes, probeKeyTypes
hasFixedSizeKeyColumn = true
} else {
meta.isJoinKeysFixedLength = false
+ varLengthKeyNumber++
}
if !prop.canBeInlined {
meta.isJoinKeysInlined = false
@@ -355,8 +359,9 @@ func newTableMeta(buildKeyIndex []int, buildTypes, buildKeyTypes, probeKeyTypes
if !prop.isKeyInteger {
isAllKeyInteger = false
}
- if meta.isJoinKeysInlined && prop.isStringRelatedType {
- meta.serializeModes = append(meta.serializeModes, codec.KeepStringLength)
+ if prop.keyLength == chunk.VarElemLen {
+ // keep var column by default for var length column
+ meta.serializeModes = append(meta.serializeModes, codec.KeepVarColumnLength)
} else {
meta.serializeModes = append(meta.serializeModes, codec.Normal)
}
@@ -371,9 +376,12 @@ func newTableMeta(buildKeyIndex []int, buildTypes, buildKeyTypes, probeKeyTypes
meta.isJoinKeysInlined = false
}
if !meta.isJoinKeysInlined {
- for i := 0; i < len(buildKeyIndex); i++ {
- if meta.serializeModes[i] == codec.KeepStringLength {
- meta.serializeModes[i] = codec.Normal
+ if varLengthKeyNumber == 1 {
+ // if key is not inlined and there is only one var-length key, then don't need to record the var length
+ for i := 0; i < len(buildKeyIndex); i++ {
+ if meta.serializeModes[i] == codec.KeepVarColumnLength {
+ meta.serializeModes[i] = codec.Normal
+ }
}
}
} else {
diff --git a/pkg/executor/join/join_row_table_test.go b/pkg/executor/join/join_row_table_test.go
index d5b1cf5eb0e7a..7b9fb361c624c 100644
--- a/pkg/executor/join/join_row_table_test.go
+++ b/pkg/executor/join/join_row_table_test.go
@@ -24,6 +24,10 @@ import (
"github.com/stretchr/testify/require"
)
+func TestHeapObjectCanMove(t *testing.T) {
+ require.Equal(t, false, heapObjectsCanMove())
+}
+
func TestFixedOffsetInRowLayout(t *testing.T) {
require.Equal(t, 8, sizeOfNextPtr)
require.Equal(t, 8, sizeOfLengthField)
@@ -174,6 +178,11 @@ func TestJoinTableMetaSerializedMode(t *testing.T) {
stringTp := types.NewFieldType(mysql.TypeVarString)
binaryStringTp := types.NewFieldType(mysql.TypeBlob)
decimalTp := types.NewFieldType(mysql.TypeNewDecimal)
+ enumTp := types.NewFieldType(mysql.TypeEnum)
+ enumWithIntFlag := types.NewFieldType(mysql.TypeEnum)
+ enumWithIntFlag.AddFlag(mysql.EnumSetAsIntFlag)
+ setTp := types.NewFieldType(mysql.TypeSet)
+ jsonTp := types.NewFieldType(mysql.TypeJSON)
type testCase struct {
buildKeyIndex []int
@@ -188,12 +197,23 @@ func TestJoinTableMetaSerializedMode(t *testing.T) {
// test NeedSignFlag
{[]int{0, 1}, []*types.FieldType{uintTp, intTp}, []*types.FieldType{uintTp, intTp}, []*types.FieldType{intTp, intTp}, []codec.SerializeMode{codec.NeedSignFlag, codec.Normal}},
{[]int{0}, []*types.FieldType{uintTp}, []*types.FieldType{uintTp}, []*types.FieldType{intTp}, []codec.SerializeMode{codec.NeedSignFlag}},
- // test KeepStringLength
- {[]int{0, 1}, []*types.FieldType{intTp, binaryStringTp}, []*types.FieldType{intTp, binaryStringTp}, []*types.FieldType{intTp, binaryStringTp}, []codec.SerializeMode{codec.Normal, codec.KeepStringLength}},
- {[]int{0}, []*types.FieldType{binaryStringTp}, []*types.FieldType{binaryStringTp}, []*types.FieldType{binaryStringTp}, []codec.SerializeMode{codec.KeepStringLength}},
- // binaryString is not inlined, no need to keep string length
+ // test KeepVarColumnLength
+ {[]int{0, 1}, []*types.FieldType{intTp, binaryStringTp}, []*types.FieldType{intTp, binaryStringTp}, []*types.FieldType{intTp, binaryStringTp}, []codec.SerializeMode{codec.Normal, codec.KeepVarColumnLength}},
+ {[]int{0}, []*types.FieldType{binaryStringTp}, []*types.FieldType{binaryStringTp}, []*types.FieldType{binaryStringTp}, []codec.SerializeMode{codec.KeepVarColumnLength}},
+ // binaryString is not inlined, no need to keep var column length
{[]int{0, 1}, []*types.FieldType{intTp, binaryStringTp}, []*types.FieldType{intTp, binaryStringTp}, []*types.FieldType{uintTp, binaryStringTp}, []codec.SerializeMode{codec.NeedSignFlag, codec.Normal}},
- {[]int{0, 1}, []*types.FieldType{stringTp, binaryStringTp}, []*types.FieldType{stringTp, binaryStringTp}, []*types.FieldType{stringTp, binaryStringTp}, []codec.SerializeMode{codec.Normal, codec.Normal}},
+ // multiple var-length column, need keep var column length
+ {[]int{0, 1}, []*types.FieldType{stringTp, binaryStringTp}, []*types.FieldType{stringTp, binaryStringTp}, []*types.FieldType{stringTp, binaryStringTp}, []codec.SerializeMode{codec.KeepVarColumnLength, codec.KeepVarColumnLength}},
+ {[]int{0, 1}, []*types.FieldType{stringTp, decimalTp}, []*types.FieldType{stringTp, decimalTp}, []*types.FieldType{stringTp, decimalTp}, []codec.SerializeMode{codec.KeepVarColumnLength, codec.KeepVarColumnLength}},
+ // set/json/decimal/enum is treated as var-length column
+ {[]int{0, 1}, []*types.FieldType{setTp, jsonTp, decimalTp, enumTp}, []*types.FieldType{setTp, jsonTp, decimalTp, enumTp}, []*types.FieldType{setTp, jsonTp, decimalTp, enumTp}, []codec.SerializeMode{codec.KeepVarColumnLength, codec.KeepVarColumnLength, codec.KeepVarColumnLength, codec.KeepVarColumnLength}},
+ {[]int{0, 1}, []*types.FieldType{setTp, jsonTp, decimalTp}, []*types.FieldType{setTp, jsonTp, decimalTp}, []*types.FieldType{setTp, jsonTp, decimalTp}, []codec.SerializeMode{codec.KeepVarColumnLength, codec.KeepVarColumnLength, codec.KeepVarColumnLength}},
+ {[]int{0, 1}, []*types.FieldType{jsonTp, decimalTp}, []*types.FieldType{jsonTp, decimalTp}, []*types.FieldType{jsonTp, decimalTp}, []codec.SerializeMode{codec.KeepVarColumnLength, codec.KeepVarColumnLength}},
+ {[]int{0, 1}, []*types.FieldType{setTp, enumTp}, []*types.FieldType{setTp, enumTp}, []*types.FieldType{setTp, enumTp}, []codec.SerializeMode{codec.KeepVarColumnLength, codec.KeepVarColumnLength}},
+ // enumWithIntFlag is fix length column
+ {[]int{0, 1}, []*types.FieldType{enumWithIntFlag, enumTp}, []*types.FieldType{enumWithIntFlag, enumTp}, []*types.FieldType{enumWithIntFlag, enumTp}, []codec.SerializeMode{codec.Normal, codec.Normal}},
+ // single non-inlined var length column don't need keep var column length
+ {[]int{0, 1}, []*types.FieldType{setTp, enumWithIntFlag}, []*types.FieldType{setTp, enumWithIntFlag}, []*types.FieldType{setTp, enumWithIntFlag}, []codec.SerializeMode{codec.Normal, codec.Normal}},
}
for index, test := range testCases {
meta := newTableMeta(test.buildKeyIndex, test.buildTypes, test.buildKeyTypes, test.probeKeyTypes, nil, []int{}, false)
diff --git a/pkg/executor/join/joiner.go b/pkg/executor/join/joiner.go
index 22a76eea1486f..bae727a840d50 100644
--- a/pkg/executor/join/joiner.go
+++ b/pkg/executor/join/joiner.go
@@ -16,7 +16,7 @@ package join
import (
"github.com/pingcap/tidb/pkg/expression"
- plannercore "github.com/pingcap/tidb/pkg/planner/core"
+ "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/chunk"
@@ -116,27 +116,27 @@ type Joiner interface {
}
// JoinerType returns the join type of a Joiner.
-func JoinerType(j Joiner) plannercore.JoinType {
+func JoinerType(j Joiner) logicalop.JoinType {
switch j.(type) {
case *semiJoiner:
- return plannercore.SemiJoin
+ return logicalop.SemiJoin
case *antiSemiJoiner:
- return plannercore.AntiSemiJoin
+ return logicalop.AntiSemiJoin
case *leftOuterSemiJoiner:
- return plannercore.LeftOuterSemiJoin
+ return logicalop.LeftOuterSemiJoin
case *antiLeftOuterSemiJoiner:
- return plannercore.AntiLeftOuterSemiJoin
+ return logicalop.AntiLeftOuterSemiJoin
case *leftOuterJoiner:
- return plannercore.LeftOuterJoin
+ return logicalop.LeftOuterJoin
case *rightOuterJoiner:
- return plannercore.RightOuterJoin
+ return logicalop.RightOuterJoin
default:
- return plannercore.InnerJoin
+ return logicalop.InnerJoin
}
}
// NewJoiner create a joiner
-func NewJoiner(ctx sessionctx.Context, joinType plannercore.JoinType,
+func NewJoiner(ctx sessionctx.Context, joinType logicalop.JoinType,
outerIsRight bool, defaultInner []types.Datum, filter []expression.Expression,
lhsColTypes, rhsColTypes []*types.FieldType, childrenUsed [][]int, isNA bool) Joiner {
base := baseJoiner{
@@ -159,7 +159,7 @@ func NewJoiner(ctx sessionctx.Context, joinType plannercore.JoinType,
zap.Ints("lUsed", base.lUsed), zap.Ints("rUsed", base.rUsed),
zap.Int("lCount", len(lhsColTypes)), zap.Int("rCount", len(rhsColTypes)))
}
- if joinType == plannercore.LeftOuterJoin || joinType == plannercore.RightOuterJoin {
+ if joinType == logicalop.LeftOuterJoin || joinType == logicalop.RightOuterJoin {
innerColTypes := lhsColTypes
if !outerIsRight {
innerColTypes = rhsColTypes
@@ -173,34 +173,34 @@ func NewJoiner(ctx sessionctx.Context, joinType plannercore.JoinType,
shallowRowType = append(shallowRowType, lhsColTypes...)
shallowRowType = append(shallowRowType, rhsColTypes...)
switch joinType {
- case plannercore.SemiJoin:
+ case logicalop.SemiJoin:
base.shallowRow = chunk.MutRowFromTypes(shallowRowType)
return &semiJoiner{base}
- case plannercore.AntiSemiJoin:
+ case logicalop.AntiSemiJoin:
base.shallowRow = chunk.MutRowFromTypes(shallowRowType)
if isNA {
return &nullAwareAntiSemiJoiner{baseJoiner: base}
}
return &antiSemiJoiner{base}
- case plannercore.LeftOuterSemiJoin:
+ case logicalop.LeftOuterSemiJoin:
base.shallowRow = chunk.MutRowFromTypes(shallowRowType)
return &leftOuterSemiJoiner{base}
- case plannercore.AntiLeftOuterSemiJoin:
+ case logicalop.AntiLeftOuterSemiJoin:
base.shallowRow = chunk.MutRowFromTypes(shallowRowType)
if isNA {
return &nullAwareAntiLeftOuterSemiJoiner{baseJoiner: base}
}
return &antiLeftOuterSemiJoiner{base}
- case plannercore.LeftOuterJoin, plannercore.RightOuterJoin, plannercore.InnerJoin:
+ case logicalop.LeftOuterJoin, logicalop.RightOuterJoin, logicalop.InnerJoin:
if len(base.conditions) > 0 {
base.chk = chunk.NewChunkWithCapacity(shallowRowType, ctx.GetSessionVars().MaxChunkSize)
}
switch joinType {
- case plannercore.LeftOuterJoin:
+ case logicalop.LeftOuterJoin:
return &leftOuterJoiner{base}
- case plannercore.RightOuterJoin:
+ case logicalop.RightOuterJoin:
return &rightOuterJoiner{base}
- case plannercore.InnerJoin:
+ case logicalop.InnerJoin:
return &innerJoiner{base}
}
}
diff --git a/pkg/executor/join/joiner_test.go b/pkg/executor/join/joiner_test.go
index 626b129f22d55..796f1b17ec398 100644
--- a/pkg/executor/join/joiner_test.go
+++ b/pkg/executor/join/joiner_test.go
@@ -20,7 +20,7 @@ import (
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/parser/mysql"
- "github.com/pingcap/tidb/pkg/planner/core"
+ "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/types"
@@ -43,7 +43,7 @@ func defaultCtx() sessionctx.Context {
}
func TestRequiredRows(t *testing.T) {
- joinTypes := []core.JoinType{core.InnerJoin, core.LeftOuterJoin, core.RightOuterJoin}
+ joinTypes := []logicalop.JoinType{logicalop.InnerJoin, logicalop.LeftOuterJoin, logicalop.RightOuterJoin}
lTypes := [][]byte{
{mysql.TypeLong},
{mysql.TypeFloat},
diff --git a/pkg/executor/join/left_outer_join_probe_test.go b/pkg/executor/join/left_outer_join_probe_test.go
index ace560d9bcee2..7b3a71393a84b 100644
--- a/pkg/executor/join/left_outer_join_probe_test.go
+++ b/pkg/executor/join/left_outer_join_probe_test.go
@@ -20,7 +20,7 @@ import (
"github.com/pingcap/tidb/pkg/expression"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/mysql"
- plannercore "github.com/pingcap/tidb/pkg/planner/core"
+ "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/chunk"
@@ -152,10 +152,10 @@ func TestLeftOuterJoinProbeBasic(t *testing.T) {
leftFilter = nil
}
testJoinProbe(t, false, tc.leftKeyIndex, tc.rightKeyIndex, tc.leftKeyTypes, tc.rightKeyTypes, tc.leftTypes, tc.rightTypes, value, tc.leftUsed,
- tc.rightUsed, tc.leftUsedByOtherCondition, tc.rightUsedByOtherCondition, leftFilter, nil, tc.otherCondition, partitionNumber, plannercore.LeftOuterJoin, 200)
+ tc.rightUsed, tc.leftUsedByOtherCondition, tc.rightUsedByOtherCondition, leftFilter, nil, tc.otherCondition, partitionNumber, logicalop.LeftOuterJoin, 200)
testJoinProbe(t, false, tc.leftKeyIndex, tc.rightKeyIndex, toNullableTypes(tc.leftKeyTypes), toNullableTypes(tc.rightKeyTypes),
toNullableTypes(tc.leftTypes), toNullableTypes(tc.rightTypes), value, tc.leftUsed, tc.rightUsed, tc.leftUsedByOtherCondition, tc.rightUsedByOtherCondition,
- leftFilter, nil, tc.otherCondition, partitionNumber, plannercore.LeftOuterJoin, 200)
+ leftFilter, nil, tc.otherCondition, partitionNumber, logicalop.LeftOuterJoin, 200)
}
}
}
@@ -205,7 +205,7 @@ func TestLeftOuterJoinProbeAllJoinKeys(t *testing.T) {
rTypes := lTypes
lUsed := []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}
rUsed := lUsed
- joinType := plannercore.LeftOuterJoin
+ joinType := logicalop.LeftOuterJoin
partitionNumber := 4
rightAsBuildSide := []bool{true, false}
@@ -270,7 +270,7 @@ func TestLeftOuterJoinProbeOtherCondition(t *testing.T) {
require.NoError(t, err, "error when create other condition")
otherCondition := make(expression.CNFExprs, 0)
otherCondition = append(otherCondition, sf)
- joinType := plannercore.LeftOuterJoin
+ joinType := logicalop.LeftOuterJoin
simpleFilter := createSimpleFilter(t)
hasFilter := []bool{false, true}
rightAsBuildSide := []bool{false, true}
@@ -311,7 +311,7 @@ func TestLeftOuterJoinProbeWithSel(t *testing.T) {
require.NoError(t, err, "error when create other condition")
otherCondition := make(expression.CNFExprs, 0)
otherCondition = append(otherCondition, sf)
- joinType := plannercore.LeftOuterJoin
+ joinType := logicalop.LeftOuterJoin
rightAsBuildSide := []bool{false, true}
simpleFilter := createSimpleFilter(t)
hasFilter := []bool{false, true}
diff --git a/pkg/executor/join/outer_join_probe.go b/pkg/executor/join/outer_join_probe.go
index bccbfc4bd75ec..444a5faf85d6f 100644
--- a/pkg/executor/join/outer_join_probe.go
+++ b/pkg/executor/join/outer_join_probe.go
@@ -15,7 +15,6 @@
package join
import (
- "sync/atomic"
"unsafe"
"github.com/pingcap/tidb/pkg/expression"
@@ -255,9 +254,7 @@ func (j *outerJoinProbe) probeForInnerSideBuild(chk, joinedChk *chunk.Chunk, rem
}
j.matchedRowsForCurrentProbeRow++
} else {
- if j.ctx.stats != nil {
- atomic.AddInt64(&j.ctx.stats.hashStat.probeCollision, 1)
- }
+ j.probeCollision++
}
j.matchedRowsHeaders[j.currentProbeRow] = getNextRowAddress(candidateRow)
} else {
@@ -313,9 +310,7 @@ func (j *outerJoinProbe) probeForOuterSideBuild(chk, joinedChk *chunk.Chunk, rem
j.matchedRowsForCurrentProbeRow++
remainCap--
} else {
- if j.ctx.stats != nil {
- atomic.AddInt64(&j.ctx.stats.hashStat.probeCollision, 1)
- }
+ j.probeCollision++
}
j.matchedRowsHeaders[j.currentProbeRow] = getNextRowAddress(candidateRow)
} else {
diff --git a/pkg/executor/join/right_outer_join_probe_test.go b/pkg/executor/join/right_outer_join_probe_test.go
index b9ad5d4ada1d4..3961b27d6bd7f 100644
--- a/pkg/executor/join/right_outer_join_probe_test.go
+++ b/pkg/executor/join/right_outer_join_probe_test.go
@@ -20,7 +20,7 @@ import (
"github.com/pingcap/tidb/pkg/expression"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/mysql"
- plannercore "github.com/pingcap/tidb/pkg/planner/core"
+ "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/chunk"
@@ -152,10 +152,10 @@ func TestRightOuterJoinProbeBasic(t *testing.T) {
rightFilter = nil
}
testJoinProbe(t, false, tc.leftKeyIndex, tc.rightKeyIndex, tc.leftKeyTypes, tc.rightKeyTypes, tc.leftTypes, tc.rightTypes, value, tc.leftUsed,
- tc.rightUsed, tc.leftUsedByOtherCondition, tc.rightUsedByOtherCondition, nil, rightFilter, tc.otherCondition, partitionNumber, plannercore.RightOuterJoin, 200)
+ tc.rightUsed, tc.leftUsedByOtherCondition, tc.rightUsedByOtherCondition, nil, rightFilter, tc.otherCondition, partitionNumber, logicalop.RightOuterJoin, 200)
testJoinProbe(t, false, tc.leftKeyIndex, tc.rightKeyIndex, toNullableTypes(tc.leftKeyTypes), toNullableTypes(tc.rightKeyTypes),
toNullableTypes(tc.leftTypes), toNullableTypes(tc.rightTypes), value, tc.leftUsed, tc.rightUsed, tc.leftUsedByOtherCondition, tc.rightUsedByOtherCondition,
- nil, rightFilter, tc.otherCondition, partitionNumber, plannercore.RightOuterJoin, 200)
+ nil, rightFilter, tc.otherCondition, partitionNumber, logicalop.RightOuterJoin, 200)
}
}
}
@@ -205,7 +205,7 @@ func TestRightOuterJoinProbeAllJoinKeys(t *testing.T) {
rTypes := lTypes
lUsed := []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}
rUsed := lUsed
- joinType := plannercore.RightOuterJoin
+ joinType := logicalop.RightOuterJoin
rightAsBuildSide := []bool{true, false}
@@ -269,7 +269,7 @@ func TestRightOuterJoinProbeOtherCondition(t *testing.T) {
require.NoError(t, err, "error when create other condition")
otherCondition := make(expression.CNFExprs, 0)
otherCondition = append(otherCondition, sf)
- joinType := plannercore.RightOuterJoin
+ joinType := logicalop.RightOuterJoin
simpleFilter := createSimpleFilter(t)
hasFilter := []bool{false, true}
rightAsBuildSide := []bool{false, true}
@@ -309,7 +309,7 @@ func TestRightOuterJoinProbeWithSel(t *testing.T) {
require.NoError(t, err, "error when create other condition")
otherCondition := make(expression.CNFExprs, 0)
otherCondition = append(otherCondition, sf)
- joinType := plannercore.RightOuterJoin
+ joinType := logicalop.RightOuterJoin
rightAsBuildSide := []bool{false, true}
simpleFilter := createSimpleFilter(t)
hasFilter := []bool{false, true}
diff --git a/pkg/executor/load_data.go b/pkg/executor/load_data.go
index e42bc778f77cb..298cef32299d1 100644
--- a/pkg/executor/load_data.go
+++ b/pkg/executor/load_data.go
@@ -291,7 +291,7 @@ func initEncodeCommitWorkers(e *LoadDataWorker) (*encodeWorker, *commitWorker, e
if err2 != nil {
return nil, nil, err2
}
- colAssignExprs, exprWarnings, err2 := e.controller.CreateColAssignExprs(insertValues.Ctx())
+ colAssignExprs, exprWarnings, err2 := e.controller.CreateColAssignExprs(insertValues.Ctx().GetPlanCtx())
if err2 != nil {
return nil, nil, err2
}
@@ -647,6 +647,11 @@ func (w *commitWorker) checkAndInsertOneBatch(ctx context.Context, rows [][]type
case ast.OnDuplicateKeyHandlingIgnore:
return w.batchCheckAndInsert(ctx, rows[0:cnt], w.addRecordLD, false)
case ast.OnDuplicateKeyHandlingError:
+ txn, err := w.Ctx().Txn(true)
+ if err != nil {
+ return err
+ }
+ dupKeyCheck := optimizeDupKeyCheckForNormalInsert(w.Ctx().GetSessionVars(), txn)
for i, row := range rows[0:cnt] {
sizeHintStep := int(w.Ctx().GetSessionVars().ShardAllocateStep)
if sizeHintStep > 0 && i%sizeHintStep == 0 {
@@ -655,9 +660,9 @@ func (w *commitWorker) checkAndInsertOneBatch(ctx context.Context, rows [][]type
if sizeHint > remain {
sizeHint = remain
}
- err = w.addRecordWithAutoIDHint(ctx, row, sizeHint)
+ err = w.addRecordWithAutoIDHint(ctx, row, sizeHint, dupKeyCheck)
} else {
- err = w.addRecord(ctx, row)
+ err = w.addRecord(ctx, row, dupKeyCheck)
}
if err != nil {
return err
@@ -670,11 +675,11 @@ func (w *commitWorker) checkAndInsertOneBatch(ctx context.Context, rows [][]type
}
}
-func (w *commitWorker) addRecordLD(ctx context.Context, row []types.Datum) error {
+func (w *commitWorker) addRecordLD(ctx context.Context, row []types.Datum, dupKeyCheck table.DupKeyCheckMode) error {
if row == nil {
return nil
}
- return w.addRecord(ctx, row)
+ return w.addRecord(ctx, row, dupKeyCheck)
}
// GetInfilePath get infile path.
diff --git a/pkg/executor/memtable_reader.go b/pkg/executor/memtable_reader.go
index 8a7644c2feb54..316439a638b99 100644
--- a/pkg/executor/memtable_reader.go
+++ b/pkg/executor/memtable_reader.go
@@ -791,10 +791,7 @@ func (e *hotRegionsHistoryRetriver) retrieve(ctx context.Context, sctx sessionct
}
tz := sctx.GetSessionVars().Location()
is := sessiontxn.GetTxnManager(sctx).GetTxnInfoSchema()
- allSchemaNames := is.AllSchemaNames()
- schemas := ensureSchemaTables(ctx, is, allSchemaNames)
- schemas = tikvHelper.FilterMemDBs(schemas)
- tables := tikvHelper.GetTablesInfoWithKeyRange(schemas)
+ tables := tikvHelper.GetTablesInfoWithKeyRange(is, tikvHelper.FilterMemDBs)
for e.heap.Len() > 0 && len(finalRows) < hotRegionsHistoryBatchSize {
minTimeItem := heap.Pop(e.heap).(hotRegionsResult)
rows, err := e.getHotRegionRowWithSchemaInfo(minTimeItem.messages.HistoryHotRegion[0], tikvHelper, tables, tz)
diff --git a/pkg/executor/partition_table_test.go b/pkg/executor/partition_table_test.go
index 7a3f5f6a21849..7dbe10f818478 100644
--- a/pkg/executor/partition_table_test.go
+++ b/pkg/executor/partition_table_test.go
@@ -1184,49 +1184,49 @@ func TestPartitionTableWithDifferentJoin(t *testing.T) {
// tk.MustHavePlan(queryHash, "IndexMergeJoin")
// tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows())
tk.MustQuery(queryHash)
- tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Warning|1815|Optimizer Hint /*+ INL_MERGE_JOIN(trange, trange2) */ is inapplicable"))
+ tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Warning|1815|The INDEX MERGE JOIN hint is deprecated for usage, try other hints."))
queryHash = fmt.Sprintf("select /*+ inl_merge_join(trange, trange2) */ * from trange, trange2 where trange.a=trange2.a and trange.a > %v and trange2.a > %v;", x1, x2)
// queryRegular = fmt.Sprintf("select /*+ inl_merge_join(tregular2, tregular4) */ * from tregular2, tregular4 where tregular2.a=tregular4.a and tregular2.a > %v and tregular4.a > %v;", x1, x2)
// tk.MustHavePlan(queryHash, "IndexMergeJoin")
// tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows())
tk.MustQuery(queryHash)
- tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Warning|1815|Optimizer Hint /*+ INL_MERGE_JOIN(trange, trange2) */ is inapplicable"))
+ tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Warning|1815|The INDEX MERGE JOIN hint is deprecated for usage, try other hints."))
queryHash = fmt.Sprintf("select /*+ inl_merge_join(trange, trange2) */ * from trange, trange2 where trange.a=trange2.a and trange.a > %v and trange.b > %v;", x1, x2)
// queryRegular = fmt.Sprintf("select /*+ inl_merge_join(tregular2, tregular4) */ * from tregular2, tregular4 where tregular2.a=tregular4.a and tregular2.a > %v and tregular2.b > %v;", x1, x2)
// tk.MustHavePlan(queryHash, "IndexMergeJoin")
// tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows())
tk.MustQuery(queryHash)
- tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Warning|1815|Optimizer Hint /*+ INL_MERGE_JOIN(trange, trange2) */ is inapplicable"))
+ tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Warning|1815|The INDEX MERGE JOIN hint is deprecated for usage, try other hints."))
queryHash = fmt.Sprintf("select /*+ inl_merge_join(trange, trange2) */ * from trange, trange2 where trange.a=trange2.a and trange.a > %v and trange2.b > %v;", x1, x2)
// queryRegular = fmt.Sprintf("select /*+ inl_merge_join(tregular2, tregular4) */ * from tregular2, tregular4 where tregular2.a=tregular4.a and tregular2.a > %v and tregular4.b > %v;", x1, x2)
// tk.MustHavePlan(queryHash, "IndexMergeJoin")
// tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows())
tk.MustQuery(queryHash)
- tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Warning|1815|Optimizer Hint /*+ INL_MERGE_JOIN(trange, trange2) */ is inapplicable"))
+ tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Warning|1815|The INDEX MERGE JOIN hint is deprecated for usage, try other hints."))
// group 6
// index_merge_join range partition and regualr table
queryHash = fmt.Sprintf("select /*+ inl_merge_join(trange, tregular4) */ * from trange, tregular4 where trange.a=tregular4.a and trange.a > %v;", x1)
queryRegular = fmt.Sprintf("select /*+ inl_merge_join(tregular2, tregular4) */ * from tregular2, tregular4 where tregular2.a=tregular4.a and tregular2.a > %v;", x1)
- tk.MustHavePlan(queryHash, "IndexMergeJoin")
+ tk.MustNotHavePlan(queryHash, "IndexMergeJoin")
tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows())
queryHash = fmt.Sprintf("select /*+ inl_merge_join(trange, tregular4) */ * from trange, tregular4 where trange.a=tregular4.a and trange.a > %v and tregular4.a > %v;", x1, x2)
queryRegular = fmt.Sprintf("select /*+ inl_merge_join(tregular2, tregular4) */ * from tregular2, tregular4 where tregular2.a=tregular4.a and tregular2.a > %v and tregular4.a > %v;", x1, x2)
- tk.MustHavePlan(queryHash, "IndexMergeJoin")
+ tk.MustNotHavePlan(queryHash, "IndexMergeJoin")
tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows())
queryHash = fmt.Sprintf("select /*+ inl_merge_join(trange, tregular4) */ * from trange, tregular4 where trange.a=tregular4.a and trange.a > %v and trange.b > %v;", x1, x2)
queryRegular = fmt.Sprintf("select /*+ inl_merge_join(tregular2, tregular4) */ * from tregular2, tregular4 where tregular2.a=tregular4.a and tregular2.a > %v and tregular2.b > %v;", x1, x2)
- tk.MustHavePlan(queryHash, "IndexMergeJoin")
+ tk.MustNotHavePlan(queryHash, "IndexMergeJoin")
tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows())
queryHash = fmt.Sprintf("select /*+ inl_merge_join(trange, tregular4) */ * from trange, tregular4 where trange.a=tregular4.a and trange.a > %v and tregular4.b > %v;", x1, x2)
queryRegular = fmt.Sprintf("select /*+ inl_merge_join(tregular2, tregular4) */ * from tregular2, tregular4 where tregular2.a=tregular4.a and tregular2.a > %v and tregular4.b > %v;", x1, x2)
- tk.MustHavePlan(queryHash, "IndexMergeJoin")
+ tk.MustNotHavePlan(queryHash, "IndexMergeJoin")
tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows())
// group 7
@@ -2099,7 +2099,7 @@ func TestDropGlobalIndex(t *testing.T) {
partition p0 values less than (4),
partition p1 values less than (7),
partition p2 values less than (10))`)
- tk.MustExec("alter table p add unique idx(id)")
+ tk.MustExec("alter table p add unique idx(id) global")
failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/checkDropGlobalIndex", `return(true)`)
tk.MustExec("alter table p drop index idx")
@@ -2243,10 +2243,6 @@ func TestIssue26251(t *testing.T) {
store := testkit.CreateMockStore(t)
tk1 := testkit.NewTestKit(t, store)
- tk1.MustExec("set tidb_enable_global_index=true")
- defer func() {
- tk1.MustExec("set tidb_enable_global_index=default")
- }()
tk1.MustExec("use test")
tk1.MustExec("create table tp (id int primary key) partition by range (id) (partition p0 values less than (100));")
tk1.MustExec("create table tn (id int primary key);")
@@ -2416,7 +2412,7 @@ func TestGlobalIndexWithSelectLock(t *testing.T) {
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("set tidb_enable_global_index = true")
tk1.MustExec("use test")
- tk1.MustExec("create table t(a int, b int, unique index(b), primary key(a)) partition by hash(a) partitions 5;")
+ tk1.MustExec("create table t(a int, b int, unique index(b) global, primary key(a)) partition by hash(a) partitions 5;")
tk1.MustExec("insert into t values (1,1),(2,2),(3,3),(4,4),(5,5);")
tk1.MustExec("begin")
tk1.MustExec("select * from t use index(b) where b = 2 order by b limit 1 for update;")
diff --git a/pkg/executor/pipelined_window.go b/pkg/executor/pipelined_window.go
index 9bd7ea5256d1c..2cea85d02e284 100644
--- a/pkg/executor/pipelined_window.go
+++ b/pkg/executor/pipelined_window.go
@@ -23,7 +23,7 @@ import (
"github.com/pingcap/tidb/pkg/executor/internal/vecgroupchecker"
"github.com/pingcap/tidb/pkg/expression"
"github.com/pingcap/tidb/pkg/parser/ast"
- "github.com/pingcap/tidb/pkg/planner/core"
+ "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/util/chunk"
)
@@ -41,8 +41,8 @@ type PipelinedWindowExec struct {
windowFuncs []aggfuncs.AggFunc
slidingWindowFuncs []aggfuncs.SlidingWindowAggFunc
partialResults []aggfuncs.PartialResult
- start *core.FrameBound
- end *core.FrameBound
+ start *logicalop.FrameBound
+ end *logicalop.FrameBound
groupChecker *vecgroupchecker.VecGroupChecker
// childResult stores the child chunk. Note that even if remaining is 0, e.rows might still references rows in data[0].chk after returned it to upper executor, since there is no guarantee what the upper executor will do to the returned chunk, it might destroy the data (as in the benchmark test, it reused the chunk to pull data, and it will be chk.Reset(), causing panicking). So dataIdx, accumulated and dropped are added to ensure that chunk will only be returned if there is no row reference.
@@ -84,19 +84,10 @@ func (e *PipelinedWindowExec) Close() error {
// Open implements the Executor Open interface
func (e *PipelinedWindowExec) Open(ctx context.Context) (err error) {
- e.rowToConsume = 0
- e.done = false
- e.accumulated = 0
- e.dropped = 0
- e.data = make([]dataInfo, 0)
- e.dataIdx = 0
- e.slidingWindowFuncs = make([]aggfuncs.SlidingWindowAggFunc, len(e.windowFuncs))
- for i, windowFunc := range e.windowFuncs {
- if slidingWindowAggFunc, ok := windowFunc.(aggfuncs.SlidingWindowAggFunc); ok {
- e.slidingWindowFuncs[i] = slidingWindowAggFunc
- }
- }
- e.rows = make([]chunk.Row, 0)
+ e.done, e.newPartition, e.whole, e.initializedSlidingWindow = false, false, false, false
+ e.dataIdx, e.curRowIdx, e.dropped, e.rowToConsume, e.accumulated = 0, 0, 0, 0, 0
+ e.lastStartRow, e.lastEndRow, e.stagedStartRow, e.stagedEndRow, e.rowStart, e.rowCnt = 0, 0, 0, 0, 0, 0
+ e.rows, e.data = make([]chunk.Row, 0), make([]dataInfo, 0)
return e.BaseExecutor.Open(ctx)
}
diff --git a/pkg/executor/pkg_test.go b/pkg/executor/pkg_test.go
index 6ae1f0c3f29f5..264bee6fc3acb 100644
--- a/pkg/executor/pkg_test.go
+++ b/pkg/executor/pkg_test.go
@@ -25,7 +25,7 @@ import (
"github.com/pingcap/tidb/pkg/expression"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/mysql"
- plannercore "github.com/pingcap/tidb/pkg/planner/core"
+ "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/chunk"
"github.com/pingcap/tidb/pkg/util/mock"
@@ -63,7 +63,7 @@ func TestNestedLoopApply(t *testing.T) {
outerFilter := expression.NewFunctionInternal(sctx, ast.LT, types.NewFieldType(mysql.TypeTiny), col0, con)
innerFilter := outerFilter.Clone()
otherFilter := expression.NewFunctionInternal(sctx, ast.EQ, types.NewFieldType(mysql.TypeTiny), col0, col1)
- joiner := join.NewJoiner(sctx, plannercore.InnerJoin, false,
+ joiner := join.NewJoiner(sctx, logicalop.InnerJoin, false,
make([]types.Datum, innerExec.Schema().Len()), []expression.Expression{otherFilter},
exec.RetTypes(outerExec), exec.RetTypes(innerExec), nil, false)
joinSchema := expression.NewSchema(col0, col1)
diff --git a/pkg/executor/point_get.go b/pkg/executor/point_get.go
index ee3b7047aa89e..46d8a9af0cb03 100644
--- a/pkg/executor/point_get.go
+++ b/pkg/executor/point_get.go
@@ -659,7 +659,7 @@ func (e *PointGetExecutor) verifyTxnScope() error {
var partName string
is := e.Ctx().GetInfoSchema().(infoschema.InfoSchema)
- tblInfo, _ := is.TableByID((e.tblInfo.ID))
+ tblInfo, _ := is.TableByID(context.Background(), e.tblInfo.ID)
tblName := tblInfo.Meta().Name.String()
tblID := GetPhysID(tblInfo.Meta(), e.partitionDefIdx)
if tblID != tblInfo.Meta().ID {
diff --git a/pkg/executor/projection.go b/pkg/executor/projection.go
index 48f3992c61638..ce1113dde4776 100644
--- a/pkg/executor/projection.go
+++ b/pkg/executor/projection.go
@@ -95,7 +95,7 @@ type ProjectionExec struct {
parentReqRows int64
memTracker *memory.Tracker
- wg sync.WaitGroup
+ wg *sync.WaitGroup
calculateNoDelay bool
prepared bool
@@ -137,6 +137,8 @@ func (e *ProjectionExec) open(_ context.Context) error {
e.memTracker.Consume(e.childResult.MemoryUsage())
}
+ e.wg = &sync.WaitGroup{}
+
return nil
}
diff --git a/pkg/executor/recover_test.go b/pkg/executor/recover_test.go
index e47c94632ce76..10785751f7acf 100644
--- a/pkg/executor/recover_test.go
+++ b/pkg/executor/recover_test.go
@@ -17,6 +17,7 @@ package executor_test
import (
"context"
"fmt"
+ "sync"
"testing"
"time"
@@ -31,6 +32,7 @@ import (
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/types"
+ "github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/dbterror"
"github.com/pingcap/tidb/pkg/util/gcutil"
"github.com/stretchr/testify/require"
@@ -656,15 +658,24 @@ func TestFlashbackSchemaWithManyTables(t *testing.T) {
// Set GC enable.
require.NoError(t, gcutil.EnableGC(tk.Session()))
- for i := 0; i < 700; i++ {
- tk.MustExec(fmt.Sprintf("create table t%d (a int)", i))
+ var wg util.WaitGroupWrapper
+ for i := 0; i < 10; i++ {
+ idx := i
+ wg.Run(func() {
+ tkit := testkit.NewTestKit(t, store)
+ tkit.MustExec("use many_tables")
+ for j := 0; j < 70; j++ {
+ tkit.MustExec(fmt.Sprintf("create table t_%d_%d (a int)", idx, j))
+ }
+ })
}
+ wg.Wait()
tk.MustExec("drop database many_tables")
tk.MustExec("flashback database many_tables")
- tk.MustQuery("select count(*) from many_tables.t0").Check(testkit.Rows("0"))
+ tk.MustQuery("select count(*) from many_tables.t_0_0").Check(testkit.Rows("0"))
}
// MockGC is used to make GC work in the test environment.
@@ -690,3 +701,51 @@ func MockGC(tk *testkit.TestKit) (string, string, string, func()) {
tk.MustExec("delete from mysql.tidb where variable_name in ( 'tikv_gc_safe_point','tikv_gc_enable' )")
return timeBeforeDrop, timeAfterDrop, safePointSQL, resetGC
}
+
+func TestFlashbackClusterWithManyDBs(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ timeBeforeDrop, _, safePointSQL, resetGC := MockGC(tk)
+ defer resetGC()
+
+ // Set GC safe point.
+ tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop))
+
+ backup := kv.TxnEntrySizeLimit.Load()
+ kv.TxnEntrySizeLimit.Store(50000)
+ t.Cleanup(func() {
+ kv.TxnEntrySizeLimit.Store(backup)
+ })
+
+ tk.MustExec("set @@global.tidb_ddl_error_count_limit = 2")
+ tk.MustExec("set @@global.tidb_enable_fast_create_table=ON")
+
+ var wg sync.WaitGroup
+ dbPerWorker := 10
+ for i := 0; i < 40; i++ {
+ i := i
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ tk2 := testkit.NewTestKit(t, store)
+ for j := 0; j < dbPerWorker; j++ {
+ dbName := fmt.Sprintf("db_%d", i*dbPerWorker+j)
+ tk2.MustExec(fmt.Sprintf("create database %s", dbName))
+ }
+ }()
+ }
+
+ wg.Wait()
+
+ ts, _ := store.CurrentVersion(oracle.GlobalTxnScope)
+ flashbackTs := oracle.GetTimeFromTS(ts.Ver)
+
+ injectSafeTS := oracle.GoTimeToTS(flashbackTs.Add(10 * time.Second))
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ddl/mockFlashbackTest", `return(true)`)
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ddl/injectSafeTS",
+ fmt.Sprintf("return(%v)", injectSafeTS))
+
+ // this test will fail before the fix, because the DDL job KV entry is too large.
+ tk.MustExec(fmt.Sprintf("flashback cluster to timestamp '%s'", flashbackTs))
+}
diff --git a/pkg/executor/replace.go b/pkg/executor/replace.go
index e52dc608343bd..3b9e9b4101d34 100644
--- a/pkg/executor/replace.go
+++ b/pkg/executor/replace.go
@@ -24,6 +24,7 @@ import (
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/meta/autoid"
"github.com/pingcap/tidb/pkg/parser/mysql"
+ "github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/table/tables"
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/types"
@@ -62,7 +63,7 @@ func (e *ReplaceExec) Open(ctx context.Context) error {
}
// replaceRow removes all duplicate rows for one row, then inserts it.
-func (e *ReplaceExec) replaceRow(ctx context.Context, r toBeCheckedRow) error {
+func (e *ReplaceExec) replaceRow(ctx context.Context, r toBeCheckedRow, dupKeyCheck table.DupKeyCheckMode) error {
txn, err := e.Ctx().Txn(true)
if err != nil {
return err
@@ -105,7 +106,7 @@ func (e *ReplaceExec) replaceRow(ctx context.Context, r toBeCheckedRow) error {
}
// No duplicated rows now, insert the row.
- err = e.addRecord(ctx, r.row)
+ err = e.addRecord(ctx, r.row, dupKeyCheck)
if err != nil {
return err
}
@@ -179,9 +180,12 @@ func (e *ReplaceExec) exec(ctx context.Context, newRows [][]types.Datum) error {
if e.stats != nil {
e.stats.Prefetch = time.Since(prefetchStart)
}
- e.Ctx().GetSessionVars().StmtCtx.AddRecordRows(uint64(len(newRows)))
+ sessionVars := e.Ctx().GetSessionVars()
+ sessionVars.StmtCtx.AddRecordRows(uint64(len(newRows)))
+ // TODO: seems we can optimize it to `DupKeyCheckSkip` because all conflict rows are deleted in previous steps.
+ dupKeyCheck := optimizeDupKeyCheckForNormalInsert(sessionVars, txn)
for _, r := range toBeCheckedRows {
- err = e.replaceRow(ctx, r)
+ err = e.replaceRow(ctx, r, dupKeyCheck)
if err != nil {
return err
}
diff --git a/pkg/executor/set.go b/pkg/executor/set.go
index 89a65bb77a2e4..7649027df2abb 100644
--- a/pkg/executor/set.go
+++ b/pkg/executor/set.go
@@ -326,7 +326,11 @@ func (e *SetExecutor) loadSnapshotInfoSchemaIfNeeded(name string, snapshotTS uin
if name != variable.TiDBSnapshot && name != variable.TiDBTxnReadTS {
return nil
}
- vars := e.Ctx().GetSessionVars()
+ return loadSnapshotInfoSchemaIfNeeded(e.Ctx(), snapshotTS)
+}
+
+func loadSnapshotInfoSchemaIfNeeded(sctx sessionctx.Context, snapshotTS uint64) error {
+ vars := sctx.GetSessionVars()
if snapshotTS == 0 {
vars.SnapshotInfoschema = nil
return nil
@@ -334,12 +338,12 @@ func (e *SetExecutor) loadSnapshotInfoSchemaIfNeeded(name string, snapshotTS uin
logutil.BgLogger().Info("load snapshot info schema",
zap.Uint64("conn", vars.ConnectionID),
zap.Uint64("SnapshotTS", snapshotTS))
- dom := domain.GetDomain(e.Ctx())
+ dom := domain.GetDomain(sctx)
snapInfo, err := dom.GetSnapshotInfoSchema(snapshotTS)
if err != nil {
return err
}
- vars.SnapshotInfoschema = temptable.AttachLocalTemporaryTableInfoSchema(e.Ctx(), snapInfo)
+ vars.SnapshotInfoschema = temptable.AttachLocalTemporaryTableInfoSchema(sctx, snapInfo)
return nil
}
diff --git a/pkg/executor/set_config.go b/pkg/executor/set_config.go
index 7ed6aa119c45a..92475d457ca35 100644
--- a/pkg/executor/set_config.go
+++ b/pkg/executor/set_config.go
@@ -70,7 +70,7 @@ func (s *SetConfigExec) Open(context.Context) error {
if s.p.Type == "tiflash" {
if !strings.HasPrefix(s.p.Name, "raftstore-proxy.") {
errorBody := "This command can only change config items begin with 'raftstore-proxy'. For other TiFlash config items, please update the config file directly. Your change to the config file will take effect immediately without a restart."
- return errors.Errorf(errorBody)
+ return errors.New(errorBody)
}
s.p.Name = strings.TrimPrefix(s.p.Name, "raftstore-proxy.")
}
diff --git a/pkg/executor/set_test.go b/pkg/executor/set_test.go
index e350d0264dff8..764537b5a030c 100644
--- a/pkg/executor/set_test.go
+++ b/pkg/executor/set_test.go
@@ -766,6 +766,23 @@ func TestSetVar(t *testing.T) {
tk.MustExec("set global tidb_max_auto_analyze_time = -1")
tk.MustQuery("select @@tidb_max_auto_analyze_time").Check(testkit.Rows("0"))
+ // test for instance plan cache variables
+ tk.MustQuery("select @@global.tidb_enable_instance_plan_cache").Check(testkit.Rows("0")) // default 0
+ tk.MustQuery("select @@global.tidb_instance_plan_cache_target_mem_size").Check(testkit.Rows("104857600"))
+ tk.MustQuery("select @@global.tidb_instance_plan_cache_max_mem_size").Check(testkit.Rows("125829120"))
+ tk.MustExecToErr("set global tidb_instance_plan_cache_target_mem_size = 125829121") // target <= max
+ tk.MustExecToErr("set global tidb_instance_plan_cache_max_mem_size = 104857599")
+ tk.MustExec("set global tidb_instance_plan_cache_target_mem_size = 114857600")
+ tk.MustQuery("select @@global.tidb_instance_plan_cache_target_mem_size").Check(testkit.Rows("114857600"))
+ tk.MustExec("set global tidb_instance_plan_cache_max_mem_size = 135829120")
+ tk.MustQuery("select @@global.tidb_instance_plan_cache_max_mem_size").Check(testkit.Rows("135829120"))
+ tk.MustExec("set global tidb_instance_plan_cache_max_mem_size = 1GiB")
+ tk.MustQuery("select @@global.tidb_instance_plan_cache_max_mem_size").Check(testkit.Rows("1073741824"))
+ tk.MustExec("set global tidb_instance_plan_cache_target_mem_size = 999MiB")
+ tk.MustQuery("select @@global.tidb_instance_plan_cache_target_mem_size").Check(testkit.Rows("1047527424"))
+ tk.MustExec("set global tidb_instance_plan_cache_target_mem_size = 998MiB")
+ tk.MustQuery("select @@global.tidb_instance_plan_cache_target_mem_size").Check(testkit.Rows("1046478848"))
+
// test variables for cost model ver2
tk.MustQuery("select @@tidb_cost_model_version").Check(testkit.Rows(fmt.Sprintf("%v", variable.DefTiDBCostModelVer)))
tk.MustExec("set tidb_cost_model_version=3")
diff --git a/pkg/executor/show.go b/pkg/executor/show.go
index e7c41f4f4469e..3287a4df2108c 100644
--- a/pkg/executor/show.go
+++ b/pkg/executor/show.go
@@ -494,6 +494,67 @@ func (*ShowExec) fetchShowOpenTables() error {
return nil
}
+// showInfo represents the result of `SHOW TABLES`.
+type showInfo struct {
+ Name model.CIStr
+ // only used for show full tables
+ TableType string
+}
+
+// getTableType returns the type of the table.
+func (e *ShowExec) getTableType(tb *model.TableInfo) string {
+ switch {
+ case tb.IsView():
+ return "VIEW"
+ case tb.IsSequence():
+ return "SEQUENCE"
+ case util.IsSystemView(e.DBName.L):
+ return "SYSTEM VIEW"
+ default:
+ return "BASE TABLE"
+ }
+}
+
+// fetchShowInfoByName fetches the show info for `SHOW TABLES like 'xxx'`
+func (e *ShowExec) fetchShowInfoByName(ctx context.Context, name string) ([]*showInfo, error) {
+ tb, err := e.is.TableByName(ctx, e.DBName, model.NewCIStr(name))
+ if err != nil {
+ // do nothing if table not exists
+ if infoschema.ErrTableNotExists.Equal(err) {
+ return nil, nil
+ }
+ return nil, errors.Trace(err)
+ }
+ return []*showInfo{{Name: tb.Meta().Name, TableType: e.getTableType(tb.Meta())}}, nil
+}
+
+// fetchShowSimpleTables fetches the table info for `SHOW TABLE`.
+func (e *ShowExec) fetchShowSimpleTables(ctx context.Context) ([]*showInfo, error) {
+ tb, err := e.is.SchemaSimpleTableInfos(ctx, e.DBName)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ showInfos := make([]*showInfo, 0, len(tb))
+ for _, v := range tb {
+ // TODO: consider add type info to TableNameInfo
+ showInfos = append(showInfos, &showInfo{Name: v.Name})
+ }
+ return showInfos, nil
+}
+
+// fetchShowFullTables fetches the table info for `SHOW FULL TABLES`.
+func (e *ShowExec) fetchShowFullTables(ctx context.Context) ([]*showInfo, error) {
+ tb, err := e.is.SchemaTableInfos(ctx, e.DBName)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ showInfos := make([]*showInfo, 0, len(tb))
+ for _, v := range tb {
+ showInfos = append(showInfos, &showInfo{Name: v.Name, TableType: e.getTableType(v)})
+ }
+ return showInfos, nil
+}
+
func (e *ShowExec) fetchShowTables(ctx context.Context) error {
checker := privilege.GetPrivilegeManager(e.Ctx())
if checker != nil && e.Ctx().GetSessionVars().User != nil {
@@ -504,12 +565,11 @@ func (e *ShowExec) fetchShowTables(ctx context.Context) error {
if !e.is.SchemaExists(e.DBName) {
return exeerrors.ErrBadDB.GenWithStackByArgs(e.DBName)
}
- // sort for tables
- schemaTables, err := e.is.SchemaTableInfos(ctx, e.DBName)
- if err != nil {
- return errors.Trace(err)
- }
- tableNames := make([]string, 0, len(schemaTables))
+ var (
+ tableNames = make([]string, 0)
+ showInfos []*showInfo
+ err error
+ )
activeRoles := e.Ctx().GetSessionVars().ActiveRoles
var (
tableTypes = make(map[string]string)
@@ -521,7 +581,18 @@ func (e *ShowExec) fetchShowTables(ctx context.Context) error {
fieldFilter = e.Extractor.Field()
fieldPatternsLike = e.Extractor.FieldPatternLike()
}
- for _, v := range schemaTables {
+
+ if fieldFilter != "" {
+ showInfos, err = e.fetchShowInfoByName(ctx, fieldFilter)
+ } else if e.Full {
+ showInfos, err = e.fetchShowFullTables(ctx)
+ } else {
+ showInfos, err = e.fetchShowSimpleTables(ctx)
+ }
+ if err != nil {
+ return errors.Trace(err)
+ }
+ for _, v := range showInfos {
// Test with mysql.AllPrivMask means any privilege would be OK.
// TODO: Should consider column privileges, which also make a table visible.
if checker != nil && !checker.RequestVerification(activeRoles, e.DBName.O, v.Name.O, "", mysql.AllPrivMask) {
@@ -532,14 +603,8 @@ func (e *ShowExec) fetchShowTables(ctx context.Context) error {
continue
}
tableNames = append(tableNames, v.Name.O)
- if v.IsView() {
- tableTypes[v.Name.O] = "VIEW"
- } else if v.IsSequence() {
- tableTypes[v.Name.O] = "SEQUENCE"
- } else if util.IsSystemView(e.DBName.L) {
- tableTypes[v.Name.O] = "SYSTEM VIEW"
- } else {
- tableTypes[v.Name.O] = "BASE TABLE"
+ if e.Full {
+ tableTypes[v.Name.O] = v.TableType
}
}
slices.Sort(tableNames)
@@ -618,6 +683,11 @@ func (e *ShowExec) fetchShowColumns(ctx context.Context) error {
if err != nil {
return errors.Trace(err)
}
+ // we will fill the column type information later, so clone a new table here.
+ tb, err = table.TableFromMeta(tb.Allocators(e.Ctx().GetTableCtx()), tb.Meta().Clone())
+ if err != nil {
+ return errors.Trace(err)
+ }
var (
fieldPatternsLike collate.WildcardPattern
fieldFilter string
@@ -1170,6 +1240,9 @@ func constructResultOfShowCreateTable(ctx sessionctx.Context, dbName *model.CISt
buf.WriteString(" /*T![clustered_index] NONCLUSTERED */")
}
}
+ if idxInfo.Global {
+ buf.WriteString(" /*T![global_index] GLOBAL */")
+ }
if i != len(publicIndices)-1 {
buf.WriteString(",\n")
}
@@ -1951,7 +2024,7 @@ func (e *ShowExec) getTable() (table.Table, error) {
if e.Table == nil {
return nil, errors.New("table not found")
}
- tb, ok := e.is.TableByID(e.Table.TableInfo.ID)
+ tb, ok := e.is.TableByID(context.Background(), e.Table.TableInfo.ID)
if !ok {
return nil, errors.Errorf("table %s not found", e.Table.Name)
}
@@ -2252,7 +2325,8 @@ func (e *ShowExec) fetchShowSessionStates(ctx context.Context) error {
return nil
}
-func fillOneImportJobInfo(info *importer.JobInfo, result *chunk.Chunk, importedRowCount int64) {
+// FillOneImportJobInfo is exported for testing.
+func FillOneImportJobInfo(info *importer.JobInfo, result *chunk.Chunk, importedRowCount int64) {
fullTableName := utils.EncloseDBAndTable(info.TableSchema, info.TableName)
result.AppendInt64(0, info.ID)
result.AppendString(1, info.Parameters.FileLocation)
@@ -2293,7 +2367,7 @@ func handleImportJobInfo(ctx context.Context, info *importer.JobInfo, result *ch
}
importedRowCount = int64(rows)
}
- fillOneImportJobInfo(info, result, importedRowCount)
+ FillOneImportJobInfo(info, result, importedRowCount)
return nil
}
@@ -2383,6 +2457,10 @@ func runWithSystemSession(ctx context.Context, sctx sessionctx.Context, fn func(
return err
}
defer b.ReleaseSysSession(ctx, sysCtx)
+
+ if err = loadSnapshotInfoSchemaIfNeeded(sysCtx, sctx.GetSessionVars().SnapshotTS); err != nil {
+ return err
+ }
// `fn` may use KV transaction, so initialize the txn here
if err = sessiontxn.NewTxn(ctx, sysCtx); err != nil {
return err
diff --git a/pkg/executor/show_test.go b/pkg/executor/show_test.go
index 17c44771cea84..2b6ee9f89c9fb 100644
--- a/pkg/executor/show_test.go
+++ b/pkg/executor/show_test.go
@@ -12,14 +12,17 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package executor
+package executor_test
import (
"testing"
"time"
+ "github.com/pingcap/tidb/pkg/errno"
+ "github.com/pingcap/tidb/pkg/executor"
"github.com/pingcap/tidb/pkg/executor/importer"
"github.com/pingcap/tidb/pkg/parser/mysql"
+ "github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/chunk"
"github.com/stretchr/testify/require"
@@ -44,12 +47,12 @@ func Test_fillOneImportJobInfo(t *testing.T) {
jobInfo := &importer.JobInfo{
Parameters: importer.ImportParameters{},
}
- fillOneImportJobInfo(jobInfo, c, -1)
+ executor.FillOneImportJobInfo(jobInfo, c, -1)
require.True(t, c.GetRow(0).IsNull(7))
require.True(t, c.GetRow(0).IsNull(10))
require.True(t, c.GetRow(0).IsNull(11))
- fillOneImportJobInfo(jobInfo, c, 0)
+ executor.FillOneImportJobInfo(jobInfo, c, 0)
require.False(t, c.GetRow(1).IsNull(7))
require.Equal(t, uint64(0), c.GetRow(1).GetUint64(7))
require.True(t, c.GetRow(1).IsNull(10))
@@ -58,9 +61,32 @@ func Test_fillOneImportJobInfo(t *testing.T) {
jobInfo.Summary = &importer.JobSummary{ImportedRows: 123}
jobInfo.StartTime = types.NewTime(types.FromGoTime(time.Now()), mysql.TypeTimestamp, 0)
jobInfo.EndTime = types.NewTime(types.FromGoTime(time.Now()), mysql.TypeTimestamp, 0)
- fillOneImportJobInfo(jobInfo, c, 0)
+ executor.FillOneImportJobInfo(jobInfo, c, 0)
require.False(t, c.GetRow(2).IsNull(7))
require.Equal(t, uint64(123), c.GetRow(2).GetUint64(7))
require.False(t, c.GetRow(2).IsNull(10))
require.False(t, c.GetRow(2).IsNull(11))
}
+
+func TestShow(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ tk.MustExec("use test")
+ tk.MustExec("create table t(id int, abclmn int);")
+ tk.MustExec("create table abclmn(a int);")
+
+ tk.MustGetErrCode("show columns from t like id", errno.ErrBadField)
+ tk.MustGetErrCode("show columns from t like `id`", errno.ErrBadField)
+
+ tk.MustQuery("show tables").Check(testkit.Rows("abclmn", "t"))
+ tk.MustQuery("show full tables").Check(testkit.Rows("abclmn BASE TABLE", "t BASE TABLE"))
+ tk.MustQuery("show tables like 't'").Check(testkit.Rows("t"))
+ tk.MustQuery("show tables like 'T'").Check(testkit.Rows("t"))
+ tk.MustQuery("show tables like 'ABCLMN'").Check(testkit.Rows("abclmn"))
+ tk.MustQuery("show tables like 'ABC%'").Check(testkit.Rows("abclmn"))
+ tk.MustQuery("show tables like '%lmn'").Check(testkit.Rows("abclmn"))
+ tk.MustQuery("show full tables like '%lmn'").Check(testkit.Rows("abclmn BASE TABLE"))
+ tk.MustGetErrCode("show tables like T", errno.ErrBadField)
+ tk.MustGetErrCode("show tables like `T`", errno.ErrBadField)
+}
diff --git a/pkg/executor/simple.go b/pkg/executor/simple.go
index b5c5adecf75d3..737eedf2690c7 100644
--- a/pkg/executor/simple.go
+++ b/pkg/executor/simple.go
@@ -2592,7 +2592,7 @@ func (e *SimpleExec) executeKillStmt(ctx context.Context, s *ast.KillStmt) error
if x, ok := s.Expr.(*ast.FuncCallExpr); ok {
if x.FnName.L == ast.ConnectionID {
sm := e.Ctx().GetSessionManager()
- sm.Kill(e.Ctx().GetSessionVars().ConnectionID, s.Query, false)
+ sm.Kill(e.Ctx().GetSessionVars().ConnectionID, s.Query, false, false)
return nil
}
return errors.New("Invalid operation. Please use 'KILL TIDB [CONNECTION | QUERY] [connectionID | CONNECTION_ID()]' instead")
@@ -2604,7 +2604,7 @@ func (e *SimpleExec) executeKillStmt(ctx context.Context, s *ast.KillStmt) error
if sm == nil {
return nil
}
- sm.Kill(s.ConnectionID, s.Query, false)
+ sm.Kill(s.ConnectionID, s.Query, false, false)
} else {
err := errors.NewNoStackError("Invalid operation. Please use 'KILL TIDB [CONNECTION | QUERY] [connectionID | CONNECTION_ID()]' instead")
e.Ctx().GetSessionVars().StmtCtx.AppendWarning(err)
@@ -2619,7 +2619,7 @@ func (e *SimpleExec) executeKillStmt(ctx context.Context, s *ast.KillStmt) error
if e.IsFromRemote {
logutil.BgLogger().Info("Killing connection in current instance redirected from remote TiDB", zap.Uint64("conn", s.ConnectionID), zap.Bool("query", s.Query),
zap.String("sourceAddr", e.Ctx().GetSessionVars().SourceAddr.IP.String()))
- sm.Kill(s.ConnectionID, s.Query, false)
+ sm.Kill(s.ConnectionID, s.Query, false, false)
return nil
}
@@ -2645,7 +2645,7 @@ func (e *SimpleExec) executeKillStmt(ctx context.Context, s *ast.KillStmt) error
e.Ctx().GetSessionVars().StmtCtx.AppendWarning(err1)
}
} else {
- sm.Kill(s.ConnectionID, s.Query, false)
+ sm.Kill(s.ConnectionID, s.Query, false, false)
}
return nil
diff --git a/pkg/executor/slow_query_test.go b/pkg/executor/slow_query_test.go
index 1e6e6029640db..d6eab8b2c7e96 100644
--- a/pkg/executor/slow_query_test.go
+++ b/pkg/executor/slow_query_test.go
@@ -59,7 +59,8 @@ func parseLog(retriever *slowQueryRetriever, sctx sessionctx.Context, reader *bu
func newSlowQueryRetriever() (*slowQueryRetriever, error) {
data := infoschema.NewData()
- newISBuilder, err := infoschema.NewBuilder(nil, nil, data).InitWithDBInfos(nil, nil, nil, 0)
+ newISBuilder := infoschema.NewBuilder(nil, nil, data, variable.SchemaCacheSize.Load() > 0)
+ err := newISBuilder.InitWithDBInfos(nil, nil, nil, 0)
if err != nil {
return nil, err
}
diff --git a/pkg/executor/sortexec/BUILD.bazel b/pkg/executor/sortexec/BUILD.bazel
index ec10efd843923..e8c7dbc6a142c 100644
--- a/pkg/executor/sortexec/BUILD.bazel
+++ b/pkg/executor/sortexec/BUILD.bazel
@@ -42,7 +42,7 @@ go_test(
timeout = "short",
srcs = ["sort_test.go"],
flaky = True,
- shard_count = 14,
+ shard_count = 15,
deps = [
"//pkg/config",
"//pkg/sessionctx/variable",
diff --git a/pkg/executor/sortexec/parallel_sort_test.go b/pkg/executor/sortexec/parallel_sort_test.go
index 36d6858c28a45..0fda9162f6495 100644
--- a/pkg/executor/sortexec/parallel_sort_test.go
+++ b/pkg/executor/sortexec/parallel_sort_test.go
@@ -16,7 +16,9 @@ package sortexec_test
import (
"context"
+ "fmt"
"math/rand"
+ "sort"
"sync"
"testing"
"time"
@@ -26,6 +28,7 @@ import (
"github.com/pingcap/tidb/pkg/executor/internal/testutil"
"github.com/pingcap/tidb/pkg/executor/sortexec"
"github.com/pingcap/tidb/pkg/expression"
+ "github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/util/memory"
"github.com/pingcap/tidb/pkg/util/mock"
"github.com/stretchr/testify/require"
@@ -142,3 +145,53 @@ func TestFailpoint(t *testing.T) {
failpointTest(t, ctx, exe, sortCase, dataSource)
}
}
+
+func TestIssue55344(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("set @@tidb_max_chunk_size=32")
+ tk.MustExec("set @@tidb_init_chunk_size=1")
+ tk.MustExec("drop table if exists t0;")
+ tk.MustExec("CREATE TABLE t0(c0 BOOL);")
+ tk.MustExec("INSERT INTO mysql.opt_rule_blacklist VALUES('predicate_push_down'),('column_prune');")
+ tk.MustExec("ADMIN reload opt_rule_blacklist;")
+
+ // Should not be panic
+ tk.MustQuery("SELECT t0.c0 FROM t0 WHERE 0 ORDER BY -646041453 ASC;")
+
+ // Test correctness
+ tk.MustExec("drop table if exists t1;")
+ tk.MustExec("CREATE TABLE t1(c int);")
+ valueNum := 1000
+ insertedValues := make([]int, 0, valueNum)
+ for i := 0; i < valueNum; i++ {
+ insertedValues = append(insertedValues, rand.Intn(10000))
+ }
+
+ insertSQL := fmt.Sprintf("INSERT INTO t1 values (%d)", insertedValues[0])
+ for i := 1; i < valueNum; i++ {
+ insertSQL = fmt.Sprintf("%s, (%d)", insertSQL, insertedValues[i])
+ }
+ insertSQL += ";"
+
+ tk.MustExec(insertSQL)
+ sort.Ints(insertedValues)
+
+ expectValue := fmt.Sprintf("%d", insertedValues[0])
+ for i := 1; i < valueNum; i++ {
+ expectValue = fmt.Sprintf("%s\n%d", expectValue, insertedValues[i])
+ }
+
+ result := tk.MustQuery("select c from t1 order by c, -646041453;")
+ require.Equal(t, expectValue, result.String())
+ result = tk.MustQuery("select c from t1 order by -646041453, c;")
+ require.Equal(t, expectValue, result.String())
+ result = tk.MustQuery("select c from t1 order by c, -646041453, c+1;")
+ require.Equal(t, expectValue, result.String())
+ result = tk.MustQuery("select c from t1 order by c+1, -646041453, c;")
+ require.Equal(t, expectValue, result.String())
+
+ tk.MustExec("delete from mysql.opt_rule_blacklist where name='column_prune' or name='predicate_push_down';")
+ tk.MustExec("ADMIN reload opt_rule_blacklist;")
+}
diff --git a/pkg/executor/sortexec/sort.go b/pkg/executor/sortexec/sort.go
index 9f581cb699a9a..5ce67e68648c7 100644
--- a/pkg/executor/sortexec/sort.go
+++ b/pkg/executor/sortexec/sort.go
@@ -16,6 +16,7 @@ package sortexec
import (
"context"
+ "math/rand"
"sync"
"sync/atomic"
"time"
@@ -96,6 +97,13 @@ type SortExec struct {
enableTmpStorageOnOOM bool
}
+// When fetcher and workers are not created, we need to initiatively close these channels
+func (e *SortExec) closeChannels() {
+ close(e.Parallel.resultChannel)
+ close(e.Parallel.chunkChannel)
+ close(e.Parallel.closeSync)
+}
+
// Close implements the Executor Close interface.
func (e *SortExec) Close() error {
// TopN not initializes `e.finishCh` but it will call the Close function
@@ -112,8 +120,7 @@ func (e *SortExec) Close() error {
}
} else if e.finishCh != nil {
if e.fetched.CompareAndSwap(false, true) {
- close(e.Parallel.resultChannel)
- close(e.Parallel.chunkChannel)
+ e.closeChannels()
} else {
for range e.Parallel.chunkChannel {
e.Parallel.fetcherAndWorkerSyncer.Done()
@@ -260,10 +267,15 @@ func (e *SortExec) Next(ctx context.Context, req *chunk.Chunk) error {
if e.fetched.CompareAndSwap(false, true) {
err := e.initCompareFuncs(e.Ctx().GetExprCtx().GetEvalCtx())
if err != nil {
+ e.closeChannels()
return err
}
- e.buildKeyColumns()
+ err = e.buildKeyColumns()
+ if err != nil {
+ e.closeChannels()
+ return err
+ }
err = e.fetchChunks(ctx)
if err != nil {
return err
@@ -752,6 +764,19 @@ func (e *SortExec) fetchChunksFromChild(ctx context.Context) {
}
func (e *SortExec) initCompareFuncs(ctx expression.EvalContext) error {
+ var err error
+ failpoint.Inject("ParallelSortRandomFail", func(val failpoint.Value) {
+ if val.(bool) {
+ randNum := rand.Int31n(10000)
+ if randNum < 500 {
+ err = errors.NewNoStackError("return error by random failpoint")
+ }
+ }
+ })
+ if err != nil {
+ return err
+ }
+
e.keyCmpFuncs = make([]chunk.CompareFunc, len(e.ByItems))
for i := range e.ByItems {
keyType := e.ByItems[i].Expr.GetType(ctx)
@@ -763,12 +788,19 @@ func (e *SortExec) initCompareFuncs(ctx expression.EvalContext) error {
return nil
}
-func (e *SortExec) buildKeyColumns() {
+func (e *SortExec) buildKeyColumns() error {
e.keyColumns = make([]int, 0, len(e.ByItems))
for _, by := range e.ByItems {
- col := by.Expr.(*expression.Column)
- e.keyColumns = append(e.keyColumns, col.Index)
+ switch col := by.Expr.(type) {
+ case *expression.Column:
+ e.keyColumns = append(e.keyColumns, col.Index)
+ case *expression.Constant:
+ // Ignore constant as constant can not affect the sorted result
+ default:
+ return errors.NewNoStackError("Get unexpected expression")
+ }
}
+ return nil
}
func (e *SortExec) lessRow(rowI, rowJ chunk.Row) int {
diff --git a/pkg/executor/sortexec/topn.go b/pkg/executor/sortexec/topn.go
index 5d49cb80703d4..d302fd03d4d3f 100644
--- a/pkg/executor/sortexec/topn.go
+++ b/pkg/executor/sortexec/topn.go
@@ -268,7 +268,11 @@ func (e *TopNExec) loadChunksUntilTotalLimit(ctx context.Context) error {
return err
}
- e.buildKeyColumns()
+ err = e.buildKeyColumns()
+ if err != nil {
+ return err
+ }
+
e.chkHeap.init(e, e.memTracker, e.Limit.Offset+e.Limit.Count, int(e.Limit.Offset), e.greaterRow, e.RetFieldTypes())
for uint64(e.chkHeap.rowChunks.Len()) < e.chkHeap.totalLimit {
srcChk := exec.TryNewCacheChunk(e.Children(0))
diff --git a/pkg/executor/stale_txn_test.go b/pkg/executor/stale_txn_test.go
index 8dd3613c4b5bd..0b5f959f09d40 100644
--- a/pkg/executor/stale_txn_test.go
+++ b/pkg/executor/stale_txn_test.go
@@ -15,6 +15,7 @@
package executor_test
import (
+ "bytes"
"context"
"fmt"
"testing"
@@ -23,6 +24,8 @@ import (
"github.com/docker/go-units"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
+ "github.com/pingcap/kvproto/pkg/errorpb"
+ "github.com/pingcap/kvproto/pkg/kvrpcpb"
"github.com/pingcap/tidb/pkg/config"
"github.com/pingcap/tidb/pkg/ddl/placement"
"github.com/pingcap/tidb/pkg/sessiontxn"
@@ -31,6 +34,8 @@ import (
"github.com/pingcap/tidb/pkg/types"
"github.com/stretchr/testify/require"
"github.com/tikv/client-go/v2/oracle"
+ "github.com/tikv/client-go/v2/tikvrpc"
+ "github.com/tikv/client-go/v2/tikvrpc/interceptor"
)
func TestExactStalenessTransaction(t *testing.T) {
@@ -1382,3 +1387,45 @@ func TestStaleTSO(t *testing.T) {
tk.MustQuery("select * from t as of timestamp " + expr + " order by id asc").Check(testkit.Rows("1"))
}
}
+
+func TestStaleReadNoBackoff(t *testing.T) {
+ cfg := config.GetGlobalConfig()
+ cfg.Labels = map[string]string{"zone": "us-east-1a"}
+ config.StoreGlobalConfig(cfg)
+ require.Equal(t, "us-east-1a", config.GetGlobalConfig().GetTiKVConfig().TxnScope)
+
+ store := testkit.CreateMockStore(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t;")
+ tk.MustExec("create table t (id int primary key)")
+ tk.MustExec("insert into t values (1)")
+ tk.MustExec("set session tidb_read_staleness = -1;")
+ tk.MustExec("set session tidb_replica_read='closest-replicas'")
+
+ // sleep 1s so stale read can see schema.
+ time.Sleep(time.Second)
+
+ failStaleReadCtx := interceptor.WithRPCInterceptor(context.Background(), interceptor.NewRPCInterceptor("fail-stale-read", func(next interceptor.RPCInterceptorFunc) interceptor.RPCInterceptorFunc {
+ return func(target string, req *tikvrpc.Request) (*tikvrpc.Response, error) {
+ if getRequest, ok := req.Req.(*kvrpcpb.GetRequest); ok {
+ if ctx := getRequest.GetContext(); ctx != nil && ctx.StaleRead && !ctx.IsRetryRequest {
+ return &tikvrpc.Response{Resp: &kvrpcpb.GetResponse{RegionError: &errorpb.Error{
+ DataIsNotReady: &errorpb.DataIsNotReady{},
+ }}}, nil
+ }
+ }
+ return next(target, req)
+ }
+ }))
+
+ res := tk.MustQueryWithContext(failStaleReadCtx, "explain analyze select * from t where id = 1")
+ resBuff := bytes.NewBufferString("")
+ for _, row := range res.Rows() {
+ _, _ = fmt.Fprintf(resBuff, "%s\t", row)
+ }
+ explain := resBuff.String()
+ require.Regexp(t, ".*rpc_errors:{data_is_not_ready:1.*", explain)
+ require.NotRegexp(t, ".*dataNotReady_backoff.*", explain)
+}
diff --git a/pkg/executor/staticrecordset/BUILD.bazel b/pkg/executor/staticrecordset/BUILD.bazel
index bff3c7ec97b2c..24c6d4afd75c9 100644
--- a/pkg/executor/staticrecordset/BUILD.bazel
+++ b/pkg/executor/staticrecordset/BUILD.bazel
@@ -27,6 +27,7 @@ go_test(
flaky = True,
shard_count = 7,
deps = [
+ "//pkg/parser/mysql",
"//pkg/session/cursor",
"//pkg/testkit",
"//pkg/util/sqlexec",
diff --git a/pkg/executor/staticrecordset/integration_test.go b/pkg/executor/staticrecordset/integration_test.go
index 961d7460474a6..26f35458ab5a8 100644
--- a/pkg/executor/staticrecordset/integration_test.go
+++ b/pkg/executor/staticrecordset/integration_test.go
@@ -20,6 +20,7 @@ import (
"time"
"github.com/pingcap/failpoint"
+ "github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/session/cursor"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/util/sqlexec"
@@ -31,6 +32,8 @@ func TestStaticRecordSet(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
+ tk.Session().GetSessionVars().SetStatusFlag(mysql.ServerStatusCursorExists, true)
+
tk.MustExec("use test")
tk.MustExec("create table t(id int)")
tk.MustExec("insert into t values (1), (2), (3)")
@@ -62,6 +65,8 @@ func TestStaticRecordSetWithTxn(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
+ tk.Session().GetSessionVars().SetStatusFlag(mysql.ServerStatusCursorExists, true)
+
tk.MustExec("use test")
tk.MustExec("create table t(id int)")
tk.MustExec("insert into t values (1), (2), (3)")
@@ -101,6 +106,8 @@ func TestStaticRecordSetExceedGCTime(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
+ tk.Session().GetSessionVars().SetStatusFlag(mysql.ServerStatusCursorExists, true)
+
tk.MustExec("use test")
tk.MustExec("create table t(id int)")
tk.MustExec("insert into t values (1), (2), (3)")
@@ -136,6 +143,8 @@ func TestDetachError(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
+ tk.Session().GetSessionVars().SetStatusFlag(mysql.ServerStatusCursorExists, true)
+
tk.MustExec("use test")
tk.MustExec("create table t(id int)")
tk.MustExec("insert into t values (1), (2), (3)")
@@ -155,6 +164,8 @@ func TestCursorWillBeClosed(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
+ tk.Session().GetSessionVars().SetStatusFlag(mysql.ServerStatusCursorExists, true)
+
tk.MustExec("use test")
tk.MustExec("create table t(id int)")
tk.MustExec("insert into t values (1), (2), (3)")
@@ -180,6 +191,8 @@ func TestCursorWillBlockMinStartTS(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
tk := testkit.NewTestKit(t, store)
+ tk.Session().GetSessionVars().SetStatusFlag(mysql.ServerStatusCursorExists, true)
+
tk.MustExec("use test")
tk.MustExec("create table t(id int)")
tk.MustExec("insert into t values (1), (2), (3)")
@@ -217,6 +230,8 @@ func TestFinishStmtError(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
+ tk.Session().GetSessionVars().SetStatusFlag(mysql.ServerStatusCursorExists, true)
+
tk.MustExec("use test")
tk.MustExec("create table t(id int)")
tk.MustExec("insert into t values (1), (2), (3)")
diff --git a/pkg/executor/stmtsummary_test.go b/pkg/executor/stmtsummary_test.go
index 782a7a612a8ba..9ef4960b4798d 100644
--- a/pkg/executor/stmtsummary_test.go
+++ b/pkg/executor/stmtsummary_test.go
@@ -23,6 +23,7 @@ import (
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/parser/model"
+ "github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/mock"
@@ -32,7 +33,8 @@ import (
func TestStmtSummaryRetriverV2_TableStatementsSummary(t *testing.T) {
data := infoschema.NewData()
- infoSchemaBuilder, err := infoschema.NewBuilder(nil, nil, data).InitWithDBInfos(nil, nil, nil, 0)
+ infoSchemaBuilder := infoschema.NewBuilder(nil, nil, data, variable.SchemaCacheSize.Load() > 0)
+ err := infoSchemaBuilder.InitWithDBInfos(nil, nil, nil, 0)
require.NoError(t, err)
infoSchema := infoSchemaBuilder.Build(math.MaxUint64)
table, err := infoSchema.TableByName(context.Background(), util.InformationSchemaName, model.NewCIStr(infoschema.TableStatementsSummary))
@@ -76,7 +78,8 @@ func TestStmtSummaryRetriverV2_TableStatementsSummary(t *testing.T) {
func TestStmtSummaryRetriverV2_TableStatementsSummaryEvicted(t *testing.T) {
data := infoschema.NewData()
- infoSchemaBuilder, err := infoschema.NewBuilder(nil, nil, data).InitWithDBInfos(nil, nil, nil, 0)
+ infoSchemaBuilder := infoschema.NewBuilder(nil, nil, data, variable.SchemaCacheSize.Load() > 0)
+ err := infoSchemaBuilder.InitWithDBInfos(nil, nil, nil, 0)
require.NoError(t, err)
infoSchema := infoSchemaBuilder.Build(math.MaxUint64)
table, err := infoSchema.TableByName(context.Background(), util.InformationSchemaName, model.NewCIStr(infoschema.TableStatementsSummaryEvicted))
@@ -155,7 +158,8 @@ func TestStmtSummaryRetriverV2_TableStatementsSummaryHistory(t *testing.T) {
stmtSummary.Add(stmtsummaryv2.GenerateStmtExecInfo4Test("digest3"))
data := infoschema.NewData()
- infoSchemaBuilder, err := infoschema.NewBuilder(nil, nil, data).InitWithDBInfos(nil, nil, nil, 0)
+ infoSchemaBuilder := infoschema.NewBuilder(nil, nil, data, variable.SchemaCacheSize.Load() > 0)
+ err = infoSchemaBuilder.InitWithDBInfos(nil, nil, nil, 0)
require.NoError(t, err)
infoSchema := infoSchemaBuilder.Build(math.MaxUint64)
table, err := infoSchema.TableByName(context.Background(), util.InformationSchemaName, model.NewCIStr(infoschema.TableStatementsSummaryHistory))
diff --git a/pkg/executor/test/admintest/BUILD.bazel b/pkg/executor/test/admintest/BUILD.bazel
index 6488fa5a84fd5..32e5567cedfcb 100644
--- a/pkg/executor/test/admintest/BUILD.bazel
+++ b/pkg/executor/test/admintest/BUILD.bazel
@@ -12,7 +12,6 @@ go_test(
deps = [
"//pkg/config",
"//pkg/ddl",
- "//pkg/ddl/util/callback",
"//pkg/domain",
"//pkg/errno",
"//pkg/executor",
@@ -26,6 +25,7 @@ go_test(
"//pkg/table/tables",
"//pkg/tablecodec",
"//pkg/testkit",
+ "//pkg/testkit/testfailpoint",
"//pkg/testkit/testsetup",
"//pkg/testkit/testutil",
"//pkg/types",
diff --git a/pkg/executor/test/admintest/admin_test.go b/pkg/executor/test/admintest/admin_test.go
index 736aa9b9edf5c..cee62ffd83f50 100644
--- a/pkg/executor/test/admintest/admin_test.go
+++ b/pkg/executor/test/admintest/admin_test.go
@@ -26,7 +26,6 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/ddl"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/domain"
mysql "github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/executor"
@@ -39,6 +38,7 @@ import (
"github.com/pingcap/tidb/pkg/table/tables"
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/testkit/testutil"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/codec"
@@ -1786,7 +1786,7 @@ func TestAdminCleanUpGlobalIndex(t *testing.T) {
tk.MustExec("drop table if exists admin_test")
tk.MustExec("set tidb_enable_global_index = true")
- tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a)) partition by hash(c) partitions 5")
+ tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a) global) partition by hash(c) partitions 5")
tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)")
tk.MustExec("analyze table admin_test")
@@ -1831,7 +1831,7 @@ func TestAdminRecoverGlobalIndex(t *testing.T) {
tk.MustExec("drop table if exists admin_test")
tk.MustExec("set tidb_enable_global_index = true")
- tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a)) partition by hash(c) partitions 5")
+ tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a) global) partition by hash(c) partitions 5")
tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)")
tk.MustExec("analyze table admin_test")
@@ -1882,7 +1882,7 @@ func TestAdminCheckGlobalIndex(t *testing.T) {
tk.MustExec("set tidb_enable_global_index = true")
tk.MustExec(fmt.Sprintf("set tidb_enable_fast_table_check = %v", enabled))
- tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a)) partition by hash(c) partitions 5")
+ tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a) global) partition by hash(c) partitions 5")
tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)")
// Make some corrupted index. Build the index information.
@@ -1979,7 +1979,7 @@ func TestAdminCheckGlobalIndexWithClusterIndex(t *testing.T) {
tk.MustExec("set tidb_enable_global_index = true")
tk.MustExec(fmt.Sprintf("set tidb_enable_fast_table_check = %v", enabled))
- tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a), primary key(c)) partition by hash(c) partitions 5")
+ tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a) global, primary key(c)) partition by hash(c) partitions 5")
tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)")
// Make some corrupted index. Build the index information.
@@ -2053,21 +2053,18 @@ func TestAdminCheckGlobalIndexWithClusterIndex(t *testing.T) {
}
func TestAdminCheckGlobalIndexDuringDDL(t *testing.T) {
- store, dom := testkit.CreateMockStoreAndDomain(t)
- originalHook := dom.DDL().GetHook()
+ store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
var schemaMap = make(map[model.SchemaState]struct{})
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
- hook := &callback.TestDDLCallback{Do: dom}
onJobUpdatedExportedFunc := func(job *model.Job) {
schemaMap[job.SchemaState] = struct{}{}
_, err := tk1.Exec("admin check table admin_test")
assert.NoError(t, err)
}
- hook.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
// check table after delete some index key/value pairs.
ddl.MockDMLExecution = func() {
@@ -2076,7 +2073,7 @@ func TestAdminCheckGlobalIndexDuringDDL(t *testing.T) {
}
batchSize := 32
- tk.MustExec(fmt.Sprintf("set global tidb_ddl_reorg_batch_size = %d", batchSize))
+ tk.MustExec(fmt.Sprintf("set @@tidb_ddl_reorg_batch_size = %d", batchSize))
var enableFastCheck = []bool{false, true}
for _, enabled := range enableFastCheck {
@@ -2086,17 +2083,17 @@ func TestAdminCheckGlobalIndexDuringDDL(t *testing.T) {
tk.MustExec("set tidb_enable_global_index = true")
tk.MustExec(fmt.Sprintf("set tidb_enable_fast_table_check = %v", enabled))
- tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a), primary key(c)) partition by hash(c) partitions 5")
+ tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a) global, primary key(c)) partition by hash(c) partitions 5")
tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)")
for i := 1; i <= batchSize*2; i++ {
tk.MustExec(fmt.Sprintf("insert admin_test values (%d, %d, %d)", i*5+1, i, i*5+1))
}
- dom.DDL().SetHook(hook)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", onJobUpdatedExportedFunc)
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/mockDMLExecution", "1*return(true)->return(false)"))
tk.MustExec("alter table admin_test truncate partition p1")
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/mockDMLExecution"))
- dom.DDL().SetHook(originalHook)
+ testfailpoint.Disable(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated")
// Should have 3 different schema states, `none`, `deleteOnly`, `deleteReorg`
require.Len(t, schemaMap, 3)
diff --git a/pkg/executor/test/analyzetest/BUILD.bazel b/pkg/executor/test/analyzetest/BUILD.bazel
index ea5f921883358..49cdd55c580a9 100644
--- a/pkg/executor/test/analyzetest/BUILD.bazel
+++ b/pkg/executor/test/analyzetest/BUILD.bazel
@@ -9,7 +9,7 @@ go_test(
"main_test.go",
],
flaky = True,
- shard_count = 48,
+ shard_count = 49,
deps = [
"//pkg/config",
"//pkg/domain",
@@ -26,7 +26,6 @@ go_test(
"//pkg/sessionctx",
"//pkg/sessionctx/variable",
"//pkg/statistics",
- "//pkg/statistics/handle/autoanalyze/exec",
"//pkg/testkit",
"//pkg/testkit/analyzehelper",
"//pkg/util/dbterror/exeerrors",
diff --git a/pkg/executor/test/analyzetest/analyze_test.go b/pkg/executor/test/analyzetest/analyze_test.go
index 87bd60a792983..2073703e4a699 100644
--- a/pkg/executor/test/analyzetest/analyze_test.go
+++ b/pkg/executor/test/analyzetest/analyze_test.go
@@ -39,7 +39,6 @@ import (
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/statistics"
- "github.com/pingcap/tidb/pkg/statistics/handle/autoanalyze/exec"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/analyzehelper"
"github.com/pingcap/tidb/pkg/util/dbterror/exeerrors"
@@ -703,11 +702,11 @@ func TestSavedAnalyzeOptions(t *testing.T) {
tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_ratio = %v", originalVal2))
}()
tk.MustExec("set global tidb_auto_analyze_ratio = 0.01")
- originalVal3 := exec.AutoAnalyzeMinCnt
+ originalVal3 := statistics.AutoAnalyzeMinCnt
defer func() {
- exec.AutoAnalyzeMinCnt = originalVal3
+ statistics.AutoAnalyzeMinCnt = originalVal3
}()
- exec.AutoAnalyzeMinCnt = 0
+ statistics.AutoAnalyzeMinCnt = 0
tk.MustExec("use test")
tk.MustExec("set @@session.tidb_analyze_version = 2")
@@ -1046,11 +1045,11 @@ func TestSavedAnalyzeColumnOptions(t *testing.T) {
tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_ratio = %v", originalVal2))
}()
tk.MustExec("set global tidb_auto_analyze_ratio = 0.01")
- originalVal3 := exec.AutoAnalyzeMinCnt
+ originalVal3 := statistics.AutoAnalyzeMinCnt
defer func() {
- exec.AutoAnalyzeMinCnt = originalVal3
+ statistics.AutoAnalyzeMinCnt = originalVal3
}()
- exec.AutoAnalyzeMinCnt = 0
+ statistics.AutoAnalyzeMinCnt = 0
originalVal4 := tk.MustQuery("select @@tidb_enable_column_tracking").Rows()[0][0].(string)
defer func() {
tk.MustExec(fmt.Sprintf("set global tidb_enable_column_tracking = %v", originalVal4))
@@ -1888,9 +1887,9 @@ func testKillAutoAnalyze(t *testing.T, ver int) {
tk := testkit.NewTestKit(t, store)
oriStart := tk.MustQuery("select @@tidb_auto_analyze_start_time").Rows()[0][0].(string)
oriEnd := tk.MustQuery("select @@tidb_auto_analyze_end_time").Rows()[0][0].(string)
- exec.AutoAnalyzeMinCnt = 0
+ statistics.AutoAnalyzeMinCnt = 0
defer func() {
- exec.AutoAnalyzeMinCnt = 1000
+ statistics.AutoAnalyzeMinCnt = 1000
tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_start_time='%v'", oriStart))
tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_end_time='%v'", oriEnd))
}()
@@ -1972,9 +1971,9 @@ func TestKillAutoAnalyzeIndex(t *testing.T) {
tk := testkit.NewTestKit(t, store)
oriStart := tk.MustQuery("select @@tidb_auto_analyze_start_time").Rows()[0][0].(string)
oriEnd := tk.MustQuery("select @@tidb_auto_analyze_end_time").Rows()[0][0].(string)
- exec.AutoAnalyzeMinCnt = 0
+ statistics.AutoAnalyzeMinCnt = 0
defer func() {
- exec.AutoAnalyzeMinCnt = 1000
+ statistics.AutoAnalyzeMinCnt = 1000
tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_start_time='%v'", oriStart))
tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_end_time='%v'", oriEnd))
}()
@@ -2064,8 +2063,8 @@ func TestAnalyzeJob(t *testing.T) {
require.Equal(t, addr, rows[0][9])
connID := strconv.FormatUint(tk.Session().GetSessionVars().ConnectionID, 10)
require.Equal(t, connID, rows[0][10])
-
- executor.StartAnalyzeJob(se, job)
+ statsHandle := domain.GetDomain(tk.Session()).StatsHandle()
+ statsHandle.StartAnalyzeJob(job)
ctx := context.WithValue(context.Background(), executor.AnalyzeProgressTest, 100)
rows = tk.MustQueryWithContext(ctx, "show analyze status").Rows()
checkTime := func(val any) {
@@ -2080,12 +2079,12 @@ func TestAnalyzeJob(t *testing.T) {
require.Equal(t, "0.1", rows[0][12]) // PROGRESS
require.Equal(t, "0", rows[0][13]) // ESTIMATED_TOTAL_ROWS
- // UpdateAnalyzeJob requires the interval between two updates to mysql.analyze_jobs is more than 5 second.
+ // UpdateAnalyzeJobProgress requires the interval between two updates to mysql.analyze_jobs is more than 5 second.
// Hence we fake last dump time as 10 second ago in order to make update to mysql.analyze_jobs happen.
lastDumpTime := time.Now().Add(-10 * time.Second)
job.Progress.SetLastDumpTime(lastDumpTime)
const smallCount int64 = 100
- executor.UpdateAnalyzeJob(se, job, smallCount)
+ statsHandle.UpdateAnalyzeJobProgress(job, smallCount)
// Delta count doesn't reach threshold so we don't dump it to mysql.analyze_jobs
require.Equal(t, smallCount, job.Progress.GetDeltaCount())
require.Equal(t, lastDumpTime, job.Progress.GetLastDumpTime())
@@ -2093,7 +2092,7 @@ func TestAnalyzeJob(t *testing.T) {
require.Equal(t, "0", rows[0][4])
const largeCount int64 = 15000000
- executor.UpdateAnalyzeJob(se, job, largeCount)
+ statsHandle.UpdateAnalyzeJobProgress(job, largeCount)
// Delta count reaches threshold so we dump it to mysql.analyze_jobs and update last dump time.
require.Equal(t, int64(0), job.Progress.GetDeltaCount())
require.True(t, job.Progress.GetLastDumpTime().After(lastDumpTime))
@@ -2101,7 +2100,7 @@ func TestAnalyzeJob(t *testing.T) {
rows = tk.MustQuery("show analyze status").Rows()
require.Equal(t, strconv.FormatInt(smallCount+largeCount, 10), rows[0][4])
- executor.UpdateAnalyzeJob(se, job, largeCount)
+ statsHandle.UpdateAnalyzeJobProgress(job, largeCount)
// We have just updated mysql.analyze_jobs in the previous step so we don't update it until 5 second passes or the analyze job is over.
require.Equal(t, largeCount, job.Progress.GetDeltaCount())
require.Equal(t, lastDumpTime, job.Progress.GetLastDumpTime())
@@ -2112,7 +2111,7 @@ func TestAnalyzeJob(t *testing.T) {
if result == statistics.AnalyzeFailed {
analyzeErr = errors.Errorf("analyze meets error")
}
- executor.FinishAnalyzeJob(se, job, analyzeErr)
+ statsHandle.FinishAnalyzeJob(job, analyzeErr, statistics.TableAnalysisJob)
rows = tk.MustQuery("show analyze status").Rows()
require.Equal(t, strconv.FormatInt(smallCount+2*largeCount, 10), rows[0][4])
checkTime(rows[0][6])
@@ -2733,12 +2732,12 @@ func TestAutoAnalyzeAwareGlobalVariableChange(t *testing.T) {
"3 0",
))
- originalVal1 := exec.AutoAnalyzeMinCnt
+ originalVal1 := statistics.AutoAnalyzeMinCnt
originalVal2 := tk.MustQuery("select @@global.tidb_auto_analyze_ratio").Rows()[0][0].(string)
- exec.AutoAnalyzeMinCnt = 0
+ statistics.AutoAnalyzeMinCnt = 0
tk.MustExec("set global tidb_auto_analyze_ratio = 0.001")
defer func() {
- exec.AutoAnalyzeMinCnt = originalVal1
+ statistics.AutoAnalyzeMinCnt = originalVal1
tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_ratio = %v", originalVal2))
}()
@@ -2807,11 +2806,9 @@ func TestAnalyzeColumnsSkipMVIndexJsonCol(t *testing.T) {
// TestAnalyzeMVIndex tests analyzing the mv index use some real data in the table.
// It checks the analyze jobs, async loading and the stats content in the memory.
func TestAnalyzeMVIndex(t *testing.T) {
- require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/executor/DebugAnalyzeJobOperations", "return(true)"))
- require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/statistics/handle/DebugAnalyzeJobOperations", "return(true)"))
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/statistics/handle/autoanalyze/DebugAnalyzeJobOperations", "return(true)"))
defer func() {
- require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/executor/DebugAnalyzeJobOperations"))
- require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/statistics/handle/DebugAnalyzeJobOperations"))
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/statistics/handle/autoanalyze/DebugAnalyzeJobOperations"))
}()
// 1. prepare the table and insert data
store, dom := testkit.CreateMockStoreAndDomain(t)
@@ -3131,3 +3128,12 @@ func TestAnalyzePartitionVerify(t *testing.T) {
}
}
}
+
+func TestIssue55438(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("CREATE TABLE t0(c0 NUMERIC , c1 BIGINT UNSIGNED AS ((CASE 0 WHEN false THEN 1358571571 ELSE TRIM(c0) END )));")
+ tk.MustExec("CREATE INDEX i0 ON t0(c1);")
+ tk.MustExec("analyze table t0")
+}
diff --git a/pkg/executor/test/analyzetest/memorycontrol/BUILD.bazel b/pkg/executor/test/analyzetest/memorycontrol/BUILD.bazel
index 793de08dcf3ad..b4f9ed544df6c 100644
--- a/pkg/executor/test/analyzetest/memorycontrol/BUILD.bazel
+++ b/pkg/executor/test/analyzetest/memorycontrol/BUILD.bazel
@@ -13,7 +13,7 @@ go_test(
"//pkg/config",
"//pkg/executor",
"//pkg/sessionctx/variable",
- "//pkg/statistics/handle/autoanalyze/exec",
+ "//pkg/statistics",
"//pkg/testkit",
"//pkg/util",
"@com_github_pingcap_failpoint//:failpoint",
diff --git a/pkg/executor/test/analyzetest/memorycontrol/memory_control_test.go b/pkg/executor/test/analyzetest/memorycontrol/memory_control_test.go
index 3687531bb6b38..1982e8a974871 100644
--- a/pkg/executor/test/analyzetest/memorycontrol/memory_control_test.go
+++ b/pkg/executor/test/analyzetest/memorycontrol/memory_control_test.go
@@ -23,7 +23,7 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/executor"
- "github.com/pingcap/tidb/pkg/statistics/handle/autoanalyze/exec"
+ "github.com/pingcap/tidb/pkg/statistics"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/util"
"github.com/stretchr/testify/require"
@@ -144,12 +144,12 @@ func TestGlobalMemoryControlForAutoAnalyze(t *testing.T) {
require.Len(t, rs0.Rows(), 0)
h := dom.StatsHandle()
- originalVal4 := exec.AutoAnalyzeMinCnt
+ originalVal4 := statistics.AutoAnalyzeMinCnt
originalVal5 := tk.MustQuery("select @@global.tidb_auto_analyze_ratio").Rows()[0][0].(string)
- exec.AutoAnalyzeMinCnt = 0
+ statistics.AutoAnalyzeMinCnt = 0
tk.MustExec("set global tidb_auto_analyze_ratio = 0.001")
defer func() {
- exec.AutoAnalyzeMinCnt = originalVal4
+ statistics.AutoAnalyzeMinCnt = originalVal4
tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_ratio = %v", originalVal5))
}()
diff --git a/pkg/executor/test/ddl/ddl_test.go b/pkg/executor/test/ddl/ddl_test.go
index 62d9764a723a4..449cf435a5e83 100644
--- a/pkg/executor/test/ddl/ddl_test.go
+++ b/pkg/executor/test/ddl/ddl_test.go
@@ -813,6 +813,11 @@ func TestSetDDLReorgWorkerCnt(t *testing.T) {
tk.MustExec("set @@global.tidb_ddl_reorg_worker_cnt = 257")
tk.MustQuery("SHOW WARNINGS").Check(testkit.Rows("Warning 1292 Truncated incorrect tidb_ddl_reorg_worker_cnt value: '257'"))
tk.MustQuery("select @@global.tidb_ddl_reorg_worker_cnt").Check(testkit.Rows("256"))
+
+ tk.MustExec("set @@tidb_ddl_reorg_worker_cnt = 10;")
+ tk.MustQuery("select @@tidb_ddl_reorg_worker_cnt;").Check(testkit.Rows("10"))
+ tk.MustQuery("select @@global.tidb_ddl_reorg_worker_cnt;").Check(testkit.Rows("256"))
+ require.Equal(t, int32(256), variable.GetDDLReorgWorkerCounter())
}
func TestSetDDLReorgBatchSize(t *testing.T) {
@@ -850,6 +855,10 @@ func TestSetDDLReorgBatchSize(t *testing.T) {
tk.MustExec("set @@global.tidb_ddl_reorg_batch_size = 1000")
res = tk.MustQuery("select @@global.tidb_ddl_reorg_batch_size")
res.Check(testkit.Rows("1000"))
+
+ tk.MustExec("set @@tidb_ddl_reorg_batch_size = 256;")
+ tk.MustQuery("select @@tidb_ddl_reorg_batch_size").Check(testkit.Rows("256"))
+ tk.MustQuery("select @@global.tidb_ddl_reorg_batch_size").Check(testkit.Rows("1000"))
}
func TestSetDDLErrorCountLimit(t *testing.T) {
diff --git a/pkg/executor/test/executor/executor_test.go b/pkg/executor/test/executor/executor_test.go
index cefc99a70fc51..95a575a0d7078 100644
--- a/pkg/executor/test/executor/executor_test.go
+++ b/pkg/executor/test/executor/executor_test.go
@@ -18,6 +18,7 @@ import (
"archive/zip"
"context"
"fmt"
+ "math/rand"
"os"
"path/filepath"
"reflect"
@@ -324,11 +325,11 @@ func TestCheckIndex(t *testing.T) {
recordVal1 := types.MakeDatums(int64(1), int64(10), int64(11))
recordVal2 := types.MakeDatums(int64(2), int64(20), int64(21))
require.NoError(t, sessiontxn.NewTxn(context.Background(), ctx))
- _, err = tb.AddRecord(ctx.GetTableCtx(), recordVal1)
+ txn, err := ctx.Txn(true)
require.NoError(t, err)
- _, err = tb.AddRecord(ctx.GetTableCtx(), recordVal2)
+ _, err = tb.AddRecord(ctx.GetTableCtx(), txn, recordVal1)
require.NoError(t, err)
- txn, err := ctx.Txn(true)
+ _, err = tb.AddRecord(ctx.GetTableCtx(), txn, recordVal2)
require.NoError(t, err)
require.NoError(t, txn.Commit(context.Background()))
@@ -1942,6 +1943,17 @@ func TestLowResolutionTSORead(t *testing.T) {
require.Error(t, err)
}
+func TestLowResolutionTSOReadScope(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+
+ tk1 := testkit.NewTestKit(t, store)
+ require.False(t, tk1.Session().GetSessionVars().UseLowResolutionTSO())
+
+ tk1.MustExec("set global tidb_low_resolution_tso = 'on'")
+ tk2 := testkit.NewTestKit(t, store)
+ require.True(t, tk2.Session().GetSessionVars().UseLowResolutionTSO())
+}
+
func TestAdapterStatement(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
@@ -2986,3 +2998,80 @@ func TestIssue48756(t *testing.T) {
"Warning 1105 ",
))
}
+
+func TestIssue50308(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t(a timestamp);")
+ tk.MustExec("insert ignore into t values(cast('2099-01-01' as date));")
+ tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Warning 1292 Incorrect timestamp value: '2099-01-01' for column 'a' at row 1"))
+ tk.MustQuery("select * from t;").Check(testkit.Rows("0000-00-00 00:00:00"))
+ tk.MustExec("delete from t")
+ tk.MustExec("insert into t values('2000-01-01');")
+ tk.MustGetErrMsg("update t set a=cast('2099-01-01' as date)", "[types:1292]Incorrect timestamp value: '2099-01-01'")
+ tk.MustExec("update ignore t set a=cast('2099-01-01' as date);")
+ tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Warning 1292 Incorrect timestamp value: '2099-01-01'"))
+ tk.MustQuery("select * from t;").Check(testkit.Rows("0000-00-00 00:00:00"))
+}
+
+func TestQueryWithKill(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test;")
+ tk.MustExec("drop table if exists tkq;")
+ tk.MustExec("create table tkq (a int key, b int, index idx_b(b));")
+ tk.MustExec("insert into tkq values (1,1);")
+ var wg sync.WaitGroup
+ ch := make(chan context.CancelFunc, 1024)
+ testDuration := time.Second * 10
+ for i := 0; i < 10; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test;")
+ start := time.Now()
+ for {
+ ctx, cancel := context.WithCancel(context.Background())
+ ch <- cancel
+ rs, err := tk.ExecWithContext(ctx, "select a from tkq where b = 1;")
+ if err == nil {
+ require.NotNil(t, rs)
+ rows, err := session.ResultSetToStringSlice(ctx, tk.Session(), rs)
+ if err == nil {
+ require.Equal(t, 1, len(rows))
+ require.Equal(t, 1, len(rows[0]))
+ require.Equal(t, "1", fmt.Sprintf("%v", rows[0][0]))
+ }
+ }
+ if err != nil {
+ require.Equal(t, context.Canceled, err)
+ }
+ if rs != nil {
+ rs.Close()
+ }
+ if time.Since(start) > testDuration {
+ return
+ }
+ }
+ }()
+ }
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for {
+ select {
+ case cancel := <-ch:
+ // mock for random kill query
+ if len(ch) < 5 {
+ time.Sleep(time.Duration(rand.Intn(1000)) * time.Nanosecond)
+ }
+ cancel()
+ case <-time.After(time.Second):
+ return
+ }
+ }
+ }()
+ wg.Wait()
+}
diff --git a/pkg/executor/test/fktest/BUILD.bazel b/pkg/executor/test/fktest/BUILD.bazel
index e8f32b820572e..721e587b1e71e 100644
--- a/pkg/executor/test/fktest/BUILD.bazel
+++ b/pkg/executor/test/fktest/BUILD.bazel
@@ -8,7 +8,7 @@ go_test(
"main_test.go",
],
flaky = True,
- shard_count = 24,
+ shard_count = 25,
deps = [
"//pkg/config",
"//pkg/executor",
@@ -21,6 +21,7 @@ go_test(
"//pkg/parser/model",
"//pkg/parser/mysql",
"//pkg/testkit",
+ "//pkg/testkit/testfailpoint",
"//pkg/types",
"//pkg/util/dbterror/exeerrors",
"//pkg/util/dbterror/plannererrors",
diff --git a/pkg/executor/test/fktest/foreign_key_test.go b/pkg/executor/test/fktest/foreign_key_test.go
index cb44d58bd23fd..30a10b548bf31 100644
--- a/pkg/executor/test/fktest/foreign_key_test.go
+++ b/pkg/executor/test/fktest/foreign_key_test.go
@@ -33,6 +33,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/dbterror/exeerrors"
"github.com/pingcap/tidb/pkg/util/dbterror/plannererrors"
@@ -2474,3 +2475,21 @@ func TestForeignKeyAndLockView(t *testing.T) {
tk.MustGetErrMsg("update t1 set id=2", "[executor:1213]Deadlock found when trying to get lock; try restarting transaction")
wg.Wait()
}
+
+func TestFKBuild(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("set @@global.tidb_enable_foreign_key=1")
+ tk.MustExec("set @@foreign_key_checks=1")
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (id int key);")
+ tk.MustExec("create table t2 (id int key, foreign key fk (id) references t1(id) ON DELETE CASCADE ON UPDATE CASCADE);")
+
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/domain/MockTryLoadDiffError", `return("renametable")`)
+
+ tk.MustExec("rename table t1 to t3;")
+ tk.MustExec("insert into test.t3 values (1)")
+ tk.MustExec("insert into test.t2 values (1)")
+ tk.MustExec("delete from test.t3")
+ tk.MustQuery("select * from test.t2").Check(testkit.Rows())
+}
diff --git a/pkg/executor/test/issuetest/executor_issue_test.go b/pkg/executor/test/issuetest/executor_issue_test.go
index 4566ba8f08453..aea2d3b601873 100644
--- a/pkg/executor/test/issuetest/executor_issue_test.go
+++ b/pkg/executor/test/issuetest/executor_issue_test.go
@@ -178,9 +178,7 @@ func TestIssue30289(t *testing.T) {
tk.MustExec("drop table if exists t")
tk.MustExec("create table t(a int)")
require.NoError(t, failpoint.Enable(fpName, `return(true)`))
- isHashJoinV2Enabled := join.IsHashJoinV2Enabled()
defer func() {
- join.SetEnableHashJoinV2(isHashJoinV2Enabled)
require.NoError(t, failpoint.Disable(fpName))
}()
useHashJoinV2 := []bool{true, false}
@@ -199,9 +197,7 @@ func TestIssue51998(t *testing.T) {
tk.MustExec("drop table if exists t")
tk.MustExec("create table t(a int)")
require.NoError(t, failpoint.Enable(fpName, `return(true)`))
- isHashJoinV2Enabled := join.IsHashJoinV2Enabled()
defer func() {
- join.SetEnableHashJoinV2(isHashJoinV2Enabled)
require.NoError(t, failpoint.Disable(fpName))
}()
useHashJoinV2 := []bool{true, false}
@@ -621,8 +617,6 @@ func TestIssue42662(t *testing.T) {
tk.MustExec("set global tidb_server_memory_limit='1600MB'")
tk.MustExec("set global tidb_server_memory_limit_sess_min_size=128*1024*1024")
tk.MustExec("set global tidb_mem_oom_action = 'cancel'")
- isHashJoinV2Enabled := join.IsHashJoinV2Enabled()
- defer join.SetEnableHashJoinV2(isHashJoinV2Enabled)
useHashJoinV2 := []bool{true, false}
for _, hashJoinV2 := range useHashJoinV2 {
join.SetEnableHashJoinV2(hashJoinV2)
diff --git a/pkg/executor/test/jointest/hashjoin/BUILD.bazel b/pkg/executor/test/jointest/hashjoin/BUILD.bazel
index 6f845ca4af528..8a0e7b727de6a 100644
--- a/pkg/executor/test/jointest/hashjoin/BUILD.bazel
+++ b/pkg/executor/test/jointest/hashjoin/BUILD.bazel
@@ -9,7 +9,7 @@ go_test(
],
flaky = True,
race = "on",
- shard_count = 19,
+ shard_count = 20,
deps = [
"//pkg/config",
"//pkg/executor/join",
diff --git a/pkg/executor/test/jointest/hashjoin/hash_join_test.go b/pkg/executor/test/jointest/hashjoin/hash_join_test.go
index a4cf651a1983c..34777fad30c7b 100644
--- a/pkg/executor/test/jointest/hashjoin/hash_join_test.go
+++ b/pkg/executor/test/jointest/hashjoin/hash_join_test.go
@@ -378,11 +378,6 @@ func TestExplainAnalyzeJoin(t *testing.T) {
require.Equal(t, 7, len(rows))
require.Regexp(t, "HashJoin.*", rows[0][0])
require.Regexp(t, "time:.*, loops:.*, build_hash_table:{total:.*, fetch:.*, build:.*}, probe:{concurrency:5, total:.*, max:.*, probe:.*, fetch and wait:.*}", rows[0][5])
- // Test for index merge join.
- rows = tk.MustQuery("explain analyze select /*+ INL_MERGE_JOIN(t1, t2) */ * from t1,t2 where t1.a=t2.a;").Rows()
- require.Len(t, rows, 9)
- require.Regexp(t, "IndexMergeJoin_.*", rows[0][0])
- require.Regexp(t, fmt.Sprintf(".*Concurrency:%v.*", tk.Session().GetSessionVars().IndexLookupJoinConcurrency()), rows[0][5])
// TestExplainAnalyzeIndexHashJoin
// Issue 43597
@@ -416,9 +411,6 @@ func TestIssue20270(t *testing.T) {
tk.MustExec("create table t1(c1 int, c2 int)")
tk.MustExec("insert into t values(1,1),(2,2)")
tk.MustExec("insert into t1 values(2,3),(4,4)")
- enableHashJoinV2 := join.IsHashJoinV2Enabled()
- join.SetEnableHashJoinV2(false)
- defer join.SetEnableHashJoinV2(enableHashJoinV2)
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/executor/join/killedInJoin2Chunk", "return(true)"))
err := tk.QueryToErr("select /*+ HASH_JOIN(t, t1) */ * from t left join t1 on t.c1 = t1.c1 where t.c1 = 1 or t1.c2 > 20")
require.Equal(t, exeerrors.ErrQueryInterrupted, err)
@@ -492,10 +484,9 @@ func TestFinalizeCurrentSegPanic(t *testing.T) {
tk.MustExec("create table t2 (a int, b int, c int)")
tk.MustExec("insert into t1 values (1, 1, 1), (1, 2, 2), (2, 1, 3), (2, 2, 4)")
tk.MustExec("insert into t2 values (1, 1, 1), (1, 2, 2), (2, 1, 3), (2, 2, 4)")
- isHashJoinV2Enabled := join.IsHashJoinV2Enabled()
join.SetEnableHashJoinV2(true)
defer func() {
- join.SetEnableHashJoinV2(isHashJoinV2Enabled)
+ join.SetEnableHashJoinV2(false)
}()
fpName := "github.com/pingcap/tidb/pkg/executor/join/finalizeCurrentSegPanic"
require.NoError(t, failpoint.Enable(fpName, "panic(\"finalizeCurrentSegPanic\")"))
@@ -516,10 +507,9 @@ func TestSplitPartitionPanic(t *testing.T) {
tk.MustExec("create table t2 (a int, b int, c int)")
tk.MustExec("insert into t1 values (1, 1, 1), (1, 2, 2), (2, 1, 3), (2, 2, 4)")
tk.MustExec("insert into t2 values (1, 1, 1), (1, 2, 2), (2, 1, 3), (2, 2, 4)")
- isHashJoinV2Enabled := join.IsHashJoinV2Enabled()
join.SetEnableHashJoinV2(true)
defer func() {
- join.SetEnableHashJoinV2(isHashJoinV2Enabled)
+ join.SetEnableHashJoinV2(false)
}()
fpName := "github.com/pingcap/tidb/pkg/executor/join/splitPartitionPanic"
require.NoError(t, failpoint.Enable(fpName, "panic(\"splitPartitionPanic\")"))
@@ -540,10 +530,9 @@ func TestProcessOneProbeChunkPanic(t *testing.T) {
tk.MustExec("create table t2 (a int, b int, c int)")
tk.MustExec("insert into t1 values (1, 1, 1), (1, 2, 2), (2, 1, 3), (2, 2, 4)")
tk.MustExec("insert into t2 values (1, 1, 1), (1, 2, 2), (2, 1, 3), (2, 2, 4)")
- isHashJoinV2Enabled := join.IsHashJoinV2Enabled()
join.SetEnableHashJoinV2(true)
defer func() {
- join.SetEnableHashJoinV2(isHashJoinV2Enabled)
+ join.SetEnableHashJoinV2(false)
}()
fpName := "github.com/pingcap/tidb/pkg/executor/join/processOneProbeChunkPanic"
require.NoError(t, failpoint.Enable(fpName, "panic(\"processOneProbeChunkPanic\")"))
@@ -564,10 +553,9 @@ func TestCreateTasksPanic(t *testing.T) {
tk.MustExec("create table t2 (a int, b int, c int)")
tk.MustExec("insert into t1 values (1, 1, 1), (1, 2, 2), (2, 1, 3), (2, 2, 4)")
tk.MustExec("insert into t2 values (1, 1, 1), (1, 2, 2), (2, 1, 3), (2, 2, 4)")
- isHashJoinV2Enabled := join.IsHashJoinV2Enabled()
join.SetEnableHashJoinV2(true)
defer func() {
- join.SetEnableHashJoinV2(isHashJoinV2Enabled)
+ join.SetEnableHashJoinV2(false)
}()
fpName := "github.com/pingcap/tidb/pkg/executor/join/createTasksPanic"
require.NoError(t, failpoint.Enable(fpName, "panic(\"createTasksPanic\")"))
@@ -588,10 +576,9 @@ func TestBuildHashTablePanic(t *testing.T) {
tk.MustExec("create table t2 (a int, b int, c int)")
tk.MustExec("insert into t1 values (1, 1, 1), (1, 2, 2), (2, 1, 3), (2, 2, 4)")
tk.MustExec("insert into t2 values (1, 1, 1), (1, 2, 2), (2, 1, 3), (2, 2, 4)")
- isHashJoinV2Enabled := join.IsHashJoinV2Enabled()
join.SetEnableHashJoinV2(true)
defer func() {
- join.SetEnableHashJoinV2(isHashJoinV2Enabled)
+ join.SetEnableHashJoinV2(false)
}()
fpName := "github.com/pingcap/tidb/pkg/executor/join/buildHashTablePanic"
require.NoError(t, failpoint.Enable(fpName, "panic(\"buildHashTablePanic\")"))
@@ -612,10 +599,9 @@ func TestKillDuringProbe(t *testing.T) {
tk.MustExec("create table t1(c1 int, c2 int)")
tk.MustExec("insert into t values(1,1),(2,2)")
tk.MustExec("insert into t1 values(2,3),(4,4)")
- isHashJoinV2Enabled := join.IsHashJoinV2Enabled()
join.SetEnableHashJoinV2(true)
defer func() {
- join.SetEnableHashJoinV2(isHashJoinV2Enabled)
+ join.SetEnableHashJoinV2(false)
}()
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/executor/join/killedDuringProbe", "return(true)"))
defer func() {
@@ -646,10 +632,9 @@ func TestKillDuringBuild(t *testing.T) {
tk.MustExec("create table t1(c1 int, c2 int)")
tk.MustExec("insert into t values(1,1),(2,2)")
tk.MustExec("insert into t1 values(2,3),(4,4)")
- isHashJoinV2Enabled := join.IsHashJoinV2Enabled()
join.SetEnableHashJoinV2(true)
defer func() {
- join.SetEnableHashJoinV2(isHashJoinV2Enabled)
+ join.SetEnableHashJoinV2(false)
}()
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/executor/join/killedDuringBuild", "return(true)"))
defer func() {
@@ -685,3 +670,21 @@ func TestIssue54755(t *testing.T) {
// left join
tk.MustQuery("select max(SQ1_alias2.col_int_nokey) as SQ1_field1 from ( t1 as SQ1_alias2 left join t2 as SQ1_alias1 on ( SQ1_alias2.col_varchar_key = SQ1_alias1.col_varchar_nokey ))").Check(testkit.Rows("150"))
}
+
+func TestIssue55016(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t;")
+ tk.MustExec("create table t(a varchar(10), b char(10))")
+ tk.MustExec("insert into t values('aa','a')")
+ isHashJoinV2Enabled := join.IsHashJoinV2Enabled()
+ defer func() {
+ join.SetEnableHashJoinV2(isHashJoinV2Enabled)
+ }()
+ hashJoinV2Enable := []bool{true, false}
+ for _, enableHashJoinV2 := range hashJoinV2Enable {
+ join.SetEnableHashJoinV2(enableHashJoinV2)
+ tk.MustQuery("select count(*) from t t1 join t t2 on t1.a = t2.b and t2.a = t1.b").Check(testkit.Rows("0"))
+ }
+}
diff --git a/pkg/executor/test/jointest/join_test.go b/pkg/executor/test/jointest/join_test.go
index 2c9d873f6f69f..634e3c5e8fef8 100644
--- a/pkg/executor/test/jointest/join_test.go
+++ b/pkg/executor/test/jointest/join_test.go
@@ -136,10 +136,10 @@ func TestJoin2(t *testing.T) {
// The physical plans of the two sql are tested at physical_plan_test.go
tk.MustQuery("select /*+ INL_JOIN(t, t1) */ * from t join t1 on t.a=t1.a").Check(testkit.Rows("1 1 1 2", "1 1 1 3", "1 1 1 4", "3 3 3 4"))
tk.MustQuery("select /*+ INL_HASH_JOIN(t, t1) */ * from t join t1 on t.a=t1.a").Sort().Check(testkit.Rows("1 1 1 2", "1 1 1 3", "1 1 1 4", "3 3 3 4"))
- tk.MustQuery("select /*+ INL_MERGE_JOIN(t, t1) */ * from t join t1 on t.a=t1.a").Check(testkit.Rows("1 1 1 2", "1 1 1 3", "1 1 1 4", "3 3 3 4"))
+ tk.MustQuery("select /*+ INL_MERGE_JOIN(t, t1) */ * from t join t1 on t.a=t1.a").Check(testkit.Rows("1 1 1 4", "1 1 1 3", "1 1 1 2", "3 3 3 4"))
tk.MustQuery("select /*+ INL_JOIN(t) */ * from t1 join t on t.a=t1.a and t.a < t1.b").Check(testkit.Rows("1 2 1 1", "1 3 1 1", "1 4 1 1", "3 4 3 3"))
tk.MustQuery("select /*+ INL_HASH_JOIN(t) */ * from t1 join t on t.a=t1.a and t.a < t1.b").Sort().Check(testkit.Rows("1 2 1 1", "1 3 1 1", "1 4 1 1", "3 4 3 3"))
- tk.MustQuery("select /*+ INL_MERGE_JOIN(t) */ * from t1 join t on t.a=t1.a and t.a < t1.b").Check(testkit.Rows("1 2 1 1", "1 3 1 1", "1 4 1 1", "3 4 3 3"))
+ tk.MustQuery("select /*+ INL_MERGE_JOIN(t) */ * from t1 join t on t.a=t1.a and t.a < t1.b").Check(testkit.Rows("1 4 1 1", "1 3 1 1", "1 2 1 1", "3 4 3 3"))
// Test single index reader.
tk.MustQuery("select /*+ INL_JOIN(t, t1) */ t1.b from t1 join t on t.b=t1.b").Check(testkit.Rows("2", "3"))
tk.MustQuery("select /*+ INL_HASH_JOIN(t, t1) */ t1.b from t1 join t on t.b=t1.b").Sort().Check(testkit.Rows("2", "3"))
diff --git a/pkg/executor/test/seqtest/seq_executor_test.go b/pkg/executor/test/seqtest/seq_executor_test.go
index 14f68aaa0abd3..df85c0db9f254 100644
--- a/pkg/executor/test/seqtest/seq_executor_test.go
+++ b/pkg/executor/test/seqtest/seq_executor_test.go
@@ -1299,9 +1299,7 @@ func TestOOMPanicInHashJoinWhenFetchBuildRows(t *testing.T) {
tk.MustExec("insert into t values(1,1),(2,2)")
fpName := "github.com/pingcap/tidb/pkg/executor/join/errorFetchBuildSideRowsMockOOMPanic"
require.NoError(t, failpoint.Enable(fpName, `panic("ERROR 1105 (HY000): Out Of Memory Quota![conn=1]")`))
- isHashJoinV2Enabled := join.IsHashJoinV2Enabled()
defer func() {
- join.SetEnableHashJoinV2(isHashJoinV2Enabled)
require.NoError(t, failpoint.Disable(fpName))
}()
useHashJoinV2 := []bool{true, false}
diff --git a/pkg/executor/test/writetest/write_test.go b/pkg/executor/test/writetest/write_test.go
index 3ea9a6e5f7133..d4013c145afc4 100644
--- a/pkg/executor/test/writetest/write_test.go
+++ b/pkg/executor/test/writetest/write_test.go
@@ -373,11 +373,11 @@ func TestRebaseIfNeeded(t *testing.T) {
tbl, err := domain.InfoSchema().TableByName(context.Background(), model.NewCIStr("test"), model.NewCIStr("t"))
require.NoError(t, err)
require.Nil(t, sessiontxn.NewTxn(context.Background(), ctx))
+ txn, err := ctx.Txn(true)
+ require.NoError(t, err)
// AddRecord directly here will skip to rebase the auto ID in the insert statement,
// which could simulate another TiDB adds a large auto ID.
- _, err = tbl.AddRecord(ctx.GetTableCtx(), types.MakeDatums(30001, 2))
- require.NoError(t, err)
- txn, err := ctx.Txn(true)
+ _, err = tbl.AddRecord(ctx.GetTableCtx(), txn, types.MakeDatums(30001, 2))
require.NoError(t, err)
require.NoError(t, txn.Commit(context.Background()))
diff --git a/pkg/executor/update.go b/pkg/executor/update.go
index bda1be8422ea2..adb813f5911c6 100644
--- a/pkg/executor/update.go
+++ b/pkg/executor/update.go
@@ -72,6 +72,8 @@ type UpdateExec struct {
fkChecks map[int64][]*FKCheckExec
// fkCascades contains the foreign key cascade. the map is tableID -> []*FKCascadeExec
fkCascades map[int64][]*FKCascadeExec
+
+ IgnoreError bool
}
// prepare `handles`, `tableUpdatable`, `changed` to avoid re-computations.
@@ -172,7 +174,7 @@ func (e *UpdateExec) merge(row, newData []types.Datum, mergeGenerated bool) erro
return nil
}
-func (e *UpdateExec) exec(ctx context.Context, _ *expression.Schema, row, newData []types.Datum) error {
+func (e *UpdateExec) exec(ctx context.Context, _ *expression.Schema, row, newData []types.Datum, dupKeyCheck table.DupKeyCheckMode) error {
defer trace.StartRegion(ctx, "UpdateExec").End()
bAssignFlag := make([]bool, len(e.assignFlag))
for i, flag := range e.assignFlag {
@@ -205,7 +207,7 @@ func (e *UpdateExec) exec(ctx context.Context, _ *expression.Schema, row, newDat
// Update row
fkChecks := e.fkChecks[content.TblID]
fkCascades := e.fkCascades[content.TblID]
- changed, err1 := updateRecord(ctx, e.Ctx(), handle, oldData, newTableData, flags, tbl, false, e.memTracker, fkChecks, fkCascades)
+ changed, err1 := updateRecord(ctx, e.Ctx(), handle, oldData, newTableData, flags, tbl, false, e.memTracker, fkChecks, fkCascades, dupKeyCheck)
if err1 == nil {
_, exist := e.updatedRowKeys[content.Start].Get(handle)
memDelta := e.updatedRowKeys[content.Start].Set(handle, changed)
@@ -273,6 +275,13 @@ func (e *UpdateExec) updateRows(ctx context.Context) (int, error) {
}
memUsageOfChk := int64(0)
totalNumRows := 0
+
+ txn, err := e.Ctx().Txn(true)
+ if err != nil {
+ return 0, err
+ }
+
+ dupKeyCheck := optimizeDupKeyCheckForUpdate(txn, e.IgnoreError)
for {
e.memTracker.Consume(-memUsageOfChk)
err := exec.Next(ctx, e.Children(0), chk)
@@ -286,14 +295,12 @@ func (e *UpdateExec) updateRows(ctx context.Context) (int, error) {
memUsageOfChk = chk.MemoryUsage()
e.memTracker.Consume(memUsageOfChk)
if e.collectRuntimeStatsEnabled() {
- txn, err := e.Ctx().Txn(true)
- if err == nil && txn.GetSnapshot() != nil {
- txn.GetSnapshot().SetOption(kv.CollectRuntimeStats, e.stats.SnapshotRuntimeStats)
+ if snap := txn.GetSnapshot(); snap != nil {
+ snap.SetOption(kv.CollectRuntimeStats, e.stats.SnapshotRuntimeStats)
}
}
- txn, err := e.Ctx().Txn(true)
// pipelined dml may already flush in background, don't touch it to avoid race.
- if err == nil && !txn.IsPipelined() {
+ if !txn.IsPipelined() {
sc := e.Ctx().GetSessionVars().StmtCtx
txn.SetOption(kv.ResourceGroupTagger, sc.GetResourceGroupTagger())
if sc.KvExecCounter != nil {
@@ -329,7 +336,7 @@ func (e *UpdateExec) updateRows(ctx context.Context) (int, error) {
}
}
// write to table
- if err := e.exec(ctx, e.Children(0).Schema(), datumRow, newRow); err != nil {
+ if err := e.exec(ctx, e.Children(0).Schema(), datumRow, newRow, dupKeyCheck); err != nil {
return 0, err
}
}
@@ -339,7 +346,7 @@ func (e *UpdateExec) updateRows(ctx context.Context) (int, error) {
return totalNumRows, nil
}
-func (*UpdateExec) handleErr(colName model.CIStr, rowIdx int, err error) error {
+func (e *UpdateExec) handleErr(colName model.CIStr, col *table.Column, rowIdx int, err error) error {
if err == nil {
return nil
}
@@ -351,7 +358,10 @@ func (*UpdateExec) handleErr(colName model.CIStr, rowIdx int, err error) error {
if types.ErrOverflow.Equal(err) {
return types.ErrWarnDataOutOfRange.GenWithStackByArgs(colName.O, rowIdx+1)
}
-
+ if types.ErrTruncatedWrongVal.Equal(err) && col != nil && col.ColumnInfo != nil && col.ColumnInfo.GetType() == mysql.TypeTimestamp {
+ ec := e.Ctx().GetSessionVars().StmtCtx.ErrCtx()
+ return errors.AddStack(ec.HandleErrorWithAlias(kv.ErrKeyExists, err, err))
+ }
return err
}
@@ -364,7 +374,7 @@ func (e *UpdateExec) fastComposeNewRow(rowIdx int, oldRow []types.Datum, cols []
}
con := assign.Expr.(*expression.Constant)
val, err := con.Eval(e.Ctx().GetExprCtx().GetEvalCtx(), emptyRow)
- if err = e.handleErr(assign.ColName, rowIdx, err); err != nil {
+ if err = e.handleErr(assign.ColName, cols[assign.Col.Index], rowIdx, err); err != nil {
return nil, err
}
@@ -372,7 +382,7 @@ func (e *UpdateExec) fastComposeNewRow(rowIdx int, oldRow []types.Datum, cols []
// No need to cast `_tidb_rowid` column value.
if cols[assign.Col.Index] != nil {
val, err = table.CastValue(e.Ctx(), val, cols[assign.Col.Index].ColumnInfo, false, false)
- if err = e.handleErr(assign.ColName, rowIdx, err); err != nil {
+ if err = e.handleErr(assign.ColName, cols[assign.Col.Index], rowIdx, err); err != nil {
return nil, err
}
}
@@ -399,7 +409,7 @@ func (e *UpdateExec) composeNewRow(rowIdx int, oldRow []types.Datum, cols []*tab
// No need to cast `_tidb_rowid` column value.
if cols[assign.Col.Index] != nil {
val, err = table.CastValue(e.Ctx(), val, cols[assign.Col.Index].ColumnInfo, false, false)
- if err = e.handleErr(assign.ColName, rowIdx, err); err != nil {
+ if err = e.handleErr(assign.ColName, cols[assign.Col.Index], rowIdx, err); err != nil {
return nil, err
}
}
@@ -420,7 +430,7 @@ func (e *UpdateExec) composeGeneratedColumns(rowIdx int, newRowData []types.Datu
continue
}
val, err := assign.Expr.Eval(e.Ctx().GetExprCtx().GetEvalCtx(), e.evalBuffer.ToRow())
- if err = e.handleErr(assign.ColName, rowIdx, err); err != nil {
+ if err = e.handleErr(assign.ColName, cols[assign.Col.Index], rowIdx, err); err != nil {
return nil, err
}
@@ -428,7 +438,7 @@ func (e *UpdateExec) composeGeneratedColumns(rowIdx int, newRowData []types.Datu
// No need to cast `_tidb_rowid` column value.
if cols[assign.Col.Index] != nil {
val, err = table.CastValue(e.Ctx(), val, cols[assign.Col.Index].ColumnInfo, false, false)
- if err = e.handleErr(assign.ColName, rowIdx, err); err != nil {
+ if err = e.handleErr(assign.ColName, cols[assign.Col.Index], rowIdx, err); err != nil {
return nil, err
}
}
@@ -574,3 +584,40 @@ func (e *UpdateExec) GetFKCascades() []*FKCascadeExec {
func (e *UpdateExec) HasFKCascades() bool {
return len(e.fkCascades) > 0
}
+
+// optimizeDupKeyCheckForUpdate trys to optimize the DupKeyCheckMode for an update statement.
+// If the DupKeyCheckMode of the current statement can be optimized, it will return `DupKeyCheckLazy` to avoid the
+// redundant requests to TiKV, otherwise, `DupKeyCheckInPlace` will be returned.
+// The second argument `ignoreNeedsCheckInPlace` is true if `IGNORE` keyword is used in the update statement.
+func optimizeDupKeyCheckForUpdate(txn kv.Transaction, ignoreNeedsCheckInPlace bool) table.DupKeyCheckMode {
+ if txn.IsPipelined() {
+ // It means `@@tidb_dml_type='bulk'` which indicates to insert rows in "bulk" mode.
+ // At this time, `DupKeyCheckLazy` should be used to improve the performance.
+ // If "bulk" mode and IGNORE keyword are used together, "bulk" is prior, see:
+ // https://github.com/pingcap/tidb/issues/55187#issuecomment-2268356459
+ return table.DupKeyCheckLazy
+ }
+
+ if ignoreNeedsCheckInPlace {
+ // For `UPDATE IGNORE ...` and `INSERT IGNORE ... ON DUPLICATE KEY UPDATE ...` statements,
+ // `DupKeyCheckInPlace` should be used to make sure the executor can get the error
+ // immediately and ignore it then.
+ return table.DupKeyCheckInPlace
+ }
+
+ if txn.IsPessimistic() {
+ // We can just check duplicated key lazily without keys in storage for the below cases:
+ // - `txn.Pipelined()` is true.
+ // It means the user is using `@@tidb_dml_type="bulk"` to insert rows in bulk mode.
+ // DupKeyCheckLazy should be used to improve the performance.
+ // - The current transaction is pessimistic.
+ // The duplicate key check can be postponed to the lock stage.
+ // Please notice that for optimistic transaction, it always returns `DupKeyCheckInPlace` even if
+ // `tidb_constraint_check_in_place` is `OFF`.
+ // That is because `tidb_constraint_check_in_place` is only designed for insert cases, see comments in issue:
+ // https://github.com/pingcap/tidb/issues/54492#issuecomment-2229941881
+ return table.DupKeyCheckLazy
+ }
+
+ return table.DupKeyCheckInPlace
+}
diff --git a/pkg/executor/window.go b/pkg/executor/window.go
index 42c46cba9f49e..4b7c7b68a09ef 100644
--- a/pkg/executor/window.go
+++ b/pkg/executor/window.go
@@ -23,7 +23,7 @@ import (
"github.com/pingcap/tidb/pkg/executor/internal/vecgroupchecker"
"github.com/pingcap/tidb/pkg/expression"
"github.com/pingcap/tidb/pkg/parser/ast"
- "github.com/pingcap/tidb/pkg/planner/core"
+ "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/util/chunk"
)
@@ -237,8 +237,8 @@ func (p *aggWindowProcessor) resetPartialResult() {
type rowFrameWindowProcessor struct {
windowFuncs []aggfuncs.AggFunc
partialResults []aggfuncs.PartialResult
- start *core.FrameBound
- end *core.FrameBound
+ start *logicalop.FrameBound
+ end *logicalop.FrameBound
curRowIdx uint64
}
@@ -379,8 +379,8 @@ func (p *rowFrameWindowProcessor) resetPartialResult() {
type rangeFrameWindowProcessor struct {
windowFuncs []aggfuncs.AggFunc
partialResults []aggfuncs.PartialResult
- start *core.FrameBound
- end *core.FrameBound
+ start *logicalop.FrameBound
+ end *logicalop.FrameBound
curRowIdx uint64
lastStartOffset uint64
lastEndOffset uint64
diff --git a/pkg/executor/write.go b/pkg/executor/write.go
index 98d8518bd4215..56538d29a7c35 100644
--- a/pkg/executor/write.go
+++ b/pkg/executor/write.go
@@ -56,12 +56,13 @@ var (
func updateRecord(
ctx context.Context, sctx sessionctx.Context, h kv.Handle, oldData, newData []types.Datum, modified []bool,
t table.Table,
- onDup bool, _ *memory.Tracker, fkChecks []*FKCheckExec, fkCascades []*FKCascadeExec,
+ onDup bool, _ *memory.Tracker, fkChecks []*FKCheckExec, fkCascades []*FKCascadeExec, dupKeyMode table.DupKeyCheckMode,
) (bool, error) {
r, ctx := tracing.StartRegionEx(ctx, "executor.updateRecord")
defer r.End()
- sc := sctx.GetSessionVars().StmtCtx
+ sessVars := sctx.GetSessionVars()
+ sc := sessVars.StmtCtx
changed, handleChanged := false, false
// onUpdateSpecified is for "UPDATE SET ts_field = old_value", the
// timestamp field is explicitly set, but not changed in fact.
@@ -82,7 +83,7 @@ func updateRecord(
// Handle exchange partition
tbl := t.Meta()
if tbl.ExchangePartitionInfo != nil && tbl.GetPartitionInfo() == nil {
- if err := checkRowForExchangePartition(sctx.GetTableCtx(), newData, tbl); err != nil {
+ if err := checkRowForExchangePartition(sctx, newData, tbl); err != nil {
return false, err
}
}
@@ -130,11 +131,11 @@ func updateRecord(
// If no changes, nothing to do, return directly.
if !changed {
// See https://dev.mysql.com/doc/refman/5.7/en/mysql-real-connect.html CLIENT_FOUND_ROWS
- if sctx.GetSessionVars().ClientCapability&mysql.ClientFoundRows > 0 {
+ if sessVars.ClientCapability&mysql.ClientFoundRows > 0 {
sc.AddAffectedRows(1)
}
keySet := lockRowKey
- if sctx.GetSessionVars().LockUnchangedKeys {
+ if sessVars.LockUnchangedKeys {
keySet |= lockUniqueKeys
}
_, err := addUnchangedKeysForLockByRow(sctx, t, h, oldData, keySet)
@@ -161,25 +162,26 @@ func updateRecord(
}
}
+ pessimisticLazyCheck := getPessimisticLazyCheckMode(sessVars)
+ txn, err := sctx.Txn(true)
+ if err != nil {
+ return false, err
+ }
// If handle changed, remove the old then add the new record, otherwise update the record.
if handleChanged {
// For `UPDATE IGNORE`/`INSERT IGNORE ON DUPLICATE KEY UPDATE`
// we use the staging buffer so that we don't need to precheck the existence of handle or unique keys by sending
// extra kv requests, and the remove action will not take effect if there are conflicts.
if updated, err := func() (bool, error) {
- txn, err := sctx.Txn(true)
- if err != nil {
- return false, err
- }
memBuffer := txn.GetMemBuffer()
sh := memBuffer.Staging()
defer memBuffer.Cleanup(sh)
- if err = t.RemoveRecord(sctx.GetTableCtx(), h, oldData); err != nil {
+ if err = t.RemoveRecord(sctx.GetTableCtx(), txn, h, oldData); err != nil {
return false, err
}
- _, err = t.AddRecord(sctx.GetTableCtx(), newData, table.IsUpdate, table.WithCtx(ctx))
+ _, err = t.AddRecord(sctx.GetTableCtx(), txn, newData, table.IsUpdate, table.WithCtx(ctx), dupKeyMode, pessimisticLazyCheck)
if err != nil {
return false, err
}
@@ -187,21 +189,32 @@ func updateRecord(
return true, nil
}(); err != nil {
if terr, ok := errors.Cause(err).(*terror.Error); ok && (terr.Code() == errno.ErrNoPartitionForGivenValue || terr.Code() == errno.ErrRowDoesNotMatchGivenPartitionSet) {
- ec := sctx.GetSessionVars().StmtCtx.ErrCtx()
+ ec := sc.ErrCtx()
return false, ec.HandleError(err)
}
return updated, err
}
} else {
+ var opts []table.UpdateRecordOption
+ if sessVars.InTxn() || sc.InHandleForeignKeyTrigger || sc.ForeignKeyTriggerCtx.HasFKCascades {
+ // If txn is auto commit and index is untouched, no need to write index value.
+ // If InHandleForeignKeyTrigger or ForeignKeyTriggerCtx.HasFKCascades is true indicate we may have
+ // foreign key cascade need to handle later, then we still need to write index value,
+ // otherwise, the later foreign cascade executor may see data-index inconsistency in txn-mem-buffer.
+ opts = []table.UpdateRecordOption{table.WithCtx(ctx), dupKeyMode, pessimisticLazyCheck}
+ } else {
+ opts = []table.UpdateRecordOption{table.WithCtx(ctx), dupKeyMode, pessimisticLazyCheck, table.SkipWriteUntouchedIndices}
+ }
+
// Update record to new value and update index.
- if err := t.UpdateRecord(ctx, sctx.GetTableCtx(), h, oldData, newData, modified); err != nil {
+ if err := t.UpdateRecord(sctx.GetTableCtx(), txn, h, oldData, newData, modified, opts...); err != nil {
if terr, ok := errors.Cause(err).(*terror.Error); ok && (terr.Code() == errno.ErrNoPartitionForGivenValue || terr.Code() == errno.ErrRowDoesNotMatchGivenPartitionSet) {
- ec := sctx.GetSessionVars().StmtCtx.ErrCtx()
+ ec := sc.ErrCtx()
return false, ec.HandleError(err)
}
return false, err
}
- if sctx.GetSessionVars().LockUnchangedKeys {
+ if sessVars.LockUnchangedKeys {
// Lock unique keys when handle unchanged
if _, err := addUnchangedKeysForLockByRow(sctx, t, h, oldData, lockUniqueKeys); err != nil {
return false, err
@@ -319,9 +332,9 @@ func resetErrDataTooLong(colName string, rowIdx int, _ error) error {
// checkRowForExchangePartition is only used for ExchangePartition by non-partitionTable during write only state.
// It check if rowData inserted or updated violate partition definition or checkConstraints of partitionTable.
-func checkRowForExchangePartition(sctx table.MutateContext, row []types.Datum, tbl *model.TableInfo) error {
+func checkRowForExchangePartition(sctx sessionctx.Context, row []types.Datum, tbl *model.TableInfo) error {
is := sctx.GetDomainInfoSchema().(infoschema.InfoSchema)
- pt, tableFound := is.TableByID(tbl.ExchangePartitionInfo.ExchangePartitionTableID)
+ pt, tableFound := is.TableByID(context.Background(), tbl.ExchangePartitionInfo.ExchangePartitionTableID)
if !tableFound {
return errors.Errorf("exchange partition process table by id failed")
}
diff --git a/pkg/expression/builtin.go b/pkg/expression/builtin.go
index e01405efda29a..e5f640da0c3c6 100644
--- a/pkg/expression/builtin.go
+++ b/pkg/expression/builtin.go
@@ -50,7 +50,7 @@ import (
type baseBuiltinFunc struct {
bufAllocator columnBufferAllocator
args []Expression
- tp *types.FieldType
+ tp *types.FieldType `plan-cache-clone:"shallow"`
pbCode tipb.ScalarFuncSig
ctor collate.Collator
@@ -427,7 +427,9 @@ func (b *baseBuiltinFunc) cloneFrom(from *baseBuiltinFunc) {
b.pbCode = from.pbCode
b.bufAllocator = newLocalColumnPool()
b.childrenVectorizedOnce = new(sync.Once)
- b.ctor = from.ctor
+ if from.ctor != nil {
+ b.ctor = from.ctor.Clone()
+ }
}
func (*baseBuiltinFunc) Clone() builtinFunc {
diff --git a/pkg/expression/builtin_other.go b/pkg/expression/builtin_other.go
index e0375bda60a1e..a7b43062640d2 100644
--- a/pkg/expression/builtin_other.go
+++ b/pkg/expression/builtin_other.go
@@ -1696,7 +1696,6 @@ func (c *getParamFunctionClass) getFunction(ctx BuildContext, args []Expression)
type builtinGetParamStringSig struct {
baseBuiltinFunc
- contextopt.SessionVarsPropReader
}
func (b *builtinGetParamStringSig) Clone() builtinFunc {
@@ -1705,20 +1704,16 @@ func (b *builtinGetParamStringSig) Clone() builtinFunc {
return newSig
}
-func (b *builtinGetParamStringSig) RequiredOptionalEvalProps() OptionalEvalPropKeySet {
- return b.SessionVarsPropReader.RequiredOptionalEvalProps()
-}
-
func (b *builtinGetParamStringSig) evalString(ctx EvalContext, row chunk.Row) (string, bool, error) {
- sessionVars, err := b.GetSessionVars(ctx)
- if err != nil {
- return "", true, err
- }
idx, isNull, err := b.args[0].EvalInt(ctx, row)
if isNull || err != nil {
return "", isNull, err
}
- v := sessionVars.PlanCacheParams.GetParamValue(int(idx))
+
+ v, err := ctx.GetParamValue(int(idx))
+ if err != nil {
+ return "", true, err
+ }
str, err := v.ToString()
if err != nil {
diff --git a/pkg/expression/builtin_other_test.go b/pkg/expression/builtin_other_test.go
index cd99de923ff39..4b3528dc01351 100644
--- a/pkg/expression/builtin_other_test.go
+++ b/pkg/expression/builtin_other_test.go
@@ -19,6 +19,7 @@ import (
"testing"
"time"
+ exprctx "github.com/pingcap/tidb/pkg/expression/context"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/types"
@@ -342,3 +343,30 @@ func TestInFunc(t *testing.T) {
require.NoError(t, err)
require.Equal(t, int64(1), chk2.Column(0).GetInt64(0))
}
+
+func TestGetParam(t *testing.T) {
+ ctx := createContext(t)
+ params := []types.Datum{
+ types.NewIntDatum(123),
+ types.NewStringDatum("abc"),
+ }
+ ctx.GetSessionVars().PlanCacheParams.Append(params...)
+ fc := funcs[ast.GetParam]
+
+ for i := range params {
+ fn, err := fc.getFunction(ctx, datumsToConstants(types.MakeDatums(i)))
+ require.NoError(t, err)
+ d, err := evalBuiltinFunc(fn, ctx, chunk.Row{})
+ require.NoError(t, err)
+ str, err := params[i].ToString()
+ require.NoError(t, err)
+ require.Equal(t, d.Kind(), types.KindString)
+ require.Equal(t, str, d.GetString())
+ }
+
+ fn, err := fc.getFunction(ctx, datumsToConstants(types.MakeDatums(len(params)+1)))
+ require.NoError(t, err)
+ d, err := evalBuiltinFunc(fn, ctx, chunk.Row{})
+ require.Equal(t, exprctx.ErrParamIndexExceedParamCounts, err)
+ require.True(t, d.IsNull())
+}
diff --git a/pkg/expression/builtin_other_vec.go b/pkg/expression/builtin_other_vec.go
index cc0f30b0d6085..dbc4e3ef89e85 100644
--- a/pkg/expression/builtin_other_vec.go
+++ b/pkg/expression/builtin_other_vec.go
@@ -126,10 +126,6 @@ func (b *builtinGetParamStringSig) vectorized() bool {
}
func (b *builtinGetParamStringSig) vecEvalString(ctx EvalContext, input *chunk.Chunk, result *chunk.Column) error {
- sessionVars, err := b.GetSessionVars(ctx)
- if err != nil {
- return err
- }
n := input.NumRows()
idx, err := b.bufAllocator.get()
if err != nil {
@@ -147,7 +143,11 @@ func (b *builtinGetParamStringSig) vecEvalString(ctx EvalContext, input *chunk.C
continue
}
idxI := idxIs[i]
- v := sessionVars.PlanCacheParams.GetParamValue(int(idxI))
+ v, err := ctx.GetParamValue(int(idxI))
+ if err != nil {
+ return err
+ }
+
str, err := v.ToString()
if err != nil {
result.AppendNull()
diff --git a/pkg/expression/builtin_other_vec_test.go b/pkg/expression/builtin_other_vec_test.go
index 6e3d81c2eecfe..9e817b0e028b6 100644
--- a/pkg/expression/builtin_other_vec_test.go
+++ b/pkg/expression/builtin_other_vec_test.go
@@ -19,6 +19,7 @@ import (
"math/rand"
"testing"
+ exprctx "github.com/pingcap/tidb/pkg/expression/context"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/chunk"
@@ -88,3 +89,37 @@ func TestInDecimal(t *testing.T) {
require.Equal(t, int64(1), result.GetInt64(0))
}
}
+
+func TestGetParamVec(t *testing.T) {
+ ctx := createContext(t)
+ params := []types.Datum{
+ types.NewIntDatum(123),
+ types.NewStringDatum("abc"),
+ }
+ ctx.GetSessionVars().PlanCacheParams.Append(params...)
+ ft := eType2FieldType(types.ETInt)
+ col := &Column{RetType: ft, Index: 0}
+ fn, err := funcs[ast.GetParam].getFunction(ctx, []Expression{col})
+ require.NoError(t, err)
+
+ input := chunk.NewChunkWithCapacity([]*types.FieldType{ft}, 3)
+ for i := range params {
+ input.Column(0).AppendInt64(int64(i))
+ }
+ result := chunk.NewColumn(ft, 3)
+ require.NoError(t, vecEvalType(ctx, fn, types.ETString, input, result))
+ require.Equal(t, len(params), result.Rows())
+ for i := 0; i < result.Rows(); i++ {
+ require.False(t, result.IsNull(i))
+ val := result.GetString(i)
+ str, err := params[i].ToString()
+ require.NoError(t, err)
+ require.Equal(t, str, val)
+ }
+
+ input = chunk.NewChunkWithCapacity([]*types.FieldType{ft}, 3)
+ input.Column(0).AppendInt64(1)
+ input.Column(0).AppendInt64(2)
+ input.Column(0).AppendInt64(int64(len(params)))
+ require.Equal(t, exprctx.ErrParamIndexExceedParamCounts, vecEvalType(ctx, fn, types.ETString, input, result))
+}
diff --git a/pkg/expression/builtin_time.go b/pkg/expression/builtin_time.go
index 335bf0852464c..e2c58fd32699c 100644
--- a/pkg/expression/builtin_time.go
+++ b/pkg/expression/builtin_time.go
@@ -6231,13 +6231,11 @@ func addUnitToTime(unit string, t time.Time, v float64) (time.Time, bool, error)
if !validAddMonth(v, t.Year(), int(t.Month())) {
return tb, true, nil
}
- tb = t.AddDate(0, int(v), 0)
- // For corner case: timestampadd(month,1,date '2024-01-31') = "2024-02-29", timestampadd(month,1,date '2024-01-30') = "2024-02-29"
- // `tb.Month()` refers to the actual result, `t.Month()+v` refers to the expect result.
- // Actual result may be greater than expect result, we need to judge and modify it.
- for int(tb.Month())%12 != (int(t.Month())+int(v))%12 {
- tb = tb.AddDate(0, 0, -1)
+ var err error
+ tb, err = types.AddDate(0, int64(v), 0, t)
+ if err != nil {
+ return tb, false, err
}
case "QUARTER":
if !validAddMonth(v*3, t.Year(), int(t.Month())) {
diff --git a/pkg/expression/builtin_time_test.go b/pkg/expression/builtin_time_test.go
index a25532e7737e2..60505be7f916f 100644
--- a/pkg/expression/builtin_time_test.go
+++ b/pkg/expression/builtin_time_test.go
@@ -2581,7 +2581,7 @@ func TestTimestampAdd(t *testing.T) {
{"MINUTE", 1.5, "1995-05-01 00:00:00.000000", "1995-05-01 00:02:00"},
{"MICROSECOND", -100, "1995-05-01 00:00:00.0001", "1995-05-01 00:00:00"},
- // issue41052
+ // issue 41052
{"MONTH", 1, "2024-01-31", "2024-02-29 00:00:00"},
{"MONTH", 1, "2024-01-30", "2024-02-29 00:00:00"},
{"MONTH", 1, "2024-01-29", "2024-02-29 00:00:00"},
@@ -2592,6 +2592,46 @@ func TestTimestampAdd(t *testing.T) {
{"MONTH", 10, "2024-10-31", "2025-08-31 00:00:00"},
{"MONTH", 1, "2024-11-30", "2024-12-30 00:00:00"},
{"MONTH", 13, "2024-11-30", "2025-12-30 00:00:00"},
+
+ // issue 54908
+ {"MONTH", 0, "2024-09-01", "2024-09-01 00:00:00"},
+ {"MONTH", -10, "2024-09-01", "2023-11-01 00:00:00"},
+ {"MONTH", -2, "2024-04-28", "2024-02-28 00:00:00"},
+ {"MONTH", -2, "2024-04-29", "2024-02-29 00:00:00"},
+ {"MONTH", -2, "2024-04-30", "2024-02-29 00:00:00"},
+ {"MONTH", -1, "2024-03-28", "2024-02-28 00:00:00"},
+ {"MONTH", -1, "2024-03-29", "2024-02-29 00:00:00"},
+ {"MONTH", -1, "2024-03-30", "2024-02-29 00:00:00"},
+ {"MONTH", -1, "2024-03-31", "2024-02-29 00:00:00"},
+ {"MONTH", -1, "2024-03-25", "2024-02-25 00:00:00"},
+ {"MONTH", -12, "2024-03-31", "2023-03-31 00:00:00"},
+ {"MONTH", -13, "2024-03-31", "2023-02-28 00:00:00"},
+ {"MONTH", -14, "2024-03-31", "2023-01-31 00:00:00"},
+ {"MONTH", -24, "2024-03-31", "2022-03-31 00:00:00"},
+ {"MONTH", -25, "2024-03-31", "2022-02-28 00:00:00"},
+ {"MONTH", -26, "2024-03-31", "2022-01-31 00:00:00"},
+ {"MONTH", -1, "2024-02-25", "2024-01-25 00:00:00"},
+ {"MONTH", -11, "2025-02-28", "2024-03-28 00:00:00"},
+ {"MONTH", -12, "2025-02-28", "2024-02-28 00:00:00"},
+ {"MONTH", -13, "2025-02-28", "2024-01-28 00:00:00"},
+ {"MONTH", -11, "2024-02-29", "2023-03-29 00:00:00"},
+ {"MONTH", -12, "2024-02-29", "2023-02-28 00:00:00"},
+ {"MONTH", -13, "2024-02-29", "2023-01-29 00:00:00"},
+ {"MONTH", -11, "2023-02-28", "2022-03-28 00:00:00"},
+ {"MONTH", -12, "2023-02-28", "2022-02-28 00:00:00"},
+ {"MONTH", -13, "2023-02-28", "2022-01-28 00:00:00"},
+ {"MONTH", -2, "2023-02-28", "2022-12-28 00:00:00"},
+ {"MONTH", -14, "2023-02-28", "2021-12-28 00:00:00"},
+ {"MONTH", -3, "2023-03-20", "2022-12-20 00:00:00"},
+ {"MONTH", -3, "2023-03-31", "2022-12-31 00:00:00"},
+ {"MONTH", -15, "2023-03-20", "2021-12-20 00:00:00"},
+ {"MONTH", -15, "2023-03-31", "2021-12-31 00:00:00"},
+ {"MONTH", 12, "2020-02-29", "2021-02-28 00:00:00"},
+ {"MONTH", -12, "2020-02-29", "2019-02-28 00:00:00"},
+ {"MONTH", 10000*365 + 1, "2024-10-29", ""},
+ {"MONTH", -10000*365 - 1, "2024-10-29", ""},
+ {"MONTH", 3, "9999-10-29", ""},
+ {"MONTH", -3, "0001-01-29", ""},
}
fc := funcs[ast.TimestampAdd]
diff --git a/pkg/expression/column.go b/pkg/expression/column.go
index 1a21e2afe2797..47ec7f590dab4 100644
--- a/pkg/expression/column.go
+++ b/pkg/expression/column.go
@@ -243,7 +243,7 @@ func (col *CorrelatedColumn) RemapColumn(m map[int64]*Column) (Expression, error
// Column represents a column.
type Column struct {
- RetType *types.FieldType
+ RetType *types.FieldType `plan-cache-clone:"shallow"`
// ID is used to specify whether this column is ExtraHandleColumn or to access histogram.
// We'll try to remove it in the future.
ID int64
@@ -544,6 +544,10 @@ func (col *Column) EvalVectorFloat32(ctx EvalContext, row chunk.Row) (types.Vect
// Clone implements Expression interface.
func (col *Column) Clone() Expression {
newCol := *col
+ if col.hashcode != nil {
+ newCol.hashcode = make([]byte, len(col.hashcode))
+ copy(newCol.hashcode, col.hashcode)
+ }
return &newCol
}
diff --git a/pkg/expression/constant.go b/pkg/expression/constant.go
index 18bdf185a097b..210b55186eff8 100644
--- a/pkg/expression/constant.go
+++ b/pkg/expression/constant.go
@@ -117,7 +117,7 @@ func NewNullWithFieldType(fieldType *types.FieldType) *Constant {
// Constant stands for a constant value.
type Constant struct {
Value types.Datum
- RetType *types.FieldType
+ RetType *types.FieldType `plan-cache-clone:"shallow"`
// DeferredExpr holds deferred function in PlanCache cached plan.
// it's only used to represent non-deterministic functions(see expression.DeferredFunctions)
// in PlanCache cached plan, so let them can be evaluated until cached item be used.
@@ -163,6 +163,16 @@ func (c *Constant) StringWithCtx(ctx ParamValues, redact string) string {
// Clone implements Expression interface.
func (c *Constant) Clone() Expression {
con := *c
+ if c.ParamMarker != nil {
+ con.ParamMarker = &ParamMarker{order: c.ParamMarker.order}
+ }
+ if c.DeferredExpr != nil {
+ con.DeferredExpr = c.DeferredExpr.Clone()
+ }
+ if c.hashcode != nil {
+ con.hashcode = make([]byte, len(c.hashcode))
+ copy(con.hashcode, c.hashcode)
+ }
return &con
}
diff --git a/pkg/expression/constant_propagation.go b/pkg/expression/constant_propagation.go
index 38c4cf3790a63..2db05ec13ad15 100644
--- a/pkg/expression/constant_propagation.go
+++ b/pkg/expression/constant_propagation.go
@@ -32,10 +32,10 @@ var MaxPropagateColsCnt = 100
// nolint:structcheck
type basePropConstSolver struct {
- colMapper map[int64]int // colMapper maps column to its index
- eqList []*Constant // if eqList[i] != nil, it means col_i = eqList[i]
- unionSet *disjointset.IntSet // unionSet stores the relations like col_i = col_j
- columns []*Column // columns stores all columns appearing in the conditions
+ colMapper map[int64]int // colMapper maps column to its index
+ eqList []*Constant // if eqList[i] != nil, it means col_i = eqList[i]
+ unionSet *disjointset.SimpleIntSet // unionSet stores the relations like col_i = col_j
+ columns []*Column // columns stores all columns appearing in the conditions
ctx exprctx.ExprContext
}
@@ -68,6 +68,29 @@ func (s *basePropConstSolver) tryToUpdateEQList(col *Column, con *Constant) (boo
return true, false
}
+// ValidCompareConstantPredicate checks if the predicate is an expression like [column '>'|'>='|'<'|'<='|'=' constant].
+// return param1: return true, if the predicate is a compare constant predicate.
+// return param2: return the column side of predicate.
+func ValidCompareConstantPredicate(ctx EvalContext, candidatePredicate Expression) bool {
+ scalarFunction, ok := candidatePredicate.(*ScalarFunction)
+ if !ok {
+ return false
+ }
+ if scalarFunction.FuncName.L != ast.GT && scalarFunction.FuncName.L != ast.GE &&
+ scalarFunction.FuncName.L != ast.LT && scalarFunction.FuncName.L != ast.LE &&
+ scalarFunction.FuncName.L != ast.EQ {
+ return false
+ }
+ column, _ := ValidCompareConstantPredicateHelper(ctx, scalarFunction, true)
+ if column == nil {
+ column, _ = ValidCompareConstantPredicateHelper(ctx, scalarFunction, false)
+ }
+ if column == nil {
+ return false
+ }
+ return true
+}
+
// ValidCompareConstantPredicateHelper checks if the predicate is a compare constant predicate, like "Column xxx Constant"
func ValidCompareConstantPredicateHelper(ctx EvalContext, eq *ScalarFunction, colIsLeft bool) (*Column, *Constant) {
var col *Column
diff --git a/pkg/expression/context/context.go b/pkg/expression/context/context.go
index 8079d4205c40a..7986807118270 100644
--- a/pkg/expression/context/context.go
+++ b/pkg/expression/context/context.go
@@ -231,6 +231,6 @@ func AssertLocationWithSessionVars(ctxLoc *time.Location, vars *variable.Session
stmtLocStr := vars.StmtCtx.TimeZone().String()
intest.Assert(ctxLocStr == varsLocStr && ctxLocStr == stmtLocStr,
"location mismatch, ctxLoc: %s, varsLoc: %s, stmtLoc: %s",
- ctxLoc.String(), ctxLocStr, stmtLocStr,
+ ctxLocStr, varsLocStr, stmtLocStr,
)
}
diff --git a/pkg/expression/context/param.go b/pkg/expression/context/param.go
index c60caae5d9141..b26a7c4172418 100644
--- a/pkg/expression/context/param.go
+++ b/pkg/expression/context/param.go
@@ -34,6 +34,6 @@ var EmptyParamValues ParamValues = &emptyParamValues{}
type emptyParamValues struct{}
// GetParamValue always returns the `ErrParamIndexExceedParamCounts` for any index
-func (e *emptyParamValues) GetParamValue(idx int) (types.Datum, error) {
+func (e *emptyParamValues) GetParamValue(_ int) (types.Datum, error) {
return types.Datum{}, ErrParamIndexExceedParamCounts
}
diff --git a/pkg/expression/contextstatic/exprctx.go b/pkg/expression/contextstatic/exprctx.go
index 21a6bac5d1891..14f8b8854b390 100644
--- a/pkg/expression/contextstatic/exprctx.go
+++ b/pkg/expression/contextstatic/exprctx.go
@@ -205,6 +205,11 @@ func (ctx *StaticExprContext) GetEvalCtx() exprctx.EvalContext {
return ctx.evalCtx
}
+// GetStaticEvalCtx returns the inner `StaticEvalContext`.
+func (ctx *StaticExprContext) GetStaticEvalCtx() *StaticEvalContext {
+ return ctx.evalCtx
+}
+
// GetCharsetInfo implements the `ExprContext.GetCharsetInfo`.
func (ctx *StaticExprContext) GetCharsetInfo() (string, string) {
return ctx.charset, ctx.collation
diff --git a/pkg/expression/evaluator.go b/pkg/expression/evaluator.go
index f08b1c54ae4c3..62bcb634e7913 100644
--- a/pkg/expression/evaluator.go
+++ b/pkg/expression/evaluator.go
@@ -15,12 +15,18 @@
package expression
import (
+ "sync/atomic"
+
"github.com/pingcap/tidb/pkg/expression/context"
"github.com/pingcap/tidb/pkg/util/chunk"
+ "github.com/pingcap/tidb/pkg/util/disjointset"
+ "github.com/pingcap/tidb/pkg/util/intest"
)
type columnEvaluator struct {
inputIdxToOutputIdxes map[int][]int
+ // mergedInputIdxToOutputIdxes is only determined in runtime when saw the input chunk.
+ mergedInputIdxToOutputIdxes atomic.Pointer[map[int][]int]
}
// run evaluates "Column" expressions.
@@ -28,7 +34,11 @@ type columnEvaluator struct {
//
// since it will change the content of the input Chunk.
func (e *columnEvaluator) run(ctx EvalContext, input, output *chunk.Chunk) error {
- for inputIdx, outputIdxes := range e.inputIdxToOutputIdxes {
+ // mergedInputIdxToOutputIdxes only can be determined in runtime when we saw the input chunk structure.
+ if e.mergedInputIdxToOutputIdxes.Load() == nil {
+ e.mergeInputIdxToOutputIdxes(input, e.inputIdxToOutputIdxes)
+ }
+ for inputIdx, outputIdxes := range *e.mergedInputIdxToOutputIdxes.Load() {
if err := output.SwapColumn(outputIdxes[0], input, inputIdx); err != nil {
return err
}
@@ -39,6 +49,93 @@ func (e *columnEvaluator) run(ctx EvalContext, input, output *chunk.Chunk) error
return nil
}
+// mergeInputIdxToOutputIdxes merges separate inputIdxToOutputIdxes entries when column references
+// are detected within the input chunk. This process ensures consistent handling of columns derived
+// from the same original source.
+//
+// Consider the following scenario:
+//
+// Initial scan operation produces a column 'a':
+//
+// scan: a (addr: ???)
+//
+// This column 'a' is used in the first projection (proj1) to create two columns a1 and a2, both referencing 'a':
+//
+// proj1
+// / \
+// / \
+// / \
+// a1 (addr: 0xe) a2 (addr: 0xe)
+// / \
+// / \
+// / \
+// proj2 proj2
+// / \ / \
+// / \ / \
+// a3 a4 a5 a6
+//
+// (addr: 0xe) (addr: 0xe) (addr: 0xe) (addr: 0xe)
+//
+// Here, a1 and a2 share the same address (0xe), indicating they reference the same data from the original 'a'.
+//
+// When moving to the second projection (proj2), the system tries to project these columns further:
+// - The first set (left side) consists of a3 and a4, derived from a1, both retaining the address (0xe).
+// - The second set (right side) consists of a5 and a6, derived from a2, also starting with address (0xe).
+//
+// When proj1 is complete, the output chunk contains two columns [a1, a2], both derived from the single column 'a' from the scan.
+// Since both a1 and a2 are column references with the same address (0xe), they are treated as referencing the same data.
+//
+// In proj2, two separate items are created:
+// - <0, [0,1]>: This means the 0th input column (a1) is projected twice, into the 0th and 1st columns of the output chunk.
+// - <1, [2,3]>: This means the 1st input column (a2) is projected twice, into the 2nd and 3rd columns of the output chunk.
+//
+// Due to the column swapping logic in each projection, after applying the <0, [0,1]> projection,
+// the addresses for a1 and a2 may become swapped or invalid:
+//
+// proj1: a1 (addr: invalid) a2 (addr: invalid)
+//
+// This can lead to issues in proj2, where further operations on these columns may be unsafe:
+//
+// proj2: a3 (addr: 0xe) a4 (addr: 0xe) a5 (addr: ???) a6 (addr: ???)
+//
+// Therefore, it's crucial to identify and merge the original column references early, ensuring
+// the final inputIdxToOutputIdxes mapping accurately reflects the shared origins of the data.
+// For instance, <0, [0,1,2,3]> indicates that the 0th input column (original 'a') is referenced
+// by all four output columns in the final output.
+//
+// mergeInputIdxToOutputIdxes merges inputIdxToOutputIdxes based on detected column references.
+// This ensures that columns with the same reference are correctly handled in the output chunk.
+func (e *columnEvaluator) mergeInputIdxToOutputIdxes(input *chunk.Chunk, inputIdxToOutputIdxes map[int][]int) {
+ originalDJSet := disjointset.NewSet[int](4)
+ flag := make([]bool, input.NumCols())
+ // Detect self column-references inside the input chunk by comparing column addresses
+ for i := 0; i < input.NumCols(); i++ {
+ if flag[i] {
+ continue
+ }
+ for j := i + 1; j < input.NumCols(); j++ {
+ if input.Column(i) == input.Column(j) {
+ flag[j] = true
+ originalDJSet.Union(i, j)
+ }
+ }
+ }
+ // Merge inputIdxToOutputIdxes based on the detected column references.
+ newInputIdxToOutputIdxes := make(map[int][]int, len(inputIdxToOutputIdxes))
+ for inputIdx := range inputIdxToOutputIdxes {
+ // Root idx is internal offset, not the right column index.
+ originalRootIdx := originalDJSet.FindRoot(inputIdx)
+ originalVal, ok := originalDJSet.FindVal(originalRootIdx)
+ intest.Assert(ok)
+ mergedOutputIdxes := newInputIdxToOutputIdxes[originalVal]
+ mergedOutputIdxes = append(mergedOutputIdxes, inputIdxToOutputIdxes[inputIdx]...)
+ newInputIdxToOutputIdxes[originalVal] = mergedOutputIdxes
+ }
+ // Update the merged inputIdxToOutputIdxes automatically.
+ // Once failed, it means other worker has done this job at meantime.
+ e.mergedInputIdxToOutputIdxes.CompareAndSwap(nil, &newInputIdxToOutputIdxes)
+}
+
type defaultEvaluator struct {
outputIdxes []int
exprs []Expression
@@ -79,18 +176,19 @@ func (e *defaultEvaluator) run(ctx EvalContext, vecEnabled bool, input, output *
func (e *defaultEvaluator) RequiredOptionalEvalProps() context.OptionalEvalPropKeySet {
props := context.OptionalEvalPropKeySet(0)
for _, expr := range e.exprs {
- props = props | getOptionalEvalPropsForExpr(expr)
+ props = props | GetOptionalEvalPropsForExpr(expr)
}
return props
}
-func getOptionalEvalPropsForExpr(expr Expression) context.OptionalEvalPropKeySet {
+// GetOptionalEvalPropsForExpr gets all optional evaluation properties that this expression requires.
+func GetOptionalEvalPropsForExpr(expr Expression) context.OptionalEvalPropKeySet {
switch e := expr.(type) {
case *ScalarFunction:
props := e.Function.RequiredOptionalEvalProps()
for _, arg := range e.GetArgs() {
- props = props | getOptionalEvalPropsForExpr(arg)
+ props = props | GetOptionalEvalPropsForExpr(arg)
}
return props
@@ -157,3 +255,12 @@ func (e *EvaluatorSuite) Run(ctx EvalContext, vecEnabled bool, input, output *ch
}
return nil
}
+
+// RequiredOptionalEvalProps exposes all optional evaluation properties that this evaluator requires.
+func (e *EvaluatorSuite) RequiredOptionalEvalProps() context.OptionalEvalPropKeySet {
+ if e.defaultEvaluator != nil {
+ return e.defaultEvaluator.RequiredOptionalEvalProps()
+ }
+
+ return 0
+}
diff --git a/pkg/expression/evaluator_test.go b/pkg/expression/evaluator_test.go
index 65de3187c002c..00c47c13b1d80 100644
--- a/pkg/expression/evaluator_test.go
+++ b/pkg/expression/evaluator_test.go
@@ -15,6 +15,7 @@
package expression
import (
+ "slices"
"testing"
"time"
@@ -649,12 +650,51 @@ func TestOptionalProp(t *testing.T) {
require.Equal(t, context.OptionalEvalPropKeySet(0), f.RequiredOptionalEvalProps())
require.Equal(t, context.OptPropCurrentUser.AsPropKeySet()|context.OptPropDDLOwnerInfo.AsPropKeySet(),
- getOptionalEvalPropsForExpr(fe))
+ GetOptionalEvalPropsForExpr(fe))
require.Equal(t, context.OptPropCurrentUser.AsPropKeySet()|context.OptPropDDLOwnerInfo.AsPropKeySet()|
context.OptPropAdvisoryLock.AsPropKeySet(),
- getOptionalEvalPropsForExpr(fe)|getOptionalEvalPropsForExpr(fe2))
+ GetOptionalEvalPropsForExpr(fe)|GetOptionalEvalPropsForExpr(fe2))
evalSuit := NewEvaluatorSuite([]Expression{fe, fe2}, false)
require.Equal(t, context.OptPropCurrentUser.AsPropKeySet()|context.OptPropDDLOwnerInfo.AsPropKeySet()|
context.OptPropAdvisoryLock.AsPropKeySet(), evalSuit.RequiredOptionalEvalProps())
}
+
+func TestMergeInputIdxToOutputIdxes(t *testing.T) {
+ ctx := createContext(t)
+ inputIdxToOutputIdxes := make(map[int][]int)
+ // input 0th should be column referred as 0th and 1st in output columns.
+ inputIdxToOutputIdxes[0] = []int{0, 1}
+ // input 1th should be column referred as 2nd and 3rd in output columns.
+ inputIdxToOutputIdxes[1] = []int{2, 3}
+ columnEval := columnEvaluator{inputIdxToOutputIdxes: inputIdxToOutputIdxes}
+
+ input := chunk.NewEmptyChunk([]*types.FieldType{types.NewFieldType(mysql.TypeLonglong), types.NewFieldType(mysql.TypeLonglong)})
+ input.AppendInt64(0, 99)
+ // input chunk's 0th and 1st are column referred itself.
+ input.MakeRef(0, 1)
+
+ // chunk: col1 <---(ref) col2
+ // ____________/ \___________/ \___
+ // proj: col1 col2 col3 col4
+ //
+ // original case after inputIdxToOutputIdxes[0], the original col2 will be nil pointer
+ // cause consecutive col3,col4 ref projection are invalid.
+ //
+ // after fix, the new inputIdxToOutputIdxes should be: inputIdxToOutputIdxes[0]: {0, 1, 2, 3}
+
+ output := chunk.NewEmptyChunk([]*types.FieldType{types.NewFieldType(mysql.TypeLonglong), types.NewFieldType(mysql.TypeLonglong),
+ types.NewFieldType(mysql.TypeLonglong), types.NewFieldType(mysql.TypeLonglong)})
+
+ err := columnEval.run(ctx, input, output)
+ require.NoError(t, err)
+ // all four columns are column-referred, pointing to the first one.
+ require.Equal(t, output.Column(0), output.Column(1))
+ require.Equal(t, output.Column(1), output.Column(2))
+ require.Equal(t, output.Column(2), output.Column(3))
+ require.Equal(t, output.GetRow(0).GetInt64(0), int64(99))
+
+ require.Equal(t, len(*columnEval.mergedInputIdxToOutputIdxes.Load()), 1)
+ slices.Sort((*columnEval.mergedInputIdxToOutputIdxes.Load())[0])
+ require.Equal(t, (*columnEval.mergedInputIdxToOutputIdxes.Load())[0], []int{0, 1, 2, 3})
+}
diff --git a/pkg/expression/expression.go b/pkg/expression/expression.go
index 279c6e0878fb1..b51bbec2cfd3f 100644
--- a/pkg/expression/expression.go
+++ b/pkg/expression/expression.go
@@ -846,6 +846,16 @@ type Assignment struct {
LazyErr error
}
+// Clone clones the Assignment.
+func (a *Assignment) Clone() *Assignment {
+ return &Assignment{
+ Col: a.Col.Clone().(*Column),
+ ColName: a.ColName,
+ Expr: a.Expr.Clone(),
+ LazyErr: a.LazyErr,
+ }
+}
+
// MemoryUsage return the memory usage of Assignment
func (a *Assignment) MemoryUsage() (sum int64) {
if a == nil {
diff --git a/pkg/expression/integration_test/BUILD.bazel b/pkg/expression/integration_test/BUILD.bazel
index 433770906f4ab..1a90ed073e9af 100644
--- a/pkg/expression/integration_test/BUILD.bazel
+++ b/pkg/expression/integration_test/BUILD.bazel
@@ -21,6 +21,7 @@ go_test(
"//pkg/parser/terror",
"//pkg/planner/core",
"//pkg/planner/core/base",
+ "//pkg/planner/core/operator/logicalop",
"//pkg/session",
"//pkg/sessionctx/variable",
"//pkg/store/mockstore",
diff --git a/pkg/expression/integration_test/integration_test.go b/pkg/expression/integration_test/integration_test.go
index 0c7b14e9143a9..eb38fe3495547 100644
--- a/pkg/expression/integration_test/integration_test.go
+++ b/pkg/expression/integration_test/integration_test.go
@@ -42,6 +42,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/terror"
plannercore "github.com/pingcap/tidb/pkg/planner/core"
"github.com/pingcap/tidb/pkg/planner/core/base"
+ "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop"
"github.com/pingcap/tidb/pkg/session"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/store/mockstore"
@@ -931,7 +932,7 @@ func TestFilterExtractFromDNF(t *testing.T) {
require.NoError(t, err, "error %v, for resolve name, expr %s", err, tt.exprStr)
p, err := plannercore.BuildLogicalPlanForTest(ctx, sctx, stmts[0], ret.InfoSchema)
require.NoError(t, err, "error %v, for build plan, expr %s", err, tt.exprStr)
- selection := p.(base.LogicalPlan).Children()[0].(*plannercore.LogicalSelection)
+ selection := p.(base.LogicalPlan).Children()[0].(*logicalop.LogicalSelection)
conds := make([]expression.Expression, len(selection.Conditions))
for i, cond := range selection.Conditions {
conds[i] = expression.PushDownNot(sctx.GetExprCtx(), cond)
@@ -1332,7 +1333,7 @@ func TestEnumIndex(t *testing.T) {
ops := []string{"=", "!=", ">", ">=", "<", "<="}
testElems := []string{"\"a\"", "\"b\"", "\"c\"", "\"d\"", "\"\"", "1", "2", "3", "4", "0", "-1"}
for i := 0; i < nRows; i++ {
- cond := fmt.Sprintf("e" + ops[rand.Intn(len(ops))] + testElems[rand.Intn(len(testElems))])
+ cond := "e" + ops[rand.Intn(len(ops))] + testElems[rand.Intn(len(testElems))]
result := tk.MustQuery("select * from t where " + cond).Sort().Rows()
tk.MustQuery("select * from tidx where " + cond).Sort().Check(result)
}
diff --git a/pkg/expression/scalar_function.go b/pkg/expression/scalar_function.go
index d27962eda9046..3ec1c6cb7f8cd 100644
--- a/pkg/expression/scalar_function.go
+++ b/pkg/expression/scalar_function.go
@@ -40,7 +40,7 @@ type ScalarFunction struct {
FuncName model.CIStr
// RetType is the type that ScalarFunction returns.
// TODO: Implement type inference here, now we use ast's return type temporarily.
- RetType *types.FieldType
+ RetType *types.FieldType `plan-cache-clone:"shallow"`
Function builtinFunc
hashcode []byte
canonicalhashcode []byte
@@ -343,7 +343,15 @@ func (sf *ScalarFunction) Clone() Expression {
FuncName: sf.FuncName,
RetType: sf.RetType,
Function: sf.Function.Clone(),
- hashcode: sf.hashcode,
+ }
+ // An implicit assumption: ScalarFunc.RetType == ScalarFunc.builtinFunc.RetType
+ if sf.hashcode != nil {
+ c.hashcode = make([]byte, len(sf.hashcode))
+ copy(c.hashcode, sf.hashcode)
+ }
+ if sf.canonicalhashcode != nil {
+ c.canonicalhashcode = make([]byte, len(sf.canonicalhashcode))
+ copy(c.canonicalhashcode, sf.canonicalhashcode)
}
c.SetCharsetAndCollation(sf.CharsetAndCollation())
c.SetCoercibility(sf.Coercibility())
diff --git a/pkg/expression/util.go b/pkg/expression/util.go
index f7b51e8983a3e..2af5c19d324f9 100644
--- a/pkg/expression/util.go
+++ b/pkg/expression/util.go
@@ -32,7 +32,6 @@ import (
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/parser/opcode"
"github.com/pingcap/tidb/pkg/parser/terror"
- "github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/types"
driver "github.com/pingcap/tidb/pkg/types/parser_driver"
"github.com/pingcap/tidb/pkg/util/chunk"
@@ -1196,8 +1195,8 @@ func DatumToConstant(d types.Datum, tp byte, flag uint) *Constant {
}
// ParamMarkerExpression generate a getparam function expression.
-func ParamMarkerExpression(ctx variable.SessionVarsProvider, v *driver.ParamMarkerExpr, needParam bool) (*Constant, error) {
- useCache := ctx.GetSessionVars().StmtCtx.UseCache()
+func ParamMarkerExpression(ctx BuildContext, v *driver.ParamMarkerExpr, needParam bool) (*Constant, error) {
+ useCache := ctx.IsUseCache()
tp := types.NewFieldType(mysql.TypeUnspecified)
types.InferParamTypeFromDatum(&v.Datum, tp)
value := &Constant{Value: v.Datum, RetType: tp}
@@ -1251,11 +1250,11 @@ func ConstructPositionExpr(p *driver.ParamMarkerExpr) *ast.PositionExpr {
}
// PosFromPositionExpr generates a position value from PositionExpr.
-func PosFromPositionExpr(ctx BuildContext, vars variable.SessionVarsProvider, v *ast.PositionExpr) (int, bool, error) {
+func PosFromPositionExpr(ctx BuildContext, v *ast.PositionExpr) (int, bool, error) {
if v.P == nil {
return v.N, false, nil
}
- value, err := ParamMarkerExpression(vars, v.P.(*driver.ParamMarkerExpr), false)
+ value, err := ParamMarkerExpression(ctx, v.P.(*driver.ParamMarkerExpr), false)
if err != nil {
return 0, true, err
}
diff --git a/pkg/expression/util_test.go b/pkg/expression/util_test.go
index 485faf2cb3480..f236e943a9d0b 100644
--- a/pkg/expression/util_test.go
+++ b/pkg/expression/util_test.go
@@ -613,9 +613,18 @@ func (m *MockExpr) EvalVectorFloat32(ctx EvalContext, row chunk.Row) (val types.
}
return types.ZeroVectorFloat32, m.i == nil, m.err
}
+func (m *MockExpr) GetType(_ EvalContext) *types.FieldType { return m.t }
+
+func (m *MockExpr) Clone() Expression {
+ cloned := new(MockExpr)
+ cloned.i = m.i
+ cloned.err = m.err
+ if m.t != nil {
+ cloned.t = m.t.Clone()
+ }
+ return cloned
+}
-func (m *MockExpr) GetType(_ EvalContext) *types.FieldType { return m.t }
-func (m *MockExpr) Clone() Expression { return nil }
func (m *MockExpr) Equal(ctx EvalContext, e Expression) bool { return false }
func (m *MockExpr) IsCorrelated() bool { return false }
func (m *MockExpr) ConstLevel() ConstLevel { return ConstNone }
diff --git a/pkg/infoschema/BUILD.bazel b/pkg/infoschema/BUILD.bazel
index c391c28ee5f75..e27e0475dcfa4 100644
--- a/pkg/infoschema/BUILD.bazel
+++ b/pkg/infoschema/BUILD.bazel
@@ -91,7 +91,7 @@ go_test(
],
embed = [":infoschema"],
flaky = True,
- shard_count = 21,
+ shard_count = 22,
deps = [
"//pkg/ddl/placement",
"//pkg/domain",
diff --git a/pkg/infoschema/builder.go b/pkg/infoschema/builder.go
index 4a23823581ba8..a29e40ac59935 100644
--- a/pkg/infoschema/builder.go
+++ b/pkg/infoschema/builder.go
@@ -546,9 +546,9 @@ func (b *Builder) applyDropSchema(diff *model.SchemaDiff) []int64 {
b.infoSchema.delSchema(di)
// Copy the sortedTables that contain the table we are going to drop.
- tableIDs := make([]int64, 0, len(di.Tables))
- bucketIdxMap := make(map[int]struct{}, len(di.Tables))
- for _, tbl := range di.Tables {
+ tableIDs := make([]int64, 0, len(di.Deprecated.Tables))
+ bucketIdxMap := make(map[int]struct{}, len(di.Deprecated.Tables))
+ for _, tbl := range di.Deprecated.Tables {
bucketIdxMap[tableBucketIdx(tbl.ID)] = struct{}{}
// TODO: If the table ID doesn't exist.
tableIDs = appendAffectedIDs(tableIDs, tbl)
@@ -683,7 +683,7 @@ func applyCreateTable(b *Builder, m *meta.Meta, dbInfo *model.DBInfo, tableID in
allocs = b.buildAllocsForCreateTable(tp, dbInfo, tblInfo, allocs)
- tbl, err := b.tableFromMeta(allocs, tblInfo)
+ tbl, err := tableFromMeta(allocs, b.factory, tblInfo)
if err != nil {
return nil, errors.Trace(err)
}
@@ -705,9 +705,9 @@ func applyCreateTable(b *Builder, m *meta.Meta, dbInfo *model.DBInfo, tableID in
b.addTemporaryTable(tableID)
}
- newTbl, ok := b.infoSchema.TableByID(tableID)
+ newTbl, ok := b.infoSchema.TableByID(context.Background(), tableID)
if ok {
- dbInfo.Tables = append(dbInfo.Tables, newTbl.Meta())
+ dbInfo.Deprecated.Tables = append(dbInfo.Deprecated.Tables, newTbl.Meta())
}
return affected, nil
}
@@ -768,17 +768,19 @@ func (b *Builder) applyDropTable(diff *model.SchemaDiff, dbInfo *model.DBInfo, t
}
func (b *Builder) deleteReferredForeignKeys(dbInfo *model.DBInfo, tableID int64) {
- for i, tblInfo := range dbInfo.Tables {
+ tables := dbInfo.Deprecated.Tables
+ for i, tblInfo := range tables {
if tblInfo.ID == tableID {
- if i == len(dbInfo.Tables)-1 {
- dbInfo.Tables = dbInfo.Tables[:i]
+ if i == len(tables)-1 {
+ tables = tables[:i]
} else {
- dbInfo.Tables = append(dbInfo.Tables[:i], dbInfo.Tables[i+1:]...)
+ tables = append(tables[:i], tables[i+1:]...)
}
b.infoSchema.deleteReferredForeignKeys(dbInfo.Name, tblInfo)
break
}
}
+ dbInfo.Deprecated.Tables = tables
}
// Build builds and returns the built infoschema.
@@ -793,12 +795,12 @@ func (b *Builder) Build(schemaTS uint64) InfoSchema {
}
// InitWithOldInfoSchema initializes an empty new InfoSchema by copies all the data from old InfoSchema.
-func (b *Builder) InitWithOldInfoSchema(oldSchema InfoSchema) (*Builder, error) {
+func (b *Builder) InitWithOldInfoSchema(oldSchema InfoSchema) error {
// Do not mix infoschema v1 and infoschema v2 building, this can simplify the logic.
// If we want to build infoschema v2, but the old infoschema is v1, just return error to trigger a full load.
isV2, _ := IsV2(oldSchema)
if b.enableV2 != isV2 {
- return nil, errors.Errorf("builder's (v2=%v) infoschema mismatch, return error to trigger full reload", b.enableV2)
+ return errors.Errorf("builder's (v2=%v) infoschema mismatch, return error to trigger full reload", b.enableV2)
}
if schemaV2, ok := oldSchema.(*infoschemaV2); ok {
@@ -816,7 +818,7 @@ func (b *Builder) InitWithOldInfoSchema(oldSchema InfoSchema) (*Builder, error)
b.infoSchema.referredForeignKeyMap = maps.Clone(oldIS.referredForeignKeyMap)
copy(b.infoSchema.sortedTablesBuckets, oldIS.sortedTablesBuckets)
- return b, nil
+ return nil
}
// getSchemaAndCopyIfNecessary creates a new schemaTables instance when a table in the database has changed.
@@ -858,14 +860,10 @@ func (b *Builder) sortAllTablesByID() {
}
// InitWithDBInfos initializes an empty new InfoSchema with a slice of DBInfo, all placement rules, and schema version.
-func (b *Builder) InitWithDBInfos(dbInfos []*model.DBInfo, policies []*model.PolicyInfo, resourceGroups []*model.ResourceGroupInfo, schemaVersion int64) (*Builder, error) {
+func (b *Builder) InitWithDBInfos(dbInfos []*model.DBInfo, policies []*model.PolicyInfo, resourceGroups []*model.ResourceGroupInfo, schemaVersion int64) error {
info := b.infoSchema
info.schemaMetaVersion = schemaVersion
- b.initBundleInfoBuilder()
-
- b.initMisc(dbInfos, policies, resourceGroups)
-
if b.enableV2 {
// We must not clear the historial versions like b.infoData = NewData(), because losing
// the historial versions would cause applyDiff get db not exist error and fail, then
@@ -885,31 +883,36 @@ func (b *Builder) InitWithDBInfos(dbInfos []*model.DBInfo, policies []*model.Pol
b.infoData.resetBeforeFullLoad(schemaVersion)
}
+ b.initBundleInfoBuilder()
+
for _, di := range dbInfos {
- err := b.createSchemaTablesForDB(di, b.tableFromMeta, schemaVersion)
+ err := b.createSchemaTablesForDB(di, tableFromMeta, schemaVersion)
if err != nil {
- return nil, errors.Trace(err)
+ return errors.Trace(err)
}
}
+ // initMisc depends on the tables and schemas, so it should be called after createSchemaTablesForDB
+ b.initMisc(dbInfos, policies, resourceGroups)
+
err := b.initVirtualTables(schemaVersion)
if err != nil {
- return nil, err
+ return err
}
b.sortAllTablesByID()
- return b, nil
+ return nil
}
-func (b *Builder) tableFromMeta(alloc autoid.Allocators, tblInfo *model.TableInfo) (table.Table, error) {
+func tableFromMeta(alloc autoid.Allocators, factory func() (pools.Resource, error), tblInfo *model.TableInfo) (table.Table, error) {
ret, err := tables.TableFromMeta(alloc, tblInfo)
if err != nil {
return nil, errors.Trace(err)
}
if t, ok := ret.(table.CachedTable); ok {
var tmp pools.Resource
- tmp, err = b.factory()
+ tmp, err = factory()
if err != nil {
return nil, errors.Trace(err)
}
@@ -922,17 +925,17 @@ func (b *Builder) tableFromMeta(alloc autoid.Allocators, tblInfo *model.TableInf
return ret, nil
}
-type tableFromMetaFunc func(alloc autoid.Allocators, tblInfo *model.TableInfo) (table.Table, error)
+type tableFromMetaFunc func(alloc autoid.Allocators, factory func() (pools.Resource, error), tblInfo *model.TableInfo) (table.Table, error)
func (b *Builder) createSchemaTablesForDB(di *model.DBInfo, tableFromMeta tableFromMetaFunc, schemaVersion int64) error {
schTbls := &schemaTables{
dbInfo: di,
- tables: make(map[string]table.Table, len(di.Tables)),
+ tables: make(map[string]table.Table, len(di.Deprecated.Tables)),
}
- for _, t := range di.Tables {
+ for _, t := range di.Deprecated.Tables {
allocs := autoid.NewAllocatorsFromTblInfo(b.Requirement, di.ID, t)
var tbl table.Table
- tbl, err := tableFromMeta(allocs, t)
+ tbl, err := tableFromMeta(allocs, b.factory, t)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Build table `%s`.`%s` schema failed", di.Name.O, t.Name.O))
}
@@ -951,9 +954,9 @@ func (b *Builder) createSchemaTablesForDB(di *model.DBInfo, tableFromMeta tableF
if b.enableV2 {
for name, id := range di.TableName2ID {
item := tableItem{
- dbName: di.Name.L,
+ dbName: di.Name,
dbID: di.ID,
- tableName: name,
+ tableName: model.NewCIStr(name),
tableID: id,
schemaVersion: schemaVersion,
}
@@ -981,9 +984,9 @@ func (b *Builder) addDB(schemaVersion int64, di *model.DBInfo, schTbls *schemaTa
func (b *Builder) addTable(schemaVersion int64, di *model.DBInfo, tblInfo *model.TableInfo, tbl table.Table) {
if b.enableV2 {
b.infoData.add(tableItem{
- dbName: di.Name.L,
+ dbName: di.Name,
dbID: di.ID,
- tableName: tblInfo.Name.L,
+ tableName: tblInfo.Name,
tableID: tblInfo.ID,
schemaVersion: schemaVersion,
}, tbl)
@@ -1006,18 +1009,18 @@ func RegisterVirtualTable(dbInfo *model.DBInfo, tableFromMeta tableFromMetaFunc)
}
// NewBuilder creates a new Builder with a Handle.
-func NewBuilder(r autoid.Requirement, factory func() (pools.Resource, error), infoData *Data) *Builder {
+func NewBuilder(r autoid.Requirement, factory func() (pools.Resource, error), infoData *Data, useV2 bool) *Builder {
builder := &Builder{
Requirement: r,
- infoschemaV2: NewInfoSchemaV2(r, infoData),
+ infoschemaV2: NewInfoSchemaV2(r, factory, infoData),
dirtyDB: make(map[string]bool),
factory: factory,
infoData: infoData,
+ enableV2: useV2,
}
schemaCacheSize := variable.SchemaCacheSize.Load()
if schemaCacheSize > 0 {
infoData.tableCache.SetCapacity(schemaCacheSize)
- builder.enableV2 = true
}
return builder
}
diff --git a/pkg/infoschema/builder_misc.go b/pkg/infoschema/builder_misc.go
index 2b33ff6a18d24..f3c10ffc57a3b 100644
--- a/pkg/infoschema/builder_misc.go
+++ b/pkg/infoschema/builder_misc.go
@@ -114,10 +114,18 @@ func (b *Builder) initMisc(dbInfos []*model.DBInfo, policies []*model.PolicyInfo
}
// Maintain foreign key reference information.
- rs := info.ListTablesWithSpecialAttribute(ForeignKeysAttribute)
- for _, db := range rs {
- for _, tbl := range db.TableInfos {
- info.addReferredForeignKeys(model.NewCIStr(db.DBName), tbl)
+ if b.enableV2 {
+ rs := b.ListTablesWithSpecialAttribute(ForeignKeysAttribute)
+ for _, db := range rs {
+ for _, tbl := range db.TableInfos {
+ info.addReferredForeignKeys(db.DBName, tbl)
+ }
+ }
+ return
+ }
+ for _, di := range dbInfos {
+ for _, t := range di.Deprecated.Tables {
+ b.infoSchema.addReferredForeignKeys(di.Name, t)
}
}
}
diff --git a/pkg/infoschema/bundle_builder.go b/pkg/infoschema/bundle_builder.go
index 133357709f235..a54ffc119357d 100644
--- a/pkg/infoschema/bundle_builder.go
+++ b/pkg/infoschema/bundle_builder.go
@@ -15,6 +15,8 @@
package infoschema
import (
+ "context"
+
"github.com/pingcap/errors"
"github.com/pingcap/tidb/pkg/ddl/placement"
"github.com/pingcap/tidb/pkg/parser/model"
@@ -114,7 +116,7 @@ func (b *bundleInfoBuilder) completeUpdateTables(is *infoSchema) {
func (b *bundleInfoBuilder) updateTableBundles(infoSchemaInterface InfoSchema, tableID int64) {
is := infoSchemaInterface.base()
- tbl, ok := infoSchemaInterface.TableByID(tableID)
+ tbl, ok := infoSchemaInterface.TableByID(context.Background(), tableID)
if !ok {
b.deleteBundle(is, tableID)
return
diff --git a/pkg/infoschema/cache.go b/pkg/infoschema/cache.go
index 7f8d000cc9d63..a58dca8c2e5fd 100644
--- a/pkg/infoschema/cache.go
+++ b/pkg/infoschema/cache.go
@@ -37,6 +37,10 @@ type InfoCache struct {
r autoid.Requirement
Data *Data
+
+ // first known schema version records the first known schema version, all schemas between [firstKnownSchemaVersion, latest)
+ // are known as long as we keep the DDL history correctly.
+ firstKnownSchemaVersion int64
}
type schemaAndTimestamp struct {
@@ -97,6 +101,7 @@ func (h *InfoCache) Upsert(is InfoSchema, schemaTS uint64) func() {
infoschema: is,
timestamp: int64(schemaTS),
})
+ h.firstKnownSchemaVersion = is.SchemaMetaVersion()
return func() {
// TODO: It's a bit tricky here, somewhere is holding the reference of the old infoschema.
@@ -223,8 +228,28 @@ func (h *InfoCache) getByVersionNoLock(version int64) InfoSchema {
// return h.cache[i]
// }
// ```
+ // upsert is a full reset of InfoCache, after upsert, the DDL history might lost and the assumption does not hold anymore.
+ // For example:
+ // Before
+ // infoschema 51
+ // infoschema 52
+ // infoschema 53
+ // infoschema 54
+ // infoschema 55
+ // infoschema 56
+ // After Upsert()
+ // infoschema 56
+ // Then load historial snapshot version 51
+ // infoschema 51
+ // infoschema 56
+ // Now, request for schema version 55, return infoschem 51 would be wrong!
+ //
+ if i == len(h.cache) {
+ return nil
+ }
- if i < len(h.cache) && (i != 0 || h.cache[i].infoschema.SchemaMetaVersion() == version) {
+ if h.cache[i].infoschema.SchemaMetaVersion() == version ||
+ (i != 0 && h.cache[i].infoschema.SchemaMetaVersion() >= h.firstKnownSchemaVersion) {
infoschema_metrics.HitVersionCounter.Inc()
return h.cache[i].infoschema
}
@@ -289,6 +314,9 @@ func (h *InfoCache) Insert(is InfoSchema, schemaTS uint64) bool {
infoschema: is,
timestamp: int64(schemaTS),
}
+ if len(h.cache) == 1 {
+ h.firstKnownSchemaVersion = is.SchemaMetaVersion()
+ }
} else if i < len(h.cache) {
// drop older schema
copy(h.cache[i+1:], h.cache[i:])
diff --git a/pkg/infoschema/cluster.go b/pkg/infoschema/cluster.go
index f2897eef52550..6436a17b203eb 100644
--- a/pkg/infoschema/cluster.go
+++ b/pkg/infoschema/cluster.go
@@ -108,8 +108,9 @@ func init() {
}
}
-// isClusterTableByName used to check whether the table is a cluster memory table.
-func isClusterTableByName(dbName, tableName string) bool {
+// IsClusterTableByName used to check whether the table is a cluster memory table.
+// Export for PhysicalTableScan.ExplainID
+func IsClusterTableByName(dbName, tableName string) bool {
dbName = strings.ToUpper(dbName)
switch dbName {
case util.InformationSchemaName.O, util.PerformanceSchemaName.O:
diff --git a/pkg/infoschema/context/infoschema.go b/pkg/infoschema/context/infoschema.go
index 0da24b60175eb..e808da697c9de 100644
--- a/pkg/infoschema/context/infoschema.go
+++ b/pkg/infoschema/context/infoschema.go
@@ -33,13 +33,18 @@ type MetaOnlyInfoSchema interface {
FindTableInfoByPartitionID(partitionID int64) (*model.TableInfo, *model.DBInfo, *model.PartitionDefinition)
TableExists(schema, table model.CIStr) bool
SchemaByID(id int64) (*model.DBInfo, bool)
- AllSchemas() []*model.DBInfo
+ SchemaAndTable
AllSchemaNames() []model.CIStr
- SchemaTableInfos(ctx stdctx.Context, schema model.CIStr) ([]*model.TableInfo, error)
SchemaSimpleTableInfos(ctx stdctx.Context, schema model.CIStr) ([]*model.TableNameInfo, error)
Misc
}
+// SchemaAndTable is define for iterating all the schemas and tables in the infoschema.
+type SchemaAndTable interface {
+ AllSchemas() []*model.DBInfo
+ SchemaTableInfos(ctx stdctx.Context, schema model.CIStr) ([]*model.TableInfo, error)
+}
+
// Misc contains the methods that are not closely related to InfoSchema.
type Misc interface {
PolicyByName(name model.CIStr) (*model.PolicyInfo, bool)
@@ -61,3 +66,21 @@ type Misc interface {
// GetTableReferredForeignKeys gets the table's ReferredFKInfo by lowercase schema and table name.
GetTableReferredForeignKeys(schema, table string) []*model.ReferredFKInfo
}
+
+// DBInfoAsInfoSchema is used mainly in test.
+type DBInfoAsInfoSchema []*model.DBInfo
+
+// AllSchemas implement infoschema.SchemaAndTable interface.
+func (d DBInfoAsInfoSchema) AllSchemas() []*model.DBInfo {
+ return []*model.DBInfo(d)
+}
+
+// SchemaTableInfos implement infoschema.SchemaAndTable interface.
+func (d DBInfoAsInfoSchema) SchemaTableInfos(ctx stdctx.Context, schema model.CIStr) ([]*model.TableInfo, error) {
+ for _, db := range d {
+ if db.Name == schema {
+ return db.Deprecated.Tables, nil
+ }
+ }
+ return nil, nil
+}
diff --git a/pkg/infoschema/infoschema.go b/pkg/infoschema/infoschema.go
index 76ea5ef4b8038..9051f0236608b 100644
--- a/pkg/infoschema/infoschema.go
+++ b/pkg/infoschema/infoschema.go
@@ -100,7 +100,8 @@ type SchemaAndTableName struct {
// MockInfoSchema only serves for test.
func MockInfoSchema(tbList []*model.TableInfo) InfoSchema {
result := newInfoSchema()
- dbInfo := &model.DBInfo{ID: 1, Name: model.NewCIStr("test"), Tables: tbList}
+ dbInfo := &model.DBInfo{ID: 1, Name: model.NewCIStr("test")}
+ dbInfo.Deprecated.Tables = tbList
tableNames := &schemaTables{
dbInfo: dbInfo,
tables: make(map[string]table.Table),
@@ -140,7 +141,8 @@ func MockInfoSchema(tbList []*model.TableInfo) InfoSchema {
State: model.StatePublic,
},
}
- mysqlDBInfo := &model.DBInfo{ID: 2, Name: model.NewCIStr("mysql"), Tables: tables}
+ mysqlDBInfo := &model.DBInfo{ID: 2, Name: model.NewCIStr("mysql")}
+ mysqlDBInfo.Deprecated.Tables = tables
tableNames = &schemaTables{
dbInfo: mysqlDBInfo,
tables: make(map[string]table.Table),
@@ -164,7 +166,8 @@ func MockInfoSchema(tbList []*model.TableInfo) InfoSchema {
// MockInfoSchemaWithSchemaVer only serves for test.
func MockInfoSchemaWithSchemaVer(tbList []*model.TableInfo, schemaVer int64) InfoSchema {
result := newInfoSchema()
- dbInfo := &model.DBInfo{ID: 1, Name: model.NewCIStr("test"), Tables: tbList}
+ dbInfo := &model.DBInfo{ID: 1, Name: model.NewCIStr("test")}
+ dbInfo.Deprecated.Tables = tbList
tableNames := &schemaTables{
dbInfo: dbInfo,
tables: make(map[string]table.Table),
@@ -291,7 +294,7 @@ func SchemaByTable(is InfoSchema, tableInfo *model.TableInfo) (val *model.DBInfo
return is.SchemaByID(tableInfo.DBID)
}
-func (is *infoSchema) TableByID(id int64) (val table.Table, ok bool) {
+func (is *infoSchema) TableByID(_ stdctx.Context, id int64) (val table.Table, ok bool) {
if !tableIDIsValid(id) {
return nil, false
}
@@ -306,7 +309,7 @@ func (is *infoSchema) TableByID(id int64) (val table.Table, ok bool) {
// TableInfoByID implements InfoSchema.TableInfoByID
func (is *infoSchema) TableInfoByID(id int64) (*model.TableInfo, bool) {
- tbl, ok := is.TableByID(id)
+ tbl, ok := is.TableByID(stdctx.Background(), id)
return getTableInfo(tbl), ok
}
@@ -348,14 +351,14 @@ func (is *infoSchema) SchemaSimpleTableInfos(ctx stdctx.Context, schema model.CI
}
type tableInfoResult struct {
- DBName string
+ DBName model.CIStr
TableInfos []*model.TableInfo
}
func (is *infoSchema) ListTablesWithSpecialAttribute(filter specialAttributeFilter) []tableInfoResult {
ret := make([]tableInfoResult, 0, 10)
for _, dbName := range is.AllSchemaNames() {
- res := tableInfoResult{DBName: dbName.O}
+ res := tableInfoResult{DBName: dbName}
tblInfos, err := is.SchemaTableInfos(stdctx.Background(), dbName)
terror.Log(err)
for _, tblInfo := range tblInfos {
@@ -469,8 +472,8 @@ func init() {
Name: util.InformationSchemaName,
Charset: mysql.DefaultCharset,
Collate: mysql.DefaultCollationName,
- Tables: infoSchemaTables,
}
+ infoSchemaDB.Deprecated.Tables = infoSchemaTables
RegisterVirtualTable(infoSchemaDB, createInfoSchemaTable)
util.GetSequenceByName = func(is context.MetaOnlyInfoSchema, schema, sequence model.CIStr) (util.SequenceTable, error) {
return GetSequenceByName(is.(InfoSchema), schema, sequence)
@@ -807,7 +810,7 @@ func (ts *SessionExtendedInfoSchema) TableInfoByName(schema, table model.CIStr)
// TableInfoByID implements InfoSchema.TableInfoByID
func (ts *SessionExtendedInfoSchema) TableInfoByID(id int64) (*model.TableInfo, bool) {
- tbl, ok := ts.TableByID(id)
+ tbl, ok := ts.TableByID(stdctx.Background(), id)
return getTableInfo(tbl), ok
}
@@ -820,7 +823,7 @@ func (ts *SessionExtendedInfoSchema) FindTableInfoByPartitionID(
}
// TableByID implements InfoSchema.TableByID
-func (ts *SessionExtendedInfoSchema) TableByID(id int64) (table.Table, bool) {
+func (ts *SessionExtendedInfoSchema) TableByID(ctx stdctx.Context, id int64) (table.Table, bool) {
if !tableIDIsValid(id) {
return nil, false
}
@@ -837,7 +840,7 @@ func (ts *SessionExtendedInfoSchema) TableByID(id int64) (table.Table, bool) {
}
}
- return ts.InfoSchema.TableByID(id)
+ return ts.InfoSchema.TableByID(ctx, id)
}
// SchemaByID implements InfoSchema.SchemaByID, it returns a stale DBInfo even if it's dropped.
@@ -889,7 +892,7 @@ func (ts *SessionExtendedInfoSchema) DetachTemporaryTableInfoSchema() *SessionEx
// If the id is a partition id, the corresponding table.Table and PartitionDefinition will be returned.
// If the id is not found in the InfoSchema, nil will be returned for both return values.
func FindTableByTblOrPartID(is InfoSchema, id int64) (table.Table, *model.PartitionDefinition) {
- tbl, ok := is.TableByID(id)
+ tbl, ok := is.TableByID(stdctx.Background(), id)
if ok {
return tbl, nil
}
diff --git a/pkg/infoschema/infoschema_nokit_test.go b/pkg/infoschema/infoschema_nokit_test.go
index 4983e8b275489..add60b0eb6c3e 100644
--- a/pkg/infoschema/infoschema_nokit_test.go
+++ b/pkg/infoschema/infoschema_nokit_test.go
@@ -21,6 +21,13 @@ import (
"github.com/stretchr/testify/require"
)
+// GetCache is exported for testing.
+func (is *infoschemaV2) HasCache(tableID int64, schemaVersion int64) bool {
+ key := tableCacheKey{tableID, schemaVersion}
+ _, found := is.tableCache.Get(key)
+ return found
+}
+
func TestInfoSchemaAddDel(t *testing.T) {
is := newInfoSchema()
is.addSchema(&schemaTables{
diff --git a/pkg/infoschema/infoschema_test.go b/pkg/infoschema/infoschema_test.go
index d052312df8b94..226fd33949244 100644
--- a/pkg/infoschema/infoschema_test.go
+++ b/pkg/infoschema/infoschema_test.go
@@ -17,6 +17,7 @@ package infoschema_test
import (
"context"
"encoding/json"
+ "fmt"
"math"
"strings"
"testing"
@@ -91,18 +92,19 @@ func TestBasic(t *testing.T) {
dbID, err := internal.GenGlobalID(re.Store())
require.NoError(t, err)
dbInfo := &model.DBInfo{
- ID: dbID,
- Name: dbName,
- Tables: []*model.TableInfo{tblInfo},
- State: model.StatePublic,
+ ID: dbID,
+ Name: dbName,
+ State: model.StatePublic,
}
+ dbInfo.Deprecated.Tables = []*model.TableInfo{tblInfo}
tblInfo.DBID = dbInfo.ID
dbInfos := []*model.DBInfo{dbInfo}
internal.AddDB(t, re.Store(), dbInfo)
internal.AddTable(t, re.Store(), dbInfo, tblInfo)
- builder, err := infoschema.NewBuilder(re, nil, infoschema.NewData()).InitWithDBInfos(dbInfos, nil, nil, 1)
+ builder := infoschema.NewBuilder(re, nil, infoschema.NewData(), variable.SchemaCacheSize.Load() > 0)
+ err = builder.InitWithDBInfos(dbInfos, nil, nil, 1)
require.NoError(t, err)
txn, err := re.Store().Begin()
@@ -156,7 +158,7 @@ func TestBasic(t *testing.T) {
require.False(t, infoschema.TableIsView(is, dbName, tbName))
require.False(t, infoschema.TableIsSequence(is, dbName, tbName))
- tb, ok := is.TableByID(tbID)
+ tb, ok := is.TableByID(context.Background(), tbID)
require.True(t, ok)
require.NotNil(t, tb)
@@ -164,7 +166,7 @@ func TestBasic(t *testing.T) {
require.True(t, ok)
require.Same(t, tb.Meta(), gotTblInfo)
- tb, ok = is.TableByID(dbID)
+ tb, ok = is.TableByID(context.Background(), dbID)
require.False(t, ok)
require.Nil(t, tb)
@@ -172,7 +174,7 @@ func TestBasic(t *testing.T) {
require.False(t, ok)
require.Nil(t, gotTblInfo)
- tb, ok = is.TableByID(-12345)
+ tb, ok = is.TableByID(context.Background(), -12345)
require.False(t, ok)
require.Nil(t, tb)
@@ -196,7 +198,7 @@ func TestBasic(t *testing.T) {
require.Nil(t, gotTblInfo)
// negative id should always be seen as not exists
- tb, ok = is.TableByID(-1)
+ tb, ok = is.TableByID(context.Background(), -1)
require.False(t, ok)
require.Nil(t, tb)
schema, ok = is.SchemaByID(-1)
@@ -209,9 +211,9 @@ func TestBasic(t *testing.T) {
tblInfos, err := is.SchemaTableInfos(context.Background(), dbName)
require.NoError(t, err)
require.Len(t, tblInfos, 1)
- tbl, ok := is.TableByID(tblInfos[0].ID)
+ tbl, ok := is.TableByID(context.Background(), tblInfos[0].ID)
require.True(t, ok)
- require.Same(t, tbl.Meta(), tblInfos[0])
+ require.Equal(t, tbl.Meta(), tblInfos[0]) // Equal but not Same
tblInfos, err = is.SchemaTableInfos(context.Background(), noexist)
require.NoError(t, err)
@@ -225,14 +227,22 @@ func TestBasic(t *testing.T) {
require.NoError(t, err)
txn, err = re.Store().Begin()
require.NoError(t, err)
- _, err = builder.ApplyDiff(meta.NewMeta(txn), &model.SchemaDiff{Type: model.ActionRenameTable, SchemaID: dbID, TableID: tbID, OldSchemaID: dbID})
+ _, err = builder.ApplyDiff(meta.NewMeta(txn), &model.SchemaDiff{
+ Type: model.ActionRenameTable,
+ SchemaID: dbID,
+ TableID: tbID,
+ OldSchemaID: dbID,
+ Version: is.SchemaMetaVersion() + 1,
+ })
require.NoError(t, err)
err = txn.Rollback()
require.NoError(t, err)
is = builder.Build(math.MaxUint64)
schema, ok = is.SchemaByID(dbID)
require.True(t, ok)
- require.Equal(t, 1, len(schema.Tables))
+ tbls, err := is.SchemaTableInfos(context.Background(), schema.Name)
+ require.NoError(t, err)
+ require.Equal(t, 1, len(tbls))
}
func TestMockInfoSchema(t *testing.T) {
@@ -252,7 +262,7 @@ func TestMockInfoSchema(t *testing.T) {
}
tableInfo.Columns = []*model.ColumnInfo{colInfo}
is := infoschema.MockInfoSchema([]*model.TableInfo{tableInfo})
- tbl, ok := is.TableByID(tblID)
+ tbl, ok := is.TableByID(context.Background(), tblID)
require.True(t, ok)
require.Equal(t, tblName, tbl.Meta().Name)
require.Equal(t, colInfo, tbl.Cols()[0].ColumnInfo)
@@ -279,7 +289,8 @@ func TestInfoTables(t *testing.T) {
require.NoError(t, err)
}()
- builder, err := infoschema.NewBuilder(re, nil, infoschema.NewData()).InitWithDBInfos(nil, nil, nil, 0)
+ builder := infoschema.NewBuilder(re, nil, infoschema.NewData(), variable.SchemaCacheSize.Load() > 0)
+ err := builder.InitWithDBInfos(nil, nil, nil, 0)
require.NoError(t, err)
is := builder.Build(math.MaxUint64)
@@ -336,14 +347,15 @@ func TestBuildSchemaWithGlobalTemporaryTable(t *testing.T) {
}()
dbInfo := &model.DBInfo{
- ID: 1,
- Name: model.NewCIStr("test"),
- Tables: []*model.TableInfo{},
- State: model.StatePublic,
+ ID: 1,
+ Name: model.NewCIStr("test"),
+ State: model.StatePublic,
}
+ dbInfo.Deprecated.Tables = []*model.TableInfo{}
dbInfos := []*model.DBInfo{dbInfo}
data := infoschema.NewData()
- builder, err := infoschema.NewBuilder(re, nil, data).InitWithDBInfos(dbInfos, nil, nil, 1)
+ builder := infoschema.NewBuilder(re, nil, data, variable.SchemaCacheSize.Load() > 0)
+ err := builder.InitWithDBInfos(dbInfos, nil, nil, 1)
require.NoError(t, err)
is := builder.Build(math.MaxUint64)
require.False(t, is.HasTemporaryTable())
@@ -363,7 +375,8 @@ func TestBuildSchemaWithGlobalTemporaryTable(t *testing.T) {
err := kv.RunInNewTxn(ctx, re.Store(), true, func(ctx context.Context, txn kv.Transaction) error {
m := meta.NewMeta(txn)
for _, change := range changes {
- builder, err := infoschema.NewBuilder(re, nil, data).InitWithOldInfoSchema(curIs)
+ builder = infoschema.NewBuilder(re, nil, data, variable.SchemaCacheSize.Load() > 0)
+ err := builder.InitWithOldInfoSchema(curIs)
require.NoError(t, err)
change(m, builder)
curIs = builder.Build(math.MaxUint64)
@@ -376,7 +389,7 @@ func TestBuildSchemaWithGlobalTemporaryTable(t *testing.T) {
createGlobalTemporaryTableChange := func(tblID int64) func(m *meta.Meta, builder *infoschema.Builder) {
return func(m *meta.Meta, builder *infoschema.Builder) {
- err := m.CreateTableOrView(db.ID, db.Name.L, &model.TableInfo{
+ err := m.CreateTableOrView(db.ID, &model.TableInfo{
ID: tblID,
TempTableType: model.TempTableGlobal,
State: model.StatePublic,
@@ -389,7 +402,7 @@ func TestBuildSchemaWithGlobalTemporaryTable(t *testing.T) {
createNormalTableChange := func(tblID int64) func(m *meta.Meta, builder *infoschema.Builder) {
return func(m *meta.Meta, builder *infoschema.Builder) {
- err := m.CreateTableOrView(db.ID, db.Name.L, &model.TableInfo{
+ err := m.CreateTableOrView(db.ID, &model.TableInfo{
ID: tblID,
State: model.StatePublic,
})
@@ -401,7 +414,7 @@ func TestBuildSchemaWithGlobalTemporaryTable(t *testing.T) {
dropTableChange := func(tblID int64) func(m *meta.Meta, builder *infoschema.Builder) {
return func(m *meta.Meta, builder *infoschema.Builder) {
- err := m.DropTableOrView(db.ID, db.Name.L, tblID, "")
+ err := m.DropTableOrView(db.ID, tblID)
require.NoError(t, err)
_, err = builder.ApplyDiff(m, &model.SchemaDiff{Type: model.ActionDropTable, SchemaID: db.ID, TableID: tblID, Version: 1})
require.NoError(t, err)
@@ -410,10 +423,10 @@ func TestBuildSchemaWithGlobalTemporaryTable(t *testing.T) {
truncateGlobalTemporaryTableChange := func(tblID, newTblID int64) func(m *meta.Meta, builder *infoschema.Builder) {
return func(m *meta.Meta, builder *infoschema.Builder) {
- err := m.DropTableOrView(db.ID, db.Name.L, tblID, "")
+ err := m.DropTableOrView(db.ID, tblID)
require.NoError(t, err)
- err = m.CreateTableOrView(db.ID, db.Name.L, &model.TableInfo{
+ err = m.CreateTableOrView(db.ID, &model.TableInfo{
ID: newTblID,
TempTableType: model.TempTableGlobal,
State: model.StatePublic,
@@ -444,9 +457,10 @@ func TestBuildSchemaWithGlobalTemporaryTable(t *testing.T) {
newDB, ok := newIS.SchemaByName(model.NewCIStr("test"))
tblInfos, err := newIS.SchemaTableInfos(context.Background(), newDB.Name)
require.NoError(t, err)
- newDB.Tables = tblInfos
+ newDB.Deprecated.Tables = tblInfos
require.True(t, ok)
- builder, err = infoschema.NewBuilder(re, nil, data).InitWithDBInfos([]*model.DBInfo{newDB}, newIS.AllPlacementPolicies(), newIS.AllResourceGroups(), newIS.SchemaMetaVersion())
+ builder = infoschema.NewBuilder(re, nil, data, variable.SchemaCacheSize.Load() > 0)
+ err = builder.InitWithDBInfos([]*model.DBInfo{newDB}, newIS.AllPlacementPolicies(), newIS.AllResourceGroups(), newIS.SchemaMetaVersion())
require.NoError(t, err)
require.True(t, builder.Build(math.MaxUint64).HasTemporaryTable())
@@ -571,12 +585,13 @@ func TestBuildBundle(t *testing.T) {
assertBundle(is, tbl2.Meta().ID, nil)
assertBundle(is, p1.ID, p1Bundle)
- if len(db.Tables) == 0 {
+ if len(db.Deprecated.Tables) == 0 {
var err error
- db.Tables, err = is.SchemaTableInfos(context.Background(), db.Name)
+ db.Deprecated.Tables, err = is.SchemaTableInfos(context.Background(), db.Name)
require.NoError(t, err)
}
- builder, err := infoschema.NewBuilder(dom, nil, infoschema.NewData()).InitWithDBInfos([]*model.DBInfo{db}, is.AllPlacementPolicies(), is.AllResourceGroups(), is.SchemaMetaVersion())
+ builder := infoschema.NewBuilder(dom, nil, infoschema.NewData(), variable.SchemaCacheSize.Load() > 0)
+ err = builder.InitWithDBInfos([]*model.DBInfo{db}, is.AllPlacementPolicies(), is.AllResourceGroups(), is.SchemaMetaVersion())
require.NoError(t, err)
is2 := builder.Build(math.MaxUint64)
assertBundle(is2, tbl1.Meta().ID, tb1Bundle)
@@ -584,6 +599,55 @@ func TestBuildBundle(t *testing.T) {
assertBundle(is2, p1.ID, p1Bundle)
}
+func TestWithRefillOption(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("set @@global.tidb_schema_cache_size = 512 * 1024 * 1024")
+
+ tk.MustExec("create table t1 (id int)")
+ tk.MustQuery("select * from t1").Check(testkit.Rows())
+ is := dom.InfoSchema()
+ tbl, err := is.TableByName(context.Background(), model.NewCIStr("test"), model.NewCIStr("t1"))
+ require.NoError(t, err)
+ tblInfo := tbl.Meta()
+ ok, v2 := infoschema.IsV2(is)
+ require.True(t, ok)
+
+ hit := true
+ miss := false
+ testCases := []struct {
+ OP string
+ ctx context.Context
+ expect bool
+ }{
+ {"TableByName", context.Background(), hit},
+ {"TableByName", infoschema.WithRefillOption(context.Background(), true), hit},
+ {"TableByName", infoschema.WithRefillOption(context.Background(), false), miss},
+ {"TableByID", context.Background(), miss},
+ {"TableByID", infoschema.WithRefillOption(context.Background(), true), hit},
+ {"TableByID", infoschema.WithRefillOption(context.Background(), false), miss},
+ }
+
+ for i, testCase := range testCases {
+ // Mock t1 schema cache been evicted.
+ v2.EvictTable(model.NewCIStr("test"), model.NewCIStr("t1"))
+
+ // Test the API
+ switch testCase.OP {
+ case "TableByID":
+ _, found := is.TableByID(testCase.ctx, tblInfo.ID)
+ require.True(t, found)
+ case "TableByName":
+ _, err := is.TableByName(testCase.ctx, model.NewCIStr("test"), model.NewCIStr("t1"))
+ require.NoError(t, err)
+ }
+
+ got := v2.HasCache(tblInfo.ID, is.SchemaMetaVersion())
+ require.Equal(t, testCase.expect, got, fmt.Sprintf("case %d failed", i))
+ }
+}
+
func TestLocalTemporaryTables(t *testing.T) {
re := internal.CreateAutoIDRequirement(t)
var err error
@@ -825,35 +889,35 @@ func TestLocalTemporaryTables(t *testing.T) {
require.Same(t, tbl.Meta(), gotTblInfo)
// test TableByID
- tbl, ok := is.TableByID(normalTbTestA.Meta().ID)
+ tbl, ok := is.TableByID(context.Background(), normalTbTestA.Meta().ID)
require.True(t, ok)
require.Equal(t, normalTbTestA.Meta(), tbl.Meta())
gotTblInfo, ok = is.TableInfoByID(normalTbTestA.Meta().ID)
require.True(t, ok)
require.Same(t, tbl.Meta(), gotTblInfo)
- tbl, ok = is.TableByID(normalTbTestB.Meta().ID)
+ tbl, ok = is.TableByID(context.Background(), normalTbTestB.Meta().ID)
require.True(t, ok)
require.Equal(t, normalTbTestB.Meta(), tbl.Meta())
gotTblInfo, ok = is.TableInfoByID(normalTbTestB.Meta().ID)
require.True(t, ok)
require.Same(t, tbl.Meta(), gotTblInfo)
- tbl, ok = is.TableByID(tmpTbTestA.Meta().ID)
+ tbl, ok = is.TableByID(context.Background(), tmpTbTestA.Meta().ID)
require.True(t, ok)
require.Equal(t, tmpTbTestA, tbl)
gotTblInfo, ok = is.TableInfoByID(tmpTbTestA.Meta().ID)
require.True(t, ok)
require.Same(t, tbl.Meta(), gotTblInfo)
- tbl, ok = is.TableByID(tb12.Meta().ID)
+ tbl, ok = is.TableByID(context.Background(), tb12.Meta().ID)
require.True(t, ok)
require.Equal(t, tb12, tbl)
gotTblInfo, ok = is.TableInfoByID(tb12.Meta().ID)
require.True(t, ok)
require.Same(t, tbl.Meta(), gotTblInfo)
- tbl, ok = is.TableByID(1234567)
+ tbl, ok = is.TableByID(context.Background(), 1234567)
require.False(t, ok)
require.Nil(t, tbl)
gotTblInfo, ok = is.TableInfoByID(1234567)
@@ -885,7 +949,7 @@ func TestLocalTemporaryTables(t *testing.T) {
require.Nil(t, info)
// negative id should always be seen as not exists
- tbl, ok = is.TableByID(-1)
+ tbl, ok = is.TableByID(context.Background(), -1)
require.False(t, ok)
require.Nil(t, tbl)
info, ok = is.SchemaByID(-1)
@@ -999,7 +1063,8 @@ func (tc *infoschemaTestContext) createSchema() {
internal.AddDB(tc.t, tc.re.Store(), dbInfo)
tc.dbInfo = dbInfo
// init infoschema
- builder, err := infoschema.NewBuilder(tc.re, nil, tc.data).InitWithDBInfos([]*model.DBInfo{}, nil, nil, 1)
+ builder := infoschema.NewBuilder(tc.re, nil, tc.data, variable.SchemaCacheSize.Load() > 0)
+ err := builder.InitWithDBInfos([]*model.DBInfo{}, nil, nil, 1)
require.NoError(tc.t, err)
tc.is = builder.Build(math.MaxUint64)
}
@@ -1045,7 +1110,7 @@ func (tc *infoschemaTestContext) runCreateTable(tblName string) int64 {
internal.AddTable(tc.t, tc.re.Store(), tc.dbInfo, tblInfo)
tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionCreateTable, SchemaID: tc.dbInfo.ID, TableID: tblInfo.ID}, func(tc *infoschemaTestContext) {
- tbl, ok := tc.is.TableByID(tblInfo.ID)
+ tbl, ok := tc.is.TableByID(context.Background(), tblInfo.ID)
require.True(tc.t, ok)
require.Equal(tc.t, tbl.Meta().Name.O, tblName)
})
@@ -1069,7 +1134,7 @@ func (tc *infoschemaTestContext) runCreateTables(tblNames []string) {
tc.applyDiffAndCheck(&diff, func(tc *infoschemaTestContext) {
for i, opt := range diff.AffectedOpts {
- tbl, ok := tc.is.TableByID(opt.TableID)
+ tbl, ok := tc.is.TableByID(context.Background(), opt.TableID)
require.True(tc.t, ok)
require.Equal(tc.t, tbl.Meta().Name.O, tblNames[i])
}
@@ -1083,7 +1148,7 @@ func (tc *infoschemaTestContext) runDropTable(tblName string) {
// dropTable
internal.DropTable(tc.t, tc.re.Store(), tc.dbInfo, tblID, tblName)
tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionDropTable, SchemaID: tc.dbInfo.ID, TableID: tblID}, func(tc *infoschemaTestContext) {
- tbl, ok := tc.is.TableByID(tblID)
+ tbl, ok := tc.is.TableByID(context.Background(), tblID)
require.False(tc.t, ok)
require.Nil(tc.t, tbl)
})
@@ -1106,7 +1171,7 @@ func (tc *infoschemaTestContext) runAddColumn(tblName string) {
tc.addColumn(tbl.Meta())
tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionAddColumn, SchemaID: tc.dbInfo.ID, TableID: tbl.Meta().ID}, func(tc *infoschemaTestContext) {
- tbl, ok := tc.is.TableByID(tbl.Meta().ID)
+ tbl, ok := tc.is.TableByID(context.Background(), tbl.Meta().ID)
require.True(tc.t, ok)
require.Equal(tc.t, 2, len(tbl.Cols()))
})
@@ -1139,7 +1204,7 @@ func (tc *infoschemaTestContext) runModifyColumn(tblName string) {
tc.modifyColumn(tbl.Meta())
tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionModifyColumn, SchemaID: tc.dbInfo.ID, TableID: tbl.Meta().ID}, func(tc *infoschemaTestContext) {
- tbl, ok := tc.is.TableByID(tbl.Meta().ID)
+ tbl, ok := tc.is.TableByID(context.Background(), tbl.Meta().ID)
require.True(tc.t, ok)
require.Equal(tc.t, "test", tbl.Cols()[0].Comment)
})
@@ -1183,7 +1248,8 @@ func (tc *infoschemaTestContext) applyDiffAndCheck(diff *model.SchemaDiff, check
txn, err := tc.re.Store().Begin()
require.NoError(tc.t, err)
- builder, err := infoschema.NewBuilder(tc.re, nil, tc.data).InitWithOldInfoSchema(tc.is)
+ builder := infoschema.NewBuilder(tc.re, nil, tc.data, variable.SchemaCacheSize.Load() > 0)
+ err = builder.InitWithOldInfoSchema(tc.is)
require.NoError(tc.t, err)
// applyDiff
_, err = builder.ApplyDiff(meta.NewMeta(txn), diff)
diff --git a/pkg/infoschema/infoschema_v2.go b/pkg/infoschema/infoschema_v2.go
index eb4c72ec9b411..3cdbe4235a464 100644
--- a/pkg/infoschema/infoschema_v2.go
+++ b/pkg/infoschema/infoschema_v2.go
@@ -22,6 +22,7 @@ import (
"sync"
"time"
+ "github.com/ngaut/pools"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/ddl/placement"
@@ -32,7 +33,6 @@ import (
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/terror"
"github.com/pingcap/tidb/pkg/table"
- "github.com/pingcap/tidb/pkg/table/tables"
"github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/size"
"github.com/pingcap/tidb/pkg/util/tracing"
@@ -42,9 +42,9 @@ import (
// tableItem is the btree item sorted by name or by id.
type tableItem struct {
- dbName string
+ dbName model.CIStr
dbID int64
- tableName string
+ tableName model.CIStr
tableID int64
schemaVersion int64
tomb bool
@@ -59,7 +59,7 @@ type schemaItem struct {
type schemaIDName struct {
schemaVersion int64
id int64
- name string
+ name model.CIStr
tomb bool
}
@@ -123,7 +123,7 @@ type Data struct {
}
type tableInfoItem struct {
- dbName string
+ dbName model.CIStr
tableID int64
schemaVersion int64
tableInfo *model.TableInfo
@@ -227,8 +227,8 @@ func (isd *Data) addSpecialDB(di *model.DBInfo, tables *schemaTables) {
}
func (isd *Data) addDB(schemaVersion int64, dbInfo *model.DBInfo) {
- dbInfo.Tables = nil
- isd.schemaID2Name.Set(schemaIDName{schemaVersion: schemaVersion, id: dbInfo.ID, name: dbInfo.Name.O})
+ dbInfo.Deprecated.Tables = nil
+ isd.schemaID2Name.Set(schemaIDName{schemaVersion: schemaVersion, id: dbInfo.ID, name: dbInfo.Name})
isd.schemaMap.Set(schemaItem{schemaVersion: schemaVersion, dbInfo: dbInfo})
}
@@ -248,7 +248,7 @@ func (isd *Data) remove(item tableItem) {
func (isd *Data) deleteDB(dbInfo *model.DBInfo, schemaVersion int64) {
item := schemaItem{schemaVersion: schemaVersion, dbInfo: dbInfo, tomb: true}
isd.schemaMap.Set(item)
- isd.schemaID2Name.Set(schemaIDName{schemaVersion: schemaVersion, id: dbInfo.ID, name: dbInfo.Name.O, tomb: true})
+ isd.schemaID2Name.Set(schemaIDName{schemaVersion: schemaVersion, id: dbInfo.ID, name: dbInfo.Name, tomb: true})
}
// resetBeforeFullLoad is called before a full recreate operation within builder.InitWithDBInfos().
@@ -463,17 +463,17 @@ func compareByID(a, b tableItem) bool {
}
func compareByName(a, b tableItem) bool {
- if a.dbName < b.dbName {
+ if a.dbName.L < b.dbName.L {
return true
}
- if a.dbName > b.dbName {
+ if a.dbName.L > b.dbName.L {
return false
}
- if a.tableName < b.tableName {
+ if a.tableName.L < b.tableName.L {
return true
}
- if a.tableName > b.tableName {
+ if a.tableName.L > b.tableName.L {
return false
}
@@ -481,10 +481,10 @@ func compareByName(a, b tableItem) bool {
}
func compareTableInfoItem(a, b tableInfoItem) bool {
- if a.dbName < b.dbName {
+ if a.dbName.L < b.dbName.L {
return true
}
- if a.dbName > b.dbName {
+ if a.dbName.L > b.dbName.L {
return false
}
@@ -532,16 +532,18 @@ var _ InfoSchema = &infoschemaV2{}
type infoschemaV2 struct {
*infoSchema // in fact, we only need the infoSchemaMisc inside it, but the builder rely on it.
r autoid.Requirement
+ factory func() (pools.Resource, error)
ts uint64
*Data
}
// NewInfoSchemaV2 create infoschemaV2.
-func NewInfoSchemaV2(r autoid.Requirement, infoData *Data) infoschemaV2 {
+func NewInfoSchemaV2(r autoid.Requirement, factory func() (pools.Resource, error), infoData *Data) infoschemaV2 {
return infoschemaV2{
infoSchema: newInfoSchema(),
Data: infoData,
r: r,
+ factory: factory,
}
}
@@ -586,11 +588,10 @@ func (is *infoschemaV2) CloneAndUpdateTS(startTS uint64) *infoschemaV2 {
return &tmp
}
-func (is *infoschemaV2) TableByID(id int64) (val table.Table, ok bool) {
- return is.tableByID(id, false)
-}
-
-func (is *infoschemaV2) tableByID(id int64, noRefill bool) (val table.Table, ok bool) {
+// TableByID implements the InfoSchema interface.
+// As opposed to TableByName, TableByID will not refill cache when schema cache miss,
+// unless the caller changes the behavior by passing a context use WithRefillOption.
+func (is *infoschemaV2) TableByID(ctx context.Context, id int64) (val table.Table, ok bool) {
if !tableIDIsValid(id) {
return
}
@@ -609,30 +610,36 @@ func (is *infoschemaV2) tableByID(id int64, noRefill bool) (val table.Table, ok
}
if isTableVirtual(id) {
- if raw, exist := is.Data.specials.Load(itm.dbName); exist {
+ if raw, exist := is.Data.specials.Load(itm.dbName.L); exist {
schTbls := raw.(*schemaTables)
- val, ok = schTbls.tables[itm.tableName]
+ val, ok = schTbls.tables[itm.tableName.L]
return
}
return nil, false
}
+
+ refill := false
+ if opt := ctx.Value(refillOptionKey); opt != nil {
+ refill = opt.(bool)
+ }
+
// get cache with old key
oldKey := tableCacheKey{itm.tableID, itm.schemaVersion}
tbl, found = is.tableCache.Get(oldKey)
if found && tbl != nil {
- if !noRefill {
+ if refill {
is.tableCache.Set(key, tbl)
}
return tbl, true
}
// Maybe the table is evicted? need to reload.
- ret, err := loadTableInfo(context.Background(), is.r, is.Data, id, itm.dbID, is.ts, is.infoSchema.schemaMetaVersion)
+ ret, err := is.loadTableInfo(ctx, id, itm.dbID, is.ts, is.infoSchema.schemaMetaVersion)
if err != nil || ret == nil {
return nil, false
}
- if !noRefill {
+ if refill {
is.tableCache.Set(oldKey, ret)
}
return ret, true
@@ -646,7 +653,7 @@ func IsSpecialDB(dbName string) bool {
}
// EvictTable is exported for testing only.
-func (is *infoschemaV2) EvictTable(schema, tbl string) {
+func (is *infoschemaV2) EvictTable(schema, tbl model.CIStr) {
eq := func(a, b *tableItem) bool { return a.dbName == b.dbName && a.tableName == b.tableName }
itm, ok := search(is.byName, is.infoSchema.schemaMetaVersion, tableItem{dbName: schema, tableName: tbl, schemaVersion: math.MaxInt64}, eq)
if !ok {
@@ -664,7 +671,7 @@ type tableByNameHelper struct {
}
func (h *tableByNameHelper) onItem(item tableItem) bool {
- if item.dbName != h.end.dbName || item.tableName != h.end.tableName {
+ if item.dbName.L != h.end.dbName.L || item.tableName.L != h.end.tableName.L {
h.found = false
return false
}
@@ -678,6 +685,8 @@ func (h *tableByNameHelper) onItem(item tableItem) bool {
return true
}
+// TableByName implements the InfoSchema interface.
+// When schema cache miss, it will fetch the TableInfo from TikV and refill cache.
func (is *infoschemaV2) TableByName(ctx context.Context, schema, tbl model.CIStr) (t table.Table, err error) {
if IsSpecialDB(schema.L) {
if raw, ok := is.specials.Load(schema.L); ok {
@@ -692,7 +701,7 @@ func (is *infoschemaV2) TableByName(ctx context.Context, schema, tbl model.CIStr
start := time.Now()
var h tableByNameHelper
- h.end = tableItem{dbName: schema.L, tableName: tbl.L, schemaVersion: math.MaxInt64}
+ h.end = tableItem{dbName: schema, tableName: tbl, schemaVersion: math.MaxInt64}
h.schemaVersion = is.infoSchema.schemaMetaVersion
is.byName.Descend(h.end, h.onItem)
@@ -710,11 +719,19 @@ func (is *infoschemaV2) TableByName(ctx context.Context, schema, tbl model.CIStr
}
// Maybe the table is evicted? need to reload.
- ret, err := loadTableInfo(ctx, is.r, is.Data, itm.tableID, itm.dbID, is.ts, is.infoSchema.schemaMetaVersion)
+ ret, err := is.loadTableInfo(ctx, itm.tableID, itm.dbID, is.ts, is.infoSchema.schemaMetaVersion)
if err != nil {
return nil, errors.Trace(err)
}
- is.tableCache.Set(oldKey, ret)
+
+ refill := true
+ if opt := ctx.Value(refillOptionKey); opt != nil {
+ refill = opt.(bool)
+ }
+ if refill {
+ is.tableCache.Set(oldKey, ret)
+ }
+
metrics.TableByNameMissDuration.Observe(float64(time.Since(start)))
return ret, nil
}
@@ -727,7 +744,7 @@ func (is *infoschemaV2) TableInfoByName(schema, table model.CIStr) (*model.Table
// TableInfoByID implements InfoSchema.TableInfoByID
func (is *infoschemaV2) TableInfoByID(id int64) (*model.TableInfo, bool) {
- tbl, ok := is.TableByID(id)
+ tbl, ok := is.TableByID(context.Background(), id)
return getTableInfo(tbl), ok
}
@@ -795,33 +812,34 @@ func (is *infoschemaV2) SchemaSimpleTableInfos(ctx context.Context, schema model
return nil, nil // something wrong?
}
-retry:
- dbInfo, ok := is.SchemaByName(schema)
- if !ok {
+ // Ascend is much more difficult than Descend.
+ // So the data is taken out first and then dedup in Descend order.
+ var tableItems []tableItem
+ is.byName.Ascend(tableItem{dbName: schema}, func(item tableItem) bool {
+ if item.dbName.L != schema.L {
+ return false
+ }
+ if is.infoSchema.schemaMetaVersion >= item.schemaVersion {
+ tableItems = append(tableItems, item)
+ }
+ return true
+ })
+ if len(tableItems) == 0 {
return nil, nil
}
- snapshot := is.r.Store().GetSnapshot(kv.NewVersion(is.ts))
- // Using the KV timeout read feature to address the issue of potential DDL lease expiration when
- // the meta region leader is slow.
- snapshot.SetOption(kv.TiKVClientReadTimeout, uint64(3000)) // 3000ms.
- m := meta.NewSnapshotMeta(snapshot)
- tblInfos, err := m.ListSimpleTables(dbInfo.ID)
- if err != nil {
- if meta.ErrDBNotExists.Equal(err) {
- return nil, nil
- }
- // Flashback statement could cause such kind of error.
- // In theory that error should be handled in the lower layer, like client-go.
- // But it's not done, so we retry here.
- if strings.Contains(err.Error(), "in flashback progress") {
- select {
- case <-time.After(200 * time.Millisecond):
- case <-ctx.Done():
- return nil, ctx.Err()
+ tblInfos := make([]*model.TableNameInfo, 0, len(tableItems))
+ var curr *tableItem
+ for i := len(tableItems) - 1; i >= 0; i-- {
+ item := &tableItems[i]
+ if curr == nil || curr.tableName != tableItems[i].tableName {
+ curr = item
+ if !item.tomb {
+ tblInfos = append(tblInfos, &model.TableNameInfo{
+ ID: item.tableID,
+ Name: item.tableName,
+ })
}
- goto retry
}
- return nil, errors.Trace(err)
}
return tblInfos, nil
}
@@ -935,7 +953,7 @@ func (is *infoschemaV2) FindTableByPartitionID(partitionID int64) (table.Table,
return nil, nil, nil
}
- tbl, ok := is.TableByID(pi.tableID)
+ tbl, ok := is.TableByID(context.Background(), pi.tableID)
if !ok {
// something wrong?
return nil, nil, nil
@@ -983,7 +1001,7 @@ func (is *infoschemaV2) SchemaByID(id int64) (*model.DBInfo, bool) {
return st.dbInfo, true
}
var ok bool
- var name string
+ var name model.CIStr
is.Data.schemaID2Name.Descend(schemaIDName{
id: id,
schemaVersion: math.MaxInt64,
@@ -1004,10 +1022,10 @@ func (is *infoschemaV2) SchemaByID(id int64) (*model.DBInfo, bool) {
if !ok {
return nil, false
}
- return is.SchemaByName(model.NewCIStr(name))
+ return is.SchemaByName(name)
}
-func loadTableInfo(ctx context.Context, r autoid.Requirement, infoData *Data, tblID, dbID int64, ts uint64, schemaVersion int64) (table.Table, error) {
+func (is *infoschemaV2) loadTableInfo(ctx context.Context, tblID, dbID int64, ts uint64, schemaVersion int64) (table.Table, error) {
defer tracing.StartRegion(ctx, "infoschema.loadTableInfo").End()
failpoint.Inject("mockLoadTableInfoError", func(_ failpoint.Value) {
failpoint.Return(nil, errors.New("mockLoadTableInfoError"))
@@ -1015,7 +1033,7 @@ func loadTableInfo(ctx context.Context, r autoid.Requirement, infoData *Data, tb
// Try to avoid repeated concurrency loading.
res, err, _ := loadTableSF.Do(fmt.Sprintf("%d-%d-%d", dbID, tblID, schemaVersion), func() (any, error) {
retry:
- snapshot := r.Store().GetSnapshot(kv.NewVersion(ts))
+ snapshot := is.r.Store().GetSnapshot(kv.NewVersion(ts))
// Using the KV timeout read feature to address the issue of potential DDL lease expiration when
// the meta region leader is slow.
snapshot.SetOption(kv.TiKVClientReadTimeout, uint64(3000)) // 3000ms.
@@ -1031,8 +1049,7 @@ func loadTableInfo(ctx context.Context, r autoid.Requirement, infoData *Data, tb
goto retry
}
- // TODO load table panic!!!
- panic(err)
+ return nil, errors.Trace(err)
}
// table removed.
@@ -1045,9 +1062,8 @@ func loadTableInfo(ctx context.Context, r autoid.Requirement, infoData *Data, tb
ConvertCharsetCollateToLowerCaseIfNeed(tblInfo)
ConvertOldVersionUTF8ToUTF8MB4IfNeed(tblInfo)
- allocs := autoid.NewAllocatorsFromTblInfo(r, dbID, tblInfo)
- // TODO: handle cached table!!!
- ret, err := tables.TableFromMeta(allocs, tblInfo)
+ allocs := autoid.NewAllocatorsFromTblInfo(is.r, dbID, tblInfo)
+ ret, err := tableFromMeta(allocs, is.factory, tblInfo)
if err != nil {
return nil, errors.Trace(err)
}
@@ -1192,7 +1208,7 @@ func allocByID(b *Builder, id int64) (autoid.Allocators, bool) {
} else {
is = b.infoSchema
}
- tbl, ok := is.TableByID(id)
+ tbl, ok := is.TableByID(context.Background(), id)
if !ok {
return autoid.Allocators{}, false
}
@@ -1233,7 +1249,7 @@ func (b *Builder) applyDropSchemaV2(diff *model.SchemaDiff) []int64 {
return nil
}
- tableIDs := make([]int64, 0, len(di.Tables))
+ tableIDs := make([]int64, 0, len(di.Deprecated.Tables))
tables, err := b.infoschemaV2.SchemaTableInfos(context.Background(), di.Name)
terror.Log(err)
for _, tbl := range tables {
@@ -1254,13 +1270,14 @@ func (b *Builder) applyDropTableV2(diff *model.SchemaDiff, dbInfo *model.DBInfo,
delete(b.infoSchemaMisc.temporaryTableIDs, tableID)
}
- table, ok := b.infoschemaV2.TableByID(tableID)
+ table, ok := b.infoschemaV2.TableByID(context.Background(), tableID)
if !ok {
return nil
}
+ tblInfo := table.Meta()
// The old DBInfo still holds a reference to old table info, we need to remove it.
- b.infoSchema.deleteReferredForeignKeys(dbInfo.Name, table.Meta())
+ b.infoSchema.deleteReferredForeignKeys(dbInfo.Name, tblInfo)
if pi := table.Meta().GetPartitionInfo(); pi != nil {
for _, def := range pi.Definitions {
@@ -1269,12 +1286,13 @@ func (b *Builder) applyDropTableV2(diff *model.SchemaDiff, dbInfo *model.DBInfo,
}
b.infoData.remove(tableItem{
- dbName: dbInfo.Name.L,
+ dbName: dbInfo.Name,
dbID: dbInfo.ID,
- tableName: table.Meta().Name.L,
- tableID: table.Meta().ID,
+ tableName: tblInfo.Name,
+ tableID: tblInfo.ID,
schemaVersion: diff.Version,
})
+ affected = appendAffectedIDs(affected, tblInfo)
return affected
}
@@ -1435,10 +1453,10 @@ func (is *infoschemaV2) ListTablesWithSpecialAttribute(filter specialAttributeFi
}
if currDB == "" {
- currDB = item.dbName
+ currDB = item.dbName.L
res = tableInfoResult{DBName: item.dbName}
res.TableInfos = append(res.TableInfos, item.tableInfo)
- } else if currDB == item.dbName {
+ } else if currDB == item.dbName.L {
res.TableInfos = append(res.TableInfos, item.tableInfo)
} else {
ret = append(ret, res)
@@ -1452,3 +1470,14 @@ func (is *infoschemaV2) ListTablesWithSpecialAttribute(filter specialAttributeFi
}
return ret
}
+
+type refillOption struct{}
+
+var refillOptionKey refillOption
+
+// WithRefillOption controls the infoschema v2 cache refill operation.
+// By default, TableByID does not refill schema cache, and TableByName does.
+// The behavior can be changed by providing the context.Context.
+func WithRefillOption(ctx context.Context, refill bool) context.Context {
+ return context.WithValue(ctx, refillOptionKey, refill)
+}
diff --git a/pkg/infoschema/infoschema_v2_test.go b/pkg/infoschema/infoschema_v2_test.go
index 23f4948495195..3af31750b58f3 100644
--- a/pkg/infoschema/infoschema_v2_test.go
+++ b/pkg/infoschema/infoschema_v2_test.go
@@ -23,6 +23,7 @@ import (
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/meta"
"github.com/pingcap/tidb/pkg/parser/model"
+ "github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/stretchr/testify/require"
)
@@ -31,7 +32,7 @@ func TestV2Basic(t *testing.T) {
defer func() {
r.Store().Close()
}()
- is := NewInfoSchemaV2(r, NewData())
+ is := NewInfoSchemaV2(r, nil, NewData())
schemaName := model.NewCIStr("testDB")
tableName := model.NewCIStr("test")
@@ -41,7 +42,7 @@ func TestV2Basic(t *testing.T) {
internal.AddDB(t, r.Store(), dbInfo)
tblInfo := internal.MockTableInfo(t, r.Store(), tableName.O)
tblInfo.DBID = dbInfo.ID
- is.Data.add(tableItem{schemaName.L, dbInfo.ID, tableName.L, tblInfo.ID, 2, false}, internal.MockTable(t, r.Store(), tblInfo))
+ is.Data.add(tableItem{schemaName, dbInfo.ID, tableName, tblInfo.ID, 2, false}, internal.MockTable(t, r.Store(), tblInfo))
internal.AddTable(t, r.Store(), dbInfo, tblInfo)
is.base().schemaMetaVersion = 1
require.Equal(t, 1, len(is.AllSchemas()))
@@ -76,7 +77,7 @@ func TestV2Basic(t *testing.T) {
require.True(t, ok)
require.Equal(t, dbInfo, getDBInfo)
- getTableInfo, ok = is.TableByID(tblInfo.ID)
+ getTableInfo, ok = is.TableByID(context.Background(), tblInfo.ID)
require.True(t, ok)
require.NotNil(t, getTableInfo)
@@ -85,7 +86,7 @@ func TestV2Basic(t *testing.T) {
require.Same(t, gotTblInfo, getTableInfo.Meta())
// negative id should always be seen as not exists
- getTableInfo, ok = is.TableByID(-1)
+ getTableInfo, ok = is.TableByID(context.Background(), -1)
require.False(t, ok)
require.Nil(t, getTableInfo)
gotTblInfo, ok = is.TableInfoByID(-1)
@@ -127,7 +128,8 @@ func TestMisc(t *testing.T) {
r.Store().Close()
}()
- builder, err := NewBuilder(r, nil, NewData()).InitWithDBInfos(nil, nil, nil, 1)
+ builder := NewBuilder(r, nil, NewData(), variable.SchemaCacheSize.Load() > 0)
+ err := builder.InitWithDBInfos(nil, nil, nil, 1)
require.NoError(t, err)
is := builder.Build(math.MaxUint64)
require.Len(t, is.AllResourceGroups(), 0)
@@ -249,7 +251,8 @@ func TestBundles(t *testing.T) {
schemaName := model.NewCIStr("testDB")
tableName := model.NewCIStr("test")
- builder, err := NewBuilder(r, nil, NewData()).InitWithDBInfos(nil, nil, nil, 1)
+ builder := NewBuilder(r, nil, NewData(), variable.SchemaCacheSize.Load() > 0)
+ err := builder.InitWithDBInfos(nil, nil, nil, 1)
require.NoError(t, err)
is := builder.Build(math.MaxUint64)
require.Equal(t, 2, len(is.AllSchemas()))
diff --git a/pkg/infoschema/interface.go b/pkg/infoschema/interface.go
index 0b7701532ed94..71796c0329923 100644
--- a/pkg/infoschema/interface.go
+++ b/pkg/infoschema/interface.go
@@ -28,7 +28,7 @@ import (
type InfoSchema interface {
context.MetaOnlyInfoSchema
TableByName(ctx stdctx.Context, schema, table model.CIStr) (table.Table, error)
- TableByID(id int64) (table.Table, bool)
+ TableByID(ctx stdctx.Context, id int64) (table.Table, bool)
FindTableByPartitionID(partitionID int64) (table.Table, *model.DBInfo, *model.PartitionDefinition)
ListTablesWithSpecialAttribute(filter specialAttributeFilter) []tableInfoResult
base() *infoSchema
diff --git a/pkg/infoschema/internal/testkit.go b/pkg/infoschema/internal/testkit.go
index 1f468040e7e65..c4c60a064546e 100644
--- a/pkg/infoschema/internal/testkit.go
+++ b/pkg/infoschema/internal/testkit.go
@@ -150,12 +150,13 @@ func GenGlobalID(store kv.Storage) (int64, error) {
func MockDBInfo(t testing.TB, store kv.Storage, DBName string) *model.DBInfo {
id, err := GenGlobalID(store)
require.NoError(t, err)
- return &model.DBInfo{
- ID: id,
- Name: model.NewCIStr(DBName),
- Tables: []*model.TableInfo{},
- State: model.StatePublic,
+ dbInfo := &model.DBInfo{
+ ID: id,
+ Name: model.NewCIStr(DBName),
+ State: model.StatePublic,
}
+ dbInfo.Deprecated.Tables = []*model.TableInfo{}
+ return dbInfo
}
// MockTableInfo mock TableInfo for testing.
@@ -222,7 +223,7 @@ func MockPolicyRefInfo(t *testing.T, store kv.Storage, policyName string) *model
func AddTable(t testing.TB, store kv.Storage, dbInfo *model.DBInfo, tblInfo *model.TableInfo) {
ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL)
err := kv.RunInNewTxn(ctx, store, true, func(ctx context.Context, txn kv.Transaction) error {
- err := meta.NewMeta(txn).CreateTableOrView(dbInfo.ID, dbInfo.Name.O, tblInfo)
+ err := meta.NewMeta(txn).CreateTableOrView(dbInfo.ID, tblInfo)
require.NoError(t, err)
return errors.Trace(err)
})
@@ -244,7 +245,7 @@ func UpdateTable(t *testing.T, store kv.Storage, dbInfo *model.DBInfo, tblInfo *
func DropTable(t testing.TB, store kv.Storage, dbInfo *model.DBInfo, tblID int64, tblName string) {
ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL)
err := kv.RunInNewTxn(ctx, store, true, func(ctx context.Context, txn kv.Transaction) error {
- err := meta.NewMeta(txn).DropTableOrView(dbInfo.ID, dbInfo.Name.O, tblID, tblName)
+ err := meta.NewMeta(txn).DropTableOrView(dbInfo.ID, tblID)
require.NoError(t, err)
return errors.Trace(err)
})
@@ -267,7 +268,7 @@ func DropDB(t testing.TB, store kv.Storage, dbInfo *model.DBInfo) {
ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL)
err := kv.RunInNewTxn(ctx, store, true, func(ctx context.Context, txn kv.Transaction) error {
- err := meta.NewMeta(txn).DropDatabase(dbInfo.ID, dbInfo.Name.O)
+ err := meta.NewMeta(txn).DropDatabase(dbInfo.ID)
require.NoError(t, err)
return errors.Trace(err)
})
diff --git a/pkg/infoschema/metrics_schema.go b/pkg/infoschema/metrics_schema.go
index b8cbfd4992719..3efd2724f4b3e 100644
--- a/pkg/infoschema/metrics_schema.go
+++ b/pkg/infoschema/metrics_schema.go
@@ -21,6 +21,7 @@ import (
"strconv"
"strings"
+ "github.com/ngaut/pools"
"github.com/pingcap/errors"
"github.com/pingcap/tidb/pkg/meta/autoid"
"github.com/pingcap/tidb/pkg/parser/model"
@@ -57,9 +58,9 @@ func init() {
Name: model.NewCIStr(util.MetricSchemaName.O),
Charset: mysql.DefaultCharset,
Collate: mysql.DefaultCollationName,
- Tables: metricTables,
}
- RegisterVirtualTable(dbInfo, tableFromMeta)
+ dbInfo.Deprecated.Tables = metricTables
+ RegisterVirtualTable(dbInfo, tableFromMetaForMetricsTable)
}
// MetricTableDef is the metric table define.
@@ -146,7 +147,7 @@ type metricSchemaTable struct {
infoschemaTable
}
-func tableFromMeta(alloc autoid.Allocators, meta *model.TableInfo) (table.Table, error) {
+func tableFromMetaForMetricsTable(_ autoid.Allocators, _ func() (pools.Resource, error), meta *model.TableInfo) (table.Table, error) {
columns := make([]*table.Column, 0, len(meta.Columns))
for _, colInfo := range meta.Columns {
col := table.ToColumn(colInfo)
diff --git a/pkg/infoschema/perfschema/BUILD.bazel b/pkg/infoschema/perfschema/BUILD.bazel
index 0e300561f33f5..0c3b81094b9e8 100644
--- a/pkg/infoschema/perfschema/BUILD.bazel
+++ b/pkg/infoschema/perfschema/BUILD.bazel
@@ -26,6 +26,7 @@ go_library(
"//pkg/types",
"//pkg/util",
"//pkg/util/profile",
+ "@com_github_ngaut_pools//:pools",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_tikv_pd_client//http",
diff --git a/pkg/infoschema/perfschema/init.go b/pkg/infoschema/perfschema/init.go
index 8e4f3c4852e92..f10651e84ef47 100644
--- a/pkg/infoschema/perfschema/init.go
+++ b/pkg/infoschema/perfschema/init.go
@@ -68,8 +68,8 @@ func Init() {
Name: util.PerformanceSchemaName,
Charset: mysql.DefaultCharset,
Collate: mysql.DefaultCollationName,
- Tables: tbls,
}
+ dbInfo.Deprecated.Tables = tbls
infoschema.RegisterVirtualTable(dbInfo, tableFromMeta)
}
if expression.EvalSimpleAst != nil {
diff --git a/pkg/infoschema/perfschema/tables.go b/pkg/infoschema/perfschema/tables.go
index aea648135f298..d242bd7560a71 100644
--- a/pkg/infoschema/perfschema/tables.go
+++ b/pkg/infoschema/perfschema/tables.go
@@ -24,6 +24,7 @@ import (
"sync"
"time"
+ "github.com/ngaut/pools"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/infoschema"
@@ -129,7 +130,7 @@ func IsPredefinedTable(tableName string) bool {
return ok
}
-func tableFromMeta(allocs autoid.Allocators, meta *model.TableInfo) (table.Table, error) {
+func tableFromMeta(allocs autoid.Allocators, _ func() (pools.Resource, error), meta *model.TableInfo) (table.Table, error) {
if f, ok := pluginTable[meta.Name.L]; ok {
ret, err := f(allocs, meta)
return ret, err
diff --git a/pkg/infoschema/sieve.go b/pkg/infoschema/sieve.go
index 03ba1f8d88c4b..01100ebf359d6 100644
--- a/pkg/infoschema/sieve.go
+++ b/pkg/infoschema/sieve.go
@@ -19,6 +19,7 @@ import (
"context"
"sync"
+ "github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/infoschema/internal"
)
@@ -150,6 +151,10 @@ func (s *Sieve[K, V]) Set(key K, value V) {
}
func (s *Sieve[K, V]) Get(key K) (value V, ok bool) {
+ failpoint.Inject("skipGet", func() {
+ var v V
+ failpoint.Return(v, false)
+ })
s.mu.Lock()
defer s.mu.Unlock()
if e, ok := s.items[key]; ok {
diff --git a/pkg/infoschema/tables.go b/pkg/infoschema/tables.go
index 89e5f088fac40..8d7ce2cfef688 100644
--- a/pkg/infoschema/tables.go
+++ b/pkg/infoschema/tables.go
@@ -28,6 +28,7 @@ import (
"sync"
"time"
+ "github.com/ngaut/pools"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/kvproto/pkg/diagnosticspb"
@@ -855,7 +856,7 @@ var tableProcesslistCols = []columnInfo{
{name: "TxnStart", tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, deflt: ""},
{name: "RESOURCE_GROUP", tp: mysql.TypeVarchar, size: resourcegroup.MaxGroupNameLength, flag: mysql.NotNullFlag, deflt: ""},
{name: "SESSION_ALIAS", tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, deflt: ""},
- {name: "CURRENT_AFFECTED_ROWS", tp: mysql.TypeLonglong, size: 21, flag: mysql.UnsignedFlag},
+ {name: "ROWS_AFFECTED", tp: mysql.TypeLonglong, size: 21, flag: mysql.UnsignedFlag},
}
var tableTiDBIndexesCols = []columnInfo{
@@ -2352,13 +2353,13 @@ var tableNameToColumns = map[string][]columnInfo{
TableTiDBIndexUsage: tableTiDBIndexUsage,
}
-func createInfoSchemaTable(_ autoid.Allocators, meta *model.TableInfo) (table.Table, error) {
+func createInfoSchemaTable(_ autoid.Allocators, _ func() (pools.Resource, error), meta *model.TableInfo) (table.Table, error) {
columns := make([]*table.Column, len(meta.Columns))
for i, col := range meta.Columns {
columns[i] = table.ToColumn(col)
}
tp := table.VirtualTable
- if isClusterTableByName(util.InformationSchemaName.O, meta.Name.O) {
+ if IsClusterTableByName(util.InformationSchemaName.O, meta.Name.O) {
tp = table.ClusterTable
}
return &infoschemaTable{meta: meta, cols: columns, tp: tp}, nil
@@ -2426,17 +2427,17 @@ func (it *infoschemaTable) IndexPrefix() kv.Key {
}
// AddRecord implements table.Table AddRecord interface.
-func (it *infoschemaTable) AddRecord(ctx table.MutateContext, r []types.Datum, opts ...table.AddRecordOption) (recordID kv.Handle, err error) {
+func (it *infoschemaTable) AddRecord(ctx table.MutateContext, txn kv.Transaction, r []types.Datum, opts ...table.AddRecordOption) (recordID kv.Handle, err error) {
return nil, table.ErrUnsupportedOp
}
// RemoveRecord implements table.Table RemoveRecord interface.
-func (it *infoschemaTable) RemoveRecord(ctx table.MutateContext, h kv.Handle, r []types.Datum) error {
+func (it *infoschemaTable) RemoveRecord(ctx table.MutateContext, txn kv.Transaction, h kv.Handle, r []types.Datum) error {
return table.ErrUnsupportedOp
}
// UpdateRecord implements table.Table UpdateRecord interface.
-func (it *infoschemaTable) UpdateRecord(gctx context.Context, ctx table.MutateContext, h kv.Handle, oldData, newData []types.Datum, touched []bool) error {
+func (it *infoschemaTable) UpdateRecord(ctx table.MutateContext, txn kv.Transaction, h kv.Handle, oldData, newData []types.Datum, touched []bool, opts ...table.UpdateRecordOption) error {
return table.ErrUnsupportedOp
}
@@ -2519,17 +2520,17 @@ func (vt *VirtualTable) IndexPrefix() kv.Key {
}
// AddRecord implements table.Table AddRecord interface.
-func (vt *VirtualTable) AddRecord(ctx table.MutateContext, r []types.Datum, opts ...table.AddRecordOption) (recordID kv.Handle, err error) {
+func (vt *VirtualTable) AddRecord(ctx table.MutateContext, txn kv.Transaction, r []types.Datum, opts ...table.AddRecordOption) (recordID kv.Handle, err error) {
return nil, table.ErrUnsupportedOp
}
// RemoveRecord implements table.Table RemoveRecord interface.
-func (vt *VirtualTable) RemoveRecord(ctx table.MutateContext, h kv.Handle, r []types.Datum) error {
+func (vt *VirtualTable) RemoveRecord(ctx table.MutateContext, txn kv.Transaction, h kv.Handle, r []types.Datum) error {
return table.ErrUnsupportedOp
}
// UpdateRecord implements table.Table UpdateRecord interface.
-func (vt *VirtualTable) UpdateRecord(ctx context.Context, sctx table.MutateContext, h kv.Handle, oldData, newData []types.Datum, touched []bool) error {
+func (vt *VirtualTable) UpdateRecord(ctx table.MutateContext, txn kv.Transaction, h kv.Handle, oldData, newData []types.Datum, touched []bool, opts ...table.UpdateRecordOption) error {
return table.ErrUnsupportedOp
}
diff --git a/pkg/infoschema/test/clustertablestest/cluster_tables_test.go b/pkg/infoschema/test/clustertablestest/cluster_tables_test.go
index 40b6271d5a392..f53d2d8021f4f 100644
--- a/pkg/infoschema/test/clustertablestest/cluster_tables_test.go
+++ b/pkg/infoschema/test/clustertablestest/cluster_tables_test.go
@@ -403,7 +403,7 @@ func TestStmtSummaryEvictedCountTable(t *testing.T) {
err := tk.QueryToErr("select * from information_schema.CLUSTER_STATEMENTS_SUMMARY_EVICTED")
// This error is come from cop(TiDB) fetch from rpc server.
- require.EqualError(t, err, "other error: [planner:1227]Access denied; you need (at least one of) the PROCESS privilege(s) for this operation")
+ require.ErrorContains(t, err, "other error: [planner:1227]Access denied; you need (at least one of) the PROCESS privilege(s) for this operation")
require.NoError(t, tk.Session().Auth(&auth.UserIdentity{
Username: "testuser2",
@@ -944,6 +944,8 @@ func TestQuickBinding(t *testing.T) {
tk.MustExec("use test")
tk.MustExec(`create table t1 (pk int, a int, b int, c int, primary key(pk), key k_a(a), key k_bc(b, c))`)
tk.MustExec(`create table t2 (a int, b int, c int, key k_a(a), key k_bc(b, c))`) // no primary key
+ tk.MustExec(`create table t3 (a int, b int, c int, key k_a(a), key k_bc(b, c))`)
+ tk.MustExec(`create table t4 (a int, b int, c int, key k_a(a), key k_bc(b, c))`)
type testCase struct {
template string
@@ -971,32 +973,42 @@ func TestQuickBinding(t *testing.T) {
{`select a+b+? from (select /*+ stream_agg() */ count(*) as a from t1) tt1, (select /*+ hash_agg() */ count(*) as b from t1) tt2`, "stream_agg(@`sel_2`), use_index(@`sel_2` `test`.`t1` `k_a`), no_order_index(@`sel_2` `test`.`t1` `k_a`), agg_to_cop(@`sel_2`), hash_agg(@`sel_3`), use_index(@`sel_3` `test`.`t1` `k_a`), no_order_index(@`sel_3` `test`.`t1` `k_a`), agg_to_cop(@`sel_3`)", nil},
// 2-way hash joins
- {`select /*+ hash_join(t1, t2), use_index(t1), use_index(t2) */ t1.* from t1, t2 where t1.a=t2.a and t1.a= 0 && j < len(row) {
- value, err = table.CastValue(kvcodec.SessionCtx, row[j],
+ value, err = table.CastColumnValue(kvcodec.SessionCtx.GetExprCtx(), row[j],
ExtraHandleColumnInfo, false, false)
rowValue = value.GetInt64()
} else {
diff --git a/pkg/lightning/backend/kv/sql2kv_test.go b/pkg/lightning/backend/kv/sql2kv_test.go
index 5780be319a3a6..737eb86befade 100644
--- a/pkg/lightning/backend/kv/sql2kv_test.go
+++ b/pkg/lightning/backend/kv/sql2kv_test.go
@@ -71,7 +71,7 @@ type mockTable struct {
table.Table
}
-func (mockTable) AddRecord(ctx table.MutateContext, r []types.Datum, opts ...table.AddRecordOption) (recordID kv.Handle, err error) {
+func (mockTable) AddRecord(ctx table.MutateContext, txn kv.Transaction, r []types.Datum, opts ...table.AddRecordOption) (recordID kv.Handle, err error) {
return kv.IntHandle(-1), errors.New("mock error")
}
diff --git a/pkg/lightning/backend/local/duplicate.go b/pkg/lightning/backend/local/duplicate.go
index 9cb853cd3cac0..1de2af5597547 100644
--- a/pkg/lightning/backend/local/duplicate.go
+++ b/pkg/lightning/backend/local/duplicate.go
@@ -405,11 +405,10 @@ func (s *RemoteDupKVStream) Close() error {
return nil
}
-// DupeDetector provides methods to collect and decode duplicated KV pairs into row data. The results
-// are stored into the errorMgr.
-// this object can only be used once, either for local or remote deduplication.
-// TODO(lance6716): make it private.
-type DupeDetector struct {
+// dupeDetector provides methods to collect and decode duplicated KV pairs into
+// row data. The results are stored into the errorMgr. This object can only be
+// used once, either for local or remote deduplication.
+type dupeDetector struct {
tbl table.Table
tableName string
splitCli split.SplitClient
@@ -425,7 +424,7 @@ type DupeDetector struct {
taskType string
}
-// NewDupeDetector creates a new DupeDetector.
+// NewDupeDetector creates a new dupeDetector.
func NewDupeDetector(
tbl table.Table,
tableName string,
@@ -438,13 +437,13 @@ func NewDupeDetector(
logger log.Logger,
resourceGroupName string,
taskType string,
-) (*DupeDetector, error) {
+) (*dupeDetector, error) {
logger = logger.With(zap.String("tableName", tableName))
decoder, err := kv.NewTableKVDecoder(tbl, tableName, sessOpts, logger)
if err != nil {
return nil, errors.Trace(err)
}
- return &DupeDetector{
+ return &dupeDetector{
tbl: tbl,
tableName: tableName,
splitCli: splitCli,
@@ -461,12 +460,16 @@ func NewDupeDetector(
}
// HasDuplicate returns true if there are duplicated KV pairs.
-func (m *DupeDetector) HasDuplicate() bool {
+func (m *dupeDetector) HasDuplicate() bool {
return m.hasDupe.Load()
}
// RecordDataConflictError records data conflicts to errorMgr. The key received from stream must be a row key.
-func (m *DupeDetector) RecordDataConflictError(ctx context.Context, stream DupKVStream, algorithm config.DuplicateResolutionAlgorithm) error {
+func (m *dupeDetector) RecordDataConflictError(
+ ctx context.Context,
+ stream DupKVStream,
+ algorithm config.DuplicateResolutionAlgorithm,
+) error {
//nolint: errcheck
defer stream.Close()
var dataConflictInfos []errormanager.DataConflictInfo
@@ -515,7 +518,7 @@ func (m *DupeDetector) RecordDataConflictError(ctx context.Context, stream DupKV
return nil
}
-func (m *DupeDetector) saveIndexHandles(ctx context.Context, handles pendingIndexHandles) error {
+func (m *dupeDetector) saveIndexHandles(ctx context.Context, handles pendingIndexHandles) error {
snapshot := m.tikvCli.GetSnapshot(math.MaxUint64)
batchGetMap, err := snapshot.BatchGet(ctx, handles.rawHandles)
if err != nil {
@@ -540,7 +543,7 @@ func (m *DupeDetector) saveIndexHandles(ctx context.Context, handles pendingInde
}
// RecordIndexConflictError records index conflicts to errorMgr. The key received from stream must be an index key.
-func (m *DupeDetector) RecordIndexConflictError(ctx context.Context, stream DupKVStream, tableID int64, indexInfo *model.IndexInfo, algorithm config.DuplicateResolutionAlgorithm) error {
+func (m *dupeDetector) RecordIndexConflictError(ctx context.Context, stream DupKVStream, tableID int64, indexInfo *model.IndexInfo, algorithm config.DuplicateResolutionAlgorithm) error {
//nolint: errcheck
defer stream.Close()
indexHandles := makePendingIndexHandlesWithCapacity(0)
@@ -684,7 +687,7 @@ func ConvertToErrFoundConflictRecords(originalErr error, tbl table.Table) error
}
// BuildDuplicateTaskForTest is only used for test.
-var BuildDuplicateTaskForTest = func(m *DupeDetector) ([]dupTask, error) {
+var BuildDuplicateTaskForTest = func(m *dupeDetector) ([]dupTask, error) {
return m.buildDupTasks()
}
@@ -694,7 +697,7 @@ type dupTask struct {
indexInfo *model.IndexInfo
}
-func (m *DupeDetector) buildDupTasks() ([]dupTask, error) {
+func (m *dupeDetector) buildDupTasks() ([]dupTask, error) {
if m.indexID != 0 {
return m.buildIndexDupTasks()
}
@@ -739,7 +742,7 @@ func (m *DupeDetector) buildDupTasks() ([]dupTask, error) {
return tasks, nil
}
-func (m *DupeDetector) buildIndexDupTasks() ([]dupTask, error) {
+func (m *dupeDetector) buildIndexDupTasks() ([]dupTask, error) {
for _, indexInfo := range m.tbl.Meta().Indices {
if m.indexID != indexInfo.ID {
continue
@@ -767,7 +770,7 @@ func (m *DupeDetector) buildIndexDupTasks() ([]dupTask, error) {
return nil, nil
}
-func (m *DupeDetector) splitLocalDupTaskByKeys(
+func (m *dupeDetector) splitLocalDupTaskByKeys(
task dupTask,
dupDB *pebble.DB,
keyAdapter common.KeyAdapter,
@@ -793,7 +796,7 @@ func (m *DupeDetector) splitLocalDupTaskByKeys(
return newDupTasks, nil
}
-func (m *DupeDetector) buildLocalDupTasks(dupDB *pebble.DB, keyAdapter common.KeyAdapter) ([]dupTask, error) {
+func (m *dupeDetector) buildLocalDupTasks(dupDB *pebble.DB, keyAdapter common.KeyAdapter) ([]dupTask, error) {
tasks, err := m.buildDupTasks()
if err != nil {
return nil, errors.Trace(err)
@@ -812,7 +815,7 @@ func (m *DupeDetector) buildLocalDupTasks(dupDB *pebble.DB, keyAdapter common.Ke
}
// CollectDuplicateRowsFromDupDB collects duplicates from the duplicate DB and records all duplicate row info into errorMgr.
-func (m *DupeDetector) CollectDuplicateRowsFromDupDB(ctx context.Context, dupDB *pebble.DB, keyAdapter common.KeyAdapter, algorithm config.DuplicateResolutionAlgorithm) error {
+func (m *dupeDetector) CollectDuplicateRowsFromDupDB(ctx context.Context, dupDB *pebble.DB, keyAdapter common.KeyAdapter, algorithm config.DuplicateResolutionAlgorithm) error {
tasks, err := m.buildLocalDupTasks(dupDB, keyAdapter)
if err != nil {
return errors.Trace(err)
@@ -848,7 +851,7 @@ func (m *DupeDetector) CollectDuplicateRowsFromDupDB(ctx context.Context, dupDB
return errors.Trace(g.Wait())
}
-func (m *DupeDetector) splitKeyRangeByRegions(
+func (m *dupeDetector) splitKeyRangeByRegions(
ctx context.Context, keyRange tidbkv.KeyRange,
) ([]*split.RegionInfo, []tidbkv.KeyRange, error) {
rawStartKey := codec.EncodeBytes(nil, keyRange.StartKey)
@@ -891,7 +894,7 @@ func (m *DupeDetector) splitKeyRangeByRegions(
return regions, keyRanges, nil
}
-func (m *DupeDetector) processRemoteDupTaskOnce(
+func (m *dupeDetector) processRemoteDupTaskOnce(
ctx context.Context,
task dupTask,
logger log.Logger,
@@ -969,7 +972,7 @@ func (m *DupeDetector) processRemoteDupTaskOnce(
// processRemoteDupTask processes a remoteDupTask. A task contains a key range.
// A key range is associated with multiple regions. processRemoteDupTask tries
// to collect duplicates from each region.
-func (m *DupeDetector) processRemoteDupTask(
+func (m *dupeDetector) processRemoteDupTask(
ctx context.Context,
task dupTask,
logger log.Logger,
@@ -1017,7 +1020,7 @@ func (m *DupeDetector) processRemoteDupTask(
// collectDuplicateRowsFromTiKV collects duplicates from the remote TiKV and
// records all duplicate row info into errorMgr.
-func (m *DupeDetector) collectDuplicateRowsFromTiKV(
+func (m *dupeDetector) collectDuplicateRowsFromTiKV(
ctx context.Context,
importClientFactory ImportClientFactory,
algorithm config.DuplicateResolutionAlgorithm,
diff --git a/pkg/lightning/backend/local/duplicate_test.go b/pkg/lightning/backend/local/duplicate_test.go
index 8a42e7f6f69c4..fc17ec9293308 100644
--- a/pkg/lightning/backend/local/duplicate_test.go
+++ b/pkg/lightning/backend/local/duplicate_test.go
@@ -94,7 +94,7 @@ func buildTableForTestConvertToErrFoundConflictRecords(t *testing.T, node []ast.
Logger: log.L(),
})
require.NoError(t, err)
- encoder.SessionCtx.GetSessionVars().RowEncoder.Enable = true
+ encoder.SessionCtx.GetTableCtx().GetRowEncodingConfig().RowEncoder.Enable = true
data1 := []types.Datum{
types.NewIntDatum(1),
diff --git a/pkg/lightning/backend/tidb/BUILD.bazel b/pkg/lightning/backend/tidb/BUILD.bazel
index 7fc382b7b46ee..d65342fd2a725 100644
--- a/pkg/lightning/backend/tidb/BUILD.bazel
+++ b/pkg/lightning/backend/tidb/BUILD.bazel
@@ -18,7 +18,6 @@ go_library(
"//pkg/lightning/verification",
"//pkg/parser/model",
"//pkg/parser/mysql",
- "//pkg/sessionctx",
"//pkg/table",
"//pkg/types",
"//pkg/util/dbutil",
diff --git a/pkg/lightning/backend/tidb/tidb.go b/pkg/lightning/backend/tidb/tidb.go
index 186e17d9d6d06..2c70cc1df82b9 100644
--- a/pkg/lightning/backend/tidb/tidb.go
+++ b/pkg/lightning/backend/tidb/tidb.go
@@ -39,7 +39,6 @@ import (
"github.com/pingcap/tidb/pkg/lightning/verification"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
- "github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/dbutil"
@@ -83,7 +82,6 @@ func (rows tidbRows) MarshalLogArray(encoder zapcore.ArrayEncoder) error {
type tidbEncoder struct {
mode mysql.SQLMode
tbl table.Table
- se sessionctx.Context
// the index of table columns for each data field.
// index == len(table.columns) means this field is `_tidb_rowid`
columnIdx []int
@@ -105,17 +103,10 @@ func NewEncodingBuilder() encode.EncodingBuilder {
// NewEncoder creates a KV encoder.
// It implements the `backend.EncodingBuilder` interface.
-func (*encodingBuilder) NewEncoder(ctx context.Context, config *encode.EncodingConfig) (encode.Encoder, error) {
- se := kv.NewSessionCtx(&config.SessionOptions, log.FromContext(ctx))
- if config.SQLMode.HasStrictMode() {
- se.GetSessionVars().SkipUTF8Check = false
- se.GetSessionVars().SkipASCIICheck = false
- }
-
+func (*encodingBuilder) NewEncoder(_ context.Context, config *encode.EncodingConfig) (encode.Encoder, error) {
return &tidbEncoder{
mode: config.SQLMode,
tbl: config.Table,
- se: se,
path: config.Path,
logger: config.Logger,
}, nil
diff --git a/pkg/lightning/checkpoints/BUILD.bazel b/pkg/lightning/checkpoints/BUILD.bazel
index 90c521225490a..d7c7882ee2042 100644
--- a/pkg/lightning/checkpoints/BUILD.bazel
+++ b/pkg/lightning/checkpoints/BUILD.bazel
@@ -36,7 +36,7 @@ go_test(
embed = [":checkpoints"],
flaky = True,
race = "on",
- shard_count = 23,
+ shard_count = 24,
deps = [
"//br/pkg/version/build",
"//pkg/lightning/checkpoints/checkpointspb",
diff --git a/pkg/lightning/checkpoints/checkpoints.go b/pkg/lightning/checkpoints/checkpoints.go
index 186ee9b90ffe3..81497cc4dc5c0 100644
--- a/pkg/lightning/checkpoints/checkpoints.go
+++ b/pkg/lightning/checkpoints/checkpoints.go
@@ -800,9 +800,13 @@ func (cpdb *MySQLCheckpointsDB) Initialize(ctx context.Context, cfg *config.Conf
for _, db := range dbInfo {
for _, table := range db.Tables {
tableName := common.UniqueTable(db.Name, table.Name)
- tableInfo, err := json.Marshal(table.Desired)
- if err != nil {
- return errors.Trace(err)
+ tableInfo := []byte("")
+ if cfg.TikvImporter.AddIndexBySQL {
+ // see comments in FileCheckpointsDB.Initialize
+ tableInfo, err = json.Marshal(table.Desired)
+ if err != nil {
+ return errors.Trace(err)
+ }
}
_, err = stmt.ExecContext(c, cfg.TaskID, tableName, CheckpointStatusLoaded, table.ID, tableInfo)
if err != nil {
@@ -933,8 +937,10 @@ func (cpdb *MySQLCheckpointsDB) Get(ctx context.Context, tableName string) (*Tab
return errors.NotFoundf("checkpoint for table %s", tableName)
}
}
- if err := json.Unmarshal(rawTableInfo, &cp.TableInfo); err != nil {
- return errors.Trace(err)
+ if len(rawTableInfo) > 0 {
+ if err := json.Unmarshal(rawTableInfo, &cp.TableInfo); err != nil {
+ return errors.Trace(err)
+ }
}
cp.Checksum = verify.MakeKVChecksum(bytes, kvs, checksum)
cp.Status = CheckpointStatus(status)
@@ -1246,13 +1252,22 @@ func (cpdb *FileCheckpointsDB) Initialize(_ context.Context, cfg *config.Config,
cpdb.checkpoints.Checkpoints = make(map[string]*checkpointspb.TableCheckpointModel)
}
+ var err error
for _, db := range dbInfo {
for _, table := range db.Tables {
tableName := common.UniqueTable(db.Name, table.Name)
if _, ok := cpdb.checkpoints.Checkpoints[tableName]; !ok {
- tableInfo, err := json.Marshal(table.Desired)
- if err != nil {
- return errors.Trace(err)
+ var tableInfo []byte
+ if cfg.TikvImporter.AddIndexBySQL {
+ // tableInfo is quite large in most case, when importing many
+ // small tables, writing tableInfo will make checkpoint file
+ // vary large, and save checkpoint will become bottleneck, so
+ // we only store table info if we are going to add index by SQL
+ // where it's required.
+ tableInfo, err = json.Marshal(table.Desired)
+ if err != nil {
+ return errors.Trace(err)
+ }
}
cpdb.checkpoints.Checkpoints[tableName] = &checkpointspb.TableCheckpointModel{
Status: uint32(CheckpointStatusLoaded),
@@ -1308,8 +1323,10 @@ func (cpdb *FileCheckpointsDB) Get(_ context.Context, tableName string) (*TableC
}
var tableInfo *model.TableInfo
- if err := json.Unmarshal(tableModel.TableInfo, &tableInfo); err != nil {
- return nil, errors.Trace(err)
+ if len(tableModel.TableInfo) > 0 {
+ if err := json.Unmarshal(tableModel.TableInfo, &tableInfo); err != nil {
+ return nil, errors.Trace(err)
+ }
}
cp := &TableCheckpoint{
diff --git a/pkg/lightning/checkpoints/checkpoints_file_test.go b/pkg/lightning/checkpoints/checkpoints_file_test.go
index df2b84f50a3e4..7213dc9aa3e09 100644
--- a/pkg/lightning/checkpoints/checkpoints_file_test.go
+++ b/pkg/lightning/checkpoints/checkpoints_file_test.go
@@ -41,7 +41,7 @@ func newTestConfig() *config.Config {
return cfg
}
-func newFileCheckpointsDB(t *testing.T) *checkpoints.FileCheckpointsDB {
+func newFileCheckpointsDB(t *testing.T, addIndexBySQL bool) *checkpoints.FileCheckpointsDB {
dir := t.TempDir()
ctx := context.Background()
cpdb, err := checkpoints.NewFileCheckpointsDB(ctx, filepath.Join(dir, "cp.pb"))
@@ -49,6 +49,7 @@ func newFileCheckpointsDB(t *testing.T) *checkpoints.FileCheckpointsDB {
// 2. initialize with checkpoint data.
cfg := newTestConfig()
+ cfg.TikvImporter.AddIndexBySQL = addIndexBySQL
err = cpdb.Initialize(ctx, cfg, map[string]*checkpoints.TidbDBInfo{
"db1": {
Name: "db1",
@@ -164,13 +165,7 @@ func setInvalidStatus(cpdb *checkpoints.FileCheckpointsDB) {
func TestGet(t *testing.T) {
ctx := context.Background()
- cpdb := newFileCheckpointsDB(t)
-
- // 5. get back the checkpoints
-
- cp, err := cpdb.Get(ctx, "`db1`.`t2`")
- require.NoError(t, err)
- expect := &checkpoints.TableCheckpoint{
+ expectT2 := &checkpoints.TableCheckpoint{
Status: checkpoints.CheckpointStatusAllWritten,
AllocBase: 132861,
Checksum: verification.MakeKVChecksum(4492, 686, 486070148910),
@@ -203,11 +198,18 @@ func TestGet(t *testing.T) {
},
},
}
- require.Equal(t, expect, cp)
- cp, err = cpdb.Get(ctx, "`db2`.`t3`")
- require.NoError(t, err)
- expect = &checkpoints.TableCheckpoint{
+ expectT3 := &checkpoints.TableCheckpoint{
+ Status: checkpoints.CheckpointStatusLoaded,
+ Engines: map[int32]*checkpoints.EngineCheckpoint{
+ -1: {
+ Status: checkpoints.CheckpointStatusLoaded,
+ Chunks: []*checkpoints.ChunkCheckpoint{},
+ },
+ },
+ }
+
+ expectT3AddIndexBySQL := &checkpoints.TableCheckpoint{
Status: checkpoints.CheckpointStatusLoaded,
Engines: map[int32]*checkpoints.EngineCheckpoint{
-1: {
@@ -219,16 +221,41 @@ func TestGet(t *testing.T) {
Name: model.NewCIStr("t3"),
},
}
- require.Equal(t, expect, cp)
- cp, err = cpdb.Get(ctx, "`db3`.`not-exists`")
- require.Nil(t, cp)
- require.True(t, errors.IsNotFound(err))
+ t.Run("normal", func(t *testing.T) {
+ cpdb := newFileCheckpointsDB(t, false)
+ cp, err := cpdb.Get(ctx, "`db1`.`t2`")
+ require.NoError(t, err)
+ require.Equal(t, expectT2, cp)
+
+ cp, err = cpdb.Get(ctx, "`db2`.`t3`")
+ require.NoError(t, err)
+ require.Equal(t, expectT3, cp)
+
+ cp, err = cpdb.Get(ctx, "`db3`.`not-exists`")
+ require.Nil(t, cp)
+ require.True(t, errors.IsNotFound(err))
+ })
+
+ t.Run("add-index-by-sql", func(t *testing.T) {
+ cpdb := newFileCheckpointsDB(t, true)
+ cp, err := cpdb.Get(ctx, "`db1`.`t2`")
+ require.NoError(t, err)
+ require.Equal(t, expectT2, cp)
+
+ cp, err = cpdb.Get(ctx, "`db2`.`t3`")
+ require.NoError(t, err)
+ require.Equal(t, expectT3AddIndexBySQL, cp)
+
+ cp, err = cpdb.Get(ctx, "`db3`.`not-exists`")
+ require.Nil(t, cp)
+ require.True(t, errors.IsNotFound(err))
+ })
}
func TestRemoveAllCheckpoints(t *testing.T) {
ctx := context.Background()
- cpdb := newFileCheckpointsDB(t)
+ cpdb := newFileCheckpointsDB(t, false)
err := cpdb.RemoveCheckpoint(ctx, "all")
require.NoError(t, err)
@@ -244,7 +271,7 @@ func TestRemoveAllCheckpoints(t *testing.T) {
func TestRemoveOneCheckpoint(t *testing.T) {
ctx := context.Background()
- cpdb := newFileCheckpointsDB(t)
+ cpdb := newFileCheckpointsDB(t, false)
err := cpdb.RemoveCheckpoint(ctx, "`db1`.`t2`")
require.NoError(t, err)
@@ -260,7 +287,7 @@ func TestRemoveOneCheckpoint(t *testing.T) {
func TestIgnoreAllErrorCheckpoints(t *testing.T) {
ctx := context.Background()
- cpdb := newFileCheckpointsDB(t)
+ cpdb := newFileCheckpointsDB(t, false)
setInvalidStatus(cpdb)
@@ -278,7 +305,7 @@ func TestIgnoreAllErrorCheckpoints(t *testing.T) {
func TestIgnoreOneErrorCheckpoints(t *testing.T) {
ctx := context.Background()
- cpdb := newFileCheckpointsDB(t)
+ cpdb := newFileCheckpointsDB(t, false)
setInvalidStatus(cpdb)
@@ -296,7 +323,7 @@ func TestIgnoreOneErrorCheckpoints(t *testing.T) {
func TestDestroyAllErrorCheckpoints(t *testing.T) {
ctx := context.Background()
- cpdb := newFileCheckpointsDB(t)
+ cpdb := newFileCheckpointsDB(t, false)
setInvalidStatus(cpdb)
@@ -328,7 +355,7 @@ func TestDestroyAllErrorCheckpoints(t *testing.T) {
func TestDestroyOneErrorCheckpoint(t *testing.T) {
ctx := context.Background()
- cpdb := newFileCheckpointsDB(t)
+ cpdb := newFileCheckpointsDB(t, false)
setInvalidStatus(cpdb)
diff --git a/pkg/lightning/checkpoints/checkpoints_sql_test.go b/pkg/lightning/checkpoints/checkpoints_sql_test.go
index cbd6b812ba8d6..e3e7081e768b0 100644
--- a/pkg/lightning/checkpoints/checkpoints_sql_test.go
+++ b/pkg/lightning/checkpoints/checkpoints_sql_test.go
@@ -78,6 +78,114 @@ func TestNormalOperations(t *testing.T) {
ctx := context.Background()
s := newCPSQLSuite(t)
cpdb := s.cpdb
+ s.mock.ExpectBegin()
+ initializeStmt := s.mock.ExpectPrepare(
+ "REPLACE INTO `mock-schema`\\.`task_v\\d+`")
+ initializeStmt.ExpectExec().
+ WithArgs(123, "/data", "local", "127.0.0.1:8287", "127.0.0.1", 4000, "127.0.0.1:2379", "/tmp/sorted-kv", build.ReleaseVersion).
+ WillReturnResult(sqlmock.NewResult(6, 1))
+ initializeStmt = s.mock.
+ ExpectPrepare("INSERT INTO `mock-schema`\\.`table_v\\d+`")
+ initializeStmt.ExpectExec().
+ WithArgs(123, "`db1`.`t2`", sqlmock.AnyArg(), int64(2), []byte("")).
+ WillReturnResult(sqlmock.NewResult(8, 1))
+ s.mock.ExpectCommit()
+
+ s.mock.MatchExpectationsInOrder(false)
+ cfg := newTestConfig()
+ err := cpdb.Initialize(ctx, cfg, map[string]*checkpoints.TidbDBInfo{
+ "db1": {
+ Name: "db1",
+ Tables: map[string]*checkpoints.TidbTableInfo{
+ "t2": {
+ Name: "t2",
+ ID: 2,
+ Desired: &model.TableInfo{
+ Name: model.NewCIStr("t2"),
+ },
+ },
+ },
+ },
+ })
+ s.mock.MatchExpectationsInOrder(true)
+ require.NoError(t, err)
+ require.Nil(t, s.mock.ExpectationsWereMet())
+
+ s.mock.ExpectBegin()
+ s.mock.
+ ExpectQuery("SELECT .+ FROM `mock-schema`\\.`engine_v\\d+`").
+ WithArgs("`db1`.`t2`").
+ WillReturnRows(
+ sqlmock.NewRows([]string{"engine_id", "status"}).
+ AddRow(0, 120).
+ AddRow(-1, 30),
+ )
+ s.mock.
+ ExpectQuery("SELECT (?s:.+) FROM `mock-schema`\\.`chunk_v\\d+`").
+ WithArgs("`db1`.`t2`").
+ WillReturnRows(
+ sqlmock.NewRows([]string{
+ "engine_id", "path", "offset", "type", "compression", "sort_key", "file_size", "columns",
+ "pos", "end_offset", "prev_rowid_max", "rowid_max",
+ "kvc_bytes", "kvc_kvs", "kvc_checksum", "unix_timestamp(create_time)",
+ }).
+ AddRow(
+ 0, "/tmp/path/1.sql", 0, mydump.SourceTypeSQL, 0, "", 123, "[]",
+ 55904, 102400, 681, 5000,
+ 4491, 586, 486070148917, 1234567894,
+ ),
+ )
+ s.mock.
+ ExpectQuery("SELECT .+ FROM `mock-schema`\\.`table_v\\d+`").
+ WithArgs("`db1`.`t2`").
+ WillReturnRows(
+ sqlmock.NewRows([]string{"status", "alloc_base", "table_id", "table_info", "kv_bytes", "kv_kvs", "kv_checksum"}).
+ AddRow(60, 132861, int64(2), nil, uint64(4492), uint64(686), uint64(486070148910)),
+ )
+ s.mock.ExpectCommit()
+
+ cp, err := cpdb.Get(ctx, "`db1`.`t2`")
+ require.Nil(t, err)
+ require.Equal(t, &checkpoints.TableCheckpoint{
+ Status: checkpoints.CheckpointStatusAllWritten,
+ AllocBase: 132861,
+ TableID: int64(2),
+ TableInfo: nil,
+ Engines: map[int32]*checkpoints.EngineCheckpoint{
+ -1: {Status: checkpoints.CheckpointStatusLoaded},
+ 0: {
+ Status: checkpoints.CheckpointStatusImported,
+ Chunks: []*checkpoints.ChunkCheckpoint{{
+ Key: checkpoints.ChunkCheckpointKey{
+ Path: "/tmp/path/1.sql",
+ Offset: 0,
+ },
+ FileMeta: mydump.SourceFileMeta{
+ Path: "/tmp/path/1.sql",
+ Type: mydump.SourceTypeSQL,
+ FileSize: 123,
+ },
+ ColumnPermutation: []int{},
+ Chunk: mydump.Chunk{
+ Offset: 55904,
+ EndOffset: 102400,
+ PrevRowIDMax: 681,
+ RowIDMax: 5000,
+ },
+ Checksum: verification.MakeKVChecksum(4491, 586, 486070148917),
+ Timestamp: 1234567894,
+ }},
+ },
+ },
+ Checksum: verification.MakeKVChecksum(4492, 686, 486070148910),
+ }, cp)
+ require.Nil(t, s.mock.ExpectationsWereMet())
+}
+
+func TestNormalOperationsWithAddIndexBySQL(t *testing.T) {
+ ctx := context.Background()
+ s := newCPSQLSuite(t)
+ cpdb := s.cpdb
// 2. initialize with checkpoint data.
@@ -115,6 +223,7 @@ func TestNormalOperations(t *testing.T) {
s.mock.MatchExpectationsInOrder(false)
cfg := newTestConfig()
+ cfg.TikvImporter.AddIndexBySQL = true
err = cpdb.Initialize(ctx, cfg, map[string]*checkpoints.TidbDBInfo{
"db1": {
Name: "db1",
diff --git a/pkg/lightning/common/common_test.go b/pkg/lightning/common/common_test.go
index 7da0879f3f771..fdedbebfcabcc 100644
--- a/pkg/lightning/common/common_test.go
+++ b/pkg/lightning/common/common_test.go
@@ -51,7 +51,7 @@ func newTableInfo(t *testing.T,
if err := m.CreateDatabase(&model.DBInfo{ID: dbID}); err != nil && !errors.ErrorEqual(err, meta.ErrDBExists) {
return err
}
- return m.CreateTableOrView(dbID, "", tableInfo)
+ return m.CreateTableOrView(dbID, tableInfo)
})
require.NoError(t, err)
return tableInfo
diff --git a/pkg/lightning/config/config.go b/pkg/lightning/config/config.go
index b131e0180e0f7..f7e07f256a09e 100644
--- a/pkg/lightning/config/config.go
+++ b/pkg/lightning/config/config.go
@@ -1633,13 +1633,3 @@ func (cfg *Config) Adjust(ctx context.Context) error {
}
return cfg.Conflict.adjust(&cfg.TikvImporter)
}
-
-// AdjustForDDL acts like Adjust, but DDL will not use some functionalities so
-// those members are skipped in adjusting.
-func (cfg *Config) AdjustForDDL() error {
- if err := cfg.TikvImporter.adjust(); err != nil {
- return err
- }
- cfg.App.adjust(&cfg.TikvImporter)
- return nil
-}
diff --git a/pkg/lightning/errormanager/errormanager.go b/pkg/lightning/errormanager/errormanager.go
index 21a77131d8f85..7c83d7e486a6a 100644
--- a/pkg/lightning/errormanager/errormanager.go
+++ b/pkg/lightning/errormanager/errormanager.go
@@ -611,7 +611,7 @@ func (em *ErrorManager) ReplaceConflictKeys(
if err != nil {
return errors.Trace(err)
}
- decodedData, _, err := tables.DecodeRawRowData(encoder.SessionCtx,
+ decodedData, _, err := tables.DecodeRawRowData(encoder.SessionCtx.GetExprCtx(),
tbl.Meta(), overwrittenHandle, tbl.Cols(), overwritten)
if err != nil {
return errors.Trace(err)
@@ -791,7 +791,7 @@ func (em *ErrorManager) ReplaceConflictKeys(
if err != nil {
return errors.Trace(err)
}
- decodedData, _, err := tables.DecodeRawRowData(encoder.SessionCtx,
+ decodedData, _, err := tables.DecodeRawRowData(encoder.SessionCtx.GetExprCtx(),
tbl.Meta(), handle, tbl.Cols(), latestValue)
if err != nil {
return errors.Trace(err)
@@ -820,7 +820,7 @@ func (em *ErrorManager) ReplaceConflictKeys(
if err != nil {
return errors.Trace(err)
}
- decodedData, _, err := tables.DecodeRawRowData(encoder.SessionCtx,
+ decodedData, _, err := tables.DecodeRawRowData(encoder.SessionCtx.GetExprCtx(),
tbl.Meta(), handle, tbl.Cols(), rawValue)
if err != nil {
return errors.Trace(err)
diff --git a/pkg/lightning/errormanager/errormanager_test.go b/pkg/lightning/errormanager/errormanager_test.go
index e611f24d3b1da..ed353b3629c55 100644
--- a/pkg/lightning/errormanager/errormanager_test.go
+++ b/pkg/lightning/errormanager/errormanager_test.go
@@ -231,7 +231,7 @@ func TestReplaceConflictOneKey(t *testing.T) {
Logger: log.L(),
})
require.NoError(t, err)
- encoder.SessionCtx.GetSessionVars().RowEncoder.Enable = true
+ encoder.SessionCtx.GetTableCtx().GetRowEncodingConfig().RowEncoder.Enable = true
data1 := []types.Datum{
types.NewIntDatum(1),
@@ -420,7 +420,7 @@ func TestReplaceConflictOneUniqueKey(t *testing.T) {
Logger: log.L(),
})
require.NoError(t, err)
- encoder.SessionCtx.GetSessionVars().RowEncoder.Enable = true
+ encoder.SessionCtx.GetTableCtx().GetRowEncodingConfig().RowEncoder.Enable = true
data1 := []types.Datum{
types.NewIntDatum(1),
diff --git a/pkg/lightning/errormanager/resolveconflict_test.go b/pkg/lightning/errormanager/resolveconflict_test.go
index 117420e84a90c..a76c7418aa54c 100644
--- a/pkg/lightning/errormanager/resolveconflict_test.go
+++ b/pkg/lightning/errormanager/resolveconflict_test.go
@@ -67,7 +67,7 @@ func TestReplaceConflictMultipleKeysNonclusteredPk(t *testing.T) {
Logger: log.L(),
})
require.NoError(t, err)
- encoder.SessionCtx.GetSessionVars().RowEncoder.Enable = true
+ encoder.SessionCtx.GetTableCtx().GetRowEncodingConfig().RowEncoder.Enable = true
data1 := []types.Datum{
types.NewIntDatum(1),
@@ -288,7 +288,7 @@ func TestReplaceConflictOneKeyNonclusteredPk(t *testing.T) {
Logger: log.L(),
})
require.NoError(t, err)
- encoder.SessionCtx.GetSessionVars().RowEncoder.Enable = true
+ encoder.SessionCtx.GetTableCtx().GetRowEncodingConfig().RowEncoder.Enable = true
data1 := []types.Datum{
types.NewIntDatum(1),
@@ -456,7 +456,7 @@ func TestReplaceConflictOneUniqueKeyNonclusteredPk(t *testing.T) {
Logger: log.L(),
})
require.NoError(t, err)
- encoder.SessionCtx.GetSessionVars().RowEncoder.Enable = true
+ encoder.SessionCtx.GetTableCtx().GetRowEncodingConfig().RowEncoder.Enable = true
data1 := []types.Datum{
types.NewIntDatum(1),
@@ -662,7 +662,7 @@ func TestReplaceConflictOneUniqueKeyNonclusteredVarcharPk(t *testing.T) {
Logger: log.L(),
})
require.NoError(t, err)
- encoder.SessionCtx.GetSessionVars().RowEncoder.Enable = true
+ encoder.SessionCtx.GetTableCtx().GetRowEncodingConfig().RowEncoder.Enable = true
data1 := []types.Datum{
types.NewStringDatum("x"),
diff --git a/pkg/meta/BUILD.bazel b/pkg/meta/BUILD.bazel
index 141ee455b2035..b1a0bff60e885 100644
--- a/pkg/meta/BUILD.bazel
+++ b/pkg/meta/BUILD.bazel
@@ -18,6 +18,7 @@ go_library(
"//pkg/structure",
"//pkg/util/dbterror",
"//pkg/util/hack",
+ "//pkg/util/partialjson",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_kvproto//pkg/kvrpcpb",
"@com_github_pingcap_kvproto//pkg/resource_manager",
diff --git a/pkg/meta/autoid/autoid_test.go b/pkg/meta/autoid/autoid_test.go
index f6fa527db7d36..a531c6ec1a5b4 100644
--- a/pkg/meta/autoid/autoid_test.go
+++ b/pkg/meta/autoid/autoid_test.go
@@ -65,15 +65,15 @@ func TestSignedAutoid(t *testing.T) {
m := meta.NewMeta(txn)
err = m.CreateDatabase(&model.DBInfo{ID: 1, Name: model.NewCIStr("a")})
require.NoError(t, err)
- err = m.CreateTableOrView(1, "", &model.TableInfo{ID: 1, Name: model.NewCIStr("t")})
+ err = m.CreateTableOrView(1, &model.TableInfo{ID: 1, Name: model.NewCIStr("t")})
require.NoError(t, err)
- err = m.CreateTableOrView(1, "", &model.TableInfo{ID: 2, Name: model.NewCIStr("t1")})
+ err = m.CreateTableOrView(1, &model.TableInfo{ID: 2, Name: model.NewCIStr("t1")})
require.NoError(t, err)
- err = m.CreateTableOrView(1, "", &model.TableInfo{ID: 3, Name: model.NewCIStr("t1")})
+ err = m.CreateTableOrView(1, &model.TableInfo{ID: 3, Name: model.NewCIStr("t1")})
require.NoError(t, err)
- err = m.CreateTableOrView(1, "", &model.TableInfo{ID: 4, Name: model.NewCIStr("t2")})
+ err = m.CreateTableOrView(1, &model.TableInfo{ID: 4, Name: model.NewCIStr("t2")})
require.NoError(t, err)
- err = m.CreateTableOrView(1, "", &model.TableInfo{ID: 5, Name: model.NewCIStr("t3")})
+ err = m.CreateTableOrView(1, &model.TableInfo{ID: 5, Name: model.NewCIStr("t3")})
require.NoError(t, err)
return nil
})
@@ -270,15 +270,15 @@ func TestUnsignedAutoid(t *testing.T) {
m := meta.NewMeta(txn)
err = m.CreateDatabase(&model.DBInfo{ID: 1, Name: model.NewCIStr("a")})
require.NoError(t, err)
- err = m.CreateTableOrView(1, "", &model.TableInfo{ID: 1, Name: model.NewCIStr("t")})
+ err = m.CreateTableOrView(1, &model.TableInfo{ID: 1, Name: model.NewCIStr("t")})
require.NoError(t, err)
- err = m.CreateTableOrView(1, "", &model.TableInfo{ID: 2, Name: model.NewCIStr("t1")})
+ err = m.CreateTableOrView(1, &model.TableInfo{ID: 2, Name: model.NewCIStr("t1")})
require.NoError(t, err)
- err = m.CreateTableOrView(1, "", &model.TableInfo{ID: 3, Name: model.NewCIStr("t1")})
+ err = m.CreateTableOrView(1, &model.TableInfo{ID: 3, Name: model.NewCIStr("t1")})
require.NoError(t, err)
- err = m.CreateTableOrView(1, "", &model.TableInfo{ID: 4, Name: model.NewCIStr("t2")})
+ err = m.CreateTableOrView(1, &model.TableInfo{ID: 4, Name: model.NewCIStr("t2")})
require.NoError(t, err)
- err = m.CreateTableOrView(1, "", &model.TableInfo{ID: 5, Name: model.NewCIStr("t3")})
+ err = m.CreateTableOrView(1, &model.TableInfo{ID: 5, Name: model.NewCIStr("t3")})
require.NoError(t, err)
return nil
})
@@ -434,7 +434,7 @@ func TestConcurrentAlloc(t *testing.T) {
m := meta.NewMeta(txn)
err = m.CreateDatabase(&model.DBInfo{ID: dbID, Name: model.NewCIStr("a")})
require.NoError(t, err)
- err = m.CreateTableOrView(dbID, "a", &model.TableInfo{ID: tblID, Name: model.NewCIStr("t")})
+ err = m.CreateTableOrView(dbID, &model.TableInfo{ID: tblID, Name: model.NewCIStr("t")})
require.NoError(t, err)
return nil
})
@@ -520,7 +520,7 @@ func TestRollbackAlloc(t *testing.T) {
m := meta.NewMeta(txn)
err = m.CreateDatabase(&model.DBInfo{ID: dbID, Name: model.NewCIStr("a")})
require.NoError(t, err)
- err = m.CreateTableOrView(dbID, "a", &model.TableInfo{ID: tblID, Name: model.NewCIStr("t")})
+ err = m.CreateTableOrView(dbID, &model.TableInfo{ID: tblID, Name: model.NewCIStr("t")})
require.NoError(t, err)
return nil
})
@@ -570,9 +570,9 @@ func TestAllocComputationIssue(t *testing.T) {
m := meta.NewMeta(txn)
err = m.CreateDatabase(&model.DBInfo{ID: 1, Name: model.NewCIStr("a")})
require.NoError(t, err)
- err = m.CreateTableOrView(1, "a", &model.TableInfo{ID: 1, Name: model.NewCIStr("t")})
+ err = m.CreateTableOrView(1, &model.TableInfo{ID: 1, Name: model.NewCIStr("t")})
require.NoError(t, err)
- err = m.CreateTableOrView(1, "a", &model.TableInfo{ID: 2, Name: model.NewCIStr("t1")})
+ err = m.CreateTableOrView(1, &model.TableInfo{ID: 2, Name: model.NewCIStr("t1")})
require.NoError(t, err)
return nil
})
@@ -621,7 +621,7 @@ func TestIssue40584(t *testing.T) {
m := meta.NewMeta(txn)
err = m.CreateDatabase(&model.DBInfo{ID: 1, Name: model.NewCIStr("a")})
require.NoError(t, err)
- err = m.CreateTableOrView(1, "a", &model.TableInfo{ID: 1, Name: model.NewCIStr("t")})
+ err = m.CreateTableOrView(1, &model.TableInfo{ID: 1, Name: model.NewCIStr("t")})
require.NoError(t, err)
return nil
})
diff --git a/pkg/meta/autoid/bench_test.go b/pkg/meta/autoid/bench_test.go
index cd196440bb012..f2df4e67633c2 100644
--- a/pkg/meta/autoid/bench_test.go
+++ b/pkg/meta/autoid/bench_test.go
@@ -48,7 +48,7 @@ func BenchmarkAllocator_Alloc(b *testing.B) {
if err != nil {
return err
}
- err = m.CreateTableOrView(dbID, "a", &model.TableInfo{ID: tblID, Name: model.NewCIStr("t")})
+ err = m.CreateTableOrView(dbID, &model.TableInfo{ID: tblID, Name: model.NewCIStr("t")})
if err != nil {
return err
}
@@ -103,7 +103,7 @@ func BenchmarkAllocator_SequenceAlloc(b *testing.B) {
Sequence: seq,
}
sequenceBase = seq.Start - 1
- err = m.CreateSequenceAndSetSeqValue(1, "a", seqTable, sequenceBase)
+ err = m.CreateSequenceAndSetSeqValue(1, seqTable, sequenceBase)
return err
})
if err != nil {
diff --git a/pkg/meta/autoid/seq_autoid_test.go b/pkg/meta/autoid/seq_autoid_test.go
index da3772ba9d5b6..ae0d4508a1e75 100644
--- a/pkg/meta/autoid/seq_autoid_test.go
+++ b/pkg/meta/autoid/seq_autoid_test.go
@@ -60,7 +60,7 @@ func TestSequenceAutoid(t *testing.T) {
Sequence: seq,
}
sequenceBase = seq.Start - 1
- err = m.CreateSequenceAndSetSeqValue(1, "a", seqTable, sequenceBase)
+ err = m.CreateSequenceAndSetSeqValue(1, seqTable, sequenceBase)
require.NoError(t, err)
return nil
})
@@ -189,7 +189,7 @@ func TestConcurrentAllocSequence(t *testing.T) {
} else {
sequenceBase = seq.Start + 1
}
- err1 = m.CreateSequenceAndSetSeqValue(2, "a", seqTable, sequenceBase)
+ err1 = m.CreateSequenceAndSetSeqValue(2, seqTable, sequenceBase)
require.NoError(t, err1)
return nil
})
diff --git a/pkg/meta/meta.go b/pkg/meta/meta.go
index f16af4abf23d6..f11c125f1625f 100644
--- a/pkg/meta/meta.go
+++ b/pkg/meta/meta.go
@@ -38,6 +38,7 @@ import (
"github.com/pingcap/tidb/pkg/structure"
"github.com/pingcap/tidb/pkg/util/dbterror"
"github.com/pingcap/tidb/pkg/util/hack"
+ "github.com/pingcap/tidb/pkg/util/partialjson"
)
var (
@@ -70,13 +71,8 @@ var (
mNextGlobalIDKey = []byte("NextGlobalID")
mSchemaVersionKey = []byte("SchemaVersionKey")
mDBs = []byte("DBs")
- mNames = []byte("Names")
- mDBNames = []byte("DBNames")
- mDBNameInitialized = []byte("DBNameInitialized")
- mDDLV2Initialized = []byte("DDLV2Initialized")
mDBPrefix = "DB"
mTablePrefix = "Table"
- mNameSep = []byte("\x00")
mSequencePrefix = "SID"
mSeqCyclePrefix = "SequenceCycle"
mTableIDPrefix = "TID"
@@ -176,20 +172,11 @@ func (ver DDLTableVersion) Bytes() []byte {
// Option is for Meta option.
type Option func(m *Meta)
-// WithUpdateTableName is for updating the name of the table.
-// Only used for ddl v2.
-func WithUpdateTableName() Option {
- return func(m *Meta) {
- m.needUpdateName = true
- }
-}
-
// Meta is for handling meta information in a transaction.
type Meta struct {
- txn *structure.TxStructure
- StartTS uint64 // StartTS is the txn's start TS.
- jobListKey JobListKeyType
- needUpdateName bool
+ txn *structure.TxStructure
+ StartTS uint64 // StartTS is the txn's start TS.
+ jobListKey JobListKeyType
}
// NewMeta creates a Meta in transaction txn.
@@ -681,9 +668,6 @@ func (m *Meta) CreateDatabase(dbInfo *model.DBInfo) error {
if err := m.txn.HSet(mDBs, dbKey, data); err != nil {
return errors.Trace(err)
}
- if m.needUpdateName {
- return errors.Trace(m.CreateDatabaseName(dbInfo.Name.L, dbInfo.ID))
- }
return nil
}
@@ -704,7 +688,7 @@ func (m *Meta) UpdateDatabase(dbInfo *model.DBInfo) error {
}
// CreateTableOrView creates a table with tableInfo in database.
-func (m *Meta) CreateTableOrView(dbID int64, dbName string, tableInfo *model.TableInfo) error {
+func (m *Meta) CreateTableOrView(dbID int64, tableInfo *model.TableInfo) error {
// Check if db exists.
dbKey := m.dbKey(dbID)
if err := m.checkDBExists(dbKey); err != nil {
@@ -725,9 +709,6 @@ func (m *Meta) CreateTableOrView(dbID int64, dbName string, tableInfo *model.Tab
if err := m.txn.HSet(dbKey, tableKey, data); err != nil {
return errors.Trace(err)
}
- if m.needUpdateName {
- return errors.Trace(m.CreateTableName(dbName, tableInfo.Name.L, tableInfo.ID))
- }
return nil
}
@@ -850,8 +831,8 @@ func (m *Meta) GetSchemaCacheSize() (size uint64, isNull bool, err error) {
// CreateTableAndSetAutoID creates a table with tableInfo in database,
// and rebases the table autoID.
-func (m *Meta) CreateTableAndSetAutoID(dbID int64, dbName string, tableInfo *model.TableInfo, autoIDs AutoIDGroup) error {
- err := m.CreateTableOrView(dbID, dbName, tableInfo)
+func (m *Meta) CreateTableAndSetAutoID(dbID int64, tableInfo *model.TableInfo, autoIDs AutoIDGroup) error {
+ err := m.CreateTableOrView(dbID, tableInfo)
if err != nil {
return errors.Trace(err)
}
@@ -875,8 +856,8 @@ func (m *Meta) CreateTableAndSetAutoID(dbID int64, dbName string, tableInfo *mod
}
// CreateSequenceAndSetSeqValue creates sequence with tableInfo in database, and rebase the sequence seqValue.
-func (m *Meta) CreateSequenceAndSetSeqValue(dbID int64, dbName string, tableInfo *model.TableInfo, seqValue int64) error {
- err := m.CreateTableOrView(dbID, dbName, tableInfo)
+func (m *Meta) CreateSequenceAndSetSeqValue(dbID int64, tableInfo *model.TableInfo, seqValue int64) error {
+ err := m.CreateTableOrView(dbID, tableInfo)
if err != nil {
return errors.Trace(err)
}
@@ -914,7 +895,7 @@ func (m *Meta) DropPolicy(policyID int64) error {
}
// DropDatabase drops whole database.
-func (m *Meta) DropDatabase(dbID int64, dbName string) error {
+func (m *Meta) DropDatabase(dbID int64) error {
// Check if db exists.
dbKey := m.dbKey(dbID)
if err := m.txn.HClear(dbKey); err != nil {
@@ -925,16 +906,13 @@ func (m *Meta) DropDatabase(dbID int64, dbName string) error {
return errors.Trace(err)
}
- if m.needUpdateName {
- return errors.Trace(m.DropDatabaseName(dbName))
- }
return nil
}
// DropSequence drops sequence in database.
// Sequence is made of table struct and kv value pair.
-func (m *Meta) DropSequence(dbID int64, dbName string, tblID int64, tbName string) error {
- err := m.DropTableOrView(dbID, dbName, tblID, tbName)
+func (m *Meta) DropSequence(dbID int64, tblID int64) error {
+ err := m.DropTableOrView(dbID, tblID)
if err != nil {
return err
}
@@ -949,7 +927,7 @@ func (m *Meta) DropSequence(dbID int64, dbName string, tblID int64, tbName strin
// DropTableOrView drops table in database.
// If delAutoID is true, it will delete the auto_increment id key-value of the table.
// For rename table, we do not need to rename auto_increment id key-value.
-func (m *Meta) DropTableOrView(dbID int64, dbName string, tblID int64, tbName string) error {
+func (m *Meta) DropTableOrView(dbID int64, tblID int64) error {
// Check if db exists.
dbKey := m.dbKey(dbID)
if err := m.checkDBExists(dbKey); err != nil {
@@ -965,9 +943,6 @@ func (m *Meta) DropTableOrView(dbID int64, dbName string, tblID int64, tbName st
if err := m.txn.HDel(dbKey, tableKey); err != nil {
return errors.Trace(err)
}
- if m.needUpdateName {
- return errors.Trace(m.DropTableName(dbName, tbName))
- }
return nil
}
@@ -1038,20 +1013,23 @@ func (m *Meta) GetMetasByDBID(dbID int64) ([]structure.HashPair, error) {
return res, nil
}
-var checkSubstringsInOrder = [6]string{
+var checkSubstringsInOrder = [7]string{
`"fk_info":null`,
`"partition":null`,
`"Lock":null`,
`"tiflash_replica":null`,
+ `"temp_table_type":0`,
`"policy_ref_info":null`,
`"ttl_info":null`,
}
-// CheckSpecialAttributes checks if the special attributes are in the table info.
-// Make it same as hasSpecialAttributes.
+// IsTableInfoMustLoad checks the above substrings in a table info's json representing.
+// When a table contains one of them, tidb must load the table info during schema full load.
+// hasSpecialAttributes() is a subset of it, the difference is that:
+// If a table need to be resident in-memory, its table info MUST be loaded.
+// If a table info is loaded, it's NOT NECESSARILY to be keep in-memory.
// Exported for testing.
-// It's the regexp version for hasSpecialAttributes(), please keep up-to-date with it.
-func CheckSpecialAttributes(json []byte) bool {
+func IsTableInfoMustLoad(json []byte) bool {
idx := 0
for _, substr := range checkSubstringsInOrder {
idx = bytes.Index(json, hack.Slice(substr))
@@ -1064,7 +1042,7 @@ func CheckSpecialAttributes(json []byte) bool {
}
// NameExtractRegexp is exported for testing.
-const NameExtractRegexp = `"L":"([^"\\]*(?:\\.[^"\\]*)*)"}`
+const NameExtractRegexp = `"O":"([^"\\]*(?:\\.[^"\\]*)*)",`
// Unescape is exported for testing.
func Unescape(s string) string {
@@ -1073,9 +1051,9 @@ func Unescape(s string) string {
return s
}
-// GetAllNameToIDAndSpecialAttributeInfo gets all the fields and values and table info for special attributes in a hash.
+// GetAllNameToIDAndTheMustLoadedTableInfo gets all the fields and values and table info for special attributes in a hash.
// It's used to get some infos for information schema cache in a faster way.
-func GetAllNameToIDAndSpecialAttributeInfo(m *Meta, dbID int64) (map[string]int64, []*model.TableInfo, error) {
+func GetAllNameToIDAndTheMustLoadedTableInfo(m *Meta, dbID int64) (map[string]int64, []*model.TableInfo, error) {
dbKey := m.dbKey(dbID)
if err := m.checkDBExists(dbKey); err != nil {
return nil, nil, errors.Trace(err)
@@ -1101,7 +1079,7 @@ func GetAllNameToIDAndSpecialAttributeInfo(m *Meta, dbID int64) (map[string]int6
key := Unescape(nameLMatch[1])
res[strings.Clone(key)] = int64(id)
- if CheckSpecialAttributes(value) {
+ if IsTableInfoMustLoad(value) {
tbInfo := &model.TableInfo{}
err = json.Unmarshal(value, tbInfo)
if err != nil {
@@ -1159,10 +1137,9 @@ func (m *Meta) ListSimpleTables(dbID int64) ([]*model.TableNameInfo, error) {
continue
}
- tbInfo := &model.TableNameInfo{}
- err = json.Unmarshal(r.Value, tbInfo)
- if err != nil {
- return nil, errors.Trace(err)
+ tbInfo, err2 := FastUnmarshalTableNameInfo(r.Value)
+ if err2 != nil {
+ return nil, errors.Trace(err2)
}
tables = append(tables, tbInfo)
@@ -1171,6 +1148,52 @@ func (m *Meta) ListSimpleTables(dbID int64) ([]*model.TableNameInfo, error) {
return tables, nil
}
+var tableNameInfoFields = []string{"id", "name"}
+
+// FastUnmarshalTableNameInfo is exported for testing.
+func FastUnmarshalTableNameInfo(data []byte) (*model.TableNameInfo, error) {
+ m, err := partialjson.ExtractTopLevelMembers(data, tableNameInfoFields)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ idTokens, ok := m["id"]
+ if !ok {
+ return nil, errors.New("id field not found in JSON")
+ }
+ if len(idTokens) != 1 {
+ return nil, errors.Errorf("unexpected id field in JSON, %v", idTokens)
+ }
+ num, ok := idTokens[0].(json.Number)
+ if !ok {
+ return nil, errors.Errorf(
+ "id field is not a number, got %T %v", idTokens[0], idTokens[0],
+ )
+ }
+ id, err := num.Int64()
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ nameTokens, ok := m["name"]
+ if !ok {
+ return nil, errors.New("name field not found in JSON")
+ }
+ // 6 tokens; {, O, ..., L, ..., }, the data looks like this {123,"O","t","L","t",125}
+ if len(nameTokens) != 6 {
+ return nil, errors.Errorf("unexpected name field in JSON, %v", nameTokens)
+ }
+ name, ok := nameTokens[2].(string)
+ if !ok {
+ return nil, errors.Errorf("unexpected name field in JSON, %v", nameTokens)
+ }
+
+ return &model.TableNameInfo{
+ ID: id,
+ Name: model.NewCIStr(name),
+ }, nil
+}
+
// ListDatabases shows all databases.
func (m *Meta) ListDatabases() ([]*model.DBInfo, error) {
res, err := m.txn.HGetAll(mDBs)
@@ -1288,7 +1311,7 @@ func (m *Meta) GetResourceGroup(groupID int64) (*model.ResourceGroupInfo, error)
return nil, errors.Trace(err)
}
if value == nil {
- // the default group is not persistanted to tikv by default.
+ // the default group is not persistent to tikv by default.
if groupID == defaultGroupID {
return defaultRGroupMeta, nil
}
@@ -1662,163 +1685,6 @@ func (m *Meta) SetSchemaDiff(diff *model.SchemaDiff) error {
return errors.Trace(err)
}
-// TableNameKey constructs the key for table name.
-func (*Meta) TableNameKey(dbName string, tableName string) kv.Key {
- var sb strings.Builder
- sb.Write(mNames)
- sb.WriteByte(':')
- sb.WriteString(strings.ToLower(dbName))
- sb.Write(mNameSep)
- sb.WriteString(strings.ToLower(tableName))
- return kv.Key(sb.String())
-}
-
-// DatabaseNameKey constructs the key for database name.
-func (*Meta) DatabaseNameKey(dbName string) kv.Key {
- var sb strings.Builder
- sb.Write(mDBNames)
- sb.WriteByte(':')
- sb.WriteString(strings.ToLower(dbName))
- return kv.Key(sb.String())
-}
-
-// CheckTableNameExists checks if the table name exists.
-func (m *Meta) CheckTableNameExists(name []byte) error {
- v, err := m.txn.Get(name)
- if err == nil && v == nil {
- err = ErrTableNotExists.FastGenByArgs(string(name))
- }
- return errors.Trace(err)
-}
-
-// CheckTableNameNotExists checks if the table name not exists.
-func (m *Meta) CheckTableNameNotExists(name []byte) error {
- v, err := m.txn.Get(name)
- if err == nil && v != nil {
- err = ErrTableExists.FastGenByArgs(string(name))
- }
- return errors.Trace(err)
-}
-
-// CheckDatabaseNameExists checks if the database name exists.
-func (m *Meta) CheckDatabaseNameExists(name []byte) error {
- v, err := m.txn.Get(name)
- if err == nil && v == nil {
- err = ErrDBNotExists.FastGenByArgs(string(name))
- }
- return errors.Trace(err)
-}
-
-// CheckDatabaseNameNotExists checks if the database name not exists.
-func (m *Meta) CheckDatabaseNameNotExists(name []byte) error {
- v, err := m.txn.Get(name)
- if err == nil && v != nil {
- err = ErrDBExists.FastGenByArgs(string(name))
- }
- return errors.Trace(err)
-}
-
-// CreateDatabaseName creates a database name.
-// Used by CreateDatabase/RenameDatabase
-func (m *Meta) CreateDatabaseName(dbName string, dbID int64) error {
- // Check if database exists.
- key := m.DatabaseNameKey(dbName)
- if err := m.CheckDatabaseNameNotExists(key); err != nil {
- return errors.Trace(err)
- }
- return m.txn.Set(key, []byte(strconv.FormatInt(dbID, 10)))
-}
-
-// CreateTableName creates a table name.
-// Used by CreateTable/RenameTable/TruncateTable/RecoverTable/RecoverSchema/CreateView...
-func (m *Meta) CreateTableName(dbName string, tableName string, tableID int64) error {
- // Check if table exists.
- key := m.TableNameKey(dbName, tableName)
- if err := m.CheckTableNameNotExists(key); err != nil {
- return errors.Trace(err)
- }
- return m.txn.Set(key, []byte(strconv.FormatInt(tableID, 10)))
-}
-
-// DropTableName drops a table name.
-// Used by DropTable/RenameTable/TruncateTable/DropView...
-func (m *Meta) DropTableName(dbName string, tableName string) error {
- // Check if table exists.
- key := m.TableNameKey(dbName, tableName)
- if err := m.CheckTableNameExists(key); err != nil {
- return errors.Trace(err)
- }
- return m.txn.Clear(key)
-}
-
-// DropDatabaseName drops a database name.
-// Used by DropDatabase.
-func (m *Meta) DropDatabaseName(dbName string) error {
- // Check if database exists.
- key := m.DatabaseNameKey(dbName)
- if err := m.CheckDatabaseNameExists(key); err != nil {
- return errors.Trace(err)
- }
- if err := m.txn.Clear(key); err != nil {
- return errors.Trace(err)
- }
-
- // iterate all tables
- prefix := m.TableNameKey(dbName, "")
- return m.txn.Iterate(prefix, prefix.PrefixNext(), func(key []byte, _ []byte) error {
- return m.txn.Clear(key)
- })
-}
-
-// ClearAllTableNames clears all table names.
-func (m *Meta) ClearAllTableNames() error {
- prefix := kv.Key(fmt.Sprintf("%s:", mNames))
- return m.txn.Iterate(prefix, prefix.PrefixNext(), func(key []byte, _ []byte) error {
- return m.txn.Clear(key)
- })
-}
-
-// ClearAllDatabaseNames clears all database names.
-func (m *Meta) ClearAllDatabaseNames() error {
- prefix := kv.Key(fmt.Sprintf("%s:", mDBNames))
- return m.txn.Iterate(prefix, prefix.PrefixNext(), func(key []byte, _ []byte) error {
- return m.txn.Clear(key)
- })
-}
-
-// SetFastCreateTableInitialized set fast create table initialized.
-func (m *Meta) SetFastCreateTableInitialized(b bool) error {
- var data []byte
- if b {
- data = []byte("1")
- } else {
- data = []byte("0")
- }
- if err := m.txn.Set(mDDLV2Initialized, data); err != nil {
- return errors.Trace(err)
- }
- return errors.Trace(m.txn.Set(mDBNameInitialized, data))
-}
-
-// GetFastCreateTableInitialized gets fast create table initialized.
-func (m *Meta) GetFastCreateTableInitialized() (initialized bool, err error) {
- val1, err := m.txn.Get(mDDLV2Initialized)
- if err != nil {
- return false, errors.Trace(err)
- }
- if len(val1) == 0 {
- return false, nil
- }
- val2, err := m.txn.Get(mDBNameInitialized)
- if err != nil {
- return false, errors.Trace(err)
- }
- if len(val2) == 0 {
- return false, nil
- }
- return bytes.Equal(val1, []byte("1")) && bytes.Equal(val2, []byte("1")), nil
-}
-
// GroupRUStats keeps the ru consumption statistics data.
type GroupRUStats struct {
ID int64 `json:"id"`
diff --git a/pkg/meta/meta_test.go b/pkg/meta/meta_test.go
index 34ac1e7ec9f1c..da2da89b49d7f 100644
--- a/pkg/meta/meta_test.go
+++ b/pkg/meta/meta_test.go
@@ -237,7 +237,7 @@ func TestMeta(t *testing.T) {
Name: model.NewCIStr("t"),
DBID: dbInfo.ID,
}
- err = m.CreateTableOrView(1, dbInfo.Name.L, tbInfo)
+ err = m.CreateTableOrView(1, tbInfo)
require.NoError(t, err)
n, err = m.GetAutoIDAccessors(1, 1).RowID().Inc(10)
@@ -248,7 +248,7 @@ func TestMeta(t *testing.T) {
require.NoError(t, err)
require.Equal(t, int64(10), n)
- err = m.CreateTableOrView(1, dbInfo.Name.L, tbInfo)
+ err = m.CreateTableOrView(1, tbInfo)
require.NotNil(t, err)
require.True(t, meta.ErrTableExists.Equal(err))
@@ -275,7 +275,7 @@ func TestMeta(t *testing.T) {
Name: model.NewCIStr("bb"),
DBID: dbInfo.ID,
}
- err = m.CreateTableOrView(1, dbInfo.Name.L, tbInfo2)
+ err = m.CreateTableOrView(1, tbInfo2)
require.NoError(t, err)
tblName := &model.TableNameInfo{ID: tbInfo.ID, Name: tbInfo.Name}
@@ -309,7 +309,7 @@ func TestMeta(t *testing.T) {
require.NoError(t, err)
require.Equal(t, int64(10), n)
- err = m.DropTableOrView(1, dbInfo.Name.L, tbInfo2.ID, tbInfo2.Name.L)
+ err = m.DropTableOrView(1, tbInfo2.ID)
require.NoError(t, err)
err = m.GetAutoIDAccessors(1, tbInfo2.ID).Del()
require.NoError(t, err)
@@ -341,7 +341,7 @@ func TestMeta(t *testing.T) {
Name: model.NewCIStr("t_rename"),
}
// Create table.
- err = m.CreateTableOrView(1, dbInfo.Name.L, tbInfo100)
+ err = m.CreateTableOrView(1, tbInfo100)
require.NoError(t, err)
// Update auto ID.
currentDBID := int64(1)
@@ -367,7 +367,7 @@ func TestMeta(t *testing.T) {
ID: 3,
Name: model.NewCIStr("tbl3"),
}
- err = m.CreateTableAndSetAutoID(1, dbInfo.Name.L, tbInfo3, meta.AutoIDGroup{RowID: 123, IncrementID: 0})
+ err = m.CreateTableAndSetAutoID(1, tbInfo3, meta.AutoIDGroup{RowID: 123, IncrementID: 0})
require.NoError(t, err)
id, err := m.GetAutoIDAccessors(1, tbInfo3.ID).RowID().Get()
require.NoError(t, err)
@@ -377,9 +377,9 @@ func TestMeta(t *testing.T) {
require.Equal(t, []byte(strconv.FormatInt(1234, 10)), val)
require.Equal(t, []byte{0x6d, 0x44, 0x42, 0x3a, 0x31, 0x0, 0x0, 0x0, 0x0, 0xfb, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x68, 0x54, 0x49, 0x44, 0x3a, 0x33, 0x0, 0x0, 0x0, 0xfc}, key)
- err = m.DropDatabase(1, dbInfo.Name.L)
+ err = m.DropDatabase(1)
require.NoError(t, err)
- err = m.DropDatabase(currentDBID, dbInfo.Name.L)
+ err = m.DropDatabase(currentDBID)
require.NoError(t, err)
dbs, err = m.ListDatabases()
@@ -655,160 +655,74 @@ func TestCreateMySQLDatabase(t *testing.T) {
require.NoError(t, err)
}
-func TestName(t *testing.T) {
- store, err := mockstore.NewMockStore()
- require.NoError(t, err)
- defer func() {
- require.NoError(t, store.Close())
- }()
-
- txn, err := store.Begin()
- require.NoError(t, err)
-
- // TestDatabaseNameKey
- m := meta.NewMeta(txn)
- key := m.DatabaseNameKey("db")
- require.Equal(t, string(key), "DBNames:db")
-
- // TestCheckDatabaseNameExists
- err = m.CheckDatabaseNameExists(m.DatabaseNameKey("db"))
- require.True(t, meta.ErrDBNotExists.Equal(err))
- // TestCheckDatabaseNameNotExists
- err = m.CheckDatabaseNameNotExists(m.DatabaseNameKey("db"))
- require.NoError(t, err)
- // TestCreateDatabase
- err = m.CreateDatabaseName("db", 1)
- require.NoError(t, err)
- err = m.CheckDatabaseNameExists(m.DatabaseNameKey("db"))
- require.NoError(t, err)
- err = m.CheckDatabaseNameNotExists(m.DatabaseNameKey("db"))
- require.True(t, meta.ErrDBExists.Equal(err))
-
- // TestTableNameKey
- key = m.TableNameKey("db", "tb")
- require.Equal(t, string(key), "Names:db\x00tb")
-
- // TestCheckTableNameExists
- err = m.CheckTableNameExists(m.TableNameKey("db", "tb"))
- require.True(t, meta.ErrTableNotExists.Equal(err))
- // TestCheckTableNameNotExists
- err = m.CheckTableNameNotExists(m.TableNameKey("db", "tb"))
- require.NoError(t, err)
-
- // TestCreateTable
- err = m.CreateTableName("db", "tb", 1)
- require.NoError(t, err)
- err = m.CheckTableNameExists(m.TableNameKey("db", "tb"))
- require.NoError(t, err)
- err = m.CheckTableNameNotExists(m.TableNameKey("db", "tb"))
- require.True(t, meta.ErrTableExists.Equal(err))
- err = m.CreateTableName("db", "t", 2)
- require.NoError(t, err)
-
- err = m.CreateTableName("db", "tb", 3)
- require.True(t, meta.ErrTableExists.Equal(err))
-
- err = m.CreateDatabaseName("d", 4)
- require.NoError(t, err)
- err = m.CreateTableName("d", "btb", 3)
- require.NoError(t, err)
- err = m.CheckTableNameExists(m.TableNameKey("d", "btb"))
- require.NoError(t, err)
-
- // TestDropTableName
- err = m.DropTableName("db1", "b")
- require.True(t, meta.ErrTableNotExists.Equal(err))
- err = m.DropTableName("db", "tb")
- require.NoError(t, err)
-
- // TestDropDatabaseName
- err = m.DropDatabaseName("xx")
- require.True(t, meta.ErrDBNotExists.Equal(err))
- err = m.DropDatabaseName("d")
- require.NoError(t, err)
- err = m.CheckTableNameNotExists(m.TableNameKey("d", "btb"))
- require.NoError(t, err)
- err = m.CheckTableNameExists(m.TableNameKey("db", "t"))
- require.NoError(t, err)
-
- // TestClearAllTableNames
- err = m.ClearAllTableNames()
- require.NoError(t, err)
- err = m.CheckTableNameNotExists(m.TableNameKey("db1", "t"))
- require.NoError(t, err)
-
- // TestClearAllDatabaseNames
- err = m.ClearAllDatabaseNames()
- require.NoError(t, err)
-
- // TestFastCreateTableInitialized
- v, err := m.GetFastCreateTableInitialized()
- require.NoError(t, err)
- require.Equal(t, v, false)
- err = m.SetFastCreateTableInitialized(true)
- require.NoError(t, err)
- v, err = m.GetFastCreateTableInitialized()
- require.NoError(t, err)
- require.Equal(t, v, true)
- err = m.SetFastCreateTableInitialized(false)
- require.NoError(t, err)
- v, err = m.GetFastCreateTableInitialized()
- require.NoError(t, err)
- require.Equal(t, v, false)
-
- err = txn.Rollback()
- require.NoError(t, err)
-}
-
-func TestCheckSpecialAttributes(t *testing.T) {
+func TestIsTableInfoMustLoad(t *testing.T) {
tableInfo := &model.TableInfo{
TTLInfo: &model.TTLInfo{IntervalExprStr: "1", IntervalTimeUnit: int(ast.TimeUnitDay), JobInterval: "1h"},
}
b, err := json.Marshal(tableInfo)
require.NoError(t, err)
- require.True(t, meta.CheckSpecialAttributes(b))
+ require.True(t, meta.IsTableInfoMustLoad(b))
tableInfo = &model.TableInfo{
TiFlashReplica: &model.TiFlashReplicaInfo{Count: 1},
}
b, err = json.Marshal(tableInfo)
require.NoError(t, err)
- require.True(t, meta.CheckSpecialAttributes(b))
+ require.True(t, meta.IsTableInfoMustLoad(b))
tableInfo = &model.TableInfo{
PlacementPolicyRef: &model.PolicyRefInfo{ID: 1},
}
b, err = json.Marshal(tableInfo)
require.NoError(t, err)
- require.True(t, meta.CheckSpecialAttributes(b))
+ require.True(t, meta.IsTableInfoMustLoad(b))
tableInfo = &model.TableInfo{
Partition: &model.PartitionInfo{Expr: "a"},
}
b, err = json.Marshal(tableInfo)
require.NoError(t, err)
- require.True(t, meta.CheckSpecialAttributes(b))
+ require.True(t, meta.IsTableInfoMustLoad(b))
tableInfo = &model.TableInfo{
Lock: &model.TableLockInfo{State: model.TableLockStatePreLock},
}
b, err = json.Marshal(tableInfo)
require.NoError(t, err)
- require.True(t, meta.CheckSpecialAttributes(b))
+ require.True(t, meta.IsTableInfoMustLoad(b))
tableInfo = &model.TableInfo{
ForeignKeys: []*model.FKInfo{{ID: 1}},
}
b, err = json.Marshal(tableInfo)
require.NoError(t, err)
- require.True(t, meta.CheckSpecialAttributes(b))
+ require.True(t, meta.IsTableInfoMustLoad(b))
+
+ tableInfo = &model.TableInfo{
+ TempTableType: model.TempTableGlobal,
+ }
+ b, err = json.Marshal(tableInfo)
+ require.NoError(t, err)
+ require.True(t, meta.IsTableInfoMustLoad(b))
tableInfo = &model.TableInfo{
ID: 123,
}
b, err = json.Marshal(tableInfo)
require.NoError(t, err)
- require.False(t, meta.CheckSpecialAttributes(b))
+ require.False(t, meta.IsTableInfoMustLoad(b))
+}
+
+func TestIsTableInfoMustLoadSubStringsOrder(t *testing.T) {
+ // The order matter!
+ // IsTableInfoMustLoad relies on the order of the json marshal result,
+ // or the internal of the json marshal in other words.
+ // This test cover the invariance, if Go std library changes, we can catch it.
+ tableInfo := &model.TableInfo{}
+ b, err := json.Marshal(tableInfo)
+ require.NoError(t, err)
+ expect := `{"id":0,"name":{"O":"","L":""},"charset":"","collate":"","cols":null,"index_info":null,"constraint_info":null,"fk_info":null,"state":0,"pk_is_handle":false,"is_common_handle":false,"common_handle_version":0,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":0,"max_idx_id":0,"max_fk_id":0,"max_cst_id":0,"update_timestamp":0,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"auto_random_range_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":0,"tiflash_replica":null,"is_columnar":false,"temp_table_type":0,"cache_table_status":0,"policy_ref_info":null,"stats_options":null,"exchange_partition_info":null,"ttl_info":null,"revision":0}`
+ require.Equal(t, string(b), expect)
}
func TestTableNameExtract(t *testing.T) {
@@ -851,10 +765,9 @@ func TestTableNameExtract(t *testing.T) {
require.Equal(t, `"\"啊"`, meta.Unescape(nameLMatch[1]))
}
-func BenchmarkCheckSpecialAttributes(b *testing.B) {
- benchCases := [][2]string{
- {"narrow", `CREATE TABLE t (c INT PRIMARY KEY);`},
- {"wide", `
+var benchCases = [][2]string{
+ {"narrow", `CREATE TABLE t (c INT PRIMARY KEY);`},
+ {"wide", `
CREATE TABLE t (
c BIGINT PRIMARY KEY AUTO_RANDOM,
c2 TINYINT,
@@ -875,16 +788,17 @@ CREATE TABLE t (
UNIQUE INDEX idx4(c12),
INDEX idx5((c + c2))
);`},
- }
+}
+func BenchmarkIsTableInfoMustLoad(b *testing.B) {
for _, benchCase := range benchCases {
b.Run(benchCase[0], func(b *testing.B) {
- benchCheckSpecialAttributes(b, benchCase[1])
+ benchIsTableInfoMustLoad(b, benchCase[1])
})
}
}
-func benchCheckSpecialAttributes(b *testing.B, sql string) {
+func getTableInfoJSON(b *testing.B, sql string) []byte {
p := parser.New()
stmt, err := p.ParseOneStmt(sql, "", "")
require.NoError(b, err)
@@ -894,9 +808,51 @@ func benchCheckSpecialAttributes(b *testing.B, sql string) {
data, err := json.Marshal(tblInfo)
require.NoError(b, err)
+ return data
+}
+
+func benchIsTableInfoMustLoad(b *testing.B, sql string) {
+ data := getTableInfoJSON(b, sql)
+
b.ResetTimer()
for i := 0; i < b.N; i++ {
- got := meta.CheckSpecialAttributes(data)
+ got := meta.IsTableInfoMustLoad(data)
intest.Assert(!got)
}
}
+
+func BenchmarkTableNameInfo(b *testing.B) {
+ for _, benchCase := range benchCases {
+ b.Run(benchCase[0]+"-json", func(b *testing.B) {
+ benchJSONTableNameInfo(b, benchCase[1])
+ })
+ b.Run(benchCase[0]+"-fastjson", func(b *testing.B) {
+ benchFastJSONTableNameInfo(b, benchCase[1])
+ })
+ }
+}
+
+func benchJSONTableNameInfo(b *testing.B, sql string) {
+ data := getTableInfoJSON(b, sql)
+
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ tbInfo := &model.TableNameInfo{}
+ err := json.Unmarshal(data, tbInfo)
+ intest.Assert(tbInfo.ID == 1)
+ intest.Assert(tbInfo.Name.L == "t")
+ intest.AssertNoError(err)
+ }
+}
+
+func benchFastJSONTableNameInfo(b *testing.B, sql string) {
+ data := getTableInfoJSON(b, sql)
+
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ tbInfo, err := meta.FastUnmarshalTableNameInfo(data)
+ intest.AssertNoError(err)
+ intest.Assert(tbInfo.ID == 1)
+ intest.Assert(tbInfo.Name.L == "t")
+ }
+}
diff --git a/pkg/metrics/ddl.go b/pkg/metrics/ddl.go
index 0a899fb3fc311..7e4973f55f007 100644
--- a/pkg/metrics/ddl.go
+++ b/pkg/metrics/ddl.go
@@ -43,7 +43,7 @@ var (
OwnerHandleSyncerHistogram *prometheus.HistogramVec
- // Metrics for ddl_worker.go.
+ // Metrics for job_worker.go.
WorkerNotifyDDLJob = "notify_job"
WorkerAddDDLJob = "add_job"
WorkerRunDDLJob = "run_job"
diff --git a/pkg/metrics/grafana/performance_overview.json b/pkg/metrics/grafana/performance_overview.json
index 8d5ae505a3253..4845b53fe97b6 100644
--- a/pkg/metrics/grafana/performance_overview.json
+++ b/pkg/metrics/grafana/performance_overview.json
@@ -1192,20 +1192,24 @@
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
"decimals": null,
- "description": "- avg: average cpu usage for all instance.\n- delta: max(cpu utilization) - min(cpu utilization)\n- max: max(cpu utilization)",
+ "description": "1. CPU: CPU utilization for all TiDB instances.\n- CPU-Avg: avg(cpu utilization)\n- CPU-Delta: max(cpu utilization) - min(cpu utilization)\n- CPU-Max: max(cpu utilization)\n2. Max memory usage",
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"grid": {},
"gridPos": {
- "h": 7,
- "w": 12,
+ "h": 8,
+ "w": 8,
"x": 0,
"y": 22
},
"hiddenSeries": false,
- "id": 181,
+ "id": 23763572004,
"legend": {
"alignAsTable": true,
"avg": true,
@@ -1227,47 +1231,74 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "8.0.7",
+ "pluginVersion": "7.5.17",
"pointradius": 5,
"points": false,
"renderer": "flot",
- "seriesOverrides": [],
+ "seriesOverrides": [
+ {
+ "$$hashKey": "object:371",
+ "alias": "Mem-Max",
+ "yaxis": 2
+ }
+ ],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
- "expr": "avg(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",job=\"tidb\"}[1m]))",
+ "expr": "avg(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}[1m]))",
"format": "time_series",
+ "hide": false,
"interval": "",
- "intervalFactor": 1,
- "legendFormat": "avg",
+ "intervalFactor": 2,
+ "legendFormat": "CPU-Avg",
"refId": "A",
"step": 30
},
{
"exemplar": true,
- "expr": "max(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",job=\"tidb\"}[1m]))",
+ "expr": "max(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}[1m]))",
"hide": false,
"interval": "",
- "legendFormat": "max",
+ "intervalFactor": 1,
+ "legendFormat": "CPU-Max",
"refId": "C"
},
{
"exemplar": true,
- "expr": "(max(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}[1m])) - min(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}[1m])))",
+ "expr": "(max(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}[1m])) - min(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}[1m])))",
"hide": false,
"interval": "",
- "legendFormat": "delta",
+ "legendFormat": "CPU-Delta",
"refId": "B"
+ },
+ {
+ "exemplar": true,
+ "expr": "max(tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"})",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "CPU-Quota",
+ "refId": "D",
+ "step": 30
+ },
+ {
+ "exemplar": true,
+ "expr": "max(sum((process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", job=~\"tidb\"})) by (instance))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "Mem-Max",
+ "refId": "E"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "TiDB CPU",
+ "title": "TiDB CPU/Memory",
"tooltip": {
"msResolution": false,
"shared": true,
@@ -1284,23 +1315,24 @@
},
"yaxes": [
{
- "$$hashKey": "object:188",
+ "$$hashKey": "object:208",
"decimals": null,
"format": "percentunit",
- "label": null,
+ "label": "CPU (%)",
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
- "$$hashKey": "object:189",
- "format": "short",
- "label": null,
+ "$$hashKey": "object:209",
+ "decimals": null,
+ "format": "bytes",
+ "label": "Memory",
"logBase": 1,
"max": null,
- "min": null,
- "show": false
+ "min": "0",
+ "show": true
}
],
"yaxis": {
@@ -1315,20 +1347,24 @@
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
"decimals": null,
- "description": "1. CPU: CPU utilization for all TiKV instances.\n- CPU-Avg: avg(cpu utilization)\n- CPU-Delta: max(cpu utilization) - min(cpu utilization)\n- CPU-Max: max(cpu utilization)\n2. IO MBps: The total bytes of read and write in all TiKV instances",
+ "description": "1. CPU: CPU utilization for all TiKV instances.\n- CPU-Avg: avg(cpu utilization)\n- CPU-Delta: max(cpu utilization) - min(cpu utilization)\n- CPU-Max: max(cpu utilization)\n2. Max memory usage",
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"grid": {},
"gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
+ "h": 8,
+ "w": 8,
+ "x": 8,
"y": 22
},
"hiddenSeries": false,
- "id": 182,
+ "id": 23763572003,
"legend": {
"alignAsTable": true,
"avg": true,
@@ -1350,24 +1386,14 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "8.0.7",
+ "pluginVersion": "7.5.17",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"$$hashKey": "object:371",
- "alias": "IO-Avg",
- "yaxis": 2
- },
- {
- "$$hashKey": "object:563",
- "alias": "IO-Max",
- "yaxis": 2
- },
- {
- "$$hashKey": "object:576",
- "alias": "IO-Delta",
+ "alias": "Mem-Max",
"yaxis": 2
}
],
@@ -1405,35 +1431,160 @@
},
{
"exemplar": true,
- "expr": "avg(sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[1m])) by (instance))",
+ "expr": "max(tikv_server_cpu_cores_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=~\".*tikv\"})",
"hide": false,
- "instant": false,
"interval": "",
- "legendFormat": "IO-Avg",
+ "legendFormat": "CPU-Quota",
"refId": "D"
},
{
"exemplar": true,
- "expr": "max(sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[1m])) by (instance))",
+ "expr": "max(sum((process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", job=~\".*tikv\"})) by (instance))",
"hide": false,
"interval": "",
- "legendFormat": "IO-Max",
+ "legendFormat": "Mem-Max",
"refId": "E"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "TiKV CPU/Memory",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:208",
+ "decimals": null,
+ "format": "percentunit",
+ "label": "CPU (%)",
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:209",
+ "decimals": null,
+ "format": "bytes",
+ "label": "Memory",
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "decimals": null,
+ "description": "1. CPU: CPU utilization for pd instances.\n- CPU-Max: max(cpu utilization)\n2. Max memory usage",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 22
+ },
+ "hiddenSeries": false,
+ "id": 182,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.17",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "$$hashKey": "object:371",
+ "alias": "Mem-Max",
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "max(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}[1m]))",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "CPU-Max",
+ "refId": "C"
},
{
"exemplar": true,
- "expr": "max(sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[1m])) by (instance)) - min(sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[1m])) by (instance))",
+ "expr": "max(pd_service_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"})",
"hide": false,
"interval": "",
- "legendFormat": "IO-Delta",
- "refId": "F"
+ "intervalFactor": 1,
+ "legendFormat": "CPU-Quota",
+ "refId": "A"
+ },
+ {
+ "exemplar": true,
+ "expr": "max(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"})",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "Mem-Max",
+ "refId": "E"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "TiKV CPU/IO MBps",
+ "title": "PD CPU/Memory",
"tooltip": {
"msResolution": false,
"shared": true,
@@ -1462,12 +1613,269 @@
{
"$$hashKey": "object:209",
"decimals": null,
+ "format": "bytes",
+ "label": "Memory",
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "decimals": null,
+ "description": "Out traffic statistics From TiDB to the client.\nRead Flow between TiKV and Rocksdb.",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 30
+ },
+ "hiddenSeries": false,
+ "id": 23763572000,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.17",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tidb_server_packet_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=~\"Out|read\"}[1m])) by (type)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "TiDB -> Client",
+ "refId": "A",
+ "step": 30
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"kv\", type=~\"bytes_read|iter_bytes_read\"}[1m]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "Rocksdb -> TiKV",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Read Traffic",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:188",
+ "decimals": null,
"format": "Bps",
- "label": "MBps ",
+ "label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
+ },
+ {
+ "$$hashKey": "object:189",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST-CLUSTER}",
+ "decimals": null,
+ "description": "Write traffic statistics between from Client to TiDB , the txn write rate from TiDB to TiKV, and the write flow from TiKV to RocksDB.",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 30
+ },
+ "hiddenSeries": false,
+ "id": 23763572002,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.17",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tidb_server_packet_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=~\"In|write\"}[1m])) by (type)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "Client -> TiDB",
+ "refId": "C"
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tidb_tikvclient_txn_write_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (scope)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "TiDB -> TiKV: {{scope}}",
+ "refId": "D"
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"kv\", type=\"wal_file_bytes\"}[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "TiKV -> Rocksdb",
+ "refId": "A",
+ "step": 30
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(rate(tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=~\"bytes_read|bytes_written\"}[1m]))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "RocksDB Compaction",
+ "refId": "B",
+ "step": 30
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Write Traffic",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:188",
+ "decimals": null,
+ "format": "Bps",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:189",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
}
],
"yaxis": {
@@ -1482,13 +1890,17 @@
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
"description": "TiDB query durations",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 6,
"w": 8,
"x": 0,
- "y": 29
+ "y": 37
},
"hiddenSeries": false,
"id": 80,
@@ -1513,7 +1925,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "8.0.7",
+ "pluginVersion": "7.5.17",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -1602,6 +2014,10 @@
"description": "TiDB connection idle durations",
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"grid": {},
@@ -1609,7 +2025,7 @@
"h": 6,
"w": 8,
"x": 8,
- "y": 29
+ "y": 37
},
"hiddenSeries": false,
"id": 171,
@@ -1635,7 +2051,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "8.0.7",
+ "pluginVersion": "7.5.17",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -1735,6 +2151,10 @@
"description": "TiDB current connection counts",
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"grid": {},
@@ -1742,7 +2162,7 @@
"h": 6,
"w": 8,
"x": 16,
- "y": 29
+ "y": 37
},
"hiddenSeries": false,
"id": 188,
@@ -1769,7 +2189,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "8.0.7",
+ "pluginVersion": "7.5.17",
"pointradius": 5,
"points": false,
"renderer": "flot",
diff --git a/pkg/metrics/grafana/tidb_resource_control.json b/pkg/metrics/grafana/tidb_resource_control.json
index f5bc16dd4837f..976947f43ee45 100644
--- a/pkg/metrics/grafana/tidb_resource_control.json
+++ b/pkg/metrics/grafana/tidb_resource_control.json
@@ -225,9 +225,17 @@
"editorMode": "code",
"expr": "sum(resource_manager_resource_unit_read_request_unit_max_per_sec{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (resource_group)",
"hide": false,
- "legendFormat": "{{resource_group}}",
+ "legendFormat": "{{resource_group}}-read",
"range": true,
- "refId": "C"
+ "refId": "A"
+ },
+ {
+ "editorMode": "code",
+ "expr": "sum(resource_manager_resource_unit_write_request_unit_max_per_sec{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (resource_group)",
+ "hide": false,
+ "legendFormat": "{{resource_group}}-write",
+ "range": true,
+ "refId": "B"
}
],
"thresholds": [],
diff --git a/pkg/metrics/grafana/tidb_runtime.json b/pkg/metrics/grafana/tidb_runtime.json
index bbc6b33d4fb67..ef5a4b8bca8ca 100644
--- a/pkg/metrics/grafana/tidb_runtime.json
+++ b/pkg/metrics/grafana/tidb_runtime.json
@@ -65,7 +65,7 @@
"dashLength": 10,
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
- "description": "TiDB process rss memory usage. TiDB heap memory size in use ",
+ "description": "This panel visualizes the memory usage of TiDB processes, including both resident set size (RSS) and various components of memory",
"editable": true,
"error": false,
"fill": 0,
@@ -76,6 +76,7 @@
"x": 0,
"y": 1
},
+ "hiddenSeries": false,
"id": 4,
"legend": {
"alignAsTable": false,
@@ -105,27 +106,57 @@
"repeatDirection": "h",
"seriesOverrides": [
{
- "alias": "alloc-from-os",
- "fill": 3,
+ "$$hashKey": "object:91",
+ "alias": "rss",
+ "bars": false,
+ "color": "#37872D",
"lines": true,
"stack": false
},
{
- "alias": "gc-threshold",
+ "$$hashKey": "object:92",
+ "alias": "tidb_server_memory_limit",
"bars": false,
"color": "#C4162A",
"lines": true,
- "linewidth": 2,
"stack": false
},
{
- "alias": "gc",
+ "$$hashKey": "object:215",
+ "alias": "heap_inuse",
+ "color": "#96D98D"
+ },
+ {
+ "$$hashKey": "object:247",
+ "alias": "stack_inuse",
+ "color": "#F2CC0C"
+ },
+ {
+ "$$hashKey": "object:261",
+ "alias": "heap_unused",
+ "color": "#3274D9"
+ },
+ {
+ "$$hashKey": "object:273",
+ "alias": "go_runtime_metadata",
+ "color": "#FF780A"
+ },
+ {
+ "$$hashKey": "object:283",
+ "alias": "other_memory",
+ "color": "#E02F44"
+ },
+ {
+ "$$hashKey": "object:297",
+ "alias": "free_mem_reserved_by_go",
+ "color": "rgb(112, 3, 166)"
+ },
+ {
+ "$$hashKey": "object:321",
+ "alias": "returned_to_os",
"bars": false,
- "color": "#C4162A",
- "hideTooltip": true,
- "legend": false,
- "pointradius": 3,
- "points": true,
+ "color": "#3274D9",
+ "lines": true,
"stack": false
}
],
@@ -134,76 +165,94 @@
"steppedLine": false,
"targets": [
{
+ "exemplar": true,
"expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}",
"format": "time_series",
"hide": false,
+ "interval": "",
"intervalFactor": 1,
- "legendFormat": "alloc-from-os",
+ "legendFormat": "rss",
"refId": "A"
},
{
- "expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} / (1 + tidb_server_gogc{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} / 100)",
+ "exemplar": true,
+ "expr": "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}",
"format": "time_series",
"hide": false,
+ "interval": "",
"intervalFactor": 1,
- "legendFormat": "estimate-inuse",
- "refId": "I"
+ "legendFormat": "heap_inuse",
+ "refId": "B"
},
{
- "expr": "go_memstats_alloc_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}",
+ "exemplar": true,
+ "expr": "go_memory_classes_heap_stacks_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}",
"format": "time_series",
"hide": false,
+ "interval": "",
"intervalFactor": 1,
- "legendFormat": "go-memstats-inuse",
- "refId": "H"
+ "legendFormat": "stack_inuse",
+ "refId": "C"
},
{
- "expr": "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} - go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} / (1 + tidb_server_gogc{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} / 100)",
+ "exemplar": true,
+ "expr": "go_memory_classes_heap_unused_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}",
"format": "time_series",
"hide": false,
+ "interval": "",
"intervalFactor": 1,
- "legendFormat": "estimate-garbage",
- "refId": "C"
+ "legendFormat": "heap_unused",
+ "refId": "D"
},
{
- "expr": "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} - go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} + go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}",
+ "exemplar": true,
+ "expr": "go_memory_classes_metadata_mcache_free_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}+go_memory_classes_metadata_mcache_inuse_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}+go_memory_classes_metadata_mspan_free_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}+go_memory_classes_metadata_mspan_inuse_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}+go_memory_classes_metadata_other_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}",
"format": "time_series",
"hide": false,
+ "interval": "",
"intervalFactor": 1,
- "legendFormat": "reserved-by-go",
- "refId": "B"
+ "legendFormat": "go_runtime_metadata",
+ "refId": "E"
},
{
- "expr": "go_memstats_stack_sys_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} + go_memstats_mspan_sys_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} + go_memstats_mcache_sys_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} + go_memstats_buck_hash_sys_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} + go_memstats_gc_sys_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} + go_memstats_other_sys_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}",
+ "exemplar": true,
+ "expr": "go_memory_classes_os_stacks_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}+go_memory_classes_other_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}+go_memory_classes_profiling_buckets_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}",
"format": "time_series",
"hide": false,
+ "interval": "",
"intervalFactor": 1,
- "legendFormat": "used-by-go",
- "refId": "D"
+ "legendFormat": "other_memory",
+ "refId": "F"
},
{
- "expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}",
+ "exemplar": true,
+ "expr": "go_memory_classes_heap_free_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}",
"format": "time_series",
"hide": false,
+ "interval": "",
"intervalFactor": 1,
- "legendFormat": "gc-threshold",
- "refId": "E"
+ "legendFormat": "free_mem_reserved_by_go",
+ "refId": "G"
},
{
- "expr": "(clamp_max(idelta(go_memstats_last_gc_time_seconds{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]), 1) * go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) > 0",
+ "exemplar": true,
+ "expr": "tidb_server_memory_quota_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", job=\"tidb\"}",
"format": "time_series",
- "hide": false,
+ "hide": true,
+ "interval": "",
"intervalFactor": 1,
- "legendFormat": "gc",
- "refId": "F"
+ "legendFormat": "tidb_server_memory_limit",
+ "refId": "H"
},
{
- "expr": "tidb_server_memory_quota_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", job=\"tidb\"}",
+ "exemplar": true,
+ "expr": "go_memory_classes_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", job=\"tidb\"}",
"format": "time_series",
"hide": false,
+ "interval": "",
"intervalFactor": 1,
- "legendFormat": "quota",
- "refId": "G"
+ "legendFormat": "returned_to_os",
+ "refId": "I"
}
],
"thresholds": [],
diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go
index a6a494b92092f..a2f2de523c11e 100644
--- a/pkg/metrics/metrics.go
+++ b/pkg/metrics/metrics.go
@@ -171,6 +171,8 @@ func RegisterMetrics() {
prometheus.MustRegister(PseudoEstimation)
prometheus.MustRegister(PacketIOCounter)
prometheus.MustRegister(QueryDurationHistogram)
+ prometheus.MustRegister(QueryRPCHistogram)
+ prometheus.MustRegister(QueryProcessedKeyHistogram)
prometheus.MustRegister(QueryTotalCounter)
prometheus.MustRegister(AffectedRowsCounter)
prometheus.MustRegister(SchemaLeaseErrorCounter)
diff --git a/pkg/metrics/server.go b/pkg/metrics/server.go
index 09d9b5ee3e838..d3d60e4b5657e 100644
--- a/pkg/metrics/server.go
+++ b/pkg/metrics/server.go
@@ -27,15 +27,17 @@ var (
// Metrics
var (
- PacketIOCounter *prometheus.CounterVec
- QueryDurationHistogram *prometheus.HistogramVec
- QueryTotalCounter *prometheus.CounterVec
- AffectedRowsCounter *prometheus.CounterVec
- ConnGauge *prometheus.GaugeVec
- DisconnectionCounter *prometheus.CounterVec
- PreparedStmtGauge prometheus.Gauge
- ExecuteErrorCounter *prometheus.CounterVec
- CriticalErrorCounter prometheus.Counter
+ PacketIOCounter *prometheus.CounterVec
+ QueryDurationHistogram *prometheus.HistogramVec
+ QueryRPCHistogram *prometheus.HistogramVec
+ QueryProcessedKeyHistogram *prometheus.HistogramVec
+ QueryTotalCounter *prometheus.CounterVec
+ AffectedRowsCounter *prometheus.CounterVec
+ ConnGauge *prometheus.GaugeVec
+ DisconnectionCounter *prometheus.CounterVec
+ PreparedStmtGauge prometheus.Gauge
+ ExecuteErrorCounter *prometheus.CounterVec
+ CriticalErrorCounter prometheus.Counter
ServerStart = "server-start"
ServerStop = "server-stop"
@@ -92,6 +94,24 @@ func InitServerMetrics() {
Buckets: prometheus.ExponentialBuckets(0.0005, 2, 29), // 0.5ms ~ 1.5days
}, []string{LblSQLType, LblDb, LblResourceGroup})
+ QueryRPCHistogram = NewHistogramVec(
+ prometheus.HistogramOpts{
+ Namespace: "tidb",
+ Subsystem: "server",
+ Name: "query_statement_rpc_count",
+ Help: "Bucketed histogram of execution rpc count of handled query statements.",
+ Buckets: prometheus.ExponentialBuckets(1, 1.5, 23), // 1 ~ 8388608
+ }, []string{LblSQLType, LblDb})
+
+ QueryProcessedKeyHistogram = NewHistogramVec(
+ prometheus.HistogramOpts{
+ Namespace: "tidb",
+ Subsystem: "server",
+ Name: "query_statement_processed_keys",
+ Help: "Bucketed histogram of processed key count during the scan of handled query statements.",
+ Buckets: prometheus.ExponentialBuckets(1, 2, 32),
+ }, []string{LblSQLType, LblDb})
+
QueryTotalCounter = NewCounterVec(
prometheus.CounterOpts{
Namespace: "tidb",
diff --git a/pkg/parser/ast/ddl.go b/pkg/parser/ast/ddl.go
index 90af3c4e3c2c9..0c39cbfceaf9a 100644
--- a/pkg/parser/ast/ddl.go
+++ b/pkg/parser/ast/ddl.go
@@ -573,6 +573,9 @@ func (n *ColumnOption) Restore(ctx *format.RestoreCtx) error {
return nil
})
}
+ if n.StrValue == "Global" {
+ ctx.WriteKeyWord(" GLOBAL")
+ }
case ColumnOptionNotNull:
ctx.WriteKeyWord("NOT NULL")
case ColumnOptionAutoIncrement:
@@ -596,6 +599,9 @@ func (n *ColumnOption) Restore(ctx *format.RestoreCtx) error {
}
case ColumnOptionUniqKey:
ctx.WriteKeyWord("UNIQUE KEY")
+ if n.StrValue == "Global" {
+ ctx.WriteKeyWord(" GLOBAL")
+ }
case ColumnOptionNull:
ctx.WriteKeyWord("NULL")
case ColumnOptionOnUpdate:
@@ -715,8 +721,10 @@ const (
// | index_type
// | WITH PARSER parser_name
// | COMMENT 'string'
+// | GLOBAL
//
// See http://dev.mysql.com/doc/refman/5.7/en/create-table.html
+// with the addition of Global Index
type IndexOption struct {
node
@@ -726,6 +734,22 @@ type IndexOption struct {
ParserName model.CIStr
Visibility IndexVisibility
PrimaryKeyTp model.PrimaryKeyType
+ Global bool
+}
+
+// IsEmpty is true if only default options are given
+// and it should not be added to the output
+func (n *IndexOption) IsEmpty() bool {
+ if n.PrimaryKeyTp != model.PrimaryKeyTypeDefault ||
+ n.KeyBlockSize > 0 ||
+ n.Tp != model.IndexTypeInvalid ||
+ len(n.ParserName.O) > 0 ||
+ n.Comment != "" ||
+ n.Global ||
+ n.Visibility != IndexVisibilityDefault {
+ return false
+ }
+ return true
}
// Restore implements Node interface.
@@ -774,6 +798,17 @@ func (n *IndexOption) Restore(ctx *format.RestoreCtx) error {
hasPrevOption = true
}
+ if n.Global {
+ if hasPrevOption {
+ ctx.WritePlain(" ")
+ }
+ _ = ctx.WriteWithSpecialComments(tidb.FeatureIDGlobalIndex, func() error {
+ ctx.WriteKeyWord("GLOBAL")
+ return nil
+ })
+ hasPrevOption = true
+ }
+
if n.Visibility != IndexVisibilityDefault {
if hasPrevOption {
ctx.WritePlain(" ")
@@ -920,7 +955,7 @@ func (n *Constraint) Restore(ctx *format.RestoreCtx) error {
}
}
- if n.Option != nil {
+ if n.Option != nil && !n.Option.IsEmpty() {
ctx.WritePlain(" ")
if err := n.Option.Restore(ctx); err != nil {
return errors.Annotate(err, "An error occurred while splicing Constraint Option")
@@ -1823,7 +1858,7 @@ func (n *CreateIndexStmt) Restore(ctx *format.RestoreCtx) error {
}
ctx.WritePlain(")")
- if n.IndexOption.Tp != model.IndexTypeInvalid || n.IndexOption.KeyBlockSize > 0 || n.IndexOption.Comment != "" || len(n.IndexOption.ParserName.O) > 0 || n.IndexOption.Visibility != IndexVisibilityDefault {
+ if n.IndexOption != nil && !n.IndexOption.IsEmpty() {
ctx.WritePlain(" ")
if err := n.IndexOption.Restore(ctx); err != nil {
return errors.Annotate(err, "An error occurred while restore CreateIndexStmt.IndexOption")
@@ -2202,7 +2237,11 @@ func (n *ResourceGroupOption) Restore(ctx *format.RestoreCtx) error {
case ResourceRURate:
ctx.WriteKeyWord("RU_PER_SEC ")
ctx.WritePlain("= ")
- ctx.WritePlainf("%d", n.UintValue)
+ if n.BoolValue {
+ ctx.WriteKeyWord("UNLIMITED")
+ } else {
+ ctx.WritePlainf("%d", n.UintValue)
+ }
case ResourcePriority:
ctx.WriteKeyWord("PRIORITY ")
ctx.WritePlain("= ")
@@ -4340,8 +4379,9 @@ func (n *PartitionMethod) acceptInPlace(v Visitor) bool {
// PartitionOptions specifies the partition options.
type PartitionOptions struct {
PartitionMethod
- Sub *PartitionMethod
- Definitions []*PartitionDefinition
+ Sub *PartitionMethod
+ Definitions []*PartitionDefinition
+ UpdateIndexes []*Constraint
}
// Validate checks if the partition is well-formed.
@@ -4435,6 +4475,22 @@ func (n *PartitionOptions) Restore(ctx *format.RestoreCtx) error {
ctx.WritePlain(")")
}
+ if len(n.UpdateIndexes) > 0 {
+ ctx.WritePlain(" UPDATE INDEXES (")
+ for i, update := range n.UpdateIndexes {
+ if i > 0 {
+ ctx.WritePlain(",")
+ }
+ ctx.WriteName(update.Name)
+ if update.Option != nil && update.Option.Global {
+ ctx.WritePlain(" GLOBAL")
+ } else {
+ ctx.WritePlain(" LOCAL")
+ }
+ }
+ ctx.WritePlain(")")
+ }
+
return nil
}
diff --git a/pkg/parser/ast/misc.go b/pkg/parser/ast/misc.go
index fa07d309953d0..2be542f069cfe 100644
--- a/pkg/parser/ast/misc.go
+++ b/pkg/parser/ast/misc.go
@@ -1971,6 +1971,40 @@ func (n *DropUserStmt) Accept(v Visitor) (Node, bool) {
return v.Leave(n)
}
+type StringOrUserVar struct {
+ node
+ StringLit string
+ UserVar *VariableExpr
+}
+
+func (n *StringOrUserVar) Restore(ctx *format.RestoreCtx) error {
+ if len(n.StringLit) > 0 {
+ ctx.WriteString(n.StringLit)
+ }
+ if n.UserVar != nil {
+ if err := n.UserVar.Restore(ctx); err != nil {
+ return errors.Annotate(err, "An error occurred while restore ColumnNameOrUserVar.UserVar")
+ }
+ }
+ return nil
+}
+
+func (n *StringOrUserVar) Accept(v Visitor) (node Node, ok bool) {
+ newNode, skipChild := v.Enter(n)
+ if skipChild {
+ return v.Leave(newNode)
+ }
+ n = newNode.(*StringOrUserVar)
+ if n.UserVar != nil {
+ node, ok = n.UserVar.Accept(v)
+ if !ok {
+ return node, false
+ }
+ n.UserVar = node.(*VariableExpr)
+ }
+ return v.Leave(n)
+}
+
// CreateBindingStmt creates sql binding hint.
type CreateBindingStmt struct {
stmtNode
@@ -1978,7 +2012,7 @@ type CreateBindingStmt struct {
GlobalScope bool
OriginNode StmtNode
HintedNode StmtNode
- PlanDigest string
+ PlanDigests []*StringOrUserVar
}
func (n *CreateBindingStmt) Restore(ctx *format.RestoreCtx) error {
@@ -1990,7 +2024,14 @@ func (n *CreateBindingStmt) Restore(ctx *format.RestoreCtx) error {
}
if n.OriginNode == nil {
ctx.WriteKeyWord("BINDING FROM HISTORY USING PLAN DIGEST ")
- ctx.WriteString(n.PlanDigest)
+ for i, v := range n.PlanDigests {
+ if i != 0 {
+ ctx.WritePlain(", ")
+ }
+ if err := v.Restore(ctx); err != nil {
+ return errors.Annotatef(err, "An error occurred while restore CreateBindingStmt.PlanDigests[%d]", i)
+ }
+ }
} else {
ctx.WriteKeyWord("BINDING FOR ")
if err := n.OriginNode.Restore(ctx); err != nil {
@@ -2021,6 +2062,14 @@ func (n *CreateBindingStmt) Accept(v Visitor) (Node, bool) {
return n, false
}
n.HintedNode = hintedNode.(StmtNode)
+ } else {
+ for i, digest := range n.PlanDigests {
+ newDigest, ok := digest.Accept(v)
+ if !ok {
+ return n, false
+ }
+ n.PlanDigests[i] = newDigest.(*StringOrUserVar)
+ }
}
return v.Leave(n)
}
@@ -2032,7 +2081,7 @@ type DropBindingStmt struct {
GlobalScope bool
OriginNode StmtNode
HintedNode StmtNode
- SQLDigest string
+ SQLDigests []*StringOrUserVar
}
func (n *DropBindingStmt) Restore(ctx *format.RestoreCtx) error {
@@ -2045,7 +2094,14 @@ func (n *DropBindingStmt) Restore(ctx *format.RestoreCtx) error {
ctx.WriteKeyWord("BINDING FOR ")
if n.OriginNode == nil {
ctx.WriteKeyWord("SQL DIGEST ")
- ctx.WriteString(n.SQLDigest)
+ for i, v := range n.SQLDigests {
+ if i != 0 {
+ ctx.WritePlain(", ")
+ }
+ if err := v.Restore(ctx); err != nil {
+ return errors.Annotatef(err, "An error occurred while restore CreateBindingStmt.PlanDigests[%d]", i)
+ }
+ }
} else {
if err := n.OriginNode.Restore(ctx); err != nil {
return errors.Trace(err)
@@ -2080,6 +2136,14 @@ func (n *DropBindingStmt) Accept(v Visitor) (Node, bool) {
}
n.HintedNode = hintedNode.(StmtNode)
}
+ } else {
+ for i, digest := range n.SQLDigests {
+ newDigest, ok := digest.Accept(v)
+ if !ok {
+ return n, false
+ }
+ n.SQLDigests[i] = newDigest.(*StringOrUserVar)
+ }
}
return v.Leave(n)
}
diff --git a/pkg/parser/ast/util.go b/pkg/parser/ast/util.go
index 1683041f2ae82..6ae6cf2bcde70 100644
--- a/pkg/parser/ast/util.go
+++ b/pkg/parser/ast/util.go
@@ -26,7 +26,8 @@ func IsReadOnly(node Node) bool {
case *SelectStmt:
if st.LockInfo != nil {
switch st.LockInfo.LockType {
- case SelectLockForUpdate, SelectLockForUpdateNoWait, SelectLockForUpdateWaitN:
+ case SelectLockForUpdate, SelectLockForUpdateNoWait, SelectLockForUpdateWaitN,
+ SelectLockForShare, SelectLockForShareNoWait:
return false
}
}
diff --git a/pkg/parser/generate_keyword/genkeyword.go b/pkg/parser/generate_keyword/genkeyword.go
index edfaa05dd5c8e..0c89c2b8fc3e5 100644
--- a/pkg/parser/generate_keyword/genkeyword.go
+++ b/pkg/parser/generate_keyword/genkeyword.go
@@ -36,7 +36,8 @@ const (
)
const (
- fileStart = `// Copyright 2023 PingCAP, Inc.
+ fileStart = `// Code generated by genkeyword. DO NOT EDIT.
+// Copyright 2023 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -75,8 +76,6 @@ var keywordRe *regexp.Regexp
// example data:
//
// add "ADD"
-//
-// Note that all keywords except `TiDB_CURRENT_TSO` are fully uppercase.
func parseLine(line string) string {
if keywordRe == nil {
keywordRe = regexp.MustCompile(`^\s+\w+\s+"(\w+)"$`)
diff --git a/pkg/parser/generate_keyword/genkeyword_test.go b/pkg/parser/generate_keyword/genkeyword_test.go
index 2a7a7d5b24722..8e3f49fac4a1a 100644
--- a/pkg/parser/generate_keyword/genkeyword_test.go
+++ b/pkg/parser/generate_keyword/genkeyword_test.go
@@ -10,6 +10,6 @@ func TestParseLine(t *testing.T) {
add := parseLine(" add \"ADD\"")
require.Equal(t, add, "ADD")
- tso := parseLine(" tidbCurrentTSO \"TiDB_CURRENT_TSO\"")
- require.Equal(t, tso, "TiDB_CURRENT_TSO")
+ tso := parseLine(" tidbCurrentTSO \"TIDB_CURRENT_TSO\"")
+ require.Equal(t, tso, "TIDB_CURRENT_TSO")
}
diff --git a/pkg/parser/keywords.go b/pkg/parser/keywords.go
index 74dc5a1b3214f..3eef0c62be347 100644
--- a/pkg/parser/keywords.go
+++ b/pkg/parser/keywords.go
@@ -1,3 +1,4 @@
+// Code generated by genkeyword. DO NOT EDIT.
// Copyright 2023 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
@@ -221,6 +222,7 @@ var Keywords = []KeywordsType{
{"TABLESAMPLE", true, "reserved"},
{"TERMINATED", true, "reserved"},
{"THEN", true, "reserved"},
+ {"TIDB_CURRENT_TSO", true, "reserved"},
{"TINYBLOB", true, "reserved"},
{"TINYINT", true, "reserved"},
{"TINYTEXT", true, "reserved"},
@@ -228,7 +230,6 @@ var Keywords = []KeywordsType{
{"TRAILING", true, "reserved"},
{"TRIGGER", true, "reserved"},
{"TRUE", true, "reserved"},
- {"TiDB_CURRENT_TSO", true, "reserved"},
{"UNION", true, "reserved"},
{"UNIQUE", true, "reserved"},
{"UNLOCK", true, "reserved"},
@@ -622,6 +623,7 @@ var Keywords = []KeywordsType{
{"VALIDATION", false, "unreserved"},
{"VALUE", false, "unreserved"},
{"VARIABLES", false, "unreserved"},
+ {"VECTOR", false, "unreserved"},
{"VIEW", false, "unreserved"},
{"VISIBLE", false, "unreserved"},
{"WAIT", false, "unreserved"},
diff --git a/pkg/parser/keywords_test.go b/pkg/parser/keywords_test.go
index 8caca9048b7cf..2925623635dea 100644
--- a/pkg/parser/keywords_test.go
+++ b/pkg/parser/keywords_test.go
@@ -36,7 +36,7 @@ func TestKeywords(t *testing.T) {
}
func TestKeywordsLength(t *testing.T) {
- require.Equal(t, 653, len(parser.Keywords))
+ require.Equal(t, 654, len(parser.Keywords))
reservedNr := 0
for _, kw := range parser.Keywords {
diff --git a/pkg/parser/model/ddl.go b/pkg/parser/model/ddl.go
index 6bed14a8bc88c..4a57fda2e5a91 100644
--- a/pkg/parser/model/ddl.go
+++ b/pkg/parser/model/ddl.go
@@ -456,7 +456,6 @@ func (sub *SubJob) ToProxyJob(parentJob *Job, seq int) Job {
Collate: parentJob.Collate,
AdminOperator: parentJob.AdminOperator,
TraceInfo: parentJob.TraceInfo,
- LocalMode: parentJob.LocalMode,
}
}
@@ -584,6 +583,7 @@ type Job struct {
// LocalMode = true means the job is running on the local TiDB that the client
// connects to, else it's run on the DDL owner.
// Only happens when tidb_enable_fast_create_table = on
+ // this field is useless since 8.3
LocalMode bool `json:"local_mode"`
// SQLMode for executing DDL query.
@@ -987,6 +987,11 @@ func (job *Job) NotStarted() bool {
return job.State == JobStateNone || job.State == JobStateQueueing
}
+// Started returns true if the job is started.
+func (job *Job) Started() bool {
+ return !job.NotStarted()
+}
+
// InFinalState returns whether the job is in a final state of job FSM.
// TODO JobStateRollbackDone is not a final state, maybe we should add a JobStateRollbackSynced
// state to diff between the entrance of JobStateRollbackDone and move the job to
diff --git a/pkg/parser/model/model.go b/pkg/parser/model/model.go
index 0e8440ac65471..a3f2bd138dde2 100644
--- a/pkg/parser/model/model.go
+++ b/pkg/parser/model/model.go
@@ -1173,6 +1173,13 @@ type ExchangePartitionInfo struct {
XXXExchangePartitionFlag bool `json:"exchange_partition_flag"`
}
+// UpdateIndexInfo is to carry the entries in the list of indexes in UPDATE INDEXES
+// during ALTER TABLE t PARTITION BY ... UPDATE INDEXES (idx_a GLOBAL, idx_b LOCAL...)
+type UpdateIndexInfo struct {
+ IndexName string `json:"index_name"`
+ Global bool `json:"global"`
+}
+
// PartitionInfo provides table partition info.
type PartitionInfo struct {
Type PartitionType `json:"type"`
@@ -1210,6 +1217,8 @@ type PartitionInfo struct {
DDLType PartitionType `json:"ddl_type"`
DDLExpr string `json:"ddl_expr"`
DDLColumns []CIStr `json:"ddl_columns"`
+ // For ActionAlterTablePartitioning, UPDATE INDEXES
+ DDLUpdateIndexes []UpdateIndexInfo `json:"ddl_update_indexes"`
}
// Clone clones itself.
@@ -1687,11 +1696,13 @@ func (fk *FKInfo) Clone() *FKInfo {
// DBInfo provides meta data describing a DB.
type DBInfo struct {
- ID int64 `json:"id"` // Database ID
- Name CIStr `json:"db_name"` // DB name.
- Charset string `json:"charset"`
- Collate string `json:"collate"`
- Tables []*TableInfo `json:"-"` // Tables in the DB.
+ ID int64 `json:"id"` // Database ID
+ Name CIStr `json:"db_name"` // DB name.
+ Charset string `json:"charset"`
+ Collate string `json:"collate"`
+ Deprecated struct { // Tables is not set in infoschema v2, use infoschema SchemaTableInfos() instead.
+ Tables []*TableInfo `json:"-"` // Tables in the DB.
+ }
State SchemaState `json:"state"`
PlacementPolicyRef *PolicyRefInfo `json:"policy_ref_info"`
TableName2ID map[string]int64 `json:"-"`
@@ -1700,9 +1711,9 @@ type DBInfo struct {
// Clone clones DBInfo.
func (db *DBInfo) Clone() *DBInfo {
newInfo := *db
- newInfo.Tables = make([]*TableInfo, len(db.Tables))
- for i := range db.Tables {
- newInfo.Tables[i] = db.Tables[i].Clone()
+ newInfo.Deprecated.Tables = make([]*TableInfo, len(db.Deprecated.Tables))
+ for i := range db.Deprecated.Tables {
+ newInfo.Deprecated.Tables[i] = db.Deprecated.Tables[i].Clone()
}
return &newInfo
}
@@ -1710,8 +1721,8 @@ func (db *DBInfo) Clone() *DBInfo {
// Copy shallow copies DBInfo.
func (db *DBInfo) Copy() *DBInfo {
newInfo := *db
- newInfo.Tables = make([]*TableInfo, len(db.Tables))
- copy(newInfo.Tables, db.Tables)
+ newInfo.Deprecated.Tables = make([]*TableInfo, len(db.Deprecated.Tables))
+ copy(newInfo.Deprecated.Tables, db.Deprecated.Tables)
return &newInfo
}
@@ -2000,12 +2011,6 @@ func (t RunawayActionType) String() string {
}
}
-// ResourceGroupRefInfo is the struct to refer the resource group.
-type ResourceGroupRefInfo struct {
- ID int64 `json:"id"`
- Name CIStr `json:"name"`
-}
-
// ResourceGroupRunawaySettings is the runaway settings of the resource group
type ResourceGroupRunawaySettings struct {
ExecElapsedTimeMs uint64 `json:"exec_elapsed_time_ms"`
diff --git a/pkg/parser/model/model_test.go b/pkg/parser/model/model_test.go
index 71762e97869bf..76bfd0b19790e 100644
--- a/pkg/parser/model/model_test.go
+++ b/pkg/parser/model/model_test.go
@@ -168,8 +168,8 @@ func TestModelBasic(t *testing.T) {
Name: NewCIStr("test"),
Charset: "utf8",
Collate: "utf8_bin",
- Tables: []*TableInfo{table},
}
+ dbInfo.Deprecated.Tables = []*TableInfo{table}
n := dbInfo.Clone()
require.Equal(t, dbInfo, n)
diff --git a/pkg/parser/model/reorg.go b/pkg/parser/model/reorg.go
index 9355d796271eb..bcdfa9b358efc 100644
--- a/pkg/parser/model/reorg.go
+++ b/pkg/parser/model/reorg.go
@@ -34,6 +34,26 @@ type DDLReorgMeta struct {
ResourceGroupName string `json:"resource_group_name"`
Version int64 `json:"version"`
TargetScope string `json:"target_scope"`
+ // These two variables are set when corresponding session variables are set explicitly. When they are set,
+ // user cannot change it by setting the global one. Otherwise, they can be adjusted dynamically through global var.
+ Concurrency int `json:"concurrency"`
+ BatchSize int `json:"batch_size"`
+}
+
+// GetConcurrencyOrDefault gets the concurrency from DDLReorgMeta or returns the default value.
+func (dm *DDLReorgMeta) GetConcurrencyOrDefault(defaultVal int) int {
+ if dm == nil || dm.Concurrency == 0 {
+ return defaultVal
+ }
+ return dm.Concurrency
+}
+
+// GetBatchSizeOrDefault gets the batch size from DDLReorgMeta or returns the default value.
+func (dm *DDLReorgMeta) GetBatchSizeOrDefault(defaultVal int) int {
+ if dm == nil || dm.BatchSize == 0 {
+ return defaultVal
+ }
+ return dm.BatchSize
}
const (
diff --git a/pkg/parser/parser.go b/pkg/parser/parser.go
index d79df38c6d85f..5c8c5cc545a09 100644
--- a/pkg/parser/parser.go
+++ b/pkg/parser/parser.go
@@ -814,7 +814,7 @@ const (
then = 57559
tiFlash = 58159
tidb = 58158
- tidbCurrentTSO = 57567
+ tidbCurrentTSO = 57560
tidbJson = 58065
tikvImporter = 57936
timeDuration = 58066
@@ -822,11 +822,11 @@ const (
timestampAdd = 58067
timestampDiff = 58068
timestampType = 57938
- tinyIntType = 57561
- tinyblobType = 57560
- tinytextType = 57562
+ tinyIntType = 57562
+ tinyblobType = 57561
+ tinytextType = 57563
tls = 58069
- to = 57563
+ to = 57564
toTSO = 57349
toTimestamp = 57348
tokenIssuer = 57939
@@ -846,13 +846,13 @@ const (
tpch10 = 57941
trace = 57942
traditional = 57943
- trailing = 57564
+ trailing = 57565
transaction = 57944
- trigger = 57565
+ trigger = 57566
triggers = 57945
trim = 58080
trueCardCost = 58081
- trueKwd = 57566
+ trueKwd = 57567
truncate = 57946
tsoType = 57947
ttl = 57948
@@ -922,578 +922,578 @@ const (
zerofill = 57594
yyMaxDepth = 200
- yyTabOfs = -2896
+ yyTabOfs = -2915
)
var (
yyXLAT = map[int]int{
- 59: 0, // ';' (2542x)
- 57344: 1, // $end (2529x)
- 57848: 2, // remove (2020x)
- 58148: 3, // split (2020x)
- 57777: 4, // merge (2019x)
- 57849: 5, // reorganize (2018x)
- 57650: 6, // comment (2012x)
- 57919: 7, // storage (1923x)
- 57608: 8, // autoIncrement (1912x)
- 44: 9, // ',' (1884x)
- 57717: 10, // first (1811x)
- 57598: 11, // after (1805x)
- 57882: 12, // serial (1801x)
- 57609: 13, // autoRandom (1800x)
- 57649: 14, // columnFormat (1800x)
- 57818: 15, // password (1765x)
- 57635: 16, // charsetKwd (1756x)
- 57637: 17, // checksum (1746x)
- 58033: 18, // placement (1743x)
- 57752: 19, // keyBlockSize (1727x)
- 57930: 20, // tablespace (1723x)
- 57693: 21, // encryption (1721x)
- 57698: 22, // engine (1718x)
- 57674: 23, // data (1716x)
- 57743: 24, // insertMethod (1714x)
- 57771: 25, // maxRows (1714x)
- 57781: 26, // minRows (1714x)
- 57794: 27, // nodegroup (1714x)
- 57660: 28, // connection (1706x)
- 57610: 29, // autoRandomBase (1703x)
- 58151: 30, // statsBuckets (1701x)
- 58157: 31, // statsTopN (1701x)
- 57948: 32, // ttl (1701x)
- 57607: 33, // autoIdCache (1700x)
- 57612: 34, // avgRowLength (1700x)
- 57655: 35, // compression (1700x)
- 57681: 36, // delayKeyWrite (1700x)
- 57812: 37, // packKeys (1700x)
- 57831: 38, // preSplitRegions (1700x)
- 57869: 39, // rowFormat (1700x)
- 57875: 40, // secondaryEngine (1700x)
- 57886: 41, // shardRowIDBits (1700x)
- 57911: 42, // statsAutoRecalc (1700x)
- 57912: 43, // statsColChoice (1700x)
- 57913: 44, // statsColList (1700x)
- 57915: 45, // statsPersistent (1700x)
- 57916: 46, // statsSamplePages (1700x)
- 57917: 47, // statsSampleRate (1700x)
- 57931: 48, // tableChecksum (1700x)
- 57949: 49, // ttlEnable (1700x)
- 57950: 50, // ttlJobInterval (1700x)
- 57856: 51, // resource (1679x)
- 41: 52, // ')' (1653x)
- 57605: 53, // attribute (1652x)
- 57595: 54, // account (1650x)
- 57713: 55, // failedLoginAttempts (1650x)
- 57819: 56, // passwordLockTime (1650x)
- 57346: 57, // identifier (1648x)
- 57861: 58, // resume (1636x)
- 57890: 59, // signed (1636x)
- 57896: 60, // snapshot (1634x)
- 57613: 61, // backend (1633x)
- 57636: 62, // checkpoint (1633x)
- 57638: 63, // checksumConcurrency (1633x)
- 57656: 64, // compressionLevel (1633x)
- 57657: 65, // compressionType (1633x)
- 57658: 66, // concurrency (1633x)
- 57665: 67, // csvBackslashEscape (1633x)
- 57666: 68, // csvDelimiter (1633x)
- 57667: 69, // csvHeader (1633x)
- 57668: 70, // csvNotNull (1633x)
- 57669: 71, // csvNull (1633x)
- 57670: 72, // csvSeparator (1633x)
- 57671: 73, // csvTrimLastSeparators (1633x)
- 57694: 74, // encryptionKeyFile (1633x)
- 57695: 75, // encryptionMethod (1633x)
- 58007: 76, // fullBackupStorage (1633x)
- 58008: 77, // gcTTL (1633x)
- 57737: 78, // ignoreStats (1633x)
- 57757: 79, // lastBackup (1633x)
- 57761: 80, // loadStats (1633x)
- 57809: 81, // onDuplicate (1633x)
- 57807: 82, // online (1633x)
- 57843: 83, // rateLimit (1633x)
- 58043: 84, // restoredTS (1633x)
- 57879: 85, // sendCredentialsToTiKV (1633x)
- 57893: 86, // skipSchemaFiles (1633x)
- 58051: 87, // startTS (1633x)
- 57920: 88, // strictFormat (1633x)
- 57936: 89, // tikvImporter (1633x)
- 58083: 90, // untilTS (1633x)
- 57966: 91, // waitTiflashReady (1633x)
- 57971: 92, // withSysTable (1633x)
- 57617: 93, // begin (1627x)
- 57651: 94, // commit (1627x)
- 57791: 95, // no (1627x)
- 57865: 96, // rollback (1627x)
- 57910: 97, // start (1625x)
- 57946: 98, // truncate (1624x)
- 57629: 99, // cache (1622x)
- 57792: 100, // nocache (1621x)
- 57810: 101, // open (1621x)
- 57596: 102, // action (1620x)
- 57643: 103, // close (1620x)
- 57673: 104, // cycle (1620x)
- 57780: 105, // minValue (1620x)
- 57696: 106, // end (1619x)
- 57740: 107, // increment (1619x)
- 57793: 108, // nocycle (1619x)
- 57795: 109, // nomaxvalue (1619x)
- 57796: 110, // nominvalue (1619x)
- 57601: 111, // algorithm (1617x)
- 57858: 112, // restart (1617x)
- 57951: 113, // tp (1617x)
- 57645: 114, // clustered (1616x)
- 57745: 115, // invisible (1616x)
- 57797: 116, // nonclustered (1616x)
- 58142: 117, // regions (1616x)
- 57964: 118, // visible (1616x)
- 57978: 119, // background (1614x)
- 57985: 120, // burstable (1614x)
- 58039: 121, // priority (1614x)
- 58040: 122, // queryLimit (1614x)
- 58045: 123, // ruRate (1614x)
- 57922: 124, // subpartition (1612x)
- 57817: 125, // partitions (1611x)
- 58035: 126, // plan (1611x)
- 57974: 127, // yearType (1611x)
- 57987: 128, // constraints (1609x)
- 58005: 129, // followerConstraints (1609x)
- 58006: 130, // followers (1609x)
- 58020: 131, // leaderConstraints (1609x)
- 58022: 132, // learnerConstraints (1609x)
- 58023: 133, // learners (1609x)
- 58038: 134, // primaryRegion (1609x)
- 58047: 135, // schedule (1609x)
- 57909: 136, // sqlTsiYear (1609x)
- 58062: 137, // survivalPreferences (1609x)
- 58088: 138, // voterConstraints (1609x)
- 58089: 139, // voters (1609x)
- 57648: 140, // columns (1607x)
- 57738: 141, // importKwd (1607x)
- 57963: 142, // view (1607x)
- 57677: 143, // day (1606x)
- 58091: 144, // watch (1605x)
- 57994: 145, // defined (1604x)
- 58000: 146, // execElapsed (1604x)
- 57873: 147, // second (1604x)
- 57918: 148, // status (1604x)
- 57734: 149, // hour (1603x)
- 57778: 150, // microsecond (1603x)
- 57779: 151, // minute (1603x)
- 57784: 152, // month (1603x)
- 57839: 153, // quarter (1603x)
- 57902: 154, // sqlTsiDay (1603x)
- 57903: 155, // sqlTsiHour (1603x)
- 57904: 156, // sqlTsiMinute (1603x)
- 57905: 157, // sqlTsiMonth (1603x)
- 57906: 158, // sqlTsiQuarter (1603x)
- 57907: 159, // sqlTsiSecond (1603x)
- 57908: 160, // sqlTsiWeek (1603x)
- 57968: 161, // week (1603x)
- 57604: 162, // ascii (1602x)
- 57628: 163, // byteType (1602x)
- 57929: 164, // tables (1602x)
- 57955: 165, // unicodeSym (1602x)
- 57715: 166, // fields (1601x)
- 57762: 167, // local (1600x)
- 57765: 168, // logs (1600x)
- 58066: 169, // timeDuration (1600x)
- 57841: 170, // query (1598x)
- 57880: 171, // separator (1598x)
- 57639: 172, // cipher (1597x)
- 57750: 173, // issuer (1597x)
- 57767: 174, // maxConnectionsPerHour (1597x)
- 57770: 175, // maxQueriesPerHour (1597x)
- 57772: 176, // maxUpdatesPerHour (1597x)
- 57773: 177, // maxUserConnections (1597x)
- 57828: 178, // preceding (1597x)
- 57871: 179, // san (1597x)
- 57921: 180, // subject (1597x)
- 57939: 181, // tokenIssuer (1597x)
- 57998: 182, // endTime (1596x)
- 57751: 183, // jsonType (1596x)
- 58050: 184, // startTime (1596x)
- 57676: 185, // datetimeType (1595x)
- 57675: 186, // dateType (1595x)
- 57718: 187, // fixed (1595x)
- 57937: 188, // timeType (1595x)
- 57620: 189, // bindings (1594x)
- 57672: 190, // current (1594x)
- 57680: 191, // definer (1594x)
- 57729: 192, // hash (1594x)
- 57736: 193, // identified (1594x)
- 57857: 194, // respect (1594x)
- 57864: 195, // role (1594x)
- 57938: 196, // timestampType (1594x)
- 57960: 197, // value (1594x)
- 57962: 198, // vectorType (1594x)
- 57614: 199, // backup (1593x)
- 57626: 200, // booleanType (1593x)
- 57697: 201, // enforced (1593x)
- 57720: 202, // following (1593x)
- 57758: 203, // less (1593x)
- 57799: 204, // nowait (1593x)
- 57808: 205, // only (1593x)
- 57872: 206, // savepoint (1593x)
- 57892: 207, // skip (1593x)
- 58064: 208, // taskTypes (1593x)
- 57934: 209, // textType (1593x)
- 57935: 210, // than (1593x)
- 58159: 211, // tiFlash (1593x)
- 57952: 212, // unbounded (1593x)
- 57619: 213, // binding (1592x)
- 57623: 214, // bitType (1592x)
- 57625: 215, // boolType (1592x)
- 57700: 216, // enum (1592x)
- 57726: 217, // global (1592x)
- 57735: 218, // hypo (1592x)
- 58134: 219, // job (1592x)
- 57786: 220, // national (1592x)
- 57787: 221, // ncharType (1592x)
- 58030: 222, // next_row_id (1592x)
- 57801: 223, // nvarcharType (1592x)
- 57803: 224, // offset (1592x)
- 57827: 225, // policy (1592x)
- 58037: 226, // predicate (1592x)
- 57852: 227, // replica (1592x)
- 57932: 228, // temporary (1592x)
- 57958: 229, // user (1592x)
- 57682: 230, // digest (1591x)
- 58135: 231, // jobs (1591x)
- 57763: 232, // location (1591x)
- 58034: 233, // planCache (1591x)
- 57829: 234, // prepare (1591x)
- 58150: 235, // stats (1591x)
- 57956: 236, // unknown (1591x)
- 57965: 237, // wait (1591x)
- 57627: 238, // btree (1590x)
- 57988: 239, // cooldown (1590x)
- 57679: 240, // declare (1590x)
- 57996: 241, // dryRun (1590x)
- 57721: 242, // format (1590x)
- 57749: 243, // isolation (1590x)
- 57755: 244, // last (1590x)
- 57768: 245, // max_idxnum (1590x)
- 57776: 246, // memory (1590x)
- 57789: 247, // next (1590x)
- 57802: 248, // off (1590x)
- 57811: 249, // optional (1590x)
- 57822: 250, // per_db (1590x)
- 57832: 251, // privileges (1590x)
- 57855: 252, // required (1590x)
- 57870: 253, // rtree (1590x)
- 58145: 254, // sampleRate (1590x)
- 57881: 255, // sequence (1590x)
- 57884: 256, // session (1590x)
- 57895: 257, // slow (1590x)
- 57959: 258, // validation (1590x)
- 57961: 259, // variables (1590x)
- 57606: 260, // attributes (1589x)
- 58123: 261, // cancel (1589x)
- 57653: 262, // compact (1589x)
- 58128: 263, // ddl (1589x)
- 57684: 264, // disable (1589x)
- 57688: 265, // do (1589x)
- 57690: 266, // dynamic (1589x)
- 57691: 267, // enable (1589x)
- 57701: 268, // errorKwd (1589x)
- 57999: 269, // exact (1589x)
- 57719: 270, // flush (1589x)
- 57723: 271, // full (1589x)
- 57728: 272, // handler (1589x)
- 57732: 273, // history (1589x)
- 57774: 274, // mb (1589x)
- 57782: 275, // mode (1589x)
- 57820: 276, // pause (1589x)
- 57825: 277, // plugins (1589x)
- 57834: 278, // processlist (1589x)
- 57845: 279, // recover (1589x)
- 57850: 280, // repair (1589x)
- 57851: 281, // repeatable (1589x)
- 58048: 282, // similar (1589x)
- 58149: 283, // statistics (1589x)
- 57923: 284, // subpartitions (1589x)
- 58158: 285, // tidb (1589x)
- 57970: 286, // without (1589x)
- 58092: 287, // admin (1588x)
- 58093: 288, // batch (1588x)
- 57616: 289, // bdr (1588x)
- 57622: 290, // binlog (1588x)
- 57624: 291, // block (1588x)
- 57983: 292, // br (1588x)
- 57984: 293, // briefType (1588x)
- 58094: 294, // buckets (1588x)
- 57630: 295, // calibrate (1588x)
- 57631: 296, // capture (1588x)
- 58124: 297, // cardinality (1588x)
- 57634: 298, // chain (1588x)
- 57642: 299, // clientErrorsSummary (1588x)
- 58125: 300, // cmSketch (1588x)
- 57646: 301, // coalesce (1588x)
- 57654: 302, // compressed (1588x)
- 57663: 303, // context (1588x)
- 57989: 304, // copyKwd (1588x)
- 58127: 305, // correlation (1588x)
- 57664: 306, // cpu (1588x)
- 57678: 307, // deallocate (1588x)
- 58129: 308, // dependency (1588x)
- 57683: 309, // directory (1588x)
- 57686: 310, // discard (1588x)
- 57687: 311, // disk (1588x)
- 57995: 312, // dotType (1588x)
- 58131: 313, // drainer (1588x)
- 58132: 314, // dry (1588x)
- 57689: 315, // duplicate (1588x)
- 57707: 316, // exchange (1588x)
- 57709: 317, // execute (1588x)
- 57710: 318, // expansion (1588x)
- 58003: 319, // flashback (1588x)
- 57725: 320, // general (1588x)
- 57730: 321, // help (1588x)
- 58011: 322, // high (1588x)
- 57731: 323, // histogram (1588x)
- 57733: 324, // hosts (1588x)
- 57702: 325, // identSQLErrors (1588x)
- 57741: 326, // incremental (1588x)
- 58012: 327, // inplace (1588x)
- 57744: 328, // instance (1588x)
- 58013: 329, // instant (1588x)
- 57748: 330, // ipc (1588x)
- 57753: 331, // labels (1588x)
- 57764: 332, // locked (1588x)
- 58025: 333, // low (1588x)
- 58027: 334, // medium (1588x)
- 58028: 335, // metadata (1588x)
- 57783: 336, // modify (1588x)
- 57790: 337, // nextval (1588x)
- 58136: 338, // nodeID (1588x)
- 58137: 339, // nodeState (1588x)
- 57800: 340, // nulls (1588x)
- 57813: 341, // pageSym (1588x)
- 58140: 342, // pump (1588x)
- 57838: 343, // purge (1588x)
- 57844: 344, // rebuild (1588x)
- 57846: 345, // redundant (1588x)
- 57847: 346, // reload (1588x)
- 57859: 347, // restore (1588x)
- 57867: 348, // routine (1588x)
- 58046: 349, // s3 (1588x)
- 58146: 350, // samples (1588x)
- 57876: 351, // secondaryLoad (1588x)
- 57877: 352, // secondaryUnload (1588x)
- 57887: 353, // share (1588x)
- 57889: 354, // shutdown (1588x)
- 57894: 355, // slave (1588x)
- 57898: 356, // source (1588x)
- 58152: 357, // statsExtended (1588x)
- 57914: 358, // statsOptions (1588x)
- 58056: 359, // stop (1588x)
- 57925: 360, // swaps (1588x)
- 58065: 361, // tidbJson (1588x)
- 58070: 362, // tokudbDefault (1588x)
- 58071: 363, // tokudbFast (1588x)
- 58072: 364, // tokudbLzma (1588x)
- 58073: 365, // tokudbQuickLZ (1588x)
- 58074: 366, // tokudbSmall (1588x)
- 58075: 367, // tokudbSnappy (1588x)
- 58076: 368, // tokudbUncompressed (1588x)
- 58077: 369, // tokudbZlib (1588x)
- 58078: 370, // tokudbZstd (1588x)
- 58160: 371, // topn (1588x)
- 57942: 372, // trace (1588x)
- 57943: 373, // traditional (1588x)
- 58081: 374, // trueCardCost (1588x)
- 58082: 375, // unlimited (1588x)
- 58087: 376, // verboseType (1588x)
- 57967: 377, // warnings (1588x)
- 57597: 378, // advise (1587x)
- 57599: 379, // against (1587x)
- 57600: 380, // ago (1587x)
- 57602: 381, // always (1587x)
- 57615: 382, // backups (1587x)
- 57618: 383, // bernoulli (1587x)
- 57621: 384, // bindingCache (1587x)
- 58112: 385, // builtins (1587x)
- 57632: 386, // cascaded (1587x)
- 57633: 387, // causal (1587x)
- 57640: 388, // cleanup (1587x)
- 57641: 389, // client (1587x)
- 57644: 390, // cluster (1587x)
- 57647: 391, // collation (1587x)
- 58126: 392, // columnStatsUsage (1587x)
- 57652: 393, // committed (1587x)
- 57659: 394, // config (1587x)
- 57661: 395, // consistency (1587x)
- 57662: 396, // consistent (1587x)
- 58130: 397, // depth (1587x)
- 57685: 398, // disabled (1587x)
- 57997: 399, // dump (1587x)
- 57692: 400, // enabled (1587x)
- 57699: 401, // engines (1587x)
- 57705: 402, // events (1587x)
- 57706: 403, // evolve (1587x)
- 57711: 404, // expire (1587x)
- 58001: 405, // exprPushdownBlacklist (1587x)
- 57712: 406, // extended (1587x)
- 57714: 407, // faultsSym (1587x)
- 57722: 408, // found (1587x)
- 57724: 409, // function (1587x)
- 57727: 410, // grants (1587x)
- 58133: 411, // histogramsInFlight (1587x)
- 57742: 412, // indexes (1587x)
- 58014: 413, // internal (1587x)
- 57746: 414, // invoker (1587x)
- 57747: 415, // io (1587x)
- 57754: 416, // language (1587x)
- 57759: 417, // level (1587x)
- 57760: 418, // list (1587x)
- 58024: 419, // log (1587x)
- 57766: 420, // master (1587x)
- 57769: 421, // max_minutes (1587x)
- 57788: 422, // never (1587x)
- 57798: 423, // none (1587x)
- 57804: 424, // oltpReadOnly (1587x)
- 57805: 425, // oltpReadWrite (1587x)
- 57806: 426, // oltpWriteOnly (1587x)
- 58138: 427, // optimistic (1587x)
- 58032: 428, // optRuleBlacklist (1587x)
- 57814: 429, // parser (1587x)
- 57815: 430, // partial (1587x)
- 57816: 431, // partitioning (1587x)
- 57823: 432, // per_table (1587x)
- 57821: 433, // percent (1587x)
- 58139: 434, // pessimistic (1587x)
- 57826: 435, // point (1587x)
- 57830: 436, // preserve (1587x)
- 57835: 437, // profile (1587x)
- 57836: 438, // profiles (1587x)
- 57840: 439, // queries (1587x)
- 58041: 440, // recent (1587x)
- 58141: 441, // region (1587x)
- 58042: 442, // replayer (1587x)
- 57860: 443, // restores (1587x)
- 57862: 444, // reuse (1587x)
- 57866: 445, // rollup (1587x)
- 58144: 446, // run (1587x)
- 57874: 447, // secondary (1587x)
- 57878: 448, // security (1587x)
- 57883: 449, // serializable (1587x)
- 58147: 450, // sessionStates (1587x)
- 57891: 451, // simple (1587x)
- 58153: 452, // statsHealthy (1587x)
- 58154: 453, // statsHistograms (1587x)
- 58155: 454, // statsLocked (1587x)
- 58156: 455, // statsMeta (1587x)
- 57926: 456, // switchesSym (1587x)
- 57927: 457, // system (1587x)
- 57928: 458, // systemTime (1587x)
- 58063: 459, // target (1587x)
- 57933: 460, // temptable (1587x)
- 58069: 461, // tls (1587x)
- 58079: 462, // top (1587x)
- 57940: 463, // tpcc (1587x)
- 57941: 464, // tpch10 (1587x)
- 57944: 465, // transaction (1587x)
- 57945: 466, // triggers (1587x)
- 57953: 467, // uncommitted (1587x)
- 57954: 468, // undefined (1587x)
- 57957: 469, // unset (1587x)
- 58161: 470, // width (1587x)
- 57972: 471, // workload (1587x)
- 57973: 472, // x509 (1587x)
- 57975: 473, // addDate (1586x)
- 57603: 474, // any (1586x)
- 57976: 475, // approxCountDistinct (1586x)
- 57977: 476, // approxPercentile (1586x)
- 57611: 477, // avg (1586x)
- 57979: 478, // bitAnd (1586x)
- 57980: 479, // bitOr (1586x)
- 57981: 480, // bitXor (1586x)
- 57982: 481, // bound (1586x)
- 57986: 482, // cast (1586x)
- 57990: 483, // curDate (1586x)
- 57991: 484, // curTime (1586x)
- 57992: 485, // dateAdd (1586x)
- 57993: 486, // dateSub (1586x)
- 57703: 487, // escape (1586x)
- 57704: 488, // event (1586x)
- 57708: 489, // exclusive (1586x)
- 58002: 490, // extract (1586x)
- 57716: 491, // file (1586x)
- 58004: 492, // follower (1586x)
- 58009: 493, // getFormat (1586x)
- 58010: 494, // groupConcat (1586x)
- 57739: 495, // imports (1586x)
- 58015: 496, // ioReadBandwidth (1586x)
- 58016: 497, // ioWriteBandwidth (1586x)
- 58017: 498, // jsonArrayagg (1586x)
- 58018: 499, // jsonObjectAgg (1586x)
- 57756: 500, // lastval (1586x)
- 58019: 501, // leader (1586x)
- 58021: 502, // learner (1586x)
- 58026: 503, // max (1586x)
- 57775: 504, // member (1586x)
- 58029: 505, // min (1586x)
- 57785: 506, // names (1586x)
- 58031: 507, // now (1586x)
- 58036: 508, // position (1586x)
- 57833: 509, // process (1586x)
- 57837: 510, // proxy (1586x)
- 57842: 511, // quick (1586x)
- 57853: 512, // replicas (1586x)
- 57854: 513, // replication (1586x)
- 58143: 514, // reset (1586x)
- 57863: 515, // reverse (1586x)
- 57868: 516, // rowCount (1586x)
- 58044: 517, // running (1586x)
- 57885: 518, // setval (1586x)
- 57888: 519, // shared (1586x)
- 57897: 520, // some (1586x)
- 57899: 521, // sqlBufferResult (1586x)
- 57900: 522, // sqlCache (1586x)
- 57901: 523, // sqlNoCache (1586x)
- 58049: 524, // staleness (1586x)
- 58055: 525, // std (1586x)
- 58052: 526, // stddev (1586x)
- 58053: 527, // stddevPop (1586x)
- 58054: 528, // stddevSamp (1586x)
- 58057: 529, // strict (1586x)
- 58058: 530, // strong (1586x)
- 58059: 531, // subDate (1586x)
- 58060: 532, // substring (1586x)
- 58061: 533, // sum (1586x)
- 57924: 534, // super (1586x)
- 58067: 535, // timestampAdd (1586x)
- 58068: 536, // timestampDiff (1586x)
- 58080: 537, // trim (1586x)
- 57947: 538, // tsoType (1586x)
- 58084: 539, // variance (1586x)
- 58085: 540, // varPop (1586x)
- 58086: 541, // varSamp (1586x)
- 58090: 542, // voter (1586x)
- 57969: 543, // weightString (1586x)
- 57505: 544, // on (1499x)
- 40: 545, // '(' (1496x)
- 57590: 546, // with (1364x)
- 57353: 547, // stringLit (1347x)
- 58180: 548, // not2 (1303x)
- 57405: 549, // defaultKwd (1255x)
- 57498: 550, // not (1234x)
- 57369: 551, // as (1200x)
- 57384: 552, // collate (1168x)
+ 59: 0, // ';' (2558x)
+ 57344: 1, // $end (2545x)
+ 57848: 2, // remove (2031x)
+ 58148: 3, // split (2031x)
+ 57777: 4, // merge (2030x)
+ 57849: 5, // reorganize (2029x)
+ 57650: 6, // comment (2023x)
+ 57919: 7, // storage (1932x)
+ 57608: 8, // autoIncrement (1921x)
+ 44: 9, // ',' (1906x)
+ 57717: 10, // first (1820x)
+ 57598: 11, // after (1814x)
+ 57882: 12, // serial (1810x)
+ 57609: 13, // autoRandom (1809x)
+ 57649: 14, // columnFormat (1809x)
+ 57818: 15, // password (1767x)
+ 57635: 16, // charsetKwd (1758x)
+ 57637: 17, // checksum (1748x)
+ 58033: 18, // placement (1745x)
+ 57752: 19, // keyBlockSize (1731x)
+ 57930: 20, // tablespace (1725x)
+ 57693: 21, // encryption (1723x)
+ 57698: 22, // engine (1720x)
+ 57674: 23, // data (1718x)
+ 57743: 24, // insertMethod (1716x)
+ 57771: 25, // maxRows (1716x)
+ 57781: 26, // minRows (1716x)
+ 57794: 27, // nodegroup (1716x)
+ 57660: 28, // connection (1708x)
+ 57610: 29, // autoRandomBase (1705x)
+ 58151: 30, // statsBuckets (1703x)
+ 58157: 31, // statsTopN (1703x)
+ 57948: 32, // ttl (1703x)
+ 57607: 33, // autoIdCache (1702x)
+ 57612: 34, // avgRowLength (1702x)
+ 57655: 35, // compression (1702x)
+ 57681: 36, // delayKeyWrite (1702x)
+ 57812: 37, // packKeys (1702x)
+ 57831: 38, // preSplitRegions (1702x)
+ 57869: 39, // rowFormat (1702x)
+ 57875: 40, // secondaryEngine (1702x)
+ 57886: 41, // shardRowIDBits (1702x)
+ 57911: 42, // statsAutoRecalc (1702x)
+ 57912: 43, // statsColChoice (1702x)
+ 57913: 44, // statsColList (1702x)
+ 57915: 45, // statsPersistent (1702x)
+ 57916: 46, // statsSamplePages (1702x)
+ 57917: 47, // statsSampleRate (1702x)
+ 57931: 48, // tableChecksum (1702x)
+ 57949: 49, // ttlEnable (1702x)
+ 57950: 50, // ttlJobInterval (1702x)
+ 57856: 51, // resource (1681x)
+ 41: 52, // ')' (1668x)
+ 57605: 53, // attribute (1654x)
+ 57595: 54, // account (1652x)
+ 57713: 55, // failedLoginAttempts (1652x)
+ 57819: 56, // passwordLockTime (1652x)
+ 57346: 57, // identifier (1650x)
+ 57762: 58, // local (1638x)
+ 57861: 59, // resume (1638x)
+ 57890: 60, // signed (1638x)
+ 57896: 61, // snapshot (1636x)
+ 57613: 62, // backend (1635x)
+ 57636: 63, // checkpoint (1635x)
+ 57638: 64, // checksumConcurrency (1635x)
+ 57656: 65, // compressionLevel (1635x)
+ 57657: 66, // compressionType (1635x)
+ 57658: 67, // concurrency (1635x)
+ 57665: 68, // csvBackslashEscape (1635x)
+ 57666: 69, // csvDelimiter (1635x)
+ 57667: 70, // csvHeader (1635x)
+ 57668: 71, // csvNotNull (1635x)
+ 57669: 72, // csvNull (1635x)
+ 57670: 73, // csvSeparator (1635x)
+ 57671: 74, // csvTrimLastSeparators (1635x)
+ 57694: 75, // encryptionKeyFile (1635x)
+ 57695: 76, // encryptionMethod (1635x)
+ 58007: 77, // fullBackupStorage (1635x)
+ 58008: 78, // gcTTL (1635x)
+ 57737: 79, // ignoreStats (1635x)
+ 57757: 80, // lastBackup (1635x)
+ 57761: 81, // loadStats (1635x)
+ 57809: 82, // onDuplicate (1635x)
+ 57807: 83, // online (1635x)
+ 57843: 84, // rateLimit (1635x)
+ 58043: 85, // restoredTS (1635x)
+ 57879: 86, // sendCredentialsToTiKV (1635x)
+ 57893: 87, // skipSchemaFiles (1635x)
+ 58051: 88, // startTS (1635x)
+ 57920: 89, // strictFormat (1635x)
+ 57936: 90, // tikvImporter (1635x)
+ 58083: 91, // untilTS (1635x)
+ 57966: 92, // waitTiflashReady (1635x)
+ 57971: 93, // withSysTable (1635x)
+ 57726: 94, // global (1630x)
+ 57617: 95, // begin (1629x)
+ 57651: 96, // commit (1629x)
+ 57791: 97, // no (1629x)
+ 57865: 98, // rollback (1629x)
+ 57910: 99, // start (1627x)
+ 57946: 100, // truncate (1626x)
+ 57629: 101, // cache (1624x)
+ 57792: 102, // nocache (1623x)
+ 57810: 103, // open (1623x)
+ 57596: 104, // action (1622x)
+ 57643: 105, // close (1622x)
+ 57673: 106, // cycle (1622x)
+ 57780: 107, // minValue (1622x)
+ 57601: 108, // algorithm (1621x)
+ 57696: 109, // end (1621x)
+ 57740: 110, // increment (1621x)
+ 57793: 111, // nocycle (1621x)
+ 57795: 112, // nomaxvalue (1621x)
+ 57796: 113, // nominvalue (1621x)
+ 57951: 114, // tp (1621x)
+ 57645: 115, // clustered (1620x)
+ 57745: 116, // invisible (1620x)
+ 57797: 117, // nonclustered (1620x)
+ 57964: 118, // visible (1620x)
+ 57858: 119, // restart (1619x)
+ 58142: 120, // regions (1618x)
+ 57978: 121, // background (1617x)
+ 57985: 122, // burstable (1617x)
+ 58039: 123, // priority (1617x)
+ 58040: 124, // queryLimit (1617x)
+ 58045: 125, // ruRate (1617x)
+ 57922: 126, // subpartition (1614x)
+ 57817: 127, // partitions (1613x)
+ 58035: 128, // plan (1613x)
+ 57974: 129, // yearType (1613x)
+ 57987: 130, // constraints (1611x)
+ 58005: 131, // followerConstraints (1611x)
+ 58006: 132, // followers (1611x)
+ 58020: 133, // leaderConstraints (1611x)
+ 58022: 134, // learnerConstraints (1611x)
+ 58023: 135, // learners (1611x)
+ 58038: 136, // primaryRegion (1611x)
+ 58047: 137, // schedule (1611x)
+ 57909: 138, // sqlTsiYear (1611x)
+ 58062: 139, // survivalPreferences (1611x)
+ 58088: 140, // voterConstraints (1611x)
+ 58089: 141, // voters (1611x)
+ 57648: 142, // columns (1609x)
+ 57738: 143, // importKwd (1609x)
+ 57963: 144, // view (1609x)
+ 57677: 145, // day (1608x)
+ 58091: 146, // watch (1607x)
+ 57994: 147, // defined (1606x)
+ 58000: 148, // execElapsed (1606x)
+ 57873: 149, // second (1606x)
+ 57918: 150, // status (1606x)
+ 57734: 151, // hour (1605x)
+ 57778: 152, // microsecond (1605x)
+ 57779: 153, // minute (1605x)
+ 57784: 154, // month (1605x)
+ 57839: 155, // quarter (1605x)
+ 57902: 156, // sqlTsiDay (1605x)
+ 57903: 157, // sqlTsiHour (1605x)
+ 57904: 158, // sqlTsiMinute (1605x)
+ 57905: 159, // sqlTsiMonth (1605x)
+ 57906: 160, // sqlTsiQuarter (1605x)
+ 57907: 161, // sqlTsiSecond (1605x)
+ 57908: 162, // sqlTsiWeek (1605x)
+ 57968: 163, // week (1605x)
+ 57604: 164, // ascii (1604x)
+ 57628: 165, // byteType (1604x)
+ 57929: 166, // tables (1604x)
+ 57955: 167, // unicodeSym (1604x)
+ 57715: 168, // fields (1603x)
+ 57765: 169, // logs (1602x)
+ 58066: 170, // timeDuration (1602x)
+ 57841: 171, // query (1600x)
+ 57880: 172, // separator (1600x)
+ 57639: 173, // cipher (1599x)
+ 57750: 174, // issuer (1599x)
+ 57767: 175, // maxConnectionsPerHour (1599x)
+ 57770: 176, // maxQueriesPerHour (1599x)
+ 57772: 177, // maxUpdatesPerHour (1599x)
+ 57773: 178, // maxUserConnections (1599x)
+ 57828: 179, // preceding (1599x)
+ 57871: 180, // san (1599x)
+ 57921: 181, // subject (1599x)
+ 57939: 182, // tokenIssuer (1599x)
+ 57998: 183, // endTime (1598x)
+ 57751: 184, // jsonType (1598x)
+ 58050: 185, // startTime (1598x)
+ 57676: 186, // datetimeType (1597x)
+ 57675: 187, // dateType (1597x)
+ 57718: 188, // fixed (1597x)
+ 57937: 189, // timeType (1597x)
+ 57620: 190, // bindings (1596x)
+ 57672: 191, // current (1596x)
+ 57680: 192, // definer (1596x)
+ 57729: 193, // hash (1596x)
+ 57736: 194, // identified (1596x)
+ 57857: 195, // respect (1596x)
+ 57864: 196, // role (1596x)
+ 57938: 197, // timestampType (1596x)
+ 57960: 198, // value (1596x)
+ 57962: 199, // vectorType (1596x)
+ 57614: 200, // backup (1595x)
+ 57626: 201, // booleanType (1595x)
+ 57697: 202, // enforced (1595x)
+ 57720: 203, // following (1595x)
+ 57758: 204, // less (1595x)
+ 57799: 205, // nowait (1595x)
+ 57808: 206, // only (1595x)
+ 57872: 207, // savepoint (1595x)
+ 57892: 208, // skip (1595x)
+ 58064: 209, // taskTypes (1595x)
+ 57934: 210, // textType (1595x)
+ 57935: 211, // than (1595x)
+ 58159: 212, // tiFlash (1595x)
+ 57952: 213, // unbounded (1595x)
+ 57619: 214, // binding (1594x)
+ 57623: 215, // bitType (1594x)
+ 57625: 216, // boolType (1594x)
+ 57700: 217, // enum (1594x)
+ 57735: 218, // hypo (1594x)
+ 58134: 219, // job (1594x)
+ 57786: 220, // national (1594x)
+ 57787: 221, // ncharType (1594x)
+ 58030: 222, // next_row_id (1594x)
+ 57801: 223, // nvarcharType (1594x)
+ 57803: 224, // offset (1594x)
+ 57827: 225, // policy (1594x)
+ 58037: 226, // predicate (1594x)
+ 57852: 227, // replica (1594x)
+ 57932: 228, // temporary (1594x)
+ 57958: 229, // user (1594x)
+ 57682: 230, // digest (1593x)
+ 58135: 231, // jobs (1593x)
+ 57763: 232, // location (1593x)
+ 58034: 233, // planCache (1593x)
+ 57829: 234, // prepare (1593x)
+ 58150: 235, // stats (1593x)
+ 57956: 236, // unknown (1593x)
+ 57965: 237, // wait (1593x)
+ 57627: 238, // btree (1592x)
+ 57988: 239, // cooldown (1592x)
+ 57679: 240, // declare (1592x)
+ 57996: 241, // dryRun (1592x)
+ 57721: 242, // format (1592x)
+ 57749: 243, // isolation (1592x)
+ 57755: 244, // last (1592x)
+ 57768: 245, // max_idxnum (1592x)
+ 57776: 246, // memory (1592x)
+ 57789: 247, // next (1592x)
+ 57802: 248, // off (1592x)
+ 57811: 249, // optional (1592x)
+ 57822: 250, // per_db (1592x)
+ 57832: 251, // privileges (1592x)
+ 57855: 252, // required (1592x)
+ 57870: 253, // rtree (1592x)
+ 58145: 254, // sampleRate (1592x)
+ 57881: 255, // sequence (1592x)
+ 57884: 256, // session (1592x)
+ 57895: 257, // slow (1592x)
+ 58082: 258, // unlimited (1592x)
+ 57959: 259, // validation (1592x)
+ 57961: 260, // variables (1592x)
+ 57606: 261, // attributes (1591x)
+ 58123: 262, // cancel (1591x)
+ 57653: 263, // compact (1591x)
+ 58128: 264, // ddl (1591x)
+ 57684: 265, // disable (1591x)
+ 57688: 266, // do (1591x)
+ 57690: 267, // dynamic (1591x)
+ 57691: 268, // enable (1591x)
+ 57701: 269, // errorKwd (1591x)
+ 57999: 270, // exact (1591x)
+ 57719: 271, // flush (1591x)
+ 57723: 272, // full (1591x)
+ 57728: 273, // handler (1591x)
+ 57732: 274, // history (1591x)
+ 57774: 275, // mb (1591x)
+ 57782: 276, // mode (1591x)
+ 57820: 277, // pause (1591x)
+ 57825: 278, // plugins (1591x)
+ 57834: 279, // processlist (1591x)
+ 57845: 280, // recover (1591x)
+ 57850: 281, // repair (1591x)
+ 57851: 282, // repeatable (1591x)
+ 58048: 283, // similar (1591x)
+ 58149: 284, // statistics (1591x)
+ 57923: 285, // subpartitions (1591x)
+ 58158: 286, // tidb (1591x)
+ 57970: 287, // without (1591x)
+ 58092: 288, // admin (1590x)
+ 58093: 289, // batch (1590x)
+ 57616: 290, // bdr (1590x)
+ 57622: 291, // binlog (1590x)
+ 57624: 292, // block (1590x)
+ 57983: 293, // br (1590x)
+ 57984: 294, // briefType (1590x)
+ 58094: 295, // buckets (1590x)
+ 57630: 296, // calibrate (1590x)
+ 57631: 297, // capture (1590x)
+ 58124: 298, // cardinality (1590x)
+ 57634: 299, // chain (1590x)
+ 57642: 300, // clientErrorsSummary (1590x)
+ 58125: 301, // cmSketch (1590x)
+ 57646: 302, // coalesce (1590x)
+ 57654: 303, // compressed (1590x)
+ 57663: 304, // context (1590x)
+ 57989: 305, // copyKwd (1590x)
+ 58127: 306, // correlation (1590x)
+ 57664: 307, // cpu (1590x)
+ 57678: 308, // deallocate (1590x)
+ 58129: 309, // dependency (1590x)
+ 57683: 310, // directory (1590x)
+ 57686: 311, // discard (1590x)
+ 57687: 312, // disk (1590x)
+ 57995: 313, // dotType (1590x)
+ 58131: 314, // drainer (1590x)
+ 58132: 315, // dry (1590x)
+ 57689: 316, // duplicate (1590x)
+ 57707: 317, // exchange (1590x)
+ 57709: 318, // execute (1590x)
+ 57710: 319, // expansion (1590x)
+ 58003: 320, // flashback (1590x)
+ 57725: 321, // general (1590x)
+ 57730: 322, // help (1590x)
+ 58011: 323, // high (1590x)
+ 57731: 324, // histogram (1590x)
+ 57733: 325, // hosts (1590x)
+ 57702: 326, // identSQLErrors (1590x)
+ 57741: 327, // incremental (1590x)
+ 57742: 328, // indexes (1590x)
+ 58012: 329, // inplace (1590x)
+ 57744: 330, // instance (1590x)
+ 58013: 331, // instant (1590x)
+ 57748: 332, // ipc (1590x)
+ 57753: 333, // labels (1590x)
+ 57764: 334, // locked (1590x)
+ 58025: 335, // low (1590x)
+ 58027: 336, // medium (1590x)
+ 58028: 337, // metadata (1590x)
+ 57783: 338, // modify (1590x)
+ 57790: 339, // nextval (1590x)
+ 58136: 340, // nodeID (1590x)
+ 58137: 341, // nodeState (1590x)
+ 57800: 342, // nulls (1590x)
+ 57813: 343, // pageSym (1590x)
+ 58140: 344, // pump (1590x)
+ 57838: 345, // purge (1590x)
+ 57844: 346, // rebuild (1590x)
+ 57846: 347, // redundant (1590x)
+ 57847: 348, // reload (1590x)
+ 57859: 349, // restore (1590x)
+ 57867: 350, // routine (1590x)
+ 58046: 351, // s3 (1590x)
+ 58146: 352, // samples (1590x)
+ 57876: 353, // secondaryLoad (1590x)
+ 57877: 354, // secondaryUnload (1590x)
+ 57887: 355, // share (1590x)
+ 57889: 356, // shutdown (1590x)
+ 57894: 357, // slave (1590x)
+ 57898: 358, // source (1590x)
+ 58152: 359, // statsExtended (1590x)
+ 57914: 360, // statsOptions (1590x)
+ 58056: 361, // stop (1590x)
+ 57925: 362, // swaps (1590x)
+ 58065: 363, // tidbJson (1590x)
+ 58070: 364, // tokudbDefault (1590x)
+ 58071: 365, // tokudbFast (1590x)
+ 58072: 366, // tokudbLzma (1590x)
+ 58073: 367, // tokudbQuickLZ (1590x)
+ 58074: 368, // tokudbSmall (1590x)
+ 58075: 369, // tokudbSnappy (1590x)
+ 58076: 370, // tokudbUncompressed (1590x)
+ 58077: 371, // tokudbZlib (1590x)
+ 58078: 372, // tokudbZstd (1590x)
+ 58160: 373, // topn (1590x)
+ 57942: 374, // trace (1590x)
+ 57943: 375, // traditional (1590x)
+ 58081: 376, // trueCardCost (1590x)
+ 58087: 377, // verboseType (1590x)
+ 57967: 378, // warnings (1590x)
+ 57597: 379, // advise (1589x)
+ 57599: 380, // against (1589x)
+ 57600: 381, // ago (1589x)
+ 57602: 382, // always (1589x)
+ 57615: 383, // backups (1589x)
+ 57618: 384, // bernoulli (1589x)
+ 57621: 385, // bindingCache (1589x)
+ 58112: 386, // builtins (1589x)
+ 57632: 387, // cascaded (1589x)
+ 57633: 388, // causal (1589x)
+ 57640: 389, // cleanup (1589x)
+ 57641: 390, // client (1589x)
+ 57644: 391, // cluster (1589x)
+ 57647: 392, // collation (1589x)
+ 58126: 393, // columnStatsUsage (1589x)
+ 57652: 394, // committed (1589x)
+ 57659: 395, // config (1589x)
+ 57661: 396, // consistency (1589x)
+ 57662: 397, // consistent (1589x)
+ 58130: 398, // depth (1589x)
+ 57685: 399, // disabled (1589x)
+ 57997: 400, // dump (1589x)
+ 57692: 401, // enabled (1589x)
+ 57699: 402, // engines (1589x)
+ 57705: 403, // events (1589x)
+ 57706: 404, // evolve (1589x)
+ 57711: 405, // expire (1589x)
+ 58001: 406, // exprPushdownBlacklist (1589x)
+ 57712: 407, // extended (1589x)
+ 57714: 408, // faultsSym (1589x)
+ 57722: 409, // found (1589x)
+ 57724: 410, // function (1589x)
+ 57727: 411, // grants (1589x)
+ 58133: 412, // histogramsInFlight (1589x)
+ 58014: 413, // internal (1589x)
+ 57746: 414, // invoker (1589x)
+ 57747: 415, // io (1589x)
+ 57754: 416, // language (1589x)
+ 57759: 417, // level (1589x)
+ 57760: 418, // list (1589x)
+ 58024: 419, // log (1589x)
+ 57766: 420, // master (1589x)
+ 57769: 421, // max_minutes (1589x)
+ 57788: 422, // never (1589x)
+ 57798: 423, // none (1589x)
+ 57804: 424, // oltpReadOnly (1589x)
+ 57805: 425, // oltpReadWrite (1589x)
+ 57806: 426, // oltpWriteOnly (1589x)
+ 58138: 427, // optimistic (1589x)
+ 58032: 428, // optRuleBlacklist (1589x)
+ 57814: 429, // parser (1589x)
+ 57815: 430, // partial (1589x)
+ 57816: 431, // partitioning (1589x)
+ 57823: 432, // per_table (1589x)
+ 57821: 433, // percent (1589x)
+ 58139: 434, // pessimistic (1589x)
+ 57826: 435, // point (1589x)
+ 57830: 436, // preserve (1589x)
+ 57835: 437, // profile (1589x)
+ 57836: 438, // profiles (1589x)
+ 57840: 439, // queries (1589x)
+ 58041: 440, // recent (1589x)
+ 58141: 441, // region (1589x)
+ 58042: 442, // replayer (1589x)
+ 57860: 443, // restores (1589x)
+ 57862: 444, // reuse (1589x)
+ 57866: 445, // rollup (1589x)
+ 58144: 446, // run (1589x)
+ 57874: 447, // secondary (1589x)
+ 57878: 448, // security (1589x)
+ 57883: 449, // serializable (1589x)
+ 58147: 450, // sessionStates (1589x)
+ 57891: 451, // simple (1589x)
+ 58153: 452, // statsHealthy (1589x)
+ 58154: 453, // statsHistograms (1589x)
+ 58155: 454, // statsLocked (1589x)
+ 58156: 455, // statsMeta (1589x)
+ 57926: 456, // switchesSym (1589x)
+ 57927: 457, // system (1589x)
+ 57928: 458, // systemTime (1589x)
+ 58063: 459, // target (1589x)
+ 57933: 460, // temptable (1589x)
+ 58069: 461, // tls (1589x)
+ 58079: 462, // top (1589x)
+ 57940: 463, // tpcc (1589x)
+ 57941: 464, // tpch10 (1589x)
+ 57944: 465, // transaction (1589x)
+ 57945: 466, // triggers (1589x)
+ 57953: 467, // uncommitted (1589x)
+ 57954: 468, // undefined (1589x)
+ 57957: 469, // unset (1589x)
+ 58161: 470, // width (1589x)
+ 57972: 471, // workload (1589x)
+ 57973: 472, // x509 (1589x)
+ 57975: 473, // addDate (1588x)
+ 57603: 474, // any (1588x)
+ 57976: 475, // approxCountDistinct (1588x)
+ 57977: 476, // approxPercentile (1588x)
+ 57611: 477, // avg (1588x)
+ 57979: 478, // bitAnd (1588x)
+ 57980: 479, // bitOr (1588x)
+ 57981: 480, // bitXor (1588x)
+ 57982: 481, // bound (1588x)
+ 57986: 482, // cast (1588x)
+ 57990: 483, // curDate (1588x)
+ 57991: 484, // curTime (1588x)
+ 57992: 485, // dateAdd (1588x)
+ 57993: 486, // dateSub (1588x)
+ 57703: 487, // escape (1588x)
+ 57704: 488, // event (1588x)
+ 57708: 489, // exclusive (1588x)
+ 58002: 490, // extract (1588x)
+ 57716: 491, // file (1588x)
+ 58004: 492, // follower (1588x)
+ 58009: 493, // getFormat (1588x)
+ 58010: 494, // groupConcat (1588x)
+ 57739: 495, // imports (1588x)
+ 58015: 496, // ioReadBandwidth (1588x)
+ 58016: 497, // ioWriteBandwidth (1588x)
+ 58017: 498, // jsonArrayagg (1588x)
+ 58018: 499, // jsonObjectAgg (1588x)
+ 57756: 500, // lastval (1588x)
+ 58019: 501, // leader (1588x)
+ 58021: 502, // learner (1588x)
+ 58026: 503, // max (1588x)
+ 57775: 504, // member (1588x)
+ 58029: 505, // min (1588x)
+ 57785: 506, // names (1588x)
+ 58031: 507, // now (1588x)
+ 58036: 508, // position (1588x)
+ 57833: 509, // process (1588x)
+ 57837: 510, // proxy (1588x)
+ 57842: 511, // quick (1588x)
+ 57853: 512, // replicas (1588x)
+ 57854: 513, // replication (1588x)
+ 58143: 514, // reset (1588x)
+ 57863: 515, // reverse (1588x)
+ 57868: 516, // rowCount (1588x)
+ 58044: 517, // running (1588x)
+ 57885: 518, // setval (1588x)
+ 57888: 519, // shared (1588x)
+ 57897: 520, // some (1588x)
+ 57899: 521, // sqlBufferResult (1588x)
+ 57900: 522, // sqlCache (1588x)
+ 57901: 523, // sqlNoCache (1588x)
+ 58049: 524, // staleness (1588x)
+ 58055: 525, // std (1588x)
+ 58052: 526, // stddev (1588x)
+ 58053: 527, // stddevPop (1588x)
+ 58054: 528, // stddevSamp (1588x)
+ 58057: 529, // strict (1588x)
+ 58058: 530, // strong (1588x)
+ 58059: 531, // subDate (1588x)
+ 58060: 532, // substring (1588x)
+ 58061: 533, // sum (1588x)
+ 57924: 534, // super (1588x)
+ 58067: 535, // timestampAdd (1588x)
+ 58068: 536, // timestampDiff (1588x)
+ 58080: 537, // trim (1588x)
+ 57947: 538, // tsoType (1588x)
+ 58084: 539, // variance (1588x)
+ 58085: 540, // varPop (1588x)
+ 58086: 541, // varSamp (1588x)
+ 58090: 542, // voter (1588x)
+ 57969: 543, // weightString (1588x)
+ 57505: 544, // on (1508x)
+ 40: 545, // '(' (1499x)
+ 57590: 546, // with (1368x)
+ 57353: 547, // stringLit (1348x)
+ 58180: 548, // not2 (1310x)
+ 57405: 549, // defaultKwd (1262x)
+ 57498: 550, // not (1241x)
+ 57369: 551, // as (1209x)
+ 57384: 552, // collate (1175x)
57568: 553, // union (1153x)
57475: 554, // left (1149x)
57534: 555, // right (1149x)
- 57576: 556, // using (1138x)
+ 57576: 556, // using (1140x)
43: 557, // '+' (1124x)
45: 558, // '-' (1122x)
57496: 559, // mod (1102x)
- 57515: 560, // partition (1085x)
- 57502: 561, // null (1063x)
- 57580: 562, // values (1059x)
- 57446: 563, // ignore (1045x)
+ 57515: 560, // partition (1094x)
+ 57502: 561, // null (1070x)
+ 57580: 562, // values (1061x)
+ 57446: 563, // ignore (1047x)
57421: 564, // except (1042x)
57461: 565, // intersect (1041x)
- 57530: 566, // replace (1039x)
+ 57530: 566, // replace (1041x)
57381: 567, // charType (1028x)
57426: 568, // fetch (1023x)
58169: 569, // eq (1021x)
@@ -1504,7 +1504,7 @@ var (
57463: 574, // into (1007x)
42: 575, // '*' (1005x)
57434: 576, // from (1002x)
- 57483: 577, // lock (998x)
+ 57483: 577, // lock (1000x)
57587: 578, // where (990x)
57510: 579, // order (986x)
57432: 580, // force (979x)
@@ -1549,7 +1549,7 @@ var (
57370: 619, // asc (856x)
57448: 620, // in (850x)
57559: 621, // then (850x)
- 57556: 622, // tableKwd (847x)
+ 57556: 622, // tableKwd (849x)
47: 623, // '/' (842x)
60: 624, // '<