diff --git a/Makefile b/Makefile index 04f824b6943..13821324ee2 100644 --- a/Makefile +++ b/Makefile @@ -361,6 +361,17 @@ install-schema-mysql: temporal-sql-tool ./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) --db $(VISIBILITY_DB) setup-schema -v 0.0 ./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) --db $(VISIBILITY_DB) update-schema -d ./schema/mysql/v57/visibility/versioned +install-schema-mysql8: temporal-sql-tool + @printf $(COLOR) "Install MySQL schema..." + ./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) --db $(TEMPORAL_DB) drop -f + ./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) --db $(TEMPORAL_DB) create + ./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) --db $(TEMPORAL_DB) setup-schema -v 0.0 + ./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) --db $(TEMPORAL_DB) update-schema -d ./schema/mysql/v8/temporal/versioned + ./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) --db $(VISIBILITY_DB) drop -f + ./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) --db $(VISIBILITY_DB) create + ./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) --db $(VISIBILITY_DB) setup-schema -v 0.0 + ./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) --db $(VISIBILITY_DB) update-schema -d ./schema/mysql/v8/visibility/versioned + install-schema-postgresql: temporal-sql-tool @printf $(COLOR) "Install Postgres schema..." ./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) -p 5432 --pl postgres --db $(TEMPORAL_DB) drop -f @@ -430,6 +441,9 @@ start-es: temporal-server start-mysql: temporal-server ./temporal-server --env development-mysql --allow-no-auth start +start-mysql8: temporal-server + ./temporal-server --env development-mysql8 --allow-no-auth start + start-mysql-es: temporal-server ./temporal-server --env development-mysql-es --allow-no-auth start diff --git a/common/persistence/persistence-tests/persistenceTestBase.go b/common/persistence/persistence-tests/persistenceTestBase.go index 58e214c35cb..ca62d23a96c 100644 --- a/common/persistence/persistence-tests/persistenceTestBase.go +++ b/common/persistence/persistence-tests/persistenceTestBase.go @@ -136,7 +136,7 @@ func NewTestBaseWithSQL(options *TestBaseOptions) TestBase { if options.DBPort == 0 { switch options.SQLDBPluginName { - case mysql.PluginName: + case mysql.PluginName, mysql.PluginNameV8: options.DBPort = environment.GetMySQLPort() case postgresql.PluginName: options.DBPort = environment.GetPostgreSQLPort() @@ -148,7 +148,7 @@ func NewTestBaseWithSQL(options *TestBaseOptions) TestBase { } if options.DBHost == "" { switch options.SQLDBPluginName { - case mysql.PluginName: + case mysql.PluginName, mysql.PluginNameV8: options.DBHost = environment.GetMySQLAddress() case postgresql.PluginName: options.DBHost = environment.GetPostgreSQLAddress() diff --git a/common/persistence/persistence-tests/setup.go b/common/persistence/persistence-tests/setup.go index 04c0cac6f32..85a8cac31d0 100644 --- a/common/persistence/persistence-tests/setup.go +++ b/common/persistence/persistence-tests/setup.go @@ -33,9 +33,10 @@ import ( ) const ( - testMySQLUser = "temporal" - testMySQLPassword = "temporal" - testMySQLSchemaDir = "schema/mysql/v57" + testMySQLUser = "temporal" + testMySQLPassword = "temporal" + testMySQLSchemaDir = "schema/mysql/v57" + testMySQL8SchemaDir = "schema/mysql/v8" testPostgreSQLUser = "temporal" testPostgreSQLPassword = "temporal" @@ -61,6 +62,19 @@ func GetMySQLTestClusterOption() *TestBaseOptions { } } +// GetMySQL8TestClusterOption return test options +func GetMySQL8TestClusterOption() *TestBaseOptions { + return &TestBaseOptions{ + SQLDBPluginName: mysql.PluginNameV8, + DBUsername: testMySQLUser, + DBPassword: testMySQLPassword, + DBHost: environment.GetMySQLAddress(), + DBPort: environment.GetMySQLPort(), + SchemaDir: testMySQL8SchemaDir, + StoreType: config.StoreTypeSQL, + } +} + // GetPostgreSQLTestClusterOption return test options func GetPostgreSQLTestClusterOption() *TestBaseOptions { return &TestBaseOptions{ diff --git a/common/persistence/sql/sqlplugin/mysql/db.go b/common/persistence/sql/sqlplugin/mysql/db.go index 115721c7076..3e1e567df00 100644 --- a/common/persistence/sql/sqlplugin/mysql/db.go +++ b/common/persistence/sql/sqlplugin/mysql/db.go @@ -33,7 +33,7 @@ import ( "go.temporal.io/server/common/persistence/schema" "go.temporal.io/server/common/persistence/sql/sqlplugin" - mysqlschema "go.temporal.io/server/schema/mysql" + mysqlschemaV57 "go.temporal.io/server/schema/mysql/v57" ) // db represents a logical connection to mysql database @@ -120,9 +120,9 @@ func (mdb *db) DbName() string { func (mdb *db) ExpectedVersion() string { switch mdb.dbKind { case sqlplugin.DbKindMain: - return mysqlschema.Version + return mysqlschemaV57.Version case sqlplugin.DbKindVisibility: - return mysqlschema.VisibilityVersion + return mysqlschemaV57.VisibilityVersion default: panic(fmt.Sprintf("unknown db kind %v", mdb.dbKind)) } diff --git a/common/persistence/sql/sqlplugin/mysql/db_v8.go b/common/persistence/sql/sqlplugin/mysql/db_v8.go new file mode 100644 index 00000000000..2aefd906a77 --- /dev/null +++ b/common/persistence/sql/sqlplugin/mysql/db_v8.go @@ -0,0 +1,101 @@ +// The MIT License +// +// Copyright (c) 2020 Temporal Technologies Inc. All rights reserved. +// +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package mysql + +import ( + "context" + "fmt" + + "github.com/jmoiron/sqlx" + + "go.temporal.io/server/common/persistence/schema" + "go.temporal.io/server/common/persistence/sql/sqlplugin" + mysqlschemaV8 "go.temporal.io/server/schema/mysql/v8" +) + +// db represents a logical connection to mysql database +type dbV8 struct { + db +} + +var _ sqlplugin.AdminDB = (*dbV8)(nil) +var _ sqlplugin.DB = (*dbV8)(nil) +var _ sqlplugin.Tx = (*dbV8)(nil) + +// newDB returns an instance of DB, which is a logical +// connection to the underlying mysql database +func newDBV8( + dbKind sqlplugin.DbKind, + dbName string, + xdb *sqlx.DB, + tx *sqlx.Tx, +) *dbV8 { + mdb := &dbV8{ + db: db{ + dbKind: dbKind, + dbName: dbName, + db: xdb, + tx: tx, + }, + } + mdb.conn = xdb + if tx != nil { + mdb.conn = tx + } + mdb.converter = &converter{} + return mdb +} + +// BeginTx starts a new transaction and returns a reference to the Tx object +func (mdb *dbV8) BeginTx(ctx context.Context) (sqlplugin.Tx, error) { + xtx, err := mdb.db.db.BeginTxx(ctx, nil) + if err != nil { + return nil, err + } + return newDBV8(mdb.dbKind, mdb.dbName, mdb.db.db, xtx), nil +} + +// PluginName returns the name of the mysql plugin +func (mdb *dbV8) PluginName() string { + return PluginNameV8 +} + +// ExpectedVersion returns expected version. +func (mdb *dbV8) ExpectedVersion() string { + switch mdb.dbKind { + case sqlplugin.DbKindMain: + return mysqlschemaV8.Version + case sqlplugin.DbKindVisibility: + return mysqlschemaV8.VisibilityVersion + default: + panic(fmt.Sprintf("unknown db kind %v", mdb.dbKind)) + } +} + +// VerifyVersion verify schema version is up to date +func (mdb *dbV8) VerifyVersion() error { + expectedVersion := mdb.ExpectedVersion() + return schema.VerifyCompatibleVersion(mdb, mdb.dbName, expectedVersion) +} diff --git a/common/persistence/sql/sqlplugin/mysql/plugin_v8.go b/common/persistence/sql/sqlplugin/mysql/plugin_v8.go new file mode 100644 index 00000000000..6146fbb864e --- /dev/null +++ b/common/persistence/sql/sqlplugin/mysql/plugin_v8.go @@ -0,0 +1,75 @@ +// The MIT License +// +// Copyright (c) 2020 Temporal Technologies Inc. All rights reserved. +// +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package mysql + +import ( + "go.temporal.io/server/common/config" + "go.temporal.io/server/common/persistence/sql" + "go.temporal.io/server/common/persistence/sql/sqlplugin" + "go.temporal.io/server/common/resolver" +) + +const ( + // PluginName is the name of the plugin + PluginNameV8 = "mysql8" +) + +type pluginV8 struct { + plugin +} + +var _ sqlplugin.Plugin = (*pluginV8)(nil) + +func init() { + sql.RegisterPlugin(PluginNameV8, &pluginV8{}) +} + +// CreateDB initialize the db object +func (p *pluginV8) CreateDB( + dbKind sqlplugin.DbKind, + cfg *config.SQL, + r resolver.ServiceResolver, +) (sqlplugin.DB, error) { + conn, err := p.createDBConnection(cfg, r) + if err != nil { + return nil, err + } + db := newDBV8(dbKind, cfg.DatabaseName, conn, nil) + return db, nil +} + +// CreateAdminDB initialize the db object +func (p *pluginV8) CreateAdminDB( + dbKind sqlplugin.DbKind, + cfg *config.SQL, + r resolver.ServiceResolver, +) (sqlplugin.AdminDB, error) { + conn, err := p.createDBConnection(cfg, r) + if err != nil { + return nil, err + } + db := newDBV8(dbKind, cfg.DatabaseName, conn, nil) + return db, nil +} diff --git a/common/persistence/tests/mysql_test.go b/common/persistence/tests/mysql_test.go index 58431ce48cd..225c39592c5 100644 --- a/common/persistence/tests/mysql_test.go +++ b/common/persistence/tests/mysql_test.go @@ -152,6 +152,13 @@ func TestMySQLVisibilityPersistenceSuite(t *testing.T) { suite.Run(t, s) } +func TestMySQL8VisibilityPersistenceSuite(t *testing.T) { + s := &VisibilityPersistenceSuite{ + TestBase: persistencetests.NewTestBaseWithSQL(persistencetests.GetMySQL8TestClusterOption()), + } + suite.Run(t, s) +} + // TODO: Merge persistence-tests into the tests directory. func TestMySQLHistoryV2PersistenceSuite(t *testing.T) { @@ -182,6 +189,34 @@ func TestMySQLClusterMetadataPersistence(t *testing.T) { suite.Run(t, s) } +func TestMySQL8HistoryV2PersistenceSuite(t *testing.T) { + s := new(persistencetests.HistoryV2PersistenceSuite) + s.TestBase = persistencetests.NewTestBaseWithSQL(persistencetests.GetMySQL8TestClusterOption()) + s.TestBase.Setup(nil) + suite.Run(t, s) +} + +func TestMySQL8MetadataPersistenceSuiteV2(t *testing.T) { + s := new(persistencetests.MetadataPersistenceSuiteV2) + s.TestBase = persistencetests.NewTestBaseWithSQL(persistencetests.GetMySQL8TestClusterOption()) + s.TestBase.Setup(nil) + suite.Run(t, s) +} + +func TestMySQL8QueuePersistence(t *testing.T) { + s := new(persistencetests.QueuePersistenceSuite) + s.TestBase = persistencetests.NewTestBaseWithSQL(persistencetests.GetMySQL8TestClusterOption()) + s.TestBase.Setup(nil) + suite.Run(t, s) +} + +func TestMySQL8ClusterMetadataPersistence(t *testing.T) { + s := new(persistencetests.ClusterMetadataManagerSuite) + s.TestBase = persistencetests.NewTestBaseWithSQL(persistencetests.GetMySQL8TestClusterOption()) + s.TestBase.Setup(nil) + suite.Run(t, s) +} + // SQL Store tests func TestMySQLNamespaceSuite(t *testing.T) { diff --git a/common/persistence/tests/mysql_test_util.go b/common/persistence/tests/mysql_test_util.go index 6ed4b788246..7b563a40292 100644 --- a/common/persistence/tests/mysql_test_util.go +++ b/common/persistence/tests/mysql_test_util.go @@ -38,7 +38,7 @@ import ( p "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/persistence/sql" "go.temporal.io/server/common/persistence/sql/sqlplugin" - _ "go.temporal.io/server/common/persistence/sql/sqlplugin/mysql" + "go.temporal.io/server/common/persistence/sql/sqlplugin/mysql" "go.temporal.io/server/common/resolver" "go.temporal.io/server/common/shuffle" "go.temporal.io/server/environment" @@ -56,8 +56,8 @@ const ( // TODO hard code this dir for now // need to merge persistence test config / initialization in one place - testMySQLExecutionSchema = "../../../schema/mysql/v57/temporal/schema.sql" - testMySQLVisibilitySchema = "../../../schema/mysql/v57/visibility/schema.sql" + testMySQLExecutionSchema = "../../../schema/mysql/v8/temporal/schema.sql" + testMySQLVisibilitySchema = "../../../schema/mysql/v8/visibility/schema.sql" ) type ( @@ -100,7 +100,7 @@ func NewMySQLConfig() *config.SQL { strconv.Itoa(environment.GetMySQLPort()), ), ConnectProtocol: testMySQLConnectionProtocol, - PluginName: "mysql", + PluginName: mysql.PluginNameV8, DatabaseName: testMySQLDatabaseNamePrefix + shuffle.String(testMySQLDatabaseNameSuffix), } } diff --git a/config/development-mysql8.yaml b/config/development-mysql8.yaml new file mode 100644 index 00000000000..393437603e3 --- /dev/null +++ b/config/development-mysql8.yaml @@ -0,0 +1,117 @@ +log: + stdout: true + level: info + +persistence: + defaultStore: mysql-default + visibilityStore: mysql-visibility + numHistoryShards: 4 + datastores: + mysql-default: + sql: + pluginName: "mysql8" + databaseName: "temporal" + connectAddr: "127.0.0.1:3306" + connectProtocol: "tcp" + user: "temporal" + password: "temporal" + maxConns: 20 + maxIdleConns: 20 + maxConnLifetime: "1h" + mysql-visibility: + sql: + pluginName: "mysql8" + databaseName: "temporal_visibility" + connectAddr: "127.0.0.1:3306" + connectProtocol: "tcp" + user: "temporal" + password: "temporal" + maxConns: 2 + maxIdleConns: 2 + maxConnLifetime: "1h" + +global: + membership: + maxJoinDuration: 30s + broadcastAddress: "127.0.0.1" + pprof: + port: 7936 + metrics: + prometheus: +# # specify framework to use new approach for initializing metrics and/or use opentelemetry +# framework: "opentelemetry" + framework: "tally" + timerType: "histogram" + listenAddress: "127.0.0.1:8000" + +services: + frontend: + rpc: + grpcPort: 7233 + membershipPort: 6933 + bindOnLocalHost: true + + matching: + rpc: + grpcPort: 7235 + membershipPort: 6935 + bindOnLocalHost: true + + history: + rpc: + grpcPort: 7234 + membershipPort: 6934 + bindOnLocalHost: true + + worker: + rpc: + grpcPort: 7239 + membershipPort: 6939 + bindOnLocalHost: true + +clusterMetadata: + enableGlobalNamespace: false + failoverVersionIncrement: 10 + masterClusterName: "active" + currentClusterName: "active" + clusterInformation: + active: + enabled: true + initialFailoverVersion: 1 + rpcName: "frontend" + rpcAddress: "localhost:7233" + +dcRedirectionPolicy: + policy: "noop" + toDC: "" + +archival: + history: + state: "enabled" + enableRead: true + provider: + filestore: + fileMode: "0666" + dirMode: "0766" + gstorage: + credentialsPath: "/tmp/gcloud/keyfile.json" + visibility: + state: "enabled" + enableRead: true + provider: + filestore: + fileMode: "0666" + dirMode: "0766" + +namespaceDefaults: + archival: + history: + state: "disabled" + URI: "file:///tmp/temporal_archival/development" + visibility: + state: "disabled" + URI: "file:///tmp/temporal_vis_archival/development" + +dynamicConfigClient: + filepath: "config/dynamicconfig/development-sql.yaml" + pollInterval: "10s" diff --git a/schema/mysql/version.go b/schema/mysql/v57/version.go similarity index 99% rename from schema/mysql/version.go rename to schema/mysql/v57/version.go index 202d98ab91d..d4a22d65241 100644 --- a/schema/mysql/version.go +++ b/schema/mysql/v57/version.go @@ -22,7 +22,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -package mysql +package v57 // NOTE: whenever there is a new database schema update, plz update the following versions diff --git a/schema/mysql/v8/temporal/database.sql b/schema/mysql/v8/temporal/database.sql new file mode 100644 index 00000000000..f38c409fa70 --- /dev/null +++ b/schema/mysql/v8/temporal/database.sql @@ -0,0 +1 @@ +CREATE DATABASE temporal CHARACTER SET utf8mb4; diff --git a/schema/mysql/v8/temporal/schema.sql b/schema/mysql/v8/temporal/schema.sql new file mode 100644 index 00000000000..3e4b8dd4170 --- /dev/null +++ b/schema/mysql/v8/temporal/schema.sql @@ -0,0 +1,296 @@ +CREATE TABLE namespaces( + partition_id INT NOT NULL, + id BINARY(16) NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + notification_version BIGINT NOT NULL, + -- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + is_global TINYINT(1) NOT NULL, + PRIMARY KEY(partition_id, id) +); + +CREATE TABLE namespace_metadata ( + partition_id INT NOT NULL, + notification_version BIGINT NOT NULL, + PRIMARY KEY(partition_id) +); + +INSERT INTO namespace_metadata (partition_id, notification_version) VALUES (54321, 1); + +CREATE TABLE shards ( + shard_id INT NOT NULL, + -- + range_id BIGINT NOT NULL, + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id) +); + +CREATE TABLE executions( + shard_id INT NOT NULL, + namespace_id BINARY(16) NOT NULL, + workflow_id VARCHAR(255) NOT NULL, + run_id BINARY(16) NOT NULL, + -- + next_event_id BIGINT NOT NULL, + last_write_version BIGINT NOT NULL, + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + state MEDIUMBLOB NOT NULL, + state_encoding VARCHAR(16) NOT NULL, + db_record_version BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (shard_id, namespace_id, workflow_id, run_id) +); + +CREATE TABLE current_executions( + shard_id INT NOT NULL, + namespace_id BINARY(16) NOT NULL, + workflow_id VARCHAR(255) NOT NULL, + -- + run_id BINARY(16) NOT NULL, + create_request_id VARCHAR(255) NOT NULL, + state INT NOT NULL, + status INT NOT NULL, + start_version BIGINT NOT NULL DEFAULT 0, + last_write_version BIGINT NOT NULL, + PRIMARY KEY (shard_id, namespace_id, workflow_id) +); + +CREATE TABLE buffered_events ( + shard_id INT NOT NULL, + namespace_id BINARY(16) NOT NULL, + workflow_id VARCHAR(255) NOT NULL, + run_id BINARY(16) NOT NULL, + id BIGINT AUTO_INCREMENT NOT NULL UNIQUE, + -- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, namespace_id, workflow_id, run_id, id) +); + +CREATE TABLE tasks ( + range_hash INT UNSIGNED NOT NULL, + task_queue_id VARBINARY(272) NOT NULL, + task_id BIGINT NOT NULL, + -- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (range_hash, task_queue_id, task_id) +); + +CREATE TABLE task_queues ( + range_hash INT UNSIGNED NOT NULL, + task_queue_id VARBINARY(272) NOT NULL, + -- + range_id BIGINT NOT NULL, + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (range_hash, task_queue_id) +); + +CREATE TABLE history_immediate_tasks( + shard_id INT NOT NULL, + category_id INT NOT NULL, + task_id BIGINT NOT NULL, + -- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, category_id, task_id) +); + +CREATE TABLE history_scheduled_tasks ( + shard_id INT NOT NULL, + category_id INT NOT NULL, + visibility_timestamp DATETIME(6) NOT NULL, + task_id BIGINT NOT NULL, + -- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, category_id, visibility_timestamp, task_id) +); + +CREATE TABLE transfer_tasks( + shard_id INT NOT NULL, + task_id BIGINT NOT NULL, + -- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, task_id) +); + +CREATE TABLE timer_tasks ( + shard_id INT NOT NULL, + visibility_timestamp DATETIME(6) NOT NULL, + task_id BIGINT NOT NULL, + -- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, visibility_timestamp, task_id) +); + +CREATE TABLE replication_tasks ( + shard_id INT NOT NULL, + task_id BIGINT NOT NULL, + -- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, task_id) +); + +CREATE TABLE replication_tasks_dlq ( + source_cluster_name VARCHAR(255) NOT NULL, + shard_id INT NOT NULL, + task_id BIGINT NOT NULL, + -- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (source_cluster_name, shard_id, task_id) +); + +CREATE TABLE visibility_tasks( + shard_id INT NOT NULL, + task_id BIGINT NOT NULL, + -- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, task_id) +); + +CREATE TABLE activity_info_maps ( +-- each row corresponds to one key of one map + shard_id INT NOT NULL, + namespace_id BINARY(16) NOT NULL, + workflow_id VARCHAR(255) NOT NULL, + run_id BINARY(16) NOT NULL, + schedule_id BIGINT NOT NULL, +-- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16), + PRIMARY KEY (shard_id, namespace_id, workflow_id, run_id, schedule_id) +); + +CREATE TABLE timer_info_maps ( + shard_id INT NOT NULL, + namespace_id BINARY(16) NOT NULL, + workflow_id VARCHAR(255) NOT NULL, + run_id BINARY(16) NOT NULL, + timer_id VARCHAR(255) NOT NULL, +-- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16), + PRIMARY KEY (shard_id, namespace_id, workflow_id, run_id, timer_id) +); + +CREATE TABLE child_execution_info_maps ( + shard_id INT NOT NULL, + namespace_id BINARY(16) NOT NULL, + workflow_id VARCHAR(255) NOT NULL, + run_id BINARY(16) NOT NULL, + initiated_id BIGINT NOT NULL, +-- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16), + PRIMARY KEY (shard_id, namespace_id, workflow_id, run_id, initiated_id) +); + +CREATE TABLE request_cancel_info_maps ( + shard_id INT NOT NULL, + namespace_id BINARY(16) NOT NULL, + workflow_id VARCHAR(255) NOT NULL, + run_id BINARY(16) NOT NULL, + initiated_id BIGINT NOT NULL, +-- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16), + PRIMARY KEY (shard_id, namespace_id, workflow_id, run_id, initiated_id) +); + +CREATE TABLE signal_info_maps ( + shard_id INT NOT NULL, + namespace_id BINARY(16) NOT NULL, + workflow_id VARCHAR(255) NOT NULL, + run_id BINARY(16) NOT NULL, + initiated_id BIGINT NOT NULL, +-- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16), + PRIMARY KEY (shard_id, namespace_id, workflow_id, run_id, initiated_id) +); + +CREATE TABLE signals_requested_sets ( + shard_id INT NOT NULL, + namespace_id BINARY(16) NOT NULL, + workflow_id VARCHAR(255) NOT NULL, + run_id BINARY(16) NOT NULL, + signal_id VARCHAR(255) NOT NULL, + -- + PRIMARY KEY (shard_id, namespace_id, workflow_id, run_id, signal_id) +); + +-- history eventsV2: history_node stores history event data +CREATE TABLE history_node ( + shard_id INT NOT NULL, + tree_id BINARY(16) NOT NULL, + branch_id BINARY(16) NOT NULL, + node_id BIGINT NOT NULL, + txn_id BIGINT NOT NULL, + -- + prev_txn_id BIGINT NOT NULL DEFAULT 0, + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, tree_id, branch_id, node_id, txn_id) +); + +-- history eventsV2: history_tree stores branch metadata +CREATE TABLE history_tree ( + shard_id INT NOT NULL, + tree_id BINARY(16) NOT NULL, + branch_id BINARY(16) NOT NULL, + -- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, tree_id, branch_id) +); + +CREATE TABLE queue ( + queue_type INT NOT NULL, + message_id BIGINT NOT NULL, + message_payload MEDIUMBLOB NOT NULL, + message_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY(queue_type, message_id) +); + +CREATE TABLE queue_metadata ( + queue_type INT NOT NULL, + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + version BIGINT NOT NULL, + PRIMARY KEY(queue_type) +); + +CREATE TABLE cluster_metadata_info ( + metadata_partition INT NOT NULL, + cluster_name VARCHAR(255) NOT NULL, + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + version BIGINT NOT NULL, + PRIMARY KEY(metadata_partition, cluster_name) +); + +CREATE TABLE cluster_membership ( + membership_partition INT NOT NULL, + host_id BINARY(16) NOT NULL, + rpc_address VARCHAR(128) NOT NULL, + rpc_port SMALLINT NOT NULL, + role TINYINT NOT NULL, + session_start TIMESTAMP DEFAULT '1970-01-02 00:00:01', + last_heartbeat TIMESTAMP DEFAULT '1970-01-02 00:00:01', + record_expiry TIMESTAMP DEFAULT '1970-01-02 00:00:01', + INDEX (role, host_id), + INDEX (role, last_heartbeat), + INDEX (rpc_address, role), + INDEX (last_heartbeat), + INDEX (record_expiry), + PRIMARY KEY (membership_partition, host_id) +); diff --git a/schema/mysql/v8/temporal/versioned/v1.0/manifest.json b/schema/mysql/v8/temporal/versioned/v1.0/manifest.json new file mode 100644 index 00000000000..698c16b092b --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.0/manifest.json @@ -0,0 +1,8 @@ +{ + "CurrVersion": "1.0", + "MinCompatibleVersion": "0.1", + "Description": "base version of schema", + "SchemaUpdateCqlFiles": [ + "schema.sql" + ] +} diff --git a/schema/mysql/v8/temporal/versioned/v1.0/schema.sql b/schema/mysql/v8/temporal/versioned/v1.0/schema.sql new file mode 100644 index 00000000000..84a1f8c2c6c --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.0/schema.sql @@ -0,0 +1,260 @@ +CREATE TABLE namespaces( + partition_id INT NOT NULL, + id BINARY(16) NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + notification_version BIGINT NOT NULL, + -- + data BLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + is_global TINYINT(1) NOT NULL, + PRIMARY KEY(partition_id, id) +); + +CREATE TABLE namespace_metadata ( + partition_id INT NOT NULL, + notification_version BIGINT NOT NULL, + PRIMARY KEY(partition_id) +); + +INSERT INTO namespace_metadata (partition_id, notification_version) VALUES (54321, 1); + +CREATE TABLE shards ( + shard_id INT NOT NULL, + -- + range_id BIGINT NOT NULL, + data BLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id) +); + +CREATE TABLE transfer_tasks( + shard_id INT NOT NULL, + task_id BIGINT NOT NULL, + -- + data BLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, task_id) +); + +CREATE TABLE executions( + shard_id INT NOT NULL, + namespace_id BINARY(16) NOT NULL, + workflow_id VARCHAR(255) NOT NULL, + run_id BINARY(16) NOT NULL, + -- + next_event_id BIGINT NOT NULL, + last_write_version BIGINT NOT NULL, + data BLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + state BLOB NOT NULL, + state_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, namespace_id, workflow_id, run_id) +); + +CREATE TABLE current_executions( + shard_id INT NOT NULL, + namespace_id BINARY(16) NOT NULL, + workflow_id VARCHAR(255) NOT NULL, + -- + run_id BINARY(16) NOT NULL, + create_request_id VARCHAR(64) NOT NULL, + state INT NOT NULL, + status INT NOT NULL, + start_version BIGINT NOT NULL, + last_write_version BIGINT NOT NULL, + PRIMARY KEY (shard_id, namespace_id, workflow_id) +); + +CREATE TABLE buffered_events ( + shard_id INT NOT NULL, + namespace_id BINARY(16) NOT NULL, + workflow_id VARCHAR(255) NOT NULL, + run_id BINARY(16) NOT NULL, + id BIGINT AUTO_INCREMENT NOT NULL UNIQUE, + -- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, namespace_id, workflow_id, run_id, id) +); + +CREATE TABLE tasks ( + range_hash INT UNSIGNED NOT NULL, + task_queue_id VARBINARY(272) NOT NULL, + task_id BIGINT NOT NULL, + -- + data BLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (range_hash, task_queue_id, task_id) +); + +CREATE TABLE task_queues ( + range_hash INT UNSIGNED NOT NULL, + task_queue_id VARBINARY(272) NOT NULL, + -- + range_id BIGINT NOT NULL, + data BLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (range_hash, task_queue_id) +); + +CREATE TABLE replication_tasks ( + shard_id INT NOT NULL, + task_id BIGINT NOT NULL, + -- + data BLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, task_id) +); + +CREATE TABLE replication_tasks_dlq ( + source_cluster_name VARCHAR(255) NOT NULL, + shard_id INT NOT NULL, + task_id BIGINT NOT NULL, + -- + data BLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (source_cluster_name, shard_id, task_id) +); + +CREATE TABLE timer_tasks ( + shard_id INT NOT NULL, + visibility_timestamp DATETIME(6) NOT NULL, + task_id BIGINT NOT NULL, + -- + data BLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, visibility_timestamp, task_id) +); + +CREATE TABLE activity_info_maps ( +-- each row corresponds to one key of one map + shard_id INT NOT NULL, + namespace_id BINARY(16) NOT NULL, + workflow_id VARCHAR(255) NOT NULL, + run_id BINARY(16) NOT NULL, + schedule_id BIGINT NOT NULL, +-- + data BLOB NOT NULL, + data_encoding VARCHAR(16), + PRIMARY KEY (shard_id, namespace_id, workflow_id, run_id, schedule_id) +); + +CREATE TABLE timer_info_maps ( + shard_id INT NOT NULL, + namespace_id BINARY(16) NOT NULL, + workflow_id VARCHAR(255) NOT NULL, + run_id BINARY(16) NOT NULL, + timer_id VARCHAR(255) NOT NULL, +-- + data BLOB NOT NULL, + data_encoding VARCHAR(16), + PRIMARY KEY (shard_id, namespace_id, workflow_id, run_id, timer_id) +); + +CREATE TABLE child_execution_info_maps ( + shard_id INT NOT NULL, + namespace_id BINARY(16) NOT NULL, + workflow_id VARCHAR(255) NOT NULL, + run_id BINARY(16) NOT NULL, + initiated_id BIGINT NOT NULL, +-- + data BLOB NOT NULL, + data_encoding VARCHAR(16), + PRIMARY KEY (shard_id, namespace_id, workflow_id, run_id, initiated_id) +); + +CREATE TABLE request_cancel_info_maps ( + shard_id INT NOT NULL, + namespace_id BINARY(16) NOT NULL, + workflow_id VARCHAR(255) NOT NULL, + run_id BINARY(16) NOT NULL, + initiated_id BIGINT NOT NULL, +-- + data BLOB NOT NULL, + data_encoding VARCHAR(16), + PRIMARY KEY (shard_id, namespace_id, workflow_id, run_id, initiated_id) +); + +CREATE TABLE signal_info_maps ( + shard_id INT NOT NULL, + namespace_id BINARY(16) NOT NULL, + workflow_id VARCHAR(255) NOT NULL, + run_id BINARY(16) NOT NULL, + initiated_id BIGINT NOT NULL, +-- + data BLOB NOT NULL, + data_encoding VARCHAR(16), + PRIMARY KEY (shard_id, namespace_id, workflow_id, run_id, initiated_id) +); + +CREATE TABLE signals_requested_sets ( + shard_id INT NOT NULL, + namespace_id BINARY(16) NOT NULL, + workflow_id VARCHAR(255) NOT NULL, + run_id BINARY(16) NOT NULL, + signal_id VARCHAR(64) NOT NULL, + -- + PRIMARY KEY (shard_id, namespace_id, workflow_id, run_id, signal_id) +); + +-- history eventsV2: history_node stores history event data +CREATE TABLE history_node ( + shard_id INT NOT NULL, + tree_id BINARY(16) NOT NULL, + branch_id BINARY(16) NOT NULL, + node_id BIGINT NOT NULL, + txn_id BIGINT NOT NULL, + -- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, tree_id, branch_id, node_id, txn_id) +); + +-- history eventsV2: history_tree stores branch metadata +CREATE TABLE history_tree ( + shard_id INT NOT NULL, + tree_id BINARY(16) NOT NULL, + branch_id BINARY(16) NOT NULL, + -- + data BLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, tree_id, branch_id) +); + +CREATE TABLE queue ( + queue_type INT NOT NULL, + message_id BIGINT NOT NULL, + message_payload BLOB NOT NULL, + PRIMARY KEY(queue_type, message_id) +); + +CREATE TABLE queue_metadata ( + queue_type INT NOT NULL, + data BLOB NOT NULL, + PRIMARY KEY(queue_type) +); + +CREATE TABLE cluster_metadata ( + metadata_partition INT NOT NULL, + immutable_data BLOB NOT NULL, + immutable_data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY(metadata_partition) +); + +CREATE TABLE cluster_membership +( + membership_partition INT NOT NULL, + host_id BINARY(16) NOT NULL, + rpc_address VARCHAR(15) NOT NULL, + rpc_port SMALLINT NOT NULL, + role TINYINT NOT NULL, + session_start TIMESTAMP DEFAULT '1970-01-02 00:00:01', + last_heartbeat TIMESTAMP DEFAULT '1970-01-02 00:00:01', + record_expiry TIMESTAMP DEFAULT '1970-01-02 00:00:01', + INDEX (role, host_id), + INDEX (role, last_heartbeat), + INDEX (rpc_address, role), + INDEX (last_heartbeat), + INDEX (record_expiry), + PRIMARY KEY (membership_partition, host_id) +); diff --git a/schema/mysql/v8/temporal/versioned/v1.1/cluster_metadata.sql b/schema/mysql/v8/temporal/versioned/v1.1/cluster_metadata.sql new file mode 100644 index 00000000000..36db5eb44c6 --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.1/cluster_metadata.sql @@ -0,0 +1,3 @@ +ALTER TABLE cluster_metadata ADD data BLOB NOT NULL; +ALTER TABLE cluster_metadata ADD data_encoding VARCHAR(16) NOT NULL DEFAULT 'Proto3'; +ALTER TABLE cluster_metadata ADD version BIGINT NOT NULL DEFAULT 1; \ No newline at end of file diff --git a/schema/mysql/v8/temporal/versioned/v1.1/manifest.json b/schema/mysql/v8/temporal/versioned/v1.1/manifest.json new file mode 100644 index 00000000000..ebcefd1c74e --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.1/manifest.json @@ -0,0 +1,8 @@ +{ + "CurrVersion": "1.1", + "MinCompatibleVersion": "1.0", + "Description": "schema update for cluster metadata", + "SchemaUpdateCqlFiles": [ + "cluster_metadata.sql" + ] +} diff --git a/schema/mysql/v8/temporal/versioned/v1.2/blob_size.sql b/schema/mysql/v8/temporal/versioned/v1.2/blob_size.sql new file mode 100644 index 00000000000..51d3fcebee4 --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.2/blob_size.sql @@ -0,0 +1,20 @@ +ALTER TABLE namespaces MODIFY COLUMN data MEDIUMBLOB; +ALTER TABLE shards MODIFY COLUMN data MEDIUMBLOB; +ALTER TABLE transfer_tasks MODIFY COLUMN data MEDIUMBLOB; +ALTER TABLE executions MODIFY COLUMN data MEDIUMBLOB; +ALTER TABLE executions MODIFY COLUMN state MEDIUMBLOB; +ALTER TABLE tasks MODIFY COLUMN data MEDIUMBLOB; +ALTER TABLE task_queues MODIFY COLUMN data MEDIUMBLOB; +ALTER TABLE replication_tasks MODIFY COLUMN data MEDIUMBLOB; +ALTER TABLE replication_tasks_dlq MODIFY COLUMN data MEDIUMBLOB; +ALTER TABLE timer_tasks MODIFY COLUMN data MEDIUMBLOB; +ALTER TABLE activity_info_maps MODIFY COLUMN data MEDIUMBLOB; +ALTER TABLE timer_info_maps MODIFY COLUMN data MEDIUMBLOB; +ALTER TABLE child_execution_info_maps MODIFY COLUMN data MEDIUMBLOB; +ALTER TABLE request_cancel_info_maps MODIFY COLUMN data MEDIUMBLOB; +ALTER TABLE signal_info_maps MODIFY COLUMN data MEDIUMBLOB; +ALTER TABLE history_tree MODIFY COLUMN data MEDIUMBLOB; +ALTER TABLE queue MODIFY COLUMN message_payload MEDIUMBLOB; +ALTER TABLE queue_metadata MODIFY COLUMN data MEDIUMBLOB; +ALTER TABLE cluster_metadata MODIFY COLUMN data MEDIUMBLOB; +ALTER TABLE cluster_metadata MODIFY COLUMN immutable_data MEDIUMBLOB; diff --git a/schema/mysql/v8/temporal/versioned/v1.2/manifest.json b/schema/mysql/v8/temporal/versioned/v1.2/manifest.json new file mode 100644 index 00000000000..d25c966894a --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.2/manifest.json @@ -0,0 +1,9 @@ +{ + "CurrVersion": "1.2", + "MinCompatibleVersion": "1.0", + "Description": "schema update for RPC replication and blob size adjustments", + "SchemaUpdateCqlFiles": [ + "queue.sql", + "blob_size.sql" + ] +} diff --git a/schema/mysql/v8/temporal/versioned/v1.2/queue.sql b/schema/mysql/v8/temporal/versioned/v1.2/queue.sql new file mode 100644 index 00000000000..e3f1587fbb0 --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.2/queue.sql @@ -0,0 +1,2 @@ +ALTER TABLE queue ADD message_encoding VARCHAR(16) NOT NULL DEFAULT 'Json'; +ALTER TABLE queue_metadata ADD data_encoding VARCHAR(16) NOT NULL DEFAULT 'Json'; diff --git a/schema/mysql/v8/temporal/versioned/v1.3/manifest.json b/schema/mysql/v8/temporal/versioned/v1.3/manifest.json new file mode 100644 index 00000000000..bf9e4fd3f89 --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.3/manifest.json @@ -0,0 +1,8 @@ +{ + "CurrVersion": "1.3", + "MinCompatibleVersion": "1.0", + "Description": "schema update for kafka deprecation", + "SchemaUpdateCqlFiles": [ + "visibility_tasks.sql" + ] +} \ No newline at end of file diff --git a/schema/mysql/v8/temporal/versioned/v1.3/visibility_tasks.sql b/schema/mysql/v8/temporal/versioned/v1.3/visibility_tasks.sql new file mode 100644 index 00000000000..b8ebcdd696b --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.3/visibility_tasks.sql @@ -0,0 +1,8 @@ +CREATE TABLE visibility_tasks( + shard_id INT NOT NULL, + task_id BIGINT NOT NULL, + -- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, task_id) +); \ No newline at end of file diff --git a/schema/mysql/v8/temporal/versioned/v1.4/cluster_metadata.sql b/schema/mysql/v8/temporal/versioned/v1.4/cluster_metadata.sql new file mode 100644 index 00000000000..b909e3a8cfa --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.4/cluster_metadata.sql @@ -0,0 +1,2 @@ +ALTER TABLE cluster_metadata DROP immutable_data; +ALTER TABLE cluster_metadata DROP immutable_data_encoding; diff --git a/schema/mysql/v8/temporal/versioned/v1.4/manifest.json b/schema/mysql/v8/temporal/versioned/v1.4/manifest.json new file mode 100644 index 00000000000..3f2a8f4c07f --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.4/manifest.json @@ -0,0 +1,8 @@ +{ + "CurrVersion": "1.4", + "MinCompatibleVersion": "1.0", + "Description": "schema update for cluster metadata cleanup", + "SchemaUpdateCqlFiles": [ + "cluster_metadata.sql" + ] +} diff --git a/schema/mysql/v8/temporal/versioned/v1.5/cluster_membership.sql b/schema/mysql/v8/temporal/versioned/v1.5/cluster_membership.sql new file mode 100644 index 00000000000..04485b82cf0 --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.5/cluster_membership.sql @@ -0,0 +1 @@ +ALTER TABLE cluster_membership MODIFY COLUMN rpc_address VARCHAR(128); diff --git a/schema/mysql/v8/temporal/versioned/v1.5/event.sql b/schema/mysql/v8/temporal/versioned/v1.5/event.sql new file mode 100644 index 00000000000..6437b811b76 --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.5/event.sql @@ -0,0 +1 @@ +ALTER TABLE history_node ADD prev_txn_id BIGINT NOT NULL DEFAULT 0; diff --git a/schema/mysql/v8/temporal/versioned/v1.5/executions.sql b/schema/mysql/v8/temporal/versioned/v1.5/executions.sql new file mode 100644 index 00000000000..dbdd9e52af7 --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.5/executions.sql @@ -0,0 +1 @@ +ALTER TABLE executions ADD db_record_version BIGINT NOT NULL DEFAULT 0; diff --git a/schema/mysql/v8/temporal/versioned/v1.5/manifest.json b/schema/mysql/v8/temporal/versioned/v1.5/manifest.json new file mode 100644 index 00000000000..56fc8d59f2e --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.5/manifest.json @@ -0,0 +1,10 @@ +{ + "CurrVersion": "1.5", + "MinCompatibleVersion": "1.0", + "Description": "schema update for cluster_membership, executions and history_node tables", + "SchemaUpdateCqlFiles": [ + "event.sql", + "executions.sql", + "cluster_membership.sql" + ] +} diff --git a/schema/mysql/v8/temporal/versioned/v1.6/manifest.json b/schema/mysql/v8/temporal/versioned/v1.6/manifest.json new file mode 100644 index 00000000000..bbd66684b2e --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.6/manifest.json @@ -0,0 +1,8 @@ +{ + "CurrVersion": "1.6", + "MinCompatibleVersion": "1.0", + "Description": "schema update for queue_metadata", + "SchemaUpdateCqlFiles": [ + "queue_metadata.sql" + ] +} diff --git a/schema/mysql/v8/temporal/versioned/v1.6/queue_metadata.sql b/schema/mysql/v8/temporal/versioned/v1.6/queue_metadata.sql new file mode 100644 index 00000000000..33830d21b0d --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.6/queue_metadata.sql @@ -0,0 +1 @@ +ALTER TABLE queue_metadata ADD version BIGINT NOT NULL DEFAULT 0; diff --git a/schema/mysql/v8/temporal/versioned/v1.7/cluster_metadata_info.sql b/schema/mysql/v8/temporal/versioned/v1.7/cluster_metadata_info.sql new file mode 100644 index 00000000000..e7141a7ee48 --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.7/cluster_metadata_info.sql @@ -0,0 +1,8 @@ +CREATE TABLE cluster_metadata_info ( + metadata_partition INT NOT NULL, + cluster_name VARCHAR(255) NOT NULL, + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + version BIGINT NOT NULL, + PRIMARY KEY(metadata_partition, cluster_name) +); \ No newline at end of file diff --git a/schema/mysql/v8/temporal/versioned/v1.7/manifest.json b/schema/mysql/v8/temporal/versioned/v1.7/manifest.json new file mode 100644 index 00000000000..5d99bfe1134 --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.7/manifest.json @@ -0,0 +1,10 @@ +{ + "CurrVersion": "1.7", + "MinCompatibleVersion": "1.0", + "Description": "create cluster metadata info table to store cluster information and executions to store tiered storage queue", + "SchemaUpdateCqlFiles": [ + "cluster_metadata_info.sql", + "no_start_version.sql", + "tiered_storage_tasks.sql" + ] +} diff --git a/schema/mysql/v8/temporal/versioned/v1.7/no_start_version.sql b/schema/mysql/v8/temporal/versioned/v1.7/no_start_version.sql new file mode 100644 index 00000000000..d58adf0c319 --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.7/no_start_version.sql @@ -0,0 +1 @@ +ALTER TABLE current_executions ALTER COLUMN start_version SET DEFAULT 0; \ No newline at end of file diff --git a/schema/mysql/v8/temporal/versioned/v1.7/tiered_storage_tasks.sql b/schema/mysql/v8/temporal/versioned/v1.7/tiered_storage_tasks.sql new file mode 100644 index 00000000000..8f9dc7b461f --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.7/tiered_storage_tasks.sql @@ -0,0 +1,8 @@ +CREATE TABLE tiered_storage_tasks ( + shard_id INT NOT NULL, + task_id BIGINT NOT NULL, + -- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, task_id) +); \ No newline at end of file diff --git a/schema/mysql/v8/temporal/versioned/v1.8/alter_columns.sql b/schema/mysql/v8/temporal/versioned/v1.8/alter_columns.sql new file mode 100644 index 00000000000..47943f1b550 --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.8/alter_columns.sql @@ -0,0 +1,2 @@ +ALTER TABLE current_executions MODIFY COLUMN create_request_id VARCHAR(255); +ALTER TABLE signals_requested_sets MODIFY COLUMN signal_id VARCHAR(255); diff --git a/schema/mysql/v8/temporal/versioned/v1.8/drop_unused_tasks_table.sql b/schema/mysql/v8/temporal/versioned/v1.8/drop_unused_tasks_table.sql new file mode 100644 index 00000000000..b4b35b2ce5e --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.8/drop_unused_tasks_table.sql @@ -0,0 +1 @@ +DROP TABLE tiered_storage_tasks; \ No newline at end of file diff --git a/schema/mysql/v8/temporal/versioned/v1.8/manifest.json b/schema/mysql/v8/temporal/versioned/v1.8/manifest.json new file mode 100644 index 00000000000..d49fed323fc --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.8/manifest.json @@ -0,0 +1,9 @@ +{ + "CurrVersion": "1.8", + "MinCompatibleVersion": "1.0", + "Description": "drop unused tasks table; expand VARCHAR columns governed by maxIDLength to VARCHAR(255)", + "SchemaUpdateCqlFiles": [ + "drop_unused_tasks_table.sql", + "alter_columns.sql" + ] +} diff --git a/schema/mysql/v8/temporal/versioned/v1.9/history_tasks_table.sql b/schema/mysql/v8/temporal/versioned/v1.9/history_tasks_table.sql new file mode 100644 index 00000000000..6c7cd15480c --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.9/history_tasks_table.sql @@ -0,0 +1,20 @@ +CREATE TABLE history_immediate_tasks( + shard_id INT NOT NULL, + category_id INT NOT NULL, + task_id BIGINT NOT NULL, + -- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, category_id, task_id) +); + +CREATE TABLE history_scheduled_tasks ( + shard_id INT NOT NULL, + category_id INT NOT NULL, + visibility_timestamp DATETIME(6) NOT NULL, + task_id BIGINT NOT NULL, + -- + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, category_id, visibility_timestamp, task_id) +); \ No newline at end of file diff --git a/schema/mysql/v8/temporal/versioned/v1.9/manifest.json b/schema/mysql/v8/temporal/versioned/v1.9/manifest.json new file mode 100644 index 00000000000..9df45be3080 --- /dev/null +++ b/schema/mysql/v8/temporal/versioned/v1.9/manifest.json @@ -0,0 +1,8 @@ +{ + "CurrVersion": "1.9", + "MinCompatibleVersion": "1.0", + "Description": "add history tasks table", + "SchemaUpdateCqlFiles": [ + "history_tasks_table.sql" + ] +} diff --git a/schema/mysql/v8/version.go b/schema/mysql/v8/version.go new file mode 100644 index 00000000000..a7f1301985e --- /dev/null +++ b/schema/mysql/v8/version.go @@ -0,0 +1,33 @@ +// The MIT License +// +// Copyright (c) 2020 Temporal Technologies Inc. All rights reserved. +// +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package v8 + +// NOTE: whenever there is a new database schema update, plz update the following versions + +// Version is the MySQL database release version +const Version = "1.9" + +// VisibilityVersion is the MySQL visibility database release version +const VisibilityVersion = "1.2" diff --git a/schema/mysql/v8/visibility/database.sql b/schema/mysql/v8/visibility/database.sql new file mode 100644 index 00000000000..4cb0a650d0e --- /dev/null +++ b/schema/mysql/v8/visibility/database.sql @@ -0,0 +1 @@ +CREATE DATABASE temporal_visibility CHARACTER SET utf8mb4; diff --git a/schema/mysql/v8/visibility/schema.sql b/schema/mysql/v8/visibility/schema.sql new file mode 100644 index 00000000000..2b91b0782fb --- /dev/null +++ b/schema/mysql/v8/visibility/schema.sql @@ -0,0 +1,143 @@ +CREATE TABLE executions_visibility ( + namespace_id CHAR(64) NOT NULL, + run_id CHAR(64) NOT NULL, + start_time DATETIME(6) NOT NULL, + execution_time DATETIME(6) NOT NULL, + workflow_id VARCHAR(255) NOT NULL, + workflow_type_name VARCHAR(255) NOT NULL, + status INT NOT NULL, -- enum WorkflowExecutionStatus {RUNNING, COMPLETED, FAILED, CANCELED, TERMINATED, CONTINUED_AS_NEW, TIMED_OUT} + close_time DATETIME(6) NULL, + history_length BIGINT NULL, + memo BLOB NULL, + encoding VARCHAR(64) NOT NULL, + task_queue VARCHAR(255) NOT NULL DEFAULT '', + search_attributes JSON NULL, + + -- Each search attribute has its own generated column. + -- For string types (keyword and text), we need to unquote the json string, + -- ie., use `->>` instead of `->` operator. + -- For text types, the generated column need to be STORED instead of VIRTUAL, + -- so we can create a full-text search index. + -- For datetime type, MySQL can't cast datetime string with timezone to + -- datetime type directly, so we need to call CONVERT_TZ to convert to UTC. + -- Check the `custom_search_attributes` table for complete set of examples. + + -- Predefined search attributes + TemporalChangeVersion JSON GENERATED ALWAYS AS (search_attributes->"$.TemporalChangeVersion"), + BinaryChecksums JSON GENERATED ALWAYS AS (search_attributes->"$.BinaryChecksums"), + BatcherUser VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.BatcherUser"), + TemporalScheduledStartTime DATETIME(6) GENERATED ALWAYS AS ( + CONVERT_TZ( + REGEXP_REPLACE(search_attributes->>"$.TemporalScheduledStartTime", 'Z|[+-][0-9]{2}:[0-9]{2}$', ''), + SUBSTR(REPLACE(search_attributes->>"$.TemporalScheduledStartTime", 'Z', '+00:00'), -6, 6), + '+00:00' + ) + ), + TemporalScheduledById VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.TemporalScheduledById"), + TemporalSchedulePaused BOOLEAN GENERATED ALWAYS AS (search_attributes->"$.TemporalSchedulePaused"), + TemporalNamespaceDivision VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.TemporalNamespaceDivision"), + + PRIMARY KEY (namespace_id, run_id) +); + +CREATE INDEX default_idx ON executions_visibility (namespace_id, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_execution_time ON executions_visibility (namespace_id, execution_time, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_workflow_id ON executions_visibility (namespace_id, workflow_id, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_workflow_type ON executions_visibility (namespace_id, workflow_type_name, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_status ON executions_visibility (namespace_id, status, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_history_length ON executions_visibility (namespace_id, history_length, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_task_queue ON executions_visibility (namespace_id, task_queue, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); + +-- Indexes for the predefined search attributes +CREATE INDEX by_temporal_change_version ON executions_visibility (namespace_id, (CAST(TemporalChangeVersion AS CHAR(255) ARRAY)), (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_binary_checksums ON executions_visibility (namespace_id, (CAST(BinaryChecksums AS CHAR(255) ARRAY)), (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_batcher_user ON executions_visibility (namespace_id, BatcherUser, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_temporal_scheduled_start_time ON executions_visibility (namespace_id, TemporalScheduledStartTime, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_temporal_scheduled_by_id ON executions_visibility (namespace_id, TemporalScheduledById, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_temporal_schedule_paused ON executions_visibility (namespace_id, TemporalSchedulePaused, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_temporal_namespace_division ON executions_visibility (namespace_id, TemporalNamespaceDivision, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); + + +CREATE TABLE custom_search_attributes ( + namespace_id CHAR(64) NOT NULL, + run_id CHAR(64) NOT NULL, + search_attributes JSON NULL, + Bool01 BOOLEAN GENERATED ALWAYS AS (search_attributes->"$.Bool01"), + Bool02 BOOLEAN GENERATED ALWAYS AS (search_attributes->"$.Bool02"), + Bool03 BOOLEAN GENERATED ALWAYS AS (search_attributes->"$.Bool03"), + Datetime01 DATETIME(6) GENERATED ALWAYS AS ( + CONVERT_TZ( + REGEXP_REPLACE(search_attributes->>"$.Datetime01", 'Z|[+-][0-9]{2}:[0-9]{2}$', ''), + SUBSTR(REPLACE(search_attributes->>"$.Datetime01", 'Z', '+00:00'), -6, 6), + '+00:00' + ) + ), + Datetime02 DATETIME(6) GENERATED ALWAYS AS ( + CONVERT_TZ( + REGEXP_REPLACE(search_attributes->>"$.Datetime02", 'Z|[+-][0-9]{2}:[0-9]{2}$', ''), + SUBSTR(REPLACE(search_attributes->>"$.Datetime02", 'Z', '+00:00'), -6, 6), + '+00:00' + ) + ), + Datetime03 DATETIME(6) GENERATED ALWAYS AS ( + CONVERT_TZ( + REGEXP_REPLACE(search_attributes->>"$.Datetime03", 'Z|[+-][0-9]{2}:[0-9]{2}$', ''), + SUBSTR(REPLACE(search_attributes->>"$.Datetime03", 'Z', '+00:00'), -6, 6), + '+00:00' + ) + ), + Double01 DECIMAL(20, 5) GENERATED ALWAYS AS (search_attributes->"$.Double01"), + Double02 DECIMAL(20, 5) GENERATED ALWAYS AS (search_attributes->"$.Double02"), + Double03 DECIMAL(20, 5) GENERATED ALWAYS AS (search_attributes->"$.Double03"), + Int01 BIGINT GENERATED ALWAYS AS (search_attributes->"$.Int01"), + Int02 BIGINT GENERATED ALWAYS AS (search_attributes->"$.Int02"), + Int03 BIGINT GENERATED ALWAYS AS (search_attributes->"$.Int03"), + Keyword01 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.Keyword01"), + Keyword02 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.Keyword02"), + Keyword03 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.Keyword03"), + Keyword04 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.Keyword04"), + Keyword05 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.Keyword05"), + Keyword06 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.Keyword06"), + Keyword07 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.Keyword07"), + Keyword08 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.Keyword08"), + Keyword09 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.Keyword09"), + Keyword10 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.Keyword10"), + Text01 TEXT GENERATED ALWAYS AS (search_attributes->>"$.Text01") STORED, + Text02 TEXT GENERATED ALWAYS AS (search_attributes->>"$.Text02") STORED, + Text03 TEXT GENERATED ALWAYS AS (search_attributes->>"$.Text03") STORED, + KeywordList01 JSON GENERATED ALWAYS AS (search_attributes->"$.KeywordList01"), + KeywordList02 JSON GENERATED ALWAYS AS (search_attributes->"$.KeywordList02"), + KeywordList03 JSON GENERATED ALWAYS AS (search_attributes->"$.KeywordList03"), + + PRIMARY KEY (namespace_id, run_id), + FOREIGN KEY (namespace_id, run_id) REFERENCES executions_visibility (namespace_id, run_id) ON DELETE CASCADE +); + +CREATE INDEX by_bool_01 ON custom_search_attributes (namespace_id, Bool01); +CREATE INDEX by_bool_02 ON custom_search_attributes (namespace_id, Bool02); +CREATE INDEX by_bool_03 ON custom_search_attributes (namespace_id, Bool03); +CREATE INDEX by_datetime_01 ON custom_search_attributes (namespace_id, Datetime01); +CREATE INDEX by_datetime_02 ON custom_search_attributes (namespace_id, Datetime02); +CREATE INDEX by_datetime_03 ON custom_search_attributes (namespace_id, Datetime03); +CREATE INDEX by_double_01 ON custom_search_attributes (namespace_id, Double01); +CREATE INDEX by_double_02 ON custom_search_attributes (namespace_id, Double02); +CREATE INDEX by_double_03 ON custom_search_attributes (namespace_id, Double03); +CREATE INDEX by_int_01 ON custom_search_attributes (namespace_id, Int01); +CREATE INDEX by_int_02 ON custom_search_attributes (namespace_id, Int02); +CREATE INDEX by_int_03 ON custom_search_attributes (namespace_id, Int03); +CREATE INDEX by_keyword_01 ON custom_search_attributes (namespace_id, Keyword01); +CREATE INDEX by_keyword_02 ON custom_search_attributes (namespace_id, Keyword02); +CREATE INDEX by_keyword_03 ON custom_search_attributes (namespace_id, Keyword03); +CREATE INDEX by_keyword_04 ON custom_search_attributes (namespace_id, Keyword04); +CREATE INDEX by_keyword_05 ON custom_search_attributes (namespace_id, Keyword05); +CREATE INDEX by_keyword_06 ON custom_search_attributes (namespace_id, Keyword06); +CREATE INDEX by_keyword_07 ON custom_search_attributes (namespace_id, Keyword07); +CREATE INDEX by_keyword_08 ON custom_search_attributes (namespace_id, Keyword08); +CREATE INDEX by_keyword_09 ON custom_search_attributes (namespace_id, Keyword09); +CREATE INDEX by_keyword_10 ON custom_search_attributes (namespace_id, Keyword10); +CREATE FULLTEXT INDEX by_text_01 ON custom_search_attributes (namespace_id, Text01); +CREATE FULLTEXT INDEX by_text_02 ON custom_search_attributes (namespace_id, Text02); +CREATE FULLTEXT INDEX by_text_03 ON custom_search_attributes (namespace_id, Text03); +CREATE INDEX by_keyword_list_01 ON custom_search_attributes (namespace_id, (CAST(KeywordList01 AS CHAR(255) ARRAY))); +CREATE INDEX by_keyword_list_02 ON custom_search_attributes (namespace_id, (CAST(KeywordList02 AS CHAR(255) ARRAY))); +CREATE INDEX by_keyword_list_03 ON custom_search_attributes (namespace_id, (CAST(KeywordList03 AS CHAR(255) ARRAY))); diff --git a/schema/mysql/v8/visibility/versioned/v1.0/manifest.json b/schema/mysql/v8/visibility/versioned/v1.0/manifest.json new file mode 100644 index 00000000000..a1ce2f782bd --- /dev/null +++ b/schema/mysql/v8/visibility/versioned/v1.0/manifest.json @@ -0,0 +1,8 @@ +{ + "CurrVersion": "1.0", + "MinCompatibleVersion": "0.1", + "Description": "base version of visibility schema", + "SchemaUpdateCqlFiles": [ + "schema.sql" + ] +} diff --git a/schema/mysql/v8/visibility/versioned/v1.0/schema.sql b/schema/mysql/v8/visibility/versioned/v1.0/schema.sql new file mode 100644 index 00000000000..fe0ce2503c3 --- /dev/null +++ b/schema/mysql/v8/visibility/versioned/v1.0/schema.sql @@ -0,0 +1,23 @@ +CREATE TABLE executions_visibility ( + namespace_id CHAR(64) NOT NULL, + run_id CHAR(64) NOT NULL, + start_time DATETIME(6) NOT NULL, + execution_time DATETIME(6) NOT NULL, + workflow_id VARCHAR(255) NOT NULL, + workflow_type_name VARCHAR(255) NOT NULL, + status INT NOT NULL, -- enum WorkflowExecutionStatus {RUNNING, COMPLETED, FAILED, CANCELED, TERMINATED, CONTINUED_AS_NEW, TIMED_OUT} + close_time DATETIME(6) NULL, + history_length BIGINT, + memo BLOB, + encoding VARCHAR(64) NOT NULL, + task_queue VARCHAR(255) DEFAULT '' NOT NULL, + + PRIMARY KEY (namespace_id, run_id) +); + +CREATE INDEX by_type_start_time ON executions_visibility (namespace_id, workflow_type_name, status, start_time DESC, run_id); +CREATE INDEX by_workflow_id_start_time ON executions_visibility (namespace_id, workflow_id, status, start_time DESC, run_id); +CREATE INDEX by_status_by_start_time ON executions_visibility (namespace_id, status, start_time DESC, run_id); +CREATE INDEX by_type_close_time ON executions_visibility (namespace_id, workflow_type_name, status, close_time DESC, run_id); +CREATE INDEX by_workflow_id_close_time ON executions_visibility (namespace_id, workflow_id, status, close_time DESC, run_id); +CREATE INDEX by_status_by_close_time ON executions_visibility (namespace_id, status, close_time DESC, run_id); diff --git a/schema/mysql/v8/visibility/versioned/v1.1/index.sql b/schema/mysql/v8/visibility/versioned/v1.1/index.sql new file mode 100644 index 00000000000..52c629705a1 --- /dev/null +++ b/schema/mysql/v8/visibility/versioned/v1.1/index.sql @@ -0,0 +1 @@ +CREATE INDEX by_close_time_by_status ON executions_visibility (namespace_id, close_time DESC, run_id, status); diff --git a/schema/mysql/v8/visibility/versioned/v1.1/manifest.json b/schema/mysql/v8/visibility/versioned/v1.1/manifest.json new file mode 100644 index 00000000000..bdd6fe5b283 --- /dev/null +++ b/schema/mysql/v8/visibility/versioned/v1.1/manifest.json @@ -0,0 +1,8 @@ +{ + "CurrVersion": "1.1", + "MinCompatibleVersion": "0.1", + "Description": "add close time & status index", + "SchemaUpdateCqlFiles": [ + "index.sql" + ] +} diff --git a/schema/mysql/v8/visibility/versioned/v1.2/advanced_visibility.sql b/schema/mysql/v8/visibility/versioned/v1.2/advanced_visibility.sql new file mode 100644 index 00000000000..c25a4213d91 --- /dev/null +++ b/schema/mysql/v8/visibility/versioned/v1.2/advanced_visibility.sql @@ -0,0 +1,129 @@ +-- Add search attributes related columns +ALTER TABLE executions_visibility + ADD COLUMN search_attributes JSON NULL, + ADD COLUMN TemporalChangeVersion JSON GENERATED ALWAYS AS (search_attributes->"$.TemporalChangeVersion"), + ADD COLUMN BinaryChecksums JSON GENERATED ALWAYS AS (search_attributes->"$.BinaryChecksums"), + ADD COLUMN BatcherUser VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.BatcherUser"), + ADD COLUMN TemporalScheduledStartTime DATETIME(6) GENERATED ALWAYS AS ( + CONVERT_TZ( + REGEXP_REPLACE(search_attributes->>"$.TemporalScheduledStartTime", 'Z|[+-][0-9]{2}:[0-9]{2}$', ''), + SUBSTR(REPLACE(search_attributes->>"$.TemporalScheduledStartTime", 'Z', '+00:00'), -6, 6), + '+00:00' + ) + ), + ADD COLUMN TemporalScheduledById VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.TemporalScheduledById"), + ADD COLUMN TemporalSchedulePaused BOOLEAN GENERATED ALWAYS AS (search_attributes->"$.TemporalSchedulePaused"), + ADD COLUMN TemporalNamespaceDivision VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.TemporalNamespaceDivision"); + +-- Drop existing indexes +DROP INDEX by_type_start_time ON executions_visibility; +DROP INDEX by_workflow_id_start_time ON executions_visibility; +DROP INDEX by_status_by_start_time ON executions_visibility; +DROP INDEX by_type_close_time ON executions_visibility; +DROP INDEX by_workflow_id_close_time ON executions_visibility; +DROP INDEX by_status_by_close_time ON executions_visibility; +DROP INDEX by_close_time_by_status ON executions_visibility; + +-- Create new indexes +CREATE INDEX default_idx ON executions_visibility (namespace_id, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_execution_time ON executions_visibility (namespace_id, execution_time, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_workflow_id ON executions_visibility (namespace_id, workflow_id, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_workflow_type ON executions_visibility (namespace_id, workflow_type_name, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_status ON executions_visibility (namespace_id, status, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_history_length ON executions_visibility (namespace_id, history_length, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_task_queue ON executions_visibility (namespace_id, task_queue, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); + +-- Indexes for the predefined search attributes +CREATE INDEX by_temporal_change_version ON executions_visibility (namespace_id, (CAST(TemporalChangeVersion AS CHAR(255) ARRAY)), (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_binary_checksums ON executions_visibility (namespace_id, (CAST(BinaryChecksums AS CHAR(255) ARRAY)), (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_batcher_user ON executions_visibility (namespace_id, BatcherUser, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_temporal_scheduled_start_time ON executions_visibility (namespace_id, TemporalScheduledStartTime, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_temporal_scheduled_by_id ON executions_visibility (namespace_id, TemporalScheduledById, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_temporal_schedule_paused ON executions_visibility (namespace_id, TemporalSchedulePaused, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_temporal_namespace_division ON executions_visibility (namespace_id, TemporalNamespaceDivision, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); + + +-- Custom search attributes +CREATE TABLE custom_search_attributes ( + namespace_id CHAR(64) NOT NULL, + run_id CHAR(64) NOT NULL, + search_attributes JSON NULL, + Bool01 BOOLEAN GENERATED ALWAYS AS (search_attributes->"$.Bool01"), + Bool02 BOOLEAN GENERATED ALWAYS AS (search_attributes->"$.Bool02"), + Bool03 BOOLEAN GENERATED ALWAYS AS (search_attributes->"$.Bool03"), + Datetime01 DATETIME(6) GENERATED ALWAYS AS ( + CONVERT_TZ( + REGEXP_REPLACE(search_attributes->>"$.Datetime01", 'Z|[+-][0-9]{2}:[0-9]{2}$', ''), + SUBSTR(REPLACE(search_attributes->>"$.Datetime01", 'Z', '+00:00'), -6, 6), + '+00:00' + ) + ), + Datetime02 DATETIME(6) GENERATED ALWAYS AS ( + CONVERT_TZ( + REGEXP_REPLACE(search_attributes->>"$.Datetime02", 'Z|[+-][0-9]{2}:[0-9]{2}$', ''), + SUBSTR(REPLACE(search_attributes->>"$.Datetime02", 'Z', '+00:00'), -6, 6), + '+00:00' + ) + ), + Datetime03 DATETIME(6) GENERATED ALWAYS AS ( + CONVERT_TZ( + REGEXP_REPLACE(search_attributes->>"$.Datetime03", 'Z|[+-][0-9]{2}:[0-9]{2}$', ''), + SUBSTR(REPLACE(search_attributes->>"$.Datetime03", 'Z', '+00:00'), -6, 6), + '+00:00' + ) + ), + Double01 DECIMAL(20, 5) GENERATED ALWAYS AS (search_attributes->"$.Double01"), + Double02 DECIMAL(20, 5) GENERATED ALWAYS AS (search_attributes->"$.Double02"), + Double03 DECIMAL(20, 5) GENERATED ALWAYS AS (search_attributes->"$.Double03"), + Int01 BIGINT GENERATED ALWAYS AS (search_attributes->"$.Int01"), + Int02 BIGINT GENERATED ALWAYS AS (search_attributes->"$.Int02"), + Int03 BIGINT GENERATED ALWAYS AS (search_attributes->"$.Int03"), + Keyword01 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.Keyword01"), + Keyword02 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.Keyword02"), + Keyword03 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.Keyword03"), + Keyword04 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.Keyword04"), + Keyword05 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.Keyword05"), + Keyword06 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.Keyword06"), + Keyword07 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.Keyword07"), + Keyword08 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.Keyword08"), + Keyword09 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.Keyword09"), + Keyword10 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.Keyword10"), + Text01 TEXT GENERATED ALWAYS AS (search_attributes->>"$.Text01") STORED, + Text02 TEXT GENERATED ALWAYS AS (search_attributes->>"$.Text02") STORED, + Text03 TEXT GENERATED ALWAYS AS (search_attributes->>"$.Text03") STORED, + KeywordList01 JSON GENERATED ALWAYS AS (search_attributes->"$.KeywordList01"), + KeywordList02 JSON GENERATED ALWAYS AS (search_attributes->"$.KeywordList02"), + KeywordList03 JSON GENERATED ALWAYS AS (search_attributes->"$.KeywordList03"), + + PRIMARY KEY (namespace_id, run_id), + FOREIGN KEY (namespace_id, run_id) REFERENCES executions_visibility (namespace_id, run_id) ON DELETE CASCADE +); + +CREATE INDEX by_bool_01 ON custom_search_attributes (namespace_id, Bool01); +CREATE INDEX by_bool_02 ON custom_search_attributes (namespace_id, Bool02); +CREATE INDEX by_bool_03 ON custom_search_attributes (namespace_id, Bool03); +CREATE INDEX by_datetime_01 ON custom_search_attributes (namespace_id, Datetime01); +CREATE INDEX by_datetime_02 ON custom_search_attributes (namespace_id, Datetime02); +CREATE INDEX by_datetime_03 ON custom_search_attributes (namespace_id, Datetime03); +CREATE INDEX by_double_01 ON custom_search_attributes (namespace_id, Double01); +CREATE INDEX by_double_02 ON custom_search_attributes (namespace_id, Double02); +CREATE INDEX by_double_03 ON custom_search_attributes (namespace_id, Double03); +CREATE INDEX by_int_01 ON custom_search_attributes (namespace_id, Int01); +CREATE INDEX by_int_02 ON custom_search_attributes (namespace_id, Int02); +CREATE INDEX by_int_03 ON custom_search_attributes (namespace_id, Int03); +CREATE INDEX by_keyword_01 ON custom_search_attributes (namespace_id, Keyword01); +CREATE INDEX by_keyword_02 ON custom_search_attributes (namespace_id, Keyword02); +CREATE INDEX by_keyword_03 ON custom_search_attributes (namespace_id, Keyword03); +CREATE INDEX by_keyword_04 ON custom_search_attributes (namespace_id, Keyword04); +CREATE INDEX by_keyword_05 ON custom_search_attributes (namespace_id, Keyword05); +CREATE INDEX by_keyword_06 ON custom_search_attributes (namespace_id, Keyword06); +CREATE INDEX by_keyword_07 ON custom_search_attributes (namespace_id, Keyword07); +CREATE INDEX by_keyword_08 ON custom_search_attributes (namespace_id, Keyword08); +CREATE INDEX by_keyword_09 ON custom_search_attributes (namespace_id, Keyword09); +CREATE INDEX by_keyword_10 ON custom_search_attributes (namespace_id, Keyword10); +CREATE FULLTEXT INDEX by_text_01 ON custom_search_attributes (namespace_id, Text01); +CREATE FULLTEXT INDEX by_text_02 ON custom_search_attributes (namespace_id, Text02); +CREATE FULLTEXT INDEX by_text_03 ON custom_search_attributes (namespace_id, Text03); +CREATE INDEX by_keyword_list_01 ON custom_search_attributes (namespace_id, (CAST(KeywordList01 AS CHAR(255) ARRAY))); +CREATE INDEX by_keyword_list_02 ON custom_search_attributes (namespace_id, (CAST(KeywordList02 AS CHAR(255) ARRAY))); +CREATE INDEX by_keyword_list_03 ON custom_search_attributes (namespace_id, (CAST(KeywordList03 AS CHAR(255) ARRAY))); diff --git a/schema/mysql/v8/visibility/versioned/v1.2/manifest.json b/schema/mysql/v8/visibility/versioned/v1.2/manifest.json new file mode 100644 index 00000000000..3d31b79f6bc --- /dev/null +++ b/schema/mysql/v8/visibility/versioned/v1.2/manifest.json @@ -0,0 +1,8 @@ +{ + "CurrVersion": "1.2", + "MinCompatibleVersion": "0.1", + "Description": "update schema to support advanced visibility", + "SchemaUpdateCqlFiles": [ + "advanced_visibility.sql" + ] +} diff --git a/tests/test_cluster.go b/tests/test_cluster.go index cff632f2323..4b3c29dadb4 100644 --- a/tests/test_cluster.go +++ b/tests/test_cluster.go @@ -125,6 +125,8 @@ func NewCluster(options *TestClusterConfig, logger log.Logger) (*TestCluster, er switch TestFlags.PersistenceDriver { case mysql.PluginName: ops = persistencetests.GetMySQLTestClusterOption() + case mysql.PluginNameV8: + ops = persistencetests.GetMySQL8TestClusterOption() case postgresql.PluginName: ops = persistencetests.GetPostgreSQLTestClusterOption() case sqlite.PluginName: diff --git a/tools/tests/mysql_cli_test.go b/tools/tests/mysql_cli_test.go index 8956e715feb..13bac176c0e 100644 --- a/tools/tests/mysql_cli_test.go +++ b/tools/tests/mysql_cli_test.go @@ -32,7 +32,8 @@ import ( "go.temporal.io/server/common/persistence/sql/sqlplugin/mysql" "go.temporal.io/server/environment" - mysqlversion "go.temporal.io/server/schema/mysql" + mysqlversionV57 "go.temporal.io/server/schema/mysql/v57" + mysqlversionV8 "go.temporal.io/server/schema/mysql/v8" "go.temporal.io/server/tools/sql/clitest" ) @@ -77,9 +78,9 @@ func TestMySQLUpdateSchemaTestSuite(t *testing.T) { mysql.PluginName, testMySQLQuery, testMySQLExecutionSchemaVersionDir, - mysqlversion.Version, + mysqlversionV57.Version, testMySQLVisibilitySchemaVersionDir, - mysqlversion.VisibilityVersion, + mysqlversionV57.VisibilityVersion, )) } @@ -94,3 +95,62 @@ func TestMySQLVersionTestSuite(t *testing.T) { testMySQLVisibilitySchemaFile, )) } + +func TestMySQL8ConnTestSuite(t *testing.T) { + suite.Run(t, clitest.NewSQLConnTestSuite( + environment.GetMySQLAddress(), + strconv.Itoa(environment.GetMySQLPort()), + mysql.PluginNameV8, + testMySQLQuery, + )) +} + +func TestMySQL8HandlerTestSuite(t *testing.T) { + suite.Run(t, clitest.NewHandlerTestSuite( + environment.GetMySQLAddress(), + strconv.Itoa(environment.GetMySQLPort()), + mysql.PluginNameV8, + )) +} + +func TestMySQL8SetupSchemaTestSuite(t *testing.T) { + t.Setenv("SQL_HOST", environment.GetMySQLAddress()) + t.Setenv("SQL_PORT", strconv.Itoa(environment.GetMySQLPort())) + t.Setenv("SQL_USER", testUser) + t.Setenv("SQL_PASSWORD", testPassword) + suite.Run(t, clitest.NewSetupSchemaTestSuite( + environment.GetMySQLAddress(), + strconv.Itoa(environment.GetMySQLPort()), + mysql.PluginNameV8, + testMySQLQuery, + )) +} + +func TestMySQL8UpdateSchemaTestSuite(t *testing.T) { + t.Setenv("SQL_HOST", environment.GetMySQLAddress()) + t.Setenv("SQL_PORT", strconv.Itoa(environment.GetMySQLPort())) + t.Setenv("SQL_USER", testUser) + t.Setenv("SQL_PASSWORD", testPassword) + suite.Run(t, clitest.NewUpdateSchemaTestSuite( + environment.GetMySQLAddress(), + strconv.Itoa(environment.GetMySQLPort()), + mysql.PluginNameV8, + testMySQLQuery, + testMySQL8ExecutionSchemaVersionDir, + mysqlversionV8.Version, + testMySQL8VisibilitySchemaVersionDir, + mysqlversionV8.VisibilityVersion, + )) +} + +func TestMySQL8VersionTestSuite(t *testing.T) { + t.Setenv("SQL_USER", testUser) + t.Setenv("SQL_PASSWORD", testPassword) + suite.Run(t, clitest.NewVersionTestSuite( + environment.GetMySQLAddress(), + strconv.Itoa(environment.GetMySQLPort()), + mysql.PluginNameV8, + testMySQL8ExecutionSchemaFile, + testMySQL8VisibilitySchemaFile, + )) +} diff --git a/tools/tests/test_data.go b/tools/tests/test_data.go index 51d3f9665a1..0797d0f191d 100644 --- a/tools/tests/test_data.go +++ b/tools/tests/test_data.go @@ -25,13 +25,20 @@ package tests const ( - testUser = "temporal" - testPassword = "temporal" + testUser = "temporal" + testPassword = "temporal" + testMySQLExecutionSchemaFile = "../../schema/mysql/v57/temporal/schema.sql" testMySQLVisibilitySchemaFile = "../../schema/mysql/v57/visibility/schema.sql" testMySQLExecutionSchemaVersionDir = "../../schema/mysql/v57/temporal/versioned" testMySQLVisibilitySchemaVersionDir = "../../schema/mysql/v57/visibility/versioned" - testMySQLQuery = ` + + testMySQL8ExecutionSchemaFile = "../../schema/mysql/v8/temporal/schema.sql" + testMySQL8VisibilitySchemaFile = "../../schema/mysql/v8/visibility/schema.sql" + testMySQL8ExecutionSchemaVersionDir = "../../schema/mysql/v8/temporal/versioned" + testMySQL8VisibilitySchemaVersionDir = "../../schema/mysql/v8/visibility/versioned" + + testMySQLQuery = ` -- test sql file content CREATE TABLE executions(