diff --git a/config/config.go b/config/config.go index e9bd29edd393f..adeb4cc1c623c 100644 --- a/config/config.go +++ b/config/config.go @@ -93,9 +93,10 @@ type Config struct { TreatOldVersionUTF8AsUTF8MB4 bool `toml:"treat-old-version-utf8-as-utf8mb4" json:"treat-old-version-utf8-as-utf8mb4"` // EnableTableLock indicate whether enable table lock. // TODO: remove this after table lock features stable. - EnableTableLock bool `toml:"enable-table-lock" json:"enable-table-lock"` - DelayCleanTableLock uint64 `toml:"delay-clean-table-lock" json:"delay-clean-table-lock"` - SplitRegionMaxNum uint64 `toml:"split-region-max-num" json:"split-region-max-num"` + EnableTableLock bool `toml:"enable-table-lock" json:"enable-table-lock"` + DelayCleanTableLock uint64 `toml:"delay-clean-table-lock" json:"delay-clean-table-lock"` + SplitRegionMaxNum uint64 `toml:"split-region-max-num" json:"split-region-max-num"` + StmtSummary StmtSummary `toml:"stmt-summary" json:"stmt-summary"` } // Log is the log section of config. @@ -316,6 +317,14 @@ type PessimisticTxn struct { TTL string `toml:"ttl" json:"ttl"` } +// StmtSummary is the config for statement summary. +type StmtSummary struct { + // The maximum number of statements kept in memory. + MaxStmtCount uint `toml:"max-stmt-count" json:"max-stmt-count"` + // The maximum length of displayed normalized SQL and sample SQL. + MaxSQLLength uint `toml:"max-sql-length" json:"max-sql-length"` +} + var defaultConf = Config{ Host: "0.0.0.0", AdvertiseAddress: "", @@ -410,6 +419,10 @@ var defaultConf = Config{ MaxRetryCount: 256, TTL: "40s", }, + StmtSummary: StmtSummary{ + MaxStmtCount: 100, + MaxSQLLength: 4096, + }, } var ( diff --git a/config/config.toml.example b/config/config.toml.example index 2997fcabe746f..f4891850d7690 100644 --- a/config/config.toml.example +++ b/config/config.toml.example @@ -315,3 +315,10 @@ max-retry-count = 256 # default TTL in milliseconds for pessimistic lock. # The value must between "15s" and "120s". ttl = "40s" + +[stmt-summary] +# max number of statements kept in memory. +max-stmt-count = 100 + +# max length of displayed normalized sql and sample sql. +max-sql-length = 4096 diff --git a/config/config_test.go b/config/config_test.go index 0d4f4cdc4ed0e..a5e0c6e79e04f 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -70,6 +70,9 @@ txn-total-size-limit=2000 [tikv-client] commit-timeout="41s" max-batch-size=128 +[stmt-summary] +max-stmt-count=1000 +max-sql-length=1024 `) c.Assert(err, IsNil) @@ -90,6 +93,8 @@ max-batch-size=128 c.Assert(conf.EnableTableLock, IsTrue) c.Assert(conf.DelayCleanTableLock, Equals, uint64(5)) c.Assert(conf.SplitRegionMaxNum, Equals, uint64(10000)) + c.Assert(conf.StmtSummary.MaxStmtCount, Equals, uint(1000)) + c.Assert(conf.StmtSummary.MaxSQLLength, Equals, uint(1024)) c.Assert(f.Close(), IsNil) c.Assert(os.Remove(configFile), IsNil) diff --git a/domain/global_vars_cache.go b/domain/global_vars_cache.go index 89cc772f8fec7..68a8863a745ee 100644 --- a/domain/global_vars_cache.go +++ b/domain/global_vars_cache.go @@ -18,7 +18,9 @@ import ( "time" "github.com/pingcap/parser/ast" + "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/stmtsummary" ) // GlobalVariableCache caches global variables. @@ -41,6 +43,8 @@ func (gvc *GlobalVariableCache) Update(rows []chunk.Row, fields []*ast.ResultFie gvc.rows = rows gvc.fields = fields gvc.Unlock() + + checkEnableStmtSummary(rows, fields) } // Get gets the global variables from cache. @@ -63,6 +67,28 @@ func (gvc *GlobalVariableCache) Disable() { return } +// checkEnableStmtSummary looks for TiDBEnableStmtSummary and notifies StmtSummary +func checkEnableStmtSummary(rows []chunk.Row, fields []*ast.ResultField) { + for _, row := range rows { + varName := row.GetString(0) + if varName == variable.TiDBEnableStmtSummary { + varVal := row.GetDatum(1, &fields[1].Column.FieldType) + + sVal := "" + if !varVal.IsNull() { + var err error + sVal, err = varVal.ToString() + if err != nil { + return + } + } + + stmtsummary.OnEnableStmtSummaryModified(sVal) + break + } + } +} + // GetGlobalVarsCache gets the global variable cache. func (do *Domain) GetGlobalVarsCache() *GlobalVariableCache { return &do.gvc diff --git a/domain/global_vars_cache_test.go b/domain/global_vars_cache_test.go index 11cb0c95a32f1..455cb30fcc28d 100644 --- a/domain/global_vars_cache_test.go +++ b/domain/global_vars_cache_test.go @@ -14,6 +14,7 @@ package domain import ( + "sync/atomic" "time" . "github.com/pingcap/check" @@ -21,6 +22,7 @@ import ( "github.com/pingcap/parser/charset" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" @@ -96,3 +98,47 @@ func getResultField(colName string, id, offset int) *ast.ResultField { DBName: model.NewCIStr("test"), } } + +func (gvcSuite *testGVCSuite) TestCheckEnableStmtSummary(c *C) { + defer testleak.AfterTest(c)() + testleak.BeforeTest() + + store, err := mockstore.NewMockTikvStore() + c.Assert(err, IsNil) + defer store.Close() + ddlLease := 50 * time.Millisecond + dom := NewDomain(store, ddlLease, 0, mockFactory) + err = dom.Init(ddlLease, sysMockFactory) + c.Assert(err, IsNil) + defer dom.Close() + + gvc := dom.GetGlobalVarsCache() + + rf := getResultField("c", 1, 0) + rf1 := getResultField("c1", 2, 1) + ft := &types.FieldType{ + Tp: mysql.TypeString, + Charset: charset.CharsetBin, + Collate: charset.CollationBin, + } + ft1 := &types.FieldType{ + Tp: mysql.TypeString, + Charset: charset.CharsetBin, + Collate: charset.CollationBin, + } + + atomic.StoreInt32(&variable.EnableStmtSummary, 0) + ck := chunk.NewChunkWithCapacity([]*types.FieldType{ft, ft1}, 1024) + ck.AppendString(0, variable.TiDBEnableStmtSummary) + ck.AppendString(1, "1") + row := ck.GetRow(0) + gvc.Update([]chunk.Row{row}, []*ast.ResultField{rf, rf1}) + c.Assert(atomic.LoadInt32(&variable.EnableStmtSummary), Equals, int32(1)) + + ck = chunk.NewChunkWithCapacity([]*types.FieldType{ft, ft1}, 1024) + ck.AppendString(0, variable.TiDBEnableStmtSummary) + ck.AppendString(1, "0") + row = ck.GetRow(0) + gvc.Update([]chunk.Row{row}, []*ast.ResultField{rf, rf1}) + c.Assert(atomic.LoadInt32(&variable.EnableStmtSummary), Equals, int32(0)) +} diff --git a/executor/adapter.go b/executor/adapter.go index c6e9c9dbd129b..7e66dea831e5f 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -46,6 +46,7 @@ import ( "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/sqlexec" + "github.com/pingcap/tidb/util/stmtsummary" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) @@ -180,6 +181,7 @@ func (a *recordSet) Close() error { a.stmt.LogSlowQuery(a.txnStartTS, a.lastErr == nil) a.stmt.Ctx.GetSessionVars().PrevStmt = a.stmt.OriginText() a.stmt.logAudit() + a.stmt.SummaryStmt() return err } @@ -779,6 +781,27 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool) { } } +// SummaryStmt collects statements for performance_schema.events_statements_summary_by_digest +func (a *ExecStmt) SummaryStmt() { + sessVars := a.Ctx.GetSessionVars() + if sessVars.InRestrictedSQL || atomic.LoadInt32(&variable.EnableStmtSummary) == 0 { + return + } + stmtCtx := sessVars.StmtCtx + normalizedSQL, digest := stmtCtx.SQLDigest() + costTime := time.Since(sessVars.StartTime) + stmtsummary.StmtSummaryByDigestMap.AddStatement(&stmtsummary.StmtExecInfo{ + SchemaName: sessVars.CurrentDB, + OriginalSQL: a.Text, + NormalizedSQL: normalizedSQL, + Digest: digest, + TotalLatency: uint64(costTime.Nanoseconds()), + AffectedRows: stmtCtx.AffectedRows(), + SentRows: 0, + StartTime: sessVars.StartTime, + }) +} + // IsPointGetWithPKOrUniqueKeyByAutoCommit returns true when meets following conditions: // 1. ctx is auto commit tagged // 2. txn is not valid diff --git a/infoschema/perfschema/const.go b/infoschema/perfschema/const.go index a5764ce79d7fc..2bb4579f08890 100644 --- a/infoschema/perfschema/const.go +++ b/infoschema/perfschema/const.go @@ -39,6 +39,7 @@ var perfSchemaTables = []string{ tableStagesCurrent, tableStagesHistory, tableStagesHistoryLong, + tableEventsStatementsSummaryByDigest, } // tableGlobalStatus contains the column name definitions for table global_status, same as MySQL. @@ -374,3 +375,19 @@ const tableStagesHistoryLong = "CREATE TABLE if not exists performance_schema.ev "WORK_ESTIMATED BIGINT(20) UNSIGNED," + "NESTING_EVENT_ID BIGINT(20) UNSIGNED," + "NESTING_EVENT_TYPE ENUM('TRANSACTION','STATEMENT','STAGE'));" + +// tableEventsStatementsSummaryByDigest contains the column name definitions for table +// events_statements_summary_by_digest, same as MySQL. +const tableEventsStatementsSummaryByDigest = "CREATE TABLE if not exists events_statements_summary_by_digest (" + + "SCHEMA_NAME VARCHAR(64) DEFAULT NULL," + + "DIGEST VARCHAR(64) DEFAULT NULL," + + "DIGEST_TEXT LONGTEXT DEFAULT NULL," + + "EXEC_COUNT BIGINT(20) UNSIGNED NOT NULL," + + "SUM_LATENCY BIGINT(20) UNSIGNED NOT NULL," + + "MAX_LATENCY BIGINT(20) UNSIGNED NOT NULL," + + "MIN_LATENCY BIGINT(20) UNSIGNED NOT NULL," + + "AVG_LATENCY BIGINT(20) UNSIGNED NOT NULL," + + "SUM_ROWS_AFFECTED BIGINT(20) UNSIGNED NOT NULL," + + "FIRST_SEEN TIMESTAMP(6) NOT NULL," + + "LAST_SEEN TIMESTAMP(6) NOT NULL," + + "QUERY_SAMPLE_TEXT LONGTEXT DEFAULT NULL);" diff --git a/infoschema/perfschema/tables.go b/infoschema/perfschema/tables.go index dbe3e68155659..65684416e3401 100644 --- a/infoschema/perfschema/tables.go +++ b/infoschema/perfschema/tables.go @@ -16,8 +16,16 @@ package perfschema import ( "github.com/pingcap/parser/model" "github.com/pingcap/tidb/infoschema" + "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta/autoid" + "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/table" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/stmtsummary" +) + +const ( + tableNameEventsStatementsSummaryByDigest = "events_statements_summary_by_digest" ) // perfSchemaTable stands for the fake table all its data is in the memory. @@ -77,3 +85,44 @@ func (vt *perfSchemaTable) GetPhysicalID() int64 { func (vt *perfSchemaTable) Meta() *model.TableInfo { return vt.meta } + +func (vt *perfSchemaTable) getRows(ctx sessionctx.Context, cols []*table.Column) (fullRows [][]types.Datum, err error) { + switch vt.meta.Name.O { + case tableNameEventsStatementsSummaryByDigest: + fullRows = stmtsummary.StmtSummaryByDigestMap.ToDatum() + } + if len(cols) == len(vt.cols) { + return + } + rows := make([][]types.Datum, len(fullRows)) + for i, fullRow := range fullRows { + row := make([]types.Datum, len(cols)) + for j, col := range cols { + row[j] = fullRow[col.Offset] + } + rows[i] = row + } + return rows, nil +} + +// IterRecords implements table.Table IterRecords interface. +func (vt *perfSchemaTable) IterRecords(ctx sessionctx.Context, startKey kv.Key, cols []*table.Column, + fn table.RecordIterFunc) error { + if len(startKey) != 0 { + return table.ErrUnsupportedOp + } + rows, err := vt.getRows(ctx, cols) + if err != nil { + return err + } + for i, row := range rows { + more, err := fn(int64(i), row, cols) + if err != nil { + return err + } + if !more { + break + } + } + return nil +} diff --git a/infoschema/perfschema/tables_test.go b/infoschema/perfschema/tables_test.go index 4349e4e77ca2c..5f7d044431570 100644 --- a/infoschema/perfschema/tables_test.go +++ b/infoschema/perfschema/tables_test.go @@ -17,6 +17,8 @@ import ( "testing" . "github.com/pingcap/check" + "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/util/testkit" @@ -28,22 +30,32 @@ func TestT(t *testing.T) { TestingT(t) } -var _ = Suite(&testSuite{}) +var _ = Suite(&testTableSuite{}) -type testSuite struct { +type testTableSuite struct { + store kv.Storage + dom *domain.Domain } -func (s *testSuite) TestPerfSchemaTables(c *C) { +func (s *testTableSuite) SetUpSuite(c *C) { testleak.BeforeTest() - defer testleak.AfterTest(c)() - store, err := mockstore.NewMockTikvStore() + + var err error + s.store, err = mockstore.NewMockTikvStore() c.Assert(err, IsNil) - defer store.Close() - do, err := session.BootstrapSession(store) + session.DisableStats4Test() + s.dom, err = session.BootstrapSession(s.store) c.Assert(err, IsNil) - defer do.Close() +} + +func (s *testTableSuite) TearDownSuite(c *C) { + defer testleak.AfterTest(c)() + s.dom.Close() + s.store.Close() +} - tk := testkit.NewTestKit(c, store) +func (s *testTableSuite) TestPerfSchemaTables(c *C) { + tk := testkit.NewTestKit(c, s.store) tk.MustExec("use performance_schema") tk.MustQuery("select * from global_status where variable_name = 'Ssl_verify_mode'").Check(testkit.Rows()) @@ -51,3 +63,63 @@ func (s *testSuite) TestPerfSchemaTables(c *C) { tk.MustQuery("select * from setup_actors").Check(testkit.Rows()) tk.MustQuery("select * from events_stages_history_long").Check(testkit.Rows()) } + +// Test events_statements_summary_by_digest +func (s *testTableSuite) TestStmtSummaryTable(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int, b varchar(10))") + + // Statement summary is disabled by default + tk.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("0")) + tk.MustExec("insert into t values(1, 'a')") + tk.MustQuery("select * from performance_schema.events_statements_summary_by_digest").Check(testkit.Rows()) + + tk.MustExec("set global tidb_enable_stmt_summary = 1") + tk.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("1")) + + // Invalidate the cache manually so that tidb_enable_stmt_summary works immediately. + s.dom.GetGlobalVarsCache().Disable() + + // Create a new session to test + tk = testkit.NewTestKitWithInit(c, s.store) + + // Test INSERT + tk.MustExec("insert into t values(1, 'a')") + tk.MustExec("insert into t values(2, 'b')") + tk.MustExec("insert into t VALUES(3, 'c')") + tk.MustExec("/**/insert into t values(4, 'd')") + tk.MustQuery(`select schema_name, exec_count, sum_rows_affected, query_sample_text + from performance_schema.events_statements_summary_by_digest + where digest_text like 'insert into t%'`, + ).Check(testkit.Rows("test 4 4 insert into t values(1, 'a')")) + + // Test SELECT + tk.MustQuery("select * from t where a=2") + tk.MustQuery(`select schema_name, exec_count, sum_rows_affected, query_sample_text + from performance_schema.events_statements_summary_by_digest + where digest_text like 'select * from t%'`, + ).Check(testkit.Rows("test 1 0 select * from t where a=2")) + + // select ... order by + tk.MustQuery(`select schema_name, exec_count, sum_rows_affected, query_sample_text + from performance_schema.events_statements_summary_by_digest + order by exec_count desc limit 1`, + ).Check(testkit.Rows("test 4 4 insert into t values(1, 'a')")) + + // Disable it again + tk.MustExec("set global tidb_enable_stmt_summary = 0") + tk.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("0")) + + // Create a new session to test + tk = testkit.NewTestKitWithInit(c, s.store) + + // This statement shouldn't be summarized + tk.MustQuery("select * from t where a=2") + + // The table should be cleared + tk.MustQuery(`select schema_name, exec_count, sum_rows_affected, query_sample_text + from performance_schema.events_statements_summary_by_digest`, + ).Check(testkit.Rows()) +} diff --git a/session/session.go b/session/session.go index cdc3783d2920f..4c25828592d63 100644 --- a/session/session.go +++ b/session/session.go @@ -1720,6 +1720,7 @@ var builtinGlobalVariable = []string{ variable.TiDBEnableNoopFuncs, variable.TiDBEnableIndexMerge, variable.TiDBTxnMode, + variable.TiDBEnableStmtSummary, } var ( diff --git a/session/tidb.go b/session/tidb.go index 054fd5c302ab4..5cafad530d09a 100644 --- a/session/tidb.go +++ b/session/tidb.go @@ -218,6 +218,7 @@ func runStmt(ctx context.Context, sctx sessionctx.Context, s sqlexec.Statement) // then it could include the transaction commit time. if rs == nil { s.(*executor.ExecStmt).LogSlowQuery(origTxnCtx.StartTS, err == nil) + s.(*executor.ExecStmt).SummaryStmt() sessVars.PrevStmt = s.OriginText() } }() diff --git a/sessionctx/variable/session_test.go b/sessionctx/variable/session_test.go index 7e1a5716a20c1..7efb93315df55 100644 --- a/sessionctx/variable/session_test.go +++ b/sessionctx/variable/session_test.go @@ -51,6 +51,7 @@ func (*testSessionSuite) TestSetSystemVariable(c *C) { {variable.TIDBMemQuotaIndexLookupReader, "1024", false}, {variable.TIDBMemQuotaIndexLookupJoin, "1024", false}, {variable.TIDBMemQuotaNestedLoopApply, "1024", false}, + {variable.TiDBEnableStmtSummary, "1", false}, } for _, t := range tests { err := variable.SetSessionSystemVar(v, t.key, types.NewDatum(t.value)) diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index e28182eba7faf..7d68607a92b96 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -117,6 +117,14 @@ func BoolToIntStr(b bool) string { return "0" } +// BoolToInt32 converts bool to int32 +func BoolToInt32(b bool) int32 { + if b { + return 1 + } + return 0 +} + // we only support MySQL now var defaultSysVars = []*SysVar{ {ScopeGlobal, "gtid_mode", "OFF"}, @@ -707,6 +715,7 @@ var defaultSysVars = []*SysVar{ {ScopeGlobal | ScopeSession, TiDBEnableNoopFuncs, BoolToIntStr(DefTiDBEnableNoopFuncs)}, {ScopeSession, TiDBReplicaRead, "leader"}, {ScopeSession, TiDBAllowRemoveAutoInc, BoolToIntStr(DefTiDBAllowRemoveAutoInc)}, + {ScopeGlobal, TiDBEnableStmtSummary, BoolToIntStr(DefTiDBEnableStmtSummary)}, } // SynonymsSysVariables is synonyms of system variables. diff --git a/sessionctx/variable/sysvar_test.go b/sessionctx/variable/sysvar_test.go index 702db0ac7fce1..5a23978f98f56 100644 --- a/sessionctx/variable/sysvar_test.go +++ b/sessionctx/variable/sysvar_test.go @@ -64,3 +64,8 @@ func (*testSysVarSuite) TestTxnMode(c *C) { err = seVar.setTxnMode("something else") c.Assert(err, NotNil) } + +func (*testSysVarSuite) TestBoolToInt32(c *C) { + c.Assert(BoolToInt32(true), Equals, int32(1)) + c.Assert(BoolToInt32(false), Equals, int32(0)) +} diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 48a454cac6734..af9c936dc4575 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -296,6 +296,9 @@ const ( // TiDBEnableNoopFuncs set true will enable using fake funcs(like get_lock release_lock) TiDBEnableNoopFuncs = "tidb_enable_noop_functions" + + // TiDBEnableStmtSummary indicates whether the statement summary is enabled. + TiDBEnableStmtSummary = "tidb_enable_stmt_summary" ) // Default TiDB system variable values. @@ -362,6 +365,7 @@ const ( DefWaitSplitRegionTimeout = 300 // 300s DefTiDBEnableNoopFuncs = false DefTiDBAllowRemoveAutoInc = false + DefTiDBEnableStmtSummary = false ) // Process global variables. @@ -381,4 +385,5 @@ var ( MaxOfMaxAllowedPacket uint64 = 1073741824 ExpensiveQueryTimeThreshold uint64 = DefTiDBExpensiveQueryTimeThreshold MinExpensiveQueryTimeThreshold uint64 = 10 //10s + EnableStmtSummary int32 = BoolToInt32(DefTiDBEnableStmtSummary) ) diff --git a/sessionctx/variable/varsutil.go b/sessionctx/variable/varsutil.go index 563f6ceb031d4..dcbd97ae1198b 100644 --- a/sessionctx/variable/varsutil.go +++ b/sessionctx/variable/varsutil.go @@ -380,7 +380,8 @@ func ValidateSetSystemVar(vars *SessionVars, name string, value string) (string, TiDBBatchInsert, TiDBDisableTxnAutoRetry, TiDBEnableStreaming, TiDBBatchDelete, TiDBBatchCommit, TiDBEnableCascadesPlanner, TiDBEnableWindowFunction, TiDBCheckMb4ValueInUTF8, TiDBLowResolutionTSO, TiDBEnableIndexMerge, TiDBEnableNoopFuncs, - TiDBScatterRegion, TiDBGeneralLog, TiDBConstraintCheckInPlace, TiDBEnableVectorizedExpression: + TiDBScatterRegion, TiDBGeneralLog, TiDBConstraintCheckInPlace, TiDBEnableVectorizedExpression, + TiDBEnableStmtSummary: fallthrough case GeneralLog, AvoidTemporalUpgrade, BigTables, CheckProxyUsers, LogBin, CoreFile, EndMakersInJSON, SQLLogBin, OfflineMode, PseudoSlaveMode, LowPriorityUpdates, diff --git a/tidb-server/main.go b/tidb-server/main.go index cdfe515e030d0..c2e2d80fbe531 100644 --- a/tidb-server/main.go +++ b/tidb-server/main.go @@ -362,7 +362,7 @@ func loadConfig() string { // hotReloadConfigItems lists all config items which support hot-reload. var hotReloadConfigItems = []string{"Performance.MaxProcs", "Performance.MaxMemory", "Performance.CrossJoin", "Performance.FeedbackProbability", "Performance.QueryFeedbackLimit", "Performance.PseudoEstimateRatio", - "OOMAction", "MemQuotaQuery"} + "OOMAction", "MemQuotaQuery", "StmtSummary.MaxStmtCount", "StmtSummary.MaxSQLLength"} func reloadConfig(nc, c *config.Config) { // Just a part of config items need to be reload explicitly. diff --git a/util/kvcache/simple_lru.go b/util/kvcache/simple_lru.go index 7120e3a5abb7c..b3f18f2871d42 100644 --- a/util/kvcache/simple_lru.go +++ b/util/kvcache/simple_lru.go @@ -38,6 +38,7 @@ type cacheEntry struct { type SimpleLRUCache struct { capacity uint size uint + // 0 indicates no quota quota uint64 guard float64 elements map[string]*list.Element @@ -88,6 +89,17 @@ func (l *SimpleLRUCache) Put(key Key, value Value) { l.elements[hash] = element l.size++ + // Getting used memory is expensive and can be avoided by setting quota to 0. + if l.quota <= 0 { + if l.size > l.capacity { + lru := l.cache.Back() + l.cache.Remove(lru) + delete(l.elements, string(lru.Value.(*cacheEntry).key.Hash())) + l.size-- + } + return + } + memUsed, err := memory.MemUsed() if err != nil { l.DeleteAll() @@ -137,3 +149,13 @@ func (l *SimpleLRUCache) DeleteAll() { func (l *SimpleLRUCache) Size() int { return int(l.size) } + +// Values return all values in cache. +func (l *SimpleLRUCache) Values() []Value { + values := make([]Value, 0, l.cache.Len()) + for ele := l.cache.Front(); ele != nil; ele = ele.Next() { + value := ele.Value.(*cacheEntry).value + values = append(values, value) + } + return values +} diff --git a/util/kvcache/simple_lru_test.go b/util/kvcache/simple_lru_test.go index 1aa17a646f8ed..14e3d629bec61 100644 --- a/util/kvcache/simple_lru_test.go +++ b/util/kvcache/simple_lru_test.go @@ -106,6 +106,22 @@ func (s *testLRUCacheSuite) TestPut(c *C) { c.Assert(root, IsNil) } +func (s *testLRUCacheSuite) TestZeroQuota(c *C) { + lru := NewSimpleLRUCache(100, 0, 0) + c.Assert(lru.capacity, Equals, uint(100)) + + keys := make([]*mockCacheKey, 100) + vals := make([]int64, 100) + + for i := 0; i < 100; i++ { + keys[i] = newMockHashKey(int64(i)) + vals[i] = int64(i) + lru.Put(keys[i], vals[i]) + } + c.Assert(lru.size, Equals, lru.capacity) + c.Assert(lru.size, Equals, uint(100)) +} + func (s *testLRUCacheSuite) TestOOMGuard(c *C) { maxMem, err := memory.MemTotal() c.Assert(err, IsNil) @@ -228,3 +244,25 @@ func (s *testLRUCacheSuite) TestDeleteAll(c *C) { c.Assert(int(lru.size), Equals, 0) } } + +func (s *testLRUCacheSuite) TestValues(c *C) { + maxMem, err := memory.MemTotal() + c.Assert(err, IsNil) + + lru := NewSimpleLRUCache(5, 0, maxMem) + + keys := make([]*mockCacheKey, 5) + vals := make([]int64, 5) + + for i := 0; i < 5; i++ { + keys[i] = newMockHashKey(int64(i)) + vals[i] = int64(i) + lru.Put(keys[i], vals[i]) + } + + values := lru.Values() + c.Assert(len(values), Equals, 5) + for i := 0; i < 5; i++ { + c.Assert(values[i], Equals, int64(4-i)) + } +} diff --git a/util/stmtsummary/statement_summary.go b/util/stmtsummary/statement_summary.go new file mode 100644 index 0000000000000..75e931db5a7c3 --- /dev/null +++ b/util/stmtsummary/statement_summary.go @@ -0,0 +1,230 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package stmtsummary + +import ( + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/sessionctx/variable" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/hack" + "github.com/pingcap/tidb/util/kvcache" +) + +// There're many types of statement summary tables in MySQL, but we have +// only implemented events_statement_summary_by_digest for now. + +// stmtSummaryByDigestKey defines key for stmtSummaryByDigestMap.summaryMap +type stmtSummaryByDigestKey struct { + // Same statements may appear in different schema, but they refer to different tables. + schemaName string + digest string + // TODO: add plan digest + // `hash` is the hash value of this object + hash []byte +} + +// Hash implements SimpleLRUCache.Key +func (key *stmtSummaryByDigestKey) Hash() []byte { + if len(key.hash) == 0 { + key.hash = make([]byte, 0, len(key.schemaName)+len(key.digest)) + key.hash = append(key.hash, hack.Slice(key.digest)...) + key.hash = append(key.hash, hack.Slice(strings.ToLower(key.schemaName))...) + } + return key.hash +} + +// stmtSummaryByDigestMap is a LRU cache that stores statement summaries. +type stmtSummaryByDigestMap struct { + // It's rare to read concurrently, so RWMutex is not needed. + sync.Mutex + summaryMap *kvcache.SimpleLRUCache +} + +// StmtSummaryByDigestMap is a global map containing all statement summaries. +var StmtSummaryByDigestMap = newStmtSummaryByDigestMap() + +// stmtSummaryByDigest is the summary for each type of statements. +type stmtSummaryByDigest struct { + // It's rare to read concurrently, so RWMutex is not needed. + sync.Mutex + schemaName string + digest string + normalizedSQL string + sampleSQL string + execCount uint64 + sumLatency uint64 + maxLatency uint64 + minLatency uint64 + sumAffectedRows uint64 + // Number of rows sent to client. + sumSentRows uint64 + // The first time this type of SQL executes. + firstSeen time.Time + // The last time this type of SQL executes. + lastSeen time.Time +} + +// StmtExecInfo records execution information of each statement. +type StmtExecInfo struct { + SchemaName string + OriginalSQL string + NormalizedSQL string + Digest string + TotalLatency uint64 + AffectedRows uint64 + // Number of rows sent to client. + SentRows uint64 + StartTime time.Time +} + +// newStmtSummaryByDigestMap creates an empty stmtSummaryByDigestMap. +func newStmtSummaryByDigestMap() *stmtSummaryByDigestMap { + maxStmtCount := config.GetGlobalConfig().StmtSummary.MaxStmtCount + return &stmtSummaryByDigestMap{ + summaryMap: kvcache.NewSimpleLRUCache(maxStmtCount, 0, 0), + } +} + +// newStmtSummaryByDigest creates a stmtSummaryByDigest from StmtExecInfo +func newStmtSummaryByDigest(sei *StmtExecInfo) *stmtSummaryByDigest { + // Trim SQL to size MaxSQLLength + maxSQLLength := config.GetGlobalConfig().StmtSummary.MaxSQLLength + normalizedSQL := sei.NormalizedSQL + if len(normalizedSQL) > int(maxSQLLength) { + normalizedSQL = normalizedSQL[:maxSQLLength] + } + sampleSQL := sei.OriginalSQL + if len(sampleSQL) > int(maxSQLLength) { + sampleSQL = sampleSQL[:maxSQLLength] + } + + return &stmtSummaryByDigest{ + schemaName: sei.SchemaName, + digest: sei.Digest, + normalizedSQL: normalizedSQL, + sampleSQL: sampleSQL, + execCount: 1, + sumLatency: sei.TotalLatency, + maxLatency: sei.TotalLatency, + minLatency: sei.TotalLatency, + sumAffectedRows: sei.AffectedRows, + sumSentRows: sei.SentRows, + firstSeen: sei.StartTime, + lastSeen: sei.StartTime, + } +} + +// Add a StmtExecInfo to stmtSummary +func (ssbd *stmtSummaryByDigest) add(sei *StmtExecInfo) { + ssbd.Lock() + + ssbd.sumLatency += sei.TotalLatency + ssbd.execCount++ + if sei.TotalLatency > ssbd.maxLatency { + ssbd.maxLatency = sei.TotalLatency + } + if sei.TotalLatency < ssbd.minLatency { + ssbd.minLatency = sei.TotalLatency + } + ssbd.sumAffectedRows += sei.AffectedRows + ssbd.sumSentRows += sei.SentRows + if sei.StartTime.Before(ssbd.firstSeen) { + ssbd.firstSeen = sei.StartTime + } + if ssbd.lastSeen.Before(sei.StartTime) { + ssbd.lastSeen = sei.StartTime + } + + ssbd.Unlock() +} + +// AddStatement adds a statement to StmtSummaryByDigestMap. +func (ssMap *stmtSummaryByDigestMap) AddStatement(sei *StmtExecInfo) { + key := &stmtSummaryByDigestKey{ + schemaName: sei.SchemaName, + digest: sei.Digest, + } + + ssMap.Lock() + // Check again. Statements could be added before disabling the flag and after Clear() + if atomic.LoadInt32(&variable.EnableStmtSummary) == 0 { + ssMap.Unlock() + return + } + value, ok := ssMap.summaryMap.Get(key) + if !ok { + newSummary := newStmtSummaryByDigest(sei) + ssMap.summaryMap.Put(key, newSummary) + } + ssMap.Unlock() + + // Lock a single entry, not the whole cache. + if ok { + value.(*stmtSummaryByDigest).add(sei) + } +} + +// Clear removes all statement summaries. +func (ssMap *stmtSummaryByDigestMap) Clear() { + ssMap.Lock() + ssMap.summaryMap.DeleteAll() + ssMap.Unlock() +} + +// Convert statement summary to Datum +func (ssMap *stmtSummaryByDigestMap) ToDatum() [][]types.Datum { + ssMap.Lock() + values := ssMap.summaryMap.Values() + ssMap.Unlock() + + rows := make([][]types.Datum, 0, len(values)) + for _, value := range values { + summary := value.(*stmtSummaryByDigest) + summary.Lock() + record := types.MakeDatums( + summary.schemaName, + summary.digest, + summary.normalizedSQL, + summary.execCount, + summary.sumLatency, + summary.maxLatency, + summary.minLatency, + summary.sumLatency/summary.execCount, // AVG_LATENCY + summary.sumAffectedRows, + types.Time{Time: types.FromGoTime(summary.firstSeen), Type: mysql.TypeTimestamp}, + types.Time{Time: types.FromGoTime(summary.lastSeen), Type: mysql.TypeTimestamp}, + summary.sampleSQL, + ) + summary.Unlock() + rows = append(rows, record) + } + + return rows +} + +// OnEnableStmtSummaryModified is triggered once EnableStmtSummary is modified. +func OnEnableStmtSummaryModified(newValue string) { + if variable.TiDBOptOn(newValue) { + atomic.StoreInt32(&variable.EnableStmtSummary, 1) + } else { + atomic.StoreInt32(&variable.EnableStmtSummary, 0) + StmtSummaryByDigestMap.Clear() + } +} diff --git a/util/stmtsummary/statement_summary_test.go b/util/stmtsummary/statement_summary_test.go new file mode 100644 index 0000000000000..614f6f7800146 --- /dev/null +++ b/util/stmtsummary/statement_summary_test.go @@ -0,0 +1,346 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package stmtsummary + +import ( + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + . "github.com/pingcap/check" + "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/sessionctx/variable" + "github.com/pingcap/tidb/types" +) + +var _ = Suite(&testStmtSummarySuite{}) + +type testStmtSummarySuite struct { + ssMap *stmtSummaryByDigestMap +} + +func (s *testStmtSummarySuite) SetUpSuite(c *C) { + s.ssMap = newStmtSummaryByDigestMap() + atomic.StoreInt32(&variable.EnableStmtSummary, 1) +} + +func TestT(t *testing.T) { + CustomVerboseFlag = true + TestingT(t) +} + +// Test stmtSummaryByDigest.AddStatement +func (s *testStmtSummarySuite) TestAddStatement(c *C) { + s.ssMap.Clear() + + // First statement + stmtExecInfo1 := &StmtExecInfo{ + SchemaName: "schema_name", + OriginalSQL: "original_sql1", + NormalizedSQL: "normalized_sql", + Digest: "digest", + TotalLatency: 10000, + AffectedRows: 100, + SentRows: 100, + StartTime: time.Date(2019, 1, 1, 10, 10, 10, 10, time.UTC), + } + key := &stmtSummaryByDigestKey{ + schemaName: stmtExecInfo1.SchemaName, + digest: stmtExecInfo1.Digest, + } + expectedSummary := stmtSummaryByDigest{ + schemaName: stmtExecInfo1.SchemaName, + digest: stmtExecInfo1.Digest, + normalizedSQL: stmtExecInfo1.NormalizedSQL, + sampleSQL: stmtExecInfo1.OriginalSQL, + execCount: 1, + sumLatency: stmtExecInfo1.TotalLatency, + maxLatency: stmtExecInfo1.TotalLatency, + minLatency: stmtExecInfo1.TotalLatency, + sumAffectedRows: stmtExecInfo1.AffectedRows, + sumSentRows: stmtExecInfo1.SentRows, + firstSeen: stmtExecInfo1.StartTime, + lastSeen: stmtExecInfo1.StartTime, + } + + s.ssMap.AddStatement(stmtExecInfo1) + summary, ok := s.ssMap.summaryMap.Get(key) + c.Assert(ok, IsTrue) + c.Assert(*summary.(*stmtSummaryByDigest) == expectedSummary, IsTrue) + + // Second statement + stmtExecInfo2 := &StmtExecInfo{ + SchemaName: "schema_name", + OriginalSQL: "original_sql2", + NormalizedSQL: "normalized_sql", + Digest: "digest", + TotalLatency: 50000, + AffectedRows: 500, + SentRows: 500, + StartTime: time.Date(2019, 1, 1, 10, 10, 20, 10, time.UTC), + } + expectedSummary.execCount++ + expectedSummary.sumLatency += stmtExecInfo2.TotalLatency + expectedSummary.maxLatency = stmtExecInfo2.TotalLatency + expectedSummary.sumAffectedRows += stmtExecInfo2.AffectedRows + expectedSummary.sumSentRows += stmtExecInfo2.SentRows + expectedSummary.lastSeen = stmtExecInfo2.StartTime + + s.ssMap.AddStatement(stmtExecInfo2) + summary, ok = s.ssMap.summaryMap.Get(key) + c.Assert(ok, IsTrue) + c.Assert(*summary.(*stmtSummaryByDigest) == expectedSummary, IsTrue) + + // Third statement + stmtExecInfo3 := &StmtExecInfo{ + SchemaName: "schema_name", + OriginalSQL: "original_sql3", + NormalizedSQL: "normalized_sql", + Digest: "digest", + TotalLatency: 1000, + AffectedRows: 10, + SentRows: 10, + StartTime: time.Date(2019, 1, 1, 10, 10, 0, 10, time.UTC), + } + expectedSummary.execCount++ + expectedSummary.sumLatency += stmtExecInfo3.TotalLatency + expectedSummary.minLatency = stmtExecInfo3.TotalLatency + expectedSummary.sumAffectedRows += stmtExecInfo3.AffectedRows + expectedSummary.sumSentRows += stmtExecInfo3.SentRows + expectedSummary.firstSeen = stmtExecInfo3.StartTime + + s.ssMap.AddStatement(stmtExecInfo3) + summary, ok = s.ssMap.summaryMap.Get(key) + c.Assert(ok, IsTrue) + c.Assert(*summary.(*stmtSummaryByDigest) == expectedSummary, IsTrue) + + // Fourth statement that in a different schema + stmtExecInfo4 := &StmtExecInfo{ + SchemaName: "schema_name2", + OriginalSQL: "original_sql1", + NormalizedSQL: "normalized_sql", + Digest: "digest", + TotalLatency: 1000, + AffectedRows: 10, + SentRows: 10, + StartTime: time.Date(2019, 1, 1, 10, 10, 0, 10, time.UTC), + } + key = &stmtSummaryByDigestKey{ + schemaName: stmtExecInfo4.SchemaName, + digest: stmtExecInfo4.Digest, + } + + s.ssMap.AddStatement(stmtExecInfo4) + c.Assert(s.ssMap.summaryMap.Size(), Equals, 2) + _, ok = s.ssMap.summaryMap.Get(key) + c.Assert(ok, IsTrue) + + // Fifth statement that has a different digest + stmtExecInfo5 := &StmtExecInfo{ + SchemaName: "schema_name", + OriginalSQL: "original_sql1", + NormalizedSQL: "normalized_sql2", + Digest: "digest2", + TotalLatency: 1000, + AffectedRows: 10, + SentRows: 10, + StartTime: time.Date(2019, 1, 1, 10, 10, 0, 10, time.UTC), + } + key = &stmtSummaryByDigestKey{ + schemaName: stmtExecInfo5.SchemaName, + digest: stmtExecInfo5.Digest, + } + + s.ssMap.AddStatement(stmtExecInfo5) + c.Assert(s.ssMap.summaryMap.Size(), Equals, 3) + _, ok = s.ssMap.summaryMap.Get(key) + c.Assert(ok, IsTrue) +} + +func match(c *C, row []types.Datum, expected ...interface{}) { + c.Assert(len(row), Equals, len(expected)) + for i := range row { + got := fmt.Sprintf("%v", row[i].GetValue()) + need := fmt.Sprintf("%v", expected[i]) + c.Assert(got, Equals, need) + } +} + +// Test stmtSummaryByDigest.ToDatum +func (s *testStmtSummarySuite) TestToDatum(c *C) { + s.ssMap.Clear() + + stmtExecInfo1 := &StmtExecInfo{ + SchemaName: "schema_name", + OriginalSQL: "original_sql1", + NormalizedSQL: "normalized_sql", + Digest: "digest", + TotalLatency: 10000, + AffectedRows: 100, + SentRows: 100, + StartTime: time.Date(2019, 1, 1, 10, 10, 10, 10, time.UTC), + } + s.ssMap.AddStatement(stmtExecInfo1) + datums := s.ssMap.ToDatum() + c.Assert(len(datums), Equals, 1) + t := types.Time{Time: types.FromGoTime(stmtExecInfo1.StartTime), Type: mysql.TypeTimestamp} + match(c, datums[0], stmtExecInfo1.SchemaName, stmtExecInfo1.Digest, stmtExecInfo1.NormalizedSQL, + 1, stmtExecInfo1.TotalLatency, stmtExecInfo1.TotalLatency, stmtExecInfo1.TotalLatency, stmtExecInfo1.TotalLatency, + stmtExecInfo1.AffectedRows, t, t, stmtExecInfo1.OriginalSQL) +} + +// Test AddStatement and ToDatum parallel +func (s *testStmtSummarySuite) TestAddStatementParallel(c *C) { + s.ssMap.Clear() + + threads := 8 + loops := 32 + wg := sync.WaitGroup{} + wg.Add(threads) + + addStmtFunc := func() { + defer wg.Done() + stmtExecInfo1 := &StmtExecInfo{ + SchemaName: "schema_name", + OriginalSQL: "original_sql1", + NormalizedSQL: "normalized_sql", + Digest: "digest", + TotalLatency: 10000, + AffectedRows: 100, + SentRows: 100, + StartTime: time.Date(2019, 1, 1, 10, 10, 10, 10, time.UTC), + } + + // Add 32 times with different digest + for i := 0; i < loops; i++ { + stmtExecInfo1.Digest = fmt.Sprintf("digest%d", i) + s.ssMap.AddStatement(stmtExecInfo1) + } + + // There would be 32 summaries + datums := s.ssMap.ToDatum() + c.Assert(len(datums), Equals, loops) + } + + for i := 0; i < threads; i++ { + go addStmtFunc() + } + wg.Wait() + + datums := s.ssMap.ToDatum() + c.Assert(len(datums), Equals, loops) +} + +// Test max number of statement count. +func (s *testStmtSummarySuite) TestMaxStmtCount(c *C) { + s.ssMap.Clear() + + stmtExecInfo1 := &StmtExecInfo{ + SchemaName: "schema_name", + OriginalSQL: "original_sql1", + NormalizedSQL: "normalized_sql", + Digest: "digest", + TotalLatency: 10000, + AffectedRows: 100, + SentRows: 100, + StartTime: time.Date(2019, 1, 1, 10, 10, 10, 10, time.UTC), + } + + maxStmtCount := config.GetGlobalConfig().StmtSummary.MaxStmtCount + + // 1000 digests + loops := int(maxStmtCount) * 10 + for i := 0; i < loops; i++ { + stmtExecInfo1.Digest = fmt.Sprintf("digest%d", i) + s.ssMap.AddStatement(stmtExecInfo1) + } + + // Summary count should be MaxStmtCount + sm := s.ssMap.summaryMap + c.Assert(sm.Size(), Equals, int(maxStmtCount)) + + // LRU cache should work + for i := loops - int(maxStmtCount); i < loops; i++ { + key := &stmtSummaryByDigestKey{ + schemaName: stmtExecInfo1.SchemaName, + digest: fmt.Sprintf("digest%d", i), + } + _, ok := sm.Get(key) + c.Assert(ok, IsTrue) + } +} + +// Test max length of normalized and sample SQL. +func (s *testStmtSummarySuite) TestMaxSQLLength(c *C) { + s.ssMap.Clear() + + // Create a long SQL + maxSQLLength := config.GetGlobalConfig().StmtSummary.MaxSQLLength + length := int(maxSQLLength) * 10 + str := strings.Repeat("a", length) + + stmtExecInfo1 := &StmtExecInfo{ + SchemaName: "schema_name", + OriginalSQL: str, + NormalizedSQL: str, + Digest: "digest", + TotalLatency: 10000, + AffectedRows: 100, + SentRows: 100, + StartTime: time.Date(2019, 1, 1, 10, 10, 10, 10, time.UTC), + } + + s.ssMap.AddStatement(stmtExecInfo1) + key := &stmtSummaryByDigestKey{ + schemaName: stmtExecInfo1.SchemaName, + digest: stmtExecInfo1.Digest, + } + value, ok := s.ssMap.summaryMap.Get(key) + c.Assert(ok, IsTrue) + // Length of normalizedSQL and sampleSQL should be maxSQLLength + summary := value.(*stmtSummaryByDigest) + c.Assert(len(summary.normalizedSQL), Equals, int(maxSQLLength)) + c.Assert(len(summary.sampleSQL), Equals, int(maxSQLLength)) +} + +// Test setting EnableStmtSummary to 0 +func (s *testStmtSummarySuite) TestDisableStmtSummary(c *C) { + s.ssMap.Clear() + OnEnableStmtSummaryModified("0") + + stmtExecInfo1 := &StmtExecInfo{ + SchemaName: "schema_name", + OriginalSQL: "original_sql1", + NormalizedSQL: "normalized_sql", + Digest: "digest", + TotalLatency: 10000, + AffectedRows: 100, + SentRows: 100, + StartTime: time.Date(2019, 1, 1, 10, 10, 10, 10, time.UTC), + } + + s.ssMap.AddStatement(stmtExecInfo1) + datums := s.ssMap.ToDatum() + c.Assert(len(datums), Equals, 0) + + OnEnableStmtSummaryModified("1") + + s.ssMap.AddStatement(stmtExecInfo1) + datums = s.ssMap.ToDatum() + c.Assert(len(datums), Equals, 1) +}