Skip to content

Commit

Permalink
Join estimation algos (#2212)
Browse files Browse the repository at this point in the history 
* starter for join estimation

* start for join estimation

* progress for bucket merge

* more progress for tests

* more fixes

* more bug fix progress

* edits

* fix test scoping, more bugs

* more tests and fixes

* [ga-format-pr] Run ./format_repo.sh to fix formatting

* more test cleanup

* better debug

* more CI debug

* back to original seed

* better comments

* [ga-format-pr] Run ./format_repo.sh to fix formatting

* float helpers

* distributions

* [ga-format-pr] Run ./format_repo.sh to fix formatting

* fix distribution table names

* fmt

* nick comments

* more nick comments

* todo for stat lower bound

---------

Co-authored-by: max-hoffman <[email protected]>
  • Loading branch information
@max-hoffman
max-hoffman and max-hoffman authored Jan 17, 2024
1 parent e7e445f commit 6800629
Show file tree
Hide file tree
Showing 38 changed files with 2,187 additions and 120 deletions.
2 changes: 1 addition & 1 deletion engine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -811,7 +811,7 @@ func (e *Engine) executeEvent(ctx *sql.Context, dbName, createEventStatement, us

// Drain the iterate to execute the event body/definition
// NOTE: No row data is returned for an event; we just need to execute the statements
_, err = sql.RowIterToRows(ctx, definitionNode.Schema(), iter)
_, err = sql.RowIterToRows(ctx, iter)
return err
}

Expand Down
35 changes: 21 additions & 14 deletions enginetest/engine_only_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,18 +133,18 @@ func TestLocks(t *testing.T) {

ctx := enginetest.NewContext(harness)
ctx.SetCurrentDatabase("db")
sch, iter, err := engine.Query(ctx, "LOCK TABLES t1 READ, t2 WRITE, t3 READ")
_, iter, err := engine.Query(ctx, "LOCK TABLES t1 READ, t2 WRITE, t3 READ")
require.NoError(err)

_, err = sql.RowIterToRows(ctx, sch, iter)
_, err = sql.RowIterToRows(ctx, iter)
require.NoError(err)

ctx = enginetest.NewContext(harness)
ctx.SetCurrentDatabase("db")
sch, iter, err = engine.Query(ctx, "UNLOCK TABLES")
_, iter, err = engine.Query(ctx, "UNLOCK TABLES")
require.NoError(err)

_, err = sql.RowIterToRows(ctx, sch, iter)
_, err = sql.RowIterToRows(ctx, iter)
require.NoError(err)

require.Equal(1, t1.readLocks)
Expand Down Expand Up @@ -184,10 +184,10 @@ func TestRootSpanFinish(t *testing.T) {
sql.WithRootSpan(fakeSpan)(sqlCtx)
sqlCtx = sqlCtx.WithContext(ctx)

sch, iter, err := e.Query(sqlCtx, "SELECT 1")
_, iter, err := e.Query(sqlCtx, "SELECT 1")
require.NoError(t, err)

_, err = sql.RowIterToRows(sqlCtx, sch, iter)
_, err = sql.RowIterToRows(sqlCtx, iter)
require.NoError(t, err)

require.True(t, fakeSpan.finished)
Expand Down Expand Up @@ -274,7 +274,7 @@ func TestShowProcessList(t *testing.T) {

iter, err := rowexec.DefaultBuilder.Build(ctx, n, nil)
require.NoError(err)
rows, err := sql.RowIterToRows(ctx, n.Schema(), iter)
rows, err := sql.RowIterToRows(ctx, iter)
require.NoError(err)

expected := []sql.Row{
Expand Down Expand Up @@ -351,7 +351,7 @@ func TestTrackProcess(t *testing.T) {

iter, err := rowexec.DefaultBuilder.Build(ctx, proc, nil)
require.NoError(err)
_, err = sql.RowIterToRows(ctx, nil, iter)
_, err = sql.RowIterToRows(ctx, iter)
require.NoError(err)

procs := ctx.ProcessList.Processes()
Expand Down Expand Up @@ -654,7 +654,14 @@ func TestTableFunctions(t *testing.T) {
harness.Setup(setup.MydbData)

databaseProvider := harness.NewDatabaseProvider()
testDatabaseProvider := NewTestProvider(&databaseProvider, SimpleTableFunction{}, memory.IntSequenceTable{}, memory.PointLookupTable{}, memory.TableFunc{})
testDatabaseProvider := NewTestProvider(
&databaseProvider,
SimpleTableFunction{},
memory.IntSequenceTable{},
memory.PointLookupTable{},
memory.TableFunc{},
memory.ExponentialDistTable{},
memory.NormalDistTable{})

engine := enginetest.NewEngineWithProvider(t, harness, testDatabaseProvider)
engine.EngineAnalyzer().ExecBuilder = rowexec.DefaultBuilder
Expand Down Expand Up @@ -747,17 +754,17 @@ func TestCollationCoercion(t *testing.T) {
collationQuery := fmt.Sprintf(`SELECT COLLATION(%s) FROM temp_tbl LIMIT 1;`, test.Parameters)
for i, query := range []string{coercibilityQuery, collationQuery} {
t.Run(query, func(t *testing.T) {
sch, iter, err := engine.Query(ctx, query)
_, iter, err := engine.Query(ctx, query)
if test.Error {
if err == nil {
_, err := sql.RowIterToRows(ctx, sch, iter)
_, err := sql.RowIterToRows(ctx, iter)
require.Error(t, err)
} else {
require.Error(t, err)
}
} else {
require.NoError(t, err)
rows, err := sql.RowIterToRows(ctx, sch, iter)
rows, err := sql.RowIterToRows(ctx, iter)
require.NoError(t, err)
require.Equal(t, 1, len(rows))
require.Equal(t, 1, len(rows[0]))
Expand Down Expand Up @@ -793,9 +800,9 @@ func TestRegex(t *testing.T) {
enginetest.TestQueryWithContext(t, ctx, engine, harness, tt.Query, tt.Expected, nil, nil)
} else {
newCtx := ctx.WithQuery(tt.Query)
sch, iter, err := engine.Query(newCtx, tt.Query)
_, iter, err := engine.Query(newCtx, tt.Query)
if err == nil {
_, err = sql.RowIterToRows(newCtx, sch, iter)
_, err = sql.RowIterToRows(newCtx, iter)
require.Error(t, err)
}
}
Expand Down
58 changes: 29 additions & 29 deletions enginetest/enginetests.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1547,9 +1547,9 @@ func TestUserPrivileges(t *testing.T, harness ClientHarness) {
})
} else if script.ExpectingErr {
t.Run(lastQuery, func(t *testing.T) {
sch, iter, err := engine.Query(ctx, lastQuery)
_, iter, err := engine.Query(ctx, lastQuery)
if err == nil {
_, err = sql.RowIterToRows(ctx, sch, iter)
_, err = sql.RowIterToRows(ctx, iter)
}
require.Error(t, err)
for _, errKind := range []*errors.Kind{
Expand All @@ -1567,7 +1567,7 @@ func TestUserPrivileges(t *testing.T, harness ClientHarness) {
t.Run(lastQuery, func(t *testing.T) {
sch, iter, err := engine.Query(ctx, lastQuery)
require.NoError(t, err)
rows, err := sql.RowIterToRows(ctx, sch, iter)
rows, err := sql.RowIterToRows(ctx, iter)
require.NoError(t, err)
// See the comment on QuickPrivilegeTest for a more in-depth explanation, but essentially we treat
// nil in script.Expected as matching "any" non-error result.
Expand Down Expand Up @@ -2133,33 +2133,33 @@ func initializeViewsForVersionedViewsTests(t *testing.T, harness VersionedDBHarn
require := require.New(t)

ctx := NewContext(harness)
sch, iter, err := e.Query(ctx, "CREATE VIEW myview1 AS SELECT * FROM myhistorytable")
_, iter, err := e.Query(ctx, "CREATE VIEW myview1 AS SELECT * FROM myhistorytable")
require.NoError(err)
_, err = sql.RowIterToRows(ctx, sch, iter)
_, err = sql.RowIterToRows(ctx, iter)
require.NoError(err)

// nested views
sch, iter, err = e.Query(ctx, "CREATE VIEW myview2 AS SELECT * FROM myview1 WHERE i = 1")
_, iter, err = e.Query(ctx, "CREATE VIEW myview2 AS SELECT * FROM myview1 WHERE i = 1")
require.NoError(err)
_, err = sql.RowIterToRows(ctx, sch, iter)
_, err = sql.RowIterToRows(ctx, iter)
require.NoError(err)

// views with unions
sch, iter, err = e.Query(ctx, "CREATE VIEW myview3 AS SELECT i from myview1 union select s from myhistorytable")
_, iter, err = e.Query(ctx, "CREATE VIEW myview3 AS SELECT i from myview1 union select s from myhistorytable")
require.NoError(err)
_, err = sql.RowIterToRows(ctx, sch, iter)
_, err = sql.RowIterToRows(ctx, iter)
require.NoError(err)

// views with subqueries
sch, iter, err = e.Query(ctx, "CREATE VIEW myview4 AS SELECT * FROM myhistorytable where i in (select distinct cast(RIGHT(s, 1) as signed) from myhistorytable)")
_, iter, err = e.Query(ctx, "CREATE VIEW myview4 AS SELECT * FROM myhistorytable where i in (select distinct cast(RIGHT(s, 1) as signed) from myhistorytable)")
require.NoError(err)
_, err = sql.RowIterToRows(ctx, sch, iter)
_, err = sql.RowIterToRows(ctx, iter)
require.NoError(err)

// views with a subquery alias
sch, iter, err = e.Query(ctx, "CREATE VIEW myview5 AS SELECT * FROM (select * from myhistorytable where i in (select distinct cast(RIGHT(s, 1) as signed))) as sq")
_, iter, err = e.Query(ctx, "CREATE VIEW myview5 AS SELECT * FROM (select * from myhistorytable where i in (select distinct cast(RIGHT(s, 1) as signed))) as sq")
require.NoError(err)
_, err = sql.RowIterToRows(ctx, sch, iter)
_, err = sql.RowIterToRows(ctx, iter)
require.NoError(err)
}

Expand Down Expand Up @@ -3842,30 +3842,30 @@ func TestClearWarnings(t *testing.T, harness Harness) {
require.NoError(err)

// this query will cause 3 warnings.
sch, iter, err := e.Query(ctx, "drop table if exists table1, table2, table3;")
_, iter, err := e.Query(ctx, "drop table if exists table1, table2, table3;")
require.NoError(err)
err = iter.Close(ctx)
require.NoError(err)

sch, iter, err = e.Query(ctx, "SHOW WARNINGS")
_, iter, err = e.Query(ctx, "SHOW WARNINGS")
require.NoError(err)
rows, err := sql.RowIterToRows(ctx, sch, iter)
rows, err := sql.RowIterToRows(ctx, iter)
require.NoError(err)
err = iter.Close(ctx)
require.NoError(err)
require.Equal(3, len(rows))

sch, iter, err = e.Query(ctx, "SHOW WARNINGS LIMIT 1")
_, iter, err = e.Query(ctx, "SHOW WARNINGS LIMIT 1")
require.NoError(err)
rows, err = sql.RowIterToRows(ctx, sch, iter)
rows, err = sql.RowIterToRows(ctx, iter)
require.NoError(err)
err = iter.Close(ctx)
require.NoError(err)
require.Equal(1, len(rows))

sch, iter, err = e.Query(ctx, "SELECT * FROM mytable LIMIT 1")
_, iter, err = e.Query(ctx, "SELECT * FROM mytable LIMIT 1")
require.NoError(err)
_, err = sql.RowIterToRows(ctx, sch, iter)
_, err = sql.RowIterToRows(ctx, iter)
require.NoError(err)
err = iter.Close(ctx)
require.NoError(err)
Expand Down Expand Up @@ -3953,19 +3953,19 @@ func TestConcurrentTransactions(t *testing.T, harness Harness) {
// We want to add the query to the process list to represent the full workflow.
clientSessionA, err = pl.BeginQuery(clientSessionA, "INSERT INTO a VALUES (1,1)")
require.NoError(err)
sch, iter, err := e.Query(clientSessionA, "INSERT INTO a VALUES (1,1)")
_, iter, err := e.Query(clientSessionA, "INSERT INTO a VALUES (1,1)")
require.NoError(err)

clientSessionB, err = pl.BeginQuery(clientSessionB, "INSERT INTO a VALUES (2,2)")
require.NoError(err)
sch2, iter2, err := e.Query(clientSessionB, "INSERT INTO a VALUES (2,2)")
_, iter2, err := e.Query(clientSessionB, "INSERT INTO a VALUES (2,2)")
require.NoError(err)

rows, err := sql.RowIterToRows(clientSessionA, sch, iter)
rows, err := sql.RowIterToRows(clientSessionA, iter)
require.NoError(err)
require.Len(rows, 1)

rows, err = sql.RowIterToRows(clientSessionB, sch2, iter2)
rows, err = sql.RowIterToRows(clientSessionB, iter2)
require.NoError(err)
require.Len(rows, 1)
}
Expand Down Expand Up @@ -4130,14 +4130,14 @@ func TestTracing(t *testing.T, harness Harness) {

sql.WithTracer(tracer)(ctx)

sch, iter, err := e.Query(ctx, `SELECT DISTINCT i
_, iter, err := e.Query(ctx, `SELECT DISTINCT i
FROM mytable
WHERE s = 'first row'
ORDER BY i DESC
LIMIT 1`)
require.NoError(t, err)

rows, err := sql.RowIterToRows(ctx, sch, iter)
rows, err := sql.RowIterToRows(ctx, iter)
require.Len(t, rows, 1)
require.NoError(t, err)

Expand Down Expand Up @@ -4801,10 +4801,10 @@ func TestCharsetCollationEngine(t *testing.T, harness Harness) {

for _, query := range script.Queries {
t.Run(query.Query, func(t *testing.T) {
sch, iter, err := engine.Query(ctx, query.Query)
_, iter, err := engine.Query(ctx, query.Query)
if query.Error || query.ErrKind != nil {
if err == nil {
_, err := sql.RowIterToRows(ctx, sch, iter)
_, err := sql.RowIterToRows(ctx, iter)
require.Error(t, err)
if query.ErrKind != nil {
require.True(t, query.ErrKind.Is(err))
Expand All @@ -4817,7 +4817,7 @@ func TestCharsetCollationEngine(t *testing.T, harness Harness) {
}
} else {
require.NoError(t, err)
rows, err := sql.RowIterToRows(ctx, sch, iter)
rows, err := sql.RowIterToRows(ctx, iter)
require.NoError(t, err)
require.Equal(t, query.Expected, rows)
}
Expand Down
20 changes: 10 additions & 10 deletions enginetest/evaluation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,9 @@ func RunQueryWithContext(t *testing.T, e QueryEngine, harness Harness, ctx *sql.
ctx = NewContext(harness)
}
ctx = ctx.WithQuery(query)
sch, iter, err := e.Query(ctx, query)
_, iter, err := e.Query(ctx, query)
require.NoError(t, err, "error running query %s: %v", query, err)
_, err = sql.RowIterToRows(ctx, sch, iter)
_, err = sql.RowIterToRows(ctx, iter)
require.NoError(t, err)
validateEngine(t, ctx, harness, e)
}
Expand Down Expand Up @@ -341,7 +341,7 @@ func TestQueryWithContext(t *testing.T, ctx *sql.Context, e QueryEngine, harness
sch, iter, err := e.QueryWithBindings(ctx, q, nil, bindings)
require.NoError(err, "Unexpected error for query %s: %s", q, err)

rows, err := sql.RowIterToRows(ctx, sch, iter)
rows, err := sql.RowIterToRows(ctx, iter)
require.NoError(err, "Unexpected error for query %s: %s", q, err)

if expected != nil {
Expand Down Expand Up @@ -384,7 +384,7 @@ func TestQueryWithIndexCheck(t *testing.T, ctx *sql.Context, e QueryEngine, harn
sch, iter, err := e.QueryWithBindings(ctx, q, nil, bindings)
require.NoError(err, "Unexpected error for query %s: %s", q, err)

rows, err := sql.RowIterToRows(ctx, sch, iter)
rows, err := sql.RowIterToRows(ctx, iter)
require.NoError(err, "Unexpected error for query %s: %s", q, err)

if expected != nil {
Expand Down Expand Up @@ -590,7 +590,7 @@ func runQueryPreparedWithCtx(t *testing.T, ctx *sql.Context, e QueryEngine, q st
return nil, nil, err
}

rows, err := sql.RowIterToRows(ctx, sch, iter)
rows, err := sql.RowIterToRows(ctx, iter)
return rows, sch, err
}

Expand Down Expand Up @@ -879,9 +879,9 @@ func AssertErr(t *testing.T, e QueryEngine, harness Harness, query string, expec
// type of error.
func AssertErrWithBindings(t *testing.T, e QueryEngine, harness Harness, query string, bindings map[string]*querypb.BindVariable, expectedErrKind *errors.Kind, errStrs ...string) {
ctx := NewContext(harness)
sch, iter, err := e.QueryWithBindings(ctx, query, nil, bindings)
_, iter, err := e.QueryWithBindings(ctx, query, nil, bindings)
if err == nil {
_, err = sql.RowIterToRows(ctx, sch, iter)
_, err = sql.RowIterToRows(ctx, iter)
}
require.Error(t, err)
if expectedErrKind != nil {
Expand All @@ -897,9 +897,9 @@ func AssertErrWithBindings(t *testing.T, e QueryEngine, harness Harness, query s
// AssertErrWithCtx is the same as AssertErr, but uses the context given instead of creating one from a harness
func AssertErrWithCtx(t *testing.T, e QueryEngine, harness Harness, ctx *sql.Context, query string, expectedErrKind *errors.Kind, errStrs ...string) {
ctx = ctx.WithQuery(query)
sch, iter, err := e.Query(ctx, query)
_, iter, err := e.Query(ctx, query)
if err == nil {
_, err = sql.RowIterToRows(ctx, sch, iter)
_, err = sql.RowIterToRows(ctx, iter)
}
require.Error(t, err)
if expectedErrKind != nil {
Expand Down Expand Up @@ -963,7 +963,7 @@ func AssertWarningAndTestQuery(
sch, iter, err := e.Query(ctx, query)
require.NoError(err, "Unexpected error for query %s", query)

rows, err := sql.RowIterToRows(ctx, sch, iter)
rows, err := sql.RowIterToRows(ctx, iter)
require.NoError(err, "Unexpected error for query %s", query)

if !IsServerEngine(e) {
Expand Down
Loading

0 comments on commit 6800629

Please sign in to comment.