Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add(typedef,generators) schema tests add #381

Merged
merged 3 commits into from
Jul 7, 2023
Merged

add(typedef,generators) schema tests add #381

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bf6e13c
add(typedef) add marshal and unmarshal test for full cases schema
illia-li Jul 7, 2023
ae1d81c
fix(generotrs): update schema generation to use standalone seed
Jul 7, 2023
e64e8e7
add(generators) add test for GenSchema with different seeds
illia-li Jul 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 1 addition & 15 deletions cmd/gemini/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ import (
"github.com/scylladb/gemini/pkg/jobs"
"github.com/scylladb/gemini/pkg/replication"
"github.com/scylladb/gemini/pkg/store"
"github.com/scylladb/gemini/pkg/tableopts"
"github.com/scylladb/gemini/pkg/typedef"
"github.com/scylladb/gemini/pkg/utils"

Expand Down Expand Up @@ -198,7 +197,7 @@ func run(_ *cobra.Command, _ []string) error {
return errors.Wrap(err, "cannot create schema")
}
} else {
schema = generators.GenSchema(schemaConfig)
schema = generators.GenSchema(schemaConfig, seed)
}

jsonSchema, _ := json.MarshalIndent(schema, "", " ")
Expand Down Expand Up @@ -392,19 +391,6 @@ func createClusters(
return testCluster, oracleCluster
}

func createTableOptions(tableOptionStrings []string, logger *zap.Logger) []tableopts.Option {
var tableOptions []tableopts.Option
for _, optionString := range tableOptionStrings {
o, err := tableopts.FromCQL(optionString)
if err != nil {
logger.Warn("invalid table option", zap.String("option", optionString), zap.Error(err))
continue
}
tableOptions = append(tableOptions, o)
}
return tableOptions
}

func getReplicationStrategy(rs string, fallback *replication.Replication, logger *zap.Logger) *replication.Replication {
switch rs {
case "network":
Expand Down
3 changes: 2 additions & 1 deletion cmd/gemini/schema.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"strings"

"github.com/scylladb/gemini/pkg/replication"
"github.com/scylladb/gemini/pkg/tableopts"
"github.com/scylladb/gemini/pkg/typedef"

"go.uber.org/zap"
Expand Down Expand Up @@ -67,7 +68,7 @@ func createDefaultSchemaConfig(logger *zap.Logger) typedef.SchemaConfig {
return typedef.SchemaConfig{
ReplicationStrategy: rs,
OracleReplicationStrategy: ors,
TableOptions: createTableOptions(tableOptions, logger),
TableOptions: tableopts.CreateTableOptions(tableOptions, logger),
MaxTables: maxTables,
MaxPartitionKeys: maxPartitionKeys,
MinPartitionKeys: minPartitionKeys,
Expand Down
3 changes: 2 additions & 1 deletion cmd/gemini/strategies_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ func TestGetReplicationStrategy(t *testing.T) {
}
}

func TestReadSchema(t *testing.T) {
// TestReadExampleSchema main task of this test to be sure that schema example (schema.json) is correct and have correct marshal, unmarshal
func TestReadExampleSchema(t *testing.T) {
filePath := "schema.json"

testSchema, err := readSchema(filePath, typedef.SchemaConfig{})
Expand Down
68 changes: 34 additions & 34 deletions pkg/generators/column_generator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,51 +26,51 @@ func GenColumnName(prefix string, idx int) string {
return fmt.Sprintf("%s%d", prefix, idx)
}

func GenColumnType(numColumns int, sc *typedef.SchemaConfig) typedef.Type {
n := rand.Intn(numColumns + 5)
func GenColumnType(numColumns int, sc *typedef.SchemaConfig, r *rand.Rand) typedef.Type {
n := r.Intn(numColumns + 5)
switch n {
case numColumns:
return GenTupleType(sc)
return GenTupleType(sc, r)
case numColumns + 1:
return GenUDTType(sc)
return GenUDTType(sc, r)
case numColumns + 2:
return GenSetType(sc)
return GenSetType(sc, r)
case numColumns + 3:
return GenListType(sc)
return GenListType(sc, r)
case numColumns + 4:
return GenMapType(sc)
return GenMapType(sc, r)
default:
return GenSimpleType(sc)
return GenSimpleType(sc, r)
}
}

func GenSimpleType(_ *typedef.SchemaConfig) typedef.SimpleType {
return typedef.AllTypes[rand.Intn(len(typedef.AllTypes))]
func GenSimpleType(_ *typedef.SchemaConfig, r *rand.Rand) typedef.SimpleType {
return typedef.AllTypes[r.Intn(len(typedef.AllTypes))]
}

func GenTupleType(sc *typedef.SchemaConfig) typedef.Type {
n := rand.Intn(sc.MaxTupleParts)
func GenTupleType(sc *typedef.SchemaConfig, r *rand.Rand) typedef.Type {
n := r.Intn(sc.MaxTupleParts)
if n < 2 {
n = 2
}
typeList := make([]typedef.SimpleType, n)
for i := 0; i < n; i++ {
typeList[i] = GenSimpleType(sc)
typeList[i] = GenSimpleType(sc, r)
}
return &typedef.TupleType{
ComplexType: typedef.TYPE_TUPLE,
ValueTypes: typeList,
Frozen: rand.Uint32()%2 == 0,
Frozen: r.Uint32()%2 == 0,
}
}

func GenUDTType(sc *typedef.SchemaConfig) *typedef.UDTType {
udtNum := rand.Uint32()
func GenUDTType(sc *typedef.SchemaConfig, r *rand.Rand) *typedef.UDTType {
udtNum := r.Uint32()
typeName := fmt.Sprintf("udt_%d", udtNum)
ts := make(map[string]typedef.SimpleType)

for i := 0; i < rand.Intn(sc.MaxUDTParts)+1; i++ {
ts[typeName+fmt.Sprintf("_%d", i)] = GenSimpleType(sc)
for i := 0; i < r.Intn(sc.MaxUDTParts)+1; i++ {
ts[typeName+fmt.Sprintf("_%d", i)] = GenSimpleType(sc, r)
}

return &typedef.UDTType{
Expand All @@ -81,51 +81,51 @@ func GenUDTType(sc *typedef.SchemaConfig) *typedef.UDTType {
}
}

func GenSetType(sc *typedef.SchemaConfig) *typedef.BagType {
return genBagType(typedef.TYPE_SET, sc)
func GenSetType(sc *typedef.SchemaConfig, r *rand.Rand) *typedef.BagType {
return genBagType(typedef.TYPE_SET, sc, r)
}

func GenListType(sc *typedef.SchemaConfig) *typedef.BagType {
return genBagType(typedef.TYPE_LIST, sc)
func GenListType(sc *typedef.SchemaConfig, r *rand.Rand) *typedef.BagType {
return genBagType(typedef.TYPE_LIST, sc, r)
}

func genBagType(kind string, sc *typedef.SchemaConfig) *typedef.BagType {
func genBagType(kind string, sc *typedef.SchemaConfig, r *rand.Rand) *typedef.BagType {
var t typedef.SimpleType
for {
t = GenSimpleType(sc)
t = GenSimpleType(sc, r)
if t != typedef.TYPE_DURATION {
break
}
}
return &typedef.BagType{
ComplexType: kind,
ValueType: t,
Frozen: rand.Uint32()%2 == 0,
Frozen: r.Uint32()%2 == 0,
}
}

func GenMapType(sc *typedef.SchemaConfig) *typedef.MapType {
t := GenSimpleType(sc)
func GenMapType(sc *typedef.SchemaConfig, r *rand.Rand) *typedef.MapType {
t := GenSimpleType(sc, r)
for {
if _, ok := typedef.TypesMapKeyBlacklist[t]; !ok {
break
}
t = GenSimpleType(sc)
t = GenSimpleType(sc, r)
}
return &typedef.MapType{
ComplexType: typedef.TYPE_MAP,
KeyType: t,
ValueType: GenSimpleType(sc),
Frozen: rand.Uint32()%2 == 0,
ValueType: GenSimpleType(sc, r),
Frozen: r.Uint32()%2 == 0,
}
}

func GenPartitionKeyColumnType() typedef.Type {
return typedef.PartitionKeyTypes[rand.Intn(len(typedef.PartitionKeyTypes))]
func GenPartitionKeyColumnType(r *rand.Rand) typedef.Type {
return typedef.PartitionKeyTypes[r.Intn(len(typedef.PartitionKeyTypes))]
}

func GenPrimaryKeyColumnType() typedef.Type {
return typedef.PkTypes[rand.Intn(len(typedef.PkTypes))]
func GenPrimaryKeyColumnType(r *rand.Rand) typedef.Type {
return typedef.PkTypes[r.Intn(len(typedef.PkTypes))]
}

func GenIndexName(prefix string, idx int) string {
Expand Down
33 changes: 18 additions & 15 deletions pkg/generators/statement_generator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,15 @@ import (
"fmt"
"strings"

"golang.org/x/exp/rand"

"github.com/scylladb/gemini/pkg/builders"
"github.com/scylladb/gemini/pkg/typedef"
"github.com/scylladb/gemini/pkg/utils"
)

func GenSchema(sc typedef.SchemaConfig) *typedef.Schema {
func GenSchema(sc typedef.SchemaConfig, seed uint64) *typedef.Schema {
r := rand.New(rand.NewSource(seed))
builder := builders.NewSchemaBuilder()
builder.Config(sc)
keyspace := typedef.Keyspace{
Expand All @@ -32,22 +35,22 @@ func GenSchema(sc typedef.SchemaConfig) *typedef.Schema {
OracleReplication: sc.OracleReplicationStrategy,
}
builder.Keyspace(keyspace)
numTables := utils.RandInt(1, sc.GetMaxTables())
numTables := utils.RandInt2(r, 1, sc.GetMaxTables())
for i := 0; i < numTables; i++ {
table := genTable(sc, fmt.Sprintf("table%d", i+1))
table := genTable(sc, fmt.Sprintf("table%d", i+1), r)
builder.Table(table)
}
return builder.Build()
}

func genTable(sc typedef.SchemaConfig, tableName string) *typedef.Table {
partitionKeys := make(typedef.Columns, utils.RandInt(sc.GetMinPartitionKeys(), sc.GetMaxPartitionKeys()))
func genTable(sc typedef.SchemaConfig, tableName string, r *rand.Rand) *typedef.Table {
partitionKeys := make(typedef.Columns, utils.RandInt2(r, sc.GetMinPartitionKeys(), sc.GetMaxPartitionKeys()))
for i := 0; i < len(partitionKeys); i++ {
partitionKeys[i] = &typedef.ColumnDef{Name: GenColumnName("pk", i), Type: GenPartitionKeyColumnType()}
partitionKeys[i] = &typedef.ColumnDef{Name: GenColumnName("pk", i), Type: GenPartitionKeyColumnType(r)}
}
clusteringKeys := make(typedef.Columns, utils.RandInt(sc.GetMinClusteringKeys(), sc.GetMaxClusteringKeys()))
clusteringKeys := make(typedef.Columns, utils.RandInt2(r, sc.GetMinClusteringKeys(), sc.GetMaxClusteringKeys()))
for i := 0; i < len(clusteringKeys); i++ {
clusteringKeys[i] = &typedef.ColumnDef{Name: GenColumnName("ck", i), Type: GenPrimaryKeyColumnType()}
clusteringKeys[i] = &typedef.ColumnDef{Name: GenColumnName("ck", i), Type: GenPrimaryKeyColumnType(r)}
}
table := typedef.Table{
Name: tableName,
Expand All @@ -71,21 +74,21 @@ func genTable(sc typedef.SchemaConfig, tableName string) *typedef.Table {
}
return &table
}
columns := make(typedef.Columns, utils.RandInt(sc.GetMinColumns(), sc.GetMaxColumns()))
columns := make(typedef.Columns, utils.RandInt2(r, sc.GetMinColumns(), sc.GetMaxColumns()))
for i := 0; i < len(columns); i++ {
columns[i] = &typedef.ColumnDef{Name: GenColumnName("col", i), Type: GenColumnType(len(columns), &sc)}
columns[i] = &typedef.ColumnDef{Name: GenColumnName("col", i), Type: GenColumnType(len(columns), &sc, r)}
}
table.Columns = columns

var indexes []typedef.IndexDef
if sc.CQLFeature > typedef.CQL_FEATURE_BASIC && len(columns) > 0 {
indexes = CreateIndexesForColumn(&table, utils.RandInt(1, len(columns)))
indexes = CreateIndexesForColumn(&table, utils.RandInt2(r, 1, len(columns)))
}
table.Indexes = indexes

var mvs []typedef.MaterializedView
if sc.CQLFeature > typedef.CQL_FEATURE_BASIC && len(clusteringKeys) > 0 {
mvs = CreateMaterializedViews(columns, table.Name, partitionKeys, clusteringKeys)
if sc.CQLFeature > typedef.CQL_FEATURE_BASIC && len(clusteringKeys) > 0 && columns.ValidColumnsForPrimaryKey().Len() != 0 {
mvs = CreateMaterializedViews(columns, table.Name, partitionKeys, clusteringKeys, r)
}

table.MaterializedViews = mvs
Expand Down Expand Up @@ -143,12 +146,12 @@ func GetDropSchema(s *typedef.Schema) []string {
}
}

func CreateMaterializedViews(c typedef.Columns, tableName string, partitionKeys, clusteringKeys typedef.Columns) []typedef.MaterializedView {
func CreateMaterializedViews(c typedef.Columns, tableName string, partitionKeys, clusteringKeys typedef.Columns, r *rand.Rand) []typedef.MaterializedView {
validColumns := c.ValidColumnsForPrimaryKey()
var mvs []typedef.MaterializedView
numMvs := 1
for i := 0; i < numMvs; i++ {
col := validColumns.Random()
col := validColumns.Random(r)
if col == nil {
fmt.Printf("unable to generate valid columns for materialized view")
continue
Expand Down
77 changes: 77 additions & 0 deletions pkg/generators/statement_generator_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,16 @@
package generators_test

import (
"encoding/json"
"testing"
"time"

"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"golang.org/x/exp/rand"

"github.com/scylladb/gemini/pkg/generators"
"github.com/scylladb/gemini/pkg/replication"
"github.com/scylladb/gemini/pkg/tableopts"
"github.com/scylladb/gemini/pkg/typedef"
)
Expand Down Expand Up @@ -180,6 +185,49 @@ func TestGetCreateSchema(t *testing.T) {
}
}

func TestGenSchema(t *testing.T) {
seeds := [10]uint64{
uint64(10 + rand.Intn(10)),
uint64(100 + rand.Intn(100)),
uint64(1000 + rand.Intn(1000)),
uint64(10000 + rand.Intn(10000)),
uint64(100000 + rand.Intn(100000)),
uint64(1000000 + rand.Intn(1000000)),
uint64(10000000 + rand.Intn(10000000)),
uint64(100000000 + rand.Intn(100000000)),
uint64(1000000000 + rand.Intn(1000000000)),
uint64(time.Now().Nanosecond()),
}

for idx := range seeds {
testSchema := generators.GenSchema(testSchemaConfig, seeds[idx])
testSchema.Config = typedef.SchemaConfig{}
transformAndDiff(t, testSchema)
}
}

func transformAndDiff(t *testing.T, testSchema *typedef.Schema) {
t.Helper()
opts := cmp.Options{
cmp.AllowUnexported(typedef.Table{}, typedef.MaterializedView{}),
cmpopts.IgnoreUnexported(typedef.Table{}, typedef.MaterializedView{}),
cmpopts.EquateEmpty(),
}

testSchemaMarshaled, err := json.MarshalIndent(testSchema, " ", " ")
if err != nil {
t.Fatalf("unable to marshal schema example json, error=%s\n", err)
}
testSchemaTransformed := typedef.Schema{}
if err = json.Unmarshal(testSchemaMarshaled, &testSchemaTransformed); err != nil {
t.Fatalf("unable to unmarshal json, error=%s\n", err)
}

if diff := cmp.Diff(*testSchema, testSchemaTransformed, opts); diff != "" {
t.Fatalf("schema not the same after marshal/unmarshal, diff=%s", diff)
}
}

func createColumns(cnt int, prefix string) typedef.Columns {
var cols typedef.Columns
for i := 0; i < cnt; i++ {
Expand All @@ -190,3 +238,32 @@ func createColumns(cnt int, prefix string) typedef.Columns {
}
return cols
}

var testSchemaConfig = typedef.SchemaConfig{
ReplicationStrategy: replication.NewSimpleStrategy(),
OracleReplicationStrategy: replication.NewSimpleStrategy(),
TableOptions: tableopts.CreateTableOptions([]string{
"compression = {'sstable_compression':'LZ4Compressor'}",
"read_repair_chance = 1.0", "comment = 'Important records'", "cdc = {'enabled':'true','preimage':'true'}",
"compaction = {'class':'LeveledCompactionStrategy','enabled':true,'sstable_size_in_mb':160,'tombstone_compaction_interval':86400,'tombstone_threshold':0.2}",
},
nil),
MaxTables: 10,
MaxPartitionKeys: 10,
MinPartitionKeys: 1,
MaxClusteringKeys: 10,
MinClusteringKeys: 1,
MaxColumns: 25,
MinColumns: 1,
MaxUDTParts: 20,
MaxTupleParts: 20,
MaxBlobLength: 1e4,
MaxStringLength: 1000,
MinBlobLength: 0,
MinStringLength: 0,
UseCounters: false,
UseLWT: false,
CQLFeature: typedef.CQL_FEATURE_NORMAL,
AsyncObjectStabilizationAttempts: 10,
AsyncObjectStabilizationDelay: 100000,
}
Loading