fix(typedef): 'time', 'timestamp', 'date' types generate fixed, added…

… convert type 'time' for JSON *added primary keys values convertation for JSON queries. *redo generation TYPE_TIME,TIME_TIMESTAMP,TYPE_DATE in accordance of scylla
scylladb · Jun 21, 2023 · aa2c7dc · aa2c7dc
1 parent f708d86
commit aa2c7dc
Show file tree

Hide file tree

Showing 20 changed files with 173 additions and 144 deletions.
diff --git a/pkg/jobs/gen_mutate_stmt.go b/pkg/jobs/gen_mutate_stmt.go
@@ -17,10 +17,10 @@ package jobs
 import (
 	"encoding/json"
 	"fmt"
-
-	"golang.org/x/exp/rand"
+	"time"
 
 	"github.com/scylladb/gocqlx/v2/qb"
+	"golang.org/x/exp/rand"
 
 	"github.com/scylladb/gemini/pkg/generators"
 	"github.com/scylladb/gemini/pkg/typedef"
@@ -125,8 +125,6 @@ func genInsertJSONStmt(
 	r *rand.Rand,
 	p *typedef.PartitionRangeConfig,
 ) (*typedef.Stmt, error) {
-	var v string
-	var ok bool
 	if table.IsCounterTable() {
 		return nil, nil
 	}
@@ -135,25 +133,11 @@ func genInsertJSONStmt(
 	for i, pk := range table.PartitionKeys {
 		switch t := pk.Type.(type) {
 		case typedef.SimpleType:
-			if t != typedef.TYPE_BLOB {
-				values[pk.Name] = vs[i]
-				continue
-			}
-			v, ok = vs[i].(string)
-			if ok {
-				values[pk.Name] = "0x" + v
-			}
+			values[pk.Name] = convertForJSON(t, vs[i])
 		case *typedef.TupleType:
 			tupVals := make([]interface{}, len(t.ValueTypes))
 			for j := 0; j < len(t.ValueTypes); j++ {
-				if t.ValueTypes[j] == typedef.TYPE_BLOB {
-					v, ok = vs[i+j].(string)
-					if ok {
-						v = "0x" + v
-					}
-					vs[i+j] = v
-				}
-				tupVals[i] = vs[i+j]
+				tupVals[i] = convertForJSON(t, vs[i])
 				i++
 			}
 			values[pk.Name] = tupVals
@@ -195,3 +179,15 @@ func genDeleteRows(_ *typedef.Schema, t *typedef.Table, valuesWithToken *typedef
 		Values:          values,
 	}, nil
 }
+
+func convertForJSON(vType typedef.Type, value interface{}) interface{} {
+	switch vType {
+	case typedef.TYPE_BLOB:
+		val, _ := value.(string)
+		return "0x" + val
+	case typedef.TYPE_TIME:
+		val, _ := value.(int64)
+		return time.Unix(0, val).UTC().Format("15:04:05.000000000")
+	}
+	return value
+}
diff --git a/pkg/jobs/gen_utils_test.go b/pkg/jobs/gen_utils_test.go
@@ -146,13 +146,20 @@ func getErrorMsgIfDifferent(t *testing.T, expected, received, errMsg string) {
 		return
 	}
 	errMsgList := make([]string, 0)
-	switch len(expected) == len(received) {
+	subString := " "
+	if strings.Count(expected, ",\"") > strings.Count(expected, subString) {
+		subString = ",\""
+	}
+	tmpExpected := strings.Split(expected, subString)
+	tmpReceived := strings.Split(received, subString)
+	switch len(tmpExpected) == len(tmpReceived) {
 	case true:
 		// Inject nice row that highlights differences if length is not changed
+		expected, received = addDiffHighlight(tmpExpected, tmpReceived, subString)
 		errMsgList = []string{
 			errMsg,
 			fmt.Sprintf("Expected   %s", expected),
-			"           " + diffHighlightString(expected, received),
+			diffHighlightString([]rune(expected), []rune(received)),
 			fmt.Sprintf("Received   %s", received),
 			"-------------------------------------------",
 		}
@@ -167,8 +174,8 @@ func getErrorMsgIfDifferent(t *testing.T, expected, received, errMsg string) {
 	t.Error(strings.Join(errMsgList, "\n"))
 }
 
-func diffHighlightString(expected, received string) string {
-	out := ""
+func diffHighlightString(expected, received []rune) string {
+	out := "Difference "
 	for idx := range expected {
 		if expected[idx] == received[idx] {
 			out += " "
@@ -179,6 +186,19 @@ func diffHighlightString(expected, received string) string {
 	return out
 }
 
+func addDiffHighlight(expected, received []string, subString string) (string, string) {
+	for idx := range expected {
+		delta := len(expected[idx]) - len(received[idx])
+		if delta > 0 {
+			received[idx] += strings.Repeat("↔", delta)
+		}
+		if delta < 0 {
+			expected[idx] += strings.Repeat("↔", -delta)
+		}
+	}
+	return strings.Join(expected, subString), strings.Join(received, subString)
+}
+
 func convertStmtsToResults(stmt interface{}) Results {
 	var out Results
 	switch stmts := stmt.(type) {

diff --git a/pkg/jobs/test_expected_data/check/clustering_range.json b/pkg/jobs/test_expected_data/check/clustering_range.json
@@ -5,7 +5,7 @@
       "TokenValues": "",
       "Query": "SELECT * FROM ks1.pk1_ck1_col1_cck1 WHERE pk0=? AND ck0\u003e? AND ck0\u003c?",
       "Names": "[pk0 ck0 ck0]",
-      "Values": "[1 1969-12-31 1969-12-31]",
+      "Values": "[1 1970-01-01 1970-01-01]",
       "Types": " bigint date date",
       "QueryType": "1"
     }
@@ -16,7 +16,7 @@
       "TokenValues": "",
       "Query": "SELECT * FROM ks1.pk1_ck1_col1cr_cckAll WHERE pk0=? AND ck0\u003e? AND ck0\u003c?",
       "Names": "[pk0 ck0 ck0]",
-      "Values": "[1 1969-12-31 1969-12-31]",
+      "Values": "[1 1970-01-01 1970-01-01]",
       "Types": " bigint date date",
       "QueryType": "1"
     }
@@ -27,7 +27,7 @@
       "TokenValues": "",
       "Query": "SELECT * FROM ks1.pk3_ck3_col3cr_cckAll WHERE pk0=? AND pk1=? AND pk2=? AND ck0=? AND ck1=? AND ck2\u003e? AND ck2\u003c?",
       "Names": "[pk0 pk1 pk2 ck0 ck1 ck2 ck2]",
-      "Values": "[1 1.110223e-16 1.1.1.1 01 1969-12-31 0.001 0.001]",
+      "Values": "[1 1.110223e-16 1.1.1.1 01 1970-01-01 0.001 0.001]",
       "Types": " bigint float inet ascii date decimal decimal",
       "QueryType": "1"
     }
@@ -49,7 +49,7 @@
       "TokenValues": "",
       "Query": "SELECT * FROM ks1.pk3_ck3_col5_cckAll WHERE pk0=? AND pk1=? AND pk2=? AND ck0=? AND ck1=? AND ck2\u003e? AND ck2\u003c?",
       "Names": "[pk0 pk1 pk2 ck0 ck1 ck2 ck2]",
-      "Values": "[1 1.110223e-16 1.1.1.1 01 1969-12-31 0.001 0.001]",
+      "Values": "[1 1.110223e-16 1.1.1.1 01 1970-01-01 0.001 0.001]",
       "Types": " bigint float inet ascii date decimal decimal",
       "QueryType": "1"
     }
@@ -60,7 +60,7 @@
       "TokenValues": "",
       "Query": "SELECT * FROM ks1.pkAll_ckAll_colAll_cck1 WHERE pk0=? AND pk1=? AND pk2=? AND pk3=? AND pk4=? AND pk5=? AND pk6=? AND pk7=? AND pk8=? AND pk9=? AND pk10=? AND pk11=? AND pk12=? AND pk13=? AND pk14=? AND pk15=? AND pk16=? AND pk17=? AND pk18=? AND ck0\u003e? AND ck0\u003c?",
       "Names": "[pk0 pk1 pk2 pk3 pk4 pk5 pk6 pk7 pk8 pk9 pk10 pk11 pk12 pk13 pk14 pk15 pk16 pk17 pk18 ck0 ck0]",
-      "Values": "[01 1 3030 false 1969-12-31 0.001 1.1102230246251565e-16 1.110223e-16 1.1.1.1 0 0 00 1969-12-31 00:00:01 +0000 UTC 00000001-0000-1000-8000-3132372e302e 0 00000001-0000-1000-8000-3132372e302e 00 1 -86399000000000 01 00]",
+      "Values": "[01 1 3030 false 1970-01-01 0.001 1.1102230246251565e-16 1.110223e-16 1.1.1.1 0 0 00 1 00000001-0000-1000-8000-3132372e302e 0 00000001-0000-1000-8000-3132372e302e 00 1 1 01 00]",
       "Types": " ascii bigint blob boolean date decimal double float inet int smallint text timestamp timeuuid tinyint uuid varchar varint time ascii ascii",
       "QueryType": "1"
     }
@@ -71,7 +71,7 @@
       "TokenValues": "",
       "Query": "SELECT * FROM ks1.pkAll_ckAll_colAll_cckAll WHERE pk0=? AND pk1=? AND pk2=? AND pk3=? AND pk4=? AND pk5=? AND pk6=? AND pk7=? AND pk8=? AND pk9=? AND pk10=? AND pk11=? AND pk12=? AND pk13=? AND pk14=? AND pk15=? AND pk16=? AND pk17=? AND pk18=? AND ck0=? AND ck1=? AND ck2=? AND ck3=? AND ck4=? AND ck5=? AND ck6=? AND ck7=? AND ck8=? AND ck9=? AND ck10=? AND ck11=? AND ck12=? AND ck13=? AND ck14=? AND ck15=? AND ck16=? AND ck17=? AND ck18\u003e? AND ck18\u003c?",
       "Names": "[pk0 pk1 pk2 pk3 pk4 pk5 pk6 pk7 pk8 pk9 pk10 pk11 pk12 pk13 pk14 pk15 pk16 pk17 pk18 ck0 ck1 ck2 ck3 ck4 ck5 ck6 ck7 ck8 ck9 ck10 ck11 ck12 ck13 ck14 ck15 ck16 ck17 ck18 ck18]",
-      "Values": "[01 1 3030 false 1969-12-31 0.001 1.1102230246251565e-16 1.110223e-16 1.1.1.1 0 0 00 1969-12-31 00:00:01 +0000 UTC 00000001-0000-1000-8000-3132372e302e 0 00000001-0000-1000-8000-3132372e302e 00 1 -86399000000000 01 1 3030 false 1969-12-31 0.001 1.1102230246251565e-16 1.110223e-16 1.1.1.1 0 0 00 1969-12-31 00:00:01 +0000 UTC 00000001-0000-1000-8000-3132372e302e 0 00000001-0000-1000-8000-3132372e302e 00 1 -86399000000000 -86399000000000]",
+      "Values": "[01 1 3030 false 1970-01-01 0.001 1.1102230246251565e-16 1.110223e-16 1.1.1.1 0 0 00 1 00000001-0000-1000-8000-3132372e302e 0 00000001-0000-1000-8000-3132372e302e 00 1 1 01 1 3030 false 1970-01-01 0.001 1.1102230246251565e-16 1.110223e-16 1.1.1.1 0 0 00 1 00000001-0000-1000-8000-3132372e302e 0 00000001-0000-1000-8000-3132372e302e 00 1 1 1]",
       "Types": " ascii bigint blob boolean date decimal double float inet int smallint text timestamp timeuuid tinyint uuid varchar varint time ascii bigint blob boolean date decimal double float inet int smallint text timestamp timeuuid tinyint uuid varchar varint time time",
       "QueryType": "1"
     }

diff --git a/pkg/jobs/test_expected_data/check/clustering_range_mv.json b/pkg/jobs/test_expected_data/check/clustering_range_mv.json
@@ -5,7 +5,7 @@
       "TokenValues": "",
       "Query": "SELECT * FROM ks1.pk1_ck1_col1_cck1.mv_mv_1 WHERE pk0=? AND ck0\u003e? AND ck0\u003c?",
       "Names": "[pk0 ck0 ck0]",
-      "Values": "[1 1969-12-31 1969-12-31]",
+      "Values": "[1 1970-01-01 1970-01-01]",
       "Types": " bigint date date",
       "QueryType": "1"
     }
@@ -16,7 +16,7 @@
       "TokenValues": "",
       "Query": "SELECT * FROM ks1.pk1_ck1_col1_cck1.mvNp_mv_1 WHERE col0=? AND pk0=? AND ck0\u003e? AND ck0\u003c?",
       "Names": "[col0 pk0 ck0 ck0]",
-      "Values": "[1969-12-31 1 1969-12-31 1969-12-31]",
+      "Values": "[1970-01-01 1 1970-01-01 1970-01-01]",
       "Types": " date bigint date date",
       "QueryType": "1"
     }
@@ -27,7 +27,7 @@
       "TokenValues": "",
       "Query": "SELECT * FROM ks1.pk1_ck1_col1cr_cckAll.mv_mv_1 WHERE pk0=? AND ck0\u003e? AND ck0\u003c?",
       "Names": "[pk0 ck0 ck0]",
-      "Values": "[1 1969-12-31 1969-12-31]",
+      "Values": "[1 1970-01-01 1970-01-01]",
       "Types": " bigint date date",
       "QueryType": "1"
     }
@@ -38,7 +38,7 @@
       "TokenValues": "",
       "Query": "SELECT * FROM ks1.pk3_ck3_col3cr_cckAll.mv_mv_1 WHERE pk0=? AND pk1=? AND pk2=? AND ck0=? AND ck1=? AND ck2\u003e? AND ck2\u003c?",
       "Names": "[pk0 pk1 pk2 ck0 ck1 ck2 ck2]",
-      "Values": "[1 1.110223e-16 1.1.1.1 01 1969-12-31 0.001 0.001]",
+      "Values": "[1 1.110223e-16 1.1.1.1 01 1970-01-01 0.001 0.001]",
       "Types": " bigint float inet ascii date decimal decimal",
       "QueryType": "1"
     }
@@ -71,7 +71,7 @@
       "TokenValues": "",
       "Query": "SELECT * FROM ks1.pk3_ck3_col5_cckAll.mv_mv_1 WHERE pk0=? AND pk1=? AND pk2=? AND ck0=? AND ck1=? AND ck2\u003e? AND ck2\u003c?",
       "Names": "[pk0 pk1 pk2 ck0 ck1 ck2 ck2]",
-      "Values": "[1 1.110223e-16 1.1.1.1 01 1969-12-31 0.001 0.001]",
+      "Values": "[1 1.110223e-16 1.1.1.1 01 1970-01-01 0.001 0.001]",
       "Types": " bigint float inet ascii date decimal decimal",
       "QueryType": "1"
     }
@@ -82,7 +82,7 @@
       "TokenValues": "",
       "Query": "SELECT * FROM ks1.pk3_ck3_col5_cckAll.mvNp_mv_1 WHERE col0=? AND pk0=? AND pk1=? AND pk2=? AND ck0=? AND ck1=? AND ck2\u003e? AND ck2\u003c?",
       "Names": "[col0 pk0 pk1 pk2 ck0 ck1 ck2 ck2]",
-      "Values": "[01 1 1.110223e-16 1.1.1.1 00 1969-12-31 0.001 0.001]",
+      "Values": "[01 1 1.110223e-16 1.1.1.1 00 1970-01-01 0.001 0.001]",
       "Types": " ascii bigint float inet ascii date decimal decimal",
       "QueryType": "1"
     }
@@ -93,7 +93,7 @@
       "TokenValues": "",
       "Query": "SELECT * FROM ks1.pkAll_ckAll_colAll_cck1.mv_mv_1 WHERE pk0=? AND pk1=? AND pk2=? AND pk3=? AND pk4=? AND pk5=? AND pk6=? AND pk7=? AND pk8=? AND pk9=? AND pk10=? AND pk11=? AND pk12=? AND pk13=? AND pk14=? AND pk15=? AND pk16=? AND pk17=? AND pk18=? AND ck0\u003e? AND ck0\u003c?",
       "Names": "[pk0 pk1 pk2 pk3 pk4 pk5 pk6 pk7 pk8 pk9 pk10 pk11 pk12 pk13 pk14 pk15 pk16 pk17 pk18 ck0 ck0]",
-      "Values": "[01 1 3030 false 1969-12-31 0.001 1.1102230246251565e-16 1.110223e-16 1.1.1.1 0 0 00 1969-12-31 00:00:01 +0000 UTC 00000001-0000-1000-8000-3132372e302e 0 00000001-0000-1000-8000-3132372e302e 00 1 -86399000000000 01 00]",
+      "Values": "[01 1 3030 false 1970-01-01 0.001 1.1102230246251565e-16 1.110223e-16 1.1.1.1 0 0 00 1 00000001-0000-1000-8000-3132372e302e 0 00000001-0000-1000-8000-3132372e302e 00 1 1 01 00]",
       "Types": " ascii bigint blob boolean date decimal double float inet int smallint text timestamp timeuuid tinyint uuid varchar varint time ascii ascii",
       "QueryType": "1"
     }
@@ -104,7 +104,7 @@
       "TokenValues": "",
       "Query": "SELECT * FROM ks1.pkAll_ckAll_colAll_cck1.mvNp_mv_1 WHERE col1=? AND pk0=? AND pk1=? AND pk2=? AND pk3=? AND pk4=? AND pk5=? AND pk6=? AND pk7=? AND pk8=? AND pk9=? AND pk10=? AND pk11=? AND pk12=? AND pk13=? AND pk14=? AND pk15=? AND pk16=? AND pk17=? AND pk18=? AND ck0\u003e? AND ck0\u003c?",
       "Names": "[col1 pk0 pk1 pk2 pk3 pk4 pk5 pk6 pk7 pk8 pk9 pk10 pk11 pk12 pk13 pk14 pk15 pk16 pk17 pk18 ck0 ck0]",
-      "Values": "[01 01 1 3030 false 1969-12-31 0.001 1.1102230246251565e-16 1.110223e-16 1.1.1.1 0 0 00 1969-12-31 00:00:01 +0000 UTC 00000001-0000-1000-8000-3132372e302e 0 00000001-0000-1000-8000-3132372e302e 00 1 -86399000000000 00 00]",
+      "Values": "[01 01 1 3030 false 1970-01-01 0.001 1.1102230246251565e-16 1.110223e-16 1.1.1.1 0 0 00 1 00000001-0000-1000-8000-3132372e302e 0 00000001-0000-1000-8000-3132372e302e 00 1 1 00 00]",
       "Types": " ascii ascii bigint blob boolean date decimal double float inet int smallint text timestamp timeuuid tinyint uuid varchar varint time ascii ascii",
       "QueryType": "1"
     }
@@ -115,7 +115,7 @@
       "TokenValues": "",
       "Query": "SELECT * FROM ks1.pkAll_ckAll_colAll_cckAll.mv_mv_1 WHERE pk0=? AND pk1=? AND pk2=? AND pk3=? AND pk4=? AND pk5=? AND pk6=? AND pk7=? AND pk8=? AND pk9=? AND pk10=? AND pk11=? AND pk12=? AND pk13=? AND pk14=? AND pk15=? AND pk16=? AND pk17=? AND pk18=? AND ck0=? AND ck1=? AND ck2=? AND ck3=? AND ck4=? AND ck5=? AND ck6=? AND ck7=? AND ck8=? AND ck9=? AND ck10=? AND ck11=? AND ck12=? AND ck13=? AND ck14=? AND ck15=? AND ck16=? AND ck17=? AND ck18\u003e? AND ck18\u003c?",
       "Names": "[pk0 pk1 pk2 pk3 pk4 pk5 pk6 pk7 pk8 pk9 pk10 pk11 pk12 pk13 pk14 pk15 pk16 pk17 pk18 ck0 ck1 ck2 ck3 ck4 ck5 ck6 ck7 ck8 ck9 ck10 ck11 ck12 ck13 ck14 ck15 ck16 ck17 ck18 ck18]",
-      "Values": "[01 1 3030 false 1969-12-31 0.001 1.1102230246251565e-16 1.110223e-16 1.1.1.1 0 0 00 1969-12-31 00:00:01 +0000 UTC 00000001-0000-1000-8000-3132372e302e 0 00000001-0000-1000-8000-3132372e302e 00 1 -86399000000000 01 1 3030 false 1969-12-31 0.001 1.1102230246251565e-16 1.110223e-16 1.1.1.1 0 0 00 1969-12-31 00:00:01 +0000 UTC 00000001-0000-1000-8000-3132372e302e 0 00000001-0000-1000-8000-3132372e302e 00 1 -86399000000000 -86399000000000]",
+      "Values": "[01 1 3030 false 1970-01-01 0.001 1.1102230246251565e-16 1.110223e-16 1.1.1.1 0 0 00 1 00000001-0000-1000-8000-3132372e302e 0 00000001-0000-1000-8000-3132372e302e 00 1 1 01 1 3030 false 1970-01-01 0.001 1.1102230246251565e-16 1.110223e-16 1.1.1.1 0 0 00 1 00000001-0000-1000-8000-3132372e302e 0 00000001-0000-1000-8000-3132372e302e 00 1 1 1]",
       "Types": " ascii bigint blob boolean date decimal double float inet int smallint text timestamp timeuuid tinyint uuid varchar varint time ascii bigint blob boolean date decimal double float inet int smallint text timestamp timeuuid tinyint uuid varchar varint time time",
       "QueryType": "1"
     }
@@ -126,7 +126,7 @@
       "TokenValues": "",
       "Query": "SELECT * FROM ks1.pkAll_ckAll_colAll_cckAll.mvNp_mv_1 WHERE col1=? AND pk0=? AND pk1=? AND pk2=? AND pk3=? AND pk4=? AND pk5=? AND pk6=? AND pk7=? AND pk8=? AND pk9=? AND pk10=? AND pk11=? AND pk12=? AND pk13=? AND pk14=? AND pk15=? AND pk16=? AND pk17=? AND pk18=? AND ck0=? AND ck1=? AND ck2=? AND ck3=? AND ck4=? AND ck5=? AND ck6=? AND ck7=? AND ck8=? AND ck9=? AND ck10=? AND ck11=? AND ck12=? AND ck13=? AND ck14=? AND ck15=? AND ck16=? AND ck17=? AND ck18\u003e? AND ck18\u003c?",
       "Names": "[col1 pk0 pk1 pk2 pk3 pk4 pk5 pk6 pk7 pk8 pk9 pk10 pk11 pk12 pk13 pk14 pk15 pk16 pk17 pk18 ck0 ck1 ck2 ck3 ck4 ck5 ck6 ck7 ck8 ck9 ck10 ck11 ck12 ck13 ck14 ck15 ck16 ck17 ck18 ck18]",
-      "Values": "[01 01 1 3030 false 1969-12-31 0.001 1.1102230246251565e-16 1.110223e-16 1.1.1.1 0 0 00 1969-12-31 00:00:01 +0000 UTC 00000001-0000-1000-8000-3132372e302e 0 00000001-0000-1000-8000-3132372e302e 00 1 -86399000000000 00 1 3030 false 1969-12-31 0.001 1.1102230246251565e-16 1.110223e-16 1.1.1.1 0 0 00 1969-12-31 00:00:01 +0000 UTC 00000001-0000-1000-8000-3132372e302e 0 00000001-0000-1000-8000-3132372e302e 01 1 -86399000000000 -86399000000000]",
+      "Values": "[01 01 1 3030 false 1970-01-01 0.001 1.1102230246251565e-16 1.110223e-16 1.1.1.1 0 0 00 1 00000001-0000-1000-8000-3132372e302e 0 00000001-0000-1000-8000-3132372e302e 00 1 1 00 1 3030 false 1970-01-01 0.001 1.1102230246251565e-16 1.110223e-16 1.1.1.1 0 0 00 1 00000001-0000-1000-8000-3132372e302e 0 00000001-0000-1000-8000-3132372e302e 01 1 1 1]",
       "Types": " ascii ascii bigint blob boolean date decimal double float inet int smallint text timestamp timeuuid tinyint uuid varchar varint time ascii bigint blob boolean date decimal double float inet int smallint text timestamp timeuuid tinyint uuid varchar varint time time",
       "QueryType": "1"
     }