fix: Tool: Add param to run in TSDB mode without TSDB

migration/cli.go

@@ -27,8 +27,9 @@ func Run(ctx context.Context, p *provider.Provider, outputPath string) error {
 	prefixParam := flag.String("prefix", defaultPrefix, "Prefix for files")
 	doFullParam := flag.Bool("full", false, "Generate initial migrations (prefix will be 'init')")
 	dialectParam := flag.String("dialect", "", "Dialect to generate initial migrations (empty: all)")
-	dsnParam := flag.String("dsn", os.Getenv("CQ_DSN"), "DSN to compare changes against in upgrade mode")
-	schemaName := flag.String("schema", "public", "Schema to compare tables from in upgrade mode")
+	dsnParam := flag.String("dsn", os.Getenv("CQ_DSN"), "DSN to compare changes against (upgrade mode)")
+	schemaName := flag.String("schema", "public", "Schema to compare tables from (upgrade mode)")
+	fakeTSDB := flag.Bool("fake-tsdb", false, "Run in TSDB mode without TSDB (upgrade mode)")
 	flag.Parse()
 	if flag.NArg() > 0 {
 		flag.Usage()
@@ -70,11 +71,16 @@ func Run(ctx context.Context, p *provider.Provider, outputPath string) error {
 	}
 	defer conn.Release()
 
-	if *dialectType == schema.TSDB && *schemaName == "public" {
+	if *fakeTSDB {
+		dt := schema.TSDB
+		dialectType = &dt
+	}
+
+	if *dialectType == schema.TSDB && *schemaName == "public" && !*fakeTSDB {
 		*schemaName = "history"
 	}
 
-	if err := GenerateDiff(ctx, hclog.L(), conn, *schemaName, *dialectType, p, *outputPathParam, *prefixParam); err != nil {
+	if err := GenerateDiff(ctx, hclog.L(), conn, *schemaName, *dialectType, p, *outputPathParam, *prefixParam, *fakeTSDB); err != nil {
 		return fmt.Errorf("failed to generate migrations: %w", err)
 	}
 

migration/migration.go

@@ -30,13 +30,13 @@ func GenerateFull(ctx context.Context, logger hclog.Logger, p *provider.Provider
 }
 
 // GenerateDiff creates incremental table migrations for the provider based on it's ResourceMap. Entities are compared to a given conn.
-func GenerateDiff(ctx context.Context, logger hclog.Logger, conn *pgxpool.Conn, schemaName string, dialectType schema.DialectType, p *provider.Provider, outputPath, prefix string) error {
+func GenerateDiff(ctx context.Context, logger hclog.Logger, conn *pgxpool.Conn, schemaName string, dialectType schema.DialectType, p *provider.Provider, outputPath, prefix string, fakeTSDB bool) error {
 	dialect, err := schema.GetDialect(dialectType)
 	if err != nil {
 		return err
 	}
 
-	err = generateDiffForDialect(ctx, logger, afero.Afero{Fs: afero.OsFs{}}, conn, schemaName, dialect, p, filepath.Join(outputPath, dialectType.MigrationDirectory()), prefix)
+	err = generateDiffForDialect(ctx, logger, afero.Afero{Fs: afero.OsFs{}}, conn, schemaName, dialect, p, filepath.Join(outputPath, dialectType.MigrationDirectory()), prefix, fakeTSDB)
 	if err == errNoChange {
 		return nil
 	}
@@ -118,7 +118,7 @@ func generateFullForDialect(ctx context.Context, logger hclog.Logger, dialect sc
 
 var errNoChange = fmt.Errorf("no change")
 
-func generateDiffForDialect(ctx context.Context, logger hclog.Logger, fs afero.Afero, conn *pgxpool.Conn, schemaName string, dialect schema.Dialect, p *provider.Provider, outputPath, prefix string) (retErr error) {
+func generateDiffForDialect(ctx context.Context, logger hclog.Logger, fs afero.Afero, conn *pgxpool.Conn, schemaName string, dialect schema.Dialect, p *provider.Provider, outputPath, prefix string, fakeTSDB bool) (retErr error) {
 	cName, dName := filepath.Join(outputPath, prefix+"up.sql"), filepath.Join(outputPath, prefix+"down.sql")
 
 	defer func() {
@@ -170,7 +170,7 @@ func generateDiffForDialect(ctx context.Context, logger hclog.Logger, fs afero.A
 	for _, resName := range resourceKeys(p.ResourceMap) {
 		table := p.ResourceMap[resName]
 
-		ups, downs, err := tc.DiffTable(ctx, conn, schemaName, table, nil)
+		ups, downs, err := tc.DiffTable(ctx, conn, schemaName, table, nil, fakeTSDB)
 		if err != nil {
 			return fmt.Errorf("DiffTable failed for %s: %w", table.Name, err)
 		}

migration/table.go

@@ -27,6 +27,8 @@ const (
 	dropConstraintFromTable = `ALTER TABLE IF EXISTS %s DROP CONSTRAINT %s;`
 
 	dropTable = `DROP TABLE IF EXISTS %s;`
+
+	fakeTSDBAssumeColumn = `cq_fetch_date`
 )
 
 // TableCreator handles creation of schema.Table in database as SQL strings
@@ -97,7 +99,8 @@ func (m TableCreator) CreateTableDefinitions(ctx context.Context, t *schema.Tabl
 // Column renames are detected (best effort) and ALTER TABLE RENAME COLUMN statements are generated as comments.
 // Table renames or removals are not detected.
 // FK changes are not detected.
-func (m TableCreator) DiffTable(ctx context.Context, conn *pgxpool.Conn, schemaName string, t, parent *schema.Table) (up, down []string, err error) {
+// if fakeTSDB is set, PKs for existing resources are assumed to have fakeTSDBAssumeColumn (`cq_fetch_date`) as the first part of composite PK.
+func (m TableCreator) DiffTable(ctx context.Context, conn *pgxpool.Conn, schemaName string, t, parent *schema.Table, fakeTSDB bool) (up, down []string, err error) {
 	rows, err := conn.Query(ctx, queryTableColumns, schemaName, t.Name)
 	if err != nil {
 		return nil, nil, err
@@ -135,7 +138,7 @@ func (m TableCreator) DiffTable(ctx context.Context, conn *pgxpool.Conn, schemaN
 	up, down = make([]string, 0, capSize), make([]string, 0, capSize)
 	downLast := make([]string, 0, capSize)
 
-	cUp, cDown, err := m.diffConstraints(ctx, conn, schemaName, t)
+	cUp, cDown, err := m.diffConstraints(ctx, conn, schemaName, t, fakeTSDB)
 	if err != nil {
 		return nil, nil, fmt.Errorf("diffConstraints failed: %w", err)
 	}
@@ -149,7 +152,11 @@ func (m TableCreator) DiffTable(ctx context.Context, conn *pgxpool.Conn, schemaN
 			m.log.Warn("column missing from table, not adding it", "table", t.Name, "column", d)
 			continue
 		}
+		if fakeTSDB && col.Name == fakeTSDBAssumeColumn {
+			continue
+		}
 		if col.Internal() {
+			m.log.Warn("table missing internal column, not adding it", "table", t.Name, "column", d)
 			continue
 		}
 
@@ -190,7 +197,7 @@ func (m TableCreator) DiffTable(ctx context.Context, conn *pgxpool.Conn, schemaN
 
 	// Do relation tables
 	for _, r := range t.Relations {
-		if cr, dr, err := m.DiffTable(ctx, conn, schemaName, r, t); err != nil {
+		if cr, dr, err := m.DiffTable(ctx, conn, schemaName, r, t, fakeTSDB); err != nil {
 			return nil, nil, err
 		} else {
 			up = append(up, cr...)
@@ -274,7 +281,7 @@ func (c constraintMigrations) Squash() constraintMigration {
 	return ret
 }
 
-func (m TableCreator) diffConstraints(ctx context.Context, conn *pgxpool.Conn, schemaName string, t *schema.Table) (up, down constraintMigrations, err error) {
+func (m TableCreator) diffConstraints(ctx context.Context, conn *pgxpool.Conn, schemaName string, t *schema.Table, fakeTSDB bool) (up, down constraintMigrations, err error) {
 	rows, err := conn.Query(ctx, queryTablePKs, schemaName, t.Name)
 	if err != nil {
 		return nil, nil, err
@@ -312,6 +319,10 @@ func (m TableCreator) diffConstraints(ctx context.Context, conn *pgxpool.Conn, s
 			}, nil
 	}
 
+	if fakeTSDB && existingPKs[0].Columns[0] != fakeTSDBAssumeColumn {
+		existingPKs[0].Columns = append([]string{fakeTSDBAssumeColumn}, existingPKs[0].Columns...)
+	}
+
 	if reflect.DeepEqual(existingPKs[0].Columns, pks) {
 		return nil, nil, nil
 	}

migration/testbuilder.go

@@ -159,7 +159,7 @@ func doMigrationsTest(t *testing.T, ctx context.Context, dsn string, prov *provi
 		dialectType, err := schema.GetDialect(dialect)
 		assert.NoError(t, err)
 
-		if err := generateDiffForDialect(ctx, hclog.NewNullLogger(), fs, conn, "public", dialectType, prov, "/", ""); err != errNoChange {
+		if err := generateDiffForDialect(ctx, hclog.NewNullLogger(), fs, conn, "public", dialectType, prov, "/", "", false); err != errNoChange {
 			assert.NoError(t, err)
 
 			mig, err := fs.ReadFile("/up.sql")