feat: migration from mongo to postgres data storage (#65)

* feat: migration from mongo to postgres data storage Signed-off-by: Lawrence Zawila <[email protected]> * fix: move models & migrations to app root Signed-off-by: Lawrence Zawila <[email protected]> * fix: lint & env var support Signed-off-by: Lawrence Zawila <[email protected]> * fix: payment enums order Signed-off-by: Lawrence Zawila <[email protected]> * fix: uint type assertion Signed-off-by: Lawrence Zawila <[email protected]> * feat: update docker-compose * feat: Update to PG14 * feat: storage mappings and usage flow adjustments Signed-off-by: Lawrence Zawila <[email protected]> * fix: drop type assertion to lower integer bounds Signed-off-by: Lawrence Zawila <[email protected]> * fix: kafka message format Signed-off-by: Lawrence Zawila <[email protected]> * fix: tasks response formatting Signed-off-by: Lawrence Zawila <[email protected]> * fix: missing provider in kafka message Signed-off-by: Lawrence Zawila <[email protected]> * fix: object instead of an array for payments kafka message Signed-off-by: Lawrence Zawila <[email protected]> * feat: drop adjustments from kafka message Signed-off-by: Lawrence Zawila <[email protected]> * feat: drop raw data in kafka message Signed-off-by: Lawrence Zawila <[email protected]> * fix: payment selection by id Signed-off-by: Lawrence Zawila <[email protected]> * feat: migration from mongo to postgres data storage Signed-off-by: Lawrence Zawila <[email protected]> * fix: move models & migrations to app root Signed-off-by: Lawrence Zawila <[email protected]> * fix: lint & env var support Signed-off-by: Lawrence Zawila <[email protected]> * fix: payment enums order Signed-off-by: Lawrence Zawila <[email protected]> * fix: uint type assertion Signed-off-by: Lawrence Zawila <[email protected]> * feat: update docker-compose * feat: Update to PG14 * feat: storage mappings and usage flow adjustments Signed-off-by: Lawrence Zawila <[email protected]> * fix: drop type assertion to lower integer bounds Signed-off-by: Lawrence Zawila <[email protected]> * fix: kafka message format Signed-off-by: Lawrence Zawila <[email protected]> * fix: tasks response formatting Signed-off-by: Lawrence Zawila <[email protected]> * fix: missing provider in kafka message Signed-off-by: Lawrence Zawila <[email protected]> * fix: object instead of an array for payments kafka message Signed-off-by: Lawrence Zawila <[email protected]> * feat: drop adjustments from kafka message Signed-off-by: Lawrence Zawila <[email protected]> * feat: drop raw data in kafka message Signed-off-by: Lawrence Zawila <[email protected]> * fix: payment selection by id Signed-off-by: Lawrence Zawila <[email protected]> * fix: router paths for connectors Signed-off-by: Lawrence Zawila <[email protected]> * fix: case insensitivity for connector paths Signed-off-by: Lawrence Zawila <[email protected]> * fix: default to empty array instead of null in json responses Signed-off-by: Lawrence Zawila <[email protected]> Signed-off-by: Lawrence Zawila <[email protected]> Co-authored-by: Maxence Maireaux <[email protected]>
formancehq · Dec 20, 2022 · 655feec · 655feec
1 parent 587160e
commit 655feec
Show file tree

Hide file tree

Showing 153 changed files with 3,212 additions and 2,991 deletions.
diff --git a/.golangci.yml b/.golangci.yml
@@ -99,6 +99,11 @@ issues:
         - goerr113
         - varnamelen
 
+    - path: internal/app/migrations/
+      linters:
+        - gochecknoinits
+        - varnamelen
+
     - linters:
       - nolintlint
       text: "should be written without leading space"

diff --git a/cmd/migrate.go b/cmd/migrate.go
@@ -0,0 +1,73 @@
+package cmd
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/spf13/viper"
+
+	// allow blank import to initiate migrations.
+	_ "github.com/formancehq/payments/internal/app/migrations"
+	_ "github.com/lib/pq"
+
+	"github.com/pressly/goose/v3"
+	"github.com/spf13/cobra"
+)
+
+func newMigrate() *cobra.Command {
+	return &cobra.Command{
+		Use:   "migrate",
+		Short: "Run migrations",
+		RunE:  runMigrate,
+	}
+}
+
+// Usage: `go run cmd/main.go migrate --postgres-uri {uri} {command}`
+/*
+Commands:
+    up                   Migrate the DB to the most recent version available
+    up-by-one            Migrate the DB up by 1
+    up-to VERSION        Migrate the DB to a specific VERSION
+    down                 Roll back the version by 1
+    down-to VERSION      Roll back to a specific VERSION
+    redo                 Re-run the latest migration
+    reset                Roll back all migrations
+    status               Dump the migration status for the current DB
+    version              Print the current version of the database
+    create NAME [sql|go] Creates new migration file with the current timestamp
+    fix                  Apply sequential ordering to migrations
+*/
+
+func runMigrate(cmd *cobra.Command, args []string) error {
+	postgresURI := viper.GetString(postgresURIFlag)
+	if postgresURI == "" {
+		postgresURI = cmd.Flag(postgresURIFlag).Value.String()
+	}
+
+	if postgresURI == "" {
+		return fmt.Errorf("postgres uri is not set")
+	}
+
+	database, err := goose.OpenDBWithDriver("postgres", postgresURI)
+	if err != nil {
+		return fmt.Errorf("failed to open database: %w", err)
+	}
+
+	defer func() {
+		if err = database.Close(); err != nil {
+			log.Fatalf("failed to close DB: %v\n", err)
+		}
+	}()
+
+	if len(args) == 0 {
+		return fmt.Errorf("missing migration direction")
+	}
+
+	command := args[0]
+
+	if err = goose.Run(command, database, ".", args[1:]...); err != nil {
+		log.Printf("migrate %v: %v", command, err)
+	}
+
+	return nil
+}
diff --git a/cmd/root.go b/cmd/root.go
@@ -35,12 +35,17 @@ func rootCommand() *cobra.Command {
 	root.AddCommand(version)
 
 	server := newServer()
-	root.AddCommand(server)
+	root.AddCommand(newServer())
+
+	migrate := newMigrate()
+	root.AddCommand(migrate)
 
 	root.PersistentFlags().Bool(debugFlag, false, "Debug mode")
+
+	migrate.Flags().String(postgresURIFlag, "postgres://localhost/payments", "PostgreSQL DB address")
+
 	server.Flags().BoolP("toggle", "t", false, "Help message for toggle")
-	server.Flags().String(mongodbURIFlag, "mongodb://localhost:27017", "MongoDB address")
-	server.Flags().String(mongodbDatabaseFlag, "payments", "MongoDB database name")
+	server.Flags().String(postgresURIFlag, "postgres://localhost/payments", "PostgreSQL DB address")
 	server.Flags().String(envFlag, "local", "Environment")
 	server.Flags().Bool(publisherKafkaEnabledFlag, false, "Publish write events to kafka")
 	server.Flags().StringSlice(publisherKafkaBrokerFlag, []string{}, "Kafka address is kafka enabled")

diff --git a/cmd/server.go b/cmd/server.go
@@ -5,7 +5,7 @@ import (
 
 	"github.com/bombsimon/logrusr/v3"
 	"github.com/formancehq/payments/internal/app/api"
-	"github.com/formancehq/payments/internal/app/database"
+	"github.com/formancehq/payments/internal/app/storage"
 	"github.com/pkg/errors"
 	"go.opentelemetry.io/otel"
 
@@ -27,34 +27,26 @@ import (
 
 //nolint:gosec // false positive
 const (
-	mongodbURIFlag                       = "mongodb-uri"
-	mongodbDatabaseFlag                  = "mongodb-database"
-	otelTracesFlag                       = "otel-traces"
-	otelTracesExporterFlag               = "otel-traces-exporter"
-	otelTracesExporterJaegerEndpointFlag = "otel-traces-exporter-jaeger-endpoint"
-	otelTracesExporterJaegerUserFlag     = "otel-traces-exporter-jaeger-user"
-	otelTracesExporterJaegerPasswordFlag = "otel-traces-exporter-jaeger-password"
-	otelTracesExporterOTLPModeFlag       = "otel-traces-exporter-otlp-mode"
-	otelTracesExporterOTLPEndpointFlag   = "otel-traces-exporter-otlp-endpoint"
-	otelTracesExporterOTLPInsecureFlag   = "otel-traces-exporter-otlp-insecure"
-	envFlag                              = "env"
-	publisherKafkaEnabledFlag            = "publisher-kafka-enabled"
-	publisherKafkaBrokerFlag             = "publisher-kafka-broker"
-	publisherKafkaSASLEnabled            = "publisher-kafka-sasl-enabled"
-	publisherKafkaSASLUsername           = "publisher-kafka-sasl-username"
-	publisherKafkaSASLPassword           = "publisher-kafka-sasl-password"
-	publisherKafkaSASLMechanism          = "publisher-kafka-sasl-mechanism"
-	publisherKafkaSASLScramSHASize       = "publisher-kafka-sasl-scram-sha-size"
-	publisherKafkaTLSEnabled             = "publisher-kafka-tls-enabled"
-	publisherTopicMappingFlag            = "publisher-topic-mapping"
-	publisherHTTPEnabledFlag             = "publisher-http-enabled"
-	authBasicEnabledFlag                 = "auth-basic-enabled"
-	authBasicCredentialsFlag             = "auth-basic-credentials"
-	authBearerEnabledFlag                = "auth-bearer-enabled"
-	authBearerIntrospectURLFlag          = "auth-bearer-introspect-url"
-	authBearerAudienceFlag               = "auth-bearer-audience"
-	authBearerAudiencesWildcardFlag      = "auth-bearer-audiences-wildcard"
-	authBearerUseScopesFlag              = "auth-bearer-use-scopes"
+	postgresURIFlag                 = "postgres-uri"
+	otelTracesFlag                  = "otel-traces"
+	envFlag                         = "env"
+	publisherKafkaEnabledFlag       = "publisher-kafka-enabled"
+	publisherKafkaBrokerFlag        = "publisher-kafka-broker"
+	publisherKafkaSASLEnabled       = "publisher-kafka-sasl-enabled"
+	publisherKafkaSASLUsername      = "publisher-kafka-sasl-username"
+	publisherKafkaSASLPassword      = "publisher-kafka-sasl-password"
+	publisherKafkaSASLMechanism     = "publisher-kafka-sasl-mechanism"
+	publisherKafkaSASLScramSHASize  = "publisher-kafka-sasl-scram-sha-size"
+	publisherKafkaTLSEnabled        = "publisher-kafka-tls-enabled"
+	publisherTopicMappingFlag       = "publisher-topic-mapping"
+	publisherHTTPEnabledFlag        = "publisher-http-enabled"
+	authBasicEnabledFlag            = "auth-basic-enabled"
+	authBasicCredentialsFlag        = "auth-basic-credentials"
+	authBearerEnabledFlag           = "auth-bearer-enabled"
+	authBearerIntrospectURLFlag     = "auth-bearer-introspect-url"
+	authBearerAudienceFlag          = "auth-bearer-audience"
+	authBearerAudiencesWildcardFlag = "auth-bearer-audiences-wildcard"
+	authBearerUseScopesFlag         = "auth-bearer-use-scopes"
 
 	serviceName = "Payments"
 )
@@ -156,17 +148,12 @@ func setLogger() {
 }
 
 func prepareDatabaseOptions() (fx.Option, error) {
-	mongodbURI := viper.GetString(mongodbURIFlag)
-	if mongodbURI == "" {
-		return nil, errors.New("missing mongodb uri")
+	postgresURI := viper.GetString(postgresURIFlag)
+	if postgresURI == "" {
+		return nil, errors.New("missing postgres uri")
 	}
 
-	mongodbDatabase := viper.GetString(mongodbDatabaseFlag)
-	if mongodbDatabase == "" {
-		return nil, errors.New("missing mongodb database name")
-	}
-
-	return database.MongoModule(mongodbURI, mongodbDatabase), nil
+	return storage.Module(postgresURI), nil
 }
 
 func topicsMapping() map[string]string {

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,30 +1,40 @@
 version: '3.8'
+volumes:
+  postgres:
 
 services:
-  mongodb:
-    image: bitnami/mongodb:4.4
-    environment:
-      MONGODB_REPLICA_SET_MODE: primary
-      MONGODB_REPLICA_SET_KEY: abcdef
-      MONGODB_ADVERTISED_HOSTNAME: mongodb
-      MONGODB_ROOT_PASSWORD: root
+  postgres:
+    image: "postgres:14-alpine"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U payments"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
     ports:
-    - "27017:27017/tcp"
+      - "5432:5432"
+    environment:
+      POSTGRES_USER: "payments"
+      POSTGRES_PASSWORD: "payments"
+      POSTGRES_DB: "payments"
+      PGDATA: /data/postgres
+    volumes:
+      - postgres:/data/postgres
 
   payments:
     image: golang:1.19.3-alpine3.16
+    command: go run ./ migrate up && go run ./ server
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://127.0.0.1:8080/_healthcheck" ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
     depends_on:
-      - mongodb
-    command:
-      - go
-      - run
-      - main.go
-      - server
+      - postgres
     ports:
       - "8080:8080"
     volumes:
-      - .:/src
-    working_dir: /src
+      - .:/app
+    working_dir: /app
     environment:
       DEBUG: ${DEBUG:-"true"}
-      MONGODB_URI: mongodb://root:root@mongodb:27017
+      POSTGRES_URI: postgres://payments:payments@postgres/payments?sslmode=disable