fix: broken plugin tests on ci (#333)

* fix: rename tag to plugins

* fix: add curr branch on workflow to test ci

* fix(clickhouse): try different network config

* ifx: revert fixed tags, to hit ci, one by one

* chore: add a new temporary workflow for tests

* fix: rename workflow

* fix(bigtable-test): update tags for docker repo

* fix: increase retry time duration

* chore: remove plugins test from the curr branch

* fix(kafka): change port, remove closing connection

* fix(kafka): missing build tags

* fix: tests for clickhouse, kafka and bigtable

* fix: incorrect port allocation

* chore: clean back unnecessary changes

* fix: wrong bramch name

* fix(bigtable): remove unused imports

* fix(bigtable): remove string as error

* chore: increase dockertest max wait to 5 mins

* fix(couchdb): add single node config in test

* fix(kafka): add healthcheck in test

* fix(test): kafka and couchdb healthcheck

* fix(couchdb): unreliable test

* fix(superset): broken test

Co-authored-by: Muhammad Abduh <[email protected]>

.github/workflows/plugin_test.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+      - fix-plugin-tests
   workflow_dispatch:
 
 jobs:

plugins/extractors/bigtable/bigtable_test.go

@@ -1,58 +1,19 @@
-//go:build integration
-// +build integration
+//go:build plugins
+// +build plugins
 
 package bigtable_test
 
 import (
 	"context"
-	"log"
-	"os"
 	"testing"
 
 	"github.com/odpf/meteor/test/utils"
 
-	"cloud.google.com/go/bigtable"
 	"github.com/odpf/meteor/plugins"
 	bt "github.com/odpf/meteor/plugins/extractors/bigtable"
-	"github.com/ory/dockertest/v3"
-	"github.com/ory/dockertest/v3/docker"
 	"github.com/stretchr/testify/assert"
 )
 
-func TestMain(m *testing.M) {
-	// setup test
-	opts := dockertest.RunOptions{
-		Repository: "shopify/bigtable-emulator",
-		Env: []string{
-			"BIGTABLE_EMULATOR_HOST=localhost:9035",
-		},
-		ExposedPorts: []string{"9035"},
-		PortBindings: map[docker.Port][]docker.PortBinding{
-			"9035": {
-				{HostIP: "0.0.0.0", HostPort: "9035"},
-			},
-		},
-		Cmd: []string{"-cf", "dev.records.data,dev.records.metadata"},
-	}
-	// exponential backoff-retry, because the application in the container might not be ready to accept connections yet
-	retryFn := func(resource *dockertest.Resource) (err error) {
-		_, err = bigtable.NewAdminClient(context.Background(), "dev", "dev")
-		return
-	}
-	purgeFn, err := utils.CreateContainer(opts, retryFn)
-	if err != nil {
-		log.Fatal("", err)
-	}
-
-	// run tests
-	code := m.Run()
-
-	if err := purgeFn(); err != nil {
-		log.Fatal("", err)
-	}
-	os.Exit(code)
-}
-
 func TestInit(t *testing.T) {
 	t.Run("should return error if no project_id in config", func(t *testing.T) {
 		err := bt.New(utils.Logger).Init(context.TODO(), map[string]interface{}{
@@ -67,6 +28,6 @@ func TestInit(t *testing.T) {
 			"project_id": "",
 		})
 
-		assert.EqualError(t, err, "invalid extractor config")
+		assert.Equal(t, plugins.InvalidConfigError{}, err)
 	})
 }

plugins/extractors/clickhouse/clickhouse_test.go

@@ -1,5 +1,5 @@
-//go:build integration
-// +build integration
+//go:build plugins
+// +build plugins
 
 package clickhouse_test
 
@@ -51,7 +51,7 @@ func TestMain(m *testing.M) {
 		Tag:          "21.7.4-alpine",
 		ExposedPorts: []string{"9000", port},
 		Mounts: []string{
-			fmt.Sprintf("%s/localConfig/users.xml:/etc/clickhouse-server/users.xml:rw", pwd),
+			fmt.Sprintf("%s/localConfig/users.xml:/etc/clickhouse-server/users.d/user.xml:rw", pwd),
 		},
 		PortBindings: map[docker.Port][]docker.PortBinding{
 			"9000": {

plugins/extractors/couchdb/couchdb_test.go

@@ -1,12 +1,13 @@
-//go:build integration
-// +build integration
+//go:build plugins
+// +build plugins
 
 package couchdb_test
 
 import (
 	"context"
 	"fmt"
 	"log"
+	"net/http"
 	"os"
 	"strconv"
 	"testing"
@@ -38,6 +39,10 @@ var (
 )
 
 func TestMain(m *testing.M) {
+	pwd, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
+	}
 	// setup test
 	opts := dockertest.RunOptions{
 		Repository: "docker.io/bitnami/couchdb",
@@ -46,7 +51,10 @@ func TestMain(m *testing.M) {
 			"COUCHDB_USER=" + user,
 			"COUCHDB_PASSWORD=" + pass,
 		},
-		ExposedPorts: []string{"4369", "5984", port},
+		Mounts: []string{
+			fmt.Sprintf("%s/localConfig:/opt/bitnami/couchdb/etc/local.d:rw", pwd),
+		},
+		ExposedPorts: []string{port},
 		PortBindings: map[docker.Port][]docker.PortBinding{
 			"5984": {
 				{HostIP: "0.0.0.0", HostPort: "5984"},
@@ -57,18 +65,15 @@ func TestMain(m *testing.M) {
 	retryFn := func(resource *dockertest.Resource) (err error) {
 		client, err = kivik.New("couch", fmt.Sprintf("http://%s:%s@%s/", user, pass, host))
 		if err != nil {
-			return err
+			return
 		}
-		_, err = client.Ping(context.TODO())
+		err = setup()
 		return
 	}
 	purgeFn, err := utils.CreateContainer(opts, retryFn)
 	if err != nil {
 		log.Fatal(err)
 	}
-	if err := setup(); err != nil {
-		log.Fatal(err)
-	}
 
 	// run tests
 	code := m.Run()
@@ -115,6 +120,10 @@ func setup() (err error) {
 	for _, database := range dbs {
 		// create database
 		err = client.CreateDB(context.TODO(), database)
+		// DB already created
+		if kivik.StatusCode(err) == http.StatusPreconditionFailed {
+			err = nil
+		}
 		if err != nil {
 			return
 		}
@@ -131,6 +140,9 @@ func setup() (err error) {
 func execute(queries []map[string]interface{}, db *kivik.DB) (err error) {
 	for _, query := range queries {
 		_, err := db.Put(context.TODO(), query["_id"].(string), query)
+		if kivik.StatusCode(err) == http.StatusConflict {
+			err = nil
+		}
 		if err != nil {
 			return err
 		}

plugins/extractors/couchdb/localConfig/local.ini

@@ -0,0 +1,7 @@
+[couchdb] single_node=true
+
+[admins]
+meteor_test_user = -pbkdf2-5ab0d9d0d782edcd1353a0b058a4ea48618e363a,11726daaa26701e6ae8eccb6748f6abb,10
+
+[chttpd_auth]
+secret = ba1f0b5979e7e969f603da7710f8d7bf

plugins/extractors/elastic/elastic_test.go

@@ -1,5 +1,5 @@
-//go:build integration
-// +build integration
+//go:build plugins
+// +build plugins
 
 package elastic_test
 

plugins/extractors/kafka/kafka_test.go

@@ -1,10 +1,11 @@
-//go:build integration
-// +build integration
+//go:build plugins
+// +build plugins
 
 package kafka_test
 
 import (
 	"context"
+	"errors"
 	"log"
 	"net"
 
@@ -54,6 +55,17 @@ func TestMain(m *testing.M) {
 		if err != nil {
 			return
 		}
+
+		// healthcheck
+		brokerList, err := conn.Brokers()
+		if err != nil {
+			return
+		}
+		if len(brokerList) == 0 {
+			err = errors.New("not ready")
+			return
+		}
+
 		broker, err = conn.Controller()
 		if err != nil {
 			conn.Close()

plugins/extractors/redshift/redshift_test.go

@@ -1,3 +1,6 @@
+//go:build plugins
+// +build plugins
+
 package redshift_test
 
 import (

plugins/extractors/superset/localConfig/.gitignore

@@ -0,0 +1 @@
+__pycache__

plugins/extractors/superset/localConfig/superset_config.py

@@ -0,0 +1,2 @@
+# Flask-WTF flag for CSRF
+WTF_CSRF_ENABLED = False