diff --git a/.expeditor/verify_private.pipeline.yml b/.expeditor/verify_private.pipeline.yml index 78b1340697c..6f8110f13c5 100644 --- a/.expeditor/verify_private.pipeline.yml +++ b/.expeditor/verify_private.pipeline.yml @@ -429,6 +429,23 @@ steps: - HAB_STUDIO_SUP=false - HAB_NONINTERACTIVE=true + - label: "infra-proxy-service" + command: + - . scripts/verify_setup.sh + - hab studio run "source scripts/verify_studio_init.sh && start_deployment_service && chef-automate dev deployinate && infra_service_integration" + timeout_in_minutes: 20 + retry: + automatic: + limit: 1 + expeditor: + executor: + docker: + privileged: true + environment: + - HAB_STUDIO_SUP=false + - HAB_NONINTERACTIVE=true + - CONTAINER_HOSTNAME=localhost + # # The following tests all use the integration test framework for # end-to-end testing. These tests all test full deployments of diff --git a/.studio/infra-proxy-service b/.studio/infra-proxy-service index 27aab1ec650..ea3d2ca349a 100644 --- a/.studio/infra-proxy-service +++ b/.studio/infra-proxy-service @@ -11,6 +11,18 @@ function start_infra_proxy_service() { chef-automate dev deploy-some $HAB_ORIGIN/infra-proxy-service --with-deps } +document "infra_service_integration" </dev/null + )" chef.automate.domain.infra_proxy.service.InfraProxyService.CreateServer >/dev/null - chef-automate dev grpcurl automate-gateway -- -d \ + chef-automate dev grpcurl infra-proxy-service -- -d \ "$(cat << EOF {"id": "$5", "name": "$6", "admin_user": "$7", "admin_key": "$8", "server_id": "$1", "projects": []} EOF - )" chef.automate.api.infra_proxy.InfraProxy.CreateOrg >/dev/null + )" chef.automate.domain.infra_proxy.service.InfraProxyService.CreateOrg >/dev/null } function infra_service_load_chef_repo() { @@ -159,6 +171,10 @@ EOH source "https://supermarket.chef.io" cookbook "audit" +cookbook "chef-client" +cookbook "iptables" +cookbook "netdev" +cookbook "sudo" EOH if ! hab pkg exec chef/chef-dk knife opc user list -c ${chef_server_pivotal_rb_path} | grep ${chef_server_test_node_name} &> /dev/null; then diff --git a/components/infra-proxy-service/Makefile b/components/infra-proxy-service/Makefile index 6dfae0070ca..95e6090925f 100644 --- a/components/infra-proxy-service/Makefile +++ b/components/infra-proxy-service/Makefile @@ -1,6 +1,6 @@ include ../../Makefile.common_go -PACKAGE_PATH = github.com/chef/automate/components/infra-proxy-service +PACKAGE_PATH = github.com/chef/automate/components/infra-proxy-service/server BINS = ${PACKAGE_PATH}/cmd/infra-proxy-service MIGRATION_READMES = storage/postgres/migration/sql/README.md diff --git a/components/infra-proxy-service/cmd/infra-proxy-service/commands/serve.go b/components/infra-proxy-service/cmd/infra-proxy-service/commands/serve.go index 4172ea0d5c3..34137ea00cb 100644 --- a/components/infra-proxy-service/cmd/infra-proxy-service/commands/serve.go +++ b/components/infra-proxy-service/cmd/infra-proxy-service/commands/serve.go @@ -11,13 +11,13 @@ import ( secrets "github.com/chef/automate/api/external/secrets" "github.com/chef/automate/api/interservice/authz" + "github.com/chef/automate/components/infra-proxy-service/config" "github.com/chef/automate/components/infra-proxy-service/server" "github.com/chef/automate/components/infra-proxy-service/service" "github.com/chef/automate/components/infra-proxy-service/storage/postgres/migration" "github.com/chef/automate/lib/grpc/secureconn" "github.com/chef/automate/lib/logger" platform_config "github.com/chef/automate/lib/platform/config" - "github.com/chef/automate/lib/tls/certs" ) var serveCmd = &cobra.Command{ @@ -27,18 +27,6 @@ var serveCmd = &cobra.Command{ Args: cobra.ExactArgs(1), } -type config struct { - GRPC string `mapstructure:"grpc"` - LogFormat string `mapstructure:"log-format"` - LogLevel string `mapstructure:"log-level"` - certs.TLSConfig `mapstructure:"tls"` - PGURL string `mapstructure:"pg_url"` - Database string `mapstructure:"database"` - MigrationsPath string `mapstructure:"migrations-path"` - AuthzAddress string `mapstructure:"authz-address"` - SecretsAddress string `mapstructure:"secrets-address"` -} - func serve(cmd *cobra.Command, args []string) { cmd.PersistentFlags().StringP("log-level", "l", "info", "log level") cmd.PersistentFlags().StringP("log-format", "f", "text", "log format") @@ -53,7 +41,7 @@ func serve(cmd *cobra.Command, args []string) { fail(errors.Wrap(err, `Could not read config file. Please pass a config file as the only argument to this command.`)) } - cfg := config{} + cfg := config.Service{} if err := viper.Unmarshal(&cfg); err != nil { fail(errors.Wrap(err, "couldn't parse configuration file")) } diff --git a/components/infra-proxy-service/config/service.go b/components/infra-proxy-service/config/service.go new file mode 100644 index 00000000000..7040f7d09be --- /dev/null +++ b/components/infra-proxy-service/config/service.go @@ -0,0 +1,124 @@ +package config + +import ( + "fmt" + "net/url" + "os" + + "github.com/pkg/errors" + "github.com/spf13/viper" + + secrets "github.com/chef/automate/api/external/secrets" + "github.com/chef/automate/api/interservice/authz" + "github.com/chef/automate/components/infra-proxy-service/service" + "github.com/chef/automate/components/infra-proxy-service/storage/postgres/migration" + "github.com/chef/automate/lib/grpc/secureconn" + "github.com/chef/automate/lib/logger" + platform_config "github.com/chef/automate/lib/platform/config" + "github.com/chef/automate/lib/tls/certs" +) + +// Service config options +type Service struct { + GRPC string `mapstructure:"grpc"` + LogFormat string `mapstructure:"log-format"` + LogLevel string `mapstructure:"log-level"` + certs.TLSConfig `mapstructure:"tls"` + PGURL string `mapstructure:"pg_url"` + Database string `mapstructure:"database"` + MigrationsPath string `mapstructure:"migrations-path"` + AuthzAddress string `mapstructure:"authz-address"` + SecretsAddress string `mapstructure:"secrets-address"` +} + +// ConfigFromViper returns a Service instance from the current viper config +func ConfigFromViper(configFile string) (*service.Service, error) { + // Set the file name of the configurations file + viper.SetConfigName("config") + // Set the configuration file type + viper.SetConfigType("yaml") + // Set the path to look for the configurations file + viper.AddConfigPath("../dev") + + if err := viper.ReadInConfig(); err != nil { + fail(errors.Wrap(err, `Could not read config file. Please pass a config file as the only argument to this command.`)) + } + + cfg := Service{} + if err := viper.Unmarshal(&cfg); err != nil { + fail(errors.Wrap(err, "couldn't parse configuration file")) + } + + pgURL, err := platform_config.PGURIFromEnvironment(cfg.Database) + if err != nil { + fail(errors.Wrap(err, "Failed to get pg uri")) + } + cfg.PGURL = pgURL + + l, err := logger.NewLogger(cfg.LogFormat, cfg.LogLevel) + if err != nil { + fail(errors.Wrap(err, "couldn't initialize logger")) + } + + cfg.FixupRelativeTLSPaths(configFile) + serviceCerts, err := cfg.ReadCerts() + if err != nil { + fail(errors.Wrap(err, "Could not read certs")) + } + connFactory := secureconn.NewFactory(*serviceCerts) + + mustBeADirectory(cfg.MigrationsPath) + u, err := url.Parse(cfg.PGURL) + if err != nil { + fail(errors.Wrapf(err, "could not parse pg_url %s from config", cfg.PGURL)) + } + migrationConfig := migration.Config{ + Path: cfg.MigrationsPath, + PGURL: u, + Logger: l, + } + + if cfg.AuthzAddress == "" { + fail(errors.New("missing required config authz_address")) + } + authzConn, err := connFactory.Dial("authz-service", cfg.AuthzAddress) + if err != nil { + fail(errors.Wrapf(err, "failed to dial authz-service at (%s)", cfg.AuthzAddress)) + } + authzClient := authz.NewAuthorizationServiceClient(authzConn) + + if cfg.SecretsAddress == "" { + fail(errors.New("missing required config secrets_address")) + } + secretsConn, err := connFactory.Dial("secrets-service", cfg.SecretsAddress) + if err != nil { + fail(errors.Wrapf(err, "failed to dial secrets-service at (%s)", cfg.SecretsAddress)) + } + + // gets secrets client + secretsClient := secrets.NewSecretsServiceClient(secretsConn) + + service, err := service.Start(l, migrationConfig, connFactory, secretsClient, authzClient) + if err != nil { + fail(errors.Wrap(err, "could not initialize storage")) + } + + return service, nil +} + +// fail outputs the error and exits with a non-zero code +func fail(err error) { + // no error check: if this goes wrong, we're in trouble anyways + fmt.Fprint(os.Stderr, err.Error()) // nolint: gas + os.Exit(1) +} + +func mustBeADirectory(path string) { + stat, err := os.Stat(path) + if err == nil && stat.IsDir() { + return // everything's in its right place + } else if err != nil { + fail(errors.Wrapf(err, "open path %#v", path)) + } + fail(fmt.Errorf("path %#v is not a directory", path)) +} diff --git a/components/infra-proxy-service/dev/config.yml b/components/infra-proxy-service/dev/config.yml index 8f60479ec10..8e89b4bb4ba 100644 --- a/components/infra-proxy-service/dev/config.yml +++ b/components/infra-proxy-service/dev/config.yml @@ -4,12 +4,12 @@ log-level: "info" log-format: "text" tls: - cert_path: ../../../dev/certs/infra-proxy-service.crt - key_path: ../../../dev/certs/infra-proxy-service.key - root_ca_path: ../../../dev/certs/Chef_Automate_FAKE_Dev.crt + cert_path: /hab/svc/infra-proxy-service/config/service.crt + key_path: /hab/svc/infra-proxy-service/config/service.key + root_ca_path: /hab/svc/infra-proxy-service/config/root_ca.crt pg_url: "postgresql://postgres@127.0.0.1:5432/infra_proxy_test?sslmode=disable" -database: infra_proxy_test -migrations-path: storage/postgres/migration/sql/ +database: chef_infra_proxy +migrations-path: /src/components/infra-proxy-service/storage/postgres/migration/sql/ authz-address: "0.0.0.0:10130" secrets-address: "0.0.0.0:10131" diff --git a/components/infra-proxy-service/integration_test/clients_test.go b/components/infra-proxy-service/integration_test/clients_test.go new file mode 100644 index 00000000000..767bb5d8100 --- /dev/null +++ b/components/infra-proxy-service/integration_test/clients_test.go @@ -0,0 +1,36 @@ +package integration_test + +import ( + "context" + "testing" + + "github.com/chef/automate/api/interservice/infra_proxy/request" + "github.com/stretchr/testify/assert" +) + +func TestGetClients(t *testing.T) { + // rpc GetClients (request.Clients) returns (response.Clients) + ctx := context.Background() + req := &request.Clients{ + ServerId: autoDeployedChefServerID, + OrgId: autoDeployedChefOrganizationID, + } + res, err := infraProxy.GetClients(ctx, req) + assert.NoError(t, err) + assert.NotNil(t, res) + assert.Equal(t, 3, len(res.GetClients())) +} + +func TestGetClient(t *testing.T) { + // rpc GetClient (request.Client) returns (response.Client) + ctx := context.Background() + req := &request.Client{ + ServerId: autoDeployedChefServerID, + OrgId: autoDeployedChefOrganizationID, + Name: "chef-load-1", + } + res, err := infraProxy.GetClient(ctx, req) + assert.NoError(t, err) + assert.NotNil(t, res) + assert.Equal(t, "chef-load-1", res.Name) +} diff --git a/components/infra-proxy-service/integration_test/global_test.go b/components/infra-proxy-service/integration_test/global_test.go new file mode 100644 index 00000000000..d81ca36a4ad --- /dev/null +++ b/components/infra-proxy-service/integration_test/global_test.go @@ -0,0 +1,33 @@ +package integration_test + +import ( + "os" + "testing" +) + +// Global variables +var ( + autoDeployedChefServerID = "local-dev" + autoDeployedChefOrganizationID = "test-org" + cFile = "/src/components/infra-proxy-service/dev/config" + // This suite variable will be available for every single test as long as they + // belong to the 'integration_test' package. + suite = NewSuite() +) + +func TestMain(m *testing.M) { + // Global Setup hook: Here is where you can initialize anythings you need + // for your tests to run, things like; Initialize ES indices, insert + // nodes or runs, etc. + suite.GlobalSetup() + + // Execute the test suite and record the exit code + exitCode := m.Run() + + // Teardown hook: It says it all, this hook should clean documents + // from ES so that the next test can run on a clean env. + suite.GlobalTeardown() + + // call with result of m.Run() + os.Exit(exitCode) +} diff --git a/components/infra-proxy-service/integration_test/suite_test.go b/components/infra-proxy-service/integration_test/suite_test.go new file mode 100644 index 00000000000..d4c34a931f1 --- /dev/null +++ b/components/infra-proxy-service/integration_test/suite_test.go @@ -0,0 +1,52 @@ +package integration_test + +import ( + "github.com/chef/automate/components/infra-proxy-service/config" + "github.com/chef/automate/components/infra-proxy-service/server" +) + +// Global variables +var ( + // A global Infra Proxy Server instance to call any rpc function + // + // From any test you can directly call: + // ``` + // res, err := infraProxy.GetOrgs(ctx, &req) + // ``` + infraProxy *server.Server +) + +type Suite struct{} + +// Just returns a new struct. You have to call GlobalSetup() to setup +func NewSuite() *Suite { + return new(Suite) +} + +// GlobalSetup makes all connections. +func (s *Suite) GlobalSetup() error { + var err error + // set global infraProxy + infraProxy, err = newInfraProxyServer() + + if err != nil { + return err + } + + return nil +} + +// GlobalTeardown is the place where you tear everything down after we have finished +func (s *Suite) GlobalTeardown() {} + +// newInfraProxyServer initializes a InfraProxyServer with the default config +func newInfraProxyServer() (*server.Server, error) { + service, err := config.ConfigFromViper(cFile) + if err != nil { + return nil, err + } + + gRPC := server.NewServer(service) + + return gRPC, nil +} diff --git a/components/infra-proxy-service/server/clients.go b/components/infra-proxy-service/server/clients.go index 4dd97475010..d4a7a872501 100644 --- a/components/infra-proxy-service/server/clients.go +++ b/components/infra-proxy-service/server/clients.go @@ -13,6 +13,7 @@ import ( // GetClients get clients list func (s *Server) GetClients(ctx context.Context, req *request.Clients) (*response.Clients, error) { c, err := s.createClient(ctx, req.OrgId, req.ServerId) + if err != nil { return nil, err } diff --git a/components/infra-proxy-service/storage/postgres/orgs.go b/components/infra-proxy-service/storage/postgres/orgs.go index 8d2fc007a86..b325fcb4172 100644 --- a/components/infra-proxy-service/storage/postgres/orgs.go +++ b/components/infra-proxy-service/storage/postgres/orgs.go @@ -25,8 +25,13 @@ func (p *postgres) insertOrg(ctx context.Context, projects = []string{} } + // Adding the subjects if missing if gRPC calls from internal service level + subjects := auth_context.FromContext(auth_context.FromIncomingMetadata(ctx)).Subjects + if len(subjects) == 0 { + subjects = []string{"tls:service:compliance-service:internal"} + } _, err := p.authzClient.ValidateProjectAssignment(ctx, &authz.ValidateProjectAssignmentReq{ - Subjects: auth_context.FromContext(auth_context.FromIncomingMetadata(ctx)).Subjects, + Subjects: subjects, OldProjects: []string{}, NewProjects: projects, }) diff --git a/dev-docs/adding-data/infra/chef-repo/clients/chef-load-1.json b/dev-docs/adding-data/infra/chef-repo/clients/chef-load-1.json new file mode 100644 index 00000000000..c2ae2a8b327 --- /dev/null +++ b/dev-docs/adding-data/infra/chef-repo/clients/chef-load-1.json @@ -0,0 +1,7 @@ +{ + "name": "chef-load-1", + "validator": false, + "admin": false, + "chef_type": "client", + "create_key": false +} diff --git a/dev-docs/adding-data/infra/chef-repo/clients/chef-load-2.json b/dev-docs/adding-data/infra/chef-repo/clients/chef-load-2.json new file mode 100644 index 00000000000..0f7bb638212 --- /dev/null +++ b/dev-docs/adding-data/infra/chef-repo/clients/chef-load-2.json @@ -0,0 +1,7 @@ +{ + "name": "chef-load-2", + "validator": false, + "admin": false, + "chef_type": "client", + "create_key": false +}