Skip to content

Commit

Permalink
Merge pull request #15672 from deads2k/tsb-06-separate
Browse files Browse the repository at this point in the history 
Automatic merge from submit-queue

Run the TSB in a separate pod

This completes the work necessary to run the TSB in a pod.  We should continue to remove the cruft from the old path, but I've included a template which creates a running TSB server in the `openshift-template-service-broker` namespace.

@bparees @jim-minter
  • Loading branch information
@openshift-merge-robot
openshift-merge-robot authored Aug 9, 2017
2 parents 423e664 + edda314 commit 07ec3d2
Show file tree
Hide file tree
Showing 6 changed files with 149 additions and 92 deletions.
28 changes: 0 additions & 28 deletions contrib/completions/bash/openshift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38516,30 +38516,8 @@ _openshift_start_template-service-broker()
local_nonpersistent_flags+=("--client-ca-file=")
flags+=("--contention-profiling")
local_nonpersistent_flags+=("--contention-profiling")
flags+=("--delete-collection-workers=")
local_nonpersistent_flags+=("--delete-collection-workers=")
flags+=("--deserialization-cache-size=")
local_nonpersistent_flags+=("--deserialization-cache-size=")
flags+=("--enable-garbage-collector")
local_nonpersistent_flags+=("--enable-garbage-collector")
flags+=("--enable-swagger-ui")
local_nonpersistent_flags+=("--enable-swagger-ui")
flags+=("--etcd-cafile=")
local_nonpersistent_flags+=("--etcd-cafile=")
flags+=("--etcd-certfile=")
local_nonpersistent_flags+=("--etcd-certfile=")
flags+=("--etcd-keyfile=")
local_nonpersistent_flags+=("--etcd-keyfile=")
flags+=("--etcd-prefix=")
local_nonpersistent_flags+=("--etcd-prefix=")
flags+=("--etcd-quorum-read")
local_nonpersistent_flags+=("--etcd-quorum-read")
flags+=("--etcd-servers=")
local_nonpersistent_flags+=("--etcd-servers=")
flags+=("--etcd-servers-overrides=")
local_nonpersistent_flags+=("--etcd-servers-overrides=")
flags+=("--experimental-encryption-provider-config=")
local_nonpersistent_flags+=("--experimental-encryption-provider-config=")
flags+=("--profiling")
local_nonpersistent_flags+=("--profiling")
flags+=("--requestheader-allowed-names=")
Expand All @@ -38554,10 +38532,6 @@ _openshift_start_template-service-broker()
local_nonpersistent_flags+=("--requestheader-username-headers=")
flags+=("--secure-port=")
local_nonpersistent_flags+=("--secure-port=")
flags+=("--storage-backend=")
local_nonpersistent_flags+=("--storage-backend=")
flags+=("--storage-media-type=")
local_nonpersistent_flags+=("--storage-media-type=")
flags+=("--template-namespace=")
local_nonpersistent_flags+=("--template-namespace=")
flags+=("--tls-ca-file=")
Expand All @@ -38568,8 +38542,6 @@ _openshift_start_template-service-broker()
local_nonpersistent_flags+=("--tls-private-key-file=")
flags+=("--tls-sni-cert-key=")
local_nonpersistent_flags+=("--tls-sni-cert-key=")
flags+=("--watch-cache")
local_nonpersistent_flags+=("--watch-cache")
flags+=("--azure-container-registry-config=")
flags+=("--google-json-key=")
flags+=("--log-flush-frequency=")
Expand Down
28 changes: 0 additions & 28 deletions contrib/completions/zsh/openshift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38665,30 +38665,8 @@ _openshift_start_template-service-broker()
local_nonpersistent_flags+=("--client-ca-file=")
flags+=("--contention-profiling")
local_nonpersistent_flags+=("--contention-profiling")
flags+=("--delete-collection-workers=")
local_nonpersistent_flags+=("--delete-collection-workers=")
flags+=("--deserialization-cache-size=")
local_nonpersistent_flags+=("--deserialization-cache-size=")
flags+=("--enable-garbage-collector")
local_nonpersistent_flags+=("--enable-garbage-collector")
flags+=("--enable-swagger-ui")
local_nonpersistent_flags+=("--enable-swagger-ui")
flags+=("--etcd-cafile=")
local_nonpersistent_flags+=("--etcd-cafile=")
flags+=("--etcd-certfile=")
local_nonpersistent_flags+=("--etcd-certfile=")
flags+=("--etcd-keyfile=")
local_nonpersistent_flags+=("--etcd-keyfile=")
flags+=("--etcd-prefix=")
local_nonpersistent_flags+=("--etcd-prefix=")
flags+=("--etcd-quorum-read")
local_nonpersistent_flags+=("--etcd-quorum-read")
flags+=("--etcd-servers=")
local_nonpersistent_flags+=("--etcd-servers=")
flags+=("--etcd-servers-overrides=")
local_nonpersistent_flags+=("--etcd-servers-overrides=")
flags+=("--experimental-encryption-provider-config=")
local_nonpersistent_flags+=("--experimental-encryption-provider-config=")
flags+=("--profiling")
local_nonpersistent_flags+=("--profiling")
flags+=("--requestheader-allowed-names=")
Expand All @@ -38703,10 +38681,6 @@ _openshift_start_template-service-broker()
local_nonpersistent_flags+=("--requestheader-username-headers=")
flags+=("--secure-port=")
local_nonpersistent_flags+=("--secure-port=")
flags+=("--storage-backend=")
local_nonpersistent_flags+=("--storage-backend=")
flags+=("--storage-media-type=")
local_nonpersistent_flags+=("--storage-media-type=")
flags+=("--template-namespace=")
local_nonpersistent_flags+=("--template-namespace=")
flags+=("--tls-ca-file=")
Expand All @@ -38717,8 +38691,6 @@ _openshift_start_template-service-broker()
local_nonpersistent_flags+=("--tls-private-key-file=")
flags+=("--tls-sni-cert-key=")
local_nonpersistent_flags+=("--tls-sni-cert-key=")
flags+=("--watch-cache")
local_nonpersistent_flags+=("--watch-cache")
flags+=("--azure-container-registry-config=")
flags+=("--google-json-key=")
flags+=("--log-flush-frequency=")
Expand Down
2 changes: 1 addition & 1 deletion pkg/cmd/server/origin/master.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func (c *MasterConfig) newOpenshiftNonAPIConfig(kubeAPIServerConfig apiserver.Co
func (c *MasterConfig) newTemplateServiceBrokerConfig(kubeAPIServerConfig apiserver.Config) *openservicebrokerserver.TemplateServiceBrokerConfig {
ret := &openservicebrokerserver.TemplateServiceBrokerConfig{
GenericConfig: &kubeAPIServerConfig,
PrivilegedKubeClientConfig: *kubeAPIServerConfig.LoopbackClientConfig,
PrivilegedKubeClientConfig: kubeAPIServerConfig.LoopbackClientConfig,
TemplateInformers: c.TemplateInformers,
TemplateNamespaces: c.Options.TemplateServiceBrokerConfig.TemplateNamespaces,
}
Expand Down
37 changes: 30 additions & 7 deletions pkg/openservicebroker/cmd/server/start.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,13 @@ import (
"github.com/openshift/origin/pkg/openservicebroker/server"
)

const defaultEtcdPathPrefix = "/registry/templateservicebroker.openshift.io"

type TemplateServiceBrokerServerOptions struct {
RecommendedOptions *genericoptions.RecommendedOptions
// we don't have any storage, so we shouldn't use the recommended options
SecureServing *genericoptions.SecureServingOptions
Authentication *genericoptions.DelegatingAuthenticationOptions
Authorization *genericoptions.DelegatingAuthorizationOptions
Audit *genericoptions.AuditOptions
Features *genericoptions.FeatureOptions

StdOut io.Writer
StdErr io.Writer
Expand All @@ -26,7 +29,11 @@ type TemplateServiceBrokerServerOptions struct {

func NewTemplateServiceBrokerServerOptions(out, errOut io.Writer) *TemplateServiceBrokerServerOptions {
o := &TemplateServiceBrokerServerOptions{
RecommendedOptions: genericoptions.NewRecommendedOptions(defaultEtcdPathPrefix, server.Scheme, server.Codecs.LegacyCodec()),
SecureServing: genericoptions.NewSecureServingOptions(),
Authentication: genericoptions.NewDelegatingAuthenticationOptions(),
Authorization: genericoptions.NewDelegatingAuthorizationOptions(),
Audit: genericoptions.NewAuditOptions(),
Features: genericoptions.NewFeatureOptions(),

StdOut: out,
StdErr: errOut,
Expand Down Expand Up @@ -57,7 +64,11 @@ func NewCommandStartTemplateServiceBrokerServer(out, errOut io.Writer, stopCh <-
}

flags := cmd.Flags()
o.RecommendedOptions.AddFlags(flags)
o.SecureServing.AddFlags(flags)
o.Authentication.AddFlags(flags)
o.Authorization.AddFlags(flags)
o.Audit.AddFlags(flags)
o.Features.AddFlags(flags)
flags.StringSliceVar(&o.TemplateNamespaces, "template-namespace", o.TemplateNamespaces, "TemplateNamespaces indicates the namespace(s) in which the template service broker looks for templates to serve to the catalog.")

return cmd
Expand All @@ -73,12 +84,24 @@ func (o *TemplateServiceBrokerServerOptions) Complete() error {

func (o TemplateServiceBrokerServerOptions) Config() (*server.TemplateServiceBrokerConfig, error) {
// TODO have a "real" external address
if err := o.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil {
if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil {
return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
}

serverConfig := genericapiserver.NewConfig(server.Codecs)
if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil {
if err := o.SecureServing.ApplyTo(serverConfig); err != nil {
return nil, err
}
if err := o.Authentication.ApplyTo(serverConfig); err != nil {
return nil, err
}
if err := o.Authorization.ApplyTo(serverConfig); err != nil {
return nil, err
}
if err := o.Audit.ApplyTo(serverConfig); err != nil {
return nil, err
}
if err := o.Features.ApplyTo(serverConfig); err != nil {
return nil, err
}

Expand Down
105 changes: 77 additions & 28 deletions pkg/openservicebroker/server/apiserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,15 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
genericapiserver "k8s.io/apiserver/pkg/server"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
"k8s.io/kubernetes/pkg/api"
kclientsetinternal "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
"k8s.io/kubernetes/pkg/controller"

"github.com/openshift/origin/pkg/cmd/server/bootstrappolicy"
templateapi "github.com/openshift/origin/pkg/template/apis/template"
templateinformer "github.com/openshift/origin/pkg/template/generated/informers/internalversion"
templateinternalclientset "github.com/openshift/origin/pkg/template/generated/internalclientset"
templateservicebroker "github.com/openshift/origin/pkg/template/servicebroker"
)

Expand Down Expand Up @@ -51,7 +54,7 @@ type TemplateServiceBrokerConfig struct {

// PrivilegedKubeClientConfig is *not* a loopback config, since it needs to point to the kube apiserver
// TODO remove this and use the SA that start us instead of trying to cyclically find an SA token
PrivilegedKubeClientConfig restclient.Config
PrivilegedKubeClientConfig *restclient.Config

TemplateInformers templateinformer.SharedInformerFactory
TemplateNamespaces []string
Expand Down Expand Up @@ -87,11 +90,55 @@ func (c completedTemplateServiceBrokerConfig) New(delegationTarget genericapiser
GenericAPIServer: genericServer,
}

broker := templateservicebroker.DeprecatedNewBrokerInsideAPIServer(
c.PrivilegedKubeClientConfig,
c.TemplateInformers.Template().InternalVersion().Templates(),
c.TemplateNamespaces,
)
inCluster := false
var broker *templateservicebroker.Broker
// TODO, this block drops out after the server is moved.
if c.PrivilegedKubeClientConfig != nil {
broker = templateservicebroker.DeprecatedNewBrokerInsideAPIServer(
*c.PrivilegedKubeClientConfig,
c.TemplateInformers.Template().InternalVersion().Templates(),
c.TemplateNamespaces,
)
// make sure no one else uses it
c.TemplateInformers = nil

} else {
// we're running in cluster
// in this case we actually want to construct our own template informer
// eventually the config value drops out as it isn't supported server side
inCluster = true
clientConfig, err := restclient.InClusterConfig()
if err != nil {
return nil, err
}
templateClient, err := templateinternalclientset.NewForConfig(clientConfig)
if err != nil {
return nil, err
}
templateInformers := templateinformer.NewSharedInformerFactory(templateClient, 5*time.Minute)
templateInformers.Template().InternalVersion().Templates().Informer().AddIndexers(cache.Indexers{
templateapi.TemplateUIDIndex: func(obj interface{}) ([]string, error) {
return []string{string(obj.(*templateapi.Template).UID)}, nil
},
})

broker, err = templateservicebroker.NewBroker(
clientConfig,
templateInformers.Template().InternalVersion().Templates(),
c.TemplateNamespaces,
)
if err != nil {
return nil, err
}

s.GenericAPIServer.AddPostStartHook("template-service-broker-synctemplates", func(context genericapiserver.PostStartHookContext) error {
templateInformers.Start(context.StopCh)
if !controller.WaitForCacheSync("tsb", context.StopCh, templateInformers.Template().InternalVersion().Templates().Informer().HasSynced) {
return fmt.Errorf("unable to sync caches")
}
return nil
})
}

Route(
s.GenericAPIServer.Handler.GoRestfulContainer,
Expand All @@ -100,32 +147,34 @@ func (c completedTemplateServiceBrokerConfig) New(delegationTarget genericapiser
)

// TODO, when the TSB becomes a separate entity, this should stop creating the SA and use its container pod SA identity instead
s.GenericAPIServer.AddPostStartHook("template-service-broker-ensure-service-account", func(context genericapiserver.PostStartHookContext) error {
kc, err := kclientsetinternal.NewForConfig(context.LoopbackClientConfig)
if err != nil {
utilruntime.HandleError(fmt.Errorf("template service broker: failed to get client: %v", err))
return err
}
if !inCluster {
s.GenericAPIServer.AddPostStartHook("template-service-broker-ensure-service-account", func(context genericapiserver.PostStartHookContext) error {
kc, err := kclientsetinternal.NewForConfig(context.LoopbackClientConfig)
if err != nil {
utilruntime.HandleError(fmt.Errorf("template service broker: failed to get client: %v", err))
return err
}

err = wait.PollImmediate(time.Second, 30*time.Second, func() (done bool, err error) {
kc.Namespaces().Create(&api.Namespace{ObjectMeta: metav1.ObjectMeta{Name: bootstrappolicy.DefaultOpenShiftInfraNamespace}})
err = wait.PollImmediate(time.Second, 30*time.Second, func() (done bool, err error) {
kc.Namespaces().Create(&api.Namespace{ObjectMeta: metav1.ObjectMeta{Name: bootstrappolicy.DefaultOpenShiftInfraNamespace}})

_, err = kc.ServiceAccounts(bootstrappolicy.DefaultOpenShiftInfraNamespace).Create(&api.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: bootstrappolicy.InfraTemplateServiceBrokerServiceAccountName}})
switch {
case err == nil || kapierrors.IsAlreadyExists(err):
done, err = true, nil
case kapierrors.IsNotFound(err):
err = nil
}
_, err = kc.ServiceAccounts(bootstrappolicy.DefaultOpenShiftInfraNamespace).Create(&api.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: bootstrappolicy.InfraTemplateServiceBrokerServiceAccountName}})
switch {
case err == nil || kapierrors.IsAlreadyExists(err):
done, err = true, nil
case kapierrors.IsNotFound(err):
err = nil
}

return
})
return
})

if err != nil {
utilruntime.HandleError(fmt.Errorf("creation of template-service-broker SA failed: %v", err))
}
return err
})
if err != nil {
utilruntime.HandleError(fmt.Errorf("creation of template-service-broker SA failed: %v", err))
}
return err
})
}

return s, nil
}
41 changes: 41 additions & 0 deletions pkg/template/servicebroker/servicebroker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,47 @@ func DeprecatedNewBrokerInsideAPIServer(privilegedKubeClientConfig restclient.Co
return b
}

func NewBroker(saKubeClientConfig *restclient.Config, informer templateinformer.TemplateInformer, namespaces []string) (*Broker, error) {
templateNamespaces := map[string]struct{}{}
for _, namespace := range namespaces {
templateNamespaces[namespace] = struct{}{}
}

internalKubeClient, err := kclientset.NewForConfig(saKubeClientConfig)
if err != nil {
return nil, err
}
externalKubeClient, err := kclientsetexternal.NewForConfig(saKubeClientConfig)
if err != nil {
return nil, err
}
extrouteclientset, err := extrouteclientset.NewForConfig(saKubeClientConfig)
if err != nil {
return nil, err
}
templateClient, err := templateclientset.NewForConfig(saKubeClientConfig)
if err != nil {
return nil, err
}

b := &Broker{
kc: internalKubeClient,
extkc: externalKubeClient,
extrouteclient: extrouteclientset,
templateclient: templateClient.Template(),
lister: informer.Lister(),
hasSynced: informer.Informer().HasSynced,
templateNamespaces: templateNamespaces,
ready: make(chan struct{}),
}

// TODO this is an intermediate state. Once we're out of tree, there won't be a ready
// for now, this skips the hassynced on the lister since we'll be managing that as a poststarthook
close(b.ready)

return b, nil
}

// MakeReady actually makes the broker functional
func (b *Broker) MakeReady() error {
select {
Expand Down

0 comments on commit 07ec3d2

Please sign in to comment.