diff --git a/.dockerignore b/.dockerignore index dd8d4b04c..c7934b448 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,7 @@ !config !hack !app +!internal !pkg !sfyra !templates diff --git a/Dockerfile b/Dockerfile index 659ac644d..de2551c58 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,6 +45,7 @@ COPY ./go.sum ./ RUN --mount=type=cache,target=/.cache go mod download RUN --mount=type=cache,target=/.cache go mod verify COPY ./app/ ./app/ +COPY ./internal/ ./internal/ COPY ./hack/ ./hack/ RUN --mount=type=cache,target=/.cache go list -mod=readonly all >/dev/null RUN --mount=type=cache,target=/.cache ! go mod tidy -v 2>&1 | grep . diff --git a/app/cluster-api-provider-sidero/api/v1alpha3/metalmachinetemplate_types.go b/app/cluster-api-provider-sidero/api/v1alpha3/metalmachinetemplate_types.go index 10f199ab5..b768efa23 100644 --- a/app/cluster-api-provider-sidero/api/v1alpha3/metalmachinetemplate_types.go +++ b/app/cluster-api-provider-sidero/api/v1alpha3/metalmachinetemplate_types.go @@ -8,9 +8,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. - // MetalMachineTemplateSpec defines the desired state of MetalMachineTemplate. type MetalMachineTemplateSpec struct { Template MetalMachineTemplateResource `json:"template"` diff --git a/app/metal-controller-manager/api/v1alpha1/environment_types.go b/app/metal-controller-manager/api/v1alpha1/environment_types.go index 250e46b93..a5962deb4 100644 --- a/app/metal-controller-manager/api/v1alpha1/environment_types.go +++ b/app/metal-controller-manager/api/v1alpha1/environment_types.go @@ -8,9 +8,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. - type Asset struct { URL string `json:"url,omitempty"` SHA512 string `json:"sha512,omitempty"` diff --git a/app/metal-controller-manager/api/v1alpha1/server_types.go b/app/metal-controller-manager/api/v1alpha1/server_types.go index 4c8831907..5d25e4044 100644 --- a/app/metal-controller-manager/api/v1alpha1/server_types.go +++ b/app/metal-controller-manager/api/v1alpha1/server_types.go @@ -16,9 +16,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. - // BMC defines data about how to talk to the node via ipmitool. type BMC struct { // BMC endpoint. diff --git a/app/metal-controller-manager/api/v1alpha1/server_types_test.go b/app/metal-controller-manager/api/v1alpha1/server_types_test.go index 9720e0e15..65fdbb739 100644 --- a/app/metal-controller-manager/api/v1alpha1/server_types_test.go +++ b/app/metal-controller-manager/api/v1alpha1/server_types_test.go @@ -58,19 +58,6 @@ func Test_PartialEqual(t *testing.T) { }, want: false, }, - { - name: "partially equal value", - args: args{ - a: v1alpha1.CPUInformation{ - Manufacturer: "QEMU", - }, - b: v1alpha1.CPUInformation{ - Manufacturer: "QEMU", - Version: "1.2.0", - }, - }, - want: true, - }, } for _, tt := range tests { diff --git a/app/metal-controller-manager/api/v1alpha1/serverclass_filter.go b/app/metal-controller-manager/api/v1alpha1/serverclass_filter.go new file mode 100644 index 000000000..c4f422563 --- /dev/null +++ b/app/metal-controller-manager/api/v1alpha1/serverclass_filter.go @@ -0,0 +1,75 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package v1alpha1 + +import "sort" + +// FilterAcceptedServers returns a new slice of Servers that are accepted and qualify. +// +// Returned Servers are always sorted by name for stable results. +func FilterAcceptedServers(servers []Server, q Qualifiers) []Server { + res := make([]Server, 0, len(servers)) + + for _, server := range servers { + // skip non-accepted servers + if !server.Spec.Accepted { + continue + } + + // check CPU qualifiers if they are present + if filters := q.CPU; len(filters) > 0 { + var match bool + + for _, filter := range filters { + if cpu := server.Spec.CPU; cpu != nil && filter.PartialEqual(cpu) { + match = true + break + } + } + + if !match { + continue + } + } + + if filters := q.SystemInformation; len(filters) > 0 { + var match bool + + for _, filter := range filters { + if sysInfo := server.Spec.SystemInformation; sysInfo != nil && filter.PartialEqual(sysInfo) { + match = true + break + } + } + + if !match { + continue + } + } + + if filters := q.LabelSelectors; len(filters) > 0 { + var match bool + + for _, filter := range filters { + for labelKey, labelVal := range filter { + if val, ok := server.ObjectMeta.Labels[labelKey]; ok && labelVal == val { + match = true + break + } + } + } + + if !match { + continue + } + } + + res = append(res, server) + } + + sort.Slice(res, func(i, j int) bool { return res[i].Name < res[j].Name }) + + return res +} diff --git a/app/metal-controller-manager/api/v1alpha1/serverclass_filter_test.go b/app/metal-controller-manager/api/v1alpha1/serverclass_filter_test.go new file mode 100644 index 000000000..985935534 --- /dev/null +++ b/app/metal-controller-manager/api/v1alpha1/serverclass_filter_test.go @@ -0,0 +1,113 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package v1alpha1_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + metalv1alpha1 "github.com/talos-systems/sidero/app/metal-controller-manager/api/v1alpha1" +) + +func TestFilterAcceptedServers(t *testing.T) { + t.Parallel() + + atom := metalv1alpha1.Server{ + Spec: metalv1alpha1.ServerSpec{ + Accepted: true, + CPU: &metalv1alpha1.CPUInformation{ + Manufacturer: "Intel(R) Corporation", + Version: "Intel(R) Atom(TM) CPU C3558 @ 2.20GHz", + }, + }, + } + ryzen := metalv1alpha1.Server{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "my-server-label": "true", + }, + }, + Spec: metalv1alpha1.ServerSpec{ + Accepted: true, + CPU: &metalv1alpha1.CPUInformation{ + Manufacturer: "Advanced Micro Devices, Inc.", + Version: "AMD Ryzen 7 2700X Eight-Core Processor", + }, + SystemInformation: &metalv1alpha1.SystemInformation{ + Manufacturer: "QEMU", + }, + }, + } + notAccepted := metalv1alpha1.Server{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "my-server-label": "true", + }, + }, + Spec: metalv1alpha1.ServerSpec{ + Accepted: false, + CPU: &metalv1alpha1.CPUInformation{ + Manufacturer: "Advanced Micro Devices, Inc.", + Version: "AMD Ryzen 7 2700X Eight-Core Processor", + }, + SystemInformation: &metalv1alpha1.SystemInformation{ + Manufacturer: "QEMU", + }, + }, + } + + servers := []metalv1alpha1.Server{atom, ryzen, notAccepted} + + testdata := map[string]struct { + q metalv1alpha1.Qualifiers + expected []metalv1alpha1.Server + }{ + "Intel only": { + q: metalv1alpha1.Qualifiers{ + CPU: []metalv1alpha1.CPUInformation{ + { + Manufacturer: "Intel(R) Corporation", + }, + }, + }, + expected: []metalv1alpha1.Server{atom}, + }, + "QEMU only": { + q: metalv1alpha1.Qualifiers{ + SystemInformation: []metalv1alpha1.SystemInformation{ + { + Manufacturer: "QEMU", + }, + }, + }, + expected: []metalv1alpha1.Server{ryzen}, + }, + "with label": { + q: metalv1alpha1.Qualifiers{ + LabelSelectors: []map[string]string{ + { + "my-server-label": "true", + }, + }, + }, + expected: []metalv1alpha1.Server{ryzen}, + }, + metalv1alpha1.ServerClassAny: { + expected: []metalv1alpha1.Server{atom, ryzen}, + }, + } + + for name, td := range testdata { + name, td := name, td + t.Run(name, func(t *testing.T) { + t.Parallel() + + actual := metalv1alpha1.FilterAcceptedServers(servers, td.q) + assert.Equal(t, actual, td.expected) + }) + } +} diff --git a/app/metal-controller-manager/api/v1alpha1/serverclass_types.go b/app/metal-controller-manager/api/v1alpha1/serverclass_types.go index ddebb6ce3..9023d50a6 100644 --- a/app/metal-controller-manager/api/v1alpha1/serverclass_types.go +++ b/app/metal-controller-manager/api/v1alpha1/serverclass_types.go @@ -9,6 +9,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// ServerClassAny is an automatically created ServerClass that includes all Servers. +const ServerClassAny = "any" + type Qualifiers struct { CPU []CPUInformation `json:"cpu,omitempty"` SystemInformation []SystemInformation `json:"systemInformation,omitempty"` diff --git a/app/metal-controller-manager/api/v1alpha1/v1alpha1_test.go b/app/metal-controller-manager/api/v1alpha1/v1alpha1_test.go deleted file mode 100644 index 795266ba7..000000000 --- a/app/metal-controller-manager/api/v1alpha1/v1alpha1_test.go +++ /dev/null @@ -1,14 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -package v1alpha1_test - -import "testing" - -func TestEmpty(t *testing.T) { - // added for accurate coverage estimation - // - // please remove it once any unit-test is added - // for this package -} diff --git a/app/metal-controller-manager/controllers/serverclass_controller.go b/app/metal-controller-manager/controllers/serverclass_controller.go index 48d01f347..a97c35620 100644 --- a/app/metal-controller-manager/controllers/serverclass_controller.go +++ b/app/metal-controller-manager/controllers/serverclass_controller.go @@ -7,9 +7,9 @@ package controllers import ( "context" "fmt" - "sort" "github.com/go-logr/logr" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/cluster-api/util/patch" @@ -30,113 +30,6 @@ type ServerClassReconciler struct { Scheme *runtime.Scheme } -type serverFilter interface { - filterCPU([]metalv1alpha1.CPUInformation) serverFilter - filterSysInfo([]metalv1alpha1.SystemInformation) serverFilter - filterLabels([]map[string]string) serverFilter - fetchItems() map[string]metalv1alpha1.Server -} - -type serverResults struct { - items map[string]metalv1alpha1.Server -} - -func newServerFilter(sl *metalv1alpha1.ServerList) serverFilter { - newSF := &serverResults{ - items: make(map[string]metalv1alpha1.Server), - } - - // Add all servers to serverclass, but only if they've been accepted first - for _, server := range sl.Items { - if server.Spec.Accepted { - newSF.items[server.Name] = server - } - } - - return newSF -} - -func (sr *serverResults) filterCPU(filters []metalv1alpha1.CPUInformation) serverFilter { - if len(filters) == 0 { - return sr - } - - for _, server := range sr.items { - var match bool - - for _, cpu := range filters { - if server.Spec.CPU != nil && cpu.PartialEqual(server.Spec.CPU) { - match = true - - break - } - } - - if !match { - // Remove from results list if it's there since it's not a match for this qualifier - delete(sr.items, server.ObjectMeta.Name) - } - } - - return sr -} - -func (sr *serverResults) filterSysInfo(filters []metalv1alpha1.SystemInformation) serverFilter { - if len(filters) == 0 { - return sr - } - - for _, server := range sr.items { - var match bool - - for _, sysInfo := range filters { - if server.Spec.SystemInformation != nil && sysInfo.PartialEqual(server.Spec.SystemInformation) { - match = true - break - } - } - - if !match { - // Remove from results list if it's there since it's not a match for this qualifier - delete(sr.items, server.ObjectMeta.Name) - } - } - - return sr -} - -func (sr *serverResults) filterLabels(filters []map[string]string) serverFilter { - if len(filters) == 0 { - return sr - } - - for _, server := range sr.items { - var match bool - - for _, label := range filters { - for labelKey, labelVal := range label { - if val, ok := server.ObjectMeta.Labels[labelKey]; ok { - if labelVal == val { - match = true - break - } - } - } - } - - if !match { - // Remove from results list if it's there since it's not a match for this qualifier - delete(sr.items, server.ObjectMeta.Name) - } - } - - return sr -} - -func (sr *serverResults) fetchItems() map[string]metalv1alpha1.Server { - return sr.items -} - // +kubebuilder:rbac:groups=metal.sidero.dev,resources=serverclasses,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=metal.sidero.dev,resources=serverclasses/status,verbs=get;update;patch // +kubebuilder:rbac:groups=metal.sidero.dev,resources=servers,verbs=get;list;watch;create;update;patch;delete @@ -144,13 +37,25 @@ func (sr *serverResults) fetchItems() map[string]metalv1alpha1.Server { func (r *ServerClassReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { ctx := context.Background() + l := r.Log.WithValues("serverclass", req.NamespacedName) + l.Info("reconciling") + + //nolint:godox + // TODO: We probably should use admission webhooks instead (or in additional) to prevent + // unwanted edits instead of "fixing" the resource after the fact. + if req.Name == metalv1alpha1.ServerClassAny { + if err := ReconcileServerClassAny(ctx, r.Client); err != nil { + return ctrl.Result{}, err + } - l.Info("fetching serverclass", "serverclass", req.NamespacedName) + // do not return; re-reconcile it to update status + } //nolint:wsl sc := metalv1alpha1.ServerClass{} if err := r.Get(ctx, req.NamespacedName, &sc); err != nil { + l.Error(err, "failed fetching resource") return ctrl.Result{}, client.IgnoreNotFound(err) } @@ -165,18 +70,12 @@ func (r *ServerClassReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) return ctrl.Result{}, fmt.Errorf("unable to get serverclass: %w", err) } - // Create serverResults struct and seed items with all known, accepted servers - results := newServerFilter(sl) - - // Filter servers down based on qualifiers - results = results.filterCPU(sc.Spec.Qualifiers.CPU) - results = results.filterSysInfo(sc.Spec.Qualifiers.SystemInformation) - results = results.filterLabels(sc.Spec.Qualifiers.LabelSelectors) + results := metalv1alpha1.FilterAcceptedServers(sl.Items, sc.Spec.Qualifiers) avail := []string{} used := []string{} - for _, server := range results.fetchItems() { + for _, server := range results { if server.Status.InUse { used = append(used, server.Name) continue @@ -185,10 +84,6 @@ func (r *ServerClassReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) avail = append(avail, server.Name) } - // sort lists to avoid spurious updates due to `map` key ordering - sort.Strings(avail) - sort.Strings(used) - sc.Status.ServersAvailable = avail sc.Status.ServersInUse = used @@ -199,6 +94,36 @@ func (r *ServerClassReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) return ctrl.Result{}, nil } +// ReconcileServerClassAny ensures that ServerClass "any" exist and is in desired state. +func ReconcileServerClassAny(ctx context.Context, c client.Client) error { + key := types.NamespacedName{ + Name: metalv1alpha1.ServerClassAny, + } + + sc := metalv1alpha1.ServerClass{} + err := c.Get(ctx, key, &sc) + + switch { + case apierrors.IsNotFound(err): + sc.Name = metalv1alpha1.ServerClassAny + + return c.Create(ctx, &sc) + + case err == nil: + patchHelper, err := patch.NewHelper(&sc, c) + if err != nil { + return err + } + + sc.Spec.Qualifiers = metalv1alpha1.Qualifiers{} + + return patchHelper.Patch(ctx, &sc) + + default: + return err + } +} + func (r *ServerClassReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error { // This mapRequests handler allows us to add a watch on server resources. Upon a server resource update, // we will dump all server classes and issue a reconcile against them so that they will get updated statuses diff --git a/app/metal-controller-manager/main.go b/app/metal-controller-manager/main.go index 773431cf6..c58b7ee51 100644 --- a/app/metal-controller-manager/main.go +++ b/app/metal-controller-manager/main.go @@ -5,6 +5,7 @@ package main import ( + "context" "flag" "fmt" "os" @@ -29,6 +30,7 @@ import ( "github.com/talos-systems/sidero/app/metal-controller-manager/internal/server" "github.com/talos-systems/sidero/app/metal-controller-manager/internal/tftp" "github.com/talos-systems/sidero/app/metal-controller-manager/pkg/constants" + "github.com/talos-systems/sidero/internal/client" // +kubebuilder:scaffold:imports ) @@ -193,6 +195,17 @@ func main() { } }() + k8sClient, err := client.NewClient(nil) + if err != nil { + setupLog.Error(err, `failed to create k8s client`) + os.Exit(1) + } + + if err = controllers.ReconcileServerClassAny(context.TODO(), k8sClient); err != nil { + setupLog.Error(err, `failed to reconcile ServerClass "any"`) + os.Exit(1) + } + setupLog.Info("starting manager") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { diff --git a/app/metal-metadata-server/main.go b/app/metal-metadata-server/main.go index 66cc631b0..0513ba989 100644 --- a/app/metal-metadata-server/main.go +++ b/app/metal-metadata-server/main.go @@ -24,7 +24,7 @@ import ( "github.com/talos-systems/sidero/app/cluster-api-provider-sidero/api/v1alpha3" metalv1alpha1 "github.com/talos-systems/sidero/app/metal-controller-manager/api/v1alpha1" - "github.com/talos-systems/sidero/app/metal-metadata-server/pkg/client" + "github.com/talos-systems/sidero/internal/client" ) var ( diff --git a/docs/website/content/docs/v0.3/Configuration/serverclasses.md b/docs/website/content/docs/v0.3/Configuration/serverclasses.md index f605aaa02..97d8e0593 100644 --- a/docs/website/content/docs/v0.3/Configuration/serverclasses.md +++ b/docs/website/content/docs/v0.3/Configuration/serverclasses.md @@ -18,7 +18,7 @@ An example: apiVersion: metal.sidero.dev/v1alpha1 kind: ServerClass metadata: - name: default + name: example spec: qualifiers: cpu: @@ -31,3 +31,5 @@ spec: ``` Servers would only be added to the above class if they had _EITHER_ CPU info, _AND_ the label associated with the server resource. + +Additionally, Sidero automatically creates and maintains a server class called `"any"` that includes all (accepted) servers. Attempts to add qualifiers to it will be reverted. diff --git a/docs/website/content/docs/v0.3/Guides/bootstrapping.md b/docs/website/content/docs/v0.3/Guides/bootstrapping.md index 57b75a784..0f9789bf6 100644 --- a/docs/website/content/docs/v0.3/Guides/bootstrapping.md +++ b/docs/website/content/docs/v0.3/Guides/bootstrapping.md @@ -191,44 +191,6 @@ spec: EOF ``` -## Create Server Class - -We must now create a server class to wrap our servers we registered. -This is necessary for using the Talos control plane provider for Cluster API. -The qualifiers needed for your server class will differ based on the data provided by your registration flow. -See the [server class docs](/docs/v0.3/configuration/serverclasses) for more info on how these work. - -Here is an example of how to apply the server class once you have the proper info: - -```bash -cat < +export TAG=v0.1.0 +export REGISTRY_MIRROR_FLAGS="--registry-mirror docker.io=http://172.24.0.1:5000,k8s.gcr.io=http://172.24.0.1:5001,quay.io=http://172.24.0.1:5002,gcr.io=http://172.24.0.1:5003,ghcr.io=http://172.24.0.1:5004,127.0.0.1:5005=http://172.24.0.1:5005" +export SFYRA_EXTRA_FLAGS="--skip-teardown" +make run-sfyra +``` + +With `--skip-teardown` flag test leaves the bootstrap cluster running so that next iteration of the test can be run without waiting for the boostrap actions to be finished. It's possible to run Sfyra tests once again without bringing down the test environment, but make sure that all the clusters are deleted with `kubectl delete clusters --all`. @@ -39,7 +49,7 @@ again without bringing down the test environment, but make sure that all the clu Flag `--registry-mirror` is optional, but it speeds up provisioning significantly. See Talos guides on setting up registry pull-through caches, or just run `hack/start-registry-proxies.sh`. -Kubernetes config can be pulled with `talosconfig -n 172.24.0.2 kubeconfig --force`. +Kubernetes config can be pulled with `talosctl -n 172.24.0.2 kubeconfig --force`. When `sfyra` is not running, loadbalancer for `management-cluster` control plane is also down, it can be restarted for manual testing with `_out/sfyra loadbalancer create --kubeconfig=$HOME/.kube/config --load-balancer-port 10000`. diff --git a/sfyra/pkg/tests/server_class.go b/sfyra/pkg/tests/server_class.go index adcf9e88e..ff5e8bdea 100644 --- a/sfyra/pkg/tests/server_class.go +++ b/sfyra/pkg/tests/server_class.go @@ -43,6 +43,18 @@ const ( workloadServerClassName = "workload" ) +func TestServerClassAny(ctx context.Context, metalClient client.Client, vmSet *vm.Set) TestFunc { + return func(t *testing.T) { + var serverClass v1alpha1.ServerClass + err := metalClient.Get(ctx, types.NamespacedName{Name: v1alpha1.ServerClassAny}, &serverClass) + require.NoError(t, err) + assert.Empty(t, serverClass.Spec.Qualifiers) + + numNodes := len(vmSet.Nodes()) + assert.Len(t, append(serverClass.Status.ServersAvailable, serverClass.Status.ServersInUse...), numNodes) + } +} + // TestServerClassDefault verifies server class creation. func TestServerClassDefault(ctx context.Context, metalClient client.Client, vmSet *vm.Set) TestFunc { return func(t *testing.T) { diff --git a/sfyra/pkg/tests/tests.go b/sfyra/pkg/tests/tests.go index 72ea48fbf..a680489d4 100644 --- a/sfyra/pkg/tests/tests.go +++ b/sfyra/pkg/tests/tests.go @@ -70,6 +70,10 @@ func Run(ctx context.Context, cluster talos.Cluster, vmSet *vm.Set, capiManager "TestEnvironmentDefault", TestEnvironmentDefault(ctx, metalClient, cluster, options.KernelURL, options.InitrdURL), }, + { + "TestServerClassAny", + TestServerClassAny(ctx, metalClient, vmSet), + }, { "TestServerClassDefault", TestServerClassDefault(ctx, metalClient, vmSet),