Try to release IP if it has been allocated to current LB

pkg/controller/ippool/controller.go

@@ -15,6 +15,7 @@ import (
 	"github.com/harvester/harvester-load-balancer/pkg/controller/ippool/kubevip"
 	ctllbv1 "github.com/harvester/harvester-load-balancer/pkg/generated/controllers/loadbalancer.harvesterhci.io/v1beta1"
 	"github.com/harvester/harvester-load-balancer/pkg/ipam"
+	"github.com/harvester/harvester-load-balancer/pkg/utils"
 )
 
 const controllerName = "harvester-ipam-controller"
@@ -23,6 +24,7 @@ type Handler struct {
 	ipPoolCache  ctllbv1.IPPoolCache
 	ipPoolClient ctllbv1.IPPoolClient
 	cmClient     ctlcorev1.ConfigMapClient
+	lbCache      ctllbv1.LoadBalancerCache
 
 	allocatorMap           *ipam.SafeAllocatorMap
 	kubevipIPPoolConverter *kubevip.IPPoolConverter
@@ -31,10 +33,12 @@ type Handler struct {
 func Register(ctx context.Context, management *config.Management) error {
 	ipPools := management.LbFactory.Loadbalancer().V1beta1().IPPool()
 	configmaps := management.CoreFactory.Core().V1().ConfigMap()
+	lbCache := management.LbFactory.Loadbalancer().V1beta1().LoadBalancer().Cache()
 
 	handler := &Handler{
 		ipPoolCache:            ipPools.Cache(),
 		ipPoolClient:           ipPools,
+		lbCache:                lbCache,
 		allocatorMap:           management.AllocatorMap,
 		kubevipIPPoolConverter: kubevip.NewIPPoolConverter(configmaps),
 	}
@@ -44,6 +48,7 @@ func Register(ctx context.Context, management *config.Management) error {
 	}
 
 	ipPools.OnChange(ctx, controllerName, handler.OnChange)
+	ipPools.OnChange(ctx, controllerName, handler.OnChangeToReleaseAnIP)
 	ipPools.OnRemove(ctx, controllerName, handler.OnRemove)
 
 	return nil
@@ -152,3 +157,63 @@ func correctAllocatedHistory(pool *lbv1.IPPool) (map[string]string, error) {
 
 	return allocatedHistory, nil
 }
+
+// Due to previous version bug, the LB may be removed but the related IP still exists on pool allocation record
+// This feature gives a way to remove the IP allocation record manually
+func (h *Handler) OnChangeToReleaseAnIP(_ string, ipPool *lbv1.IPPool) (*lbv1.IPPool, error) {
+	if ipPool == nil || ipPool.DeletionTimestamp != nil || ipPool.Annotations == nil {
+		return ipPool, nil
+	}
+
+	ipStr := ipPool.Annotations[utils.AnnotationKeyManuallyReleaseIP]
+	if ipStr == "" {
+		return ipPool, nil
+	}
+
+	// check if it is a valid format, e.g.
+	// loadbalancer.harvesterhci.io/manuallyReleaseIP: "192.168.5.12: default/cluster1-lb-3"
+	ip, namespace, name, err := utils.SplitIPAllocatedString(ipStr)
+	if err != nil {
+		logrus.Infof("IP Pool %s has a manual IP release request %s, it is not valid, skip", ipPool.Name, ipStr)
+		return h.removeAnnotationKeyManuallyReleaseIP(ipPool)
+	}
+
+	// check it is a real allocation record
+	lbStr := ipPool.Status.Allocated[ip]
+	if lbStr == "" || lbStr != fmt.Sprintf("%s/%s", namespace, name) {
+		logrus.Infof("IP Pool %s has a manual IP release request %s, it has been released, skip", ipPool.Name, ipStr)
+		return h.removeAnnotationKeyManuallyReleaseIP(ipPool)
+	}
+
+	// check if the lb is existing
+	_, err = h.lbCache.Get(namespace, name)
+	if err == nil {
+		logrus.Infof("IP Pool %s has a manual IP release request %s, the lb %s/%s is still existing, skip", ipPool.Name, ipStr, namespace, name)
+		return h.removeAnnotationKeyManuallyReleaseIP(ipPool)
+	}
+
+	// return for retry
+	if !apierrors.IsNotFound(err) {
+		return ipPool, err
+	}
+
+	a := h.allocatorMap.Get(ipPool.Name)
+	if a == nil {
+		return ipPool, fmt.Errorf("IP Pool %s has a manual IP release request %s, fail to get allocator", ipPool.Name, ipStr)
+	}
+	err = a.Release(fmt.Sprintf("%s/%s", namespace, name), "")
+	if err != nil {
+		return ipPool, fmt.Errorf("IP Pool %s has a manual IP release request %s, fail to release, error:%w", ipPool.Name, ipStr, err)
+	}
+
+	logrus.Infof("IP Pool %s has a manual IP release request %s, it is successfully released", ipPool.Name, ipStr)
+
+	// note: above a.Release also updates pool internally, following call may fail at first time, as ipPool is stale
+	return h.removeAnnotationKeyManuallyReleaseIP(ipPool)
+}
+
+func (h *Handler) removeAnnotationKeyManuallyReleaseIP(ipPool *lbv1.IPPool) (*lbv1.IPPool, error) {
+	ipPoolCopy := ipPool.DeepCopy()
+	delete(ipPoolCopy.Annotations, utils.AnnotationKeyManuallyReleaseIP)
+	return h.ipPoolClient.Update(ipPoolCopy)
+}

pkg/controller/loadbalancer/controller.go

@@ -6,14 +6,14 @@ import (
 	"fmt"
 	"net"
 	"reflect"
+	"strings"
 	"time"
 
 	ctlcniv1 "github.com/harvester/harvester/pkg/generated/controllers/k8s.cni.cncf.io/v1"
 	ctlkubevirtv1 "github.com/harvester/harvester/pkg/generated/controllers/kubevirt.io/v1"
 	ctlcorev1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
 
-	lb "github.com/harvester/harvester-load-balancer/pkg/apis/loadbalancer.harvesterhci.io"
 	lbv1 "github.com/harvester/harvester-load-balancer/pkg/apis/loadbalancer.harvesterhci.io/v1beta1"
 	"github.com/harvester/harvester-load-balancer/pkg/config"
 	ctldiscoveryv1 "github.com/harvester/harvester-load-balancer/pkg/generated/controllers/discovery.k8s.io/v1"
@@ -26,10 +26,11 @@ import (
 const (
 	controllerName = "harvester-lb-controller"
 
-	AnnotationKeyNetwork   = lb.GroupName + "/network"
-	AnnotationKeyProject   = lb.GroupName + "/project"
-	AnnotationKeyNamespace = lb.GroupName + "/namespace"
-	AnnotationKeyCluster   = lb.GroupName + "/cluster"
+	// referred by cloud-provider-harvester
+	AnnotationKeyNetwork   = utils.AnnotationKeyNetwork
+	AnnotationKeyProject   = utils.AnnotationKeyProject
+	AnnotationKeyNamespace = utils.AnnotationKeyNamespace
+	AnnotationKeyCluster   = utils.AnnotationKeyCluster
 )
 
 var (
@@ -277,6 +278,15 @@ func (h *Handler) ensureAllocatedAddressPool(lbCopy, lb *lbv1.LoadBalancer) (*lb
 		ip, err := h.allocateIPFromPool(lb)
 		if err != nil {
 			logrus.Debugf("lb %s/%s fail to allocate from pool %s", lb.Namespace, lb.Name, err.Error())
+			// if unlucky the DuplicateAllocationKeyWord is reported, try to release IP, do not overwrite original error
+			if strings.Contains(err.Error(), utils.DuplicateAllocationKeyWord) {
+				pool, releaseErr := h.tryReleaseDuplicatedIPToPool(lb)
+				if releaseErr != nil {
+					logrus.Infof("lb %s/%s error: %s, try to release ip to pool %s, error: %s", lb.Namespace, lb.Name, err.Error(), pool, releaseErr.Error())
+				} else {
+					logrus.Infof("lb %s/%s error: %s, try to release ip to pool %s, ok", lb.Namespace, lb.Name, err.Error(), pool)
+				}
+			}
 			return lb, err
 		}
 
@@ -302,6 +312,24 @@ func (h *Handler) allocateIPFromPool(lb *lbv1.LoadBalancer) (*lbv1.AllocatedAddr
 	return h.requestIP(lb, pool)
 }
 
+func (h *Handler) tryReleaseDuplicatedIPToPool(lb *lbv1.LoadBalancer) (string, error) {
+	pool := lb.Spec.IPPool
+	if pool == "" {
+		// match an IP pool automatically if not specified
+		pool, err := h.selectIPPool(lb)
+		if err != nil {
+			return pool, err
+		}
+	}
+
+	// if pool is not ready, just fail and wait
+	a := h.allocatorMap.Get(pool)
+	if a == nil {
+		return pool, fmt.Errorf("fail to get allocator %s", pool)
+	}
+	return pool, a.Release(fmt.Sprintf("%s/%s", lb.Namespace, lb.Name), "")
+}
+
 func (h *Handler) requestIP(lb *lbv1.LoadBalancer, pool string) (*lbv1.AllocatedAddress, error) {
 	allocator := h.allocatorMap.Get(pool)
 	if allocator == nil {
@@ -323,10 +351,10 @@ func (h *Handler) requestIP(lb *lbv1.LoadBalancer, pool string) (*lbv1.Allocated
 
 func (h *Handler) selectIPPool(lb *lbv1.LoadBalancer) (string, error) {
 	r := &ipam.Requirement{
-		Network:   lb.Annotations[AnnotationKeyNetwork],
-		Project:   lb.Annotations[AnnotationKeyProject],
-		Namespace: lb.Annotations[AnnotationKeyNamespace],
-		Cluster:   lb.Annotations[AnnotationKeyCluster],
+		Network:   lb.Annotations[utils.AnnotationKeyNetwork],
+		Project:   lb.Annotations[utils.AnnotationKeyProject],
+		Namespace: lb.Annotations[utils.AnnotationKeyNamespace],
+		Cluster:   lb.Annotations[utils.AnnotationKeyCluster],
 	}
 	if r.Namespace == "" {
 		r.Namespace = lb.Namespace

pkg/utils/const.go

@@ -12,4 +12,9 @@ const (
 	AnnotationKeyProject   = lb.GroupName + "/project"
 	AnnotationKeyNamespace = lb.GroupName + "/namespace"
 	AnnotationKeyCluster   = lb.GroupName + "/cluster"
+
+	// value format: loadbalancer.harvesterhci.io/manuallyReleaseIP: "192.168.5.12: default/cluster1-lb-3"
+	AnnotationKeyManuallyReleaseIP = lb.GroupName + "/manuallyReleaseIP"
+
+	DuplicateAllocationKeyWord = "duplicate allocation is not allowed"
 )

pkg/utils/vid.go

@@ -47,3 +47,24 @@ func GetVid(network string, nadCache ctlcniv1.NetworkAttachmentDefinitionCache)
 	}
 	return netConf.VLAN, nil
 }
+
+// input format: "192.168.5.12: default/cluster1-lb-3"
+func SplitIPAllocatedString(ipStr string) (ip, namespace, name string, err error) {
+	ipStr = strings.Trim(ipStr, " ")
+	fields := strings.Split(ipStr, ":")
+	if len(fields) != 2 {
+		err = fmt.Errorf("%s is not a valid allocation record", ipStr)
+		return
+	}
+	nsnameStr := strings.Trim(fields[1], " ")
+	fields2 := strings.Split(nsnameStr, "/")
+	if len(fields2) != 2 {
+		err = fmt.Errorf("%s is not a valid allocation record", ipStr)
+		return
+	}
+	ip = fields[0]
+	namespace = fields2[0]
+	name = fields2[1]
+	err = nil
+	return
+}