update retry strategy

mass-deployer/pkg/mass-deployer/deployer.go

@@ -10,6 +10,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/hashicorp/go-multierror"
 	"github.com/sethvargo/go-retry"
 	"gopkg.in/yaml.v3"
 
@@ -38,25 +39,31 @@ func RunDeployer(ctx context.Context, cfg Config, output string, debug bool) err
 
 	for _, nodeGroup := range cfg.NodeGroups {
 		log.Info().Str("Node group", nodeGroup.Name).Msg("Running deployment")
+		var groupDeployments groupDeploymentsInfo
 		trial := 1
 
 		if err := retry.Do(ctx, retry.WithMaxRetries(cfg.MaxRetries, retry.NewConstant(1*time.Nanosecond)), func(ctx context.Context) error {
 			if trial != 1 {
 				log.Info().Str("Node group", nodeGroup.Name).Int("Deployment trial", trial).Msg("Retrying to deploy")
 			}
 
-			info, err := deployNodeGroup(ctx, tfPluginClient, nodeGroup, cfg.Vms, cfg.SSHKeys)
+			info, err := deployNodeGroup(ctx, tfPluginClient, &groupDeployments, nodeGroup, cfg.Vms, cfg.SSHKeys)
 			if err != nil {
 				trial++
 				log.Debug().Err(err).Str("Node group", nodeGroup.Name).Msg("failed to deploy")
 				return retry.RetryableError(err)
 			}
 
-			passedGroups[nodeGroup.Name] = info
 			log.Info().Str("Node group", nodeGroup.Name).Msg("Done deploying")
+			passedGroups[nodeGroup.Name] = info
 			return nil
 		}); err != nil {
+
 			failedGroups[nodeGroup.Name] = err.Error()
+			err := tfPluginClient.CancelByProjectName(nodeGroup.Name)
+			if err != nil {
+				log.Debug().Err(err).Send()
+			}
 		}
 	}
 
@@ -89,24 +96,32 @@ func RunDeployer(ctx context.Context, cfg Config, output string, debug bool) err
 	return os.WriteFile(output, outputBytes, 0644)
 }
 
-func deployNodeGroup(ctx context.Context, tfPluginClient deployer.TFPluginClient, nodeGroup NodesGroup, vms []Vms, sshKeys map[string]string) ([]vmOutput, error) {
+func deployNodeGroup(ctx context.Context, tfPluginClient deployer.TFPluginClient, groupDeployments *groupDeploymentsInfo, nodeGroup NodesGroup, vms []Vms, sshKeys map[string]string) ([]vmOutput, error) {
 	log.Info().Str("Node group", nodeGroup.Name).Msg("Filter nodes")
 	nodesIDs, err := filterNodes(ctx, tfPluginClient, nodeGroup)
 	if err != nil {
 		return nil, err
 	}
 	log.Debug().Ints("nodes IDs", nodesIDs).Send()
 
-	log.Debug().Str("Node group", nodeGroup.Name).Msg("Parsing vms group")
-	groupsDeployments := parseVMsGroup(vms, nodeGroup.Name, nodesIDs, sshKeys)
+	if groupDeployments.networkDeployments == nil {
+		log.Debug().Str("Node group", nodeGroup.Name).Msg("Parsing vms group")
+		*groupDeployments = parseVMsGroup(vms, nodeGroup.Name, nodesIDs, sshKeys)
+	} else {
+		log.Debug().Str("Node group", nodeGroup.Name).Msg("Updating vms group")
+		updateFailedDeployments(tfPluginClient, nodesIDs, groupDeployments)
+	}
 
 	log.Info().Str("Node group", nodeGroup.Name).Msg("Starting mass deployment")
-	info, err := massDeploy(ctx, tfPluginClient, groupsDeployments)
+	err = massDeploy(ctx, tfPluginClient, groupDeployments)
 	if err != nil {
 		return nil, err
 	}
 
-	return info, nil
+	log.Debug().Msg("Load deployments")
+	vmsInfo, err := loadGroupInfo(tfPluginClient, groupDeployments.vmDeployments)
+
+	return vmsInfo, err
 }
 
 func parseVMsGroup(vms []Vms, nodeGroup string, nodesIDs []int, sshKeys map[string]string) groupDeploymentsInfo {
@@ -121,31 +136,26 @@ func parseVMsGroup(vms []Vms, nodeGroup string, nodesIDs []int, sshKeys map[stri
 	return buildDeployments(vmsOfNodeGroup, nodeGroup, nodesIDs, sshKeys)
 }
 
-func massDeploy(ctx context.Context, tfPluginClient deployer.TFPluginClient, deployments groupDeploymentsInfo) ([]vmOutput, error) {
+func massDeploy(ctx context.Context, tfPluginClient deployer.TFPluginClient, deployments *groupDeploymentsInfo) error {
+	// deploy only contracts that need to be deployed
+	networks, vms := getNotDeployedDeployments(tfPluginClient, deployments)
+
 	log.Debug().Msg("Deploy networks")
-	err := tfPluginClient.NetworkDeployer.BatchDeploy(ctx, deployments.networkDeployments)
-	if err != nil {
-		cancelContractsOfFailedDeployments(tfPluginClient, deployments.networkDeployments, []*workloads.Deployment{})
-		return nil, err
+	if err := tfPluginClient.NetworkDeployer.BatchDeploy(ctx, networks); err != nil {
+		return err
 	}
 
 	log.Debug().Msg("Deploy virtual machines")
-	err = tfPluginClient.DeploymentDeployer.BatchDeploy(ctx, deployments.vmDeployments)
-	if err != nil {
-		cancelContractsOfFailedDeployments(tfPluginClient, deployments.networkDeployments, deployments.vmDeployments)
-		return nil, err
+	if err := tfPluginClient.DeploymentDeployer.BatchDeploy(ctx, vms); err != nil {
+		return err
 	}
 
-	log.Debug().Msg("Load deployments")
-	vmsInfo := loadDeploymentsInfo(tfPluginClient, deployments.deploymentsInfo)
-
-	return vmsInfo, nil
+	return nil
 }
 
 func buildDeployments(vms []Vms, nodeGroup string, nodesIDs []int, sshKeys map[string]string) groupDeploymentsInfo {
 	var vmDeployments []*workloads.Deployment
 	var networkDeployments []*workloads.ZNet
-	var deploymentsInfo []vmDeploymentInfo
 	var nodesIDsIdx int
 
 	// here we loop over all groups of vms within the same node group, and for every group
@@ -200,51 +210,50 @@ func buildDeployments(vms []Vms, nodeGroup string, nodesIDs []int, sshKeys map[s
 
 			vmDeployments = append(vmDeployments, &deployment)
 			networkDeployments = append(networkDeployments, &network)
-			deploymentsInfo = append(deploymentsInfo, vmDeploymentInfo{nodeID: nodeID, deploymentName: deployment.Name, vmName: vm.Name})
 		}
 	}
-	return groupDeploymentsInfo{vmDeployments: vmDeployments, networkDeployments: networkDeployments, deploymentsInfo: deploymentsInfo}
+	return groupDeploymentsInfo{vmDeployments: vmDeployments, networkDeployments: networkDeployments}
 }
 
-func cancelContractsOfFailedDeployments(tfPluginClient deployer.TFPluginClient, networkDeployments []*workloads.ZNet, vmDeployments []*workloads.Deployment) {
-	contracts := []uint64{}
-	for _, network := range networkDeployments {
-		for _, contract := range network.NodeDeploymentID {
-			if contract != 0 {
-				contracts = append(contracts, contract)
-			}
+func getNotDeployedDeployments(tfPluginClient deployer.TFPluginClient, groupDeployments *groupDeploymentsInfo) ([]*workloads.ZNet, []*workloads.Deployment) {
+	var failedVmDeployments []*workloads.Deployment
+	var failedNetworkDeployments []*workloads.ZNet
+
+	for i := range groupDeployments.networkDeployments {
+		if isFailedNetwork(*groupDeployments.networkDeployments[i]) {
+			failedNetworkDeployments = append(failedNetworkDeployments, groupDeployments.networkDeployments[i])
 		}
-	}
 
-	for _, vm := range vmDeployments {
-		if vm.ContractID != 0 {
-			contracts = append(contracts, vm.ContractID)
+		if groupDeployments.vmDeployments[i].ContractID == 0 {
+			failedVmDeployments = append(failedVmDeployments, groupDeployments.vmDeployments[i])
 		}
-	}
 
-	err := tfPluginClient.BatchCancelContract(contracts)
-	if err != nil {
-		log.Debug().Err(err)
 	}
+
+	return failedNetworkDeployments, failedVmDeployments
 }
 
-func loadDeploymentsInfo(tfPluginClient deployer.TFPluginClient, deployments []vmDeploymentInfo) []vmOutput {
+func loadGroupInfo(tfPluginClient deployer.TFPluginClient, vmDeployments []*workloads.Deployment) ([]vmOutput, error) {
 	vmsInfo := []vmOutput{}
+	var multiErr error
 	var lock sync.Mutex
 	var wg sync.WaitGroup
 
-	for _, info := range deployments {
+	for _, deployment := range vmDeployments {
 		wg.Add(1)
 
-		go func(depInfo vmDeploymentInfo) {
+		go func(deployment workloads.Deployment) {
 			defer wg.Done()
 
-			vmDeployment, err := tfPluginClient.State.LoadDeploymentFromGrid(depInfo.nodeID, depInfo.deploymentName)
+			vmDeployment, err := tfPluginClient.State.LoadDeploymentFromGrid(deployment.NodeID, deployment.Name)
 			if err != nil {
+				lock.Lock()
+				defer lock.Unlock()
+				multiErr = multierror.Append(multiErr, err)
 				log.Debug().Err(err).
-					Str("vm", depInfo.vmName).
-					Str("deployment", depInfo.deploymentName).
-					Uint32("node ID", depInfo.nodeID).
+					Str("vm", deployment.Vms[0].Name).
+					Str("deployment", deployment.Name).
+					Uint32("node ID", deployment.NodeID).
 					Msg("couldn't load from state")
 				return
 			}
@@ -265,11 +274,11 @@ func loadDeploymentsInfo(tfPluginClient deployer.TFPluginClient, deployments []v
 			lock.Lock()
 			defer lock.Unlock()
 			vmsInfo = append(vmsInfo, vmInfo)
-		}(info)
+		}(*deployment)
 	}
-
 	wg.Wait()
-	return vmsInfo
+
+	return vmsInfo, multiErr
 }
 
 func parseDisks(name string, disks []Disk) (disksWorkloads []workloads.Disk, mountsWorkloads []workloads.Mount) {
@@ -284,3 +293,56 @@ func parseDisks(name string, disks []Disk) (disksWorkloads []workloads.Disk, mou
 	}
 	return
 }
+
+func updateFailedDeployments(tfPluginClient deployer.TFPluginClient, nodesIDs []int, groupDeployments *groupDeploymentsInfo) {
+	// update nodes for failed networks and update vm deployments accordingly
+	// return if the networks were updated
+	var updated bool
+	for idx, deployment := range groupDeployments.networkDeployments {
+
+		nodeID := uint32(nodesIDs[idx%len(nodesIDs)])
+		if isFailedNetwork(*deployment) {
+			updated = true
+			groupDeployments.vmDeployments[idx].NodeID = nodeID
+			groupDeployments.networkDeployments[idx].Nodes = []uint32{nodeID}
+		}
+	}
+
+	if updated {
+		return
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	for idx, deployment := range groupDeployments.vmDeployments {
+		if deployment.ContractID == 0 {
+			for _, contract := range groupDeployments.networkDeployments[idx].NodeDeploymentID {
+				if contract != 0 {
+					err := tfPluginClient.NetworkDeployer.Cancel(ctx, groupDeployments.networkDeployments[idx])
+					if err != nil {
+						log.Debug().Err(err)
+					}
+					break
+				}
+			}
+
+			nodeID := uint32(nodesIDs[idx%len(nodesIDs)])
+			groupDeployments.vmDeployments[idx].NodeID = nodeID
+			groupDeployments.networkDeployments[idx].Nodes = []uint32{nodeID}
+		}
+	}
+}
+
+func isFailedNetwork(network workloads.ZNet) bool {
+	if len(network.NodeDeploymentID) == 0 {
+		return true
+	} else {
+		for _, contract := range network.NodeDeploymentID {
+			if contract == 0 {
+				return true
+			}
+		}
+	}
+	return false
+}

mass-deployer/pkg/mass-deployer/types.go

@@ -52,13 +52,6 @@ type Disk struct {
 type groupDeploymentsInfo struct {
 	vmDeployments      []*workloads.Deployment
 	networkDeployments []*workloads.ZNet
-	deploymentsInfo    []vmDeploymentInfo
-}
-
-type vmDeploymentInfo struct {
-	nodeID         uint32
-	vmName         string
-	deploymentName string
 }
 
 type vmOutput struct {