Paginate list API calls

Calls to Terraform Cloud API endpoints that are paginated should handle
that pagination.

.changes/unreleased/BUG FIXES-426-20240606-184719.yaml

@@ -0,0 +1,5 @@
+kind: BUG FIXES
+body: '`Project`: Fix an issue where calls to paginated API endpoints were only fetching the first page of results.'
+time: 2024-06-06T18:47:19.418960132-05:00
+custom:
+    PR: "426"

.changes/unreleased/BUG FIXES-426-20240606-184749.yaml

@@ -0,0 +1,5 @@
+kind: BUG FIXES
+body: '`AgentPool`: Fix an issue where calls to paginated API endpoints were only fetching the first page of results.'
+time: 2024-06-06T18:47:49.567157973-05:00
+custom:
+    PR: "426"

.changes/unreleased/BUG FIXES-426-20240606-184753.yaml

@@ -0,0 +1,5 @@
+kind: BUG FIXES
+body: '`Workspace`: Fix an issue where calls to paginated API endpoints were only fetching the first page of results.'
+time: 2024-06-06T18:47:53.667721493-05:00
+custom:
+    PR: "426"

controllers/agentpool_controller_autoscaling.go

@@ -18,20 +18,28 @@ import (
 	appv1alpha2 "github.com/hashicorp/terraform-cloud-operator/api/v1alpha2"
 )
 
-func computeRequiredAgentsForWorkspace(ctx context.Context, ap *agentPoolInstance, workspaceID string) (int, error) {
+func computeRequiredAgentsForWorkspace(ctx context.Context, ap *agentPoolInstance, workspaceID string) (agents int, err error) {
 	statuses := []string{
 		string(tfc.RunPlanQueued),
 		string(tfc.RunApplyQueued),
 		string(tfc.RunApplying),
 		string(tfc.RunPlanning),
 	}
-	runs, err := ap.tfClient.Client.Runs.List(ctx, workspaceID, &tfc.RunListOptions{
+	listOpts := &tfc.RunListOptions{
 		Status: strings.Join(statuses, ","),
-	})
-	if err != nil {
-		return 0, err
 	}
-	return len(runs.Items), nil
+	for {
+		runs, err := ap.tfClient.Client.Runs.List(ctx, workspaceID, listOpts)
+		if err != nil {
+			return 0, err
+		}
+		agents += len(runs.Items)
+		if runs.NextPage == 0 {
+			break
+		}
+		listOpts.PageNumber = runs.NextPage
+	}
+	return
 }
 
 func getAllAgentPoolWorkspaceIDs(ctx context.Context, ap *agentPoolInstance) ([]string, error) {
@@ -80,30 +88,33 @@ func getTargetWorkspaceID(ctx context.Context, ap *agentPoolInstance, targetWork
 	if targetWorkspace.ID != "" {
 		return targetWorkspace.ID, nil
 	}
-	list, err := ap.tfClient.Client.Workspaces.List(ctx, ap.instance.Spec.Organization, &tfc.WorkspaceListOptions{
-		Search: targetWorkspace.Name,
-	})
+	ws, err := ap.tfClient.Client.Workspaces.Read(ctx, ap.instance.Spec.Organization, targetWorkspace.Name)
+	if err == tfc.ErrResourceNotFound {
+		return "", fmt.Errorf("no such workspace found %q", targetWorkspace.Name)
+	}
 	if err != nil {
 		return "", err
 	}
-	for _, w := range list.Items {
-		if w.Name == targetWorkspace.Name {
-			return w.ID, nil
-		}
-	}
-	return "", fmt.Errorf("no such workspace found %q", targetWorkspace.Name)
+	return ws.ID, nil
 }
 
 func getTargetWorkspaceIDsByWildcardName(ctx context.Context, ap *agentPoolInstance, targetWorkspace appv1alpha2.TargetWorkspace) ([]string, error) {
-	list, err := ap.tfClient.Client.Workspaces.List(ctx, ap.instance.Spec.Organization, &tfc.WorkspaceListOptions{
+	listOpts := &tfc.WorkspaceListOptions{
 		WildcardName: targetWorkspace.WildcardName,
-	})
-	if err != nil {
-		return []string{}, err
 	}
 	workspaceIDs := []string{}
-	for _, w := range list.Items {
-		workspaceIDs = append(workspaceIDs, w.ID)
+	for {
+		list, err := ap.tfClient.Client.Workspaces.List(ctx, ap.instance.Spec.Organization, listOpts)
+		if err != nil {
+			return []string{}, err
+		}
+		for _, w := range list.Items {
+			workspaceIDs = append(workspaceIDs, w.ID)
+		}
+		if list.NextPage == 0 {
+			break
+		}
+		listOpts.PageNumber = list.NextPage
 	}
 	return workspaceIDs, nil
 }

controllers/project_controller_team_access.go

@@ -64,13 +64,19 @@ func (r *ProjectReconciler) getInstanceTeamAccess(ctx context.Context, p *projec
 func (r *ProjectReconciler) getWorkspaceTeamAccess(ctx context.Context, p *projectInstance) (map[string]*tfc.TeamProjectAccess, error) {
 	o := map[string]*tfc.TeamProjectAccess{}
 
-	t, err := p.tfClient.Client.TeamProjectAccess.List(ctx, tfc.TeamProjectAccessListOptions{ProjectID: p.instance.Status.ID})
-	if err != nil {
-		return o, err
-	}
-
-	for _, ta := range t.Items {
-		o[ta.Team.ID] = ta
+	listOpts := tfc.TeamProjectAccessListOptions{ProjectID: p.instance.Status.ID}
+	for {
+		t, err := p.tfClient.Client.TeamProjectAccess.List(ctx, listOpts)
+		if err != nil {
+			return o, err
+		}
+		for _, ta := range t.Items {
+			o[ta.Team.ID] = ta
+		}
+		if t.NextPage == 0 {
+			break
+		}
+		listOpts.PageNumber = t.NextPage
 	}
 	return o, nil
 }
@@ -85,15 +91,21 @@ func (r *ProjectReconciler) getTeams(ctx context.Context, p *projectInstance) (m
 		}
 	}
 
-	tl, err := p.tfClient.Client.Teams.List(ctx, p.instance.Spec.Organization, &tfc.TeamListOptions{
+	listOpts := &tfc.TeamListOptions{
 		Names: fTeams,
-	})
-	if err != nil {
-		return teams, err
 	}
-
-	for _, t := range tl.Items {
-		teams[t.Name] = t
+	for {
+		tl, err := p.tfClient.Client.Teams.List(ctx, p.instance.Spec.Organization, listOpts)
+		if err != nil {
+			return teams, err
+		}
+		for _, t := range tl.Items {
+			teams[t.Name] = t
+		}
+		if tl.NextPage == 0 {
+			break
+		}
+		listOpts.PageNumber = tl.NextPage
 	}
 
 	return teams, nil

controllers/workspace_controller_agents.go

@@ -13,17 +13,23 @@ import (
 func (r *WorkspaceReconciler) getAgentPoolIDByName(ctx context.Context, w *workspaceInstance) (string, error) {
 	agentPoolName := w.instance.Spec.AgentPool.Name
 
-	agentPoolIDs, err := w.tfClient.Client.AgentPools.List(ctx, w.instance.Spec.Organization, &tfc.AgentPoolListOptions{
+	listOpts := &tfc.AgentPoolListOptions{
 		Query: agentPoolName,
-	})
-	if err != nil {
-		return "", err
 	}
-
-	for _, a := range agentPoolIDs.Items {
-		if a.Name == agentPoolName {
-			return a.ID, nil
+	for {
+		agentPoolIDs, err := w.tfClient.Client.AgentPools.List(ctx, w.instance.Spec.Organization, listOpts)
+		if err != nil {
+			return "", err
+		}
+		for _, a := range agentPoolIDs.Items {
+			if a.Name == agentPoolName {
+				return a.ID, nil
+			}
+		}
+		if agentPoolIDs.NextPage == 0 {
+			break
 		}
+		listOpts.PageNumber = agentPoolIDs.NextPage
 	}
 
 	return "", fmt.Errorf("agent pool ID not found for agent pool name %q", agentPoolName)

controllers/workspace_controller_notifications.go

@@ -52,14 +52,21 @@ func (r *WorkspaceReconciler) getOrgMembers(ctx context.Context, w *workspaceIns
 			e = append(e, ne)
 		}
 	}
-	members, err := w.tfClient.Client.OrganizationMemberships.List(ctx, w.instance.Spec.Organization, &tfc.OrganizationMembershipListOptions{
+	listOpts := &tfc.OrganizationMembershipListOptions{
 		Emails: e,
-	})
-	if err != nil {
-		return nil, err
 	}
-	for _, m := range members.Items {
-		eu[m.Email] = &tfc.User{ID: m.User.ID}
+	for {
+		members, err := w.tfClient.Client.OrganizationMemberships.List(ctx, w.instance.Spec.Organization, listOpts)
+		if err != nil {
+			return nil, err
+		}
+		for _, m := range members.Items {
+			eu[m.Email] = &tfc.User{ID: m.User.ID}
+		}
+		if members.NextPage == 0 {
+			break
+		}
+		listOpts.PageNumber = members.NextPage
 	}
 	return eu, nil
 }
@@ -103,25 +110,31 @@ func (r *WorkspaceReconciler) getInstanceNotifications(ctx context.Context, w *w
 }
 
 func (r *WorkspaceReconciler) getWorkspaceNotifications(ctx context.Context, w *workspaceInstance) ([]tfc.NotificationConfiguration, error) {
-	wn, err := w.tfClient.Client.NotificationConfigurations.List(ctx, w.instance.Status.WorkspaceID, &tfc.NotificationConfigurationListOptions{})
-	if err != nil {
-		return []tfc.NotificationConfiguration{}, err
-	}
-
-	o := make([]tfc.NotificationConfiguration, len(wn.Items))
+	var o []tfc.NotificationConfiguration
 
-	for i, n := range wn.Items {
-		o[i] = tfc.NotificationConfiguration{
-			ID:              n.ID,
-			Name:            n.Name,
-			DestinationType: n.DestinationType,
-			URL:             n.URL,
-			Enabled:         n.Enabled,
-			Token:           n.Token,
-			Triggers:        n.Triggers,
-			EmailAddresses:  n.EmailAddresses,
-			EmailUsers:      n.EmailUsers,
+	listOpts := &tfc.NotificationConfigurationListOptions{}
+	for {
+		wn, err := w.tfClient.Client.NotificationConfigurations.List(ctx, w.instance.Status.WorkspaceID, listOpts)
+		if err != nil {
+			return nil, err
+		}
+		for _, n := range wn.Items {
+			o = append(o, tfc.NotificationConfiguration{
+				ID:              n.ID,
+				Name:            n.Name,
+				DestinationType: n.DestinationType,
+				URL:             n.URL,
+				Enabled:         n.Enabled,
+				Token:           n.Token,
+				Triggers:        n.Triggers,
+				EmailAddresses:  n.EmailAddresses,
+				EmailUsers:      n.EmailUsers,
+			})
+		}
+		if wn.NextPage == 0 {
+			break
 		}
+		listOpts.PageNumber = wn.NextPage
 	}
 
 	return o, nil

controllers/workspace_controller_projects.go

@@ -13,17 +13,23 @@ import (
 func (r *WorkspaceReconciler) getProjectIDByName(ctx context.Context, w *workspaceInstance) (string, error) {
 	projectName := w.instance.Spec.Project.Name
 
-	projectIDs, err := w.tfClient.Client.Projects.List(ctx, w.instance.Spec.Organization, &tfc.ProjectListOptions{
+	listOpts := &tfc.ProjectListOptions{
 		Name: projectName,
-	})
-	if err != nil {
-		return "", err
 	}
-
-	for _, p := range projectIDs.Items {
-		if p.Name == projectName {
-			return p.ID, nil
+	for {
+		projectIDs, err := w.tfClient.Client.Projects.List(ctx, w.instance.Spec.Organization, listOpts)
+		if err != nil {
+			return "", err
+		}
+		for _, p := range projectIDs.Items {
+			if p.Name == projectName {
+				return p.ID, nil
+			}
+		}
+		if projectIDs.NextPage == 0 {
+			break
 		}
+		listOpts.PageNumber = projectIDs.NextPage
 	}
 
 	return "", fmt.Errorf("project ID not found for project name %q", projectName)

controllers/workspace_controller_remote_state_sharing.go

@@ -13,16 +13,22 @@ import (
 )
 
 func (r *WorkspaceReconciler) getWorkspaces(ctx context.Context, w *workspaceInstance) (map[string]string, error) {
-	ws, err := w.tfClient.Client.Workspaces.List(ctx, w.instance.Spec.Organization, &tfc.WorkspaceListOptions{})
-	if err != nil {
-		return map[string]string{}, err
-	}
-
 	o := make(map[string]string)
 
-	for _, w := range ws.Items {
-		o[w.ID] = w.ID
-		o[w.Name] = w.ID
+	listOpts := &tfc.WorkspaceListOptions{}
+	for {
+		ws, err := w.tfClient.Client.Workspaces.List(ctx, w.instance.Spec.Organization, listOpts)
+		if err != nil {
+			return map[string]string{}, err
+		}
+		for _, w := range ws.Items {
+			o[w.ID] = w.ID
+			o[w.Name] = w.ID
+		}
+		if ws.NextPage == 0 {
+			break
+		}
+		listOpts.PageNumber = ws.NextPage
 	}
 
 	return o, nil

controllers/workspace_controller_run_tasks.go

@@ -112,12 +112,19 @@ func (r *WorkspaceReconciler) getInstanceRunTasks(ctx context.Context, w *worksp
 
 	rl := make(map[string]string)
 	if hasRunTaskName(w) {
-		rt, err := w.tfClient.Client.RunTasks.List(ctx, w.instance.Spec.Organization, &tfc.RunTaskListOptions{})
-		if err != nil {
-			return o, err
-		}
-		for _, t := range rt.Items {
-			rl[t.Name] = t.ID
+		listOpts := &tfc.RunTaskListOptions{}
+		for {
+			rt, err := w.tfClient.Client.RunTasks.List(ctx, w.instance.Spec.Organization, listOpts)
+			if err != nil {
+				return o, err
+			}
+			for _, t := range rt.Items {
+				rl[t.Name] = t.ID
+			}
+			if rt.NextPage == 0 {
+				break
+			}
+			listOpts.PageNumber = rt.NextPage
 		}
 	}
 
@@ -139,18 +146,24 @@ func (r *WorkspaceReconciler) getInstanceRunTasks(ctx context.Context, w *worksp
 func (r *WorkspaceReconciler) getWorkspaceRunTasks(ctx context.Context, w *workspaceInstance) (map[string]*tfc.WorkspaceRunTask, error) {
 	o := map[string]*tfc.WorkspaceRunTask{}
 
-	wrt, err := w.tfClient.Client.WorkspaceRunTasks.List(ctx, w.instance.Status.WorkspaceID, &tfc.WorkspaceRunTaskListOptions{})
-	if err != nil {
-		return o, err
-	}
-
-	for _, rt := range wrt.Items {
-		o[rt.RunTask.ID] = &tfc.WorkspaceRunTask{
-			ID:               rt.ID,
-			EnforcementLevel: rt.EnforcementLevel,
-			Stage:            rt.Stage,
-			RunTask:          &tfc.RunTask{ID: rt.RunTask.ID},
+	listOpts := &tfc.WorkspaceRunTaskListOptions{}
+	for {
+		wrt, err := w.tfClient.Client.WorkspaceRunTasks.List(ctx, w.instance.Status.WorkspaceID, listOpts)
+		if err != nil {
+			return o, err
+		}
+		for _, rt := range wrt.Items {
+			o[rt.RunTask.ID] = &tfc.WorkspaceRunTask{
+				ID:               rt.ID,
+				EnforcementLevel: rt.EnforcementLevel,
+				Stage:            rt.Stage,
+				RunTask:          &tfc.RunTask{ID: rt.RunTask.ID},
+			}
+		}
+		if wrt.NextPage == 0 {
+			break
 		}
+		listOpts.PageNumber = wrt.NextPage
 	}
 
 	return o, nil