[#2792] Feature/Invalidate Contributors (#2810)

- Handled signature invalidations for ghorg and domain approval removals
- Implemented ghorg client for getting users under ghorg
- Handled ICLA and ECLA signature updates

Signed-off-by: wanyaland <[email protected]>

cla-backend-go/cmd/dynamo_events_lambda/main.go

@@ -119,7 +119,7 @@ func init() {
 		projectClaGroupRepo,
 	})
 
-	signaturesRepo := signatures.NewRepository(awsSession, stage, companyRepo, usersRepo, eventsService)
+	signaturesRepo := signatures.NewRepository(awsSession, stage, companyRepo, usersRepo, eventsService, repositoriesRepo, githubOrganizationsRepo)
 
 	usersService := users.NewService(usersRepo, eventsService)
 	companyService := company.NewService(companyRepo, configFile.CorporateConsoleV1URL, userRepo, usersService)

cla-backend-go/cmd/server.go

@@ -252,7 +252,7 @@ func server(localMode bool) http.Handler {
 	})
 
 	// Signature repository handler
-	signaturesRepo := signatures.NewRepository(awsSession, stage, v1CompanyRepo, usersRepo, eventsService)
+	signaturesRepo := signatures.NewRepository(awsSession, stage, v1CompanyRepo, usersRepo, eventsService, repositoriesRepo, githubOrganizationsRepo)
 
 	// Initialize the external platform services - these are external APIs that
 	// we download the swagger specification, generate the models, and have

cla-backend-go/github/github_org.go

@@ -44,3 +44,33 @@ func GetOrganization(ctx context.Context, organizationName string) (*github.Orga
 	}
 	return org, nil
 }
+
+//GetOrganizationMembers gets members in organization
+func GetOrganizationMembers(ctx context.Context, orgName string, installationID int64) ([]string, error) {
+	f := logrus.Fields{
+		"functionName":   "GetOrganizationMembers",
+		utils.XREQUESTID: ctx.Value(utils.XREQUESTID),
+	}
+
+	client, err := NewGithubAppClient(installationID)
+	if err != nil {
+		msg := fmt.Sprintf("unable to create a github client, error: %+v", err)
+		log.WithFields(f).WithError(err).Warn(msg)
+		return nil, errors.New(msg)
+	}
+
+	users, resp, err := client.Organizations.ListMembers(ctx, orgName, nil)
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 || err != nil {
+		msg := fmt.Sprintf("List Org Members failed for Organization: %s with no success response code %d. error = %s", orgName, resp.StatusCode, err.Error())
+		log.WithFields(f).Warnf(msg)
+		return nil, errors.New(msg)
+	}
+
+	var ghUsernames []string
+	for _, user := range users {
+		log.WithFields(f).Debugf("user :%s found for organization: %s", *user.Login, orgName)
+		ghUsernames = append(ghUsernames, *user.Login)
+	}
+	return ghUsernames, nil
+}

cla-backend-go/go.sum

@@ -95,6 +95,8 @@ github.com/communitybridge/easycla v1.0.118 h1:8yrsOQ+ENUFi4RFl1krRlIxc51lzZNuti
 github.com/communitybridge/easycla v1.0.123 h1:Lh5i/9aajrTYItxNpVCmi9T1yyIfnQIOk0tC2Wtslvk=
 github.com/communitybridge/easycla v1.0.133 h1:aJulQGLLRISCMsZcCP4aIE8xGtHoBNm/EmA00n3NYVA=
 github.com/communitybridge/easycla v1.0.135 h1:Dvn8jX+7BAnpmA+jvdK0n5ajWP8SoH5vvopt7whZDEU=
+github.com/communitybridge/easycla v1.0.145 h1:ikhBSsOeEL2u3/EoyDsufh/j3HkjfFTiXAk1d61GoS8=
+github.com/communitybridge/easycla v2.0.10+incompatible h1:6eRJ5fxrMxRZHBkg8piYo+zHTcSowMrP85nZXzp5mpA=
 github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ=

cla-backend-go/signatures/models.go

@@ -16,3 +16,17 @@ type ApprovalCriteria struct {
 	UserEmail      string
 	GitHubUsername string
 }
+
+//ApprovalList ...
+type ApprovalList struct {
+	Criteria                string
+	ApprovalList            []string
+	Action                  string
+	ClaGroupID              string
+	CompanyID               string
+	DomainApprovals         []string
+	GHOrgApprovals          []string
+	GitHubUsernameApprovals []string
+	EmailApprovals          []string
+	GHUsernames             []string
+}

cla-backend-go/signatures/repository.go

@@ -21,6 +21,9 @@ import (
 
 	"github.com/communitybridge/easycla/cla-backend-go/company"
 	"github.com/communitybridge/easycla/cla-backend-go/events"
+	"github.com/communitybridge/easycla/cla-backend-go/github"
+	"github.com/communitybridge/easycla/cla-backend-go/github_organizations"
+	"github.com/communitybridge/easycla/cla-backend-go/repositories"
 
 	"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute"
 
@@ -90,17 +93,21 @@ type repository struct {
 	companyRepo        company.IRepository
 	usersRepo          users.UserRepository
 	eventsService      events.Service
+	repositoriesRepo   repositories.Repository
+	ghOrgRepo          github_organizations.Repository
 	signatureTableName string
 }
 
 // NewRepository creates a new instance of the whitelist service
-func NewRepository(awsSession *session.Session, stage string, companyRepo company.IRepository, usersRepo users.UserRepository, eventsService events.Service) SignatureRepository {
+func NewRepository(awsSession *session.Session, stage string, companyRepo company.IRepository, usersRepo users.UserRepository, eventsService events.Service, repositoriesRepo repositories.Repository, ghOrgRepo github_organizations.Repository) SignatureRepository {
 	return repository{
 		stage:              stage,
 		dynamoDBClient:     dynamodb.New(awsSession),
 		companyRepo:        companyRepo,
 		usersRepo:          usersRepo,
 		eventsService:      eventsService,
+		repositoriesRepo:   repositoriesRepo,
+		ghOrgRepo:          ghOrgRepo,
 		signatureTableName: fmt.Sprintf("cla-%s-signatures", stage),
 	}
 }
@@ -1975,6 +1982,22 @@ func (repo repository) UpdateApprovalList(ctx context.Context, claManager *model
 			companyID, projectID, signed, approved)
 	}
 
+	// Get CCLA signature - For Approval List info
+	cclaSignature, err := repo.GetCorporateSignature(ctx, projectID, companyID)
+	if err != nil {
+		msg := "unable to get corporate signature"
+		log.WithFields(f).Warn(msg)
+		return nil, errors.New(msg)
+	}
+
+	// Keep track of existing company approvals
+	approvalList := ApprovalList{
+		DomainApprovals:         cclaSignature.DomainApprovalList,
+		GHOrgApprovals:          cclaSignature.GithubOrgApprovalList,
+		GitHubUsernameApprovals: cclaSignature.GithubUsernameApprovalList,
+		EmailApprovals:          cclaSignature.EmailApprovalList,
+	}
+
 	// Just grab and use the first one - need to figure out conflict resolution if more than one
 	sig := sigs.Signatures[0]
 	expressionAttributeNames := map[string]*string{}
@@ -2049,6 +2072,7 @@ func (repo repository) UpdateApprovalList(ctx context.Context, claManager *model
 	}
 
 	if params.AddDomainApprovalList != nil || params.RemoveDomainApprovalList != nil {
+
 		columnName := "domain_whitelist"
 		attrList := buildApprovalAttributeList(ctx, sig.DomainApprovalList, params.AddDomainApprovalList, params.RemoveDomainApprovalList)
 		// If no entries after consolidating all the updates, we need to remove the column
@@ -2067,6 +2091,18 @@ func (repo repository) UpdateApprovalList(ctx context.Context, claManager *model
 			expressionAttributeValues[":d"] = attrList
 			updateExpression = updateExpression + " #D = :d, "
 		}
+		if params.RemoveDomainApprovalList != nil {
+			var invalidateErr error
+			approvalList.Criteria = utils.EmailDomainCriteria
+			approvalList.ApprovalList = params.RemoveDomainApprovalList
+			approvalList.Action = utils.RemoveApprovals
+			invalidateErr = repo.invalidateSignatures(ctx, &approvalList, claManager)
+			if invalidateErr != nil {
+				msg := fmt.Sprintf("unable to invalidate signatures based on Approval List : %+v ", approvalList)
+				log.WithFields(f).Warn(msg)
+				return nil, errors.New(msg)
+			}
+		}
 	}
 
 	if params.AddGithubUsernameApprovalList != nil || params.RemoveGithubUsernameApprovalList != nil {
@@ -2147,6 +2183,57 @@ func (repo repository) UpdateApprovalList(ctx context.Context, claManager *model
 			expressionAttributeValues[":go"] = attrList
 			updateExpression = updateExpression + " #GO = :go, "
 		}
+
+		if params.RemoveGithubOrgApprovalList != nil {
+			var invalidateErr error
+			approvalList.Criteria = utils.GitHubOrgCriteria
+			approvalList.ApprovalList = params.RemoveGithubOrgApprovalList
+			approvalList.Action = utils.RemoveApprovals
+			// Get repositories by CLAGroup
+			repositories, err := repo.repositoriesRepo.GetRepositoriesByCLAGroup(ctx, projectID, true)
+			if err != nil {
+				msg := fmt.Sprintf("unable to fetch repositories for claGroupID: %s ", projectID)
+				log.WithFields(f).Warn(msg)
+				return nil, errors.New(msg)
+			}
+			var ghOrgRepositories []*models.GithubRepository
+			var ghOrgs []*models.GithubOrganization
+			for _, repository := range repositories {
+				// Check for matching organization name in repositories table against approvalList removal GH Orgs
+				if utils.StringInSlice(repository.RepositoryOrganizationName, approvalList.ApprovalList) {
+					ghOrgRepositories = append(ghOrgRepositories, repository)
+				}
+			}
+
+			for _, ghOrgRepo := range ghOrgRepositories {
+				ghOrg, err := repo.ghOrgRepo.GetGithubOrganization(ctx, ghOrgRepo.RepositoryOrganizationName)
+				if err != nil {
+					msg := fmt.Sprintf("unable to get gh org by name: %s ", ghOrgRepo.RepositoryOrganizationName)
+					log.WithFields(f).Warn(msg)
+					return nil, errors.New(msg)
+				}
+				ghOrgs = append(ghOrgs, ghOrg)
+			}
+
+			var ghUsernames []string
+			for _, ghOrg := range ghOrgs {
+				ghOrgUsers, err := github.GetOrganizationMembers(ctx, ghOrg.OrganizationName, ghOrg.OrganizationInstallationID)
+				if err != nil {
+					msg := fmt.Sprintf("unable to fetch ghOrgUsers for org: %s ", ghOrg.OrganizationName)
+					log.WithFields(f).Warnf(msg)
+					return nil, errors.New(msg)
+				}
+				ghUsernames = append(ghUsernames, ghOrgUsers...)
+			}
+			approvalList.GHUsernames = utils.RemoveDuplicates(ghUsernames)
+
+			invalidateErr = repo.invalidateSignatures(ctx, &approvalList, claManager)
+			if invalidateErr != nil {
+				msg := fmt.Sprintf("unable to invalidate signatures based on Approval List: %+v ", approvalList)
+				log.WithFields(f).Warn(msg)
+				return nil, errors.New(msg)
+			}
+		}
 	}
 
 	// Ensure at least one value is set for us to update
@@ -2206,6 +2293,128 @@ func (repo repository) UpdateApprovalList(ctx context.Context, claManager *model
 	return updatedSig.Signatures[0], nil
 }
 
+// invalidateSignatures is a helper function that invalidates signature records based on approval list
+func (repo repository) invalidateSignatures(ctx context.Context, approvalList *ApprovalList, claManager *models.User) error {
+	f := logrus.Fields{
+		"functionName":   "invalidateSignatures",
+		utils.XREQUESTID: ctx.Value(utils.XREQUESTID),
+		"claGroupID":     &approvalList,
+	}
+
+	// Get ICLAs
+	iclas, err := repo.GetClaGroupICLASignatures(ctx, approvalList.ClaGroupID, nil)
+	if err != nil {
+		log.WithFields(f).Warn("unable to get iclas")
+		return err
+	}
+
+	// Get ECLAs
+	companyProjectParams := signatures.GetProjectCompanyEmployeeSignaturesParams{
+		CompanyID: approvalList.CompanyID,
+		ProjectID: approvalList.ClaGroupID,
+	}
+	eclas, err := repo.GetProjectCompanyEmployeeSignatures(ctx, companyProjectParams, nil, int64(10))
+	if err != nil {
+		log.WithFields(f).Warnf("unable to get cclas for company: %s and project: %s ", approvalList.CompanyID, approvalList.ClaGroupID)
+		return err
+	}
+
+	var iclaWg, eclaWg sync.WaitGroup
+
+	//Iterate iclas
+	iclaWg.Add(len(iclas.List))
+	log.WithFields(f).Debug("invalidating signature icla records... ")
+
+	for _, icla := range iclas.List {
+		go func(icla *models.IclaSignature) {
+			defer iclaWg.Done()
+			signature, err := repo.GetSignature(ctx, icla.SignatureID)
+			if err != nil {
+				log.WithFields(f).Warnf("unable to fetch signature for ID: %s ", icla.SignatureID)
+				return
+			}
+			// Grab user record
+			if signature.SignatureReferenceID == "" {
+				log.WithFields(f).Warnf("no signatureReferenceID for signature: %+v ", signature)
+				return
+			}
+			verifyErr := repo.verifyUserApprovals(ctx, signature.SignatureReferenceID, signature.SignatureID, claManager, approvalList)
+			if verifyErr != nil {
+				log.WithFields(f).Warnf("unable to verify user: %s ", signature.SignatureReferenceID)
+				return
+			}
+		}(icla)
+	}
+	iclaWg.Wait()
+
+	log.WithFields(f).Debug("invalidating signature ecla records... ")
+	// Iterate eclas
+	eclaWg.Add(len(eclas.Signatures))
+	for _, ecla := range eclas.Signatures {
+		go func(ecla *models.Signature) {
+			defer eclaWg.Done()
+			// Grab user record
+			if ecla.SignatureReferenceID == "" {
+				log.WithFields(f).Warnf("no signatureReferenceID for signature: %+v ", ecla)
+				return
+			}
+			verifyErr := repo.verifyUserApprovals(ctx, ecla.SignatureReferenceID, ecla.SignatureID, claManager, approvalList)
+			if verifyErr != nil {
+				log.WithFields(f).Warnf("unable to verify user: %s ", ecla.SignatureReferenceID)
+				return
+			}
+		}(ecla)
+	}
+	eclaWg.Wait()
+
+	return nil
+}
+
+// verify UserApprovals checks user
+func (repo repository) verifyUserApprovals(ctx context.Context, userID, signatureID string, claManager *models.User, approvalList *ApprovalList) error {
+	f := logrus.Fields{
+		"functionName":   "verifyUserApprovals",
+		utils.XREQUESTID: ctx.Value(utils.XREQUESTID),
+		"userID":         userID,
+	}
+
+	user, err := repo.usersRepo.GetUser(userID)
+	if err != nil {
+		log.WithFields(f).Warnf("unable to get user record for ID: %s ", userID)
+		return err
+	}
+
+	if approvalList.Criteria == utils.EmailDomainCriteria {
+		// Handle Domains
+		if utils.StringInSlice(getBestEmail(user), approvalList.DomainApprovals) {
+			if !utils.StringInSlice(user.GithubUsername, approvalList.GitHubUsernameApprovals) && !utils.StringInSlice(getBestEmail(user), approvalList.EmailApprovals) {
+				//Invalidate record
+				note := fmt.Sprintf("Signature invalidated (approved set to false) by %s due to %s  removal", utils.GetBestUsername(claManager), utils.EmailDomainCriteria)
+				err := repo.InvalidateProjectRecord(ctx, signatureID, note)
+				if err != nil {
+					log.WithFields(f).Warnf("unable to invalidate record for signatureID: %s ", signatureID)
+					return err
+				}
+			}
+		}
+	} else if approvalList.Criteria == utils.GitHubOrgCriteria {
+		// Handle GH Org Approvals
+		if utils.StringInSlice(user.GithubUsername, approvalList.GHUsernames) {
+			if !utils.StringInSlice(getBestEmail(user), approvalList.EmailApprovals) && !utils.StringInSlice(user.GithubUsername, approvalList.GitHubUsernameApprovals) {
+				//Invalidate record
+				note := fmt.Sprintf("Signature invalidated (approved set to false) by %s due to %s  removal", utils.GetBestUsername(claManager), utils.GitHubOrgCriteria)
+				err := repo.InvalidateProjectRecord(ctx, signatureID, note)
+				if err != nil {
+					log.WithFields(f).Warnf("unable to invalidate record for signatureID: %s ", signatureID)
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
 // removeColumn is a helper function to remove a given column when we need to zero out the column value - typically the approval list
 func (repo repository) removeColumn(ctx context.Context, signatureID, columnName string) (*models.Signature, error) {
 	f := logrus.Fields{

cla-backend-go/utils/constants.go

@@ -149,3 +149,18 @@ const EmailLabel = "Email Address"
 
 //UserLabel represents the LF/EasyCLA username
 const UserLabel = "Username"
+
+//EmailDomainCriteria represents approval based on email domain
+const EmailDomainCriteria = "Email Domain Criteria"
+
+//EmailCriteria represents approvals based on email addresses
+const EmailCriteria = "Email Criteria"
+
+//GitHubOrgCriteria represents approvals based on GH org membership
+const GitHubOrgCriteria = "GitHub Org Criteria"
+
+//AddApprovals is an action for adding approvals
+const AddApprovals = "AddApprovals"
+
+//RemoveApprovals is an action for removing approvals
+const RemoveApprovals = "RemoveApprovals"