[#3239,#3244] Bug/GitLab ECLA (#3238)

- Added v2 endpoint that fetches members of a given group id
- Updated controller for employee ack with right return_url_types(gitlab/github)
- check for gitlab group approval list criteria

Signed-off-by: Harold Wanyama <[email protected]>

cla-backend-go/github_organizations/handlers.go

@@ -176,6 +176,7 @@ func Configure(api *operations.ClaAPI, service ServiceInterface, eventService ev
 
 			return github_organizations.NewUpdateProjectGithubOrganizationConfigOK()
 		})
+
 }
 
 type codedResponse interface {

cla-backend-go/gitlab_api/client_groups.go

@@ -192,6 +192,24 @@ func GetGroupProjectListByGroupID(ctx context.Context, client *goGitLab.Client,
 	return projectList, nil
 }
 
+// ListGroupMembers lists the members of a given groupID
+func ListGroupMembers(ctx context.Context, client *goGitLab.Client, groupID int) ([]*goGitLab.GroupMember, error) {
+	f := logrus.Fields{
+		"functionName":   "gitlab_api.client_groups.GetGroupMembers",
+		utils.XREQUESTID: ctx.Value(utils.XREQUESTID),
+	}
+
+	log.WithFields(f).Debugf("fetching gitlab members for groupID: %d", groupID)
+
+	opts := &goGitLab.ListGroupMembersOptions{}
+	members, _, err := client.Groups.ListGroupMembers(groupID, opts)
+	if err != nil {
+		log.WithFields(f).Debugf("unable to fetch members for gitlab GroupID : %d", groupID)
+		return nil, err
+	}
+	return members, err
+}
+
 // ListUserProjectGroups fetches the unique groups of a gitlab users groups,
 // note: it doesn't list the projects/groups the user is member of ..., it's very limited
 func ListUserProjectGroups(ctx context.Context, client *goGitLab.Client, userID int) ([]*UserGroup, error) {

cla-backend-go/go.mod

@@ -98,9 +98,11 @@ require (
 	github.com/magiconair/properties v1.8.5 // indirect
 	github.com/mailru/easyjson v0.7.1 // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/myitcv/gobin v0.0.14 // indirect
 	github.com/pelletier/go-toml v1.9.3 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rogpeppe/go-internal v1.8.0 // indirect
 	github.com/spf13/afero v1.6.0 // indirect
 	github.com/spf13/cast v1.3.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect

cla-backend-go/go.sum

@@ -448,6 +448,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/mozillazg/request v0.8.0 h1:TbXeQUdBWr1J1df5Z+lQczDFzX9JD71kTCl7Zu/9rNM=
 github.com/mozillazg/request v0.8.0/go.mod h1:weoQ/mVFNbWgRBtivCGF1tUT9lwneFesues+CleXMWc=
+github.com/myitcv/gobin v0.0.14 h1:YkTUz0IeRspEJlly/+AXRBMA3GN7ArRVbsLJ1uYFwRk=
+github.com/myitcv/gobin v0.0.14/go.mod h1:GvHEiYCWroKI2KrMT+xQkHC3FC551wigVWeR4Sgg5P4=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nightlyone/lockfile v1.0.0/go.mod h1:rywoIealpdNse2r832aiD9jRk8ErCatROs6LzC841CI=
@@ -463,6 +465,7 @@ github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUr
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ=
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -476,6 +479,9 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
+github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
 github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=

cla-backend-go/swagger/cla.v2.yaml

@@ -1842,7 +1842,35 @@ paths:
           $ref: '#/responses/internal-server-error'
       tags:
         - gitlab-repositories
-
+
+  /gitlab/group/{gitLabGroupID}/members:
+    get:
+      summary: List members of a given GitLab group
+      description: Endpoint that returs the list of GitLab organization members
+      operationId: getGitLabGroupMembers
+      security: []
+      parameters:
+        - $ref: "#/parameters/x-request-id"
+        - name: gitLabGroupID
+          in: path
+          type: string
+          required: true
+      responses:
+        '200':
+          description: 'Success'
+          headers:
+            x-request-id:
+              type: string
+              description: The unique request ID value - assigned/set by the API Gateway based on the session
+          schema:
+            $ref: '#/definitions/gitlab-group-members-list'
+        '400':
+          $ref: '#/responses/invalid-request'
+        '404':
+          $ref: '#/responses/not-found'
+      tags:
+        - gitlab-organizations
+
   /cla-group/{claGroupID}/icla/signatures:
     get:
       summary: List individual signatures for CLA Group
@@ -4523,6 +4551,12 @@ definitions:
 
   gitlab-repositories-enroll:
     $ref: './common/gitlab-repositories-enroll.yaml'
+
+  gitlab-group-member:
+    $ref: './common/gitlab-group-member.yaml'
+
+  gitlab-group-members-list:
+    $ref: './common/gitlab-group-members-list.yaml'
 
   # ---------------------------------------------------------------------------
   # CLA Group Definitions

cla-backend-go/swagger/common/gitlab-group-member.yaml

@@ -0,0 +1,36 @@
+# Copyright The Linux Foundation and each contributor to CommunityBridge.
+# SPDX-License-Identifier: MIT
+
+type: object
+properties:
+  id:
+    type: string
+    description: user id of the gitlab member
+  username:
+    type: string
+    description: username of the gitlab member
+  name:
+    type: string
+    description: name of the gitlab member
+  state:
+    type: string
+    description: state of the gitlab member
+  avatar_url:
+    type: string
+    description: avatar_url of the gitlab member
+  web_url:
+    type: string
+    description: web_url of gitlab member
+  expired_at:
+    type: string
+    description: user expiry time
+  created_at:
+    type: string
+    description: created_at user date
+  access_level:
+    type: string
+    description: access level for user
+  group_saml_identity:
+    type: string
+    description: group saml identity 
+

cla-backend-go/swagger/common/gitlab-group-members-list.yaml

@@ -0,0 +1,9 @@
+# Copyright The Linux Foundation and each contributor to CommunityBridge.
+# SPDX-License-Identifier: MIT
+
+type: object
+properties:
+  list:
+    type: array
+    items:
+      $ref: '#/definitions/gitlab-group-member'

cla-backend-go/v2/gitlab_organizations/handlers.go

@@ -228,6 +228,23 @@ func Configure(api *operations.EasyclaAPI, service ServiceInterface, eventServic
 			return gitlab_organizations.NewAddProjectGitlabOrganizationOK().WithPayload(result)
 		})
 
+	api.GitlabOrganizationsGetGitLabGroupMembersHandler = gitlab_organizations.GetGitLabGroupMembersHandlerFunc(func(params gitlab_organizations.GetGitLabGroupMembersParams) middleware.Responder {
+		reqID := utils.GetRequestID(params.XREQUESTID)
+		ctx := utils.NewContext()
+		f := logrus.Fields{
+			"functionName":   "v2.gitlab_organizations.handlers.GitlabOrganizationsGetGitLabGroupMembersHandler",
+			utils.XREQUESTID: ctx.Value(utils.XREQUESTID),
+			"gitLabGroupID":  params.GitLabGroupID,
+		}
+		log.WithFields(f).Debug("fetching gitlab group member details")
+		memberList, err := service.GetGitLabGroupMembers(ctx, params.GitLabGroupID)
+		if err != nil {
+			msg := fmt.Sprintf("unable to get groupID: %s  member list: %+v ", params.GitLabGroupID, err)
+			return gitlab_organizations.NewGetGitLabGroupMembersBadRequest().WithPayload(utils.ErrorResponseBadRequest(reqID, msg))
+		}
+		return gitlab_organizations.NewGetGitLabGroupMembersOK().WithPayload(memberList)
+	})
+
 	api.GitlabOrganizationsUpdateProjectGitlabGroupConfigHandler = gitlab_organizations.UpdateProjectGitlabGroupConfigHandlerFunc(func(params gitlab_organizations.UpdateProjectGitlabGroupConfigParams, authUser *auth.User) middleware.Responder {
 		reqID := utils.GetRequestID(params.XREQUESTID)
 		utils.SetAuthUserProperties(authUser, params.XUSERNAME, params.XEMAIL)

cla-backend-go/v2/gitlab_organizations/service.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"net/url"
 	"sort"
+	"strconv"
 	"strings"
 
 	"github.com/communitybridge/easycla/cla-backend-go/v2/repositories"
@@ -44,6 +45,7 @@ type ServiceInterface interface {
 	GetGitLabOrganizationsEnabledWithAutoEnabled(ctx context.Context) (*v2Models.GitlabProjectOrganizations, error)
 	GetGitLabOrganizationsByProjectSFID(ctx context.Context, projectSFID string) (*v2Models.GitlabProjectOrganizations, error)
 	GetGitLabOrganizationByState(ctx context.Context, gitLabOrganizationID, authState string) (*v2Models.GitlabOrganization, error)
+	GetGitLabGroupMembers(ctx context.Context, groupID string) (*v2Models.GitlabGroupMembersList, error)
 	UpdateGitLabOrganization(ctx context.Context, input *common.GitLabAddOrganization) error
 	UpdateGitLabOrganizationAuth(ctx context.Context, gitLabOrganizationID string, oauthResp *gitlabApi.OauthSuccessResponse) error
 	DeleteGitLabOrganizationByFullPath(ctx context.Context, projectSFID string, gitlabOrgFullPath string) error
@@ -283,6 +285,52 @@ func (s *Service) GetGitLabOrganizationsEnabledWithAutoEnabled(ctx context.Conte
 	return out, nil
 }
 
+//GetGitLabGroupMembers gets group members
+func (s *Service) GetGitLabGroupMembers(ctx context.Context, groupID string) (*v2Models.GitlabGroupMembersList, error) {
+	f := logrus.Fields{
+		"functionName":   "v2.gitlab_organizations.service.GetGitLabGroupMembers",
+		utils.XREQUESTID: ctx.Value(utils.XREQUESTID),
+		"groupID":        groupID,
+	}
+	groupMemberList := make([]*v2Models.GitlabGroupMember, 0)
+	gitlabOrg, err := s.GetGitLabOrganization(ctx, groupID)
+	if err != nil {
+		log.WithFields(f).WithError(err).Warn("unable to fetch gitlab details")
+		return nil, err
+	}
+
+	if gitlabOrg != nil {
+		glClient, clientErr := gitlabApi.NewGitlabOauthClient(gitlabOrg.AuthInfo, s.gitLabApp)
+		if clientErr != nil {
+			log.WithFields(f).WithError(clientErr).Warn("problem getting gitLabClient")
+			return nil, clientErr
+		}
+
+		members, err := gitlabApi.ListGroupMembers(ctx, glClient, int(gitlabOrg.OrganizationExternalID))
+		if err != nil {
+			log.WithFields(f).WithError(err).Warn("unable to get group members list")
+			return nil, err
+		}
+
+		if len(members) > 0 {
+			for _, member := range members {
+				groupMemberList = append(groupMemberList, &v2Models.GitlabGroupMember{
+					Name:     member.Name,
+					ID:       strconv.Itoa((member.ID)),
+					Username: member.Username,
+				})
+			}
+		}
+
+	}
+
+	log.WithFields(f).Debugf("Members: %+v ", groupMemberList)
+
+	return &v2Models.GitlabGroupMembersList{
+		List: groupMemberList,
+	}, nil
+}
+
 // GetGitLabOrganizationsByProjectSFID returns a collection of GitLab organizations based on the specified project SFID value
 func (s *Service) GetGitLabOrganizationsByProjectSFID(ctx context.Context, projectSFID string) (*v2Models.GitlabProjectOrganizations, error) {
 	f := logrus.Fields{

cla-backend/cla/controllers/signing.py

@@ -123,7 +123,7 @@ def request_employee_signature(project_id, company_id, user_id, return_url_type,
                                                                  return_url)
     elif return_url_type is not None and (return_url_type.lower() == "github" or return_url_type.lower() == "gitlab"):
         cla.log.error(f'{fn} - return type is github - invoking: request_employee_signature')
-        return signing_service.request_employee_signature(str(project_id), str(company_id), str(user_id), return_url)
+        return signing_service.request_employee_signature(str(project_id), str(company_id), str(user_id), return_url, return_url_type=return_url_type)
 
     else:
         msg = (f'{fn} - unsupported return type {return_url_type} for '

cla-backend/cla/models/dynamo_models.py

@@ -480,6 +480,21 @@ class Meta:
 
     organization_name_lower = UnicodeAttribute(hash_key=True)
 
+class GitlabExternalGroupIDIndex(GlobalSecondaryIndex):
+    """
+    This class represents a global secondary index for querying gitlab organizations by group ID
+    """
+
+    class Meta:
+        """Meta class for external ID for gitlab group id index"""
+
+        index_name = "gitlab-external-group-id-index"
+        write_capacity_units = int(cla.conf["DYNAMO_WRITE_UNITS"])
+        read_capacity_units = int(cla.conf["DYNAMO_READ_UNITS"])
+        projection = AllProjection()
+
+    external_gitlab_group_id = NumberAttribute(hash_key=True)
+
 
 class GerritProjectIDIndex(GlobalSecondaryIndex):
     """
@@ -2094,7 +2109,19 @@ def is_approved(self, ccla_signature: Signature) -> bool:
         else:
             cla.log.debug(f'{fn} - users github_username is not defined - '
                           'skipping github username approval list check')
-
+
+        if gitlab_username is not None :
+            cla.log.debug(f'{fn} fetching gitlab org approval list items to search by username: {gitlab_username}')
+            gitlab_org_approval_lists = ccla_signature.get_gitlab_org_approval_list()
+            if gitlab_org_approval_lists:
+                for gl_name in gitlab_org_approval_lists:
+                    gl_org = GitlabOrg().search_organization_by_lower_name(gl_name)
+                    cla.log.debugf(f"{fn} checking gitlab_username against approval list for company: {gl_org}")
+                    gl_list = list(filter(lambda gl_user: gl_user.get_gitlab_username() == gitlab_username, cla.utils.lookup_gitlab_org_members(gl_org.get_organization_id())))
+                    if len(gl_list) > 0:
+                        cla.models.debug(f'{fn} - found gitlab username in gitlab approval list')
+                        return True
+
 
         cla.log.debug(f'{fn} - unable to find user in any whitelist')
         return False
@@ -3710,9 +3737,11 @@ class Meta:
     organization_name_lower = UnicodeAttribute(null=True)
     organization_sfid = UnicodeAttribute()
     project_sfid = UnicodeAttribute()
+    auth_info = UnicodeAttribute()
     organization_sfid_index = GitlabOrgSFIndex()
     project_sfid_organization_name_index = GitlabOrgProjectSfidOrganizationNameIndex()
     organization_name_lowe_index = GitlabOrganizationNameLowerIndex()
+    gitlab_external_group_id_index = GitlabExternalGroupIDIndex()
     auto_enabled = BooleanAttribute(null=True)
     auto_enabled_cla_group_id = UnicodeAttribute(null=True)
     branch_protection_enabled = BooleanAttribute(null=True)
@@ -3909,7 +3938,7 @@ class GitlabOrg(model_interfaces.GitlabOrg):  # pylint: disable=too-many-public-
     """
 
     def __init__(
-            self, organization_id=None, organization_name=None, organization_sfid=None,
+            self, organization_id=None, organization_name=None, organization_sfid=None, auth_info=None,
             project_sfid=None, auto_enabled=False, branch_protection_enabled=False, note=None, enabled=True
     ):
         super(GitlabOrg).__init__()
@@ -3928,6 +3957,7 @@ def __init__(
         self.model.branch_protection_enabled = branch_protection_enabled
         self.model.enabled = enabled
         self.model.note = note
+        self.model.auth_info = auth_info
 
     def __str__(self):
         return (
@@ -3937,7 +3967,8 @@ def __str__(self):
             f'auto_enabled: {self.model.auto_enabled},'
             f'branch_protection_enabled: {self.model.branch_protection_enabled},'
             f'enabled: {self.model.enabled},'
-            f'note: {self.model.note}'
+            f'note: {self.model.note}',
+            f'auth_info: {self.model.auth_info}'
         )
 
     def to_dict(self):
@@ -3988,6 +4019,9 @@ def get_note(self):
         :rtype: str
         """
         return self.model.note
+
+    def get_auth_info(self):
+        return self.model.auth_info
 
     def get_enabled(self):
         return self.model.enabled
@@ -4017,6 +4051,17 @@ def set_note(self, note):
 
     def set_enabled(self, enabled):
         self.model.enabled = enabled
+
+    def set_auth_info(self, auth_info):
+        self.model.auth_info = auth_info
+
+    def get_organization_by_groupid(self, groupid):
+        org_generator = self.model.gitlab_external_group_id_index.query(groupid)
+        for org_model in org_generator:
+            org = GitlabOrg()
+            org.model = org_model
+            return org
+        return None
 
     def get_organization_by_sfid(self, sfid) -> List:
         organization_generator = self.model.organization_sfid_index.query(sfid)

cla-backend/cla/utils.py

@@ -1496,6 +1496,16 @@ def lookup_github_organizations(github_username: str):
         return {'error': 'Could not get user github org: {}'.format(err)}
     return [github_org['login'] for github_org in r.json()]
 
+def lookup_gitlab_org_members(organization_id):
+    # Use the v2 Endpoint thats a wrapper for Gitlab Group member query
+    try:
+        r = requests.get(f'{cla.config.PLATFORM_GATEWAY_URL}/cla-service/v4/gitlab/group/{organization_id}/members')
+        r.raise_for_status()
+    except requests.exceptions.HTTPError as err:
+        cla.log.warning(f'Could not fetch gitlab org users: {err}')
+        return {f'error: Could not get user gitlab group id: {organization_id} members: {err}'}
+    return r.json()['list']
+
 
 def update_github_username(github_user: dict, user: User):
     """