Added Get Company by ID/SFID + Added GitHub v4 GraphQL Lib (#2541)

- Added v2 Get Company by ID/SFID swagger/API endpoints
- Updated company not found error/404 logic
- Added 404 error to v2 endponts that were missing it
- Added initial release of GitHub v4 GraphQL API library and init logic

Signed-off-by: makkalot <[email protected]>
Signed-off-by: David Deal <[email protected]>

Co-authored-by: Denis Kyorov <[email protected]>

.circleci/config.yml

@@ -205,6 +205,7 @@ jobs:
       - add_ssh_keys:
           fingerprints:
             - "e9:13:85:f1:b1:a1:25:bf:f5:44:34:66:82:1e:31:59"
+      - *setup_aws
       - run: echo 'export NVM_DIR=${HOME}/.nvm' >> $BASH_ENV
       - *install-node-12
       - run: echo 'export GO111MODULE=on' >> $BASH_ENV
@@ -287,16 +288,28 @@ jobs:
     <<: *buildGoBackendAnchor
     environment:
       STAGE: dev
+      AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_DEV
+      AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_DEV
+      AWS_PROFILE: lf-cla
+      AWS_REGION: us-east-1
 
   buildGoBackendStaging:
     <<: *buildGoBackendAnchor
     environment:
       STAGE: staging
+      AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_STAGING
+      AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_STAGING
+      AWS_PROFILE: lf-cla
+      AWS_REGION: us-east-1
 
   buildGoBackendProd:
     <<: *buildGoBackendAnchor
     environment:
       STAGE: prod
+      AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_PROD
+      AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_PROD
+      AWS_PROFILE: lf-cla
+      AWS_REGION: us-east-1
 
   # Deploys
   deployBackend: &deployBackendAnchor

cla-backend-go/Makefile

@@ -23,7 +23,6 @@ LINT_VERSION=v1.29.0
 SWAGGER_TOOL_VERSION=v0.24.0
 GO_PKGS=$(shell go list ./... | grep -v /vendor/ | grep -v /node_modules/)
 GO_FILES=$(shell find . -type f -name '*.go' -not -path './vendor/*')
-TEST_ENV=AWS_REGION=us-east-1 DYNAMODB_AWS_REGION=us-east-1 AWS_PROFILE=bar AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar
 
 .PHONY: generate setup tool-setup setup-dev setup-deploy clean-all clean swagger up fmt test run deps build build-mac build-aws-lambda user-subscribe-lambda qc lint
 
@@ -170,7 +169,7 @@ fmt:
 
 test:
 	@echo "Running unit tests..."
-	@ $(TEST_ENV) go test -v $(shell go list ./... | grep -v /vendor/ | grep -v /node_modules/) -coverprofile=cover.out
+	@go test -v $(shell go list ./... | grep -v /vendor/ | grep -v /node_modules/) -coverprofile=cover.out
 
 mock:
 	@echo "Re-Generating mocks"

cla-backend-go/cmd/dynamo_events_lambda/main.go

@@ -91,7 +91,7 @@ func init() {
 	githubOrganizationsRepo := github_organizations.NewRepository(awsSession, stage)
 
 	token.Init(configFile.Auth0Platform.ClientID, configFile.Auth0Platform.ClientSecret, configFile.Auth0Platform.URL, configFile.Auth0Platform.Audience)
-	github.Init(configFile.Github.AppID, configFile.Github.AppPrivateKey, configFile.Github.AccessToken)
+	github.Init(configFile.GitHub.AppID, configFile.GitHub.AppPrivateKey, configFile.GitHub.AccessToken)
 
 	user_service.InitClient(configFile.APIGatewayURL, configFile.AcsAPIKey)
 	project_service.InitClient(configFile.APIGatewayURL)

cla-backend-go/cmd/server.go

@@ -223,7 +223,7 @@ func server(localMode bool) http.Handler {
 	if err != nil {
 		logrus.Panic(err)
 	}
-	github.Init(configFile.Github.AppID, configFile.Github.AppPrivateKey, configFile.Github.AccessToken)
+	github.Init(configFile.GitHub.AppID, configFile.GitHub.AppPrivateKey, configFile.GitHub.AccessToken)
 
 	// Our backend repository handlers
 	userRepo := user.NewDynamoRepository(awsSession, stage)
@@ -304,7 +304,7 @@ func server(localMode bool) http.Handler {
 	v2Health.Configure(v2API, healthService)
 	template.Configure(api, templateService, eventsService)
 	v2Template.Configure(v2API, templateService, eventsService)
-	github.Configure(api, configFile.Github.ClientID, configFile.Github.ClientSecret, configFile.Github.AccessToken, sessionStore)
+	github.Configure(api, configFile.GitHub.ClientID, configFile.GitHub.ClientSecret, configFile.GitHub.AccessToken, sessionStore)
 	signatures.Configure(api, v1SignaturesService, sessionStore, eventsService)
 	v2Signatures.Configure(v2API, v1ProjectService, projectRepo, v1CompanyService, v1SignaturesService, sessionStore, eventsService, v2SignatureService, projectClaGroupRepo)
 	approval_list.Configure(api, v1ApprovalListService, sessionStore, v1SignaturesService, eventsService)

cla-backend-go/company/repository.go

@@ -5,7 +5,6 @@ package company
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"strings"
 
@@ -28,11 +27,6 @@ import (
 	"github.com/gofrs/uuid"
 )
 
-// errors
-var (
-	ErrCompanyDoesNotExist = errors.New("company does not exist")
-)
-
 // IRepository interface methods
 type IRepository interface { //nolint
 	CreateCompany(ctx context.Context, in *models.Company) (*models.Company, error)
@@ -176,8 +170,12 @@ func (repo repository) GetCompanyByExternalID(ctx context.Context, companySFID s
 	}
 	if len(companyRecords) == 0 {
 		log.WithFields(f).Debug("no records found")
-		return nil, ErrCompanyDoesNotExist
+		return nil, &utils.CompanyNotFound{
+			Message:   "no company records found for SFID",
+			CompanyID: companySFID,
+		}
 	}
+
 	log.WithFields(f).Debugf("loaded %d records", len(companyRecords))
 	// For debug when problems occur
 	f["companyName"] = companyRecords[0].CompanyName
@@ -195,7 +193,10 @@ func (repo repository) GetCompanyByExternalID(ctx context.Context, companySFID s
 	}
 	f["signingEntityNames"] = strings.Join(signingEntityNames, ";")
 	log.WithFields(f).Warning("unable to match company name with existing signing entity names")
-	return nil, ErrCompanyDoesNotExist
+	return nil, &utils.CompanyNotFound{
+		Message:   "company record not found - unable to match company name with existing signing entity names",
+		CompanyID: companySFID,
+	}
 }
 
 // GetCompaniesByExternalID returns a list of companies based on the company external ID. A company will have more than one if/when the SF record has multiple entity names - for which we create separate EasyCLA company records
@@ -232,9 +233,13 @@ func (repo repository) GetCompaniesByExternalID(ctx context.Context, companySFID
 	}
 
 	if len(results.Items) == 0 {
-		log.WithFields(f).Debug("no records found")
-		return nil, ErrCompanyDoesNotExist
+		log.WithFields(f).Debug("no company records found")
+		return nil, &utils.CompanyNotFound{
+			Message:     "no company records found with matching external SFID",
+			CompanySFID: companySFID,
+		}
 	}
+
 	var dbCompanyModels []DBModel
 	err = dynamodbattribute.UnmarshalListOfMaps(results.Items, &dbCompanyModels)
 	if err != nil {
@@ -279,8 +284,13 @@ func (repo repository) GetCompanyBySigningEntityName(ctx context.Context, signin
 	}
 
 	if len(results.Items) == 0 {
-		return nil, ErrCompanyDoesNotExist
+		return nil, &utils.CompanyNotFound{
+			Message:                  "no company with signing entity name found",
+			CompanySigningEntityName: signingEntityName,
+			Err:                      nil,
+		}
 	}
+
 	dbCompanyModel := DBModel{}
 	err = dynamodbattribute.UnmarshalMap(results.Items[0], &dbCompanyModel)
 	if err != nil {
@@ -366,7 +376,10 @@ func (repo repository) GetCompany(ctx context.Context, companyID string) (*model
 	}
 
 	if len(companyTableData.Item) == 0 {
-		return nil, ErrCompanyDoesNotExist
+		return nil, &utils.CompanyNotFound{
+			Message:   "no company matching company record",
+			CompanyID: companyID,
+		}
 	}
 
 	dbCompanyModel := DBModel{}

cla-backend-go/company/service.go

@@ -634,7 +634,7 @@ func (s service) GetCompanyByExternalID(ctx context.Context, companySFID string)
 		return comp, nil
 	}
 
-	if err == ErrCompanyDoesNotExist {
+	if _, ok := err.(*utils.CompanyNotFound); ok {
 		comp, err = s.CreateOrgFromExternalID(ctx, "", companySFID)
 		if err != nil {
 			return comp, err
@@ -675,7 +675,7 @@ func (s service) GetCompanyBySigningEntityName(ctx context.Context, signingEntit
 		return comp, nil
 	}
 
-	if err == ErrCompanyDoesNotExist {
+	if _, ok := err.(*utils.CompanyNotFound); ok {
 		log.WithFields(f).Debugf("Company with signing entity name %s does not exist", signingEntityName)
 		comp, err = s.CreateOrgFromExternalID(ctx, signingEntityName, companySFID)
 		if err != nil {

cla-backend-go/config/config.go

@@ -38,8 +38,8 @@ type Config struct {
 	// AWS
 	AWS AWS `json:"aws"`
 
-	// Github Application
-	Github Github `json:"github"`
+	// GitHub Application
+	GitHub GitHub `json:"github"`
 
 	// Dynamo Session Store
 	SessionStoreTableName string `json:"sessionStoreTableName"`
@@ -110,13 +110,17 @@ type AWS struct {
 	Region string `json:"region"`
 }
 
-// Github model
-type Github struct {
-	ClientID      string `json:"clientId"`
-	ClientSecret  string `json:"clientSecret"`
-	AccessToken   string `json:"accessToken"`
-	AppID         int    `json:"app_id"`
-	AppPrivateKey string `json:"app_private_key"`
+// GitHub model
+type GitHub struct {
+	ClientID                       string `json:"clientId"`
+	ClientSecret                   string `json:"clientSecret"`
+	AccessToken                    string `json:"accessToken"`
+	AppID                          int    `json:"app_id"`
+	AppPrivateKey                  string `json:"app_private_key"`
+	TestOrganization               string `json:"test_organization"`
+	TestOrganizationInstallationID string `json:"test_organization_installation_id"`
+	TestRepository                 string `json:"test_repository"`
+	TestRepositoryID               string `json:"test_repository_id"`
 }
 
 // MetricsReport keeps the config needed to send the metrics data report

cla-backend-go/config/ssm.go

@@ -65,6 +65,10 @@ func loadSSMConfig(awsSession *session.Session, stage string) Config { //nolint
 		fmt.Sprintf("cla-gh-access-token-%s", stage),
 		fmt.Sprintf("cla-gh-app-id-%s", stage),
 		fmt.Sprintf("cla-gh-app-private-key-%s", stage),
+		fmt.Sprintf("cla-gh-test-organization-%s", stage),
+		fmt.Sprintf("cla-gh-test-organization-installation-id-%s", stage),
+		fmt.Sprintf("cla-gh-test-repository-%s", stage),
+		fmt.Sprintf("cla-gh-test-repository-id-%s", stage),
 		fmt.Sprintf("cla-corporate-base-%s", stage),
 		fmt.Sprintf("cla-corporate-v2-base-%s", stage),
 		fmt.Sprintf("cla-doc-raptor-api-key-%s", stage),
@@ -118,20 +122,28 @@ func loadSSMConfig(awsSession *session.Session, stage string) Config { //nolint
 		case fmt.Sprintf("cla-auth0-algorithm-%s", stage):
 			config.Auth0.Algorithm = resp.value
 		case fmt.Sprintf("cla-gh-oauth-client-id-go-backend-%s", stage):
-			config.Github.ClientID = resp.value
+			config.GitHub.ClientID = resp.value
 		case fmt.Sprintf("cla-gh-oauth-secret-go-backend-%s", stage):
-			config.Github.ClientSecret = resp.value
+			config.GitHub.ClientSecret = resp.value
 		case fmt.Sprintf("cla-gh-access-token-%s", stage):
-			config.Github.AccessToken = resp.value
+			config.GitHub.AccessToken = resp.value
 		case fmt.Sprintf("cla-gh-app-id-%s", stage):
 			githubAppID, err := strconv.Atoi(resp.value)
 			if err != nil {
 				errMsg := fmt.Sprintf("invalid value of key: %s", fmt.Sprintf("cla-gh-app-id-%s", stage))
 				log.WithFields(f).WithError(err).Fatal(errMsg)
 			}
-			config.Github.AppID = githubAppID
+			config.GitHub.AppID = githubAppID
 		case fmt.Sprintf("cla-gh-app-private-key-%s", stage):
-			config.Github.AppPrivateKey = resp.value
+			config.GitHub.AppPrivateKey = resp.value
+		case fmt.Sprintf("cla-gh-test-organization-%s", stage):
+			config.GitHub.TestOrganization = resp.value
+		case fmt.Sprintf("cla-gh-test-organization-installation-id-%s", stage):
+			config.GitHub.TestOrganizationInstallationID = resp.value
+		case fmt.Sprintf("cla-gh-test-repository-%s", stage):
+			config.GitHub.TestRepository = resp.value
+		case fmt.Sprintf("cla-gh-test-repository-id-%s", stage):
+			config.GitHub.TestRepositoryID = resp.value
 
 		case fmt.Sprintf("cla-corporate-base-%s", stage):
 			corporateConsoleURLValue := resp.value

cla-backend-go/github/.graphqlconfig

@@ -0,0 +1,16 @@
+{
+  "name": "GitHub API V4 GraphQL Schema",
+  "schemaPath": "github-schema.graphql",
+  "extensions": {
+    "endpoints": {
+      "GitHub API V4 GraphQL Endpoint": {
+        "url": "https://api.github.com/graphql",
+        "headers": {
+          "Authorization": "Bearer ${env:GITHUB-ACCESS-TOKEN}",
+          "user-agent": "JS GraphQL"
+        },
+        "introspect": false
+      }
+    }
+  }
+}

cla-backend-go/github/client.go

@@ -8,6 +8,9 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"time"
+
+	"github.com/shurcooL/githubv4"
 
 	"github.com/bradleyfalzon/ghinstallation"
 	"github.com/google/go-github/v33/github"
@@ -77,6 +80,15 @@ func NewGithubAppClient(installationID int64) (*github.Client, error) {
 	return github.NewClient(&http.Client{Transport: itr}), nil
 }
 
+// NewGithubV4AppClient creates a new github v4 client from the supplied installationID
+func NewGithubV4AppClient(installationID int64) (*githubv4.Client, error) {
+	authTransport, err := ghinstallation.New(http.DefaultTransport, int64(getGithubAppID()), installationID, []byte(getGithubAppPrivateKey()))
+	if err != nil {
+		return nil, err
+	}
+	return githubv4.NewClient(&http.Client{Transport: authTransport, Timeout: 5 * time.Second}), nil
+}
+
 // NewGithubOauthClient creates github client from global accessToken
 func NewGithubOauthClient() *github.Client {
 	return NewGithubOauthClientWithAccessToken(getSecretAccessToken())

cla-backend-go/github/github-query.graphql

@@ -0,0 +1,25 @@
+query BranchProtectionRule($organizationOwner: String!, $repositoryName: String!) {
+    repository(owner: $organizationOwner, name: $repositoryName) {
+        id
+        createdAt
+        branchProtectionRules(first: 100) {
+            totalCount
+            nodes {
+                pattern
+                id
+                allowsDeletions
+                requiredApprovingReviewCount
+                requiredStatusCheckContexts
+            }
+            edges {
+                node {
+                    allowsDeletions
+                    id
+                    pattern
+                }
+            }
+        }
+        diskUsage
+        hasIssuesEnabled
+    }
+}