From b3975ffe0cce3e0f39fc99f7948abed7e500172d Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Thu, 12 Sep 2024 15:30:40 +0300 Subject: [PATCH] Implement basic repo registration with gitlab This adds the needed pieces for the GitLab provider to do entity registration. In the case of repos, this was the implementation of the repo lister trait. Signed-off-by: Juan Antonio Osorio --- internal/providers/gitlab/gitlab.go | 22 ++++-- internal/providers/gitlab/gitlab_rest.go | 38 +++++++++ internal/providers/gitlab/manager/manager.go | 5 +- internal/providers/gitlab/properties.go | 25 +++++- internal/providers/gitlab/properties_test.go | 7 +- internal/providers/gitlab/repo_lister.go | 78 +++++++++++++++++++ .../providers/gitlab/repository_properties.go | 78 ++++++++++++++++++- 7 files changed, 238 insertions(+), 15 deletions(-) create mode 100644 internal/providers/gitlab/repo_lister.go diff --git a/internal/providers/gitlab/gitlab.go b/internal/providers/gitlab/gitlab.go index a1058ee1d7..effd2da25c 100644 --- a/internal/providers/gitlab/gitlab.go +++ b/internal/providers/gitlab/gitlab.go @@ -19,6 +19,7 @@ package gitlab import ( "context" "encoding/json" + "errors" "net/http" "golang.org/x/oauth2" @@ -37,6 +38,7 @@ const Class = "gitlab" var Implements = []db.ProviderType{ db.ProviderTypeGit, db.ProviderTypeRest, + db.ProviderTypeRepoLister, } // AuthorizationFlows is the list of authorization flows that the DockerHub provider supports @@ -48,8 +50,7 @@ var AuthorizationFlows = []db.AuthorizationFlow{ // Ensure that the GitLab provider implements the right interfaces var _ provifv1.Git = (*gitlabClient)(nil) var _ provifv1.REST = (*gitlabClient)(nil) - -// var _ provifv1.RepoLister = (*gitlabClient)(nil) +var _ provifv1.RepoLister = (*gitlabClient)(nil) type gitlabClient struct { cred provifv1.GitLabCredential @@ -87,6 +88,12 @@ func ParseV1Config(rawCfg json.RawMessage) (*minderv1.GitLabProviderConfig, erro if err := json.Unmarshal(rawCfg, &cfg); err != nil { return nil, err } + + if cfg.GitLab == nil { + // Return a default but working config + return &minderv1.GitLabProviderConfig{}, nil + } + return cfg.GitLab, nil } @@ -123,11 +130,16 @@ func (_ *gitlabClient) SupportsEntity(entType minderv1.Entity) bool { } // RegisterEntity implements the Provider interface -func (_ *gitlabClient) RegisterEntity( - _ context.Context, _ minderv1.Entity, _ *properties.Properties, +func (c *gitlabClient) RegisterEntity( + _ context.Context, entType minderv1.Entity, props *properties.Properties, ) (*properties.Properties, error) { + if !c.SupportsEntity(entType) { + return nil, errors.New("unsupported entity type") + } + // TODO: implement - return nil, nil + + return props, nil } // DeregisterEntity implements the Provider interface diff --git a/internal/providers/gitlab/gitlab_rest.go b/internal/providers/gitlab/gitlab_rest.go index 605bab041e..8a3e8c41f2 100644 --- a/internal/providers/gitlab/gitlab_rest.go +++ b/internal/providers/gitlab/gitlab_rest.go @@ -23,6 +23,8 @@ import ( "io" "net/http" "net/url" + + provifv1 "github.com/stacklok/minder/pkg/providers/v1" ) // Do implements the REST provider interface @@ -76,3 +78,39 @@ func (c *gitlabClient) NewRequest(method, requestUrl string, body any) (*http.Re return req, nil } + +type genericRESTClient interface { + // Do sends an HTTP request and returns an HTTP response + Do(ctx context.Context, req *http.Request) (*http.Response, error) + NewRequest(method, requestUrl string, body any) (*http.Request, error) +} + +// NOTE: We're not using github.com/xanzy/go-gitlab to do the actual +// request here because of the way they form authentication for requests. +// It would be ideal to use it, so we should consider contributing and making +// that part more pluggable. +func glREST[T any](ctx context.Context, cli genericRESTClient, method, requestUrl string, body any, out T) error { + req, err := cli.NewRequest(method, requestUrl, body) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + resp, err := cli.Do(ctx, req) + if err != nil { + return fmt.Errorf("failed to get projects: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusNotFound { + return provifv1.ErrEntityNotFound + } + return fmt.Errorf("failed to get projects: %s", resp.Status) + } + + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return fmt.Errorf("failed to decode response: %w", err) + } + + return nil +} diff --git a/internal/providers/gitlab/manager/manager.go b/internal/providers/gitlab/manager/manager.go index bd2090ca43..0e039bc8cb 100644 --- a/internal/providers/gitlab/manager/manager.go +++ b/internal/providers/gitlab/manager/manager.go @@ -74,10 +74,7 @@ func (g *providerClassManager) Build(ctx context.Context, config *db.Provider) ( return nil, fmt.Errorf("error parsing gitlab config: %w", err) } - cli, err := gitlab.New( - creds, - cfg, - ) + cli, err := gitlab.New(creds, cfg) if err != nil { return nil, fmt.Errorf("error creating gitlab client: %w", err) } diff --git a/internal/providers/gitlab/properties.go b/internal/providers/gitlab/properties.go index 1d344430dc..e2ff6c0885 100644 --- a/internal/providers/gitlab/properties.go +++ b/internal/providers/gitlab/properties.go @@ -20,15 +20,23 @@ import ( "errors" "fmt" + "google.golang.org/protobuf/reflect/protoreflect" + "github.com/stacklok/minder/internal/entities/properties" minderv1 "github.com/stacklok/minder/pkg/api/protobuf/go/minder/v1" ) const ( - // RepoPropertyGroupName represents the gitlab group - RepoPropertyGroupName = "gitlab/group_name" // RepoPropertyProjectName represents the gitlab project RepoPropertyProjectName = "gitlab/project_name" + // RepoPropertyDefaultBranch represents the gitlab default branch + RepoPropertyDefaultBranch = "gitlab/default_branch" + // RepoPropertyNamespace represents the gitlab repo namespace + RepoPropertyNamespace = "gitlab/namespace" + // RepoPropertyLicense represents the gitlab repo license + RepoPropertyLicense = "gitlab/license" + // RepoPropertyCloneURL represents the gitlab repo clone URL + RepoPropertyCloneURL = "gitlab/clone_url" ) // FetchAllProperties implements the provider interface @@ -59,7 +67,7 @@ func (c *gitlabClient) GetEntityName(entityType minderv1.Entity, props *properti return "", fmt.Errorf("entity type %s not supported", entityType) } - groupName, err := getStringProp(props, RepoPropertyGroupName) + groupName, err := getStringProp(props, RepoPropertyNamespace) if err != nil { return "", err } @@ -72,6 +80,17 @@ func (c *gitlabClient) GetEntityName(entityType minderv1.Entity, props *properti return formatRepoName(groupName, projectName), nil } +// PropertiesToProtoMessage implements the ProtoMessageConverter interface +func (c *gitlabClient) PropertiesToProtoMessage( + entType minderv1.Entity, props *properties.Properties, +) (protoreflect.ProtoMessage, error) { + if !c.SupportsEntity(entType) { + return nil, fmt.Errorf("entity type %s is not supported by the gitlab provider", entType) + } + + return repoV1FromProperties(props) +} + func getStringProp(props *properties.Properties, key string) (string, error) { value, err := props.GetProperty(key).AsString() if err != nil { diff --git a/internal/providers/gitlab/properties_test.go b/internal/providers/gitlab/properties_test.go index b056687660..48f375710f 100644 --- a/internal/providers/gitlab/properties_test.go +++ b/internal/providers/gitlab/properties_test.go @@ -57,7 +57,7 @@ func Test_gitlabClient_GetEntityName(t *testing.T) { args: args{ entityType: minderv1.Entity_ENTITY_REPOSITORIES, props: MustNewProperties(map[string]any{ - RepoPropertyGroupName: "group", + RepoPropertyNamespace: "group", RepoPropertyProjectName: "project", }), }, @@ -69,7 +69,7 @@ func Test_gitlabClient_GetEntityName(t *testing.T) { args: args{ entityType: minderv1.Entity_ENTITY_REPOSITORIES, props: MustNewProperties(map[string]any{ - RepoPropertyGroupName: "group", + RepoPropertyNamespace: "group", }), }, want: "", @@ -164,6 +164,9 @@ func Test_gitlabClient_FetchAllProperties(t *testing.T) { Visibility: gitlab.PrivateVisibility, Archived: false, ForkedFromProject: nil, + Namespace: &gitlab.ProjectNamespace{ + Path: "group", + }, } w.Header().Set("Content-Type", "application/json") diff --git a/internal/providers/gitlab/repo_lister.go b/internal/providers/gitlab/repo_lister.go new file mode 100644 index 0000000000..c69a06329f --- /dev/null +++ b/internal/providers/gitlab/repo_lister.go @@ -0,0 +1,78 @@ +// +// Copyright 2024 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gitlab + +import ( + "context" + "fmt" + "net/http" + "net/url" + + "github.com/rs/zerolog" + "github.com/xanzy/go-gitlab" + + minderv1 "github.com/stacklok/minder/pkg/api/protobuf/go/minder/v1" +) + +func (c *gitlabClient) ListAllRepositories(ctx context.Context) ([]*minderv1.Repository, error) { + groups := []*gitlab.Group{} + if err := glREST(ctx, c, http.MethodGet, "groups", nil, &groups); err != nil { + return nil, fmt.Errorf("failed to get groups: %w", err) + } + + if len(groups) == 0 { + zerolog.Ctx(ctx).Debug().Msg("no groups found") + return nil, nil + } + + var repos []*minderv1.Repository + for _, g := range groups { + projs := []*gitlab.Project{} + path, err := url.JoinPath("groups", fmt.Sprintf("%d", g.ID), "projects") + if err != nil { + return nil, fmt.Errorf("failed to join URL path for projects: %w", err) + } + if err := glREST(ctx, c, http.MethodGet, path, nil, &projs); err != nil { + return nil, fmt.Errorf("failed to get projects for group %s: %w", g.FullPath, err) + } + + if repos == nil { + repos = make([]*minderv1.Repository, 0, len(projs)) + } + + if len(projs) == 0 { + zerolog.Ctx(ctx).Debug().Msgf("no projects found for group %s", g.FullPath) + } + + for _, p := range projs { + props, err := gitlabProjectToProperties(p) + if err != nil { + return nil, fmt.Errorf("failed to convert project to properties: %w", err) + } + + outRep, err := repoV1FromProperties(props) + if err != nil { + return nil, fmt.Errorf("failed to convert properties to repository: %w", err) + } + + repos = append(repos, outRep) + } + } + + zerolog.Ctx(ctx).Debug().Int("num_repos", len(repos)).Msg("found repositories in gitlab provider") + + return repos, nil +} diff --git a/internal/providers/gitlab/repository_properties.go b/internal/providers/gitlab/repository_properties.go index d6d74cd19c..91199a1e6c 100644 --- a/internal/providers/gitlab/repository_properties.go +++ b/internal/providers/gitlab/repository_properties.go @@ -21,10 +21,12 @@ import ( "fmt" "net/http" "net/url" + "strconv" "github.com/xanzy/go-gitlab" "github.com/stacklok/minder/internal/entities/properties" + minderv1 "github.com/stacklok/minder/pkg/api/protobuf/go/minder/v1" provifv1 "github.com/stacklok/minder/pkg/providers/v1" ) @@ -68,14 +70,88 @@ func (c *gitlabClient) getPropertiesForRepo( return nil, fmt.Errorf("failed to decode response: %w", err) } + outProps, err := gitlabProjectToProperties(proj) + if err != nil { + return nil, fmt.Errorf("failed to convert project to properties: %w", err) + } + + return getByProps.Merge(outProps), nil +} + +func gitlabProjectToProperties(proj *gitlab.Project) (*properties.Properties, error) { + ns := proj.Namespace + if ns == nil { + return nil, fmt.Errorf("gitlab project %d has no namespace", proj.ID) + } + owner := ns.Path + + var license string + if proj.License != nil { + license = proj.License.Name + } + outProps, err := properties.NewProperties(map[string]any{ + properties.PropertyUpstreamID: fmt.Sprintf("%d", proj.ID), + properties.PropertyName: formatRepoName(owner, proj.Name), properties.RepoPropertyIsPrivate: proj.Visibility == gitlab.PrivateVisibility, properties.RepoPropertyIsArchived: proj.Archived, properties.RepoPropertyIsFork: proj.ForkedFromProject != nil, + RepoPropertyDefaultBranch: proj.DefaultBranch, + RepoPropertyNamespace: owner, + RepoPropertyProjectName: proj.Name, + RepoPropertyLicense: license, + RepoPropertyCloneURL: proj.HTTPURLToRepo, }) if err != nil { return nil, fmt.Errorf("failed to create properties: %w", err) } - return getByProps.Merge(outProps), nil + return outProps, nil +} + +func repoV1FromProperties(repoProperties *properties.Properties) (*minderv1.Repository, error) { + upstreamID, err := repoProperties.GetProperty(properties.PropertyUpstreamID).AsString() + if err != nil { + return nil, fmt.Errorf("error fetching upstream ID property: %w", err) + } + + // convert the upstream ID to an int64 + repoId, err := strconv.ParseInt(upstreamID, 10, 64) + if err != nil { + return nil, fmt.Errorf("error converting upstream ID to int64: %w", err) + } + + name, err := repoProperties.GetProperty(RepoPropertyProjectName).AsString() + if err != nil { + return nil, fmt.Errorf("error fetching project property: %w", err) + } + + owner, err := repoProperties.GetProperty(RepoPropertyNamespace).AsString() + if err != nil { + return nil, fmt.Errorf("error fetching namespace property: %w", err) + } + + isPrivate, err := repoProperties.GetProperty(properties.RepoPropertyIsPrivate).AsBool() + if err != nil { + return nil, fmt.Errorf("error fetching is_private property: %w", err) + } + + isFork, err := repoProperties.GetProperty(properties.RepoPropertyIsFork).AsBool() + if err != nil { + return nil, fmt.Errorf("error fetching is_fork property: %w", err) + } + + pbRepo := &minderv1.Repository{ + Name: name, + Owner: owner, + RepoId: repoId, + CloneUrl: repoProperties.GetProperty(RepoPropertyCloneURL).GetString(), + IsPrivate: isPrivate, + IsFork: isFork, + DefaultBranch: repoProperties.GetProperty(RepoPropertyDefaultBranch).GetString(), + License: repoProperties.GetProperty(RepoPropertyLicense).GetString(), + Properties: repoProperties.ToProtoStruct(), + } + + return pbRepo, nil }