diff --git a/internal/providers/gitlab/gitlab.go b/internal/providers/gitlab/gitlab.go index a1058ee1d7..effd2da25c 100644 --- a/internal/providers/gitlab/gitlab.go +++ b/internal/providers/gitlab/gitlab.go @@ -19,6 +19,7 @@ package gitlab import ( "context" "encoding/json" + "errors" "net/http" "golang.org/x/oauth2" @@ -37,6 +38,7 @@ const Class = "gitlab" var Implements = []db.ProviderType{ db.ProviderTypeGit, db.ProviderTypeRest, + db.ProviderTypeRepoLister, } // AuthorizationFlows is the list of authorization flows that the DockerHub provider supports @@ -48,8 +50,7 @@ var AuthorizationFlows = []db.AuthorizationFlow{ // Ensure that the GitLab provider implements the right interfaces var _ provifv1.Git = (*gitlabClient)(nil) var _ provifv1.REST = (*gitlabClient)(nil) - -// var _ provifv1.RepoLister = (*gitlabClient)(nil) +var _ provifv1.RepoLister = (*gitlabClient)(nil) type gitlabClient struct { cred provifv1.GitLabCredential @@ -87,6 +88,12 @@ func ParseV1Config(rawCfg json.RawMessage) (*minderv1.GitLabProviderConfig, erro if err := json.Unmarshal(rawCfg, &cfg); err != nil { return nil, err } + + if cfg.GitLab == nil { + // Return a default but working config + return &minderv1.GitLabProviderConfig{}, nil + } + return cfg.GitLab, nil } @@ -123,11 +130,16 @@ func (_ *gitlabClient) SupportsEntity(entType minderv1.Entity) bool { } // RegisterEntity implements the Provider interface -func (_ *gitlabClient) RegisterEntity( - _ context.Context, _ minderv1.Entity, _ *properties.Properties, +func (c *gitlabClient) RegisterEntity( + _ context.Context, entType minderv1.Entity, props *properties.Properties, ) (*properties.Properties, error) { + if !c.SupportsEntity(entType) { + return nil, errors.New("unsupported entity type") + } + // TODO: implement - return nil, nil + + return props, nil } // DeregisterEntity implements the Provider interface diff --git a/internal/providers/gitlab/gitlab_rest.go b/internal/providers/gitlab/gitlab_rest.go index 605bab041e..8a3e8c41f2 100644 --- a/internal/providers/gitlab/gitlab_rest.go +++ b/internal/providers/gitlab/gitlab_rest.go @@ -23,6 +23,8 @@ import ( "io" "net/http" "net/url" + + provifv1 "github.com/stacklok/minder/pkg/providers/v1" ) // Do implements the REST provider interface @@ -76,3 +78,39 @@ func (c *gitlabClient) NewRequest(method, requestUrl string, body any) (*http.Re return req, nil } + +type genericRESTClient interface { + // Do sends an HTTP request and returns an HTTP response + Do(ctx context.Context, req *http.Request) (*http.Response, error) + NewRequest(method, requestUrl string, body any) (*http.Request, error) +} + +// NOTE: We're not using github.com/xanzy/go-gitlab to do the actual +// request here because of the way they form authentication for requests. +// It would be ideal to use it, so we should consider contributing and making +// that part more pluggable. +func glREST[T any](ctx context.Context, cli genericRESTClient, method, requestUrl string, body any, out T) error { + req, err := cli.NewRequest(method, requestUrl, body) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + resp, err := cli.Do(ctx, req) + if err != nil { + return fmt.Errorf("failed to get projects: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusNotFound { + return provifv1.ErrEntityNotFound + } + return fmt.Errorf("failed to get projects: %s", resp.Status) + } + + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return fmt.Errorf("failed to decode response: %w", err) + } + + return nil +} diff --git a/internal/providers/gitlab/manager/manager.go b/internal/providers/gitlab/manager/manager.go index bd2090ca43..0e039bc8cb 100644 --- a/internal/providers/gitlab/manager/manager.go +++ b/internal/providers/gitlab/manager/manager.go @@ -74,10 +74,7 @@ func (g *providerClassManager) Build(ctx context.Context, config *db.Provider) ( return nil, fmt.Errorf("error parsing gitlab config: %w", err) } - cli, err := gitlab.New( - creds, - cfg, - ) + cli, err := gitlab.New(creds, cfg) if err != nil { return nil, fmt.Errorf("error creating gitlab client: %w", err) } diff --git a/internal/providers/gitlab/properties.go b/internal/providers/gitlab/properties.go index 1d344430dc..e2ff6c0885 100644 --- a/internal/providers/gitlab/properties.go +++ b/internal/providers/gitlab/properties.go @@ -20,15 +20,23 @@ import ( "errors" "fmt" + "google.golang.org/protobuf/reflect/protoreflect" + "github.com/stacklok/minder/internal/entities/properties" minderv1 "github.com/stacklok/minder/pkg/api/protobuf/go/minder/v1" ) const ( - // RepoPropertyGroupName represents the gitlab group - RepoPropertyGroupName = "gitlab/group_name" // RepoPropertyProjectName represents the gitlab project RepoPropertyProjectName = "gitlab/project_name" + // RepoPropertyDefaultBranch represents the gitlab default branch + RepoPropertyDefaultBranch = "gitlab/default_branch" + // RepoPropertyNamespace represents the gitlab repo namespace + RepoPropertyNamespace = "gitlab/namespace" + // RepoPropertyLicense represents the gitlab repo license + RepoPropertyLicense = "gitlab/license" + // RepoPropertyCloneURL represents the gitlab repo clone URL + RepoPropertyCloneURL = "gitlab/clone_url" ) // FetchAllProperties implements the provider interface @@ -59,7 +67,7 @@ func (c *gitlabClient) GetEntityName(entityType minderv1.Entity, props *properti return "", fmt.Errorf("entity type %s not supported", entityType) } - groupName, err := getStringProp(props, RepoPropertyGroupName) + groupName, err := getStringProp(props, RepoPropertyNamespace) if err != nil { return "", err } @@ -72,6 +80,17 @@ func (c *gitlabClient) GetEntityName(entityType minderv1.Entity, props *properti return formatRepoName(groupName, projectName), nil } +// PropertiesToProtoMessage implements the ProtoMessageConverter interface +func (c *gitlabClient) PropertiesToProtoMessage( + entType minderv1.Entity, props *properties.Properties, +) (protoreflect.ProtoMessage, error) { + if !c.SupportsEntity(entType) { + return nil, fmt.Errorf("entity type %s is not supported by the gitlab provider", entType) + } + + return repoV1FromProperties(props) +} + func getStringProp(props *properties.Properties, key string) (string, error) { value, err := props.GetProperty(key).AsString() if err != nil { diff --git a/internal/providers/gitlab/properties_test.go b/internal/providers/gitlab/properties_test.go index b056687660..48f375710f 100644 --- a/internal/providers/gitlab/properties_test.go +++ b/internal/providers/gitlab/properties_test.go @@ -57,7 +57,7 @@ func Test_gitlabClient_GetEntityName(t *testing.T) { args: args{ entityType: minderv1.Entity_ENTITY_REPOSITORIES, props: MustNewProperties(map[string]any{ - RepoPropertyGroupName: "group", + RepoPropertyNamespace: "group", RepoPropertyProjectName: "project", }), }, @@ -69,7 +69,7 @@ func Test_gitlabClient_GetEntityName(t *testing.T) { args: args{ entityType: minderv1.Entity_ENTITY_REPOSITORIES, props: MustNewProperties(map[string]any{ - RepoPropertyGroupName: "group", + RepoPropertyNamespace: "group", }), }, want: "", @@ -164,6 +164,9 @@ func Test_gitlabClient_FetchAllProperties(t *testing.T) { Visibility: gitlab.PrivateVisibility, Archived: false, ForkedFromProject: nil, + Namespace: &gitlab.ProjectNamespace{ + Path: "group", + }, } w.Header().Set("Content-Type", "application/json") diff --git a/internal/providers/gitlab/repo_lister.go b/internal/providers/gitlab/repo_lister.go new file mode 100644 index 0000000000..c69a06329f --- /dev/null +++ b/internal/providers/gitlab/repo_lister.go @@ -0,0 +1,78 @@ +// +// Copyright 2024 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gitlab + +import ( + "context" + "fmt" + "net/http" + "net/url" + + "github.com/rs/zerolog" + "github.com/xanzy/go-gitlab" + + minderv1 "github.com/stacklok/minder/pkg/api/protobuf/go/minder/v1" +) + +func (c *gitlabClient) ListAllRepositories(ctx context.Context) ([]*minderv1.Repository, error) { + groups := []*gitlab.Group{} + if err := glREST(ctx, c, http.MethodGet, "groups", nil, &groups); err != nil { + return nil, fmt.Errorf("failed to get groups: %w", err) + } + + if len(groups) == 0 { + zerolog.Ctx(ctx).Debug().Msg("no groups found") + return nil, nil + } + + var repos []*minderv1.Repository + for _, g := range groups { + projs := []*gitlab.Project{} + path, err := url.JoinPath("groups", fmt.Sprintf("%d", g.ID), "projects") + if err != nil { + return nil, fmt.Errorf("failed to join URL path for projects: %w", err) + } + if err := glREST(ctx, c, http.MethodGet, path, nil, &projs); err != nil { + return nil, fmt.Errorf("failed to get projects for group %s: %w", g.FullPath, err) + } + + if repos == nil { + repos = make([]*minderv1.Repository, 0, len(projs)) + } + + if len(projs) == 0 { + zerolog.Ctx(ctx).Debug().Msgf("no projects found for group %s", g.FullPath) + } + + for _, p := range projs { + props, err := gitlabProjectToProperties(p) + if err != nil { + return nil, fmt.Errorf("failed to convert project to properties: %w", err) + } + + outRep, err := repoV1FromProperties(props) + if err != nil { + return nil, fmt.Errorf("failed to convert properties to repository: %w", err) + } + + repos = append(repos, outRep) + } + } + + zerolog.Ctx(ctx).Debug().Int("num_repos", len(repos)).Msg("found repositories in gitlab provider") + + return repos, nil +} diff --git a/internal/providers/gitlab/repository_properties.go b/internal/providers/gitlab/repository_properties.go index d6d74cd19c..91199a1e6c 100644 --- a/internal/providers/gitlab/repository_properties.go +++ b/internal/providers/gitlab/repository_properties.go @@ -21,10 +21,12 @@ import ( "fmt" "net/http" "net/url" + "strconv" "github.com/xanzy/go-gitlab" "github.com/stacklok/minder/internal/entities/properties" + minderv1 "github.com/stacklok/minder/pkg/api/protobuf/go/minder/v1" provifv1 "github.com/stacklok/minder/pkg/providers/v1" ) @@ -68,14 +70,88 @@ func (c *gitlabClient) getPropertiesForRepo( return nil, fmt.Errorf("failed to decode response: %w", err) } + outProps, err := gitlabProjectToProperties(proj) + if err != nil { + return nil, fmt.Errorf("failed to convert project to properties: %w", err) + } + + return getByProps.Merge(outProps), nil +} + +func gitlabProjectToProperties(proj *gitlab.Project) (*properties.Properties, error) { + ns := proj.Namespace + if ns == nil { + return nil, fmt.Errorf("gitlab project %d has no namespace", proj.ID) + } + owner := ns.Path + + var license string + if proj.License != nil { + license = proj.License.Name + } + outProps, err := properties.NewProperties(map[string]any{ + properties.PropertyUpstreamID: fmt.Sprintf("%d", proj.ID), + properties.PropertyName: formatRepoName(owner, proj.Name), properties.RepoPropertyIsPrivate: proj.Visibility == gitlab.PrivateVisibility, properties.RepoPropertyIsArchived: proj.Archived, properties.RepoPropertyIsFork: proj.ForkedFromProject != nil, + RepoPropertyDefaultBranch: proj.DefaultBranch, + RepoPropertyNamespace: owner, + RepoPropertyProjectName: proj.Name, + RepoPropertyLicense: license, + RepoPropertyCloneURL: proj.HTTPURLToRepo, }) if err != nil { return nil, fmt.Errorf("failed to create properties: %w", err) } - return getByProps.Merge(outProps), nil + return outProps, nil +} + +func repoV1FromProperties(repoProperties *properties.Properties) (*minderv1.Repository, error) { + upstreamID, err := repoProperties.GetProperty(properties.PropertyUpstreamID).AsString() + if err != nil { + return nil, fmt.Errorf("error fetching upstream ID property: %w", err) + } + + // convert the upstream ID to an int64 + repoId, err := strconv.ParseInt(upstreamID, 10, 64) + if err != nil { + return nil, fmt.Errorf("error converting upstream ID to int64: %w", err) + } + + name, err := repoProperties.GetProperty(RepoPropertyProjectName).AsString() + if err != nil { + return nil, fmt.Errorf("error fetching project property: %w", err) + } + + owner, err := repoProperties.GetProperty(RepoPropertyNamespace).AsString() + if err != nil { + return nil, fmt.Errorf("error fetching namespace property: %w", err) + } + + isPrivate, err := repoProperties.GetProperty(properties.RepoPropertyIsPrivate).AsBool() + if err != nil { + return nil, fmt.Errorf("error fetching is_private property: %w", err) + } + + isFork, err := repoProperties.GetProperty(properties.RepoPropertyIsFork).AsBool() + if err != nil { + return nil, fmt.Errorf("error fetching is_fork property: %w", err) + } + + pbRepo := &minderv1.Repository{ + Name: name, + Owner: owner, + RepoId: repoId, + CloneUrl: repoProperties.GetProperty(RepoPropertyCloneURL).GetString(), + IsPrivate: isPrivate, + IsFork: isFork, + DefaultBranch: repoProperties.GetProperty(RepoPropertyDefaultBranch).GetString(), + License: repoProperties.GetProperty(RepoPropertyLicense).GetString(), + Properties: repoProperties.ToProtoStruct(), + } + + return pbRepo, nil }