Implement basic repo registration with gitlab

This adds the needed pieces for the GitLab provider to do entity
registration. In the case of repos, this was the implementation of the
repo lister trait.

Signed-off-by: Juan Antonio Osorio <[email protected]>

internal/providers/gitlab/gitlab.go

@@ -19,6 +19,7 @@ package gitlab
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"net/http"
 
 	"golang.org/x/oauth2"
@@ -37,6 +38,7 @@ const Class = "gitlab"
 var Implements = []db.ProviderType{
 	db.ProviderTypeGit,
 	db.ProviderTypeRest,
+	db.ProviderTypeRepoLister,
 }
 
 // AuthorizationFlows is the list of authorization flows that the DockerHub provider supports
@@ -48,8 +50,7 @@ var AuthorizationFlows = []db.AuthorizationFlow{
 // Ensure that the GitLab provider implements the right interfaces
 var _ provifv1.Git = (*gitlabClient)(nil)
 var _ provifv1.REST = (*gitlabClient)(nil)
-
-// var _ provifv1.RepoLister = (*gitlabClient)(nil)
+var _ provifv1.RepoLister = (*gitlabClient)(nil)
 
 type gitlabClient struct {
 	cred      provifv1.GitLabCredential
@@ -87,6 +88,12 @@ func ParseV1Config(rawCfg json.RawMessage) (*minderv1.GitLabProviderConfig, erro
 	if err := json.Unmarshal(rawCfg, &cfg); err != nil {
 		return nil, err
 	}
+
+	if cfg.GitLab == nil {
+		// Return a default but working config
+		return &minderv1.GitLabProviderConfig{}, nil
+	}
+
 	return cfg.GitLab, nil
 }
 
@@ -123,11 +130,16 @@ func (_ *gitlabClient) SupportsEntity(entType minderv1.Entity) bool {
 }
 
 // RegisterEntity implements the Provider interface
-func (_ *gitlabClient) RegisterEntity(
-	_ context.Context, _ minderv1.Entity, _ *properties.Properties,
+func (c *gitlabClient) RegisterEntity(
+	_ context.Context, entType minderv1.Entity, props *properties.Properties,
 ) (*properties.Properties, error) {
+	if !c.SupportsEntity(entType) {
+		return nil, errors.New("unsupported entity type")
+	}
+
 	// TODO: implement
-	return nil, nil
+
+	return props, nil
 }
 
 // DeregisterEntity implements the Provider interface

internal/providers/gitlab/gitlab_rest.go

@@ -23,6 +23,8 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+
+	provifv1 "github.com/stacklok/minder/pkg/providers/v1"
 )
 
 // Do implements the REST provider interface
@@ -76,3 +78,39 @@ func (c *gitlabClient) NewRequest(method, requestUrl string, body any) (*http.Re
 
 	return req, nil
 }
+
+type genericRESTClient interface {
+	// Do sends an HTTP request and returns an HTTP response
+	Do(ctx context.Context, req *http.Request) (*http.Response, error)
+	NewRequest(method, requestUrl string, body any) (*http.Request, error)
+}
+
+// NOTE: We're not using github.com/xanzy/go-gitlab to do the actual
+// request here because of the way they form authentication for requests.
+// It would be ideal to use it, so we should consider contributing and making
+// that part more pluggable.
+func glREST[T any](ctx context.Context, cli genericRESTClient, method, requestUrl string, body any, out T) error {
+	req, err := cli.NewRequest(method, requestUrl, body)
+	if err != nil {
+		return fmt.Errorf("failed to create request: %w", err)
+	}
+
+	resp, err := cli.Do(ctx, req)
+	if err != nil {
+		return fmt.Errorf("failed to get projects: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		if resp.StatusCode == http.StatusNotFound {
+			return provifv1.ErrEntityNotFound
+		}
+		return fmt.Errorf("failed to get projects: %s", resp.Status)
+	}
+
+	if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
+		return fmt.Errorf("failed to decode response: %w", err)
+	}
+
+	return nil
+}

internal/providers/gitlab/manager/manager.go

@@ -74,10 +74,7 @@ func (g *providerClassManager) Build(ctx context.Context, config *db.Provider) (
 		return nil, fmt.Errorf("error parsing gitlab config: %w", err)
 	}
 
-	cli, err := gitlab.New(
-		creds,
-		cfg,
-	)
+	cli, err := gitlab.New(creds, cfg)
 	if err != nil {
 		return nil, fmt.Errorf("error creating gitlab client: %w", err)
 	}

internal/providers/gitlab/properties.go

@@ -20,15 +20,23 @@ import (
 	"errors"
 	"fmt"
 
+	"google.golang.org/protobuf/reflect/protoreflect"
+
 	"github.com/stacklok/minder/internal/entities/properties"
 	minderv1 "github.com/stacklok/minder/pkg/api/protobuf/go/minder/v1"
 )
 
 const (
-	// RepoPropertyGroupName represents the gitlab group
-	RepoPropertyGroupName = "gitlab/group_name"
 	// RepoPropertyProjectName represents the gitlab project
 	RepoPropertyProjectName = "gitlab/project_name"
+	// RepoPropertyDefaultBranch represents the gitlab default branch
+	RepoPropertyDefaultBranch = "gitlab/default_branch"
+	// RepoPropertyNamespace represents the gitlab repo namespace
+	RepoPropertyNamespace = "gitlab/namespace"
+	// RepoPropertyLicense represents the gitlab repo license
+	RepoPropertyLicense = "gitlab/license"
+	// RepoPropertyCloneURL represents the gitlab repo clone URL
+	RepoPropertyCloneURL = "gitlab/clone_url"
 )
 
 // FetchAllProperties implements the provider interface
@@ -59,7 +67,7 @@ func (c *gitlabClient) GetEntityName(entityType minderv1.Entity, props *properti
 		return "", fmt.Errorf("entity type %s not supported", entityType)
 	}
 
-	groupName, err := getStringProp(props, RepoPropertyGroupName)
+	groupName, err := getStringProp(props, RepoPropertyNamespace)
 	if err != nil {
 		return "", err
 	}
@@ -72,6 +80,17 @@ func (c *gitlabClient) GetEntityName(entityType minderv1.Entity, props *properti
 	return formatRepoName(groupName, projectName), nil
 }
 
+// PropertiesToProtoMessage implements the ProtoMessageConverter interface
+func (c *gitlabClient) PropertiesToProtoMessage(
+	entType minderv1.Entity, props *properties.Properties,
+) (protoreflect.ProtoMessage, error) {
+	if !c.SupportsEntity(entType) {
+		return nil, fmt.Errorf("entity type %s is not supported by the gitlab provider", entType)
+	}
+
+	return repoV1FromProperties(props)
+}
+
 func getStringProp(props *properties.Properties, key string) (string, error) {
 	value, err := props.GetProperty(key).AsString()
 	if err != nil {

internal/providers/gitlab/properties_test.go

@@ -57,7 +57,7 @@ func Test_gitlabClient_GetEntityName(t *testing.T) {
 			args: args{
 				entityType: minderv1.Entity_ENTITY_REPOSITORIES,
 				props: MustNewProperties(map[string]any{
-					RepoPropertyGroupName:   "group",
+					RepoPropertyNamespace:   "group",
 					RepoPropertyProjectName: "project",
 				}),
 			},
@@ -69,7 +69,7 @@ func Test_gitlabClient_GetEntityName(t *testing.T) {
 			args: args{
 				entityType: minderv1.Entity_ENTITY_REPOSITORIES,
 				props: MustNewProperties(map[string]any{
-					RepoPropertyGroupName: "group",
+					RepoPropertyNamespace: "group",
 				}),
 			},
 			want:    "",
@@ -164,6 +164,9 @@ func Test_gitlabClient_FetchAllProperties(t *testing.T) {
 					Visibility:        gitlab.PrivateVisibility,
 					Archived:          false,
 					ForkedFromProject: nil,
+					Namespace: &gitlab.ProjectNamespace{
+						Path: "group",
+					},
 				}
 
 				w.Header().Set("Content-Type", "application/json")

internal/providers/gitlab/repo_lister.go

@@ -0,0 +1,78 @@
+//
+// Copyright 2024 Stacklok, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gitlab
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+
+	"github.com/rs/zerolog"
+	"github.com/xanzy/go-gitlab"
+
+	minderv1 "github.com/stacklok/minder/pkg/api/protobuf/go/minder/v1"
+)
+
+func (c *gitlabClient) ListAllRepositories(ctx context.Context) ([]*minderv1.Repository, error) {
+	groups := []*gitlab.Group{}
+	if err := glREST(ctx, c, http.MethodGet, "groups", nil, &groups); err != nil {
+		return nil, fmt.Errorf("failed to get groups: %w", err)
+	}
+
+	if len(groups) == 0 {
+		zerolog.Ctx(ctx).Debug().Msg("no groups found")
+		return nil, nil
+	}
+
+	var repos []*minderv1.Repository
+	for _, g := range groups {
+		projs := []*gitlab.Project{}
+		path, err := url.JoinPath("groups", fmt.Sprintf("%d", g.ID), "projects")
+		if err != nil {
+			return nil, fmt.Errorf("failed to join URL path for projects: %w", err)
+		}
+		if err := glREST(ctx, c, http.MethodGet, path, nil, &projs); err != nil {
+			return nil, fmt.Errorf("failed to get projects for group %s: %w", g.FullPath, err)
+		}
+
+		if repos == nil {
+			repos = make([]*minderv1.Repository, 0, len(projs))
+		}
+
+		if len(projs) == 0 {
+			zerolog.Ctx(ctx).Debug().Msgf("no projects found for group %s", g.FullPath)
+		}
+
+		for _, p := range projs {
+			props, err := gitlabProjectToProperties(p)
+			if err != nil {
+				return nil, fmt.Errorf("failed to convert project to properties: %w", err)
+			}
+
+			outRep, err := repoV1FromProperties(props)
+			if err != nil {
+				return nil, fmt.Errorf("failed to convert properties to repository: %w", err)
+			}
+
+			repos = append(repos, outRep)
+		}
+	}
+
+	zerolog.Ctx(ctx).Debug().Int("num_repos", len(repos)).Msg("found repositories in gitlab provider")
+
+	return repos, nil
+}

internal/providers/gitlab/repository_properties.go

@@ -21,10 +21,12 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"strconv"
 
 	"github.com/xanzy/go-gitlab"
 
 	"github.com/stacklok/minder/internal/entities/properties"
+	minderv1 "github.com/stacklok/minder/pkg/api/protobuf/go/minder/v1"
 	provifv1 "github.com/stacklok/minder/pkg/providers/v1"
 )
 
@@ -68,14 +70,88 @@ func (c *gitlabClient) getPropertiesForRepo(
 		return nil, fmt.Errorf("failed to decode response: %w", err)
 	}
 
+	outProps, err := gitlabProjectToProperties(proj)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert project to properties: %w", err)
+	}
+
+	return getByProps.Merge(outProps), nil
+}
+
+func gitlabProjectToProperties(proj *gitlab.Project) (*properties.Properties, error) {
+	ns := proj.Namespace
+	if ns == nil {
+		return nil, fmt.Errorf("gitlab project %d has no namespace", proj.ID)
+	}
+	owner := ns.Path
+
+	var license string
+	if proj.License != nil {
+		license = proj.License.Name
+	}
+
 	outProps, err := properties.NewProperties(map[string]any{
+		properties.PropertyUpstreamID:     fmt.Sprintf("%d", proj.ID),
+		properties.PropertyName:           formatRepoName(owner, proj.Name),
 		properties.RepoPropertyIsPrivate:  proj.Visibility == gitlab.PrivateVisibility,
 		properties.RepoPropertyIsArchived: proj.Archived,
 		properties.RepoPropertyIsFork:     proj.ForkedFromProject != nil,
+		RepoPropertyDefaultBranch:         proj.DefaultBranch,
+		RepoPropertyNamespace:             owner,
+		RepoPropertyProjectName:           proj.Name,
+		RepoPropertyLicense:               license,
+		RepoPropertyCloneURL:              proj.HTTPURLToRepo,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to create properties: %w", err)
 	}
 
-	return getByProps.Merge(outProps), nil
+	return outProps, nil
+}
+
+func repoV1FromProperties(repoProperties *properties.Properties) (*minderv1.Repository, error) {
+	upstreamID, err := repoProperties.GetProperty(properties.PropertyUpstreamID).AsString()
+	if err != nil {
+		return nil, fmt.Errorf("error fetching upstream ID property: %w", err)
+	}
+
+	// convert the upstream ID to an int64
+	repoId, err := strconv.ParseInt(upstreamID, 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("error converting upstream ID to int64: %w", err)
+	}
+
+	name, err := repoProperties.GetProperty(RepoPropertyProjectName).AsString()
+	if err != nil {
+		return nil, fmt.Errorf("error fetching project property: %w", err)
+	}
+
+	owner, err := repoProperties.GetProperty(RepoPropertyNamespace).AsString()
+	if err != nil {
+		return nil, fmt.Errorf("error fetching namespace property: %w", err)
+	}
+
+	isPrivate, err := repoProperties.GetProperty(properties.RepoPropertyIsPrivate).AsBool()
+	if err != nil {
+		return nil, fmt.Errorf("error fetching is_private property: %w", err)
+	}
+
+	isFork, err := repoProperties.GetProperty(properties.RepoPropertyIsFork).AsBool()
+	if err != nil {
+		return nil, fmt.Errorf("error fetching is_fork property: %w", err)
+	}
+
+	pbRepo := &minderv1.Repository{
+		Name:          name,
+		Owner:         owner,
+		RepoId:        repoId,
+		CloneUrl:      repoProperties.GetProperty(RepoPropertyCloneURL).GetString(),
+		IsPrivate:     isPrivate,
+		IsFork:        isFork,
+		DefaultBranch: repoProperties.GetProperty(RepoPropertyDefaultBranch).GetString(),
+		License:       repoProperties.GetProperty(RepoPropertyLicense).GetString(),
+		Properties:    repoProperties.ToProtoStruct(),
+	}
+
+	return pbRepo, nil
 }