sourcegraph: Implement ADO client (#46615)

Commit: e1dc53373e2b16ffc01b6cd97330d44b2e0177c4
sgtest · Jan 19, 2023 · 6d0743b · 6d0743b
1 parent e0a079c
commit 6d0743b
Show file tree

Hide file tree

Showing 5 changed files with 324 additions and 4 deletions.
diff --git a/sourcegraph/internal/extsvc/azuredevops/client.go b/sourcegraph/internal/extsvc/azuredevops/client.go
@@ -0,0 +1,145 @@
+//nolint:bodyclose // Body is closed in Client.Do, but the response is still returned to provide access to the headers
+package azuredevops
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+
+	"github.com/sourcegraph/sourcegraph/internal/httpcli"
+	"github.com/sourcegraph/sourcegraph/internal/ratelimit"
+)
+
+// Client used to access an ADO code host via the REST API.
+type Client struct {
+	// HTTP Client used to communicate with the API.
+	httpClient httpcli.Doer
+
+	// Config is the code host connection config for this client.
+	Config *ADOConnection
+
+	// URL is the base URL of ADO.
+	URL *url.URL
+
+	// RateLimit is the self-imposed rate limiter (since ADO does not have a concept
+	// of rate limiting in HTTP response headers).
+	rateLimit *ratelimit.InstrumentedLimiter
+}
+
+// TODO: @varsanojidan remove this when the shcema is updated to include ADO: https://github.com/sourcegraph/sourcegraph/issues/46266.
+type ADOConnection struct {
+	Username string
+	Token    string
+}
+
+// NewClient returns an authenticated ADO API client with
+// the provided configuration. If a nil httpClient is provided, http.DefaultClient
+// will be used.
+func NewClient(urn string, config *ADOConnection, httpClient httpcli.Doer) (*Client, error) {
+	u, err := url.Parse("https://dev.azure.com")
+	if err != nil {
+		return nil, err
+	}
+
+	if httpClient == nil {
+		httpClient = httpcli.ExternalDoer
+	}
+
+	return &Client{
+		httpClient: httpClient,
+		Config:     config,
+		URL:        u,
+		rateLimit:  ratelimit.DefaultRegistry.Get(urn),
+	}, nil
+}
+
+// ListRepositoriesByProjectOrOrgArgs defines options to be set on the ListRepositories methods' calls.
+type ListRepositoriesByProjectOrOrgArgs struct {
+	// Should be in the form of 'org/project' for projects and 'org' for orgs.
+	ProjectOrOrgName string
+}
+
+func (c *Client) ListRepositoriesByProjectOrOrg(ctx context.Context, opts ListRepositoriesByProjectOrOrgArgs) (*ListRepositoriesResponse, error) {
+	qs := make(url.Values)
+
+	// TODO: @varsanojidan look into which API version/s we want to support.
+	qs.Set("api-version", "7.0")
+
+	urlRepositoriesByProjects := url.URL{Path: fmt.Sprintf("%s/_apis/git/repositories", opts.ProjectOrOrgName), RawQuery: qs.Encode()}
+
+	req, err := http.NewRequest("GET", urlRepositoriesByProjects.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	var repos ListRepositoriesResponse
+	if _, err = c.do(ctx, req, &repos); err != nil {
+		return nil, err
+	}
+
+	return &repos, nil
+}
+
+//nolint:unparam // http.Response is never used, but it makes sense API wise.
+func (c *Client) do(ctx context.Context, req *http.Request, result any) (*http.Response, error) {
+	req.URL = c.URL.ResolveReference(req.URL)
+
+	// Add Basic Auth headers for authenticated requests.
+	req.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(c.Config.Username+":"+c.Config.Token)))
+
+	if err := c.rateLimit.Wait(ctx); err != nil {
+		return nil, err
+	}
+
+	resp, err := c.httpClient.Do(req)
+
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+
+	bs, err := io.ReadAll(resp.Body)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 400 {
+		return nil, &httpError{
+			URL:        req.URL,
+			StatusCode: resp.StatusCode,
+			Body:       bs,
+		}
+	}
+
+	return resp, json.Unmarshal(bs, result)
+}
+
+type ListRepositoriesResponse struct {
+	Value []RepositoriesValue `json:"value"`
+	Count int                 `json:"count"`
+}
+
+type RepositoriesValue struct {
+	ID         string `json:"id"`
+	Name       string `json:"name"`
+	APIURL     string `json:"url"`
+	SSHURL     string `json:"sshUrl"`
+	WebURL     string `json:"webUrl"`
+	IsDisabled bool   `json:"isDisabled"`
+}
+
+type httpError struct {
+	StatusCode int
+	URL        *url.URL
+	Body       []byte
+}
+
+func (e *httpError) Error() string {
+	return fmt.Sprintf("ADO API HTTP error: code=%d url=%q body=%q", e.StatusCode, e.URL, e.Body)
+}
diff --git a/sourcegraph/internal/extsvc/azuredevops/client_test.go b/sourcegraph/internal/extsvc/azuredevops/client_test.go
@@ -0,0 +1,96 @@
+package azuredevops
+
+import (
+	"context"
+	"flag"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/dnaeon/go-vcr/cassette"
+	"github.com/sourcegraph/sourcegraph/internal/httpcli"
+	"github.com/sourcegraph/sourcegraph/internal/httptestutil"
+	"github.com/sourcegraph/sourcegraph/internal/lazyregexp"
+	"github.com/sourcegraph/sourcegraph/internal/testutil"
+)
+
+var update = flag.Bool("update", false, "update testdata")
+
+func TestClient_ListRepositoriesByProjectOrOrg(t *testing.T) {
+	cli, save := NewTestClient(t, "ListRepositoriesByProjectOrOrg", *update)
+	defer save()
+
+	ctx := context.Background()
+
+	opts := ListRepositoriesByProjectOrOrgArgs{
+		// TODO: use an sg owned org rather than a personal.
+		ProjectOrOrgName: "sgadotest",
+	}
+
+	resp, err := cli.ListRepositoriesByProjectOrOrg(ctx, opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testutil.AssertGolden(t, "testdata/golden/ListProjects.json", *update, resp)
+}
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+	os.Exit(m.Run())
+}
+
+// NewTestClient returns an azuredevops.Client that records its interactions
+// to testdata/vcr/.
+func NewTestClient(t testing.TB, name string, update bool) (*Client, func()) {
+	t.Helper()
+
+	cassete := filepath.Join("testdata/vcr/", normalize(name))
+	rec, err := httptestutil.NewRecorder(cassete, update)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rec.SetMatcher(ignoreHostMatcher)
+
+	hc, err := httpcli.NewFactory(nil, httptestutil.NewRecorderOpt(rec)).Doer()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c := &ADOConnection{
+		Username: "testuser",
+		Token:    "testpassword",
+	}
+
+	cli, err := NewClient("urn", c, hc)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return cli, func() {
+		if err := rec.Stop(); err != nil {
+			t.Errorf("failed to update test data: %s", err)
+		}
+	}
+}
+
+var normalizer = lazyregexp.New("[^A-Za-z0-9-]+")
+
+func normalize(path string) string {
+	return normalizer.ReplaceAllLiteralString(path, "-")
+}
+
+func ignoreHostMatcher(r *http.Request, i cassette.Request) bool {
+	if r.Method != i.Method {
+		return false
+	}
+	u, err := url.Parse(i.URL)
+	if err != nil {
+		return false
+	}
+	u.Host = r.URL.Host
+	u.Scheme = r.URL.Scheme
+	return r.URL.String() == u.String()
+}
diff --git a/sourcegraph/internal/extsvc/azuredevops/testdata/golden/ListProjects.json b/sourcegraph/internal/extsvc/azuredevops/testdata/golden/ListProjects.json
@@ -0,0 +1,29 @@
+{
+  "value": [
+   {
+    "id": "d43b669c-d6da-4c9e-9952-11f411153cf9",
+    "name": "sgadotest",
+    "url": "https://dev.azure.com/sgadotest/889cc0c6-c7ed-40f0-b6f1-053acfc1397d/_apis/git/repositories/d43b669c-d6da-4c9e-9952-11f411153cf9",
+    "sshUrl": "[email protected]:v3/sgadotest/sgadotest/sgadotest",
+    "webUrl": "https://dev.azure.com/sgadotest/sgadotest/_git/sgadotest",
+    "isDisabled": false
+   },
+   {
+    "id": "47c49ea1-5dc8-468e-ae6d-52410a1084a7",
+    "name": "idano",
+    "url": "https://dev.azure.com/sgadotest/889cc0c6-c7ed-40f0-b6f1-053acfc1397d/_apis/git/repositories/47c49ea1-5dc8-468e-ae6d-52410a1084a7",
+    "sshUrl": "[email protected]:v3/sgadotest/sgadotest/idano",
+    "webUrl": "https://dev.azure.com/sgadotest/sgadotest/_git/idano",
+    "isDisabled": false
+   },
+   {
+    "id": "2c8bc3c0-841b-478e-ae1b-f5736b4f9cf5",
+    "name": "sgadotest2",
+    "url": "https://dev.azure.com/sgadotest/450fc69b-ff88-4604-88d3-84b488e290b2/_apis/git/repositories/2c8bc3c0-841b-478e-ae1b-f5736b4f9cf5",
+    "sshUrl": "[email protected]:v3/sgadotest/sgadotest2/sgadotest2",
+    "webUrl": "https://dev.azure.com/sgadotest/sgadotest2/_git/sgadotest2",
+    "isDisabled": false
+   }
+  ],
+  "count": 3
+ }
diff --git a/sourcegraph/internal/extsvc/azuredevops/testdata/vcr/ListRepositoriesByProjectOrOrg.yaml b/sourcegraph/internal/extsvc/azuredevops/testdata/vcr/ListRepositoriesByProjectOrOrg.yaml
@@ -0,0 +1,54 @@
+---
+version: 1
+interactions:
+- request:
+    body: ""
+    form: {}
+    headers: {}
+    url: https://dev.azure.com/sgadotest/_apis/git/repositories?api-version=7.0
+    method: GET
+  response:
+    body: '{"value":[{"id":"d43b669c-d6da-4c9e-9952-11f411153cf9","name":"sgadotest","url":"https://dev.azure.com/sgadotest/889cc0c6-c7ed-40f0-b6f1-053acfc1397d/_apis/git/repositories/d43b669c-d6da-4c9e-9952-11f411153cf9","project":{"id":"889cc0c6-c7ed-40f0-b6f1-053acfc1397d","name":"sgadotest","url":"https://dev.azure.com/sgadotest/_apis/projects/889cc0c6-c7ed-40f0-b6f1-053acfc1397d","state":"wellFormed","revision":11,"visibility":"private","lastUpdateTime":"2023-01-17T22:13:58.063Z"},"size":0,"remoteUrl":"https://[email protected]/sgadotest/sgadotest/_git/sgadotest","sshUrl":"[email protected]:v3/sgadotest/sgadotest/sgadotest","webUrl":"https://dev.azure.com/sgadotest/sgadotest/_git/sgadotest","isDisabled":false,"isInMaintenance":false},{"id":"47c49ea1-5dc8-468e-ae6d-52410a1084a7","name":"idano","url":"https://dev.azure.com/sgadotest/889cc0c6-c7ed-40f0-b6f1-053acfc1397d/_apis/git/repositories/47c49ea1-5dc8-468e-ae6d-52410a1084a7","project":{"id":"889cc0c6-c7ed-40f0-b6f1-053acfc1397d","name":"sgadotest","url":"https://dev.azure.com/sgadotest/_apis/projects/889cc0c6-c7ed-40f0-b6f1-053acfc1397d","state":"wellFormed","revision":11,"visibility":"private","lastUpdateTime":"2023-01-17T22:13:58.063Z"},"defaultBranch":"refs/heads/main","size":726,"remoteUrl":"https://[email protected]/sgadotest/sgadotest/_git/idano","sshUrl":"[email protected]:v3/sgadotest/sgadotest/idano","webUrl":"https://dev.azure.com/sgadotest/sgadotest/_git/idano","isDisabled":false,"isInMaintenance":false},{"id":"2c8bc3c0-841b-478e-ae1b-f5736b4f9cf5","name":"sgadotest2","url":"https://dev.azure.com/sgadotest/450fc69b-ff88-4604-88d3-84b488e290b2/_apis/git/repositories/2c8bc3c0-841b-478e-ae1b-f5736b4f9cf5","project":{"id":"450fc69b-ff88-4604-88d3-84b488e290b2","name":"sgadotest2","url":"https://dev.azure.com/sgadotest/_apis/projects/450fc69b-ff88-4604-88d3-84b488e290b2","state":"wellFormed","revision":19,"visibility":"private","lastUpdateTime":"2023-01-18T05:29:24.46Z"},"size":0,"remoteUrl":"https://[email protected]/sgadotest/sgadotest2/_git/sgadotest2","sshUrl":"[email protected]:v3/sgadotest/sgadotest2/sgadotest2","webUrl":"https://dev.azure.com/sgadotest/sgadotest2/_git/sgadotest2","isDisabled":false,"isInMaintenance":false}],"count":3}'
+    headers:
+      Access-Control-Expose-Headers:
+      - Request-Context
+      Activityid:
+      - bb687111-e265-4a33-8b7f-5b4d55d3b878
+      Cache-Control:
+      - no-cache, no-store, must-revalidate
+      Content-Type:
+      - application/json; charset=utf-8; api-version=7.0
+      Date:
+      - Wed, 18 Jan 2023 14:49:16 GMT
+      Expires:
+      - "-1"
+      P3p:
+      - CP="CAO DSP COR ADMa DEV CONo TELo CUR PSA PSD TAI IVDo OUR SAMi BUS DEM NAV
+        STA UNI COM INT PHY ONL FIN PUR LOC CNT"
+      Pragma:
+      - no-cache
+      Request-Context:
+      - appId=cid-v1:72d31d95-1757-44f5-b910-a46611808454
+      Strict-Transport-Security:
+      - max-age=31536000; includeSubDomains
+      X-Cache:
+      - CONFIG_NOCACHE
+      X-Content-Type-Options:
+      - nosniff
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Msedge-Ref:
+      - 'Ref A: D4246B6C514F430BA3FB288B164F66E5 Ref B: YTO01EDGE0510 Ref C: 2023-01-18T14:49:15Z'
+      X-Tfs-Processid:
+      - a9e61ddf-ab88-4476-9b7e-1af9921bf7ed
+      X-Tfs-Session:
+      - bb687111-e265-4a33-8b7f-5b4d55d3b878
+      X-Vss-E2eid:
+      - bb687111-e265-4a33-8b7f-5b4d55d3b878
+      X-Vss-Senderdeploymentid:
+      - 4ff21e82-8865-0b2e-ffe8-9598818f8190
+      X-Vss-Userdata:
+      - 003fb9ec-23b7-699d-9973-544b77eb2595:[email protected]
+    status: 200 OK
+    code: 200
+    duration: ""
diff --git a/sourcegraph/internal/extsvc/gerrit/client_test.go b/sourcegraph/internal/extsvc/gerrit/client_test.go
@@ -10,7 +10,6 @@ import (
 	"testing"
 
 	"github.com/dnaeon/go-vcr/cassette"
-	"github.com/inconshreveable/log15"
 	"github.com/sourcegraph/sourcegraph/internal/httpcli"
 	"github.com/sourcegraph/sourcegraph/internal/httptestutil"
 	"github.com/sourcegraph/sourcegraph/internal/lazyregexp"
@@ -40,9 +39,6 @@ func TestClient_ListProjects(t *testing.T) {
 
 func TestMain(m *testing.M) {
 	flag.Parse()
-	if !testing.Verbose() {
-		log15.Root().SetHandler(log15.LvlFilterHandler(log15.LvlError, log15.Root().GetHandler()))
-	}
 	os.Exit(m.Run())
 }