feat(app): switch git proxy to golang (#993)

Co-authored-by: Viktor Gal <[email protected]>

.github/workflows/ci.yml

@@ -3,19 +3,11 @@ name: CI
 on: [push]
 
 jobs:
-  cleanup-runs:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: rokroskar/[email protected]
-      env:
-        GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-    if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master'"
-
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@master
-    - name: Set up Python ${{ matrix.python-version }}
+    - uses: actions/checkout@v2
+    - name: Set up Python
       uses: actions/setup-python@v1
       with:
         python-version: '3.7'
@@ -27,21 +19,47 @@ jobs:
       run: |
         pytest tests/unit renku_notebooks
 
+  test-git-proxy:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-go@v3
+      with:
+        go-version: '>=1.18.0'
+    - name: Test git proxy
+      run: |
+        cd git-https-proxy
+        go test -v
+  
+  test-git-services:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: '3.7'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip poetry
+        cd git_services
+        poetry install
+    - name: Test git services
+      run: |
+        cd git_services
+        poetry run pytest -v tests 
+
   test-chart:
     needs: test
     runs-on: ubuntu-latest
     if: "startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/master'"
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
           python-version: 3.7
       - name: Install helm
-        env:
-          HELM_URL: https://storage.googleapis.com/kubernetes-helm
-          HELM_TGZ: helm-v2.17.0-linux-amd64.tar.gz
-          TEMP_DIR: ${{ runner.temp }}
-        run: ./install_helm.sh
+        uses: azure/setup-helm@v1
       - name: Test chart
         run: |
           PATH=${{ runner.temp }}/linux-amd64/:$PATH
@@ -54,7 +72,11 @@ jobs:
 
   publish-chart-tagged:
     runs-on: ubuntu-latest
-    needs: test-chart
+    needs: 
+    - test-chart
+    - test
+    - test-git-proxy
+    - test-git-services
     if: "startsWith(github.ref, 'refs/tags/')"
     steps:
     - uses: actions/checkout@v2

git-https-proxy/Dockerfile

@@ -1,8 +1,8 @@
-FROM node:15.14-alpine3.12
+FROM golang:1.18.0-alpine3.15 as builder
+COPY . /src
+WORKDIR /src
+RUN go build -o /git-http-proxy main.go 
 
-LABEL maintainer="Swiss Data Science Center <[email protected]>"
-
-COPY package.json package-lock.json mitmproxy.js ./
-RUN npm ci && npm cache clean --force
-
-CMD ["node", "--use-openssl-ca", "/mitmproxy.js"]
+FROM alpine:3.15
+COPY --from=builder /git-http-proxy /git-http-proxy
+ENTRYPOINT ["/git-http-proxy"]

git-https-proxy/go.mod

@@ -0,0 +1,14 @@
+module SwissDataScienceCenter/renku-notebooks/git-https-proxy
+
+go 1.18
+
+require (
+	github.com/elazarl/goproxy v0.0.0-20220328115640-894aeddb713e
+	github.com/stretchr/testify v1.7.1
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
+)

git-https-proxy/go.sum

@@ -0,0 +1,16 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/elazarl/goproxy v0.0.0-20220328115640-894aeddb713e h1:99KFda6F/mw8xSfceY2JEVCrYWX7l+Ms6BcO5wEct+Q=
+github.com/elazarl/goproxy v0.0.0-20220328115640-894aeddb713e/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
+github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM=
+github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

git-https-proxy/main.go

@@ -0,0 +1,190 @@
+package main
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/elazarl/goproxy"
+)
+
+func main() {
+	config := parseEnv()
+	proxyHandler := getProxyHandler(config)
+	proxyServer := http.Server{
+		Addr:    fmt.Sprintf("0.0.0.0:%s", config.ProxyPort),
+		Handler: proxyHandler,
+	}
+	healthHandler := getHealthHandler(config)
+	healthServer := http.Server{
+		Addr:    fmt.Sprintf("0.0.0.0:%s", config.HealthPort),
+		Handler: healthHandler,
+	}
+	go func() {
+		// Run the health server in the "background"
+		log.Println("Health server active on port", config.HealthPort)
+		log.Fatalln(healthServer.ListenAndServe())
+	}()
+	log.Println("Git proxy active on port", config.ProxyPort)
+	log.Println("Repo Url:", config.RepoUrl, "anonymous session:", config.AnonymousSession)
+	log.Fatalln(proxyServer.ListenAndServe())
+}
+
+type gitProxyConfig struct {
+	ProxyPort          string
+	HealthPort         string
+	AnonymousSession   bool
+	EncodedCredentials string
+	RepoUrl            *url.URL
+}
+
+// Parse the environment variables used as the configuration for the proxy.
+func parseEnv() *gitProxyConfig {
+	var ok, anonymousSession bool
+	var gitlabOauthToken, proxyPort, healthPort, anonymousSessionStr, encodedCredentials string
+	var repoUrl *url.URL
+	if proxyPort, ok = os.LookupEnv("MITM_PROXY_PORT"); !ok {
+		proxyPort = "8080"
+	}
+	if healthPort, ok = os.LookupEnv("HEALTH_PORT"); !ok {
+		healthPort = "8081"
+	}
+	if anonymousSessionStr, ok = os.LookupEnv("ANONYMOUS_SESSION"); !ok {
+		anonymousSessionStr = "true"
+	}
+	anonymousSession = anonymousSessionStr == "true"
+	gitlabOauthToken = os.Getenv("GITLAB_OAUTH_TOKEN")
+	encodedCredentials = encodeCredentials(gitlabOauthToken)
+	repoUrl, err := url.Parse(os.Getenv("REPOSITORY_URL"))
+	if err != nil {
+		log.Fatal(err)
+	}
+	return &gitProxyConfig{
+		ProxyPort:          proxyPort,
+		HealthPort:         healthPort,
+		AnonymousSession:   anonymousSession,
+		EncodedCredentials: encodedCredentials,
+		RepoUrl:            repoUrl,
+	}
+}
+
+func encodeCredentials(token string) string {
+	return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("oauth2:%s", token)))
+}
+
+// Infer port if not explicitly specified
+func getPort(urlAddress *url.URL) string {
+	if urlAddress.Port() == "" {
+		if urlAddress.Scheme == "http" {
+			return "80"
+		} else if urlAddress.Scheme == "https" {
+			return "443"
+		}
+	} 
+	return urlAddress.Port()
+}
+
+// Ensure that hosts name watch with/without. I.e. 
+// ensure www.hostname.com matches hostname.com and vice versa
+func hostsMatch(url1 *url.URL, url2 *url.URL) bool {
+	var err error
+	var url1ContainsWww, url2ContainsWww bool
+	wwwRegex := fmt.Sprintf("^%s", regexp.QuoteMeta("www."))
+	url1ContainsWww, err = regexp.MatchString(wwwRegex, url1.Hostname())
+	if err != nil {
+		log.Fatalln(err)
+	}
+	url2ContainsWww, err = regexp.MatchString(wwwRegex, url2.Hostname())
+	if err != nil {
+		log.Fatalln(err)
+	}
+	if url1ContainsWww && !url2ContainsWww {
+		return url1.Hostname() == fmt.Sprintf("www.%s", url2.Hostname())
+	} else if !url1ContainsWww && url2ContainsWww {
+		return fmt.Sprintf("www.%s", url1.Hostname()) == url2.Hostname()
+	} else {
+		return url1.Hostname() == url2.Hostname()
+	}
+}
+
+// Return a server handler that contains the proxy that injects the Git aithorization header when
+// the conditions for doing so are met.
+func getProxyHandler(config *gitProxyConfig) *goproxy.ProxyHttpServer {
+	proxyHandler := goproxy.NewProxyHttpServer()
+	proxyHandler.Verbose = true
+	gitRepoHostWithWww := fmt.Sprintf("www.%s", config.RepoUrl.Hostname())
+	handlerFunc := func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
+		var validGitRequest bool
+		validGitRequest = r.URL.Scheme == config.RepoUrl.Scheme &&
+			hostsMatch(r.URL, config.RepoUrl) &&
+			getPort(r.URL) == getPort(config.RepoUrl) &&
+			strings.HasPrefix(strings.TrimLeft(r.URL.Path, "/"), strings.TrimLeft(config.RepoUrl.Path, "/"))
+		if config.AnonymousSession {
+			log.Print("Anonymous session, not adding auth headers, letting request through without adding auth headers.\n")
+			return r, nil
+		}
+		if !validGitRequest {
+			log.Println("The request", r.URL, "does not match the git repository", config.RepoUrl, ", letting request through without adding auth headers")
+			return r, nil
+		}
+		log.Println("Adding auth header to request:", r.URL)
+		r.Header.Set("Authorization", fmt.Sprintf("Basic %s", config.EncodedCredentials))
+		return r, nil
+	}
+	// NOTE: We need to eavesdrop on the HTTPS connection to insert the Auth header
+	// we do this only for the case where the request host matches the host of the git repo
+	// in all other cases we leave the request alone.
+	proxyHandler.OnRequest(goproxy.ReqHostIs(
+		config.RepoUrl.Hostname(), 
+		gitRepoHostWithWww,
+		fmt.Sprintf("%s:443", config.RepoUrl.Hostname()), 
+		fmt.Sprintf("%s:443", gitRepoHostWithWww), 
+	)).HandleConnect(goproxy.AlwaysMitm)
+	proxyHandler.OnRequest().DoFunc(handlerFunc)	
+	return proxyHandler
+}
+
+// The proxy does not expose a health endpoint. Therefore the purpose of this server
+// handler is to just fill that functionality. To ensure that the proxy is fully up
+// and running the health server will use the proxy as a proxy for the health endpoint.
+// This is necessary because sending any requests directly to the proxy results in a 500
+// with a message that the proxy only accepts proxy requests and no direct requests.
+func getHealthHandler(config *gitProxyConfig) *http.ServeMux {
+	handler := http.NewServeMux()
+	handler.HandleFunc("/ping", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		resp := make(map[string]string)
+		resp["message"] = "pong"
+		jsonResp, err := json.Marshal(resp)
+		if err != nil {
+			log.Fatalf("Error happened in JSON marshal. Err: %s", err)
+		}
+		w.Write(jsonResp)
+	})
+	handler.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
+		proxyUrl, err := url.Parse(fmt.Sprintf("http://localhost:%s", config.ProxyPort))
+		if err != nil {
+			log.Fatalln(err)
+		}
+		client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}}
+		resp, err := client.Get(fmt.Sprintf("http://localhost:%s/ping", config.HealthPort))
+		if err != nil {
+			log.Println("The GET request to /ping from within /health failed with:", err)
+			w.WriteHeader(http.StatusBadRequest)
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode >= 200 && resp.StatusCode <= 400 {
+			w.WriteHeader(http.StatusOK)
+		} else {
+			w.WriteHeader(http.StatusBadRequest)
+		}
+	})
+
+	return handler
+}