Add preheat e2e test (#582)

* feat: create preheat_test.go Signed-off-by: zzy987 <[email protected]> * feat: add post request Signed-off-by: zzy987 <[email protected]> * fix: try to fix header name Signed-off-by: zzy987 <[email protected]> * fix: try fix page not found Signed-off-by: zzy987 <[email protected]> * fix: try to install kubectl in cdn pod Signed-off-by: zzy987 <[email protected]> * fix: remove sudo Signed-off-by: zzy987 <[email protected]> * chore: put curl in manager Signed-off-by: zzy987 <[email protected]> * fix: correct json content Signed-off-by: zzy987 <[email protected]> * fix: correct request url Signed-off-by: zzy987 <[email protected]> * chore: check curl result Signed-off-by: zzy987 <[email protected]> * fix: correct request url Signed-off-by: zzy987 <[email protected]> * chore: check the cmd string in github environment Signed-off-by: zzy987 <[email protected]> * fix: use service Signed-off-by: zzy987 <[email protected]> * fix: delete cluster-name Signed-off-by: zzy987 <[email protected]> * chore: use " in header Signed-off-by: zzy987 <[email protected]> * fix: curl command format Signed-off-by: zzy987 <[email protected]> * feat: use idgen.taskid to get the filename Signed-off-by: zzy987 <[email protected]> * chore: add an error message Signed-off-by: zzy987 <[email protected]> * feat: add concurrency test Signed-off-by: zzy987 <[email protected]> * fix: ab use proxy Signed-off-by: zzy987 <[email protected]> * feat: add curl pod Signed-off-by: zzy987 <[email protected]> * feat: add function CurlCommand Signed-off-by: zzy987 <[email protected]> * fix: variable usage Signed-off-by: zzy987 <[email protected]> * chore: remove curl in cdn pod Signed-off-by: zzy987 <[email protected]> * chore: modify code format Signed-off-by: zzy987 <[email protected]> * feat: use curl in file-server Signed-off-by: zzy987 <[email protected]> * fix: correct command in file-server Signed-off-by: zzy987 <[email protected]> * chore: add a e2e-test, check if it is the reason of no preheat Signed-off-by: zzy987 <[email protected]> * chore: can test preheat now Signed-off-by: zzy987 <[email protected]> * fix: check result after concurrency test Signed-off-by: zzy987 <[email protected]> * chore: add comments, add function check preheat result Signed-off-by: zzy987 <[email protected]> * chore: delete e2e_test in e2e/manager Signed-off-by: zzy987 <[email protected]> * chore: refactor curlCommand, add func getCDNs getFS Signed-off-by: zzy987 <[email protected]> * feat: add image preheat Signed-off-by: zzy987 <[email protected]> * chore: rename functions Signed-off-by: zzy987 <[email protected]> * fix: delete functions, use stable task id in test Signed-off-by: zzy987 <[email protected]>
dragonflyoss · Jun 28, 2023 · 74696f3 · 74696f3
1 parent b18c642
commit 74696f3
Show file tree

Hide file tree

Showing 6 changed files with 288 additions and 9 deletions.
diff --git a/test/e2e/concurrency_test.go b/test/e2e/concurrency_test.go
@@ -24,11 +24,6 @@ import (
 	. "github.com/onsi/gomega" //nolint
 )
 
-const (
-	hostnameFilePath = "/etc/hostname"
-	proxy            = "localhost:65001"
-)
-
 var _ = Describe("Download concurrency", func() {
 	Context("ab", func() {
 		It("concurrent 100 should be ok", func() {

diff --git a/test/e2e/dfget_test.go b/test/e2e/dfget_test.go
@@ -25,10 +25,6 @@ import (
 	. "github.com/onsi/gomega" //nolint
 )
 
-const (
-	dragonflyNamespace = "dragonfly-system"
-)
-
 var _ = Describe("Download with dfget", func() {
 	Context("dfget", func() {
 		It("dfget download should be ok", func() {

diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go
@@ -24,6 +24,14 @@ import (
 	"d7y.io/dragonfly/v2/test/e2e/e2eutil"
 	. "github.com/onsi/ginkgo" //nolint
 	. "github.com/onsi/gomega" //nolint
+
+	_ "d7y.io/dragonfly/v2/test/e2e/manager"
+)
+
+const (
+	proxy              = "localhost:65001"
+	hostnameFilePath   = "/etc/hostname"
+	dragonflyNamespace = "dragonfly-system"
 )
 
 var _ = BeforeSuite(func() {

diff --git a/test/e2e/e2eutil/exec.go b/test/e2e/e2eutil/exec.go
@@ -1,6 +1,8 @@
 package e2eutil
 
 import (
+	"encoding/json"
+	"fmt"
 	"os/exec"
 )
 
@@ -55,6 +57,23 @@ func (p *PodExec) Command(arg ...string) *exec.Cmd {
 	return KubeCtlCommand(extArgs...)
 }
 
+func (p *PodExec) CurlCommand(method string, header map[string]string, data map[string]interface{}, target string) *exec.Cmd {
+	extArgs := []string{"/usr/bin/curl", target, "-s"}
+	if method != "" {
+		extArgs = append(extArgs, "-X", method)
+	}
+	if header != nil {
+		for k, v := range header {
+			extArgs = append(extArgs, "-H", fmt.Sprintf("%s:%s", k, v))
+		}
+	}
+	if data != nil {
+		jsonData, _ := json.Marshal(data)
+		extArgs = append(extArgs, "-d", string(jsonData))
+	}
+	return p.Command(extArgs...)
+}
+
 func KubeCtlCopyCommand(ns, pod, source, target string) *exec.Cmd {
 	return KubeCtlCommand("-n", ns, "cp", pod+":"+source, target)
 }
diff --git a/test/e2e/manager/constants.go b/test/e2e/manager/constants.go
@@ -0,0 +1,16 @@
+package manager
+
+const (
+	cdnCachePath = "/tmp/cdn/download"
+
+	managerService = "dragonfly-manager.dragonfly-system.svc"
+	managerPort    = "8080"
+	preheatPath    = "api/v1/preheats"
+	managerTag     = "d7y/manager"
+
+	dragonflyNamespace = "dragonfly-system"
+	e2eNamespace       = "dragonfly-e2e"
+
+	proxy            = "localhost:65001"
+	hostnameFilePath = "/etc/hostname"
+)
diff --git a/test/e2e/manager/preheat.go b/test/e2e/manager/preheat.go
@@ -0,0 +1,245 @@
+package manager
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"d7y.io/dragonfly/v2/internal/idgen"
+	"d7y.io/dragonfly/v2/manager/types"
+	"d7y.io/dragonfly/v2/pkg/rpc/base"
+	"d7y.io/dragonfly/v2/test/e2e/e2eutil"
+	machineryv1tasks "github.com/RichardKnop/machinery/v1/tasks"
+	. "github.com/onsi/ginkgo" //nolint
+	. "github.com/onsi/gomega" //nolint
+)
+
+var _ = Describe("Preheat with manager", func() {
+	Context("preheat", func() {
+		It("preheat files should be ok", func() {
+			var cdnPods [3]*e2eutil.PodExec
+			for i := 0; i < 3; i++ {
+				cdnPods[i] = getCDNExec(i)
+			}
+			fsPod := getFileServerExec()
+
+			for _, v := range e2eutil.GetFileList() {
+				url := e2eutil.GetFileURL(v)
+				fmt.Println("download url " + url)
+
+				// get original file digest
+				out, err := e2eutil.DockerCommand("sha256sum", v).CombinedOutput()
+				fmt.Println(string(out))
+				Expect(err).NotTo(HaveOccurred())
+				sha256sum1 := strings.Split(string(out), " ")[0]
+
+				// preheat file
+				out, err = fsPod.CurlCommand("POST", map[string]string{"Content-Type": "application/json"},
+					map[string]interface{}{"type": "file", "url": url},
+					fmt.Sprintf("http://%s:%s/%s", managerService, managerPort, preheatPath)).CombinedOutput()
+				fmt.Println(string(out))
+				Expect(err).NotTo(HaveOccurred())
+
+				// wait for success
+				preheatJob := &types.Preheat{}
+				err = json.Unmarshal(out, preheatJob)
+				Expect(err).NotTo(HaveOccurred())
+				done := waitForDone(preheatJob, fsPod)
+				Expect(done).Should(BeTrue())
+
+				// generate task_id, also the filename
+				cdnTaskID := idgen.TaskID(url, &base.UrlMeta{Tag: managerTag})
+				fmt.Println(cdnTaskID)
+
+				sha256sum2 := checkPreheatResult(cdnPods, cdnTaskID)
+				if sha256sum2 == "" {
+					fmt.Println("preheat file not found")
+				}
+				Expect(sha256sum1).To(Equal(sha256sum2))
+			}
+		})
+
+		It("preheat image should be ok", func() {
+			url := "https://registry-1.docker.io/v2/library/alpine/manifests/3.14"
+			fmt.Println("download image " + url)
+
+			var (
+				cdnTaskIDs = []string{
+					"effb4ac6e36d9a2a425ab142ba0a21fd0d49feea67a839fbd776ebb04e6f9eb7",
+					"ceaaf57ceba7221c2d54c62d77860e28b091837f235ba802c0722c522d6c7a8a",
+				}
+				sha256sum1 = []string{
+					"14119a10abf4669e8cdbdff324a9f9605d99697215a0d21c360fe8dfa8471bab",
+					"a0d0a0d46f8b52473982a3c466318f479767577551a53ffc9074c9fa7035982e",
+				}
+			)
+
+			var cdnPods [3]*e2eutil.PodExec
+			for i := 0; i < 3; i++ {
+				cdnPods[i] = getCDNExec(i)
+			}
+			fsPod := getFileServerExec()
+
+			// preheat file
+			out, err := fsPod.CurlCommand("POST", map[string]string{"Content-Type": "application/json"},
+				map[string]interface{}{"type": "image", "url": url},
+				fmt.Sprintf("http://%s:%s/%s", managerService, managerPort, preheatPath)).CombinedOutput()
+			fmt.Println(string(out))
+			Expect(err).NotTo(HaveOccurred())
+
+			// wait for success
+			preheatJob := &types.Preheat{}
+			err = json.Unmarshal(out, preheatJob)
+			Expect(err).NotTo(HaveOccurred())
+			done := waitForDone(preheatJob, fsPod)
+			Expect(done).Should(BeTrue())
+
+			for i, cdnTaskID := range cdnTaskIDs {
+				sha256sum2 := checkPreheatResult(cdnPods, cdnTaskID)
+				if sha256sum2 == "" {
+					fmt.Println("preheat file not found")
+				}
+				Expect(sha256sum1[i]).To(Equal(sha256sum2))
+			}
+		})
+
+		It("concurrency 100 preheat should be ok", func() {
+			// generate the data file
+			url := e2eutil.GetFileURL(hostnameFilePath)
+			fmt.Println("download url " + url)
+			dataFilePath := "post_data"
+			fd, err := os.Create(dataFilePath)
+			Expect(err).NotTo(HaveOccurred())
+			_, err = fd.WriteString(fmt.Sprintf(`{"type":"file","url":"%s"}`, url))
+			fd.Close()
+			Expect(err).NotTo(HaveOccurred())
+
+			// use ab to post the data file to manager concurrently
+			out, err := e2eutil.ABCommand("-c", "100", "-n", "200", "-T", "application/json", "-p", dataFilePath, "-X", proxy, fmt.Sprintf("http://%s:%s/%s", managerService, managerPort, preheatPath)).CombinedOutput()
+			fmt.Println(string(out))
+			Expect(err).NotTo(HaveOccurred())
+			os.Remove(dataFilePath)
+			Expect(err).NotTo(HaveOccurred())
+
+			// get original file digest
+			out, err = e2eutil.DockerCommand("sha256sum", hostnameFilePath).CombinedOutput()
+			fmt.Println(string(out))
+			Expect(err).NotTo(HaveOccurred())
+			sha256sum1 := strings.Split(string(out), " ")[0]
+
+			var cdnPods [3]*e2eutil.PodExec
+			for i := 0; i < 3; i++ {
+				cdnPods[i] = getCDNExec(i)
+			}
+			fsPod := getFileServerExec()
+
+			// use a curl to preheat the same file, git a id to wait for success
+			out, err = fsPod.CurlCommand("POST", map[string]string{"Content-Type": "application/json"},
+				map[string]interface{}{"type": "file", "url": url},
+				fmt.Sprintf("http://%s:%s/%s", managerService, managerPort, preheatPath)).CombinedOutput()
+			fmt.Println(string(out))
+			Expect(err).NotTo(HaveOccurred())
+
+			// wait for success
+			preheatJob := &types.Preheat{}
+			err = json.Unmarshal(out, preheatJob)
+			Expect(err).NotTo(HaveOccurred())
+			done := waitForDone(preheatJob, fsPod)
+			Expect(done).Should(BeTrue())
+
+			// generate task id to find the file
+			cdnTaskID := idgen.TaskID(url, &base.UrlMeta{Tag: managerTag})
+			fmt.Println(cdnTaskID)
+
+			sha256sum2 := checkPreheatResult(cdnPods, cdnTaskID)
+			if sha256sum2 == "" {
+				fmt.Println("preheat file not found")
+			}
+			Expect(sha256sum1).To(Equal(sha256sum2))
+		})
+	})
+})
+
+func waitForDone(preheat *types.Preheat, pod *e2eutil.PodExec) bool {
+	ticker := time.NewTicker(time.Second * 3)
+	defer ticker.Stop()
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5)
+	defer cancel()
+	for {
+		select {
+		case <-ctx.Done():
+			return false
+		case <-ticker.C:
+			out, err := pod.CurlCommand("", nil, nil,
+				fmt.Sprintf("http://%s:%s/%s/%s", managerService, managerPort, preheatPath, preheat.ID)).CombinedOutput()
+			fmt.Println(string(out))
+			Expect(err).NotTo(HaveOccurred())
+			err = json.Unmarshal(out, preheat)
+			Expect(err).NotTo(HaveOccurred())
+			switch preheat.Status {
+			case machineryv1tasks.StateSuccess:
+				return true
+			case machineryv1tasks.StateFailure:
+				return false
+			default:
+			}
+		}
+	}
+}
+
+func checkPreheatResult(cdnPods [3]*e2eutil.PodExec, cdnTaskID string) string {
+	var sha256sum2 string
+	for _, cdn := range cdnPods {
+		out, err := cdn.Command("ls", cdnCachePath).CombinedOutput()
+		if err != nil {
+			// if the directory does not exist, skip this cdn
+			continue
+		}
+		// directory name is the first three characters of the task id
+		dir := cdnTaskID[0:3]
+		if !strings.Contains(string(out), dir) {
+			continue
+		}
+
+		out, err = cdn.Command("ls", fmt.Sprintf("%s/%s", cdnCachePath, dir)).CombinedOutput()
+		Expect(err).NotTo(HaveOccurred())
+		// file name is the same as task id
+		file := cdnTaskID
+		if !strings.Contains(string(out), file) {
+			continue
+		}
+
+		// calculate digest of downloaded file
+		out, err = cdn.Command("sha256sum", fmt.Sprintf("%s/%s/%s", cdnCachePath, dir, file)).CombinedOutput()
+		fmt.Println(string(out))
+		Expect(err).NotTo(HaveOccurred())
+		sha256sum2 = strings.Split(string(out), " ")[0]
+		break
+	}
+	return sha256sum2
+}
+
+// getCDNExec get cdn pods
+func getCDNExec(n int) *e2eutil.PodExec {
+	out, err := e2eutil.KubeCtlCommand("-n", dragonflyNamespace, "get", "pod", "-l", "component=cdn",
+		"-o", fmt.Sprintf("jsonpath='{range .items[%d]}{.metadata.name}{end}'", n)).CombinedOutput()
+	podName := strings.Trim(string(out), "'")
+	Expect(err).NotTo(HaveOccurred())
+	fmt.Println(podName)
+	Expect(strings.HasPrefix(podName, "dragonfly-cdn-")).Should(BeTrue())
+	return e2eutil.NewPodExec(dragonflyNamespace, podName, "cdn")
+}
+
+// getFileServerExec get the file-server pod for curl
+func getFileServerExec() *e2eutil.PodExec {
+	out, err := e2eutil.KubeCtlCommand("-n", e2eNamespace, "get", "pod", "-l", "component=file-server",
+		"-o", "jsonpath='{range .items[*]}{.metadata.name}{end}'").CombinedOutput()
+	podName := strings.Trim(string(out), "'")
+	Expect(err).NotTo(HaveOccurred())
+	fmt.Println(podName)
+	Expect(strings.HasPrefix(podName, "file-server-")).Should(BeTrue())
+	return e2eutil.NewPodExec(e2eNamespace, podName, "")
+}