diff --git a/.github/workflows/compatibility-e2e.yml b/.github/workflows/compatibility-e2e.yml index c44ad0d5d74..18ff72831d1 100644 --- a/.github/workflows/compatibility-e2e.yml +++ b/.github/workflows/compatibility-e2e.yml @@ -17,6 +17,7 @@ env: DRAGONFLY_CHARTS_PATH: deploy/helm-charts/charts/dragonfly DRAGONFLY_CHARTS_CONFIG_PATH: test/testdata/charts/config.yaml DRAGONFLY_FILE_SERVER_PATH: test/testdata/k8s/file-server.yaml + DRAGONFLY_PROXY_SERVER_PATH: test/testdata/k8s/proxy.yaml jobs: compatibility_e2e_tests: @@ -51,7 +52,7 @@ jobs: - name: Build images run: | - make docker-build + make docker-build docker-build-testing-tools docker pull dragonflyoss/${{ matrix.module }}:${{ env.DRAGONFLY_STABLE_IMAGE_TAG }} - name: Prepare kind environment @@ -63,7 +64,12 @@ jobs: run: | helm install --wait --timeout 10m --dependency-update --create-namespace --namespace dragonfly-system --set ${{ matrix.module }}.tag=${{ env.DRAGONFLY_STABLE_IMAGE_TAG }} --set ${{ matrix.module }}.image=dragonflyoss/${{ matrix.module }} -f ${{ env.DRAGONFLY_CHARTS_CONFIG_PATH }} dragonfly ${{ env.DRAGONFLY_CHARTS_PATH }} kubectl apply -f ${{ env.DRAGONFLY_FILE_SERVER_PATH }} + kubectl apply -f ${{ env.DRAGONFLY_PROXY_SERVER_PATH }} kubectl wait po file-server-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m + kubectl wait po file-server-no-content-length-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m + kubectl wait po proxy-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m + kubectl wait po proxy-1 --namespace dragonfly-e2e --for=condition=ready --timeout=10m + kubectl wait po proxy-2 --namespace dragonfly-e2e --for=condition=ready --timeout=10m - name: Run Compatibility E2E test env: diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 09e1a9a5dd4..aee7f9aa512 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -16,6 +16,7 @@ env: DRAGONFLY_CHARTS_PATH: deploy/helm-charts/charts/dragonfly DRAGONFLY_CHARTS_CONFIG_PATH: test/testdata/charts/config.yaml DRAGONFLY_FILE_SERVER_PATH: test/testdata/k8s/file-server.yaml + DRAGONFLY_PROXY_SERVER_PATH: test/testdata/k8s/proxy.yaml jobs: e2e_tests: @@ -45,7 +46,7 @@ jobs: config: ${{ env.KIND_CONFIG_PATH }} - name: Build images - run: make docker-build + run: make docker-build docker-build-testing-tools - name: Prepare kind environment run: make kind-load @@ -54,7 +55,12 @@ jobs: run: | helm install --wait --timeout 10m --dependency-update --create-namespace --namespace dragonfly-system -f ${{ env.DRAGONFLY_CHARTS_CONFIG_PATH }} dragonfly ${{ env.DRAGONFLY_CHARTS_PATH }} kubectl apply -f ${{ env.DRAGONFLY_FILE_SERVER_PATH }} + kubectl apply -f ${{ env.DRAGONFLY_PROXY_SERVER_PATH }} kubectl wait po file-server-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m + kubectl wait po file-server-no-content-length-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m + kubectl wait po proxy-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m + kubectl wait po proxy-1 --namespace dragonfly-e2e --for=condition=ready --timeout=10m + kubectl wait po proxy-2 --namespace dragonfly-e2e --for=condition=ready --timeout=10m - name: Run E2E test run: make actions-e2e-test-coverage diff --git a/Makefile b/Makefile index c4e06c86c5b..36623653af4 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,12 @@ docker-build-manager: ./hack/docker-build.sh manager .PHONY: docker-build-manager +# Build testing tools image +docker-build-testing-tools: build-dirs + @echo "Begin to testing tools image." + ./test/tools/no-content-length/build.sh +.PHONY: docker-build-testing-tools + # Push cdn image docker-push-cdn: docker-build-cdn @echo "Begin to push cdn docker image." @@ -218,9 +224,9 @@ clean-e2e-test: .PHONY: clean-e2e-test # Kind load dragonlfy -kind-load: kind-load-cdn kind-load-scheduler kind-load-dfdaemon kind-load-manager +kind-load: kind-load-cdn kind-load-scheduler kind-load-dfdaemon kind-load-manager kind-load-testing-tools @echo "Kind load image done." -.PHONY: docker-build +.PHONY: kind-load # Run kind load docker-image cdn kind-load-cdn: @@ -242,6 +248,11 @@ kind-load-manager: @./hack/kind-load.sh manager .PHONY: kind-load-manager +# Run kind load docker testing tools +kind-load-testing-tools: + @./hack/kind-load.sh no-content-length +.PHONY: kind-load-testing-tools + # Run code lint lint: markdownlint @echo "Begin to golangci-lint." @@ -302,11 +313,12 @@ help: @echo "make e2e-test run e2e tests" @echo "make e2e-test-coverage run e2e tests with coverage" @echo "make clean-e2e-test clean e2e tests" - @echo "make kind-load-image kind load docker image" + @echo "make kind-load kind load docker image" @echo "make kind-load-cdn kind load cdn docker image" @echo "make kind-load-scheduler kind load scheduler docker image" @echo "make kind-load-dfdaemon kind load dfdaemon docker image" @echo "make kind-load-manager kind load manager docker image" + @echo "make kind-load-testing-tools kind load testing tools docker image" @echo "make lint run code lint" @echo "make markdownlint run markdown lint" @echo "make swag generate swagger api docs" diff --git a/client/daemon/peer/peertask_stream.go b/client/daemon/peer/peertask_stream.go index fcb22f7ecc3..82e4bf0ed37 100644 --- a/client/daemon/peer/peertask_stream.go +++ b/client/daemon/peer/peertask_stream.go @@ -379,15 +379,7 @@ func (s *streamPeerTask) backSource() { defer backSourceSpan.End() s.contentLength.Store(-1) _ = s.callback.Init(s) - reportPieceCtx, reportPieceSpan := tracer.Start(backSourceCtx, config.SpanReportPieceResult) - defer reportPieceSpan.End() - if peerPacketStream, err := s.schedulerClient.ReportPieceResult(reportPieceCtx, s.taskID, s.request); err != nil { - logger.Errorf("step 2: peer %s report piece failed: err", s.request.PeerId, err) - } else { - s.peerPacketStream = peerPacketStream - } - logger.Infof("step 2: start report peer %s back source piece result", s.request.PeerId) - err := s.pieceManager.DownloadSource(s.ctx, s, s.request) + err := s.pieceManager.DownloadSource(backSourceCtx, s, s.request) if err != nil { s.Errorf("download from source error: %s", err) s.failedReason = err.Error() diff --git a/hack/install-e2e-test.sh b/hack/install-e2e-test.sh index 5284b6efd00..629eaa27205 100755 --- a/hack/install-e2e-test.sh +++ b/hack/install-e2e-test.sh @@ -7,6 +7,7 @@ set -o pipefail KIND_CONFIG_PATH="test/testdata/kind/config.yaml" CHARTS_CONFIG_PATH="test/testdata/charts/config.yaml" FILE_SERVER_CONFIG_PATH="test/testdata/k8s/file-server.yaml" +PROXY_SERVER_CONFIG_PATH="test/testdata/k8s/proxy.yaml" CHARTS_PATH="deploy/helm-charts/charts/dragonfly" NAMESPACE="dragonfly-system" E2E_NAMESPACE="dragonfly-e2e" @@ -39,8 +40,24 @@ install-helm() { install-file-server() { kubectl apply -f ${FILE_SERVER_CONFIG_PATH} kubectl wait --namespace ${E2E_NAMESPACE} \ - --for=condition=ready pod ${FILE_SERVER_NAME} \ - --timeout=10m + --for=condition=ready pod ${FILE_SERVER_NAME} \ + --timeout=10m + kubectl wait --namespace ${E2E_NAMESPACE} \ + --for=condition=ready pod file-server-no-content-length-0 \ + --timeout=10m +} + +install-proxy-server() { + kubectl apply -f ${PROXY_SERVER_CONFIG_PATH} + kubectl wait --namespace ${E2E_NAMESPACE} \ + --for=condition=ready pod proxy-0 \ + --timeout=10m + kubectl wait --namespace ${E2E_NAMESPACE} \ + --for=condition=ready pod proxy-1 \ + --timeout=10m + kubectl wait --namespace ${E2E_NAMESPACE} \ + --for=condition=ready pod proxy-2 \ + --timeout=10m } install-ginkgo() { @@ -72,7 +89,7 @@ main() { install-kind print_step_info "start building docker images" - make docker-build + make docker-build docker-build-testing-tools print_step_info "start loading image for kind" make kind-load @@ -83,6 +100,9 @@ main() { print_step_info "start install file server" install-file-server + print_step_info "start install proxy server" + install-proxy-server + print_step_info "start install ginkgo" install-ginkgo diff --git a/hack/kind-load.sh b/hack/kind-load.sh index 35075b0fac1..7cc923820a8 100755 --- a/hack/kind-load.sh +++ b/hack/kind-load.sh @@ -24,6 +24,9 @@ main() { ;; manager) kind-load manager + ;; + no-content-length) + kind-load no-content-length esac } diff --git a/test/e2e/dfget_test.go b/test/e2e/dfget_test.go index 52b5c2acde4..0ff57ca4bf0 100644 --- a/test/e2e/dfget_test.go +++ b/test/e2e/dfget_test.go @@ -19,6 +19,7 @@ package e2e import ( "fmt" "strings" + "time" . "github.com/onsi/ginkgo" //nolint . "github.com/onsi/gomega" //nolint @@ -26,40 +27,88 @@ import ( "d7y.io/dragonfly/v2/test/e2e/e2eutil" ) -var _ = Describe("Download with dfget", func() { +var _ = Describe("Download with dfget and proxy", func() { Context("dfget", func() { - It("dfget download should be ok", func() { - out, err := e2eutil.KubeCtlCommand("-n", dragonflyNamespace, "get", "pod", "-l", "component=dfdaemon", - "-o", "jsonpath='{range .items[*]}{.metadata.name}{end}'").CombinedOutput() - podName := strings.Trim(string(out), "'") - Expect(err).NotTo(HaveOccurred()) - fmt.Println(podName) - Expect(strings.HasPrefix(podName, "dragonfly-dfdaemon-")).Should(BeTrue()) - pod := e2eutil.NewPodExec(dragonflyNamespace, podName, "dfdaemon") + singleDfgetTest("dfget daemon download should be ok", + dragonflyNamespace, "component=dfdaemon", + "dragonfly-dfdaemon-", "dfdaemon") + for i := 0; i < 3; i++ { + singleDfgetTest( + fmt.Sprintf("dfget daemon proxy-%d should be ok", i), + dragonflyE2ENamespace, + fmt.Sprintf("statefulset.kubernetes.io/pod-name=proxy-%d", i), + "proxy-", "proxy") + } + }) +}) - for _, v := range e2eutil.GetFileList() { - url := e2eutil.GetFileURL(v) - fmt.Println("download url " + url) +func singleDfgetTest(name, ns, label, podNamePrefix, container string) { + It(name, func() { + out, err := e2eutil.KubeCtlCommand("-n", ns, "get", "pod", "-l", label, + "-o", "jsonpath='{range .items[*]}{.metadata.name}{end}'").CombinedOutput() + podName := strings.Trim(string(out), "'") + Expect(err).NotTo(HaveOccurred()) + fmt.Println("test in pod: " + podName) + Expect(strings.HasPrefix(podName, podNamePrefix)).Should(BeTrue()) + pod := e2eutil.NewPodExec(ns, podName, container) - // get original file digest - out, err = e2eutil.DockerCommand("sha256sum", v).CombinedOutput() - fmt.Println(string(out)) - Expect(err).NotTo(HaveOccurred()) - sha256sum1 := strings.Split(string(out), " ")[0] + var urls = map[string]string{} + for _, v := range e2eutil.GetFileList() { + urls[e2eutil.GetFileURL(v)] = v + urls[e2eutil.GetNoContentLengthFileURL(v)] = v + } - // download file - out, err = pod.Command("dfget", "-O", "/tmp/d7y.out", url).CombinedOutput() - fmt.Println(string(out)) - Expect(err).NotTo(HaveOccurred()) + for url, path := range urls { + fmt.Println("download url: " + url) + // get original file digest + out, err = e2eutil.DockerCommand("sha256sum", path).CombinedOutput() + fmt.Println("original sha256sum: " + string(out)) + Expect(err).NotTo(HaveOccurred()) + sha256sum1 := strings.Split(string(out), " ")[0] - // get downloaded file digest - out, err = pod.Command("sha256sum", "/tmp/d7y.out").CombinedOutput() - fmt.Println(string(out)) - Expect(err).NotTo(HaveOccurred()) - sha256sum2 := strings.Split(string(out), " ")[0] + var ( + start time.Time + end time.Time + ) + // download file via dfget + start = time.Now() + out, err = pod.Command("dfget", "-O", "/tmp/d7y.out", url).CombinedOutput() + end = time.Now() + fmt.Println(string(out)) + Expect(err).NotTo(HaveOccurred()) + + // get dfget downloaded file digest + out, err = pod.Command("sha256sum", "/tmp/d7y.out").CombinedOutput() + fmt.Println("dfget sha256sum: " + string(out)) + Expect(err).NotTo(HaveOccurred()) + sha256sum2 := strings.Split(string(out), " ")[0] + Expect(sha256sum1).To(Equal(sha256sum2)) - Expect(sha256sum1).To(Equal(sha256sum2)) + // slow download + Expect(end.Sub(start).Seconds() < 30.0).To(Equal(true)) + + // skip dfdaemon + if ns == dragonflyNamespace { + continue } - }) + // download file via proxy + start = time.Now() + out, err = pod.Command("sh", "-c", fmt.Sprintf(` + export http_proxy=http://127.0.0.1:65001 + wget -O /tmp/wget.out %s`, url)).CombinedOutput() + end = time.Now() + fmt.Println(string(out)) + Expect(err).NotTo(HaveOccurred()) + + // get proxy downloaded file digest + out, err = pod.Command("sha256sum", "/tmp/wget.out").CombinedOutput() + fmt.Println("wget sha256sum: " + string(out)) + Expect(err).NotTo(HaveOccurred()) + sha256sum3 := strings.Split(string(out), " ")[0] + Expect(sha256sum1).To(Equal(sha256sum3)) + + // slow download + Expect(end.Sub(start).Seconds() < 30.0).To(Equal(true)) + } }) -}) +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 7146e98b283..467d052669d 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -30,9 +30,10 @@ import ( ) const ( - proxy = "localhost:65001" - hostnameFilePath = "/etc/hostname" - dragonflyNamespace = "dragonfly-system" + proxy = "localhost:65001" + hostnameFilePath = "/etc/hostname" + dragonflyNamespace = "dragonfly-system" + dragonflyE2ENamespace = "dragonfly-e2e" ) const ( diff --git a/test/e2e/e2eutil/file_server.go b/test/e2e/e2eutil/file_server.go index d1e638be973..c3a35443c8d 100644 --- a/test/e2e/e2eutil/file_server.go +++ b/test/e2e/e2eutil/file_server.go @@ -40,3 +40,8 @@ func GetFileURL(filePath string) string { baseURL := "http://file-server.dragonfly-e2e.svc/kind" return fmt.Sprintf("%s%s", baseURL, filePath) } + +func GetNoContentLengthFileURL(filePath string) string { + baseURL := "http://file-server-no-content-length.dragonfly-e2e.svc/kind" + return fmt.Sprintf("%s%s", baseURL, filePath) +} diff --git a/test/e2e/manager/preheat.go b/test/e2e/manager/preheat.go index 49020364cf2..049d05fe051 100644 --- a/test/e2e/manager/preheat.go +++ b/test/e2e/manager/preheat.go @@ -48,11 +48,11 @@ var _ = Describe("Preheat with manager", func() { for _, v := range e2eutil.GetFileList() { url := e2eutil.GetFileURL(v) - fmt.Println("download url " + url) + fmt.Println("download url: " + url) // get original file digest out, err := e2eutil.DockerCommand("sha256sum", v).CombinedOutput() - fmt.Println(string(out)) + fmt.Println("original sha256sum: " + string(out)) Expect(err).NotTo(HaveOccurred()) sha256sum1 := strings.Split(string(out), " ")[0] @@ -92,7 +92,7 @@ var _ = Describe("Preheat with manager", func() { It("preheat image should be ok", func() { url := "https://registry-1.docker.io/v2/library/alpine/manifests/3.14" - fmt.Println("download image " + url) + fmt.Println("download image: " + url) var ( cdnTaskIDs = []string{ @@ -145,7 +145,7 @@ var _ = Describe("Preheat with manager", func() { It("concurrency 100 preheat should be ok", func() { // generate the data file url := e2eutil.GetFileURL(hostnameFilePath) - fmt.Println("download url " + url) + fmt.Println("download url: " + url) dataFilePath := "post_data" fd, err := os.Create(dataFilePath) Expect(err).NotTo(HaveOccurred()) @@ -162,7 +162,7 @@ var _ = Describe("Preheat with manager", func() { // get original file digest out, err = e2eutil.DockerCommand("sha256sum", hostnameFilePath).CombinedOutput() - fmt.Println(string(out)) + fmt.Println("original sha256sum: " + string(out)) Expect(err).NotTo(HaveOccurred()) sha256sum1 := strings.Split(string(out), " ")[0] @@ -259,10 +259,9 @@ func checkPreheatResult(cdnPods [3]*e2eutil.PodExec, cdnTaskID string) string { // calculate digest of downloaded file out, err = cdn.Command("sha256sum", fmt.Sprintf("%s/%s/%s", cdnCachePath, dir, file)).CombinedOutput() - fmt.Println(string(out)) + fmt.Println("preheat sha256sum: " + string(out)) Expect(err).NotTo(HaveOccurred()) sha256sum2 = strings.Split(string(out), " ")[0] - fmt.Println(string(sha256sum2)) break } return sha256sum2 diff --git a/test/testdata/k8s/file-server.yaml b/test/testdata/k8s/file-server.yaml index 9c82b3c89dd..7d1b3d2df9d 100644 --- a/test/testdata/k8s/file-server.yaml +++ b/test/testdata/k8s/file-server.yaml @@ -4,6 +4,7 @@ metadata: name: dragonfly-e2e --- + apiVersion: v1 kind: Service metadata: @@ -21,6 +22,7 @@ spec: targetPort: 80 --- + apiVersion: apps/v1 kind: StatefulSet metadata: @@ -53,3 +55,55 @@ spec: - name: files hostPath: path: / + +--- + +apiVersion: v1 +kind: Service +metadata: + name: file-server-no-content-length + namespace: dragonfly-e2e +spec: + selector: + app: dragonfly + component: file-server-no-content-length + type: ClusterIP + ports: + - name: server + port: 80 + protocol: TCP + targetPort: 80 + +--- + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: file-server-no-content-length + namespace: dragonfly-e2e +spec: + serviceName: file-server-no-content-length + selector: + matchLabels: + app: dragonfly + component: file-server-no-content-length + replicas: 1 + template: + metadata: + labels: + app: dragonfly + component: file-server-no-content-length + spec: + containers: + - name: server + image: d7yio/no-content-length:latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 80 + volumeMounts: + - name: files + mountPath: /static/kind + volumes: + - name: files + hostPath: + path: / \ No newline at end of file diff --git a/test/testdata/k8s/proxy.yaml b/test/testdata/k8s/proxy.yaml new file mode 100644 index 00000000000..2b268f95313 --- /dev/null +++ b/test/testdata/k8s/proxy.yaml @@ -0,0 +1,132 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: proxy + namespace: dragonfly-e2e +data: + dfget.yaml: |- + aliveTime: 0s + gcInterval: 1m0s + keepStorage: false + dataDir: /root/.dragonfly/dfget-daemon/ + workDir: /root/.dragonfly/dfget-daemon/ + verbose: true + pprof-port: 0 + scheduler: + manager: + enable: true + netAddrs: + - type: tcp + addr: dragonfly-manager.dragonfly-system.svc.cluster.local:65003 + refreshInterval: 5m + scheduleTimeout: 30s + disableAutoBackSource: true + host: + advertiseIP: 0.0.0.0 + idc: "" + listenIP: 0.0.0.0 + location: "" + netTopology: "" + securityDomain: "" + download: + calculateDigest: true + downloadGRPC: + security: + insecure: true + unixListen: + socket: /tmp/dfdamon.sock + peerGRPC: + security: + insecure: true + tcpListen: + listen: 0.0.0.0 + port: 65000 + perPeerRateLimit: 100Mi + totalRateLimit: 200Mi + upload: + rateLimit: 100Mi + security: + insecure: true + tcpListen: + listen: 0.0.0.0 + port: 65002 + storage: + diskGCThreshold: 50Gi + multiplex: true + strategy: io.d7y.storage.v2.simple + taskExpireTime: 6h + proxy: + defaultFilter: Expires&Signature + tcpListen: + listen: 0.0.0.0 + port: 65001 + security: + insecure: true + registryMirror: + dynamic: true + insecure: false + url: https://index.docker.io + proxies: + - regx: blobs/sha256.* + - regx: file-server + +--- + +apiVersion: v1 +kind: Service +metadata: + name: proxy + namespace: dragonfly-e2e +spec: + selector: + app: dragonfly + component: proxy + type: ClusterIP + ports: + - name: proxy + port: 65001 + protocol: TCP + targetPort: 65001 + +--- + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: proxy + namespace: dragonfly-e2e +spec: + serviceName: proxy + selector: + matchLabels: + app: dragonfly + component: proxy + replicas: 3 + template: + metadata: + labels: + app: dragonfly + component: proxy + spec: + containers: + - name: proxy + image: d7yio/dfdaemon:latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 65001 + volumeMounts: + - mountPath: /etc/dragonfly + name: config + - mountPath: /var/log/dragonfly/daemon + name: logs + - mountPath: /root/.dragonfly/dfget-daemon/ + name: data + volumes: + - configMap: + defaultMode: 420 + name: proxy + name: config + - emptyDir: {} + name: data + - emptyDir: {} + name: logs diff --git a/test/tools/no-content-length/Dockerfile b/test/tools/no-content-length/Dockerfile new file mode 100644 index 00000000000..eebf3d52ee0 --- /dev/null +++ b/test/tools/no-content-length/Dockerfile @@ -0,0 +1,24 @@ +ARG BASE_IMAGE=alpine:3.14 + +FROM golang:1.17.3-alpine3.14 as builder + +COPY . /go/src/ + +ARG GOPROXY +ARG GOTAGS +ARG GOGCFLAGS + +RUN cd /go/src/ && GO111MODULE=off go build -o /tmp/no-content-length . + +FROM ${BASE_IMAGE} + +COPY --from=builder /tmp/no-content-length /usr/local/bin/ + +RUN echo "hosts: files dns" > /etc/nsswitch.conf + +EXPOSE 80 + +WORKDIR / + +ENTRYPOINT ["/usr/local/bin/no-content-length"] + diff --git a/test/tools/no-content-length/build.sh b/test/tools/no-content-length/build.sh new file mode 100755 index 00000000000..cb4d94a79d9 --- /dev/null +++ b/test/tools/no-content-length/build.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +docker build -t "${D7Y_REGISTRY:-d7yio}/no-content-length:${D7Y_VERSION:-latest}" \ + -f test/tools/no-content-length/Dockerfile \ + test/tools/no-content-length \ No newline at end of file diff --git a/test/tools/no-content-length/main.go b/test/tools/no-content-length/main.go new file mode 100644 index 00000000000..1137fd45fd1 --- /dev/null +++ b/test/tools/no-content-length/main.go @@ -0,0 +1,87 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "flag" + "fmt" + "io" + "net/http" + "os" + "path" + "path/filepath" + "strings" +) + +var port = flag.Int("port", 80, "") + +func main() { + http.Handle("/", &fileHandler{dir: "/static"}) + fmt.Printf("listen on %d", *port) + err := http.ListenAndServe(fmt.Sprintf(":%d", *port), nil) + if err != nil { + panic(err) + } +} + +type fileHandler struct { + dir string + enableContentLength bool +} + +func (f *fileHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + upath := filepath.Clean(r.URL.Path) + if !strings.HasPrefix(upath, "/") { + upath = "/" + upath + r.URL.Path = upath + } + + filePath := path.Join(f.dir, upath) + if !strings.HasPrefix(filePath, f.dir) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(fmt.Sprintf("target is not in correct dir"))) + return + } + fileInfo, err := os.Stat(filePath) + if err != nil { + if os.IsNotExist(err) { + w.WriteHeader(http.StatusNotFound) + } else { + w.WriteHeader(http.StatusInternalServerError) + } + _, _ = w.Write([]byte(fmt.Sprintf("%s", err))) + return + } + if fileInfo.IsDir() { + // todo list files + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(fmt.Sprintf("target is dir not file"))) + return + } + file, err := os.Open(filePath) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(fmt.Sprintf("%s", err))) + return + } + defer file.Close() + + if f.enableContentLength { + w.Header().Set("Content-Length", fmt.Sprintf("%d", fileInfo.Size())) + } + _, _ = io.Copy(w, file) +}