From cc007940905f6a34f50650b2720a122b948b1599 Mon Sep 17 00:00:00 2001 From: spencercjh Date: Fri, 27 Dec 2024 08:56:17 +0000 Subject: [PATCH 1/6] feat: introduce test_filter_by_comm.sh Signed-off-by: spencercjh --- .github/workflows/test.yml | 100 ++++++++++++++++++-------------- testdata/test_filter_by_comm.sh | 38 ++++++++++++ 2 files changed, 95 insertions(+), 43 deletions(-) create mode 100644 testdata/test_filter_by_comm.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 748199b4..e2897a7d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,40 +18,40 @@ jobs: build: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - with: - submodules: recursive - - - name: Set up Go - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5 - with: - go-version: '1.22.4' - - - name: Build - run: | - sudo apt update - sudo apt install -y git - sudo apt-get -y install pkg-config - sudo apt install -y libelf-dev - - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add - - sudo add-apt-repository "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main" - sudo apt update - - sudo apt install -y llvm - sudo apt install -y clang - pwd - ls -l - make clean && make build-bpf && make - - # - name: Test - # run: make test - - - name: Store executable - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4 - with: - name: kyanos - path: kyanos + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + submodules: recursive + + - name: Set up Go + uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5 + with: + go-version: '1.22.4' + + - name: Build + run: | + sudo apt update + sudo apt install -y git + sudo apt-get -y install pkg-config + sudo apt install -y libelf-dev + + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add - + sudo add-apt-repository "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main" + sudo apt update + + sudo apt install -y llvm + sudo apt install -y clang + pwd + ls -l + make clean && make build-bpf && make + + # - name: Test + # run: make test + + - name: Store executable + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4 + with: + name: kyanos + path: kyanos e2e-test: @@ -101,7 +101,7 @@ jobs: install-dependencies: 'true' cmd: | chmod +x /host/kyanos/kyanos - + - name: download btf file if: ${{ startsWith(matrix.kernel, '4.') }} run: | @@ -113,7 +113,7 @@ jobs: ls -la data/ find data/ -path "*vmlinuz*" -type f find data/ -path "*btf*" -type f - + - name: copy btf file if: ${{ startsWith(matrix.kernel, '4.') }} uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19 @@ -125,7 +125,7 @@ jobs: cat /etc/os-release sudo mkdir -p /var/lib/kyanos/btf/ - + sudo cp /host/data/kernels/4.*/boot/btf-4.* /var/lib/kyanos/btf/current.btf # btf_file=$(find /host/ -path "*btf*" -type f) # sudo cp $btf_file /var/lib/ptcpdump/btf/vmlinux @@ -155,8 +155,22 @@ jobs: #install python pip sudo apt install -y python3 python3-pip pipx - - + - name: Test filter by comm + uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19 + with: + provision: 'false' + cmd: | + set -ex + uname -a + cat /etc/issue + pushd /host + if [ -f "/var/lib/kyanos/btf/current.btf" ]; then + bash /host/testdata/test_filter_by_comm.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' + else + bash /host/testdata/test_filter_by_comm.sh '/host/kyanos/kyanos $kyanos_log_option' + fi + popd + - name: Test gotls uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19 with: @@ -172,7 +186,7 @@ jobs: bash /host/testdata/test_gotls.sh '/host/kyanos/kyanos $kyanos_log_option' fi popd - + - name: Test https uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19 with: @@ -188,7 +202,7 @@ jobs: bash /host/testdata/test_https.sh '/host/kyanos/kyanos $kyanos_log_option' fi popd - + - name: Test side uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19 with: @@ -263,7 +277,7 @@ jobs: else bash /host/testdata/test_kern_evt.sh '/host/kyanos/kyanos $kyanos_log_option' fi - + - name: Test test docker filter by container id uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19 with: @@ -347,7 +361,7 @@ jobs: else bash /host/testdata/test_redis.sh '/host/kyanos/kyanos $kyanos_log_option' fi - + - name: Test k8s if: ${{ startsWith(matrix.kernel, '6.') }} uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19 diff --git a/testdata/test_filter_by_comm.sh b/testdata/test_filter_by_comm.sh new file mode 100644 index 00000000..367375f3 --- /dev/null +++ b/testdata/test_filter_by_comm.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +. $(dirname "$0")/common.sh +set -ex + +CMD="$1" +FILE_PREFIX="/tmp/kyanos" +BEFORE_LNAME="${FILE_PREFIX}_filter_by_comm_before.log" +AFTER_LNAME="${FILE_PREFIX}_filter_by_comm_after.log" + +function test_filter_by_comm() { + openssl req -x509 -newkey rsa:2048 -keyout server.pem -out server.pem -days 365 -nodes -subj "/C=US/ST=California/L=San Francisco/O=My Company/CN=localhost" + + pip install --break-system-packages ssl || true + + # server start before kyanos + timeout 40 python3 ./testdata/start_https_server.py & + timeout 30 ${CMD} watch --debug-output http --comm python3 2>&1 | tee "${BEFORE_LNAME}" & + sleep 10 + curl --insecure https://127.0.0.1:4443 &>/dev/null || true + wait + + cat "${BEFORE_LNAME}" | grep "127.0.0.1:4443" + + # server start after kyanos + timeout 40 ${CMD} watch --debug-output http --comm python3 2>&1 | tee "${AFTER_LNAME}" & + timeout 30 python3 ./testdata/start_https_server.py & + sleep 10 + curl --insecure https://127.0.0.1:4443 &>/dev/null || true + wait + + cat "${AFTER_LNAME}" | grep "127.0.0.1:4443" +} + +function main() { + test_filter_by_comm +} + +main \ No newline at end of file From f1a54e1f977427de625dcca179bab9207ee80d8a Mon Sep 17 00:00:00 2001 From: spencercjh Date: Mon, 30 Dec 2024 07:38:54 +0000 Subject: [PATCH 2/6] fix: better bash set flags Signed-off-by: spencercjh --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e2897a7d..3c9dcc6c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -160,7 +160,7 @@ jobs: with: provision: 'false' cmd: | - set -ex + set -euxo pipefail uname -a cat /etc/issue pushd /host From 9eca3db35a339532c55aa899963e4c6fe8b1b7a4 Mon Sep 17 00:00:00 2001 From: spencercjh Date: Mon, 30 Dec 2024 08:34:03 +0000 Subject: [PATCH 3/6] fix: correctly implement the test Signed-off-by: spencercjh --- testdata/start_http_server.py | 27 +++++++++++++++++++++++++++ testdata/test_filter_by_comm.sh | 21 +++++++++------------ 2 files changed, 36 insertions(+), 12 deletions(-) create mode 100644 testdata/start_http_server.py diff --git a/testdata/start_http_server.py b/testdata/start_http_server.py new file mode 100644 index 00000000..0c186a84 --- /dev/null +++ b/testdata/start_http_server.py @@ -0,0 +1,27 @@ +import http.server +import ssl +from socketserver import ThreadingMixIn + +# 创建自定义的 HTTP 服务器类，支持线程以处理多个连接 +class ThreadedHTTPServer(ThreadingMixIn, http.server.HTTPServer): + # 设置 allow_reuse_address 以支持长连接 + allow_reuse_address = True + +class KeepAliveHandler(http.server.SimpleHTTPRequestHandler): + # 设置响应头以启用长连接 + + # 重写 `do_GET` 方法处理 GET 请求 + def do_GET(self): + self.send_response(200) + self.send_header("Content-type", "text/html") + self.send_header("Connection", "keep-alive") + self.send_header("Content-Length", str(len("Hello, this is an HTTP server with keep-alive support!"))) + self.end_headers() + self.wfile.write(b"Hello, this is an HTTP server with keep-alive support!") + +# 服务器地址和端口 +server_address = ('localhost', 8080) +httpd = ThreadedHTTPServer(server_address, KeepAliveHandler) + +print("HTTP server running on http://localhost:8080 with keep-alive support") +httpd.serve_forever() diff --git a/testdata/test_filter_by_comm.sh b/testdata/test_filter_by_comm.sh index 367375f3..3b3ae45a 100644 --- a/testdata/test_filter_by_comm.sh +++ b/testdata/test_filter_by_comm.sh @@ -8,27 +8,24 @@ BEFORE_LNAME="${FILE_PREFIX}_filter_by_comm_before.log" AFTER_LNAME="${FILE_PREFIX}_filter_by_comm_after.log" function test_filter_by_comm() { - openssl req -x509 -newkey rsa:2048 -keyout server.pem -out server.pem -days 365 -nodes -subj "/C=US/ST=California/L=San Francisco/O=My Company/CN=localhost" - - pip install --break-system-packages ssl || true - # server start before kyanos - timeout 40 python3 ./testdata/start_https_server.py & + timeout 40 python3 ./testdata/start_http_server.py & timeout 30 ${CMD} watch --debug-output http --comm python3 2>&1 | tee "${BEFORE_LNAME}" & sleep 10 - curl --insecure https://127.0.0.1:4443 &>/dev/null || true + curl http://127.0.0.1:8080 &>/dev/null || true wait - cat "${BEFORE_LNAME}" | grep "127.0.0.1:4443" + cat "${BEFORE_LNAME}" + cat "${BEFORE_LNAME}" | grep "Host: 127.0.0.1:8080" | grep "\\[side\\]=server" - # server start after kyanos - timeout 40 ${CMD} watch --debug-output http --comm python3 2>&1 | tee "${AFTER_LNAME}" & - timeout 30 python3 ./testdata/start_https_server.py & + # client start after kyanos + timeout 40 ${CMD} watch --debug-output http --comm curl 2>&1 | tee "${AFTER_LNAME}" & sleep 10 - curl --insecure https://127.0.0.1:4443 &>/dev/null || true + curl http://github.com &>/dev/null || true wait - cat "${AFTER_LNAME}" | grep "127.0.0.1:4443" + cat "${AFTER_LNAME}" + cat "${AFTER_LNAME}" | grep "Host: github.com" | grep "\\[side\\]=client" } function main() { From 481f27712dd07788d33c8570f2d89aeab24804d4 Mon Sep 17 00:00:00 2001 From: spencercjh Date: Mon, 30 Dec 2024 09:25:03 +0000 Subject: [PATCH 4/6] fix: use https-request instead of curl to send HTTP requests Signed-off-by: spencercjh --- testdata/https-request/http_request.go | 4 ++-- testdata/test_filter_by_comm.sh | 15 ++++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/testdata/https-request/http_request.go b/testdata/https-request/http_request.go index 2445c825..9ece8812 100644 --- a/testdata/https-request/http_request.go +++ b/testdata/https-request/http_request.go @@ -2,7 +2,7 @@ package main import ( "fmt" - "io/ioutil" + "io" "net/http" "os" "strconv" @@ -42,7 +42,7 @@ func main() { } // 读取并打印响应内容 - body, err := ioutil.ReadAll(response.Body) + body, err := io.ReadAll(response.Body) if err != nil { fmt.Printf("Failed to read response %d: %v\n", i+1, err) response.Body.Close() diff --git a/testdata/test_filter_by_comm.sh b/testdata/test_filter_by_comm.sh index 3b3ae45a..3cb58188 100644 --- a/testdata/test_filter_by_comm.sh +++ b/testdata/test_filter_by_comm.sh @@ -7,21 +7,24 @@ FILE_PREFIX="/tmp/kyanos" BEFORE_LNAME="${FILE_PREFIX}_filter_by_comm_before.log" AFTER_LNAME="${FILE_PREFIX}_filter_by_comm_after.log" -function test_filter_by_comm() { +function test_filter_by_server_comm() { # server start before kyanos timeout 40 python3 ./testdata/start_http_server.py & timeout 30 ${CMD} watch --debug-output http --comm python3 2>&1 | tee "${BEFORE_LNAME}" & + sleep 2 + timeout 25 ./testdata/https-request/https-request 'http://127.0.0.1:8080' 40 & sleep 10 - curl http://127.0.0.1:8080 &>/dev/null || true wait cat "${BEFORE_LNAME}" cat "${BEFORE_LNAME}" | grep "Host: 127.0.0.1:8080" | grep "\\[side\\]=server" +} +function test_filter_by_client_comm() { # client start after kyanos - timeout 40 ${CMD} watch --debug-output http --comm curl 2>&1 | tee "${AFTER_LNAME}" & + timeout 40 ${CMD} watch --debug-output http --comm https-request 2>&1 | tee "${AFTER_LNAME}" & sleep 10 - curl http://github.com &>/dev/null || true + timeout 30 ./testdata/https-request/https-request 'http://github.com' 40 & wait cat "${AFTER_LNAME}" @@ -29,7 +32,9 @@ function test_filter_by_comm() { } function main() { - test_filter_by_comm + test_filter_by_server_comm + sleep 10 + test_filter_by_client_comm } main \ No newline at end of file From 2a3fcc878f1866d22132b78b8282a46b3c0289b2 Mon Sep 17 00:00:00 2001 From: spencercjh Date: Tue, 31 Dec 2024 11:10:24 +0800 Subject: [PATCH 5/6] fix: GET ipinfo.io instead of github.com Signed-off-by: spencercjh --- testdata/test_filter_by_comm.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testdata/test_filter_by_comm.sh b/testdata/test_filter_by_comm.sh index 3cb58188..95c50f7b 100644 --- a/testdata/test_filter_by_comm.sh +++ b/testdata/test_filter_by_comm.sh @@ -24,11 +24,11 @@ function test_filter_by_client_comm() { # client start after kyanos timeout 40 ${CMD} watch --debug-output http --comm https-request 2>&1 | tee "${AFTER_LNAME}" & sleep 10 - timeout 30 ./testdata/https-request/https-request 'http://github.com' 40 & + timeout 30 ./testdata/https-request/https-request 'http://ipinfo.io' 40 & wait cat "${AFTER_LNAME}" - cat "${AFTER_LNAME}" | grep "Host: github.com" | grep "\\[side\\]=client" + cat "${AFTER_LNAME}" | grep "Host: ipinfo.io" | grep "\\[side\\]=client" } function main() { From c40996f14ea1259d72c881ad30567a13f12cb8e3 Mon Sep 17 00:00:00 2001 From: spencercjh Date: Tue, 31 Dec 2024 11:45:29 +0800 Subject: [PATCH 6/6] fix: skipping tests that can't be passed Signed-off-by: spencercjh --- testdata/test_filter_by_comm.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testdata/test_filter_by_comm.sh b/testdata/test_filter_by_comm.sh index 95c50f7b..1833a505 100644 --- a/testdata/test_filter_by_comm.sh +++ b/testdata/test_filter_by_comm.sh @@ -20,6 +20,7 @@ function test_filter_by_server_comm() { cat "${BEFORE_LNAME}" | grep "Host: 127.0.0.1:8080" | grep "\\[side\\]=server" } +# skip for https://github.com/hengyoush/kyanos/pull/222#issuecomment-2566106756 function test_filter_by_client_comm() { # client start after kyanos timeout 40 ${CMD} watch --debug-output http --comm https-request 2>&1 | tee "${AFTER_LNAME}" & @@ -33,8 +34,6 @@ function test_filter_by_client_comm() { function main() { test_filter_by_server_comm - sleep 10 - test_filter_by_client_comm } main \ No newline at end of file