From f90565c9db64dbb6bf583b015cbcec1a2392a478 Mon Sep 17 00:00:00 2001 From: Miles Granger Date: Tue, 30 Mar 2021 20:02:05 +0200 Subject: [PATCH] Implement PyBufferProtocol for RustyBuffer (#48) * Always return cramjam.Buffer for non _into variants * cramjam.Buffer support buffer protocol * Fix CI after dropping abi3 support (#49) * CI: Add python-version matrix * CI: Fix linux cross build * CI: Use maturin 0.10 prerelease, it has i686 wheels * AsBytes; explicit fail if not all bytes are in memory * Buffer/File impl __len__, __repr__ and __bool__ Co-authored-by: messense --- .github/workflows/CI.yml | 49 +++-- Cargo.toml | 7 +- Makefile | 2 +- README.md | 8 +- benchmarks/README.md | 448 ++++++++++++++++++--------------------- benchmarks/test_bench.py | 64 +++--- pypy_patch.py | 3 +- src/brotli.rs | 26 +-- src/deflate.rs | 26 +-- src/exceptions.rs | 1 + src/gzip.rs | 26 +-- src/io.rs | 183 ++++++++++++++-- src/lib.rs | 131 +++++------- src/lz4.rs | 27 +-- src/snappy.rs | 180 +++------------- src/zstd.rs | 40 ++-- tests/test_variants.py | 28 ++- 17 files changed, 614 insertions(+), 635 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 899112d1..0cfa4321 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -13,11 +13,14 @@ on: jobs: macos: runs-on: macos-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.6 + python-version: ${{ matrix.python-version }} - name: Install Rust toolchain uses: actions-rs/toolchain@v1 with: @@ -59,6 +62,7 @@ jobs: runs-on: windows-latest strategy: matrix: + python-version: [3.6, 3.7, 3.8, 3.9] platform: [ { python-architecture: "x64", target: "x86_64-pc-windows-msvc" }, { python-architecture: "x86", target: "i686-pc-windows-msvc" }, @@ -67,7 +71,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.6 + python-version: ${{ matrix.python-version }} architecture: ${{ matrix.platform.python-architecture }} - name: Install Rust toolchain uses: actions-rs/toolchain@v1 @@ -83,7 +87,7 @@ jobs: if: matrix.platform.python-architecture == 'x64' run: cargo test --no-default-features --release - name: Install maturin - run: pip install maturin + run: pip install --pre maturin - name: Build wheels run: | maturin build -i python --release --out dist --no-sdist --target ${{ matrix.platform.target }} @@ -102,6 +106,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: + python-version: [3.6, 3.7, 3.8, 3.9] platform: [ { manylinux: '2010', target: "x86_64-unknown-linux-gnu", arch: "x86_64" }, { manylinux: '2010', target: "i686-unknown-linux-gnu", arch: "i686" }, @@ -120,14 +125,14 @@ jobs: run: cargo test --no-default-features - uses: actions/setup-python@v2 with: - python-version: 3.6 + python-version: ${{ matrix.python-version }} - name: Build Wheels run: | echo 'curl --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable source ~/.cargo/env - export PATH=/opt/python/cp38-cp38/bin:$PATH - pip install maturin - maturin build -i python --release --out dist --no-sdist --target ${{ matrix.platform.target }} --manylinux ${{ matrix.platform.manylinux }} + export PATH=/opt/python/cp36-cp36m/bin:/opt/python/cp37-cp37m/bin:/opt/python/cp38-cp38/bin:/opt/python/cp39-cp39/bin:$PATH + pip install --pre maturin + maturin build -i python${{ matrix.python-version }} --release --out dist --no-sdist --target ${{ matrix.platform.target }} --manylinux ${{ matrix.platform.manylinux }} ' > build-wheel.sh docker run --rm -v "$PWD":/io -w /io quay.io/pypa/manylinux${{ matrix.platform.manylinux }}_${{ matrix.platform.arch }} bash build-wheel.sh @@ -147,21 +152,31 @@ jobs: runs-on: ubuntu-latest strategy: matrix: + python: [ + { version: '3.6', abi: 'm' }, + { version: '3.7', abi: 'm' }, + { version: '3.8', abi: '' }, + { version: '3.9', abi: '' }, + ] platform: [ { manylinux: "2014", target: "aarch64-unknown-linux-gnu", arch: "aarch64" }, { manylinux: "2014", target: "armv7-unknown-linux-gnueabihf", arch: "armv7" }, ] steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: 3.6 - name: Build Wheels run: | echo 'curl -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable source ~/.cargo/env rustup target add ${{ matrix.platform.target }} - maturin build -i python --release --out dist --no-sdist --target ${{ matrix.platform.target }} --manylinux ${{ matrix.platform.manylinux }} --cargo-extra-args="--no-default-features" --cargo-extra-args="--features=abi3,extension-module" # disable mimallocator + + export PYO3_CROSS_PYTHON_VERSION=${{ matrix.python.version }} + PY_VER="${PYO3_CROSS_PYTHON_VERSION//./}" + PY_ABI=${{ matrix.python.abi }} + PY_TAG="cp$PY_VER-cp$PY_VER$PY_ABI" + export PYO3_CROSS_LIB_DIR=/opt/python/$PY_TAG/lib + + maturin build -i "python$PYO3_CROSS_PYTHON_VERSION" --release --out dist --no-sdist --target ${{ matrix.platform.target }} --manylinux ${{ matrix.platform.manylinux }} --cargo-extra-args="--no-default-features" --cargo-extra-args="--features=extension-module" # disable mimallocator ' > build-wheel.sh docker run --rm -v "$PWD":/io -w /io messense/manylinux2014-cross:${{ matrix.platform.arch }} bash build-wheel.sh @@ -169,19 +184,23 @@ jobs: name: Install built wheel with: arch: ${{ matrix.platform.arch }} - distro: ubuntu18.04 + distro: ubuntu20.04 githubToken: ${{ github.token }} # Mount the dist directory as /artifacts in the container dockerRunArgs: | --volume "${PWD}/dist:/artifacts" install: | apt-get update - apt-get install -y --no-install-recommends python3 python3-pip + apt-get install -y --no-install-recommends python3 python3-pip software-properties-common pip3 install -U pip + add-apt-repository ppa:deadsnakes/ppa + apt-get update + apt-get install -y python3.6 python3.7 python3.9 run: | ls -lrth /artifacts - pip3 install cramjam --no-index --find-links /artifacts --force-reinstall - python3 -c "import cramjam" + PYTHON=python${{ matrix.python.version }} + $PYTHON -m pip install cramjam --no-index --find-links /artifacts --force-reinstall + $PYTHON -c 'import cramjam' - name: Upload wheels uses: actions/upload-artifact@v2 with: diff --git a/Cargo.toml b/Cargo.toml index ffab3274..bb8751b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cramjam" -version = "2.1.0" +version = "2.2.0" authors = ["Miles Granger "] edition = "2018" license-file = "LICENSE" @@ -12,8 +12,7 @@ readme = "README.md" crate-type = ["cdylib"] [features] -default = ["abi3", "mimallocator", "extension-module"] -abi3 = ["pyo3/abi3-py36"] +default = ["mimallocator", "extension-module"] mimallocator = ["mimalloc"] extension-module = ["pyo3/extension-module"] @@ -28,7 +27,7 @@ snap = "^1" brotli2 = "^0.3" lz4 = "^1" flate2 = "^1" -zstd = "0.6.0+zstd.1.4.8" +zstd = "0.6.1+zstd.1.4.9" numpy = "0.13.0" [dependencies.mimalloc] diff --git a/Makefile b/Makefile index 9923a3bb..857adc95 100644 --- a/Makefile +++ b/Makefile @@ -34,5 +34,5 @@ dev-install: pip install cramjam --no-index --find-links dist/ pypy-build: - maturin build -i $(shell which pypy) --release --out dist --cargo-extra-args="--no-default-features" --cargo-extra-args="--features=mimallocator,extension-module" # disable abi3 + maturin build -i $(shell which pypy) --release --out dist pypy ./pypy_patch.py diff --git a/README.md b/README.md index f35488a0..27a6e429 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,15 @@ All available for use as: ```python >>> import cramjam +>>> import numpy as np >>> compressed = cramjam.snappy.compress(b"bytes here") ->>> cramjam.snappy.decompress(compressed) +>>> decompressed = cramjam.snappy.decompress(compressed) +>>> decompressed +cramjam.Buffer(len=10) # an object which implements the buffer protocol +>>> bytes(decompressed) b"bytes here" +>>> np.frombuffer(decompressed, dtype=np.uint8) +array([ 98, 121, 116, 101, 115, 32, 104, 101, 114, 101], dtype=uint8) ``` Where the API is `cramjam..compress/decompress` and accepts diff --git a/benchmarks/README.md b/benchmarks/README.md index 381032ab..da207671 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -37,66 +37,38 @@ we automatically estimate the buffer size and can resize the `bytearray` after. `make bench-gzip` - -Notes ---- -This benchmark is parameterized over setting `output_len=True/False` as an example -of the performance benefit that can be expected from various files/sizes. - - ```bash ----------------------------------------------------------------------------------------------------------------------- benchmark: 48 tests ---------------------------------------------------------------------------------------------------------------------- -Name (time in us) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ -test_gzip[Mark.Twain-Tom.Sawyer.txt-used-output_len=False-cramjam] 498.6320 (1.0) 904.3340 (1.02) 538.3901 (1.02) 48.5933 (1.17) 524.5310 (1.01) 37.6930 (1.09) 152;105 1,857.3891 (0.98) 1505 1 -test_gzip[Mark.Twain-Tom.Sawyer.txt-used-output_len=False-gzip] 574.8140 (1.15) 1,032.3880 (1.16) 616.4469 (1.17) 53.9548 (1.30) 602.6820 (1.17) 45.1167 (1.31) 122;85 1,622.1997 (0.86) 1475 1 -test_gzip[Mark.Twain-Tom.Sawyer.txt-used-output_len=True-cramjam] 499.6490 (1.00) 887.7370 (1.0) 528.1259 (1.0) 41.6166 (1.0) 517.1500 (1.0) 34.4905 (1.0) 149;93 1,893.4877 (1.0) 1676 1 -test_gzip[Mark.Twain-Tom.Sawyer.txt-used-output_len=True-gzip] 575.3320 (1.15) 961.8130 (1.08) 617.2423 (1.17) 48.7969 (1.17) 601.3870 (1.16) 41.9803 (1.22) 127;71 1,620.1094 (0.86) 1329 1 -test_gzip[alice29.txt-used-output_len=False-cramjam] 10,896.4080 (21.85) 12,870.5330 (14.50) 11,282.3297 (21.36) 330.3431 (7.94) 11,201.8925 (21.66) 364.0470 (10.55) 15;3 88.6342 (0.05) 86 1 -test_gzip[alice29.txt-used-output_len=False-gzip] 12,027.0880 (24.12) 14,208.4480 (16.01) 12,427.6071 (23.53) 395.2144 (9.50) 12,312.2350 (23.81) 363.9955 (10.55) 12;3 80.4660 (0.04) 80 1 -test_gzip[alice29.txt-used-output_len=True-cramjam] 10,912.5080 (21.88) 13,144.6680 (14.81) 11,313.8446 (21.42) 331.2081 (7.96) 11,236.6165 (21.73) 315.2310 (9.14) 11;3 88.3873 (0.05) 90 1 -test_gzip[alice29.txt-used-output_len=True-gzip] 11,991.7980 (24.05) 14,076.5000 (15.86) 12,457.9527 (23.59) 439.7621 (10.57) 12,263.0020 (23.71) 461.3358 (13.38) 11;5 80.2700 (0.04) 81 1 -test_gzip[asyoulik.txt-used-output_len=False-cramjam] 8,340.2480 (16.73) 9,570.2210 (10.78) 8,648.5055 (16.38) 233.7639 (5.62) 8,591.8990 (16.61) 269.5990 (7.82) 27;5 115.6269 (0.06) 118 1 -test_gzip[asyoulik.txt-used-output_len=False-gzip] 9,158.9250 (18.37) 9,988.6440 (11.25) 9,459.7682 (17.91) 186.1147 (4.47) 9,412.7690 (18.20) 276.7113 (8.02) 38;0 105.7108 (0.06) 107 1 -test_gzip[asyoulik.txt-used-output_len=True-cramjam] 8,319.9830 (16.69) 10,737.6260 (12.10) 8,666.3427 (16.41) 322.9455 (7.76) 8,599.2010 (16.63) 258.3250 (7.49) 12;3 115.3889 (0.06) 113 1 -test_gzip[asyoulik.txt-used-output_len=True-gzip] 9,188.2840 (18.43) 11,275.7290 (12.70) 9,525.4683 (18.04) 351.3017 (8.44) 9,419.5680 (18.21) 268.0395 (7.77) 8;6 104.9817 (0.06) 100 1 -test_gzip[fireworks.jpeg-used-output_len=False-cramjam] 3,934.7320 (7.89) 6,228.3300 (7.02) 4,123.2676 (7.81) 232.6841 (5.59) 4,057.8555 (7.85) 139.8660 (4.06) 21;20 242.5261 (0.13) 238 1 -test_gzip[fireworks.jpeg-used-output_len=False-gzip] 3,225.5690 (6.47) 5,195.1360 (5.85) 3,414.4240 (6.47) 234.7346 (5.64) 3,347.1610 (6.47) 129.6175 (3.76) 25;26 292.8752 (0.15) 285 1 -test_gzip[fireworks.jpeg-used-output_len=True-cramjam] 3,937.0940 (7.90) 6,554.8870 (7.38) 4,114.5315 (7.79) 247.2504 (5.94) 4,073.0180 (7.88) 147.3470 (4.27) 7;7 243.0410 (0.13) 131 1 -test_gzip[fireworks.jpeg-used-output_len=True-gzip] 3,198.9300 (6.42) 5,349.8960 (6.03) 3,404.4112 (6.45) 244.9796 (5.89) 3,344.0150 (6.47) 147.7962 (4.29) 19;18 293.7366 (0.16) 279 1 -test_gzip[geo.protodata-used-output_len=False-cramjam] 1,585.4920 (3.18) 2,736.4010 (3.08) 1,661.7925 (3.15) 102.5084 (2.46) 1,640.5520 (3.17) 73.7418 (2.14) 37;31 601.7598 (0.32) 597 1 -test_gzip[geo.protodata-used-output_len=False-gzip] 2,139.4700 (4.29) 3,134.1810 (3.53) 2,243.2067 (4.25) 124.7361 (3.00) 2,215.2610 (4.28) 91.4940 (2.65) 33;21 445.7904 (0.24) 442 1 -test_gzip[geo.protodata-used-output_len=True-cramjam] 1,581.0920 (3.17) 2,745.9970 (3.09) 1,656.5480 (3.14) 113.7994 (2.73) 1,627.7865 (3.15) 66.2845 (1.92) 41;41 603.6650 (0.32) 612 1 -test_gzip[geo.protodata-used-output_len=True-gzip] 2,150.2560 (4.31) 3,237.2160 (3.65) 2,254.2443 (4.27) 149.2253 (3.59) 2,217.4510 (4.29) 92.2995 (2.68) 30;29 443.6076 (0.23) 437 1 -test_gzip[html-used-output_len=False-cramjam] 1,960.5420 (3.93) 3,141.3660 (3.54) 2,048.0674 (3.88) 109.5254 (2.63) 2,023.5200 (3.91) 78.3633 (2.27) 35;27 488.2652 (0.26) 495 1 -test_gzip[html-used-output_len=False-gzip] 2,473.2450 (4.96) 3,558.9080 (4.01) 2,594.7358 (4.91) 128.5785 (3.09) 2,570.0210 (4.97) 103.8637 (3.01) 33;20 385.3957 (0.20) 379 1 -test_gzip[html-used-output_len=True-cramjam] 1,958.1090 (3.93) 3,045.2460 (3.43) 2,048.7684 (3.88) 118.1116 (2.84) 2,024.7970 (3.92) 73.6460 (2.14) 25;25 488.0981 (0.26) 460 1 -test_gzip[html-used-output_len=True-gzip] 2,459.9110 (4.93) 3,534.8000 (3.98) 2,597.0396 (4.92) 153.6156 (3.69) 2,559.4020 (4.95) 97.3550 (2.82) 31;30 385.0538 (0.20) 387 1 -test_gzip[html_x_4-used-output_len=False-cramjam] 9,386.4630 (18.82) 10,598.7610 (11.94) 9,729.7528 (18.42) 273.3211 (6.57) 9,641.9175 (18.64) 232.4350 (6.74) 18;9 102.7775 (0.05) 100 1 -test_gzip[html_x_4-used-output_len=False-gzip] 10,954.5380 (21.97) 12,229.1090 (13.78) 11,388.3416 (21.56) 314.4430 (7.56) 11,291.9835 (21.84) 441.7480 (12.81) 28;0 87.8091 (0.05) 88 1 -test_gzip[html_x_4-used-output_len=True-cramjam] 9,362.4440 (18.78) 10,832.3570 (12.20) 9,753.7416 (18.47) 237.0947 (5.70) 9,740.3650 (18.83) 287.9575 (8.35) 20;4 102.5248 (0.05) 99 1 -test_gzip[html_x_4-used-output_len=True-gzip] 10,925.2580 (21.91) 12,430.4210 (14.00) 11,443.7624 (21.67) 264.1632 (6.35) 11,395.8990 (22.04) 319.3560 (9.26) 22;2 87.3838 (0.05) 87 1 -test_gzip[kppkn.gtb-used-output_len=False-cramjam] 39,132.5940 (78.48) 40,894.0630 (46.07) 39,748.0645 (75.26) 372.1767 (8.94) 39,747.7660 (76.86) 304.5605 (8.83) 6;1 25.1585 (0.01) 25 1 -test_gzip[kppkn.gtb-used-output_len=False-gzip] 63,357.7700 (127.06) 64,867.0860 (73.07) 63,931.2074 (121.05) 431.4790 (10.37) 63,917.2790 (123.60) 455.9250 (13.22) 5;2 15.6418 (0.01) 16 1 -test_gzip[kppkn.gtb-used-output_len=True-cramjam] 39,015.7680 (78.25) 42,225.1180 (47.56) 39,948.5314 (75.64) 763.3876 (18.34) 39,720.7660 (76.81) 784.1145 (22.73) 6;2 25.0322 (0.01) 25 1 -test_gzip[kppkn.gtb-used-output_len=True-gzip] 63,601.0280 (127.55) 65,441.2490 (73.72) 64,262.9742 (121.68) 533.9574 (12.83) 64,180.3945 (124.10) 821.0655 (23.81) 5;0 15.5611 (0.01) 16 1 -test_gzip[lcet10.txt-used-output_len=False-cramjam] 26,893.0920 (53.93) 28,413.5420 (32.01) 27,353.3749 (51.79) 383.5836 (9.22) 27,217.9150 (52.63) 500.9510 (14.52) 9;2 36.5586 (0.02) 37 1 -test_gzip[lcet10.txt-used-output_len=False-gzip] 29,453.3110 (59.07) 32,535.7740 (36.65) 30,217.1804 (57.22) 580.9736 (13.96) 30,086.3775 (58.18) 727.5420 (21.09) 6;1 33.0938 (0.02) 34 1 -test_gzip[lcet10.txt-used-output_len=True-cramjam] 26,531.6900 (53.21) 27,972.4670 (31.51) 27,264.7757 (51.63) 359.6821 (8.64) 27,271.8585 (52.73) 541.0055 (15.69) 12;0 36.6774 (0.02) 36 1 -test_gzip[lcet10.txt-used-output_len=True-gzip] 29,633.5980 (59.43) 31,146.0220 (35.08) 30,229.0893 (57.24) 363.9110 (8.74) 30,206.0570 (58.41) 489.7215 (14.20) 10;0 33.0807 (0.02) 33 1 -test_gzip[paper-100k.pdf-used-output_len=False-cramjam] 2,830.4720 (5.68) 3,538.2900 (3.99) 2,964.3271 (5.61) 107.5574 (2.58) 2,941.1885 (5.69) 120.8570 (3.50) 64;15 337.3447 (0.18) 318 1 -test_gzip[paper-100k.pdf-used-output_len=False-gzip] 3,236.7040 (6.49) 4,065.8930 (4.58) 3,425.2506 (6.49) 142.2416 (3.42) 3,401.9400 (6.58) 147.6630 (4.28) 69;16 291.9494 (0.15) 290 1 -test_gzip[paper-100k.pdf-used-output_len=True-cramjam] 2,829.1540 (5.67) 3,389.4550 (3.82) 2,957.7600 (5.60) 108.1616 (2.60) 2,929.9220 (5.67) 113.1508 (3.28) 58;24 338.0937 (0.18) 331 1 -test_gzip[paper-100k.pdf-used-output_len=True-gzip] 3,244.5330 (6.51) 5,180.7690 (5.84) 3,436.5281 (6.51) 201.4648 (4.84) 3,392.0260 (6.56) 125.4383 (3.64) 26;26 290.9914 (0.15) 261 1 -test_gzip[plrabn12.txt-used-output_len=False-cramjam] 49,320.9690 (98.91) 74,427.5540 (83.84) 53,266.0228 (100.86) 5,882.3811 (141.35) 51,283.4770 (99.17) 2,127.7145 (61.69) 2;3 18.7737 (0.01) 20 1 -test_gzip[plrabn12.txt-used-output_len=False-gzip] 58,243.8110 (116.81) 62,047.1040 (69.89) 60,300.9332 (114.18) 1,419.1243 (34.10) 60,473.5410 (116.94) 2,739.9335 (79.44) 7;0 16.5835 (0.01) 16 1 -test_gzip[plrabn12.txt-used-output_len=True-cramjam] 49,100.9320 (98.47) 63,868.7720 (71.95) 52,104.9511 (98.66) 4,869.3169 (117.00) 49,982.7855 (96.65) 1,824.6450 (52.90) 3;3 19.1920 (0.01) 16 1 -test_gzip[plrabn12.txt-used-output_len=True-gzip] 58,425.0160 (117.17) 63,285.9410 (71.29) 59,995.6096 (113.60) 1,132.8758 (27.22) 60,199.5660 (116.41) 1,314.8300 (38.12) 4;1 16.6679 (0.01) 17 1 -test_gzip[urls.10K-used-output_len=False-cramjam] 36,073.3510 (72.34) 41,615.1860 (46.88) 38,465.0136 (72.83) 1,701.3922 (40.88) 38,056.7860 (73.59) 3,375.4912 (97.87) 10;0 25.9977 (0.01) 23 1 -test_gzip[urls.10K-used-output_len=False-gzip] 40,484.9950 (81.19) 42,920.4170 (48.35) 41,712.5562 (78.98) 718.7713 (17.27) 41,697.8900 (80.63) 1,178.5225 (34.17) 7;0 23.9736 (0.01) 25 1 -test_gzip[urls.10K-used-output_len=True-cramjam] 36,067.1570 (72.33) 39,557.0310 (44.56) 37,054.0519 (70.16) 729.0753 (17.52) 36,985.7040 (71.52) 730.4390 (21.18) 7;1 26.9876 (0.01) 27 1 -test_gzip[urls.10K-used-output_len=True-gzip] 40,734.0010 (81.69) 43,980.7760 (49.54) 42,111.1947 (79.74) 836.1285 (20.09) 42,234.7625 (81.67) 1,114.0880 (32.30) 9;0 23.7467 (0.01) 24 1 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ +------------------------------------------------------------------------------------------------------------------- benchmark: 28 tests ------------------------------------------------------------------------------------------------------------------- +Name (time in us) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +test_gzip[Mark.Twain-Tom.Sawyer.txt-cramjam] 499.5080 (1.0) 728.4359 (1.0) 512.5226 (1.0) 18.5201 (1.0) 502.8080 (1.0) 19.4933 (1.0) 208;63 1,951.1333 (1.0) 1469 1 +test_gzip[Mark.Twain-Tom.Sawyer.txt-gzip] 574.5530 (1.15) 800.9881 (1.10) 590.6570 (1.15) 20.6061 (1.11) 579.6610 (1.15) 21.3466 (1.10) 204;62 1,693.0301 (0.87) 1384 1 +test_gzip[alice29.txt-cramjam] 10,784.0190 (21.59) 11,725.0331 (16.10) 11,019.1375 (21.50) 159.9243 (8.64) 10,992.1545 (21.86) 139.6049 (7.16) 17;7 90.7512 (0.05) 90 1 +test_gzip[alice29.txt-gzip] 11,856.6670 (23.74) 12,723.6000 (17.47) 12,117.6208 (23.64) 154.9040 (8.36) 12,090.8240 (24.05) 148.3495 (7.61) 14;5 82.5245 (0.04) 81 1 +test_gzip[asyoulik.txt-cramjam] 8,255.5760 (16.53) 8,808.0181 (12.09) 8,388.3462 (16.37) 89.2145 (4.82) 8,374.5810 (16.66) 102.5778 (5.26) 30;4 119.2130 (0.06) 119 1 +test_gzip[asyoulik.txt-gzip] 9,073.6510 (18.17) 9,783.6760 (13.43) 9,242.8460 (18.03) 109.5283 (5.91) 9,221.6395 (18.34) 117.2170 (6.01) 23;5 108.1918 (0.06) 106 1 +test_gzip[fifty-four-mb-random-cramjam] 1,918,547.3130 (>1000.0) 2,281,174.9650 (>1000.0) 2,015,150.7522 (>1000.0) 151,686.4135 (>1000.0) 1,968,377.9971 (>1000.0) 140,688.2324 (>1000.0) 1;1 0.4962 (0.00) 5 1 +test_gzip[fifty-four-mb-random-gzip] 1,665,076.1690 (>1000.0) 1,747,323.1020 (>1000.0) 1,705,157.4009 (>1000.0) 29,886.1207 (>1000.0) 1,704,222.0499 (>1000.0) 35,055.7323 (>1000.0) 2;0 0.5865 (0.00) 5 1 +test_gzip[fifty-four-mb-repeating-cramjam] 202,266.0240 (404.93) 203,656.1889 (279.58) 202,876.8688 (395.84) 586.9810 (31.69) 202,665.2470 (403.07) 976.8272 (50.11) 2;0 4.9291 (0.00) 5 1 +test_gzip[fifty-four-mb-repeating-gzip] 406,177.8371 (813.16) 408,499.9190 (560.79) 406,954.2060 (794.02) 941.9838 (50.86) 406,857.7340 (809.17) 1,182.0850 (60.64) 1;0 2.4573 (0.00) 5 1 +test_gzip[fireworks.jpeg-cramjam] 3,925.8640 (7.86) 5,138.1841 (7.05) 4,011.1756 (7.83) 110.0326 (5.94) 3,984.3025 (7.92) 83.2741 (4.27) 20;11 249.3035 (0.13) 246 1 +test_gzip[fireworks.jpeg-gzip] 3,184.8371 (6.38) 3,616.7459 (4.97) 3,262.3774 (6.37) 59.6370 (3.22) 3,251.3360 (6.47) 72.2620 (3.71) 62;8 306.5249 (0.16) 302 1 +test_gzip[geo.protodata-cramjam] 1,572.8211 (3.15) 1,828.2511 (2.51) 1,610.3785 (3.14) 36.8006 (1.99) 1,599.6840 (3.18) 44.1979 (2.27) 101;18 620.9720 (0.32) 609 1 +test_gzip[geo.protodata-gzip] 2,136.3030 (4.28) 2,421.2720 (3.32) 2,184.2864 (4.26) 51.3550 (2.77) 2,165.2665 (4.31) 64.8794 (3.33) 64;14 457.8154 (0.23) 436 1 +test_gzip[html-cramjam] 1,950.0050 (3.90) 2,314.5130 (3.18) 2,001.8077 (3.91) 46.9272 (2.53) 1,987.3570 (3.95) 61.8350 (3.17) 81;12 499.5485 (0.26) 480 1 +test_gzip[html-gzip] 2,466.7480 (4.94) 4,897.7289 (6.72) 2,561.2299 (5.00) 230.8277 (12.46) 2,516.4341 (5.00) 77.6293 (3.98) 10;18 390.4374 (0.20) 391 1 +test_gzip[html_x_4-cramjam] 9,295.9950 (18.61) 10,382.4440 (14.25) 9,466.6361 (18.47) 166.3815 (8.98) 9,422.9169 (18.74) 134.9731 (6.92) 12;9 105.6341 (0.05) 107 1 +test_gzip[html_x_4-gzip] 10,916.7770 (21.86) 11,705.1441 (16.07) 11,102.0056 (21.66) 138.3860 (7.47) 11,072.2100 (22.02) 140.6029 (7.21) 17;4 90.0738 (0.05) 86 1 +test_gzip[kppkn.gtb-cramjam] 38,534.4781 (77.14) 39,374.4000 (54.05) 38,952.7990 (76.00) 200.1030 (10.80) 38,965.1706 (77.50) 238.4051 (12.23) 7;0 25.6721 (0.01) 26 1 +test_gzip[kppkn.gtb-gzip] 62,682.8249 (125.49) 64,644.2850 (88.74) 63,360.6900 (123.63) 646.7011 (34.92) 63,061.5850 (125.42) 938.3130 (48.14) 4;0 15.7827 (0.01) 16 1 +test_gzip[lcet10.txt-cramjam] 26,310.1180 (52.67) 27,208.7040 (37.35) 26,565.8054 (51.83) 223.3611 (12.06) 26,516.8336 (52.74) 235.3310 (12.07) 10;3 37.6424 (0.02) 38 1 +test_gzip[lcet10.txt-gzip] 29,205.7050 (58.47) 30,147.2150 (41.39) 29,597.1343 (57.75) 193.6721 (10.46) 29,561.2030 (58.79) 219.0185 (11.24) 5;2 33.7871 (0.02) 32 1 +test_gzip[paper-100k.pdf-cramjam] 2,809.4910 (5.62) 3,251.7480 (4.46) 2,873.6302 (5.61) 51.4007 (2.78) 2,861.0771 (5.69) 55.6436 (2.85) 50;15 347.9919 (0.18) 343 1 +test_gzip[paper-100k.pdf-gzip] 3,216.7250 (6.44) 3,755.8790 (5.16) 3,286.1528 (6.41) 71.3635 (3.85) 3,266.0410 (6.50) 74.8790 (3.84) 37;12 304.3072 (0.16) 298 1 +test_gzip[plrabn12.txt-cramjam] 47,907.7781 (95.91) 50,276.0150 (69.02) 48,341.7333 (94.32) 489.6980 (26.44) 48,157.5651 (95.78) 338.0665 (17.34) 1;1 20.6861 (0.01) 21 1 +test_gzip[plrabn12.txt-gzip] 52,439.5080 (104.98) 55,409.8539 (76.07) 53,299.8328 (104.00) 872.4323 (47.11) 53,018.4830 (105.44) 1,305.1905 (66.96) 4;0 18.7618 (0.01) 19 1 +test_gzip[urls.10K-cramjam] 35,436.3660 (70.94) 36,588.2240 (50.23) 35,778.6046 (69.81) 306.0787 (16.53) 35,707.5301 (71.02) 246.1017 (12.62) 6;3 27.9497 (0.01) 27 1 +test_gzip[urls.10K-gzip] 40,238.6910 (80.56) 41,588.8891 (57.09) 40,757.2876 (79.52) 266.3417 (14.38) 40,718.7125 (80.98) 206.5550 (10.60) 3;3 24.5355 (0.01) 24 1 ``` #### Snappy @@ -109,77 +81,77 @@ There are two types here, `framed` and `raw`; the recommended one being snappy's ------------------------------------------------------------------------------------------------------------------------- benchmark: 28 tests ------------------------------------------------------------------------------------------------------------------------- Name (time in us) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -test_snappy_framed[Mark.Twain-Tom.Sawyer.txt-cramjam] 66.3110 (1.0) 132.0670 (1.0) 71.5088 (1.0) 8.0543 (1.0) 68.8310 (1.0) 2.5820 (1.0) 472;777 13,984.2887 (1.0) 4909 1 -test_snappy_framed[Mark.Twain-Tom.Sawyer.txt-snappy] 128.1280 (1.93) 280.2570 (2.12) 141.0132 (1.97) 13.1797 (1.64) 136.6060 (1.98) 10.3185 (4.00) 445;314 7,091.5369 (0.51) 3901 1 -test_snappy_framed[alice29.txt-cramjam] 667.8120 (10.07) 1,122.6380 (8.50) 728.5935 (10.19) 70.0799 (8.70) 709.3380 (10.31) 42.2965 (16.38) 152;155 1,372.5075 (0.10) 1407 1 -test_snappy_framed[alice29.txt-snappy] 1,363.7070 (20.57) 1,933.3930 (14.64) 1,463.5245 (20.47) 79.0356 (9.81) 1,446.3070 (21.01) 80.3440 (31.12) 101;31 683.2820 (0.05) 544 1 -test_snappy_framed[asyoulik.txt-cramjam] 580.7250 (8.76) 1,038.6950 (7.86) 614.7546 (8.60) 42.1717 (5.24) 604.4420 (8.78) 39.3620 (15.24) 152;95 1,626.6652 (0.12) 1648 1 -test_snappy_framed[asyoulik.txt-snappy] 1,158.6120 (17.47) 1,628.6620 (12.33) 1,227.1376 (17.16) 68.0189 (8.45) 1,213.2820 (17.63) 52.7795 (20.44) 89;68 814.9045 (0.06) 780 1 -test_snappy_framed[fifty-four-mb-random-cramjam] 77,210.1300 (>1000.0) 81,672.0980 (618.41) 78,480.3618 (>1000.0) 1,149.2224 (142.68) 78,176.9650 (>1000.0) 885.1638 (342.82) 2;1 12.7420 (0.00) 13 1 -test_snappy_framed[fifty-four-mb-random-snappy] 5,559,679.8660 (>1000.0) 8,568,313.0780 (>1000.0) 7,917,293.5574 (>1000.0) 1,319,007.0957 (>1000.0) 8,465,963.1640 (>1000.0) 830,941.5408 (>1000.0) 1;1 0.1263 (0.00) 5 1 -test_snappy_framed[fifty-four-mb-repeating-cramjam] 53,625.9030 (808.70) 56,103.9790 (424.81) 54,536.0511 (762.65) 865.3173 (107.44) 54,204.6480 (787.50) 1,152.4509 (446.34) 5;0 18.3365 (0.00) 18 1 -test_snappy_framed[fifty-four-mb-repeating-snappy] 408,793.3940 (>1000.0) 413,610.6820 (>1000.0) 411,025.2038 (>1000.0) 2,027.2435 (251.70) 410,251.0240 (>1000.0) 3,348.1738 (>1000.0) 2;0 2.4329 (0.00) 5 1 -test_snappy_framed[fireworks.jpeg-cramjam] 93.1590 (1.40) 229.4800 (1.74) 102.4807 (1.43) 11.1360 (1.38) 99.3520 (1.44) 6.0148 (2.33) 863;919 9,757.9334 (0.70) 8747 1 -test_snappy_framed[fireworks.jpeg-snappy] 634.3260 (9.57) 969.7800 (7.34) 670.6240 (9.38) 41.6387 (5.17) 659.5460 (9.58) 43.4297 (16.82) 158;81 1,491.1486 (0.11) 1377 1 -test_snappy_framed[geo.protodata-cramjam] 202.6670 (3.06) 448.9580 (3.40) 216.7288 (3.03) 19.7819 (2.46) 211.0980 (3.07) 12.7265 (4.93) 409;402 4,614.0605 (0.33) 4327 1 -test_snappy_framed[geo.protodata-snappy] 725.8820 (10.95) 1,141.0820 (8.64) 772.4960 (10.80) 44.3410 (5.51) 768.2020 (11.16) 35.2232 (13.64) 269;96 1,294.5051 (0.09) 1283 1 -test_snappy_framed[html-cramjam] 210.3770 (3.17) 385.4390 (2.92) 223.8841 (3.13) 20.1802 (2.51) 213.4705 (3.10) 14.0190 (5.43) 456;391 4,466.5962 (0.32) 3874 1 -test_snappy_framed[html-snappy] 661.9320 (9.98) 1,073.4110 (8.13) 699.2566 (9.78) 44.5358 (5.53) 687.3445 (9.99) 46.4350 (17.98) 151;66 1,430.0901 (0.10) 1360 1 -test_snappy_framed[html_x_4-cramjam] 799.6210 (12.06) 1,471.1290 (11.14) 876.7533 (12.26) 94.8879 (11.78) 851.4875 (12.37) 51.6310 (20.00) 118;123 1,140.5717 (0.08) 1202 1 -test_snappy_framed[html_x_4-snappy] 2,718.3700 (40.99) 3,591.7680 (27.20) 2,945.9552 (41.20) 133.4797 (16.57) 2,910.0090 (42.28) 125.9050 (48.76) 77;28 339.4485 (0.02) 334 1 -test_snappy_framed[kppkn.gtb-cramjam] 570.4850 (8.60) 942.4960 (7.14) 604.2190 (8.45) 37.0589 (4.60) 599.5330 (8.71) 39.6085 (15.34) 157;79 1,655.0290 (0.12) 1568 1 -test_snappy_framed[kppkn.gtb-snappy] 1,418.7900 (21.40) 1,968.2600 (14.90) 1,498.6198 (20.96) 70.0844 (8.70) 1,487.2870 (21.61) 58.7050 (22.74) 144;47 667.2807 (0.05) 619 1 -test_snappy_framed[lcet10.txt-cramjam] 1,753.5210 (26.44) 2,618.3510 (19.83) 1,927.7128 (26.96) 161.5183 (20.05) 1,880.7385 (27.32) 111.6060 (43.22) 78;55 518.7495 (0.04) 494 1 -test_snappy_framed[lcet10.txt-snappy] 3,772.7230 (56.89) 4,574.4210 (34.64) 4,001.1636 (55.95) 142.0375 (17.63) 3,985.7400 (57.91) 172.0220 (66.62) 65;9 249.9273 (0.02) 239 1 -test_snappy_framed[paper-100k.pdf-cramjam] 71.3860 (1.08) 282.0590 (2.14) 81.4524 (1.14) 10.4778 (1.30) 79.0920 (1.15) 6.5700 (2.54) 918;899 12,277.1087 (0.88) 9596 1 -test_snappy_framed[paper-100k.pdf-snappy] 533.3570 (8.04) 821.8800 (6.22) 563.8659 (7.89) 35.0754 (4.35) 556.9420 (8.09) 38.9318 (15.08) 146;75 1,773.4712 (0.13) 1609 1 -test_snappy_framed[plrabn12.txt-cramjam] 2,542.3630 (38.34) 3,281.8630 (24.85) 2,737.9968 (38.29) 111.7024 (13.87) 2,720.9985 (39.53) 138.1680 (53.51) 89;11 365.2305 (0.03) 354 1 -test_snappy_framed[plrabn12.txt-snappy] 5,096.0520 (76.85) 6,126.9320 (46.39) 5,441.2705 (76.09) 190.2210 (23.62) 5,427.8250 (78.86) 247.3365 (95.79) 53;3 183.7806 (0.01) 181 1 -test_snappy_framed[urls.10K-cramjam] 2,274.0300 (34.29) 3,110.8600 (23.56) 2,521.7336 (35.26) 130.5882 (16.21) 2,493.3860 (36.22) 139.6460 (54.08) 103;21 396.5526 (0.03) 396 1 -test_snappy_framed[urls.10K-snappy] 5,976.5190 (90.13) 6,779.0760 (51.33) 6,312.5501 (88.28) 177.9607 (22.10) 6,286.4100 (91.33) 233.4860 (90.43) 49;1 158.4146 (0.01) 152 1 +test_snappy_framed[Mark.Twain-Tom.Sawyer.txt-cramjam] 67.8750 (1.0) 184.4690 (1.0) 69.9458 (1.0) 5.8189 (1.0) 68.5219 (1.0) 0.4220 (1.72) 394;606 14,296.7757 (1.0) 5157 1 +test_snappy_framed[Mark.Twain-Tom.Sawyer.txt-snappy] 127.5920 (1.88) 321.8240 (1.74) 131.6162 (1.88) 9.4019 (1.62) 128.8950 (1.88) 0.7390 (3.01) 407;905 7,597.8511 (0.53) 5234 1 +test_snappy_framed[alice29.txt-cramjam] 676.6730 (9.97) 965.0890 (5.23) 693.1161 (9.91) 27.2101 (4.68) 679.8870 (9.92) 20.7548 (84.45) 143;90 1,442.7596 (0.10) 1423 1 +test_snappy_framed[alice29.txt-snappy] 1,362.6880 (20.08) 1,884.4430 (10.22) 1,394.3984 (19.94) 43.0275 (7.39) 1,385.3500 (20.22) 39.8223 (162.04) 62;37 717.1552 (0.05) 679 1 +test_snappy_framed[asyoulik.txt-cramjam] 593.6730 (8.75) 886.0190 (4.80) 607.4296 (8.68) 24.3360 (4.18) 596.2565 (8.70) 19.8330 (80.70) 174;85 1,646.2814 (0.12) 1612 1 +test_snappy_framed[asyoulik.txt-snappy] 1,154.8221 (17.01) 1,474.2750 (7.99) 1,179.9538 (16.87) 33.2390 (5.71) 1,169.8285 (17.07) 33.0758 (134.59) 86;34 847.4908 (0.06) 814 1 +test_snappy_framed[fifty-four-mb-random-cramjam] 106,125.5330 (>1000.0) 122,712.4280 (665.22) 113,907.3788 (>1000.0) 4,144.4500 (712.24) 113,855.9120 (>1000.0) 2,535.2445 (>1000.0) 3;2 8.7791 (0.00) 11 1 +test_snappy_framed[fifty-four-mb-random-snappy] 5,736,722.0209 (>1000.0) 8,884,037.3160 (>1000.0) 8,029,947.1742 (>1000.0) 1,294,217.3836 (>1000.0) 8,468,064.0850 (>1000.0) 931,786.2789 (>1000.0) 1;1 0.1245 (0.00) 5 1 +test_snappy_framed[fifty-four-mb-repeating-cramjam] 60,967.8230 (898.24) 82,541.3500 (447.45) 68,490.7448 (979.20) 8,907.2626 (>1000.0) 62,633.5774 (914.07) 16,801.8445 (>1000.0) 4;0 14.6005 (0.00) 12 1 +test_snappy_framed[fifty-four-mb-repeating-snappy] 416,213.7700 (>1000.0) 445,269.2589 (>1000.0) 425,080.6548 (>1000.0) 12,411.2216 (>1000.0) 418,530.8550 (>1000.0) 16,490.2027 (>1000.0) 1;0 2.3525 (0.00) 5 1 +test_snappy_framed[fireworks.jpeg-cramjam] 90.7030 (1.34) 265.3861 (1.44) 93.6783 (1.34) 6.9734 (1.20) 91.9680 (1.34) 0.3370 (1.37) 696;1196 10,674.8307 (0.75) 8521 1 +test_snappy_framed[fireworks.jpeg-snappy] 634.0879 (9.34) 905.7500 (4.91) 653.3697 (9.34) 24.7685 (4.26) 643.7055 (9.39) 14.7215 (59.90) 151;145 1,530.5271 (0.11) 1472 1 +test_snappy_framed[geo.protodata-cramjam] 204.8140 (3.02) 484.7000 (2.63) 210.1287 (3.00) 12.0245 (2.07) 206.3761 (3.01) 0.9029 (3.67) 410;868 4,758.9888 (0.33) 4299 1 +test_snappy_framed[geo.protodata-snappy] 727.0420 (10.71) 979.4390 (5.31) 744.5916 (10.65) 25.8275 (4.44) 732.3179 (10.69) 22.5262 (91.66) 155;72 1,343.0181 (0.09) 1299 1 +test_snappy_framed[html-cramjam] 211.0000 (3.11) 400.2160 (2.17) 216.8802 (3.10) 12.6234 (2.17) 212.4980 (3.10) 0.7945 (3.23) 435;805 4,610.8402 (0.32) 3736 1 +test_snappy_framed[html-snappy] 662.6330 (9.76) 1,020.1860 (5.53) 695.4484 (9.94) 29.1009 (5.00) 686.1100 (10.01) 23.4039 (95.23) 345;76 1,437.9211 (0.10) 1350 1 +test_snappy_framed[html_x_4-cramjam] 814.7360 (12.00) 1,156.0221 (6.27) 833.7065 (11.92) 34.9787 (6.01) 819.8969 (11.97) 21.4603 (87.32) 81;76 1,199.4629 (0.08) 1083 1 +test_snappy_framed[html_x_4-snappy] 2,654.4550 (39.11) 3,212.9940 (17.42) 2,729.8276 (39.03) 92.7821 (15.94) 2,701.6510 (39.43) 73.2925 (298.24) 31;25 366.3235 (0.03) 336 1 +test_snappy_framed[kppkn.gtb-cramjam] 582.3770 (8.58) 868.2209 (4.71) 596.1833 (8.52) 23.0826 (3.97) 584.8820 (8.54) 19.5719 (79.64) 194;81 1,677.3366 (0.12) 1515 1 +test_snappy_framed[kppkn.gtb-snappy] 1,413.1500 (20.82) 1,795.0049 (9.73) 1,441.8433 (20.61) 38.8178 (6.67) 1,435.4640 (20.95) 34.1205 (138.84) 54;30 693.5566 (0.05) 603 1 +test_snappy_framed[lcet10.txt-cramjam] 1,766.2169 (26.02) 2,194.9910 (11.90) 1,809.2666 (25.87) 52.9258 (9.10) 1,795.1445 (26.20) 50.4309 (205.21) 42;24 552.7101 (0.04) 526 1 +test_snappy_framed[lcet10.txt-snappy] 3,776.0150 (55.63) 4,416.6800 (23.94) 3,865.1326 (55.26) 90.4870 (15.55) 3,843.4530 (56.09) 87.1853 (354.77) 22;12 258.7233 (0.02) 235 1 +test_snappy_framed[paper-100k.pdf-cramjam] 84.9301 (1.25) 217.1260 (1.18) 87.1440 (1.25) 6.4667 (1.11) 85.6020 (1.25) 0.2458 (1.0) 593;1125 11,475.2531 (0.80) 8555 1 +test_snappy_framed[paper-100k.pdf-snappy] 533.0379 (7.85) 845.9381 (4.59) 546.8427 (7.82) 24.6972 (4.24) 538.5000 (7.86) 16.9175 (68.84) 144;115 1,828.6796 (0.13) 1636 1 +test_snappy_framed[plrabn12.txt-cramjam] 2,386.8741 (35.17) 3,894.9240 (21.11) 2,475.3461 (35.39) 169.0649 (29.05) 2,435.1560 (35.54) 63.3658 (257.84) 20;30 403.9839 (0.03) 339 1 +test_snappy_framed[plrabn12.txt-snappy] 5,019.8400 (73.96) 5,844.5639 (31.68) 5,131.9534 (73.37) 112.6590 (19.36) 5,106.9070 (74.53) 89.4035 (363.79) 15;13 194.8576 (0.01) 188 1 +test_snappy_framed[urls.10K-cramjam] 2,149.3380 (31.67) 2,757.3260 (14.95) 2,204.8847 (31.52) 69.6202 (11.96) 2,185.4395 (31.89) 61.6229 (250.75) 30;15 453.5384 (0.03) 406 1 +test_snappy_framed[urls.10K-snappy] 5,894.0420 (86.84) 6,559.6290 (35.56) 6,029.6206 (86.20) 120.3123 (20.68) 5,994.6314 (87.48) 84.0485 (342.00) 26;18 165.8479 (0.01) 164 1 ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ``` `make bench-snappy-raw` ```bash -------------------------------------------------------------------------------------------------------------- benchmark: 28 tests -------------------------------------------------------------------------------------------------------------- -Name (time in us) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -test_snappy_raw[Mark.Twain-Tom.Sawyer.txt-cramjam] 51.3180 (3.37) 106.5710 (2.12) 54.9889 (3.29) 6.7249 (2.09) 52.6220 (3.32) 1.8468 (2.57) 627;721 18,185.5023 (0.30) 5755 1 -test_snappy_raw[Mark.Twain-Tom.Sawyer.txt-snappy] 52.5980 (3.45) 110.6400 (2.20) 56.2056 (3.36) 6.7379 (2.10) 53.6730 (3.39) 1.6682 (2.32) 1269;1560 17,791.8193 (0.30) 11281 1 -test_snappy_raw[alice29.txt-cramjam] 599.5640 (39.35) 955.8480 (18.98) 634.3983 (37.97) 39.8069 (12.38) 624.0110 (39.38) 40.4120 (56.29) 134;81 1,576.2968 (0.03) 1469 1 -test_snappy_raw[alice29.txt-snappy] 600.3480 (39.41) 1,046.0080 (20.77) 641.8216 (38.42) 52.8951 (16.45) 626.3260 (39.52) 41.8230 (58.25) 148;121 1,558.0655 (0.03) 1631 1 -test_snappy_raw[asyoulik.txt-cramjam] 531.3770 (34.88) 950.9300 (18.88) 567.7280 (33.98) 53.5173 (16.65) 553.5235 (34.93) 38.8605 (54.13) 150;129 1,761.4068 (0.03) 1664 1 -test_snappy_raw[asyoulik.txt-snappy] 532.8600 (34.98) 826.0870 (16.41) 566.0961 (33.88) 42.7028 (13.28) 553.0790 (34.90) 38.2127 (53.23) 97;70 1,766.4846 (0.03) 981 1 -test_snappy_raw[fifty-four-mb-random-cramjam] 64,900.1350 (>1000.0) 66,259.3000 (>1000.0) 65,499.6381 (>1000.0) 488.8944 (152.06) 65,556.7080 (>1000.0) 900.7005 (>1000.0) 9;0 15.2673 (0.00) 16 1 -test_snappy_raw[fifty-four-mb-random-snappy] 55,702.4500 (>1000.0) 63,684.2990 (>1000.0) 56,684.4748 (>1000.0) 1,802.6270 (560.68) 56,194.9530 (>1000.0) 807.8321 (>1000.0) 1;1 17.6415 (0.00) 18 1 -test_snappy_raw[fifty-four-mb-repeating-cramjam] 61,115.8000 (>1000.0) 63,571.1790 (>1000.0) 62,403.5144 (>1000.0) 668.1779 (207.83) 62,227.2180 (>1000.0) 969.2140 (>1000.0) 7;0 16.0247 (0.00) 17 1 -test_snappy_raw[fifty-four-mb-repeating-snappy] 32,122.7550 (>1000.0) 33,752.5920 (670.29) 32,768.2958 (>1000.0) 441.6249 (137.36) 32,726.9150 (>1000.0) 592.4183 (825.17) 12;0 30.5173 (0.00) 31 1 -test_snappy_raw[fireworks.jpeg-cramjam] 24.0780 (1.58) 185.5780 (3.69) 25.9138 (1.55) 4.1937 (1.30) 24.5250 (1.55) 0.8781 (1.22) 2273;2403 38,589.5241 (0.64) 25542 1 -test_snappy_raw[fireworks.jpeg-snappy] 15.2350 (1.0) 50.3550 (1.0) 16.7069 (1.0) 3.2151 (1.0) 15.8470 (1.0) 0.8130 (1.13) 1175;1223 59,855.6622 (1.0) 15115 1 -test_snappy_raw[geo.protodata-cramjam] 155.5390 (10.21) 383.3330 (7.61) 165.6040 (9.91) 16.6777 (5.19) 160.6820 (10.14) 9.3510 (13.02) 576;655 6,038.5013 (0.10) 5400 1 -test_snappy_raw[geo.protodata-snappy] 142.9101 (9.38) 268.0700 (5.32) 152.4274 (9.12) 12.7332 (3.96) 147.1700 (9.29) 8.9320 (12.44) 669;569 6,560.5007 (0.11) 6050 1 -test_snappy_raw[html-cramjam] 165.7090 (10.88) 331.3710 (6.58) 176.3145 (10.55) 16.6429 (5.18) 171.4330 (10.82) 10.0080 (13.94) 573;606 5,671.6825 (0.09) 5450 1 -test_snappy_raw[html-snappy] 156.6590 (10.28) 376.6970 (7.48) 166.9049 (9.99) 16.5424 (5.15) 162.0930 (10.23) 9.4865 (13.21) 576;624 5,991.4352 (0.10) 5383 1 -test_snappy_raw[html_x_4-cramjam] 662.5010 (43.49) 1,178.5590 (23.40) 702.4728 (42.05) 58.7407 (18.27) 685.1415 (43.23) 41.8875 (58.34) 110;94 1,423.5426 (0.02) 1140 1 -test_snappy_raw[html_x_4-snappy] 632.7050 (41.53) 1,083.1810 (21.51) 676.2055 (40.47) 59.6778 (18.56) 658.1900 (41.53) 43.0540 (59.97) 147;126 1,478.8404 (0.02) 1453 1 -test_snappy_raw[kppkn.gtb-cramjam] 500.7480 (32.87) 863.8660 (17.16) 528.4980 (31.63) 37.5560 (11.68) 517.9300 (32.68) 32.8555 (45.76) 143;96 1,892.1548 (0.03) 1692 1 -test_snappy_raw[kppkn.gtb-snappy] 504.1240 (33.09) 855.1690 (16.98) 532.1999 (31.86) 36.5428 (11.37) 522.2600 (32.96) 34.3139 (47.80) 160;103 1,878.9933 (0.03) 1834 1 -test_snappy_raw[lcet10.txt-cramjam] 1,585.7740 (104.09) 2,621.0520 (52.05) 1,683.1401 (100.75) 123.0068 (38.26) 1,654.2825 (104.39) 82.9070 (115.48) 45;40 594.1276 (0.01) 538 1 -test_snappy_raw[lcet10.txt-snappy] 1,590.9740 (104.43) 2,517.1320 (49.99) 1,686.2719 (100.93) 104.7156 (32.57) 1,664.4925 (105.04) 74.3340 (103.54) 58;42 593.0242 (0.01) 582 1 -test_snappy_raw[paper-100k.pdf-cramjam] 27.2180 (1.79) 198.5880 (3.94) 29.5554 (1.77) 4.9856 (1.55) 27.9830 (1.77) 0.9690 (1.35) 2051;2213 33,834.7155 (0.57) 20315 1 -test_snappy_raw[paper-100k.pdf-snappy] 20.1430 (1.32) 57.0260 (1.13) 21.8590 (1.31) 3.2314 (1.01) 21.0150 (1.33) 0.7179 (1.0) 1753;1970 45,747.6927 (0.76) 26477 1 -test_snappy_raw[plrabn12.txt-cramjam] 2,393.2550 (157.09) 3,470.2860 (68.92) 2,615.1149 (156.53) 126.0164 (39.20) 2,584.6890 (163.10) 128.8117 (179.42) 59;17 382.3924 (0.01) 385 1 -test_snappy_raw[plrabn12.txt-snappy] 2,185.9910 (143.48) 3,105.9010 (61.68) 2,312.9505 (138.44) 122.6518 (38.15) 2,280.8130 (143.93) 94.9631 (132.27) 48;32 432.3482 (0.01) 394 1 -test_snappy_raw[urls.10K-cramjam] 1,851.8370 (121.55) 2,722.8100 (54.07) 1,961.6530 (117.42) 100.0348 (31.11) 1,936.3300 (122.19) 84.8327 (118.16) 67;27 509.7741 (0.01) 413 1 -test_snappy_raw[urls.10K-snappy] 1,813.8860 (119.06) 2,805.6660 (55.72) 2,047.6586 (122.56) 121.4122 (37.76) 2,024.1400 (127.73) 93.0140 (129.56) 75;44 488.3627 (0.01) 498 1 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------ benchmark: 28 tests ------------------------------------------------------------------------------------------------------------- +Name (time in us) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +test_snappy_raw[Mark.Twain-Tom.Sawyer.txt-cramjam] 51.9970 (3.43) 110.7800 (2.43) 53.6142 (3.37) 3.9642 (2.32) 52.7401 (3.38) 0.3098 (2.50) 432;630 18,651.7875 (0.30) 7791 1 +test_snappy_raw[Mark.Twain-Tom.Sawyer.txt-snappy] 52.7010 (3.47) 122.6700 (2.69) 54.9113 (3.45) 4.9091 (2.88) 53.6260 (3.44) 0.4690 (3.78) 768;1676 18,211.1850 (0.29) 11113 1 +test_snappy_raw[alice29.txt-cramjam] 599.3680 (39.51) 895.2310 (19.61) 613.8907 (38.59) 22.1703 (13.00) 603.5195 (38.70) 20.4269 (164.76) 133;54 1,628.9544 (0.03) 1158 1 +test_snappy_raw[alice29.txt-snappy] 600.2350 (39.57) 775.9969 (17.00) 613.0591 (38.54) 17.8929 (10.49) 603.6960 (38.71) 19.4444 (156.83) 231;68 1,631.1640 (0.03) 1631 1 +test_snappy_raw[asyoulik.txt-cramjam] 532.1550 (35.08) 831.1180 (18.21) 545.4919 (34.29) 21.3844 (12.54) 535.0631 (34.31) 19.8718 (160.28) 167;63 1,833.2078 (0.03) 1307 1 +test_snappy_raw[asyoulik.txt-snappy] 532.5220 (35.10) 800.6040 (17.54) 544.5727 (34.24) 18.2074 (10.67) 536.1935 (34.38) 12.6080 (101.69) 219;197 1,836.3022 (0.03) 1830 1 +test_snappy_raw[fifty-four-mb-random-cramjam] 37,541.5270 (>1000.0) 38,867.6571 (851.52) 37,919.1172 (>1000.0) 307.6292 (180.35) 37,829.9700 (>1000.0) 311.4055 (>1000.0) 6;2 26.3719 (0.00) 27 1 +test_snappy_raw[fifty-four-mb-random-snappy] 54,326.6139 (>1000.0) 55,845.7799 (>1000.0) 54,884.3537 (>1000.0) 412.7079 (241.95) 54,832.6219 (>1000.0) 475.0847 (>1000.0) 6;1 18.2201 (0.00) 19 1 +test_snappy_raw[fifty-four-mb-repeating-cramjam] 17,394.3370 (>1000.0) 18,066.0550 (395.79) 17,612.4400 (>1000.0) 163.3287 (95.75) 17,561.4910 (>1000.0) 237.1181 (>1000.0) 11;0 56.7780 (0.00) 29 1 +test_snappy_raw[fifty-four-mb-repeating-snappy] 31,430.8960 (>1000.0) 32,698.4840 (716.36) 31,772.2997 (>1000.0) 271.8508 (159.38) 31,746.9710 (>1000.0) 266.7012 (>1000.0) 4;2 31.4740 (0.00) 31 1 +test_snappy_raw[fireworks.jpeg-cramjam] 25.3069 (1.67) 59.5120 (1.30) 26.2138 (1.65) 2.6992 (1.58) 25.5830 (1.64) 0.1240 (1.0) 349;901 38,147.7836 (0.61) 7113 1 +test_snappy_raw[fireworks.jpeg-snappy] 15.1701 (1.0) 45.6451 (1.0) 15.9064 (1.0) 1.7057 (1.0) 15.5960 (1.0) 0.1411 (1.14) 988;5393 62,867.9262 (1.0) 35170 1 +test_snappy_raw[geo.protodata-cramjam] 155.7680 (10.27) 250.6210 (5.49) 160.3773 (10.08) 9.2061 (5.40) 157.2450 (10.08) 0.8419 (6.79) 356;768 6,235.2982 (0.10) 3922 1 +test_snappy_raw[geo.protodata-snappy] 142.9900 (9.43) 266.7190 (5.84) 148.3392 (9.33) 8.0790 (4.74) 145.8429 (9.35) 2.2233 (17.93) 471;684 6,741.3054 (0.11) 5261 1 +test_snappy_raw[html-cramjam] 165.5300 (10.91) 266.9621 (5.85) 169.8480 (10.68) 9.1233 (5.35) 166.7660 (10.69) 0.6332 (5.11) 348;702 5,887.6187 (0.09) 3877 1 +test_snappy_raw[html-snappy] 156.3310 (10.31) 247.4729 (5.42) 160.6741 (10.10) 8.7547 (5.13) 157.9490 (10.13) 0.7500 (6.05) 480;904 6,223.7766 (0.10) 5697 1 +test_snappy_raw[html_x_4-cramjam] 660.6469 (43.55) 1,195.4820 (26.19) 678.1065 (42.63) 39.0818 (22.91) 664.1049 (42.58) 20.3864 (164.43) 42;47 1,474.6945 (0.02) 1035 1 +test_snappy_raw[html_x_4-snappy] 633.0810 (41.73) 917.3179 (20.10) 648.8233 (40.79) 21.6437 (12.69) 638.4600 (40.94) 20.0127 (161.42) 193;63 1,541.2517 (0.02) 1451 1 +test_snappy_raw[kppkn.gtb-cramjam] 499.2380 (32.91) 708.5471 (15.52) 510.3782 (32.09) 18.7145 (10.97) 501.3411 (32.15) 14.9945 (120.94) 175;131 1,959.3315 (0.03) 1355 1 +test_snappy_raw[kppkn.gtb-snappy] 503.4071 (33.18) 733.6430 (16.07) 514.8920 (32.37) 18.8445 (11.05) 505.8900 (32.44) 14.7057 (118.61) 223;174 1,942.1549 (0.03) 1907 1 +test_snappy_raw[lcet10.txt-cramjam] 1,585.7200 (104.53) 1,902.8189 (41.69) 1,620.1644 (101.86) 42.2322 (24.76) 1,608.8235 (103.16) 43.0678 (347.37) 57;18 617.2213 (0.01) 486 1 +test_snappy_raw[lcet10.txt-snappy] 1,590.4551 (104.84) 2,116.2720 (46.36) 1,627.2067 (102.30) 47.9340 (28.10) 1,616.7820 (103.67) 41.8220 (337.32) 48;22 614.5501 (0.01) 534 1 +test_snappy_raw[paper-100k.pdf-cramjam] 28.1520 (1.86) 69.1239 (1.51) 29.0821 (1.83) 2.6677 (1.56) 28.4630 (1.83) 0.1448 (1.17) 344;1243 34,385.3648 (0.55) 7894 1 +test_snappy_raw[paper-100k.pdf-snappy] 20.3360 (1.34) 85.0500 (1.86) 21.3502 (1.34) 2.1718 (1.27) 20.9491 (1.34) 0.4959 (4.00) 930;1321 46,838.0012 (0.75) 25100 1 +test_snappy_raw[plrabn12.txt-cramjam] 2,169.4830 (143.01) 4,583.7279 (100.42) 2,236.1366 (140.58) 180.7923 (105.99) 2,212.1780 (141.84) 60.5080 (488.04) 7;10 447.1999 (0.01) 318 1 +test_snappy_raw[plrabn12.txt-snappy] 2,187.1999 (144.18) 2,717.8620 (59.54) 2,246.2940 (141.22) 67.8218 (39.76) 2,230.6465 (143.03) 47.7184 (384.88) 38;30 445.1777 (0.01) 392 1 +test_snappy_raw[urls.10K-cramjam] 1,852.2060 (122.10) 2,464.6381 (54.00) 1,899.2503 (119.40) 65.4195 (38.35) 1,882.6990 (120.72) 52.5715 (424.02) 31;18 526.5235 (0.01) 424 1 +test_snappy_raw[urls.10K-snappy] 1,814.2400 (119.59) 2,366.7030 (51.85) 1,855.7408 (116.67) 63.5185 (37.24) 1,840.4050 (118.00) 45.1091 (363.83) 25;20 538.8684 (0.01) 438 1 +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ``` ##### Benchmarks for using `compress_into` and `decompress_into` -All variants except `lz4` for now, implement a `compress_into` and `decompress_into` +All variants implement a `compress_into` and `decompress_into` This is where, if you have a Python numpy array pre-allocated to the size of the de/compress data output, we can place the bytes directly into that buffer. @@ -187,38 +159,38 @@ Again, since basically no variants implement similar functionality as we saw in benchmarks, this benchmark is specific to `cramjam` ```bash ---------------------------------------------------------------------------------------------------------------------------- benchmark: 28 tests --------------------------------------------------------------------------------------------------------------------------- -Name (time in us) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -test_cramjam_snappy_de_compress_into[Mark.Twain-Tom.Sawyer.txt-compress_into] 39.5430 (1.89) 226.3750 (4.12) 42.8299 (1.93) 6.0547 (2.06) 41.0640 (1.93) 1.4650 (1.96) 2225;2454 23,348.1766 (0.52) 18846 1 -test_cramjam_snappy_de_compress_into[Mark.Twain-Tom.Sawyer.txt-decompress_into] 20.8690 (1.0) 54.9380 (1.0) 22.1814 (1.0) 2.9375 (1.0) 21.2590 (1.0) 0.7462 (1.0) 463;481 45,082.7264 (1.0) 7705 1 -test_cramjam_snappy_de_compress_into[alice29.txt-compress_into] 463.9240 (22.23) 777.6850 (14.16) 492.9410 (22.22) 32.7011 (11.13) 483.5000 (22.74) 31.2150 (41.83) 197;115 2,028.6405 (0.04) 2053 1 -test_cramjam_snappy_de_compress_into[alice29.txt-decompress_into] 186.3790 (8.93) 382.7770 (6.97) 199.2822 (8.98) 21.8619 (7.44) 192.5040 (9.06) 11.0680 (14.83) 436;575 5,018.0092 (0.11) 4754 1 -test_cramjam_snappy_de_compress_into[asyoulik.txt-compress_into] 412.3180 (19.76) 713.2160 (12.98) 438.1621 (19.75) 36.8647 (12.55) 426.4940 (20.06) 29.6507 (39.73) 226;170 2,282.2606 (0.05) 2321 1 -test_cramjam_snappy_de_compress_into[asyoulik.txt-decompress_into] 163.3530 (7.83) 314.8180 (5.73) 173.4995 (7.82) 18.4431 (6.28) 165.1880 (7.77) 9.7200 (13.03) 567;637 5,763.7058 (0.13) 5530 1 -test_cramjam_snappy_de_compress_into[fifty-four-mb-random-compress_into] 21,370.5300 (>1000.0) 23,421.1310 (426.32) 22,271.0823 (>1000.0) 581.2127 (197.86) 22,076.6660 (>1000.0) 839.1198 (>1000.0) 9;0 44.9013 (0.00) 25 1 -test_cramjam_snappy_de_compress_into[fifty-four-mb-random-decompress_into] 14,683.7930 (703.62) 15,945.4020 (290.24) 15,056.4776 (678.79) 266.1213 (90.59) 15,000.5550 (705.61) 379.6751 (508.79) 7;1 66.4166 (0.00) 30 1 -test_cramjam_snappy_de_compress_into[fifty-four-mb-repeating-compress_into] 14,899.9090 (713.97) 16,099.5890 (293.05) 15,429.4449 (695.60) 276.9102 (94.27) 15,329.7200 (721.09) 336.6940 (451.19) 20;0 64.8111 (0.00) 63 1 -test_cramjam_snappy_de_compress_into[fifty-four-mb-repeating-decompress_into] 17,915.9630 (858.50) 19,089.1940 (347.47) 18,389.6426 (829.06) 278.4235 (94.78) 18,373.7230 (864.28) 418.8805 (561.32) 11;0 54.3784 (0.00) 29 1 -test_cramjam_snappy_de_compress_into[fireworks.jpeg-compress_into] 43.9390 (2.11) 118.4420 (2.16) 47.8558 (2.16) 5.6904 (1.94) 46.2460 (2.18) 1.9707 (2.64) 1996;2197 20,896.1152 (0.46) 19715 1 -test_cramjam_snappy_de_compress_into[fireworks.jpeg-decompress_into] 29.7500 (1.43) 73.8140 (1.34) 31.8771 (1.44) 3.9980 (1.36) 30.5820 (1.44) 1.2860 (1.72) 2432;2732 31,370.4800 (0.70) 27124 1 -test_cramjam_snappy_de_compress_into[geo.protodata-compress_into] 121.8950 (5.84) 260.3290 (4.74) 129.7199 (5.85) 12.8547 (4.38) 124.3525 (5.85) 7.0450 (9.44) 799;848 7,708.9151 (0.17) 7196 1 -test_cramjam_snappy_de_compress_into[geo.protodata-decompress_into] 61.7630 (2.96) 152.9090 (2.78) 65.6442 (2.96) 7.3924 (2.52) 62.6280 (2.95) 2.0600 (2.76) 1269;1729 15,233.6341 (0.34) 12907 1 -test_cramjam_snappy_de_compress_into[html-compress_into] 129.4560 (6.20) 252.4930 (4.60) 138.6755 (6.25) 14.4065 (4.90) 133.9925 (6.30) 7.5980 (10.18) 834;1040 7,211.0811 (0.16) 7318 1 -test_cramjam_snappy_de_compress_into[html-decompress_into] 62.3400 (2.99) 154.2940 (2.81) 66.4681 (3.00) 8.0530 (2.74) 64.3110 (3.03) 3.5030 (4.69) 1064;1480 15,044.8144 (0.33) 11675 1 -test_cramjam_snappy_de_compress_into[html_x_4-compress_into] 517.2050 (24.78) 888.9890 (16.18) 570.9151 (25.74) 50.9777 (17.35) 554.5600 (26.09) 44.1403 (59.15) 353;194 1,751.5740 (0.04) 1857 1 -test_cramjam_snappy_de_compress_into[html_x_4-decompress_into] 257.5690 (12.34) 529.3130 (9.63) 283.4639 (12.78) 37.2057 (12.67) 266.7790 (12.55) 26.8391 (35.97) 301;283 3,527.7862 (0.08) 2446 1 -test_cramjam_snappy_de_compress_into[kppkn.gtb-compress_into] 363.2850 (17.41) 611.8500 (11.14) 383.6018 (17.29) 26.0204 (8.86) 375.6370 (17.67) 23.2070 (31.10) 295;182 2,606.8697 (0.06) 2684 1 -test_cramjam_snappy_de_compress_into[kppkn.gtb-decompress_into] 195.4920 (9.37) 389.6650 (7.09) 206.6503 (9.32) 18.2140 (6.20) 201.5670 (9.48) 11.5785 (15.52) 518;513 4,839.0933 (0.11) 4671 1 -test_cramjam_snappy_de_compress_into[lcet10.txt-compress_into] 1,230.0970 (58.94) 1,987.3830 (36.18) 1,324.3744 (59.71) 91.6116 (31.19) 1,300.6140 (61.18) 70.7862 (94.86) 91;53 755.0735 (0.02) 671 1 -test_cramjam_snappy_de_compress_into[lcet10.txt-decompress_into] 489.8850 (23.47) 866.4870 (15.77) 531.4879 (23.96) 58.0469 (19.76) 516.8210 (24.31) 32.6948 (43.81) 126;136 1,881.5104 (0.04) 1385 1 -test_cramjam_snappy_de_compress_into[paper-100k.pdf-compress_into] 38.2830 (1.83) 97.7700 (1.78) 42.4763 (1.91) 5.6442 (1.92) 40.7560 (1.92) 2.0365 (2.73) 2347;2564 23,542.5644 (0.52) 19856 1 -test_cramjam_snappy_de_compress_into[paper-100k.pdf-decompress_into] 29.9950 (1.44) 80.4800 (1.46) 32.0370 (1.44) 4.0740 (1.39) 31.0030 (1.46) 0.9880 (1.32) 2199;2610 31,213.8860 (0.69) 26692 1 -test_cramjam_snappy_de_compress_into[plrabn12.txt-compress_into] 1,670.0190 (80.02) 2,474.3870 (45.04) 1,797.8845 (81.05) 104.9018 (35.71) 1,773.9745 (83.45) 89.2560 (119.61) 104;39 556.2093 (0.01) 580 1 -test_cramjam_snappy_de_compress_into[plrabn12.txt-decompress_into] 657.2370 (31.49) 1,143.2750 (20.81) 709.0987 (31.97) 61.9860 (21.10) 697.8510 (32.83) 33.1955 (44.48) 87;104 1,410.2409 (0.03) 1088 1 -test_cramjam_snappy_de_compress_into[urls.10K-compress_into] 1,454.5130 (69.70) 2,186.5280 (39.80) 1,609.2278 (72.55) 129.4296 (44.06) 1,569.5480 (73.83) 100.0898 (134.13) 54;36 621.4161 (0.01) 467 1 -test_cramjam_snappy_de_compress_into[urls.10K-decompress_into] 612.5320 (29.35) 1,063.2690 (19.35) 670.1230 (30.21) 62.6793 (21.34) 654.0500 (30.77) 43.8632 (58.78) 104;89 1,492.2634 (0.03) 1135 1 ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +--------------------------------------------------------------------------------------------------------------------------- benchmark: 28 tests ---------------------------------------------------------------------------------------------------------------------------- +Name (time in us) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +test_cramjam_snappy_de_compress_into[Mark.Twain-Tom.Sawyer.txt-compress_into] 40.5740 (6.10) 80.9841 (4.98) 42.0519 (6.02) 3.3541 (3.54) 41.1549 (5.99) 0.3109 (4.71) 1006;2779 23,780.1522 (0.17) 17961 1 +test_cramjam_snappy_de_compress_into[Mark.Twain-Tom.Sawyer.txt-decompress_into] 6.7360 (1.01) 21.8459 (1.34) 6.9848 (1.0) 0.9464 (1.0) 6.8730 (1.0) 0.0660 (1.0) 122;700 143,168.7285 (1.0) 9684 1 +test_cramjam_snappy_de_compress_into[alice29.txt-compress_into] 461.2940 (69.40) 669.6520 (41.18) 473.9482 (67.85) 18.6320 (19.69) 464.7850 (67.62) 15.4326 (233.80) 240;175 2,109.9353 (0.01) 1992 1 +test_cramjam_snappy_de_compress_into[alice29.txt-decompress_into] 6.7560 (1.02) 22.4550 (1.38) 7.1457 (1.02) 0.9911 (1.05) 7.0620 (1.03) 0.1279 (1.94) 51;203 139,943.6715 (0.98) 3866 1 +test_cramjam_snappy_de_compress_into[asyoulik.txt-compress_into] 408.1880 (61.41) 628.2150 (38.64) 420.7408 (60.24) 17.0570 (18.02) 411.6645 (59.90) 19.7020 (298.48) 301;60 2,376.7602 (0.02) 2338 1 +test_cramjam_snappy_de_compress_into[asyoulik.txt-decompress_into] 6.6471 (1.0) 35.4199 (2.18) 7.1268 (1.02) 1.2031 (1.27) 7.0110 (1.02) 0.0992 (1.50) 86;954 140,315.1785 (0.98) 5545 1 +test_cramjam_snappy_de_compress_into[fifty-four-mb-random-compress_into] 21,292.6060 (>1000.0) 22,134.7660 (>1000.0) 21,556.6603 (>1000.0) 195.2096 (206.28) 21,524.6500 (>1000.0) 282.1560 (>1000.0) 7;1 46.3894 (0.00) 26 1 +test_cramjam_snappy_de_compress_into[fifty-four-mb-random-decompress_into] 6.8980 (1.04) 17.6111 (1.08) 7.4460 (1.07) 1.8980 (2.01) 7.0570 (1.03) 0.2597 (3.93) 1;2 134,299.5668 (0.94) 31 1 +test_cramjam_snappy_de_compress_into[fifty-four-mb-repeating-compress_into] 14,725.2050 (>1000.0) 15,681.2500 (964.41) 14,952.5908 (>1000.0) 172.2539 (182.02) 14,909.9910 (>1000.0) 154.4059 (>1000.0) 8;3 66.8780 (0.00) 62 1 +test_cramjam_snappy_de_compress_into[fifty-four-mb-repeating-decompress_into] 6.8520 (1.03) 16.2600 (1.0) 7.4351 (1.06) 1.7599 (1.86) 7.0125 (1.02) 0.1526 (2.31) 1;5 134,496.9292 (0.94) 28 1 +test_cramjam_snappy_de_compress_into[fireworks.jpeg-compress_into] 45.8390 (6.90) 94.1610 (5.79) 47.7454 (6.84) 3.5484 (3.75) 46.8700 (6.82) 0.8349 (12.65) 1167;2927 20,944.4108 (0.15) 19418 1 +test_cramjam_snappy_de_compress_into[fireworks.jpeg-decompress_into] 6.7510 (1.02) 41.1160 (2.53) 7.1202 (1.02) 1.1970 (1.26) 6.9160 (1.01) 0.1888 (2.86) 515;974 140,445.6319 (0.98) 25688 1 +test_cramjam_snappy_de_compress_into[geo.protodata-compress_into] 120.1930 (18.08) 224.7400 (13.82) 124.6344 (17.84) 7.8673 (8.31) 122.0175 (17.75) 1.2110 (18.35) 666;1483 8,023.4694 (0.06) 7554 1 +test_cramjam_snappy_de_compress_into[geo.protodata-decompress_into] 6.7500 (1.02) 28.4720 (1.75) 7.2207 (1.03) 1.2935 (1.37) 7.0831 (1.03) 0.2290 (3.47) 260;297 138,490.1876 (0.97) 11741 1 +test_cramjam_snappy_de_compress_into[html-compress_into] 128.7600 (19.37) 240.4600 (14.79) 133.1894 (19.07) 8.4420 (8.92) 130.3609 (18.97) 0.9710 (14.71) 619;1486 7,508.1029 (0.05) 7215 1 +test_cramjam_snappy_de_compress_into[html-decompress_into] 6.7130 (1.01) 29.0590 (1.79) 7.3321 (1.05) 1.5681 (1.66) 6.9850 (1.02) 0.2070 (3.14) 878;946 136,386.3798 (0.95) 12593 1 +test_cramjam_snappy_de_compress_into[html_x_4-compress_into] 510.7940 (76.84) 783.9740 (48.21) 525.5117 (75.24) 21.4496 (22.67) 514.8470 (74.91) 20.4052 (309.13) 227;82 1,902.9072 (0.01) 1807 1 +test_cramjam_snappy_de_compress_into[html_x_4-decompress_into] 6.7271 (1.01) 26.0500 (1.60) 7.2545 (1.04) 1.3682 (1.45) 6.9321 (1.01) 0.3700 (5.60) 89;281 137,844.8162 (0.96) 3292 1 +test_cramjam_snappy_de_compress_into[kppkn.gtb-compress_into] 360.5250 (54.24) 599.6680 (36.88) 371.5869 (53.20) 15.3924 (16.26) 363.2680 (52.85) 20.0116 (303.17) 306;42 2,691.1603 (0.02) 2497 1 +test_cramjam_snappy_de_compress_into[kppkn.gtb-decompress_into] 6.7559 (1.02) 28.5150 (1.75) 7.2983 (1.04) 1.3938 (1.47) 7.1101 (1.03) 0.2340 (3.54) 129;254 137,018.7128 (0.96) 4798 1 +test_cramjam_snappy_de_compress_into[lcet10.txt-compress_into] 1,220.3240 (183.59) 1,442.1020 (88.69) 1,252.7170 (179.35) 34.3613 (36.31) 1,244.1790 (181.02) 44.0781 (667.77) 138;22 798.2649 (0.01) 814 1 +test_cramjam_snappy_de_compress_into[lcet10.txt-decompress_into] 6.7160 (1.01) 27.3180 (1.68) 7.2492 (1.04) 1.3898 (1.47) 7.0255 (1.02) 0.3814 (5.78) 60;125 137,946.6539 (0.96) 1868 1 +test_cramjam_snappy_de_compress_into[paper-100k.pdf-compress_into] 39.6080 (5.96) 91.3910 (5.62) 41.3625 (5.92) 3.3869 (3.58) 40.6440 (5.91) 0.9619 (14.57) 1132;1662 24,176.4958 (0.17) 20478 1 +test_cramjam_snappy_de_compress_into[paper-100k.pdf-decompress_into] 6.7310 (1.01) 31.5481 (1.94) 7.1218 (1.02) 1.1942 (1.26) 6.9310 (1.01) 0.1870 (2.83) 440;725 140,414.1376 (0.98) 23261 1 +test_cramjam_snappy_de_compress_into[plrabn12.txt-compress_into] 1,651.1610 (248.40) 2,189.8210 (134.68) 1,689.8730 (241.94) 46.3566 (48.98) 1,677.2315 (244.03) 50.6585 (767.47) 69;14 591.7604 (0.00) 572 1 +test_cramjam_snappy_de_compress_into[plrabn12.txt-decompress_into] 6.6870 (1.01) 36.4761 (2.24) 7.1576 (1.02) 1.5466 (1.63) 6.8745 (1.00) 0.2451 (3.71) 44;53 139,711.1538 (0.98) 1228 1 +test_cramjam_snappy_de_compress_into[urls.10K-compress_into] 1,445.5540 (217.47) 1,788.2879 (109.98) 1,486.7689 (212.86) 41.1075 (43.44) 1,474.6580 (214.56) 52.1065 (789.40) 88;14 672.5995 (0.00) 636 1 +test_cramjam_snappy_de_compress_into[urls.10K-decompress_into] 6.7020 (1.01) 31.9510 (1.97) 7.3564 (1.05) 1.7521 (1.85) 6.8840 (1.00) 0.2775 (4.20) 85;117 135,935.3108 (0.95) 1117 1 +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ``` #### Lz4 @@ -226,34 +198,38 @@ test_cramjam_snappy_de_compress_into[urls.10K-decompress_into] `make bench-lz4` ```bash ----------------------------------------------------------------------------------------------------------- benchmark: 24 tests ---------------------------------------------------------------------------------------------------------- -Name (time in us) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ -test_lz4[Mark.Twain-Tom.Sawyer.txt-cramjam] 265.4710 (1.29) 1,709.2960 (3.48) 283.5229 (1.30) 36.6529 (1.45) 273.4540 (1.29) 20.1355 (1.62) 288;307 3,527.0517 (0.77) 3672 1 -test_lz4[Mark.Twain-Tom.Sawyer.txt-python-lz4] 206.1390 (1.0) 491.0620 (1.0) 218.6584 (1.0) 25.2966 (1.0) 212.6340 (1.0) 12.4433 (1.0) 71;88 4,573.3440 (1.0) 969 1 -test_lz4[alice29.txt-cramjam] 1,591.6270 (7.72) 2,422.1640 (4.93) 1,670.5685 (7.64) 105.3038 (4.16) 1,641.9690 (7.72) 69.4932 (5.58) 47;41 598.5986 (0.13) 583 1 -test_lz4[alice29.txt-python-lz4] 3,014.7940 (14.63) 5,390.9890 (10.98) 3,217.0799 (14.71) 294.7618 (11.65) 3,150.8370 (14.82) 147.8563 (11.88) 20;26 310.8409 (0.07) 315 1 -test_lz4[asyoulik.txt-cramjam] 1,341.9520 (6.51) 2,609.5330 (5.31) 1,435.5756 (6.57) 147.8273 (5.84) 1,403.8710 (6.60) 73.3415 (5.89) 39;52 696.5847 (0.15) 700 1 -test_lz4[asyoulik.txt-python-lz4] 2,624.6560 (12.73) 4,103.2770 (8.36) 2,832.4582 (12.95) 198.1849 (7.83) 2,785.6950 (13.10) 156.1410 (12.55) 42;25 353.0502 (0.08) 351 1 -test_lz4[fireworks.jpeg-cramjam] 279.7980 (1.36) 1,311.0030 (2.67) 312.3522 (1.43) 38.4700 (1.52) 300.8090 (1.41) 26.9838 (2.17) 231;178 3,201.5137 (0.70) 2419 1 -test_lz4[fireworks.jpeg-python-lz4] 2,608.3740 (12.65) 4,068.8360 (8.29) 2,757.1025 (12.61) 187.2245 (7.40) 2,709.8345 (12.74) 98.5295 (7.92) 26;28 362.6996 (0.08) 340 1 -test_lz4[geo.protodata-cramjam] 540.2170 (2.62) 954.5990 (1.94) 576.6946 (2.64) 59.2219 (2.34) 558.5890 (2.63) 40.7170 (3.27) 135;113 1,734.0201 (0.38) 1662 1 -test_lz4[geo.protodata-python-lz4] 670.0600 (3.25) 1,260.5120 (2.57) 706.9607 (3.23) 64.6132 (2.55) 691.1930 (3.25) 34.7968 (2.80) 62;67 1,414.5058 (0.31) 1371 1 -test_lz4[html-cramjam] 557.7070 (2.71) 952.2150 (1.94) 589.8092 (2.70) 40.2866 (1.59) 580.9055 (2.73) 41.6970 (3.35) 131;55 1,695.4636 (0.37) 1478 1 -test_lz4[html-python-lz4] 693.6250 (3.36) 1,243.8680 (2.53) 731.9760 (3.35) 57.6403 (2.28) 717.0350 (3.37) 42.7220 (3.43) 79;68 1,366.1650 (0.30) 1381 1 -test_lz4[html_x_4-cramjam] 1,834.1350 (8.90) 3,089.3180 (6.29) 1,947.9892 (8.91) 130.8457 (5.17) 1,913.9850 (9.00) 93.6955 (7.53) 41;30 513.3499 (0.11) 480 1 -test_lz4[html_x_4-python-lz4] 2,873.1570 (13.94) 4,251.4200 (8.66) 3,030.2486 (13.86) 164.9341 (6.52) 2,984.9600 (14.04) 127.3245 (10.23) 25;20 330.0059 (0.07) 301 1 -test_lz4[kppkn.gtb-cramjam] 1,491.9130 (7.24) 2,781.5860 (5.66) 1,570.2841 (7.18) 87.5343 (3.46) 1,553.1860 (7.30) 72.3630 (5.82) 49;32 636.8274 (0.14) 571 1 -test_lz4[kppkn.gtb-python-lz4] 3,199.8520 (15.52) 4,217.5750 (8.59) 3,359.6494 (15.36) 124.7223 (4.93) 3,325.6840 (15.64) 118.4477 (9.52) 39;21 297.6501 (0.07) 297 1 -test_lz4[lcet10.txt-cramjam] 4,124.9650 (20.01) 5,752.2620 (11.71) 4,313.8181 (19.73) 200.0237 (7.91) 4,261.9530 (20.04) 164.7980 (13.24) 16;9 231.8132 (0.05) 149 1 -test_lz4[lcet10.txt-python-lz4] 8,179.7110 (39.68) 9,336.4010 (19.01) 8,561.2170 (39.15) 252.1781 (9.97) 8,493.2420 (39.94) 365.4150 (29.37) 27;1 116.8058 (0.03) 115 1 -test_lz4[paper-100k.pdf-cramjam] 430.1570 (2.09) 892.4910 (1.82) 460.2170 (2.10) 50.9308 (2.01) 445.1600 (2.09) 32.3620 (2.60) 109;104 2,172.8878 (0.48) 1670 1 -test_lz4[paper-100k.pdf-python-lz4] 1,705.9020 (8.28) 3,126.1760 (6.37) 1,840.7896 (8.42) 179.6683 (7.10) 1,791.6345 (8.43) 88.1440 (7.08) 53;65 543.2451 (0.12) 570 1 -test_lz4[plrabn12.txt-cramjam] 5,020.0420 (24.35) 6,864.6530 (13.98) 5,292.4698 (24.20) 253.9889 (10.04) 5,205.3250 (24.48) 209.6257 (16.85) 18;16 188.9477 (0.04) 161 1 -test_lz4[plrabn12.txt-python-lz4] 11,864.6600 (57.56) 13,558.0060 (27.61) 12,295.0743 (56.23) 312.6359 (12.36) 12,212.9405 (57.44) 337.8520 (27.15) 19;4 81.3334 (0.02) 78 1 -test_lz4[urls.10K-cramjam] 4,636.7460 (22.49) 5,603.8480 (11.41) 4,867.3892 (22.26) 182.9029 (7.23) 4,814.5570 (22.64) 200.3785 (16.10) 27;6 205.4489 (0.04) 125 1 -test_lz4[urls.10K-python-lz4] 9,835.2580 (47.71) 12,131.1310 (24.70) 10,280.7427 (47.02) 417.8376 (16.52) 10,100.7300 (47.50) 393.7080 (31.64) 14;6 97.2692 (0.02) 74 1 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ +------------------------------------------------------------------------------------------------------------------- benchmark: 28 tests ------------------------------------------------------------------------------------------------------------------- +Name (time in us) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +test_lz4[Mark.Twain-Tom.Sawyer.txt-cramjam] 245.3330 (1.0) 7,722.4160 (21.29) 364.4594 (1.05) 240.3840 (33.07) 368.7030 (1.06) 18.0650 (2.54) 3;265 2,743.7899 (0.95) 1155 1 +test_lz4[Mark.Twain-Tom.Sawyer.txt-python-lz4] 335.9900 (1.37) 362.7009 (1.0) 347.2828 (1.0) 7.2700 (1.0) 346.6295 (1.0) 7.1260 (1.0) 2;1 2,879.4977 (1.0) 10 1 +test_lz4[alice29.txt-cramjam] 2,814.1720 (11.47) 4,936.4029 (13.61) 3,046.3559 (8.77) 379.7652 (52.24) 2,920.2300 (8.42) 70.8259 (9.94) 33;46 328.2611 (0.11) 338 1 +test_lz4[alice29.txt-python-lz4] 3,121.6950 (12.72) 5,633.2920 (15.53) 3,441.3794 (9.91) 478.2910 (65.79) 3,252.9815 (9.38) 140.1610 (19.67) 34;50 290.5812 (0.10) 300 1 +test_lz4[asyoulik.txt-cramjam] 2,314.5641 (9.43) 3,103.1220 (8.56) 2,382.4872 (6.86) 92.2261 (12.69) 2,359.6610 (6.81) 74.5750 (10.47) 29;18 419.7294 (0.15) 404 1 +test_lz4[asyoulik.txt-python-lz4] 2,633.2920 (10.73) 4,455.6119 (12.28) 2,722.9006 (7.84) 196.3372 (27.01) 2,682.5680 (7.74) 65.2385 (9.15) 10;21 367.2554 (0.13) 367 1 +test_lz4[fifty-four-mb-random-cramjam] 757,487.6130 (>1000.0) 792,758.5000 (>1000.0) 772,111.8064 (>1000.0) 13,251.3858 (>1000.0) 769,733.5170 (>1000.0) 16,313.7843 (>1000.0) 2;0 1.2951 (0.00) 5 1 +test_lz4[fifty-four-mb-random-python-lz4] 1,597,680.0489 (>1000.0) 1,637,720.9669 (>1000.0) 1,613,587.9904 (>1000.0) 17,438.2782 (>1000.0) 1,606,424.0520 (>1000.0) 29,249.6240 (>1000.0) 1;0 0.6197 (0.00) 5 1 +test_lz4[fifty-four-mb-repeating-cramjam] 222,541.1070 (907.10) 237,449.5791 (654.67) 228,523.9746 (658.03) 6,592.2073 (906.77) 227,023.9190 (654.95) 11,598.6770 (>1000.0) 1;0 4.3759 (0.00) 5 1 +test_lz4[fifty-four-mb-repeating-python-lz4] 161,025.9259 (656.36) 356,819.4229 (983.78) 214,613.3904 (617.98) 73,690.5136 (>1000.0) 186,189.4020 (537.14) 90,134.4944 (>1000.0) 1;0 4.6595 (0.00) 7 1 +test_lz4[fireworks.jpeg-cramjam] 1,556.7919 (6.35) 3,704.2680 (10.21) 1,665.9848 (4.80) 218.8115 (30.10) 1,613.6711 (4.66) 33.3220 (4.68) 26;59 600.2456 (0.21) 569 1 +test_lz4[fireworks.jpeg-python-lz4] 2,682.0981 (10.93) 5,521.3780 (15.22) 2,876.1775 (8.28) 411.3732 (56.59) 2,762.2810 (7.97) 68.5747 (9.62) 22;43 347.6837 (0.12) 365 1 +test_lz4[geo.protodata-cramjam] 744.4300 (3.03) 1,499.9070 (4.14) 799.9747 (2.30) 82.7017 (11.38) 779.4825 (2.25) 27.9890 (3.93) 58;87 1,250.0395 (0.43) 1174 1 +test_lz4[geo.protodata-python-lz4] 694.2170 (2.83) 1,546.8430 (4.26) 722.2650 (2.08) 47.9471 (6.60) 718.2080 (2.07) 22.3070 (3.13) 51;71 1,384.5333 (0.48) 1383 1 +test_lz4[html-cramjam] 778.6801 (3.17) 1,917.2900 (5.29) 832.4302 (2.40) 111.0058 (15.27) 805.6900 (2.32) 13.8602 (1.95) 58;284 1,201.3020 (0.42) 1162 1 +test_lz4[html-python-lz4] 716.7751 (2.92) 1,543.9009 (4.26) 757.5458 (2.18) 105.6609 (14.53) 730.4699 (2.11) 27.0145 (3.79) 63;104 1,320.0522 (0.46) 1340 1 +test_lz4[html_x_4-cramjam] 3,105.4450 (12.66) 4,875.9540 (13.44) 3,301.6318 (9.51) 241.6346 (33.24) 3,248.0890 (9.37) 129.7960 (18.21) 20;21 302.8805 (0.11) 289 1 +test_lz4[html_x_4-python-lz4] 3,016.9110 (12.30) 5,823.1900 (16.06) 3,218.6525 (9.27) 294.3938 (40.49) 3,157.1820 (9.11) 103.9958 (14.59) 20;23 310.6890 (0.11) 255 1 +test_lz4[kppkn.gtb-cramjam] 3,345.9300 (13.64) 5,890.7979 (16.24) 3,538.4708 (10.19) 358.4893 (49.31) 3,434.2399 (9.91) 107.9319 (15.15) 21;28 282.6080 (0.10) 286 1 +test_lz4[kppkn.gtb-python-lz4] 3,323.8300 (13.55) 6,563.6940 (18.10) 3,649.2799 (10.51) 507.6818 (69.83) 3,453.0250 (9.96) 141.2800 (19.83) 26;48 274.0267 (0.10) 274 1 +test_lz4[lcet10.txt-cramjam] 7,444.1610 (30.34) 13,636.0540 (37.60) 8,044.9605 (23.17) 1,071.5044 (147.39) 7,670.1291 (22.13) 302.9506 (42.51) 15;22 124.3014 (0.04) 129 1 +test_lz4[lcet10.txt-python-lz4] 8,787.4030 (35.82) 15,124.7031 (41.70) 9,523.0310 (27.42) 1,039.6363 (143.00) 9,116.1070 (26.30) 404.7097 (56.79) 15;20 105.0086 (0.04) 113 1 +test_lz4[paper-100k.pdf-cramjam] 1,165.2170 (4.75) 3,391.3370 (9.35) 1,238.3142 (3.57) 187.5870 (25.80) 1,197.4330 (3.45) 45.5229 (6.39) 41;60 807.5495 (0.28) 806 1 +test_lz4[paper-100k.pdf-python-lz4] 1,786.4701 (7.28) 3,989.5640 (11.00) 1,930.3730 (5.56) 270.5717 (37.22) 1,852.4720 (5.34) 43.4450 (6.10) 38;75 518.0346 (0.18) 533 1 +test_lz4[plrabn12.txt-cramjam] 10,025.1100 (40.86) 15,823.3760 (43.63) 11,022.9072 (31.74) 1,388.2368 (190.95) 10,650.3405 (30.73) 757.5119 (106.30) 7;7 90.7202 (0.03) 62 1 +test_lz4[plrabn12.txt-python-lz4] 11,853.4301 (48.32) 17,182.6570 (47.37) 13,388.9093 (38.55) 1,159.1757 (159.45) 12,978.2920 (37.44) 1,572.6237 (220.69) 18;1 74.6887 (0.03) 77 1 +test_lz4[urls.10K-cramjam] 8,241.6250 (33.59) 15,744.7390 (43.41) 8,967.4984 (25.82) 1,247.4730 (171.59) 8,477.2279 (24.46) 717.1374 (100.64) 9;9 111.5138 (0.04) 104 1 +test_lz4[urls.10K-python-lz4] 10,320.9730 (42.07) 33,072.6539 (91.18) 14,466.2239 (41.66) 4,628.6794 (636.68) 12,947.5440 (37.35) 4,977.0970 (698.44) 4;4 69.1265 (0.02) 94 1 +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ``` #### Brotli @@ -264,30 +240,30 @@ test_lz4[urls.10K-python-lz4] 9,835.2580 (47.71) 12,131 ----------------------------------------------------------------------------------------------------- benchmark: 24 tests ------------------------------------------------------------------------------------------------------ Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -test_brotli[Mark.Twain-Tom.Sawyer.txt-brotli] 19.3134 (1.05) 27.2349 (1.30) 21.3094 (1.10) 2.6975 (4.93) 19.9170 (1.03) 2.5291 (4.19) 4;2 46.9276 (0.91) 19 1 -test_brotli[Mark.Twain-Tom.Sawyer.txt-cramjam] 18.4340 (1.0) 21.0191 (1.0) 19.3708 (1.0) 0.5474 (1.0) 19.3012 (1.0) 0.6043 (1.0) 19;1 51.6241 (1.0) 49 1 -test_brotli[alice29.txt-brotli] 224.1868 (12.16) 227.8789 (10.84) 225.7909 (11.66) 1.3771 (2.52) 225.6526 (11.69) 1.7266 (2.86) 2;0 4.4289 (0.09) 5 1 -test_brotli[alice29.txt-cramjam] 231.9518 (12.58) 234.0236 (11.13) 233.0551 (12.03) 0.9388 (1.72) 233.5518 (12.10) 1.6091 (2.66) 2;0 4.2908 (0.08) 5 1 -test_brotli[asyoulik.txt-brotli] 179.9550 (9.76) 184.0013 (8.75) 182.1853 (9.41) 1.4951 (2.73) 182.4221 (9.45) 2.1882 (3.62) 2;0 5.4889 (0.11) 6 1 -test_brotli[asyoulik.txt-cramjam] 184.3326 (10.00) 186.8581 (8.89) 185.3278 (9.57) 0.9119 (1.67) 185.2473 (9.60) 1.0866 (1.80) 2;0 5.3958 (0.10) 6 1 -test_brotli[fireworks.jpeg-brotli] 71.5389 (3.88) 75.3505 (3.58) 73.0897 (3.77) 0.9971 (1.82) 72.9035 (3.78) 0.9791 (1.62) 4;1 13.6818 (0.27) 14 1 -test_brotli[fireworks.jpeg-cramjam] 72.5284 (3.93) 79.9106 (3.80) 73.9391 (3.82) 1.8016 (3.29) 73.4397 (3.80) 0.7829 (1.30) 1;1 13.5246 (0.26) 14 1 -test_brotli[geo.protodata-brotli] 126.7453 (6.88) 131.5479 (6.26) 127.8292 (6.60) 1.5408 (2.81) 127.3805 (6.60) 0.6052 (1.00) 1;1 7.8229 (0.15) 8 1 -test_brotli[geo.protodata-cramjam] 130.8282 (7.10) 133.5629 (6.35) 131.9438 (6.81) 0.9135 (1.67) 131.6701 (6.82) 1.2637 (2.09) 3;0 7.5790 (0.15) 8 1 -test_brotli[html-brotli] 134.3089 (7.29) 136.8154 (6.51) 135.5654 (7.00) 0.9767 (1.78) 135.6640 (7.03) 1.7840 (2.95) 4;0 7.3765 (0.14) 8 1 -test_brotli[html-cramjam] 137.2950 (7.45) 141.9890 (6.76) 138.5814 (7.15) 1.6047 (2.93) 137.9258 (7.15) 1.2205 (2.02) 1;1 7.2160 (0.14) 7 1 -test_brotli[html_x_4-brotli] 164.2704 (8.91) 167.2311 (7.96) 165.5355 (8.55) 1.1562 (2.11) 165.5598 (8.58) 2.0072 (3.32) 3;0 6.0410 (0.12) 6 1 -test_brotli[html_x_4-cramjam] 167.8992 (9.11) 172.8526 (8.22) 169.7993 (8.77) 1.6606 (3.03) 169.4014 (8.78) 0.9248 (1.53) 2;1 5.8893 (0.11) 6 1 -test_brotli[kppkn.gtb-brotli] 427.2876 (23.18) 436.4801 (20.77) 431.6189 (22.28) 3.8362 (7.01) 430.7206 (22.32) 6.4873 (10.73) 2;0 2.3169 (0.04) 5 1 -test_brotli[kppkn.gtb-cramjam] 466.2677 (25.29) 497.4289 (23.67) 478.3085 (24.69) 13.1943 (24.10) 475.5884 (24.64) 21.5743 (35.70) 1;0 2.0907 (0.04) 5 1 -test_brotli[lcet10.txt-brotli] 745.3311 (40.43) 848.7560 (40.38) 799.6870 (41.28) 48.8617 (89.26) 809.2787 (41.93) 93.6578 (154.98) 2;0 1.2505 (0.02) 5 1 -test_brotli[lcet10.txt-cramjam] 762.7996 (41.38) 880.4882 (41.89) 798.9608 (41.25) 47.1400 (86.11) 788.5498 (40.85) 44.9762 (74.43) 1;0 1.2516 (0.02) 5 1 -test_brotli[paper-100k.pdf-brotli] 404.2915 (21.93) 409.0611 (19.46) 406.8449 (21.00) 1.9765 (3.61) 407.1359 (21.09) 3.3423 (5.53) 2;0 2.4579 (0.05) 5 1 -test_brotli[paper-100k.pdf-cramjam] 406.3386 (22.04) 410.8830 (19.55) 408.1782 (21.07) 1.8307 (3.34) 407.8163 (21.13) 2.7956 (4.63) 2;0 2.4499 (0.05) 5 1 -test_brotli[plrabn12.txt-brotli] 784.4900 (42.56) 797.3552 (37.93) 789.0102 (40.73) 4.9220 (8.99) 787.9514 (40.82) 4.6725 (7.73) 1;0 1.2674 (0.02) 5 1 -test_brotli[plrabn12.txt-cramjam] 793.1241 (43.02) 845.8696 (40.24) 816.3642 (42.14) 26.2341 (47.92) 802.2609 (41.57) 48.5658 (80.37) 2;0 1.2249 (0.02) 5 1 -test_brotli[urls.10K-brotli] 1,251.5425 (67.89) 1,265.7612 (60.22) 1,257.9783 (64.94) 6.7086 (12.26) 1,256.7268 (65.11) 12.8291 (21.23) 1;0 0.7949 (0.02) 5 1 -test_brotli[urls.10K-cramjam] 1,290.7796 (70.02) 1,365.9914 (64.99) 1,322.5113 (68.27) 33.0967 (60.46) 1,304.2875 (67.58) 54.4744 (90.14) 1;0 0.7561 (0.01) 5 1 +test_brotli[Mark.Twain-Tom.Sawyer.txt-brotli] 17.8897 (1.0) 21.5322 (1.05) 18.4333 (1.0) 0.6208 (1.44) 18.2443 (1.0) 0.4319 (1.0) 5;5 54.2498 (1.0) 50 1 +test_brotli[Mark.Twain-Tom.Sawyer.txt-cramjam] 18.1533 (1.01) 20.4369 (1.0) 18.7759 (1.02) 0.5898 (1.37) 18.6078 (1.02) 0.8347 (1.93) 10;1 53.2598 (0.98) 50 1 +test_brotli[alice29.txt-brotli] 220.3170 (12.32) 230.1574 (11.26) 224.9658 (12.20) 3.8282 (8.88) 223.9629 (12.28) 5.6323 (13.04) 2;0 4.4451 (0.08) 5 1 +test_brotli[alice29.txt-cramjam] 226.4241 (12.66) 228.6188 (11.19) 227.6423 (12.35) 0.9194 (2.13) 227.9714 (12.50) 1.5160 (3.51) 2;0 4.3929 (0.08) 5 1 +test_brotli[asyoulik.txt-brotli] 176.8532 (9.89) 178.5816 (8.74) 177.9033 (9.65) 0.6775 (1.57) 177.9574 (9.75) 0.9254 (2.14) 2;0 5.6210 (0.10) 6 1 +test_brotli[asyoulik.txt-cramjam] 181.3138 (10.14) 185.9610 (9.10) 183.0713 (9.93) 1.8366 (4.26) 182.5439 (10.01) 2.9700 (6.88) 1;0 5.4624 (0.10) 6 1 +test_brotli[fireworks.jpeg-brotli] 70.7069 (3.95) 75.6588 (3.70) 71.9962 (3.91) 1.1901 (2.76) 71.7863 (3.93) 0.6675 (1.55) 2;1 13.8896 (0.26) 14 1 +test_brotli[fireworks.jpeg-cramjam] 71.3536 (3.99) 73.7503 (3.61) 71.8630 (3.90) 0.6269 (1.45) 71.7001 (3.93) 0.5713 (1.32) 1;1 13.9154 (0.26) 14 1 +test_brotli[geo.protodata-brotli] 124.7704 (6.97) 126.1844 (6.17) 125.5772 (6.81) 0.4313 (1.0) 125.5776 (6.88) 0.4719 (1.09) 2;0 7.9632 (0.15) 8 1 +test_brotli[geo.protodata-cramjam] 128.6971 (7.19) 130.4920 (6.39) 129.6153 (7.03) 0.6275 (1.45) 129.5793 (7.10) 1.0149 (2.35) 4;0 7.7151 (0.14) 8 1 +test_brotli[html-brotli] 133.0865 (7.44) 137.1792 (6.71) 134.8985 (7.32) 1.1569 (2.68) 134.9101 (7.39) 0.7996 (1.85) 2;2 7.4130 (0.14) 8 1 +test_brotli[html-cramjam] 136.2971 (7.62) 141.6241 (6.93) 138.3340 (7.50) 2.2464 (5.21) 137.6318 (7.54) 3.8429 (8.90) 2;0 7.2289 (0.13) 8 1 +test_brotli[html_x_4-brotli] 162.2292 (9.07) 164.1336 (8.03) 163.3756 (8.86) 0.7556 (1.75) 163.5865 (8.97) 1.3218 (3.06) 2;0 6.1209 (0.11) 6 1 +test_brotli[html_x_4-cramjam] 166.7431 (9.32) 168.3913 (8.24) 167.3817 (9.08) 0.6918 (1.60) 167.2347 (9.17) 1.1951 (2.77) 1;0 5.9744 (0.11) 6 1 +test_brotli[kppkn.gtb-brotli] 416.0493 (23.26) 420.2574 (20.56) 417.5143 (22.65) 1.6582 (3.84) 417.0724 (22.86) 1.9902 (4.61) 1;0 2.3951 (0.04) 5 1 +test_brotli[kppkn.gtb-cramjam] 432.7377 (24.19) 438.8289 (21.47) 434.3502 (23.56) 2.5329 (5.87) 433.5837 (23.77) 2.0103 (4.65) 1;1 2.3023 (0.04) 5 1 +test_brotli[lcet10.txt-brotli] 689.3127 (38.53) 695.1439 (34.01) 691.5692 (37.52) 2.2512 (5.22) 691.0756 (37.88) 2.8880 (6.69) 2;0 1.4460 (0.03) 5 1 +test_brotli[lcet10.txt-cramjam] 706.3514 (39.48) 730.3781 (35.74) 713.2318 (38.69) 9.7223 (22.54) 709.3515 (38.88) 7.2208 (16.72) 1;1 1.4021 (0.03) 5 1 +test_brotli[paper-100k.pdf-brotli] 397.2979 (22.21) 399.6303 (19.55) 398.6496 (21.63) 0.9994 (2.32) 398.5795 (21.85) 1.6975 (3.93) 1;0 2.5085 (0.05) 5 1 +test_brotli[paper-100k.pdf-cramjam] 397.1881 (22.20) 487.4731 (23.85) 418.1530 (22.68) 38.8914 (90.17) 401.4564 (22.00) 27.9225 (64.65) 1;1 2.3915 (0.04) 5 1 +test_brotli[plrabn12.txt-brotli] 766.4343 (42.84) 965.1578 (47.23) 822.5129 (44.62) 81.3593 (188.62) 785.6867 (43.06) 70.6752 (163.63) 1;1 1.2158 (0.02) 5 1 +test_brotli[plrabn12.txt-cramjam] 776.7893 (43.42) 791.7569 (38.74) 780.8096 (42.36) 6.4344 (14.92) 777.1387 (42.60) 7.2310 (16.74) 1;0 1.2807 (0.02) 5 1 +test_brotli[urls.10K-brotli] 1,232.4364 (68.89) 1,281.6436 (62.71) 1,251.4702 (67.89) 21.9635 (50.92) 1,239.3781 (67.93) 36.2594 (83.95) 1;0 0.7991 (0.01) 5 1 +test_brotli[urls.10K-cramjam] 1,256.7332 (70.25) 1,348.1630 (65.97) 1,284.4820 (69.68) 36.9337 (85.63) 1,274.7599 (69.87) 37.5966 (87.05) 1;0 0.7785 (0.01) 5 1 -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ``` @@ -296,36 +272,36 @@ test_brotli[urls.10K-cramjam] 1,290.7796 (70.02) 1,365.9 `make bench-zstd` ```bash -------------------------------------------------------------------------------------------------------- benchmark: 24 tests ------------------------------------------------------------------------------------------------------- -Name (time in us) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ -test_zstd[Mark.Twain-Tom.Sawyer.txt-cramjam] 154.4260 (1.0) 429.8220 (1.61) 165.7084 (1.0) 18.0132 (1.0) 159.4600 (1.0) 11.8590 (1.0) 211;207 6,034.6953 (1.0) 1685 1 -test_zstd[Mark.Twain-Tom.Sawyer.txt-zstd] 191.9860 (1.24) 267.6270 (1.0) 219.1654 (1.32) 28.9777 (1.61) 197.9910 (1.24) 56.6957 (4.78) 22;0 4,562.7633 (0.76) 53 1 -test_zstd[alice29.txt-cramjam] 1,362.2300 (8.82) 2,102.8580 (7.86) 1,439.4675 (8.69) 73.2922 (4.07) 1,422.7575 (8.92) 70.1800 (5.92) 134;36 694.7014 (0.12) 684 1 -test_zstd[alice29.txt-zstd] 1,315.1700 (8.52) 2,365.9580 (8.84) 1,403.9396 (8.47) 120.7466 (6.70) 1,372.8835 (8.61) 70.7280 (5.96) 72;76 712.2813 (0.12) 744 1 -test_zstd[asyoulik.txt-cramjam] 1,211.2850 (7.84) 1,841.2200 (6.88) 1,284.1431 (7.75) 77.5367 (4.30) 1,270.1210 (7.97) 59.2710 (5.00) 78;53 778.7294 (0.13) 757 1 -test_zstd[asyoulik.txt-zstd] 1,157.6700 (7.50) 1,653.1200 (6.18) 1,220.0159 (7.36) 65.1603 (3.62) 1,204.0080 (7.55) 55.9547 (4.72) 92;49 819.6615 (0.14) 805 1 -test_zstd[fireworks.jpeg-cramjam] 292.3470 (1.89) 602.9980 (2.25) 313.0014 (1.89) 33.5971 (1.87) 302.4510 (1.90) 20.7312 (1.75) 287;295 3,194.8734 (0.53) 3057 1 -test_zstd[fireworks.jpeg-zstd] 216.5420 (1.40) 452.4620 (1.69) 229.1745 (1.38) 22.7349 (1.26) 222.9190 (1.40) 12.9210 (1.09) 378;420 4,363.4868 (0.72) 3623 1 -test_zstd[geo.protodata-cramjam] 324.0560 (2.10) 672.1640 (2.51) 354.0909 (2.14) 42.7156 (2.37) 340.2680 (2.13) 27.8100 (2.35) 216;209 2,824.1338 (0.47) 2621 1 -test_zstd[geo.protodata-zstd] 281.7450 (1.82) 535.1640 (2.00) 300.6179 (1.81) 29.3579 (1.63) 291.3200 (1.83) 18.6490 (1.57) 247;242 3,326.4816 (0.55) 2894 1 -test_zstd[html-cramjam] 349.8810 (2.27) 769.5230 (2.88) 376.6673 (2.27) 43.1311 (2.39) 362.2320 (2.27) 27.6060 (2.33) 188;186 2,654.8631 (0.44) 2332 1 -test_zstd[html-zstd] 312.4120 (2.02) 580.8570 (2.17) 333.4397 (2.01) 32.0637 (1.78) 323.5910 (2.03) 21.6778 (1.83) 253;222 2,999.0433 (0.50) 2851 1 -test_zstd[html_x_4-cramjam] 476.8870 (3.09) 887.5280 (3.32) 537.3586 (3.24) 76.9432 (4.27) 509.1320 (3.19) 46.7175 (3.94) 115;122 1,860.9548 (0.31) 909 1 -test_zstd[html_x_4-zstd] 364.0010 (2.36) 797.9640 (2.98) 389.8506 (2.35) 37.9180 (2.11) 377.5145 (2.37) 28.5240 (2.41) 215;177 2,565.0855 (0.43) 2332 1 -test_zstd[kppkn.gtb-cramjam] 1,082.0570 (7.01) 1,696.9090 (6.34) 1,148.6073 (6.93) 72.4650 (4.02) 1,131.7090 (7.10) 57.8217 (4.88) 85;62 870.6196 (0.14) 749 1 -test_zstd[kppkn.gtb-zstd] 1,043.9460 (6.76) 1,771.5120 (6.62) 1,110.4534 (6.70) 81.5983 (4.53) 1,090.0220 (6.84) 54.9060 (4.63) 95;82 900.5331 (0.15) 879 1 -test_zstd[lcet10.txt-cramjam] 3,509.2670 (22.72) 5,409.0240 (20.21) 3,734.3465 (22.54) 230.2715 (12.78) 3,684.9145 (23.11) 163.8595 (13.82) 19;14 267.7845 (0.04) 268 1 -test_zstd[lcet10.txt-zstd] 3,387.3740 (21.94) 5,608.0490 (20.95) 3,595.1312 (21.70) 246.2549 (13.67) 3,542.5960 (22.22) 180.9382 (15.26) 19;12 278.1540 (0.05) 283 1 -test_zstd[paper-100k.pdf-cramjam] 488.0500 (3.16) 942.0480 (3.52) 520.3597 (3.14) 44.4527 (2.47) 508.0725 (3.19) 36.1535 (3.05) 166;116 1,921.7476 (0.32) 1900 1 -test_zstd[paper-100k.pdf-zstd] 479.8510 (3.11) 868.9010 (3.25) 520.1514 (3.14) 46.2761 (2.57) 509.5520 (3.20) 28.6118 (2.41) 197;190 1,922.5171 (0.32) 1977 1 -test_zstd[plrabn12.txt-cramjam] 4,742.2590 (30.71) 6,472.1400 (24.18) 5,019.8933 (30.29) 234.2066 (13.00) 4,958.0030 (31.09) 198.3953 (16.73) 26;12 199.2074 (0.03) 167 1 -test_zstd[plrabn12.txt-zstd] 4,586.0060 (29.70) 6,362.5020 (23.77) 4,846.4730 (29.25) 238.6297 (13.25) 4,803.3360 (30.12) 206.9840 (17.45) 26;9 206.3356 (0.03) 203 1 -test_zstd[urls.10K-cramjam] 4,455.0140 (28.85) 8,134.9550 (30.40) 4,870.4095 (29.39) 491.9910 (27.31) 4,727.1960 (29.65) 212.0393 (17.88) 23;28 205.3215 (0.03) 201 1 -test_zstd[urls.10K-zstd] 4,177.0250 (27.05) 5,455.3670 (20.38) 4,437.3242 (26.78) 210.9385 (11.71) 4,382.1610 (27.48) 231.2377 (19.50) 44;11 225.3610 (0.04) 227 1 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ +------------------------------------------------------------------------------------------------------------- benchmark: 28 tests ------------------------------------------------------------------------------------------------------------- +Name (time in us) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +test_zstd[Mark.Twain-Tom.Sawyer.txt-cramjam] 162.9620 (1.09) 406.7180 (1.0) 176.6457 (1.0) 18.1423 (1.14) 169.1270 (1.0) 10.4961 (30.47) 201;206 5,661.0504 (1.0) 1524 1 +test_zstd[Mark.Twain-Tom.Sawyer.txt-zstd] 150.1160 (1.0) 476.3250 (1.17) 254.5834 (1.44) 73.5374 (4.64) 286.3150 (1.69) 111.8249 (324.63) 15;1 3,927.9858 (0.69) 50 1 +test_zstd[alice29.txt-cramjam] 1,375.7630 (9.16) 1,755.1420 (4.32) 1,409.3451 (7.98) 44.2825 (2.79) 1,399.8351 (8.28) 39.0127 (113.25) 63;29 709.5494 (0.13) 675 1 +test_zstd[alice29.txt-zstd] 1,325.4730 (8.83) 2,010.5341 (4.94) 1,360.4852 (7.70) 62.9252 (3.97) 1,345.8120 (7.96) 39.4245 (114.45) 35;29 735.0319 (0.13) 691 1 +test_zstd[asyoulik.txt-cramjam] 1,215.3659 (8.10) 1,828.1790 (4.49) 1,250.6696 (7.08) 57.2588 (3.61) 1,238.3070 (7.32) 38.2224 (110.96) 53;47 799.5717 (0.14) 759 1 +test_zstd[asyoulik.txt-zstd] 1,159.6669 (7.73) 1,858.6870 (4.57) 1,189.6754 (6.73) 54.2058 (3.42) 1,175.6300 (6.95) 34.5710 (100.36) 54;39 840.5654 (0.15) 816 1 +test_zstd[fifty-four-mb-random-cramjam] 126,119.5041 (840.15) 127,714.4219 (314.01) 126,973.2383 (718.80) 588.7249 (37.11) 127,016.0485 (751.01) 922.4786 (>1000.0) 4;0 7.8757 (0.00) 8 1 +test_zstd[fifty-four-mb-random-zstd] 54,271.3830 (361.53) 67,254.4950 (165.36) 62,470.3614 (353.65) 3,707.2937 (233.68) 63,691.2219 (376.59) 4,924.4625 (>1000.0) 4;0 16.0076 (0.00) 16 1 +test_zstd[fifty-four-mb-repeating-cramjam] 49,074.5769 (326.91) 50,540.2781 (124.26) 49,350.9156 (279.38) 374.5157 (23.61) 49,230.9110 (291.09) 269.6409 (782.76) 1;1 20.2630 (0.00) 14 1 +test_zstd[fifty-four-mb-repeating-zstd] 27,456.1089 (182.90) 30,666.1710 (75.40) 28,946.7121 (163.87) 620.6539 (39.12) 29,004.8160 (171.50) 532.4374 (>1000.0) 8;4 34.5462 (0.01) 33 1 +test_zstd[fireworks.jpeg-cramjam] 287.1940 (1.91) 503.2960 (1.24) 296.2061 (1.68) 19.8144 (1.25) 288.1080 (1.70) 8.4531 (24.54) 228;298 3,376.0276 (0.60) 2354 1 +test_zstd[fireworks.jpeg-zstd] 216.6729 (1.44) 594.6030 (1.46) 221.7168 (1.26) 16.1432 (1.02) 217.1519 (1.28) 0.3445 (1.0) 280;881 4,510.2582 (0.80) 4208 1 +test_zstd[geo.protodata-cramjam] 327.6070 (2.18) 517.6039 (1.27) 338.9284 (1.92) 17.7418 (1.12) 332.8320 (1.97) 8.2538 (23.96) 259;399 2,950.4755 (0.52) 2635 1 +test_zstd[geo.protodata-zstd] 280.3099 (1.87) 669.7009 (1.65) 289.7946 (1.64) 15.8649 (1.0) 285.2400 (1.69) 2.6374 (7.66) 288;645 3,450.7199 (0.61) 3200 1 +test_zstd[html-cramjam] 356.0591 (2.37) 772.1799 (1.90) 366.8109 (2.08) 23.6813 (1.49) 359.2240 (2.12) 5.6227 (16.32) 175;413 2,726.2002 (0.48) 2443 1 +test_zstd[html-zstd] 312.1750 (2.08) 533.3930 (1.31) 321.4469 (1.82) 19.8513 (1.25) 314.5090 (1.86) 3.7401 (10.86) 246;582 3,110.9334 (0.55) 2934 1 +test_zstd[html_x_4-cramjam] 476.6970 (3.18) 719.8390 (1.77) 490.4110 (2.78) 21.8491 (1.38) 481.2561 (2.85) 15.6217 (45.35) 192;152 2,039.1062 (0.36) 1797 1 +test_zstd[html_x_4-zstd] 364.6320 (2.43) 644.5120 (1.58) 374.6641 (2.12) 19.1959 (1.21) 367.3399 (2.17) 5.6954 (16.53) 231;447 2,669.0574 (0.47) 2477 1 +test_zstd[kppkn.gtb-cramjam] 1,093.0240 (7.28) 1,506.8919 (3.71) 1,131.5039 (6.41) 49.7900 (3.14) 1,117.8980 (6.61) 44.4521 (129.04) 92;52 883.7795 (0.16) 843 1 +test_zstd[kppkn.gtb-zstd] 1,044.6580 (6.96) 1,529.9510 (3.76) 1,069.5185 (6.05) 38.3409 (2.42) 1,053.7340 (6.23) 28.2210 (81.93) 77;49 935.0002 (0.17) 896 1 +test_zstd[lcet10.txt-cramjam] 3,546.4830 (23.62) 4,082.9000 (10.04) 3,626.4003 (20.53) 78.0370 (4.92) 3,605.1755 (21.32) 78.6935 (228.45) 31;11 275.7555 (0.05) 260 1 +test_zstd[lcet10.txt-zstd] 3,407.8470 (22.70) 4,383.9460 (10.78) 3,496.2894 (19.79) 105.5919 (6.66) 3,474.2125 (20.54) 70.1590 (203.67) 22;22 286.0175 (0.05) 286 1 +test_zstd[paper-100k.pdf-cramjam] 494.6779 (3.30) 958.8940 (2.36) 510.4041 (2.89) 32.5814 (2.05) 499.8320 (2.96) 13.1470 (38.17) 95;186 1,959.2320 (0.35) 1821 1 +test_zstd[paper-100k.pdf-zstd] 480.3210 (3.20) 801.3280 (1.97) 493.8572 (2.80) 26.0754 (1.64) 484.5071 (2.86) 14.3057 (41.53) 133;137 2,024.8767 (0.36) 1655 1 +test_zstd[plrabn12.txt-cramjam] 4,773.0900 (31.80) 5,415.0301 (13.31) 4,930.6590 (27.91) 134.2914 (8.46) 4,886.1710 (28.89) 140.8120 (408.78) 38;10 202.8126 (0.04) 165 1 +test_zstd[plrabn12.txt-zstd] 4,617.4621 (30.76) 5,391.8620 (13.26) 4,735.0956 (26.81) 101.1250 (6.37) 4,712.4890 (27.86) 77.8873 (226.11) 28;11 211.1890 (0.04) 203 1 +test_zstd[urls.10K-cramjam] 4,454.6180 (29.67) 6,157.7980 (15.14) 4,578.2559 (25.92) 153.7447 (9.69) 4,547.1541 (26.89) 93.7060 (272.03) 12;11 218.4238 (0.04) 194 1 +test_zstd[urls.10K-zstd] 4,201.4410 (27.99) 4,969.7730 (12.22) 4,298.6442 (24.33) 103.9367 (6.55) 4,271.2825 (25.25) 69.8585 (202.80) 18;13 232.6315 (0.04) 228 1 +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ``` - - - - diff --git a/benchmarks/test_bench.py b/benchmarks/test_bench.py index 9cfae1ee..2151afd0 100644 --- a/benchmarks/test_bench.py +++ b/benchmarks/test_bench.py @@ -3,7 +3,7 @@ import cramjam import pathlib import numpy as np - +from memory_profiler import profile FILES = [ f @@ -51,7 +51,7 @@ def test_snappy_raw(benchmark, file, use_cramjam: bool): """ import snappy - data = bytearray(file.read_bytes()) + data = file.read_bytes() if use_cramjam: benchmark( round_trip, @@ -97,7 +97,7 @@ def test_snappy_framed(benchmark, file, use_cramjam: bool): ) -@pytest.mark.parametrize("op", ("decompress_into", "compress_into")) +@pytest.mark.parametrize("op", ("decompress_raw_into", "compress_raw_into")) @pytest.mark.parametrize("file", FILES, ids=lambda val: val.name) def test_cramjam_snappy_de_compress_into(benchmark, op, file): """ @@ -123,33 +123,17 @@ def test_cramjam_snappy_de_compress_into(benchmark, op, file): @pytest.mark.parametrize( "use_cramjam", (True, False), ids=lambda val: "cramjam" if val else "gzip" ) -@pytest.mark.parametrize( - "set_output_len", (True, False), ids=lambda val: f"used-output_len={val}" -) @pytest.mark.parametrize("file", FILES, ids=lambda val: val.name) -def test_gzip(benchmark, file, use_cramjam: bool, set_output_len: bool): +def test_gzip(benchmark, file, use_cramjam: bool): data = file.read_bytes() if use_cramjam: - if set_output_len: - compressed_len = len(cramjam.gzip.compress(data)) - benchmark( - round_trip, - compress=lambda bytes: cramjam.gzip.compress( - bytes, level=9, output_len=compressed_len - ), - decompress=lambda bytes: cramjam.gzip.decompress( - bytes, output_len=len(data) - ), - data=data, - ) - else: - benchmark( - round_trip, - compress=cramjam.gzip.compress, - decompress=cramjam.gzip.decompress, - data=data, - level=9, - ) + benchmark( + round_trip, + compress=cramjam.gzip.compress, + decompress=cramjam.gzip.decompress, + data=data, + level=9, + ) else: benchmark( round_trip, @@ -189,7 +173,15 @@ def test_lz4(benchmark, file, use_cramjam: bool): @pytest.mark.parametrize( "use_cramjam", (True, False), ids=lambda val: "cramjam" if val else "brotli" ) -@pytest.mark.parametrize("file", FILES, ids=lambda val: val.name) +@pytest.mark.parametrize( + "file", + [ + f + for f in FILES + if not (isinstance(f, FiftyFourMbRandom) or isinstance(f, FiftyFourMbRepeating)) + ], + ids=lambda val: val.name, +) def test_brotli(benchmark, file, use_cramjam: bool): import brotli @@ -229,3 +221,19 @@ def test_zstd(benchmark, file, use_cramjam: bool): benchmark( round_trip, compress=zstd.compress, decompress=zstd.decompress, data=data, ) + + +@profile +def memory_profile(): + + import snappy + + data = bytearray(FILES[-1].read_bytes()) + out1 = bytes(cramjam.snappy.compress_raw(data)) + _out1 = bytes(cramjam.snappy.decompress_raw(out1)) + out2 = snappy.compress(data) + _ou2 = snappy.decompress(out2) + + +if __name__ == "__main__": + memory_profile() diff --git a/pypy_patch.py b/pypy_patch.py index 6e8ca356..144935e6 100644 --- a/pypy_patch.py +++ b/pypy_patch.py @@ -21,6 +21,7 @@ for file in pathlib.Path("./dist").iterdir(): if file.name.endswith(".whl"): new_name = regex.sub(f"pp{major}{minor}-{abi}", file.name) - new_name = new_name.replace("linux", "manylinux2010") + if "manylinux" not in new_name and "linux" in new_name: + new_name = new_name.replace("linux", "manylinux2010") print(f"Renaming {file.name} -> {new_name}") file.rename(file.parent.joinpath(new_name)) diff --git a/src/brotli.rs b/src/brotli.rs index 53186d07..e6e06ee0 100644 --- a/src/brotli.rs +++ b/src/brotli.rs @@ -1,10 +1,10 @@ //! brotli de/compression interface use crate::exceptions::{CompressionError, DecompressionError}; +use crate::io::RustyBuffer; use crate::{to_py_err, BytesType}; use pyo3::prelude::*; -use pyo3::types::PyBytes; use pyo3::wrap_pyfunction; -use pyo3::{PyResult, Python}; +use pyo3::PyResult; use std::io::Cursor; pub(crate) fn init_py_module(m: &PyModule) -> PyResult<()> { @@ -23,8 +23,8 @@ pub(crate) fn init_py_module(m: &PyModule) -> PyResult<()> { /// >>> cramjam.brotli.decompress(compressed_bytes, output_len=Optional[int]) /// ``` #[pyfunction] -pub fn decompress<'a>(py: Python<'a>, data: BytesType<'a>, output_len: Option) -> PyResult> { - crate::generic!(decompress(data), py = py, output_len = output_len) +pub fn decompress(data: BytesType, output_len: Option) -> PyResult { + crate::generic!(decompress(data), output_len = output_len) } /// Brotli compression. @@ -35,30 +35,20 @@ pub fn decompress<'a>(py: Python<'a>, data: BytesType<'a>, output_len: Option>> cramjam.brotli.compress(b'some bytes here', level=9, output_len=Option[int]) # level defaults to 11 /// ``` #[pyfunction] -pub fn compress<'a>( - py: Python<'a>, - data: BytesType<'a>, - level: Option, - output_len: Option, -) -> PyResult> { - crate::generic!(compress(data), py = py, output_len = output_len, level = level) +pub fn compress(data: BytesType, level: Option, output_len: Option) -> PyResult { + crate::generic!(compress(data), output_len = output_len, level = level) } /// Compress directly into an output buffer #[pyfunction] -pub fn compress_into<'a>( - _py: Python<'a>, - input: BytesType<'a>, - mut output: BytesType<'a>, - level: Option, -) -> PyResult { +pub fn compress_into(input: BytesType, mut output: BytesType, level: Option) -> PyResult { let r = internal::compress(input, &mut output, level)?; Ok(r) } /// Decompress directly into an output buffer #[pyfunction] -pub fn decompress_into<'a>(_py: Python<'a>, input: BytesType<'a>, mut output: BytesType<'a>) -> PyResult { +pub fn decompress_into(input: BytesType, mut output: BytesType) -> PyResult { let r = internal::decompress(input, &mut output)?; Ok(r) } diff --git a/src/deflate.rs b/src/deflate.rs index 6484b6fa..c8e7d4e0 100644 --- a/src/deflate.rs +++ b/src/deflate.rs @@ -1,10 +1,10 @@ //! deflate de/compression interface use crate::exceptions::{CompressionError, DecompressionError}; +use crate::io::RustyBuffer; use crate::{to_py_err, BytesType}; use pyo3::prelude::*; -use pyo3::types::PyBytes; use pyo3::wrap_pyfunction; -use pyo3::{PyResult, Python}; +use pyo3::PyResult; use std::io::Cursor; pub(crate) fn init_py_module(m: &PyModule) -> PyResult<()> { @@ -23,8 +23,8 @@ pub(crate) fn init_py_module(m: &PyModule) -> PyResult<()> { /// >>> cramjam.deflate.decompress(compressed_bytes, output_len=Optional[int]) /// ``` #[pyfunction] -pub fn decompress<'a>(py: Python<'a>, data: BytesType<'a>, output_len: Option) -> PyResult> { - crate::generic!(decompress(data), py = py, output_len = output_len) +pub fn decompress(data: BytesType, output_len: Option) -> PyResult { + crate::generic!(decompress(data), output_len = output_len) } /// Deflate compression. @@ -35,30 +35,20 @@ pub fn decompress<'a>(py: Python<'a>, data: BytesType<'a>, output_len: Option>> cramjam.deflate.compress(b'some bytes here', level=5, output_len=Optional[int]) # level defaults to 6 /// ``` #[pyfunction] -pub fn compress<'a>( - py: Python<'a>, - data: BytesType<'a>, - level: Option, - output_len: Option, -) -> PyResult> { - crate::generic!(compress(data), py = py, output_len = output_len, level = level) +pub fn compress(data: BytesType, level: Option, output_len: Option) -> PyResult { + crate::generic!(compress(data), output_len = output_len, level = level) } /// Compress directly into an output buffer #[pyfunction] -pub fn compress_into<'a>( - _py: Python<'a>, - input: BytesType<'a>, - mut output: BytesType<'a>, - level: Option, -) -> PyResult { +pub fn compress_into(input: BytesType, mut output: BytesType, level: Option) -> PyResult { let r = internal::compress(input, &mut output, level)?; Ok(r) } /// Decompress directly into an output buffer #[pyfunction] -pub fn decompress_into<'a>(_py: Python<'a>, input: BytesType<'a>, mut output: BytesType<'a>) -> PyResult { +pub fn decompress_into(input: BytesType, mut output: BytesType) -> PyResult { let r = internal::decompress(input, &mut output)?; Ok(r) } diff --git a/src/exceptions.rs b/src/exceptions.rs index f9870d3b..ab30d4e2 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -5,3 +5,4 @@ use pyo3::exceptions::PyException; create_exception!(cramjam, CompressionError, PyException); create_exception!(cramjam, DecompressionError, PyException); +create_exception!(cramjam, CramjamError, PyException); diff --git a/src/gzip.rs b/src/gzip.rs index db951cbe..ebf416fb 100644 --- a/src/gzip.rs +++ b/src/gzip.rs @@ -1,10 +1,10 @@ //! gzip de/compression interface use crate::exceptions::{CompressionError, DecompressionError}; +use crate::io::RustyBuffer; use crate::{to_py_err, BytesType}; use pyo3::prelude::*; -use pyo3::types::PyBytes; use pyo3::wrap_pyfunction; -use pyo3::{PyResult, Python}; +use pyo3::PyResult; use std::io::Cursor; pub(crate) fn init_py_module(m: &PyModule) -> PyResult<()> { @@ -23,8 +23,8 @@ pub(crate) fn init_py_module(m: &PyModule) -> PyResult<()> { /// >>> cramjam.gzip.decompress(compressed_bytes, output_len=Optional[int]) /// ``` #[pyfunction] -pub fn decompress<'a>(py: Python<'a>, data: BytesType<'a>, output_len: Option) -> PyResult> { - crate::generic!(decompress(data), py = py, output_len = output_len) +pub fn decompress(data: BytesType, output_len: Option) -> PyResult { + crate::generic!(decompress(data), output_len = output_len) } /// Gzip compression. @@ -35,30 +35,20 @@ pub fn decompress<'a>(py: Python<'a>, data: BytesType<'a>, output_len: Option>> cramjam.gzip.compress(b'some bytes here', level=2, output_len=Optional[int]) # Level defaults to 6 /// ``` #[pyfunction] -pub fn compress<'a>( - py: Python<'a>, - data: BytesType<'a>, - level: Option, - output_len: Option, -) -> PyResult> { - crate::generic!(compress(data), py = py, output_len = output_len, level = level) +pub fn compress(data: BytesType, level: Option, output_len: Option) -> PyResult { + crate::generic!(compress(data), output_len = output_len, level = level) } /// Compress directly into an output buffer #[pyfunction] -pub fn compress_into<'a>( - _py: Python<'a>, - input: BytesType<'a>, - mut output: BytesType<'a>, - level: Option, -) -> PyResult { +pub fn compress_into(input: BytesType, mut output: BytesType, level: Option) -> PyResult { let r = internal::compress(input, &mut output, level)?; Ok(r) } /// Decompress directly into an output buffer #[pyfunction] -pub fn decompress_into<'a>(_py: Python<'a>, input: BytesType<'a>, mut output: BytesType<'a>) -> PyResult { +pub fn decompress_into(input: BytesType, mut output: BytesType) -> PyResult { let r = internal::decompress(input, &mut output)?; Ok(r) } diff --git a/src/io.rs b/src/io.rs index 23398148..511a421c 100644 --- a/src/io.rs +++ b/src/io.rs @@ -6,23 +6,34 @@ use std::fs::{File, OpenOptions}; use std::io::{copy, Cursor, Read, Seek, SeekFrom, Write}; use crate::BytesType; +use crate::exceptions::CramjamError; use numpy::PyArray1; +use pyo3::class::buffer::PyBufferProtocol; use pyo3::prelude::*; use pyo3::types::{PyByteArray, PyBytes}; +use pyo3::{ffi, PySequenceProtocol}; +use pyo3::{AsPyPointer, PyObjectProtocol}; +use std::path::PathBuf; + +pub(crate) trait AsBytes { + fn as_bytes(&self) -> &[u8]; + fn as_bytes_mut(&mut self) -> &mut [u8]; +} /// Internal wrapper for `numpy.array`/`PyArray1`, to provide Read + Write and other traits pub struct RustyNumpyArray<'a> { pub(crate) inner: &'a PyArray1, pub(crate) cursor: Cursor<&'a mut [u8]>, } -impl<'a> RustyNumpyArray<'a> { - pub(crate) fn from_vec(py: Python<'a>, v: Vec) -> Self { - let inner = PyArray1::from_vec(py, v); - Self { - inner, - cursor: Cursor::new(unsafe { inner.as_slice_mut().unwrap() }), - } +impl<'a> AsBytes for RustyNumpyArray<'a> { + fn as_bytes(&self) -> &[u8] { + self.cursor.get_ref().as_ref() } + fn as_bytes_mut(&mut self) -> &mut [u8] { + self.cursor.get_mut() + } +} +impl<'a> RustyNumpyArray<'a> { pub(crate) fn as_bytes(&self) -> &[u8] { unsafe { self.inner.as_slice().unwrap() } } @@ -71,10 +82,13 @@ pub struct RustyPyBytes<'a> { pub(crate) inner: &'a PyBytes, pub(crate) cursor: Cursor<&'a mut [u8]>, } -impl<'a> RustyPyBytes<'a> { - pub(crate) fn as_bytes(&self) -> &[u8] { +impl<'a> AsBytes for RustyPyBytes<'a> { + fn as_bytes(&self) -> &[u8] { self.inner.as_bytes() } + fn as_bytes_mut(&mut self) -> &mut [u8] { + self.cursor.get_mut() + } } impl<'a> From<&'a PyBytes> for RustyPyBytes<'a> { fn from(inner: &'a PyBytes) -> Self { @@ -120,16 +134,12 @@ pub struct RustyPyByteArray<'a> { pub(crate) inner: &'a PyByteArray, pub(crate) cursor: Cursor<&'a mut [u8]>, } -impl<'a> RustyPyByteArray<'a> { - pub(crate) fn new(py: Python<'a>, len: usize) -> Self { - let inner = PyByteArray::new_with(py, len, |_| Ok(())).unwrap(); - Self { - inner, - cursor: Cursor::new(unsafe { inner.as_bytes_mut() }), - } +impl<'a> AsBytes for RustyPyByteArray<'a> { + fn as_bytes(&self) -> &[u8] { + self.cursor.get_ref() } - pub(crate) fn as_bytes(&self) -> &[u8] { - unsafe { self.inner.as_bytes() } + fn as_bytes_mut(&mut self) -> &mut [u8] { + self.cursor.get_mut() } } impl<'a> From<&'a PyByteArray> for RustyPyByteArray<'a> { @@ -200,9 +210,47 @@ impl<'a> Seek for RustyPyByteArray<'a> { /// #[pyclass(name = "File")] pub struct RustyFile { + pub(crate) path: PathBuf, pub(crate) inner: File, } +#[pyproto] +impl PyObjectProtocol for RustyFile { + fn __repr__(&self) -> PyResult { + let path = match self.path.as_path().to_str() { + Some(path) => path.to_string(), + None => self.path.to_string_lossy().to_string(), + }; + let repr = format!("cramjam.File(path={}, len={:?})", path, self.len()?); + Ok(repr) + } + fn __bool__(&self) -> PyResult { + Ok(self.len()? > 0) + } +} + +#[pyproto] +impl PySequenceProtocol for RustyFile { + fn __len__(&self) -> PyResult { + self.len() + } +} + +impl AsBytes for RustyFile { + fn as_bytes(&self) -> &[u8] { + unimplemented!( + "Converting a File to bytes is not supported, as it'd require reading the \ + entire file into memory; consider using cramjam.Buffer" + ) + } + fn as_bytes_mut(&mut self) -> &mut [u8] { + unimplemented!( + "Converting a File to bytes is not supported, as it'd require reading the \ + entire file into memory; consider using cramjam.Buffer" + ) + } +} + #[pymethods] impl RustyFile { /// ### Example @@ -223,6 +271,7 @@ impl RustyFile { append: Option, ) -> PyResult { Ok(Self { + path: PathBuf::from(path), inner: OpenOptions::new() .read(read.unwrap_or_else(|| true)) .write(write.unwrap_or_else(|| true)) @@ -287,6 +336,14 @@ impl RustyFile { pub fn truncate(&mut self) -> PyResult<()> { self.set_len(0) } + /// Length of the file in bytes + pub fn len(&self) -> PyResult { + let meta = self + .inner + .metadata() + .map_err(|e| CramjamError::new_err(e.to_string()))?; + Ok(meta.len() as usize) + } } /// A native Rust file-like object. Reading and writing takes place @@ -307,6 +364,90 @@ pub struct RustyBuffer { pub(crate) inner: Cursor>, } +impl AsBytes for RustyBuffer { + fn as_bytes(&self) -> &[u8] { + self.inner.get_ref().as_slice() + } + fn as_bytes_mut(&mut self) -> &mut [u8] { + self.inner.get_mut().as_mut_slice() + } +} + +impl From> for RustyBuffer { + fn from(v: Vec) -> Self { + Self { inner: Cursor::new(v) } + } +} + +#[pyproto] +impl PyBufferProtocol for RustyBuffer { + fn bf_getbuffer(slf: PyRefMut, view: *mut ffi::Py_buffer, flags: std::os::raw::c_int) -> PyResult<()> { + if view.is_null() { + return Err(pyo3::exceptions::PyBufferError::new_err("View is null")); + } + + if (flags & ffi::PyBUF_WRITABLE) == ffi::PyBUF_WRITABLE { + return Err(pyo3::exceptions::PyBufferError::new_err("Object is not writable")); + } + + unsafe { + (*view).obj = slf.as_ptr(); + ffi::Py_INCREF((*view).obj); + } + + let bytes = slf.inner.get_ref().as_slice(); + + unsafe { + (*view).buf = bytes.as_ptr() as *mut std::os::raw::c_void; + (*view).len = bytes.len() as isize; + (*view).readonly = 1; + (*view).itemsize = 1; + + (*view).format = std::ptr::null_mut(); + if (flags & ffi::PyBUF_FORMAT) == ffi::PyBUF_FORMAT { + let msg = std::ffi::CStr::from_bytes_with_nul(b"B\0").unwrap(); + (*view).format = msg.as_ptr() as *mut _; + } + + (*view).ndim = 1; + (*view).shape = std::ptr::null_mut(); + if (flags & ffi::PyBUF_ND) == ffi::PyBUF_ND { + (*view).shape = (&((*view).len)) as *const _ as *mut _; + } + + (*view).strides = std::ptr::null_mut(); + if (flags & ffi::PyBUF_STRIDES) == ffi::PyBUF_STRIDES { + (*view).strides = &((*view).itemsize) as *const _ as *mut _; + } + + (*view).suboffsets = std::ptr::null_mut(); + (*view).internal = std::ptr::null_mut(); + } + Ok(()) + } + fn bf_releasebuffer(_slf: PyRefMut, _view: *mut ffi::Py_buffer) {} +} + +#[pyproto] +impl PySequenceProtocol for RustyBuffer { + fn __len__(&self) -> usize { + self.len() + } + fn __contains__(&self, x: u8) -> bool { + self.inner.get_ref().contains(&x) + } +} + +#[pyproto] +impl PyObjectProtocol for RustyBuffer { + fn __repr__(&self) -> String { + format!("cramjam.Buffer(len={:?})", self.len()) + } + fn __bool__(&self) -> bool { + self.len() > 0 + } +} + /// A Buffer object, similar to [cramjam.File](struct.RustyFile.html) only the bytes are held in-memory /// /// ### Example @@ -329,6 +470,12 @@ impl RustyBuffer { inner: Cursor::new(buf), }) } + + /// Length of the underlying buffer + pub fn len(&self) -> usize { + self.inner.get_ref().len() + } + /// Write some bytes to the buffer, where input data can be anything in [BytesType](../enum.BytesType.html) pub fn write(&mut self, mut input: BytesType) -> PyResult { let r = write(&mut input, self)?; diff --git a/src/lib.rs b/src/lib.rs index 1a2c5fda..1304cc79 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,7 +18,7 @@ //! >>> data = b'some bytes here' //! >>> compressed = cramjam.snappy.compress(data) //! >>> decompressed = cramjam.snappy.decompress(compressed) -//! >>> assert data == decompressed +//! >>> assert bytes(data) == bytes(decompressed) //! >>> //! ``` //! @@ -62,7 +62,7 @@ pub mod zstd; use pyo3::prelude::*; -use crate::io::{RustyBuffer, RustyFile, RustyNumpyArray, RustyPyByteArray, RustyPyBytes}; +use crate::io::{AsBytes, RustyBuffer, RustyFile, RustyNumpyArray, RustyPyByteArray, RustyPyBytes}; use exceptions::{CompressionError, DecompressionError}; use std::io::{Read, Seek, SeekFrom, Write}; @@ -92,6 +92,43 @@ pub enum BytesType<'a> { RustyBuffer(&'a PyCell), } +impl<'a> AsBytes for BytesType<'a> { + fn as_bytes(&self) -> &[u8] { + match self { + BytesType::Bytes(b) => b.as_bytes(), + BytesType::ByteArray(b) => b.as_bytes(), + BytesType::NumpyArray(b) => b.as_bytes(), + BytesType::RustyBuffer(b) => { + let py_ref = b.borrow(); + let bytes = py_ref.as_bytes(); + unsafe { std::slice::from_raw_parts(bytes.as_ptr(), bytes.len()) } + } + BytesType::RustyFile(b) => { + let py_ref = b.borrow(); + let bytes = py_ref.as_bytes(); + unsafe { std::slice::from_raw_parts(bytes.as_ptr(), bytes.len()) } + } + } + } + fn as_bytes_mut(&mut self) -> &mut [u8] { + match self { + BytesType::Bytes(b) => b.as_bytes_mut(), + BytesType::ByteArray(b) => b.as_bytes_mut(), + BytesType::NumpyArray(b) => b.as_bytes_mut(), + BytesType::RustyBuffer(b) => { + let mut py_ref = b.borrow_mut(); + let bytes = py_ref.as_bytes_mut(); + unsafe { std::slice::from_raw_parts_mut(bytes.as_mut_ptr(), bytes.len()) } + } + BytesType::RustyFile(b) => { + let mut py_ref = b.borrow_mut(); + let bytes = py_ref.as_bytes_mut(); + unsafe { std::slice::from_raw_parts_mut(bytes.as_mut_ptr(), bytes.len()) } + } + } + } +} + impl<'a> Write for BytesType<'a> { fn write(&mut self, buf: &[u8]) -> std::io::Result { let result = match self { @@ -135,26 +172,18 @@ impl<'a> Seek for BytesType<'a> { } impl<'a> BytesType<'a> { - #[allow(dead_code)] fn len(&self) -> usize { self.as_bytes().len() } - fn as_bytes(&self) -> &'_ [u8] { - match self { - Self::Bytes(b) => b.as_bytes(), - Self::ByteArray(b) => b.as_bytes(), - _ => unimplemented!("Converting Rust native types to bytes is not supported"), - } - } } impl<'a> IntoPy for BytesType<'a> { fn into_py(self, py: Python) -> PyObject { match self { - Self::Bytes(bytes) => bytes.to_object(py), - Self::ByteArray(byte_array) => byte_array.to_object(py), + Self::Bytes(bytes) => bytes.inner.into(), + Self::ByteArray(byte_array) => byte_array.inner.into(), Self::RustyFile(file) => file.to_object(py), - Self::RustyBuffer(buffer) => buffer.to_object(py), + Self::RustyBuffer(buffer) => buffer.into_py(py), Self::NumpyArray(array) => array.to_object(py), } } @@ -163,74 +192,20 @@ impl<'a> IntoPy for BytesType<'a> { /// Macro for generating the implementation of de/compression against a variant interface #[macro_export] macro_rules! generic { - ($op:ident($input:expr), py=$py:ident, output_len=$output_len:ident $(, level=$level:ident)?) => { + ($op:ident($input:expr), output_len=$output_len:ident $(, level=$level:ident)?) => { { - use crate::io::{RustyPyBytes, RustyPyByteArray, RustyNumpyArray}; - - match $input { - BytesType::Bytes(_) => { - match $output_len { - Some(len) => { - let pybytes = PyBytes::new_with($py, len, |buffer| { - let mut cursor = Cursor::new(buffer); - if stringify!($op) == "compress" { - to_py_err!(CompressionError -> self::internal::$op($input, &mut cursor $(, $level)? ))?; - } else { - to_py_err!(DecompressionError -> self::internal::$op($input, &mut cursor $(, $level)? ))?; - } - Ok(()) - })?; - Ok(BytesType::Bytes(RustyPyBytes::from(pybytes))) - } - None => { - let mut buffer = Vec::new(); - if stringify!($op) == "compress" { - to_py_err!(CompressionError -> self::internal::$op($input, &mut Cursor::new(&mut buffer) $(, $level)? ))?; - } else { - to_py_err!(DecompressionError -> self::internal::$op($input, &mut Cursor::new(&mut buffer) $(, $level)? ))?; - } + use crate::io::RustyBuffer; - Ok(BytesType::Bytes(RustyPyBytes::from(PyBytes::new($py, &buffer)))) - } - } - }, - BytesType::ByteArray(_) => { - let mut pybytes = RustyPyByteArray::new($py, $output_len.unwrap_or_else(|| 0)); - if stringify!($op) == "compress" { - to_py_err!(CompressionError -> self::internal::$op($input, &mut pybytes $(, $level)? ))?; - } else { - to_py_err!(DecompressionError -> self::internal::$op($input, &mut pybytes $(, $level)? ))?; - } - Ok(BytesType::ByteArray(pybytes)) - }, - BytesType::NumpyArray(_) => { - let mut output = Vec::new(); - if stringify!($op) == "compress" { - to_py_err!(CompressionError -> self::internal::$op($input, &mut Cursor::new(&mut output) $(, $level)? ))?; - } else { - to_py_err!(DecompressionError -> self::internal::$op($input, &mut Cursor::new(&mut output) $(, $level)? ))?; - } - Ok(BytesType::NumpyArray(RustyNumpyArray::from_vec($py, output))) - }, - BytesType::RustyFile(_) => { - let mut output = crate::io::RustyBuffer::default(); - if stringify!($op) == "compress" { - to_py_err!(CompressionError -> self::internal::$op($input, &mut output $(, $level)? ))?; - } else { - to_py_err!(DecompressionError -> self::internal::$op($input, &mut output $(, $level)? ))?; - } - Ok(BytesType::RustyBuffer(PyCell::new($py, output).unwrap())) - }, - BytesType::RustyBuffer(_) => { - let mut output = crate::io::RustyBuffer::default(); - if stringify!($op) == "compress" { - to_py_err!(CompressionError -> self::internal::$op($input, &mut output $(, $level)? ))?; - } else { - to_py_err!(DecompressionError -> self::internal::$op($input, &mut output $(, $level)? ))?; - } - Ok(BytesType::RustyBuffer(PyCell::new($py, output).unwrap())) - } + let mut output: Vec = match $output_len { + Some(len) => vec![0; len], + None => vec![] + }; + if stringify!($op) == "compress" { + to_py_err!(CompressionError -> self::internal::$op($input, &mut Cursor::new(&mut output) $(, $level)? ))?; + } else { + to_py_err!(DecompressionError -> self::internal::$op($input, &mut Cursor::new(&mut output) $(, $level)? ))?; } + Ok(RustyBuffer::from(output)) } } } diff --git a/src/lz4.rs b/src/lz4.rs index e21b0ec2..c0744144 100644 --- a/src/lz4.rs +++ b/src/lz4.rs @@ -1,10 +1,10 @@ //! lz4 de/compression interface use crate::exceptions::{CompressionError, DecompressionError}; +use crate::io::RustyBuffer; use crate::{to_py_err, BytesType}; use pyo3::prelude::*; -use pyo3::types::PyBytes; use pyo3::wrap_pyfunction; -use pyo3::{PyResult, Python}; +use pyo3::PyResult; use std::io::Cursor; pub(crate) fn init_py_module(m: &PyModule) -> PyResult<()> { @@ -24,9 +24,8 @@ pub(crate) fn init_py_module(m: &PyModule) -> PyResult<()> { /// >>> cramjam.lz4.decompress(compressed_bytes, output_len=Optional[int]) /// ``` #[pyfunction] -#[allow(unused_variables)] // TODO: Make use of output_len for lz4 -pub fn decompress<'a>(py: Python<'a>, data: BytesType<'a>, output_len: Option) -> PyResult> { - crate::generic!(decompress(data), py = py, output_len = output_len) +pub fn decompress(data: BytesType, output_len: Option) -> PyResult { + crate::generic!(decompress(data), output_len = output_len) } /// lZ4 compression. @@ -38,30 +37,20 @@ pub fn decompress<'a>(py: Python<'a>, data: BytesType<'a>, output_len: Option>> cramjam.lz4.compress(b'some bytes here', output_len=Optional[int]) /// ``` #[pyfunction] -pub fn compress<'a>( - py: Python<'a>, - mut data: BytesType<'a>, - level: Option, - output_len: Option, -) -> PyResult> { - crate::generic!(compress(&mut data), py = py, output_len = output_len, level = level) +pub fn compress(mut data: BytesType, level: Option, output_len: Option) -> PyResult { + crate::generic!(compress(&mut data), output_len = output_len, level = level) } /// Compress directly into an output buffer #[pyfunction] -pub fn compress_into<'a>( - _py: Python<'a>, - mut input: BytesType<'a>, - mut output: BytesType<'a>, - level: Option, -) -> PyResult { +pub fn compress_into(mut input: BytesType, mut output: BytesType, level: Option) -> PyResult { let r = internal::compress(&mut input, &mut output, level)?; Ok(r) } /// Decompress directly into an output buffer #[pyfunction] -pub fn decompress_into<'a>(_py: Python<'a>, input: BytesType<'a>, mut output: BytesType<'a>) -> PyResult { +pub fn decompress_into(input: BytesType, mut output: BytesType) -> PyResult { let r = internal::decompress(input, &mut output)?; Ok(r) } diff --git a/src/snappy.rs b/src/snappy.rs index 534d96f5..e83181ba 100644 --- a/src/snappy.rs +++ b/src/snappy.rs @@ -1,10 +1,10 @@ //! snappy de/compression interface use crate::exceptions::{CompressionError, DecompressionError}; +use crate::io::{AsBytes, RustyBuffer}; use crate::{to_py_err, BytesType}; use pyo3::prelude::*; -use pyo3::types::PyBytes; use pyo3::wrap_pyfunction; -use pyo3::{PyResult, Python}; +use pyo3::PyResult; use std::io::Cursor; pub(crate) fn init_py_module(m: &PyModule) -> PyResult<()> { @@ -30,8 +30,8 @@ pub(crate) fn init_py_module(m: &PyModule) -> PyResult<()> { /// >>> cramjam.snappy.decompress(compressed_bytes, output_len=Optional[None]) /// ``` #[pyfunction] -pub fn decompress<'a>(py: Python<'a>, data: BytesType<'a>, output_len: Option) -> PyResult> { - crate::generic!(decompress(data), py = py, output_len = output_len) +pub fn decompress(data: BytesType, output_len: Option) -> PyResult { + crate::generic!(decompress(data), output_len = output_len) } /// Snappy compression. @@ -43,8 +43,8 @@ pub fn decompress<'a>(py: Python<'a>, data: BytesType<'a>, output_len: Option>> _ = cramjam.snappy.compress(bytearray(b'this avoids double allocation in rust side, and thus faster!')) # <- use bytearray where possible /// ``` #[pyfunction] -pub fn compress<'a>(py: Python<'a>, data: BytesType<'a>, output_len: Option) -> PyResult> { - crate::generic!(compress(data), py = py, output_len = output_len) +pub fn compress(data: BytesType, output_len: Option) -> PyResult { + crate::generic!(compress(data), output_len = output_len) } /// Snappy decompression, raw @@ -56,8 +56,11 @@ pub fn compress<'a>(py: Python<'a>, data: BytesType<'a>, output_len: Option>> cramjam.snappy.decompress_raw(compressed_raw_bytes) /// ``` #[pyfunction] -pub fn decompress_raw<'a>(py: Python<'a>, data: BytesType<'a>, output_len: Option) -> PyResult> { - crate::generic!(decompress_raw(data), py = py, output_len = output_len) +#[allow(unused_variables)] +pub fn decompress_raw(data: BytesType, output_len: Option) -> PyResult { + let mut decoder = snap::raw::Decoder::new(); + let output = to_py_err!(DecompressionError -> decoder.decompress_vec(data.as_bytes()))?; + Ok(RustyBuffer::from(output)) } /// Snappy compression raw. @@ -69,185 +72,60 @@ pub fn decompress_raw<'a>(py: Python<'a>, data: BytesType<'a>, output_len: Optio /// >>> cramjam.snappy.compress_raw(b'some bytes here') /// ``` #[pyfunction] -pub fn compress_raw<'a>(py: Python<'a>, data: BytesType<'a>, output_len: Option) -> PyResult> { - crate::generic!(compress_raw(data), py = py, output_len = output_len) +#[allow(unused_variables)] +pub fn compress_raw(data: BytesType, output_len: Option) -> PyResult { + let mut encoder = snap::raw::Encoder::new(); + let output = to_py_err!(CompressionError -> encoder.compress_vec(data.as_bytes()))?; + Ok(RustyBuffer::from(output)) } /// Compress directly into an output buffer #[pyfunction] -pub fn compress_into<'a>(_py: Python<'a>, input: BytesType<'a>, mut output: BytesType<'a>) -> PyResult { +pub fn compress_into(input: BytesType, mut output: BytesType) -> PyResult { let r = internal::compress(input, &mut output)?; Ok(r) } /// Decompress directly into an output buffer #[pyfunction] -pub fn decompress_into<'a>(_py: Python<'a>, input: BytesType<'a>, mut output: BytesType<'a>) -> PyResult { +pub fn decompress_into(input: BytesType, mut output: BytesType) -> PyResult { let r = internal::decompress(input, &mut output)?; Ok(r as usize) } /// Compress raw format directly into an output buffer #[pyfunction] -pub fn compress_raw_into<'a>(_py: Python<'a>, input: BytesType<'a>, mut output: BytesType<'a>) -> PyResult { - let r = to_py_err!(CompressionError -> internal::compress_raw(input, &mut output))?; - Ok(r) +pub fn compress_raw_into(input: BytesType, mut output: BytesType) -> PyResult { + let mut encoder = snap::raw::Encoder::new(); + let output = encoder.compress(input.as_bytes(), output.as_bytes_mut()); + to_py_err!(CompressionError -> output) } /// Decompress raw format directly into an output buffer #[pyfunction] -pub fn decompress_raw_into<'a>(_py: Python<'a>, input: BytesType<'a>, mut output: BytesType<'a>) -> PyResult { - let r = to_py_err!(DecompressionError -> internal::decompress_raw(input, &mut output))?; - Ok(r) +pub fn decompress_raw_into(input: BytesType, mut output: BytesType) -> PyResult { + let mut decoder = snap::raw::Decoder::new(); + let output = decoder.decompress(input.as_bytes(), output.as_bytes_mut()); + to_py_err!(DecompressionError -> output) } /// Get the expected max compressed length for snappy raw compression; this is the size /// of buffer that should be passed to `compress_raw_into` #[pyfunction] -pub fn compress_raw_max_len<'a>(_py: Python<'a>, data: BytesType<'a>) -> usize { +pub fn compress_raw_max_len(data: BytesType) -> usize { snap::raw::max_compress_len(data.len()) } /// Get the decompressed length for the given data. This is the size of buffer /// that should be passed to `decompress_raw_into` #[pyfunction] -pub fn decompress_raw_len<'a>(_py: Python<'a>, data: BytesType<'a>) -> PyResult { +pub fn decompress_raw_len(data: BytesType) -> PyResult { to_py_err!(DecompressionError -> snap::raw::decompress_len(data.as_bytes())) } pub(crate) mod internal { - use crate::BytesType; - use snap::raw::{decompress_len, max_compress_len, Decoder, Encoder}; use snap::read::{FrameDecoder, FrameEncoder}; - use std::io::{Cursor, Error, Read, Write}; - - pub(crate) struct RawEncoder<'a, 'b> { - inner: &'b BytesType<'a>, - overflow: Option>>, - encoder: Encoder, - is_finished: bool, - } - impl<'a, 'b> RawEncoder<'a, 'b> { - pub fn new(inner: &'b BytesType<'a>) -> Self { - Self { - inner, - encoder: Encoder::new(), - overflow: None, - is_finished: false, - } - } - fn init_read(&mut self, bytes: &[u8], buf: &mut [u8]) -> std::io::Result { - let len = max_compress_len(bytes.len()); - if buf.len() >= len { - let n = self.encoder.compress(bytes, buf)?; - self.is_finished = true; // if overflow is set, it will return 0 next iter - Ok(n) - } else { - let mut overflow = vec![0; len]; - let n = self.encoder.compress(bytes, overflow.as_mut_slice())?; - overflow.truncate(n); - let mut overflow = Cursor::new(overflow); - let r = overflow.read(buf)?; - self.overflow = Some(overflow); - Ok(r) - } - } - } - impl<'a, 'b> Read for RawEncoder<'a, 'b> { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - if self.is_finished { - return Ok(0); - } - match self.overflow.as_mut() { - Some(overflow) => overflow.read(buf), - None => match self.inner { - BytesType::Bytes(pybytes) => self.init_read(pybytes.as_bytes(), buf), - BytesType::ByteArray(pybytes) => self.init_read(pybytes.as_bytes(), buf), - BytesType::NumpyArray(array) => self.init_read(array.as_bytes(), buf), - BytesType::RustyBuffer(buffer) => { - let buffer_ref = buffer.borrow(); - self.init_read(buffer_ref.inner.get_ref(), buf) - } - BytesType::RustyFile(file) => { - let mut buffer = vec![]; - file.borrow_mut().read_to_end(&mut buffer)?; - self.init_read(buffer.as_slice(), buf) - } - }, - } - } - } - - pub(crate) struct RawDecoder<'a, 'b> { - inner: &'b BytesType<'a>, - overflow: Option>>, - decoder: Decoder, - is_finished: bool, - } - impl<'a, 'b> RawDecoder<'a, 'b> { - pub fn new(inner: &'b BytesType<'a>) -> Self { - Self { - inner, - decoder: Decoder::new(), - overflow: None, - is_finished: false, - } - } - fn init_read(&mut self, bytes: &[u8], buf: &mut [u8]) -> std::io::Result { - let len = decompress_len(bytes)?; - if buf.len() >= len { - let n = self.decoder.decompress(bytes, buf)?; - self.is_finished = true; // if overflow is set, it will return 0 next iter - Ok(n) - } else { - let mut overflow = vec![0; len]; - let n = self.decoder.decompress(bytes, overflow.as_mut_slice())?; - overflow.truncate(n); - let mut overflow = Cursor::new(overflow); - let r = overflow.read(buf)?; - self.overflow = Some(overflow); - Ok(r) - } - } - } - impl<'a, 'b> Read for RawDecoder<'a, 'b> { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - if self.is_finished { - return Ok(0); - } - match self.overflow.as_mut() { - Some(overflow) => overflow.read(buf), - None => match self.inner { - BytesType::Bytes(pybytes) => self.init_read(pybytes.as_bytes(), buf), - BytesType::ByteArray(pybytes) => self.init_read(pybytes.as_bytes(), buf), - BytesType::NumpyArray(array) => self.init_read(array.as_bytes(), buf), - BytesType::RustyBuffer(buffer) => { - let buffer_ref = buffer.borrow(); - self.init_read(buffer_ref.inner.get_ref(), buf) - } - BytesType::RustyFile(file) => { - let mut buffer = vec![]; - file.borrow_mut().read_to_end(&mut buffer)?; - self.init_read(buffer.as_slice(), buf) - } - }, - } - } - } - - /// Decompress snappy data raw into a mutable slice - pub fn decompress_raw<'a, W: Write>(input: BytesType<'a>, output: &mut W) -> std::io::Result { - let mut decoder = RawDecoder::new(&input); - let n_bytes = std::io::copy(&mut decoder, output)?; - Ok(n_bytes as usize) - } - - /// Compress snappy data raw into a mutable slice - pub fn compress_raw<'a, W: Write>(input: BytesType<'a>, output: &'a mut W) -> std::io::Result { - let mut encoder = RawEncoder::new(&input); - let n_bytes = std::io::copy(&mut encoder, output)?; - Ok(n_bytes as usize) - } + use std::io::{Error, Read, Write}; /// Decompress snappy data framed pub fn decompress(input: R, output: &mut W) -> Result { diff --git a/src/zstd.rs b/src/zstd.rs index 00a290fc..bceee059 100644 --- a/src/zstd.rs +++ b/src/zstd.rs @@ -1,10 +1,10 @@ //! zstd de/compression interface use crate::exceptions::{CompressionError, DecompressionError}; +use crate::io::RustyBuffer; use crate::{to_py_err, BytesType}; use pyo3::prelude::*; -use pyo3::types::PyBytes; use pyo3::wrap_pyfunction; -use pyo3::{PyResult, Python}; +use pyo3::PyResult; use std::io::Cursor; pub(crate) fn init_py_module(m: &PyModule) -> PyResult<()> { @@ -23,8 +23,8 @@ pub(crate) fn init_py_module(m: &PyModule) -> PyResult<()> { /// >>> cramjam.zstd.decompress(compressed_bytes, output_len=Optional[int]) /// ``` #[pyfunction] -pub fn decompress<'a>(py: Python<'a>, data: BytesType<'a>, output_len: Option) -> PyResult> { - crate::generic!(decompress(data), py = py, output_len = output_len) +pub fn decompress(data: BytesType, output_len: Option) -> PyResult { + crate::generic!(decompress(data), output_len = output_len) } /// ZSTD compression. @@ -35,27 +35,24 @@ pub fn decompress<'a>(py: Python<'a>, data: BytesType<'a>, output_len: Option>> cramjam.zstd.compress(b'some bytes here', level=0, output_len=Optional[int]) # level defaults to 11 /// ``` #[pyfunction] -pub fn compress<'a>( - py: Python<'a>, - data: BytesType<'a>, - level: Option, - output_len: Option, -) -> PyResult> { - crate::generic!(compress(data), py = py, output_len = output_len, level = level) +pub fn compress(data: BytesType, level: Option, output_len: Option) -> PyResult { + crate::generic!(compress(data), output_len = output_len, level = level) } /// Compress directly into an output buffer #[pyfunction] -pub fn compress_into<'a>( - _py: Python<'a>, - input: BytesType<'a>, - mut output: BytesType<'a>, - level: Option, -) -> PyResult { +pub fn compress_into(input: BytesType, mut output: BytesType, level: Option) -> PyResult { let r = internal::compress(input, &mut output, level)?; Ok(r) } +/// Decompress directly into an output buffer +#[pyfunction] +pub fn decompress_into<'a>(_py: Python<'a>, input: BytesType<'a>, mut output: BytesType<'a>) -> PyResult { + let r = internal::decompress(input, &mut output)?; + Ok(r) +} + pub(crate) mod internal { use std::io::{Error, Read, Write}; @@ -69,16 +66,9 @@ pub(crate) mod internal { /// Compress gzip data pub fn compress(input: R, output: &mut W, level: Option) -> Result { - let level = level.unwrap_or_else(|| 0); // 0 will use zstd's default, currently 11 + let level = level.unwrap_or_else(|| 0); // 0 will use zstd's default, currently 3 let mut encoder = zstd::stream::read::Encoder::new(input, level)?; let n_bytes = std::io::copy(&mut encoder, output)?; Ok(n_bytes as usize) } } - -/// Decompress directly into an output buffer -#[pyfunction] -pub fn decompress_into<'a>(_py: Python<'a>, input: BytesType<'a>, mut output: BytesType<'a>) -> PyResult { - let r = internal::decompress(input, &mut output)?; - Ok(r) -} diff --git a/tests/test_variants.py b/tests/test_variants.py index 2d1062e8..c273e757 100644 --- a/tests/test_variants.py +++ b/tests/test_variants.py @@ -27,12 +27,13 @@ def test_variants_simple(variant_str, is_bytearray): uncompressed = bytearray(uncompressed) compressed = variant.compress(uncompressed) - assert compressed != uncompressed - assert type(compressed) == type(uncompressed) + assert compressed.read() != uncompressed + compressed.seek(0) + assert isinstance(compressed, cramjam.Buffer) decompressed = variant.decompress(compressed, output_len=len(uncompressed)) - assert decompressed == uncompressed - assert type(decompressed) == type(uncompressed) + assert decompressed.read() == uncompressed + assert isinstance(decompressed, cramjam.Buffer) @pytest.mark.parametrize( @@ -170,3 +171,22 @@ def test_variant_snappy_raw_into(): assert n_bytes == len(data) assert same_same(decompressed_buffer[:n_bytes], data) + + +@pytest.mark.parametrize("Obj", (cramjam.File, cramjam.Buffer)) +def test_dunders(Obj, tmpdir): + if Obj == cramjam.File: + path = str(tmpdir.join("tmp.txt")) + obj = Obj(path) + else: + obj = Obj() + + assert len(obj) == 0 + assert bool(obj) is False + obj.write(b"12345") + assert len(obj) == 5 + assert bool(obj) is True + + assert "len=5" in str(obj) + if isinstance(obj, cramjam.File): + assert f"path={path}" in str(obj)