Rust file-like objects and accept bytes/bytearray/numpy

.github/workflows/CI.yml

@@ -26,9 +26,9 @@ jobs:
           profile: minimal
           default: true
       - name: Build
-        run: cargo build
+        run: cargo build --release
       - name: Tests
-        run: cargo test
+        run: cargo test --no-default-features --release
       - name: Install maturin
         run: pip install maturin
       - name: Build wheels - x86_64
@@ -78,10 +78,10 @@ jobs:
           default: true
       - name: Build
         if: matrix.platform.python-architecture == 'x64'
-        run: cargo build
+        run: cargo build --release
       - name: Tests
         if: matrix.platform.python-architecture == 'x64'
-        run: cargo test
+        run: cargo test --no-default-features --release
       - name: Install maturin
         run: pip install maturin
       - name: Build wheels
@@ -115,9 +115,9 @@ jobs:
         profile: minimal
         default: true
     - name: Build
-      run: cargo build
+      run: cargo build --release
     - name: Tests
-      run: cargo test
+      run: cargo test --no-default-features
     - uses: actions/setup-python@v2
       with:
         python-version: 3.6
@@ -220,9 +220,9 @@ jobs:
         profile: minimal
         default: true
     - name: Build
-      run: cargo build
+      run: cargo build --release
     - name: Tests
-      run: cargo test
+      run: cargo test --no-default-features
     - uses: actions/setup-python@v2
       with:
         python-version: pypy-3.6

Cargo.toml

@@ -9,23 +9,24 @@ readme = "README.md"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [lib]
-crate-type = ["cdylib"]
+crate-type = ["cdylib", "rlib"]
 
 [features]
-default = ["abi3", "mimallocator"]
+default = ["abi3", "mimallocator", "extension-module"]
 abi3 = ["pyo3/abi3-py36"]
 mimallocator = ["mimalloc"]
+extension-module = ["pyo3/extension-module"]
 
 [profile.release]
 lto = "fat"
 codegen-units = 1
 opt-level = 3
 
 [dependencies]
-pyo3 = { version = "0.13.2", features = ["extension-module"] }
+pyo3 = { version = "0.13.2" }
 snap = "^1"
 brotli2 = "^0.3"
-lz-fear = "0.1.1"
+lz4 = "^1"
 flate2 = "^1"
 zstd = "0.6.0+zstd.1.4.8"
 numpy = "0.13.0"

README.md

@@ -49,7 +49,7 @@ Where the API is `cramjam.<compression-variant>.compress/decompress` and accepts
 both `bytes` and `bytearray` objects.
 
 **de/compress_into**
-Additionally, all variants except for lz4, support `decompress_into` and `compress_into`.
+Additionally, support `decompress_into` and `compress_into`.
 If you have a numpy array preallocated, that can be used as the output location for de/compression.  
 Ex.
 ```python 

src/brotli.rs

@@ -68,14 +68,14 @@ pub(crate) mod internal {
     use std::io::Error;
 
     /// Decompress via Brotli
-    pub fn decompress<W: Write + ?Sized>(input: &[u8], output: &mut W) -> Result<usize, Error> {
+    pub fn decompress<W: Write + ?Sized, R: Read>(input: R, output: &mut W) -> Result<usize, Error> {
         let mut decoder = BrotliDecoder::new(input);
         let n_bytes = std::io::copy(&mut decoder, output)?;
         Ok(n_bytes as usize)
     }
 
     /// Compress via Brotli
-    pub fn compress<W: Write + ?Sized>(input: &[u8], output: &mut W, level: Option<u32>) -> Result<usize, Error> {
+    pub fn compress<W: Write + ?Sized, R: Read>(input: R, output: &mut W, level: Option<u32>) -> Result<usize, Error> {
         let level = level.unwrap_or_else(|| 11);
         let mut encoder = BrotliEncoder::new(input, level);
         let n_bytes = std::io::copy(&mut encoder, output)?;

src/deflate.rs

@@ -69,14 +69,14 @@ pub(crate) mod internal {
     use std::io::Error;
 
     /// Decompress gzip data
-    pub fn decompress<W: Write + ?Sized>(input: &[u8], output: &mut W) -> Result<usize, Error> {
+    pub fn decompress<W: Write + ?Sized, R: Read>(input: R, output: &mut W) -> Result<usize, Error> {
         let mut decoder = DeflateDecoder::new(input);
         let n_bytes = std::io::copy(&mut decoder, output)?;
         Ok(n_bytes as usize)
     }
 
     /// Compress gzip data
-    pub fn compress<W: Write + ?Sized>(input: &[u8], output: &mut W, level: Option<u32>) -> Result<usize, Error> {
+    pub fn compress<W: Write + ?Sized, R: Read>(input: R, output: &mut W, level: Option<u32>) -> Result<usize, Error> {
         let level = level.unwrap_or_else(|| 6);
 
         let mut encoder = DeflateEncoder::new(input, Compression::new(level));

src/gzip.rs

@@ -68,14 +68,14 @@ pub(crate) mod internal {
     use std::io::Error;
 
     /// Decompress gzip data
-    pub fn decompress<W: Write + ?Sized>(input: &[u8], output: &mut W) -> Result<usize, Error> {
+    pub fn decompress<W: Write + ?Sized, R: Read>(input: R, output: &mut W) -> Result<usize, Error> {
         let mut decoder = GzDecoder::new(input);
         let n_bytes = std::io::copy(&mut decoder, output)?;
         Ok(n_bytes as usize)
     }
 
     /// Compress gzip data
-    pub fn compress<W: Write + ?Sized>(input: &[u8], output: &mut W, level: Option<u32>) -> Result<usize, Error> {
+    pub fn compress<W: Write + ?Sized, R: Read>(input: R, output: &mut W, level: Option<u32>) -> Result<usize, Error> {
         let level = level.unwrap_or_else(|| 6);
         let mut encoder = GzEncoder::new(input, Compression::new(level));
         let n_bytes = std::io::copy(&mut encoder, output)?;

src/io.rs

@@ -0,0 +1,152 @@
+use std::fs::{File, OpenOptions};
+use std::io::{Cursor, Read, Seek, SeekFrom, Write};
+
+use pyo3::prelude::*;
+use pyo3::types::PyBytes;
+use pyo3::number::pos;
+
+#[pyclass(name = "File")]
+pub struct RustyFile {
+    inner: File,
+}
+
+#[pymethods]
+impl RustyFile {
+    #[new]
+    pub fn new(
+        path: &str,
+        read: Option<bool>,
+        write: Option<bool>,
+        truncate: Option<bool>,
+        append: Option<bool>,
+    ) -> PyResult<Self> {
+        Ok(Self {
+            inner: OpenOptions::new()
+                .read(read.unwrap_or_else(|| true))
+                .write(write.unwrap_or_else(|| true))
+                .truncate(truncate.unwrap_or_else(|| false))
+                .create(true) // create if doesn't exist, but open if it does.
+                .append(append.unwrap_or_else(|| false))
+                .open(path)?,
+        })
+    }
+
+    pub fn write(&mut self, buf: &[u8]) -> PyResult<usize> {
+        let r = Write::write(self, buf)?;
+        Ok(r)
+    }
+    pub fn read<'a>(&mut self, py: Python<'a>, n_bytes: Option<usize>) -> PyResult<&'a PyBytes> {
+        match n_bytes {
+            Some(n) => {
+                let mut buf = vec![0; n];
+                self.inner.read(buf.as_mut_slice())?;
+                Ok(PyBytes::new(py, buf.as_slice()))
+            }
+            None => {
+                let mut buf = vec![];
+                self.inner.read_to_end(&mut buf)?;
+                Ok(PyBytes::new(py, buf.as_slice()))
+            }
+        }
+    }
+    pub fn seek(&mut self, position: usize) -> PyResult<usize> {
+        let r = Seek::seek(self, SeekFrom::Start(position as u64))?;
+        Ok(r as usize)
+    }
+    pub fn set_len(&mut self, size: usize) -> PyResult<()> {
+        self.inner.set_len(size as u64)?;
+        Ok(())
+    }
+    pub fn truncate(&mut self) -> PyResult<()> {
+        self.set_len(0)
+    }
+}
+
+#[pyclass(name = "Buffer")]
+#[derive(Default)]
+pub struct RustyBuffer {
+    inner: Cursor<Vec<u8>>,
+}
+
+#[pymethods]
+impl RustyBuffer {
+    #[new]
+    pub fn new(len: Option<usize>) -> Self {
+        Self {
+            inner: Cursor::new(vec![0; len.unwrap_or_else(|| 0)]),
+        }
+    }
+
+    pub fn write(&mut self, buf: &[u8]) -> PyResult<usize> {
+        let r = Write::write(self, buf)?;
+        Ok(r)
+    }
+    pub fn read<'a>(&mut self, py: Python<'a>, n_bytes: Option<usize>) -> PyResult<&'a PyBytes> {
+        match n_bytes {
+            Some(n) => {
+                let mut buf = vec![0; n];
+                self.inner.read(buf.as_mut_slice())?;
+                Ok(PyBytes::new(py, buf.as_slice()))
+            }
+            None => {
+                let mut buf = vec![];
+                self.inner.read_to_end(&mut buf)?;
+                Ok(PyBytes::new(py, buf.as_slice()))
+            }
+        }
+    }
+    pub fn seek(&mut self, position: usize) -> PyResult<usize> {
+        // TODO: Support SeekFrom from python side as in IOBase.seek definition
+        let r = Seek::seek(self, SeekFrom::Start(position as u64))?;
+        Ok(r as usize)
+    }
+    pub fn set_len(&mut self, size: usize) -> PyResult<()> {
+        self.inner.get_mut().resize(size, 0);
+        Ok(())
+    }
+    pub fn truncate(&mut self) -> PyResult<()> {
+        self.inner.get_mut().truncate(0);
+        Ok(())
+    }
+}
+
+impl Seek for RustyBuffer {
+    fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
+        self.inner.seek(pos)
+    }
+}
+impl Seek for RustyFile {
+    fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
+        self.inner.seek(pos)
+    }
+}
+
+impl Write for RustyBuffer {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        self.inner.write(buf)
+    }
+    fn flush(&mut self) -> std::io::Result<()> {
+        self.inner.flush()
+    }
+}
+
+impl Write for RustyFile {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        self.inner.write(buf)
+    }
+    fn flush(&mut self) -> std::io::Result<()> {
+        self.inner.flush()
+    }
+}
+
+impl Read for RustyBuffer {
+    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+        self.inner.read(buf)
+    }
+}
+
+impl Read for RustyFile {
+    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+        self.inner.read(buf)
+    }
+}