Fix import of custom types (google#109)

docs/tutorials/quick_start/mnist_read_jax.py

@@ -22,18 +22,16 @@
 """
 import argparse
 from functools import partial
-from typing import Any, Dict, Tuple
+from typing import Any
 
 from jax import Array
 from jax.typing import ArrayLike
 from flax import nnx
 import jax.numpy as jnp
-import numpy as np
 import optax
 from tqdm import tqdm
 
 from sedpack.io import Dataset
-from sedpack.io.types import ExampleT, TFModelT
 
 
 def process_batch(d: Any) -> dict[str, Array]:
@@ -154,7 +152,8 @@ def main() -> None:
         if step > train_steps:
             break
 
-        # Run the optimization for one step and make a stateful update to the following:
+        # Run the optimization for one step and make a stateful update to the
+        # following:
         # - The train state's model parameters
         # - The optimizer state
         # - The training loss and accuracy batch metrics

docs/tutorials/quick_start/mnist_read_keras.py

@@ -19,7 +19,7 @@
     python mnist_read_keras.py -d "~/Datasets/my_new_dataset/"
 """
 import argparse
-from typing import Any, Dict, Tuple
+from typing import Any, Tuple
 
 import numpy as np
 import tensorflow as tf

docs/tutorials/quick_start/mnist_save.py

@@ -99,7 +99,7 @@ def main() -> None:
         random.shuffle(train_indices)
         validation_split_position: int = int(len(x_train) * 0.1)
         for index_position, index in enumerate(
-                tqdm(train_indices, desc='train and val')):
+                tqdm(train_indices, desc="train and val")):
 
             # Assign to either train or test (aka validation).
             split: SplitT = "test"

docs/tutorials/sca/tiny_aes.py

@@ -18,11 +18,12 @@
 The SCAAML package is a requirement: `python3 -m pip install "scaaml>=3.0.3"`.
 
 Example use:
-    python tiny_aes.py --dataset_path "~/datasets/tiny_aes_sedpack/" --original_files "~/datasets/tinyaes"
+    python tiny_aes.py --dataset_path "~/datasets/tiny_aes_sedpack/" \
+                       --original_files "~/datasets/tinyaes"
 """
 import argparse
 from pathlib import Path
-from typing import Any, get_args
+from typing import Any
 
 import keras
 import numpy as np
@@ -37,7 +38,7 @@
     DatasetStructure,
     Attribute,
 )
-from sedpack.io.typing import SplitT
+from sedpack.io.types import SplitT
 
 
 def add_shard(shard_file: Path, dataset_filler: DatasetFillerContext,
@@ -121,7 +122,7 @@ def convert_to_sedpack(dataset_path: Path, original_files: Path) -> None:
             year={2019},
             editor={DEF CON}
             }
-            """,
+            """,  # pylint: disable=line-too-long
             "original from":
                 "https://github.com/google/scaaml/tree/main/scaaml_intro",
         },
@@ -271,7 +272,7 @@ def train(dataset_path: Path) -> None:
     )
 
     # Train the model.
-    history = model.fit(
+    _ = model.fit(
         train_ds,
         steps_per_epoch=steps_per_epoch,
         epochs=epochs,

project-words.txt

@@ -59,4 +59,5 @@ tfrecord
 tfrecords
 tinyaes
 tobytes
+uoffset
 zstd

pyproject.toml

@@ -61,3 +61,6 @@ dev = [
 [project.urls]
 "Homepage" = "https://github.com/google/sedpack"
 "Bug Tracker" = "https://github.com/google/sedpack"
+
+[tool.ruff]
+target-version = "py310"

src/sedpack/io/flatbuffer/shardfile/Attribute.py

@@ -19,7 +19,6 @@
 # pylint: skip-file
 
 import flatbuffers  # type: ignore[import-untyped]
-from flatbuffers.compat import import_numpy  # type: ignore[import-untyped]
 
 import numpy as np
 import numpy.typing as npt

tests/io/itertools/test_itertools.py

@@ -12,12 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from pathlib import Path
 import random
-from typing import Any, Union
-
-import numpy as np
-import numpy.typing as npt
 
 from sedpack.io.itertools import *
 

tests/io/shard/test_shard_write_and_read.py

@@ -14,7 +14,6 @@
 
 from pathlib import Path
 import pytest
-from typing import get_args
 
 import numpy as np
 
@@ -24,7 +23,7 @@
 from sedpack.io.flatbuffer import IterateShardFlatBuffer
 from sedpack.io.npz import IterateShardNP
 from sedpack.io.tfrec import IterateShardTFRec
-from sedpack.io.shard.get_shard_writer import get_shard_writer, _SHARD_FILE_TYPE_TO_CLASS
+from sedpack.io.shard.get_shard_writer import get_shard_writer
 
 
 def shard_write_and_read(attributes: dict[str, np.ndarray], shard_file: Path,

tests/io/shard/test_shard_write_and_read_process_and_list.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 from pathlib import Path
-from typing import get_args
 
 import numpy as np
 
@@ -23,7 +22,7 @@
 from sedpack.io.flatbuffer import IterateShardFlatBuffer
 from sedpack.io.npz import IterateShardNP
 from sedpack.io.tfrec import IterateShardTFRec
-from sedpack.io.shard.get_shard_writer import get_shard_writer, _SHARD_FILE_TYPE_TO_CLASS
+from sedpack.io.shard.get_shard_writer import get_shard_writer
 
 
 def shard_write_and_read(attributes: dict[str, np.ndarray], shard_file: Path,

tests/io/shard/test_shard_write_async.py

@@ -12,9 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import asyncio
 from pathlib import Path
-from typing import get_args
 import pytest
 
 import numpy as np
@@ -24,7 +22,7 @@
 
 from sedpack.io.flatbuffer import IterateShardFlatBuffer
 from sedpack.io.npz import IterateShardNP
-from sedpack.io.shard.get_shard_writer import get_shard_writer, _SHARD_FILE_TYPE_TO_CLASS
+from sedpack.io.shard.get_shard_writer import get_shard_writer
 
 pytest_plugins = ("pytest_asyncio",)
 

tests/io/shard/test_shard_writer_base.py

@@ -15,8 +15,7 @@
 from typing import get_args
 
 from sedpack.io.types import ShardFileTypeT
-from sedpack.io.shard.shard_writer_base import ShardWriterBase
-from sedpack.io.shard.get_shard_writer import get_shard_writer, _SHARD_FILE_TYPE_TO_CLASS
+from sedpack.io.shard.get_shard_writer import _SHARD_FILE_TYPE_TO_CLASS
 
 
 def test_all_file_types_supported():

tests/io/test_bytes.py

@@ -13,14 +13,13 @@
 # limitations under the License.
 
 from pathlib import Path
-from typing import Any, Union
+from typing import Union
 
 import numpy as np
-import numpy.typing as npt
 
 import sedpack
 from sedpack.io import Dataset
-from sedpack.io import Metadata, DatasetStructure, Attribute
+from sedpack.io import Metadata
 from sedpack.io.types import TRAIN_SPLIT
 
 

tests/io/test_check_consistency.py

@@ -13,15 +13,14 @@
 # limitations under the License.
 
 from pathlib import Path
-from typing import Any, get_args, Union
+from typing import get_args, Union
 
 import pytest
 import numpy as np
-import numpy.typing as npt
 
 import sedpack
 from sedpack.io import Dataset
-from sedpack.io import Metadata, DatasetStructure, Attribute
+from sedpack.io import Metadata
 from sedpack.io.types import CompressionT, HashChecksumT, ShardFileTypeT, TRAIN_SPLIT
 
 

tests/io/test_compression.py

@@ -14,11 +14,9 @@
 
 import gzip
 from pathlib import Path
-from typing import Any, Union
+from typing import Union
 
 import pytest
-import numpy as np
-import numpy.typing as npt
 
 from sedpack.io.compress import CompressedFile
 from sedpack.io.types import CompressionT

tests/io/test_continue_writing.py

@@ -15,15 +15,13 @@
 import os
 from pathlib import Path
 import random
-from typing import Any, Union
+from typing import Union
 
 import numpy as np
-import numpy.typing as npt
 
 import sedpack
 from sedpack.io import Dataset
-from sedpack.io import Metadata, DatasetStructure, Attribute
-from sedpack.io.types import CompressionT, ShardFileTypeT
+from sedpack.io import Metadata
 
 
 def get_dataset(tmpdir: Union[str, Path]) -> Dataset:

tests/io/test_custom_metadata_type_limit.py

@@ -13,14 +13,14 @@
 # limitations under the License.
 
 from pathlib import Path
-from typing import Any, Union
+from typing import Union
 
 import numpy as np
 import numpy.typing as npt
 
 import sedpack
 from sedpack.io import Dataset
-from sedpack.io import Metadata, DatasetStructure, Attribute
+from sedpack.io import Metadata
 from sedpack.io.types import TRAIN_SPLIT, CompressionT, ShardFileTypeT
 
 

tests/io/test_end2end.py

@@ -13,14 +13,14 @@
 # limitations under the License.
 
 from pathlib import Path
-from typing import Any, Union
+from typing import Union
 
 import numpy as np
 import numpy.typing as npt
 
 import sedpack
 from sedpack.io import Dataset
-from sedpack.io import Metadata, DatasetStructure, Attribute
+from sedpack.io import Metadata
 from sedpack.io.types import TRAIN_SPLIT, CompressionT, ShardFileTypeT
 
 

tests/io/test_end2end_async.py

@@ -14,14 +14,14 @@
 
 from pathlib import Path
 import pytest
-from typing import Any, Union
+from typing import Union
 
 import numpy as np
 import numpy.typing as npt
 
 import sedpack
 from sedpack.io import Dataset
-from sedpack.io import Metadata, DatasetStructure, Attribute
+from sedpack.io import Metadata
 from sedpack.io.types import TRAIN_SPLIT, CompressionT, ShardFileTypeT
 
 pytest_plugins = ("pytest_asyncio",)

tests/io/test_end2end_shuffled.py

@@ -13,14 +13,14 @@
 # limitations under the License.
 
 from pathlib import Path
-from typing import Any, Union
+from typing import Union
 
 import numpy as np
 import numpy.typing as npt
 
 import sedpack
 from sedpack.io import Dataset
-from sedpack.io import Metadata, DatasetStructure, Attribute
+from sedpack.io import Metadata
 from sedpack.io.types import TRAIN_SPLIT, CompressionT, ShardFileTypeT
 
 

tests/io/test_end2end_wrong_type.py

@@ -14,14 +14,14 @@
 
 from pathlib import Path
 import pytest
-from typing import Any, Union
+from typing import Union
 
 import numpy as np
 import numpy.typing as npt
 
 import sedpack
 from sedpack.io import Dataset
-from sedpack.io import Metadata, DatasetStructure, Attribute
+from sedpack.io import Metadata
 from sedpack.io.types import TRAIN_SPLIT, CompressionT, ShardFileTypeT
 
 

tests/io/test_error.py

@@ -17,7 +17,7 @@
 
 import sedpack
 from sedpack.io import Dataset
-from sedpack.io import Attribute, Metadata
+from sedpack.io import Metadata
 from sedpack.io.errors import DatasetExistsError
 
 

tests/io/test_hash_checksums.py

@@ -12,12 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import gzip
 from pathlib import Path
-from typing import Any, Union
-
-import numpy as np
-import numpy.typing as npt
 
 from sedpack.io.utils import hash_checksums
 

tests/io/test_rust_iter.py

@@ -13,16 +13,15 @@
 # limitations under the License.
 
 from pathlib import Path
-from typing import Any, get_args, Union
+from typing import get_args, Union
 
 import numpy as np
 import numpy.typing as npt
 import pytest
 
 import sedpack
 from sedpack.io import Dataset
-from sedpack.io import Metadata, DatasetStructure, Attribute
-from sedpack.io.flatbuffer.iterate import IterateShardFlatBuffer
+from sedpack.io import Metadata
 from sedpack.io.types import TRAIN_SPLIT, CompressionT, ShardFileTypeT
 
 from sedpack import _sedpack_rs  # type: ignore[attr-defined]

tests/io/test_shard_custom_metadata.py

@@ -13,14 +13,13 @@
 # limitations under the License.
 
 from pathlib import Path
-from typing import Any, Union
+from typing import Union
 
 import numpy as np
-import numpy.typing as npt
 
 import sedpack
 from sedpack.io import Dataset
-from sedpack.io import Metadata, DatasetStructure, Attribute
+from sedpack.io import Metadata
 from sedpack.io.types import TRAIN_SPLIT