diff --git a/model_compression_toolkit/core/__init__.py b/model_compression_toolkit/core/__init__.py index d477d8174..2b9bf4696 100644 --- a/model_compression_toolkit/core/__init__.py +++ b/model_compression_toolkit/core/__init__.py @@ -13,7 +13,6 @@ # limitations under the License. # ============================================================================== -from model_compression_toolkit.core.common.data_loader import FolderImageLoader from model_compression_toolkit.core.common.framework_info import FrameworkInfo, ChannelAxis from model_compression_toolkit.core.common import network_editors as network_editor from model_compression_toolkit.core.common.quantization.debug_config import DebugConfig diff --git a/model_compression_toolkit/core/common/data_loader.py b/model_compression_toolkit/core/common/data_loader.py deleted file mode 100644 index fe4c09851..000000000 --- a/model_compression_toolkit/core/common/data_loader.py +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright 2021 Sony Semiconductor Israel, Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - - -import os -from typing import List, Callable - -import numpy as np -from PIL import Image -from model_compression_toolkit.logger import Logger - -#: -FILETYPES = ['jpeg', 'jpg', 'bmp', 'png'] - - -class FolderImageLoader(object): - """ - - Class for images loading, processing and retrieving. - - """ - - def __init__(self, - folder: str, - preprocessing: List[Callable], - batch_size: int, - file_types: List[str] = FILETYPES): - - """ Initialize a FolderImageLoader object. - - Args: - folder: Path of folder with images to load. The path has to exist, and has to contain at - least one image. - preprocessing: List of functions to use when processing the images before retrieving them. - batch_size: Number of images to retrieve each sample. - file_types: Files types to scan in the folder. Default list is :data:`~model_compression_toolkit.core.common.data_loader.FILETYPES` - - Examples: - - Instantiate a FolderImageLoader using a directory of images, that returns 10 images randomly each time it is sampled: - - >>> image_data_loader = FolderImageLoader('path/to/images/directory', preprocessing=[], batch_size=10) - >>> images = image_data_loader.sample() - - To preprocess the images before retrieving them, a list of preprocessing methods can be passed: - - >>> image_data_loader = FolderImageLoader('path/to/images/directory', preprocessing=[lambda x: (x-127.5)/127.5], batch_size=10) - - For the FolderImageLoader to scan only specific files extensions, a list of extensions can be passed: - - >>> image_data_loader = FolderImageLoader('path/to/images/directory', preprocessing=[], batch_size=10, file_types=['png']) - - """ - - self.folder = folder - self.image_list = [] - Logger.info(f"Starting Scanning Disk: {self.folder}") - for root, dirs, files in os.walk(self.folder): - for file in files: - file_type = file.split('.')[-1].lower() - if file_type in file_types: - self.image_list.append(os.path.join(root, file)) - self.n_files = len(self.image_list) - if self.n_files == 0: - Logger.critical(f"Expected files of type {FILETYPES}. No files of type {FILETYPES} were found.") # pragma: no cover - Logger.info(f"Finished Disk Scanning: Found {self.n_files} files") - self.preprocessing = preprocessing - self.batch_size = batch_size - - def _sample(self): - """ - Read batch_size random images from the image_list the FolderImageLoader holds. - Process them using the preprocessing list that was passed at initialization, and - prepare it for retrieving. - """ - - index = np.random.randint(0, self.n_files, self.batch_size) - image_list = [] - for i in index: - file = self.image_list[i] - img = np.uint8(np.array(Image.open(file).convert('RGB'))) - for p in self.preprocessing: # preprocess images - img = p(img) - image_list.append(img) - self.next_batch_data = np.stack(image_list, axis=0) - - def sample(self): - """ - - Returns: A sample of batch_size images from the folder the FolderImageLoader scanned. - - """ - - self._sample() - data = self.next_batch_data # get current data - return data diff --git a/tests/common_tests/function_tests/test_folder_image_loader.py b/tests/common_tests/function_tests/test_folder_image_loader.py deleted file mode 100644 index a7e53774d..000000000 --- a/tests/common_tests/function_tests/test_folder_image_loader.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2021 Sony Semiconductor Israel, Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - - -import numpy as np -import os -import shutil -import unittest -from PIL import Image -from pathlib import Path - -from model_compression_toolkit.core import FolderImageLoader - -img_path = "./test_data_loader_dir/test_img.jpeg" -img_shape = (224, 224, 3) -sample_batch = 5 - - -class TestFolderLoader(unittest.TestCase): - - @classmethod - def setUpClass(cls): - x = os.path.dirname(img_path) - Path(x).mkdir(parents=True, exist_ok=True) - im = Image.fromarray(np.random.random(img_shape).astype(np.uint8)) - im.save(img_path) - - @classmethod - def tearDownClass(cls): - shutil.rmtree(os.path.dirname(img_path)) - - def test_data_loader(self): - folder = os.path.dirname(img_path) - imgs_loader = FolderImageLoader(folder=folder, - preprocessing=[], - batch_size=sample_batch) - s = imgs_loader.sample() - self.assertTrue(isinstance(s, np.ndarray)) - self.assertTrue(s.shape == (sample_batch,) + img_shape) - - def test_empty_folder(self): - os.remove(os.path.abspath(img_path)) - folder = os.path.dirname(img_path) - with self.assertRaises(Exception): - FolderImageLoader(folder=folder, - preprocessing=[], - batch_size=sample_batch) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_suite.py b/tests/test_suite.py index 9fbad2886..7e912b787 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -19,7 +19,6 @@ import unittest from tests.common_tests.function_tests.test_collectors_manipulation import TestCollectorsManipulations -from tests.common_tests.function_tests.test_folder_image_loader import TestFolderLoader # ---------------- Individual test suites from tests.common_tests.function_tests.test_histogram_collector import TestHistogramCollector from tests.common_tests.function_tests.test_resource_utilization_object import TestResourceUtilizationObject @@ -97,7 +96,6 @@ suiteList = [] suiteList.append(unittest.TestLoader().loadTestsFromTestCase(TestHistogramCollector)) suiteList.append(unittest.TestLoader().loadTestsFromTestCase(TestCollectorsManipulations)) - suiteList.append(unittest.TestLoader().loadTestsFromTestCase(TestFolderLoader)) suiteList.append(unittest.TestLoader().loadTestsFromTestCase(TestThresholdSelection)) suiteList.append(unittest.TestLoader().loadTestsFromTestCase(TargetPlatformModelingTest)) suiteList.append(unittest.TestLoader().loadTestsFromTestCase(OpsetTest))