From 381157f67f0d6e7273b8454234927511425cf11b Mon Sep 17 00:00:00 2001
From: Ofir Gordon <Ofir.Gordon@altair-semi.com>
Date: Thu, 21 Mar 2024 17:47:48 +0200
Subject: [PATCH] Remove FolderImageLoader
---
model_compression_toolkit/core/__init__.py | 1 -
.../core/common/data_loader.py | 108 ------------------
.../test_folder_image_loader.py | 63 ----------
tests/test_suite.py | 2 -
4 files changed, 174 deletions(-)
delete mode 100644 model_compression_toolkit/core/common/data_loader.py
delete mode 100644 tests/common_tests/function_tests/test_folder_image_loader.py
diff --git a/model_compression_toolkit/core/__init__.py b/model_compression_toolkit/core/__init__.py
index d477d8174..2b9bf4696 100644
--- a/model_compression_toolkit/core/__init__.py
+++ b/model_compression_toolkit/core/__init__.py
@@ -13,7 +13,6 @@
# limitations under the License.
# ==============================================================================
-from model_compression_toolkit.core.common.data_loader import FolderImageLoader
from model_compression_toolkit.core.common.framework_info import FrameworkInfo, ChannelAxis
from model_compression_toolkit.core.common import network_editors as network_editor
from model_compression_toolkit.core.common.quantization.debug_config import DebugConfig
diff --git a/model_compression_toolkit/core/common/data_loader.py b/model_compression_toolkit/core/common/data_loader.py
deleted file mode 100644
index fe4c09851..000000000
--- a/model_compression_toolkit/core/common/data_loader.py
+++ /dev/null
@@ -1,108 +0,0 @@
-# Copyright 2021 Sony Semiconductor Israel, Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-
-import os
-from typing import List, Callable
-
-import numpy as np
-from PIL import Image
-from model_compression_toolkit.logger import Logger
-
-#:
-FILETYPES = ['jpeg', 'jpg', 'bmp', 'png']
-
-
-class FolderImageLoader(object):
- """
-
- Class for images loading, processing and retrieving.
-
- """
-
- def __init__(self,
- folder: str,
- preprocessing: List[Callable],
- batch_size: int,
- file_types: List[str] = FILETYPES):
-
- """ Initialize a FolderImageLoader object.
-
- Args:
- folder: Path of folder with images to load. The path has to exist, and has to contain at
- least one image.
- preprocessing: List of functions to use when processing the images before retrieving them.
- batch_size: Number of images to retrieve each sample.
- file_types: Files types to scan in the folder. Default list is :data:`~model_compression_toolkit.core.common.data_loader.FILETYPES`
-
- Examples:
-
- Instantiate a FolderImageLoader using a directory of images, that returns 10 images randomly each time it is sampled:
-
- >>> image_data_loader = FolderImageLoader('path/to/images/directory', preprocessing=[], batch_size=10)
- >>> images = image_data_loader.sample()
-
- To preprocess the images before retrieving them, a list of preprocessing methods can be passed:
-
- >>> image_data_loader = FolderImageLoader('path/to/images/directory', preprocessing=[lambda x: (x-127.5)/127.5], batch_size=10)
-
- For the FolderImageLoader to scan only specific files extensions, a list of extensions can be passed:
-
- >>> image_data_loader = FolderImageLoader('path/to/images/directory', preprocessing=[], batch_size=10, file_types=['png'])
-
- """
-
- self.folder = folder
- self.image_list = []
- Logger.info(f"Starting Scanning Disk: {self.folder}")
- for root, dirs, files in os.walk(self.folder):
- for file in files:
- file_type = file.split('.')[-1].lower()
- if file_type in file_types:
- self.image_list.append(os.path.join(root, file))
- self.n_files = len(self.image_list)
- if self.n_files == 0:
- Logger.critical(f"Expected files of type {FILETYPES}. No files of type {FILETYPES} were found.") # pragma: no cover
- Logger.info(f"Finished Disk Scanning: Found {self.n_files} files")
- self.preprocessing = preprocessing
- self.batch_size = batch_size
-
- def _sample(self):
- """
- Read batch_size random images from the image_list the FolderImageLoader holds.
- Process them using the preprocessing list that was passed at initialization, and
- prepare it for retrieving.
- """
-
- index = np.random.randint(0, self.n_files, self.batch_size)
- image_list = []
- for i in index:
- file = self.image_list[i]
- img = np.uint8(np.array(Image.open(file).convert('RGB')))
- for p in self.preprocessing: # preprocess images
- img = p(img)
- image_list.append(img)
- self.next_batch_data = np.stack(image_list, axis=0)
-
- def sample(self):
- """
-
- Returns: A sample of batch_size images from the folder the FolderImageLoader scanned.
-
- """
-
- self._sample()
- data = self.next_batch_data # get current data
- return data
diff --git a/tests/common_tests/function_tests/test_folder_image_loader.py b/tests/common_tests/function_tests/test_folder_image_loader.py
deleted file mode 100644
index a7e53774d..000000000
--- a/tests/common_tests/function_tests/test_folder_image_loader.py
+++ /dev/null
@@ -1,63 +0,0 @@
-# Copyright 2021 Sony Semiconductor Israel, Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-
-import numpy as np
-import os
-import shutil
-import unittest
-from PIL import Image
-from pathlib import Path
-
-from model_compression_toolkit.core import FolderImageLoader
-
-img_path = "./test_data_loader_dir/test_img.jpeg"
-img_shape = (224, 224, 3)
-sample_batch = 5
-
-
-class TestFolderLoader(unittest.TestCase):
-
- @classmethod
- def setUpClass(cls):
- x = os.path.dirname(img_path)
- Path(x).mkdir(parents=True, exist_ok=True)
- im = Image.fromarray(np.random.random(img_shape).astype(np.uint8))
- im.save(img_path)
-
- @classmethod
- def tearDownClass(cls):
- shutil.rmtree(os.path.dirname(img_path))
-
- def test_data_loader(self):
- folder = os.path.dirname(img_path)
- imgs_loader = FolderImageLoader(folder=folder,
- preprocessing=[],
- batch_size=sample_batch)
- s = imgs_loader.sample()
- self.assertTrue(isinstance(s, np.ndarray))
- self.assertTrue(s.shape == (sample_batch,) + img_shape)
-
- def test_empty_folder(self):
- os.remove(os.path.abspath(img_path))
- folder = os.path.dirname(img_path)
- with self.assertRaises(Exception):
- FolderImageLoader(folder=folder,
- preprocessing=[],
- batch_size=sample_batch)
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/tests/test_suite.py b/tests/test_suite.py
index 9fbad2886..7e912b787 100644
--- a/tests/test_suite.py
+++ b/tests/test_suite.py
@@ -19,7 +19,6 @@
import unittest
from tests.common_tests.function_tests.test_collectors_manipulation import TestCollectorsManipulations
-from tests.common_tests.function_tests.test_folder_image_loader import TestFolderLoader
# ---------------- Individual test suites
from tests.common_tests.function_tests.test_histogram_collector import TestHistogramCollector
from tests.common_tests.function_tests.test_resource_utilization_object import TestResourceUtilizationObject
@@ -97,7 +96,6 @@
suiteList = []
suiteList.append(unittest.TestLoader().loadTestsFromTestCase(TestHistogramCollector))
suiteList.append(unittest.TestLoader().loadTestsFromTestCase(TestCollectorsManipulations))
- suiteList.append(unittest.TestLoader().loadTestsFromTestCase(TestFolderLoader))
suiteList.append(unittest.TestLoader().loadTestsFromTestCase(TestThresholdSelection))
suiteList.append(unittest.TestLoader().loadTestsFromTestCase(TargetPlatformModelingTest))
suiteList.append(unittest.TestLoader().loadTestsFromTestCase(OpsetTest))