first commit

.gitignore

@@ -0,0 +1,105 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# IDE settings
+.vscode/

LICENSE

@@ -0,0 +1,20 @@
+MIT License 2020 Ceyda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice (including the next
+paragraph) shall be included in all copies or substantial portions of the
+Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,91 @@
+# Fast Bulk Image Integrity Checker
+Check for corrupted JPEG,PNG (and more) images in bulk using GPU jpeg decoding powered by NVIDIA DALI!
+
+Super fast compared to alternatives because it uses GPU decoding and checks in batches.
+
+# Requirements
+Depending on your cuda version:
+[NVIDIA DALI](https://docs.nvidia.com/deeplearning/dali/user-guide/docs/installation.html#id1)
+
+# Usage
+
+`pip3 install image-checker`
+
+or
+
+`git clone https://github.com/cceyda/image-checker.git`
+
+## CLI
+There is a CLI that takes a folder `--path` to scan for images with the given extensions `--ext`  
+outputs an `error.log`. Format of log file can be modified by `--log_conf` error handler using python logging [example](/master/dali_image_checker/logging_config.json). If you don't want to use gpu provide `--use_cpu` flag. Use `--recursive` to 
+traverse sub-folders.
+
+`image-check --path /mnt/data/dali_test/corrupt --recursive`
+
+```bash
+usage: image-checker [-h] [-p PATH] [-b BATCH_SIZE] [-g DEVICE_ID]
+                        [-ext EXTENSIONS] [-l LOG_CONG] [-r] [-d] [-c]
+
+Check a folder of images for broken/misidentified images
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -p PATH, --path PATH  Path for folder to be checked
+  -b BATCH_SIZE, --batch_size BATCH_SIZE
+                        Number of files checked per iteration (Recommend <100)
+  -g DEVICE_ID, --device_id DEVICE_ID
+                        Gpu ID
+  -ext EXTENSIONS, --extensions EXTENSIONS
+                        (comma delimited) list of extentions to test for (only types supported by DALI)
+  -l LOG_CONG, --log_conf LOG_CONF
+                        Config file path
+  -r, --recursive
+  -d, --debug
+  -c, --use_cpu
+```
+
+## Code
+
+```python
+from image_checker.checker import checker_batch,checker_single
+from image_checker.iterators import folder_iterator
+
+args = {
+     "path": "/mnt/data/images/main/",
+     "batch_size": 50,
+     "prefetch": 2,
+     "debug": False,
+     "extensions": ["jpeg", "jpg", "png"],
+     "recursive":False,
+     "log_cong":"logging_config.json",
+     "device":"mixed",
+     "device_id":0
+ }
+
+ds = folder_iterator(args["path"], args["extensions"], args["recursive"])
+bad_files=checker_batch(ds, args)
+
+```
+
+# FAQ
+
+- What kind of corrupted images will this catch?
+    - Images that can't be decoded by DALI.
+    - GIFs pretending to be JPEGs (with a jpg,jpeg extension)
+    - (Won't catch) files that can't be opened (TODO)
+    - (Won't catch) empty image files
+
+- Supported image types?
+
+        Same as [DALI supported formats](https://docs.nvidia.com/deeplearning/dali/user-guide/docs/supported_ops.html?highlight=supported%20image#nvidia.dali.ops.ImageDecoder): JPG, BMP, PNG, TIFF, PNM, PPM, PGM, PBM, JPEG 2000. Please note that GPU acceleration for JPEG 2000 decoding is only available for CUDA 11.
+
+- What is batch_size & prefetch?
+        DALI works with a batching+prefetching system. So batch_size * prefetch number of images are read at a time. If there is a corrupted file in the batch that batch is rechecked 1-by-1. So keep batch_size reasonable (0<100)
+
+
+Package versioning follows dali for major.minor (since it heavily depends on it), patch is this packages version changes.
+
+# Alternatives
+[check-media-integrity](https://github.com/ftarlao/check-media-integrity): supports more types but uses PIL thus slow.
+
+[Bad Peggy](https://github.com/llaith-oss/BadPeggy): Checks JPEG images, maybe detects more types of errors than this.

image_checker/DaliChecker.py

@@ -0,0 +1,38 @@
+import logging
+import os
+
+import nvidia.dali as dali
+import nvidia.dali.fn as fn
+import nvidia.dali.ops as ops
+from more_itertools import chunked
+from nvidia.dali.pipeline import Pipeline
+
+log = logging.getLogger("image_checker")
+
+# TODO
+# shard_id = torch.cuda.current_device()
+# num_shards = torch.cuda.device_count()
+shard_id = 0
+num_shards = 0
+local_rank = 0
+
+class DaliChecker:
+    def __init__(self, batch_size, prefetch=2, device="mixed", device_id=0):
+        log.debug("making checker")
+        self.batch_size = batch_size
+        self.prefetch = prefetch
+        self.device = device
+        self.device_id = device_id
+        self.make_pipe()
+        self.pipe.build()
+
+    def make_pipe(self):
+        log.debug("making pipe")
+        self.pipe = Pipeline(batch_size=self.batch_size, num_threads=2, device_id=self.device_id, prefetch_queue_depth=self.prefetch)
+        with self.pipe:
+            self.files = fn.external_source()
+            images = fn.image_decoder(self.files, device=self.device)
+            self.pipe.set_outputs(images)
+
+    def feed(self, images):
+        self.pipe.feed_input(self.files, images)

image_checker/checker.py

@@ -0,0 +1,126 @@
+import gc
+import logging
+import os
+import warnings
+import numpy as np
+
+# import torch
+from more_itertools import chunked, grouper
+from tqdm import tqdm
+from pathlib import Path
+from .DaliChecker import DaliChecker
+from .iterators import file_iterator
+
+# TODO handle warnings
+# TODO pretty tqdm
+# TODO better file list logging
+# TODO exact good image count (currently the batch is padded)
+
+log = logging.getLogger("image_checker")
+log_err = logging.getLogger("image_checker.errors")
+
+
+def batch(iterator, batch_size):
+    for x in grouper(iterator, batch_size, (np.zeros([1]), "fill.jpg")):
+        z = list(zip(*x))
+        yield list(z[0]), list(z[1])
+
+
+def checker_batch(ds, args):
+    if args["batch_size"]==1 and args["prefetch"]==1:
+        return checker_single(ds,args)
+    batched_iterator = batch(ds, args["batch_size"])
+    has_more = True
+    baddies = []
+    pbar = tqdm(total=0, desc=f"Good images (±{str(args['batch_size'])})", leave=True)
+    pbar_bad = tqdm(total=0, desc="Bad images", leave=True)
+
+    checker = DaliChecker(args["batch_size"], args["prefetch"],args["device"],args["device_id"])
+
+    def loop(checker):
+        nonlocal has_more
+        try:
+            # prefetch
+            image_paths = []
+            for _ in range(args["prefetch"]):
+                try:
+                    images, paths = next(batched_iterator)
+                    image_paths.extend(paths)
+                except StopIteration:
+                    has_more = False
+                checker.feed(images)
+            checker.pipe.run()
+            pbar.update(len(image_paths))
+            return None
+        except Exception as e:
+            #             log.error(e)
+            log.debug("Found bad files in batch")
+            log.debug("cleaning old pipe")
+            del checker
+            gc.collect()
+            #             torch.cuda.empty_cache()
+            log.debug("end clean")
+            return image_paths
+
+    while has_more:
+        potential_bad_paths = loop(checker)
+        if potential_bad_paths:
+            log.debug("Rescanning batch")
+            potential_bad_paths = list(map(Path, potential_bad_paths))
+            ds = file_iterator(potential_bad_paths, args["extensions"], False)
+            bad_paths = checker_single(ds,args, pbar, pbar_bad)
+            baddies.extend(bad_paths)
+            for x in bad_paths:
+                log_err.error(f"Bad file: {x}")
+            log.debug("Continuing Scan")
+            checker = DaliChecker(args["batch_size"], args["prefetch"],args["device"],args["device_id"])
+    log.info("End of Scan")
+    log.info(f"Found {len(baddies)} bad files")
+    return baddies
+
+
+def checker_single(ds,args, pbar=None, pbar_bad=None):
+    batched_iterator = batch(ds, 1)
+    has_more = True
+    bad_paths = []
+    if pbar is None:
+        pbar = tqdm(total=0, desc="Good images", leave=True)
+    if pbar_bad is None:
+        pbar_bad = tqdm(total=0, desc="Bad images", leave=True)
+
+    checker = DaliChecker(1, 1,args["device"],args["device_id"])
+
+    def loop(checker):
+        nonlocal has_more
+        try:
+            # prefetch
+            image_paths = []
+            for _ in range(1):
+                try:
+                    images, paths = next(batched_iterator)
+                    image_paths.extend(paths)
+                except StopIteration:
+                    has_more = False
+                checker.feed(images)
+            checker.pipe.run()
+            pbar.update(1)
+            return None
+        except Exception as e: 
+            bad_paths.extend(image_paths)
+            pbar_bad.update(1)
+            log.debug("Found bad file")
+            log.debug("cleaning old pipe")
+            del checker
+            gc.collect()
+            #             torch.cuda.empty_cache()
+            log.debug("end clean")
+            return True
+
+    while has_more:
+        reset = loop(checker)
+        if reset is not None:
+            checker = DaliChecker(1, 1,args["device"],args["device_id"])
+
+    del checker
+    gc.collect()
+    return bad_paths