diff --git a/tutorials/mct_model_garden/evaluation_metrics/coco_evaluation.py b/tutorials/mct_model_garden/evaluation_metrics/coco_evaluation.py index c5b06ea69..3db113e47 100644 --- a/tutorials/mct_model_garden/evaluation_metrics/coco_evaluation.py +++ b/tutorials/mct_model_garden/evaluation_metrics/coco_evaluation.py @@ -19,6 +19,7 @@ from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval from typing import List, Dict, Tuple, Callable, Any +import random def coco80_to_coco91(x: np.ndarray) -> np.ndarray: @@ -217,6 +218,115 @@ def load_and_preprocess_image(image_path: str, preprocess: Callable) -> np.ndarr image = preprocess(image) return image + +class CocoDataset: + def __init__(self, dataset_folder: str, annotation_file: str, preprocess: Callable) -> Tuple: + """ + A dataset class for handling COCO dataset images and annotations. + + Args: + dataset_folder (str): The path to the folder containing COCO dataset images. + annotation_file (str): The path to the COCO annotation file in JSON format. + preprocess (Callable): A function for preprocessing images. + """ + self.dataset_folder = dataset_folder + self.preprocess = preprocess + + # Load COCO annotations from a JSON file (e.g., 'annotations.json') + with open(annotation_file, 'r') as f: + self.coco_annotations = json.load(f) + + # Initialize a dictionary to store annotations grouped by image ID + self.annotations_by_image = {} + + # Iterate through the annotations and group them by image ID + for annotation in self.coco_annotations['annotations']: + image_id = annotation['image_id'] + if image_id not in self.annotations_by_image: + self.annotations_by_image[image_id] = [] + self.annotations_by_image[image_id].append(annotation) + + # Initialize a list to collect images and annotations for the current batch + self.total_images = len(self.coco_annotations['images']) + + def __len__(self): + return self.total_images + + def __getitem__(self, item): + """ + Returns the preprocessed image and its corresponding annotations. + + Args: + item: Index of the item to retrieve. + + Returns: + Tuple containing the preprocessed image and its annotations. + """ + image_info = self.coco_annotations['images'][item] + image_id = image_info['id'] + image = load_and_preprocess_image(os.path.join(self.dataset_folder, image_info['file_name']), self.preprocess) + annotations = self.annotations_by_image.get(image_id, []) + if len(annotations) > 0: + annotations[0]['orig_img_dims'] = (image_info['height'], image_info['width']) + return image, annotations + + def sample(self): + """ + Samples a batch of images and their corresponding annotations from the dataset. + + Returns: + Tuple containing a batch of preprocessed images and their annotations. + """ + batch_images = [] + batch_annotations = [] + + # Sample random image indexes + random_idx = random.sample(range(self.total_images), self.batch_size) + + # Get the corresponding items from dataset + for idx in random_idx: + batch_images.append(self[idx][0]) + batch_annotations.append(self[idx][1]) + + return np.array(batch_images), batch_annotations + + +def coco_dataset_loader(dataset: CocoDataset, batch_size: int = 1) -> Tuple: + """ + Generator function for loading and preprocessing images and their annotations from a COCO-style dataset. + + Args: + dataset (CocoDataset): CocoDataset object. + batch_size (int): The desired batch size. + + Yields: + Tuple[numpy.ndarray, list]: A tuple containing a batch of images (as a NumPy array) and a list of annotations + for each image in the batch. + """ + batch_images = [] + batch_annotations = [] + + for image_count, item in enumerate(dataset): + image, annotations = item + + # Add the image and annotations to the current batch + batch_images.append(image) + batch_annotations.append(annotations) + + # Check if the current batch is of the desired batch size + if len(batch_images) == batch_size: + # Yield the current batch + yield np.array(batch_images), batch_annotations + + # Reset the batch lists for the next batch + batch_images = [] + batch_annotations = [] + + # After processing all images, yield any remaining images in the last batch + if len(batch_images) > 0 and (dataset.total_images == image_count + 1): + yield np.array(batch_images), batch_annotations + + def coco_dataset_generator(dataset_folder: str, annotation_file: str, preprocess: Callable, batch_size: int = 1) -> Tuple: """ Generator function for loading and preprocessing images and their annotations from a COCO-style dataset. @@ -296,17 +406,17 @@ def coco_evaluate(model: Any, preprocess: Callable, dataset_folder: str, annotat """ # Load COCO evaluation set - val_dataset = coco_dataset_generator(dataset_folder=dataset_folder, - annotation_file=annotation_file, - preprocess=preprocess, - batch_size=batch_size) - + coco_dataset = CocoDataset(dataset_folder=dataset_folder, + annotation_file=annotation_file, + preprocess=preprocess, + batch_size=batch_size) + coco_loader = coco_dataset_loader(coco_dataset, batch_size) # Initialize the evaluation metric object coco_metric = CocoEval(annotation_file, output_resize) # Iterate and the evaluation set - for batch_idx, (images, targets) in enumerate(val_dataset): + for batch_idx, (images, targets) in enumerate(coco_loader): # Run inference on the batch outputs = model(images) @@ -316,4 +426,4 @@ def coco_evaluate(model: Any, preprocess: Callable, dataset_folder: str, annotat if (batch_idx + 1) % 100 == 0: print(f'processed {(batch_idx + 1) * batch_size} images') - return coco_metric.result() \ No newline at end of file + return coco_metric.result() diff --git a/tutorials/mct_model_garden/models_keras/yolov8/yolov8.py b/tutorials/mct_model_garden/models_keras/yolov8/yolov8.py index 0ba7d0d8b..fb31c4bd5 100644 --- a/tutorials/mct_model_garden/models_keras/yolov8/yolov8.py +++ b/tutorials/mct_model_garden/models_keras/yolov8/yolov8.py @@ -266,8 +266,8 @@ def dist2bbox(points: tf.Tensor, distance: tf.Tensor) -> tf.Tensor: """ d0, d1, d2, d3 = tf.unstack(distance, 4, -1) a0, a1 = tf.unstack(points, 2, -1) - x1 = layers.ReLU()(tf.math.add(a0, -d0)) # Adding a relu in order to force unsigned output (which is expected in this case) - y1 = layers.ReLU()(tf.math.add(a1, -d1)) + x1 = layers.ReLU()(tf.math.subtract(a0, d0)) # Adding a relu in order to force unsigned output (which is expected in this case) + y1 = layers.ReLU()(tf.math.subtract(a1, d1)) x2 = layers.ReLU()(tf.math.add(a0, d2)) y2 = layers.ReLU()(tf.math.add(a1, d3)) return tf.stack([y1, x1, y2, x2], -1) diff --git a/tutorials/notebooks/keras/gptq/keras_yolov8n_for_imx500_gptq.ipynb b/tutorials/notebooks/keras/gptq/keras_yolov8n_for_imx500_gptq.ipynb new file mode 100644 index 000000000..8b6e1cc24 --- /dev/null +++ b/tutorials/notebooks/keras/gptq/keras_yolov8n_for_imx500_gptq.ipynb @@ -0,0 +1,363 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "4c261298-309f-41e8-9338-a5e205f09b05", + "metadata": {}, + "source": [ + "# YOLOv8n Object Detection Keras Model - Quantization for IMX500\n", + "\n", + "[Run this tutorial in Google Colab](https://colab.research.google.com/github/sony/model_optimization/blob/main/tutorials/notebooks/keras/ptq/keras_yolov8n_for_imx500.ipynb)\n", + "\n", + "## Overview\n", + "\n", + "In this tutorial, we will illustrate the process of preparing a pre-trained model and optimized it for deployment using MCT. Specifically, we will demonstrate how to download a pre-trained YOLOv8n model from the MCT Models Library, compress it, and make it deployment-ready using MCT's gradient-based post-training quantization techniques.\n", + "\n", + "This tutorial notebook is similiar to [keras_yolov8n_for_imx500 notebook](https://github.com/sony/model_optimization/blob/main/tutorials/notebooks/keras/ptq/keras_yolov8n_for_imx500.ipynb) but with additional weights optimization using Gradient-Based Post-Training quantization.\n", + "\n", + "We will use an existing pre-trained YOLOv8n model based on [Ultralytics](https://github.com/ultralytics/ultralytics). The model was slightly adjusted with integrated NMS layer. We will compress this model and evaluate the performance of the floating point model and the compressed model on COCO dataset.\n", + "\n", + "\n", + "## Summary\n", + "\n", + "In this tutorial we will cover:\n", + "\n", + "1. Gradient-Based Post-Training Quantization using MCT of Keras object detection model.\n", + "2. Data preparation - loading and preprocessing validation and representative datasets from COCO.\n", + "3. Accuracy evaluation of the floating-point and the quantized models." + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Setup\n", + "### Install the relevant packages" + ], + "metadata": { + "collapsed": false + }, + "id": "d74f9c855ec54081" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "!pip install -q tensorflow\n", + "!pip install -q pycocotools" + ], + "metadata": { + "collapsed": false + }, + "id": "7c7fa04c9903736f" + }, + { + "cell_type": "markdown", + "source": [ + " Clone a copy of the [MCT](https://github.com/sony/model_optimization) (Model Compression Toolkit) into your current directory. This step ensures that you have access to [MCT Models Library](https://github.com/sony/model_optimization/tree/main/tutorials/mct_model_garden) folder which contains all the necessary utility functions for this tutorial.\n", + " **It's important to note that we use the most up-to-date MCT code available.**" + ], + "metadata": { + "collapsed": false + }, + "id": "57717bc8f59a0d85" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "!git clone https://github.com/sony/model_optimization.git local_mct\n", + "!pip install -r ./local_mct/requirements.txt\n", + "import sys\n", + "sys.path.insert(0,\"./local_mct\")" + ], + "metadata": { + "collapsed": false + }, + "id": "9728247bc20d0600" + }, + { + "cell_type": "markdown", + "source": [ + "### Download COCO evaluation set" + ], + "metadata": { + "collapsed": false + }, + "id": "7a1038b9fd98bba2" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "!wget -nc http://images.cocodataset.org/annotations/annotations_trainval2017.zip\n", + "!unzip -q -o annotations_trainval2017.zip -d ./coco\n", + "!echo Done loading annotations\n", + "!wget -nc http://images.cocodataset.org/zips/val2017.zip\n", + "!unzip -q -o val2017.zip -d ./coco\n", + "!echo Done loading val2017 images" + ], + "metadata": { + "collapsed": false + }, + "id": "8bea492d71b4060f" + }, + { + "cell_type": "markdown", + "id": "084c2b8b-3175-4d46-a18a-7c4d8b6fcb38", + "metadata": {}, + "source": [ + "## Model Quantization\n", + "\n", + "### Download a Pre-Trained Model \n", + "\n", + "We begin by loading a pre-trained [YOLOv8n](https://huggingface.co/SSI-DNN/test_keras_yolov8n_640x640) model. This implementation is based on [Ultralytics](https://github.com/ultralytics/ultralytics) and includes a slightly modified version of yolov8 detection-head (mainly the box decoding part) that was adapted for model quantization. For further insights into the model's implementation details, please refer to [MCT Models Library - yolov8](https://github.com/sony/model_optimization/tree/main/tutorials/mct_model_garden/models_keras/yolov8). " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e8395b28-4732-4d18-b081-5d3bdf508691", + "metadata": {}, + "outputs": [], + "source": [ + "from huggingface_hub import from_pretrained_keras\n", + "\n", + "model = from_pretrained_keras('SSI-DNN/keras_yolov8n_640x640_pp')" + ] + }, + { + "cell_type": "markdown", + "id": "3cde2f8e-0642-4374-a1f4-df2775fe7767", + "metadata": {}, + "source": [ + "### Post training quantization using Model Compression Toolkit \n", + "\n", + "Now, we're all set to use MCT's post-training quantization. To begin, we'll define a representative dataset and proceed with the model quantization. Please note that, for demonstration purposes, we'll use the evaluation dataset as our representative dataset. We'll calibrate the model using 100 representative images, divided into 20 iterations of 'batch_size' images each. \n", + "\n", + "Additionally, to further compress the model's memory footprint, we will employ the mixed-precision quantization technique. This method allows each layer to be quantized with different precision options: 2, 4, and 8 bits, aligning with the imx500 target platform capabilities." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "import model_compression_toolkit as mct\n", + "from tutorials.mct_model_garden.evaluation_metrics.coco_evaluation import CocoDataset\n", + "from tutorials.mct_model_garden.models_keras.yolov8.yolov8_preprocess import yolov8_preprocess\n", + "from typing import Iterator, Tuple, List\n", + "\n", + "REPRESENTATIVE_DATASET_FOLDER = './coco/val2017/'\n", + "REPRESENTATIVE_DATASET_ANNOTATION_FILE = './coco/annotations/instances_val2017.json'\n", + "BATCH_SIZE = 5\n", + "n_iters = 10\n", + "n_mp_images = 5\n", + "n_gptq_epochs = 1000\n", + "\n", + "# Load representative dataset\n", + "representative_dataset = CocoDataset(dataset_folder=REPRESENTATIVE_DATASET_FOLDER,\n", + " annotation_file=REPRESENTATIVE_DATASET_ANNOTATION_FILE,\n", + " preprocess=yolov8_preprocess,\n", + " batch_size=BATCH_SIZE)\n", + "\n", + "# Define representative dataset generator\n", + "def get_representative_dataset(n_iter: int, dataset_loader: Iterator[Tuple]):\n", + " \"\"\"\n", + " This function creates a representative dataset generator. The generator yields numpy\n", + " arrays of batches of shape: [Batch, H, W ,C].\n", + " Args:\n", + " n_iter: number of iterations for MCT to calibrate on\n", + " Returns:\n", + " A representative dataset generator\n", + " \"\"\" \n", + " def representative_dataset() -> Iterator[List]:\n", + " ds_iter = iter(dataset_loader)\n", + " for _ in range(n_iter):\n", + " yield [next(ds_iter)[0]]\n", + "\n", + " return representative_dataset\n", + "\n", + "# Get representative dataset generator\n", + "representative_dataset_gen = get_representative_dataset(n_iters, representative_dataset)\n", + "\n", + "# Set IMX500-v1 TPC\n", + "tpc = mct.get_target_platform_capabilities(\"tensorflow\", 'imx500', target_platform_version='v1')\n", + "\n", + "# Specify the necessary configuration for the model mixed-precision quantization. We'll use a `n_mp_images` set of images and omit the hessian metric for mixed precision calculations.\n", + "mp_config = mct.core.MixedPrecisionQuantizationConfig(num_of_images=n_mp_images, use_hessian_based_scores=False)\n", + "\n", + "# Specify the necessary configuration for Gradient-Based PTQ.\n", + "gptq_config = mct.gptq.get_keras_gptq_config(n_epochs=n_gptq_epochs, use_hessian_based_weights=False)\n", + "\n", + "# Specify the MCT core configuration, include the mixed precision settings and enable `shift_negative_activation_correction` for optimal quantization of non-linear activations. \n", + "config = mct.core.CoreConfig(mixed_precision_config=mp_config,\n", + " quantization_config=mct.core.QuantizationConfig(shift_negative_activation_correction=True))\n", + "\n", + "# Define memory KPI for mixed precision weights quantization (we found 76% of 'standard' 8bits quantization to be optimal)\n", + "kpi_data = mct.core.keras_kpi_data(model,\n", + " representative_dataset_gen,\n", + " config,\n", + " target_platform_capabilities=tpc)\n", + "kpi = mct.core.KPI(kpi_data.weights_memory * 0.76)\n", + "config.mixed_precision_config.set_target_kpi(kpi)\n", + "\n", + "# Perform Gradient-Based Post Training Quantization\n", + "quant_model, quantization_info = mct.gptq.keras_gradient_post_training_quantization(\n", + " model,\n", + " representative_dataset_gen,\n", + " gptq_config=gptq_config,\n", + " core_config=config,\n", + " target_platform_capabilities=tpc)\n", + "\n", + "print('Quantized model is ready')" + ], + "metadata": { + "collapsed": false + }, + "id": "56393342-cecf-4f64-b9ca-2f515c765942" + }, + { + "cell_type": "markdown", + "source": [ + "### Model Export\n", + "\n", + "Now, we can export the quantized model, ready for deployment, into a `.keras` format file. Please ensure that the `save_model_path` has been set correctly. " + ], + "metadata": { + "collapsed": false + }, + "id": "3be2016acdc9da60" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "mct.exporter.keras_export_model(model=quant_model, save_model_path='./qmodel.keras')" + ], + "metadata": { + "collapsed": false + }, + "id": "72dd885c7b92fa93" + }, + { + "cell_type": "markdown", + "id": "015e760b-6555-45b4-aaf9-500e974c1d86", + "metadata": {}, + "source": [ + "## Evaluation on COCO dataset\n", + "\n", + "### Floating point model evaluation\n", + "Next, we evaluate the floating point model by using `cocoeval` library alongside additional dataset utilities. We can verify the mAP accuracy aligns with that of the original model. \n", + "Note that we set the \"batch_size\" to 5 and the preprocessing according to [Ultralytics](https://github.com/ultralytics/ultralytics).\n", + "Please ensure that the dataset path has been set correctly before running this code cell." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01e90967-594b-480f-b2e6-45e2c9ce9cee", + "metadata": {}, + "outputs": [], + "source": [ + "from tutorials.mct_model_garden.evaluation_metrics.coco_evaluation import coco_evaluate\n", + "\n", + "EVAL_DATASET_FOLDER = './coco/val2017'\n", + "EVAL_DATASET_ANNOTATION_FILE = './coco/annotations/instances_val2017.json'\n", + "INPUT_RESOLUTION = 640\n", + "\n", + "# Define resizing information to map between the model's output and the original image dimensions\n", + "output_resize = {'shape': (INPUT_RESOLUTION, INPUT_RESOLUTION), 'aspect_ratio_preservation': True}\n", + "\n", + "# Evaluate the model on coco\n", + "eval_results = coco_evaluate(model=model,\n", + " dataset_folder=EVAL_DATASET_FOLDER,\n", + " annotation_file=EVAL_DATASET_ANNOTATION_FILE,\n", + " preprocess=yolov8_preprocess,\n", + " output_resize=output_resize,\n", + " batch_size=BATCH_SIZE)\n", + "\n", + "# Print float model mAP results\n", + "print(\"Float model mAP: {:.4f}\".format(eval_results[0]))" + ] + }, + { + "cell_type": "markdown", + "id": "4fb6bffc-23d1-4852-8ec5-9007361c8eeb", + "metadata": {}, + "source": [ + "### Quantized model evaluation\n", + "Lastly, we can evaluate the performance of the quantized model. There is a slight decrease in performance that can be further mitigated by either expanding the representative dataset or employing MCT's advanced quantization methods, such as GPTQ (Gradient-Based/Enhanced Post Training Quantization)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8dc7b87c-a9f4-4568-885a-fe009c8f4e8f", + "metadata": {}, + "outputs": [], + "source": [ + "# Evaluate the model on coco\n", + "eval_results = coco_evaluate(model=quant_model,\n", + " dataset_folder=EVAL_DATASET_FOLDER,\n", + " annotation_file=EVAL_DATASET_ANNOTATION_FILE,\n", + " preprocess=yolov8_preprocess,\n", + " output_resize=output_resize,\n", + " batch_size=BATCH_SIZE)\n", + "\n", + "# Print quantized model mAP results\n", + "print(\"Quantized model mAP: {:.4f}\".format(eval_results[0]))" + ] + }, + { + "cell_type": "markdown", + "source": [ + "\\\n", + "Copyright 2024 Sony Semiconductor Israel, Inc. All rights reserved.\n", + "\n", + "Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "you may not use this file except in compliance with the License.\n", + "You may obtain a copy of the License at\n", + "\n", + " http://www.apache.org/licenses/LICENSE-2.0\n", + "\n", + "Unless required by applicable law or agreed to in writing, software\n", + "distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "See the License for the specific language governing permissions and\n", + "limitations under the License." + ], + "metadata": { + "collapsed": false + }, + "id": "6d93352843a27433" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + }, + "colab": { + "provenance": [] + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}