guides/logit_checking/k400-logit-matching-torch-vs-keras-cv-backbone.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d2a41fe2",
   "metadata": {
    "papermill": {
     "duration": 0.008476,
     "end_time": "2024-03-31T16:56:33.025353",
     "exception": false,
     "start_time": "2024-03-31T16:56:33.016877",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# About\n",
    "\n",
    "This notebook demonstrates the identical results of vidoe swin transformer, imported from `keras-cv` and `torch-vision` libraries. The `keras-cv` version of video swin is implemented in `keras 3`, makes it able to run in multiple backend, i.e. `tensorflow`, `torch`, and `jax`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ab79b749",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2024-03-31T16:56:33.043892Z",
     "iopub.status.busy": "2024-03-31T16:56:33.043098Z",
     "iopub.status.idle": "2024-03-31T16:56:34.096735Z",
     "shell.execute_reply": "2024-03-31T16:56:34.095522Z"
    },
    "papermill": {
     "duration": 1.066465,
     "end_time": "2024-03-31T16:56:34.099589",
     "exception": false,
     "start_time": "2024-03-31T16:56:33.033124",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import os\n",
    "import warnings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "97b1c2e6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-31T16:56:34.116409Z",
     "iopub.status.busy": "2024-03-31T16:56:34.115809Z",
     "iopub.status.idle": "2024-03-31T16:56:34.121637Z",
     "shell.execute_reply": "2024-03-31T16:56:34.120415Z"
    },
    "papermill": {
     "duration": 0.016785,
     "end_time": "2024-03-31T16:56:34.123892",
     "exception": false,
     "start_time": "2024-03-31T16:56:34.107107",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "os.environ[\"KERAS_BACKEND\"] = \"torch\" # 'torch', 'tensorflow', 'jax'\n",
    "warnings.simplefilter(action=\"ignore\")\n",
    "os.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"3\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f3fe802e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-31T16:56:34.139998Z",
     "iopub.status.busy": "2024-03-31T16:56:34.139606Z",
     "iopub.status.idle": "2024-03-31T16:57:12.230426Z",
     "shell.execute_reply": "2024-03-31T16:57:12.229322Z"
    },
    "papermill": {
     "duration": 38.102278,
     "end_time": "2024-03-31T16:57:12.233346",
     "exception": false,
     "start_time": "2024-03-31T16:56:34.131068",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'keras-cv'...\r\n",
      "remote: Enumerating objects: 13766, done.\u001b[K\r\n",
      "remote: Counting objects: 100% (1903/1903), done.\u001b[K\r\n",
      "remote: Compressing objects: 100% (760/760), done.\u001b[K\r\n",
      "remote: Total 13766 (delta 1325), reused 1617 (delta 1127), pack-reused 11863\u001b[K\r\n",
      "Receiving objects: 100% (13766/13766), 25.64 MiB | 27.10 MiB/s, done.\r\n",
      "Resolving deltas: 100% (9776/9776), done.\r\n",
      "/kaggle/working/keras-cv\n"
     ]
    }
   ],
   "source": [
    "!git clone --branch video_swin https://github.com/innat/keras-cv.git\n",
    "%cd keras-cv\n",
    "!pip install -q -e ."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d595aeb",
   "metadata": {
    "papermill": {
     "duration": 0.009127,
     "end_time": "2024-03-31T16:57:12.252522",
     "exception": false,
     "start_time": "2024-03-31T16:57:12.243395",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# KerasCV: Video Swin : Pretrained: ImageNet 1K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "27594587",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-31T16:57:12.273806Z",
     "iopub.status.busy": "2024-03-31T16:57:12.273338Z",
     "iopub.status.idle": "2024-03-31T16:57:36.167429Z",
     "shell.execute_reply": "2024-03-31T16:57:36.166267Z"
    },
    "papermill": {
     "duration": 23.908121,
     "end_time": "2024-03-31T16:57:36.169910",
     "exception": false,
     "start_time": "2024-03-31T16:57:12.261789",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3.0.5'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import keras\n",
    "from keras import ops\n",
    "from keras_cv.models import VideoSwinBackbone\n",
    "from keras_cv.models import VideoClassifier\n",
    "\n",
    "keras.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0b342441",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-31T16:57:36.193323Z",
     "iopub.status.busy": "2024-03-31T16:57:36.191636Z",
     "iopub.status.idle": "2024-03-31T16:57:38.404512Z",
     "shell.execute_reply": "2024-03-31T16:57:38.403189Z"
    },
    "papermill": {
     "duration": 2.227219,
     "end_time": "2024-03-31T16:57:38.407364",
     "exception": false,
     "start_time": "2024-03-31T16:57:36.180145",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-03-31 16:57:37--  https://github.com/innat/VideoSwin/releases/download/v2.0/videoswin_tiny_kinetics400.weights.h5\r\n",
      "Resolving github.com (github.com)... 140.82.121.3\r\n",
      "Connecting to github.com (github.com)|140.82.121.3|:443... connected.\r\n",
      "HTTP request sent, awaiting response... 302 Found\r\n",
      "Location: https://objects.githubusercontent.com/github-production-release-asset-2e65be/697696973/5153e756-236b-41e7-a602-ab854a57034f?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20240331%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240331T165737Z&X-Amz-Expires=300&X-Amz-Signature=6188cd48f4cffee2ddbc4c5a3c8e4701e15588c12e3446ed5b8a52a002072164&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=697696973&response-content-disposition=attachment%3B%20filename%3Dvideoswin_tiny_kinetics400.weights.h5&response-content-type=application%2Foctet-stream [following]\r\n",
      "--2024-03-31 16:57:37--  https://objects.githubusercontent.com/github-production-release-asset-2e65be/697696973/5153e756-236b-41e7-a602-ab854a57034f?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20240331%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240331T165737Z&X-Amz-Expires=300&X-Amz-Signature=6188cd48f4cffee2ddbc4c5a3c8e4701e15588c12e3446ed5b8a52a002072164&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=697696973&response-content-disposition=attachment%3B%20filename%3Dvideoswin_tiny_kinetics400.weights.h5&response-content-type=application%2Foctet-stream\r\n",
      "Resolving objects.githubusercontent.com (objects.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\r\n",
      "Connecting to objects.githubusercontent.com (objects.githubusercontent.com)|185.199.108.133|:443... connected.\r\n",
      "HTTP request sent, awaiting response... 200 OK\r\n",
      "Length: 111855496 (107M) [application/octet-stream]\r\n",
      "Saving to: 'videoswin_tiny_kinetics400.weights.h5'\r\n",
      "\r\n",
      "videoswin_tiny_kine 100%[===================>] 106.67M   230MB/s    in 0.5s    \r\n",
      "\r\n",
      "2024-03-31 16:57:38 (230 MB/s) - 'videoswin_tiny_kinetics400.weights.h5' saved [111855496/111855496]\r\n",
      "\r\n"
     ]
    }
   ],
   "source": [
    "!wget https://github.com/innat/VideoSwin/releases/download/v2.0/videoswin_tiny_kinetics400.weights.h5\n",
    "\n",
    "def vswin_tiny():\n",
    "    backbone=VideoSwinBackbone(\n",
    "        input_shape=(32, 224, 224, 3), \n",
    "        embed_dim=96,\n",
    "        depths=[2, 2, 6, 2],\n",
    "        num_heads=[3, 6, 12, 24],\n",
    "        include_rescaling=False, \n",
    "    )\n",
    "    backbone.load_weights(\n",
    "        'videoswin_tiny_kinetics400.weights.h5'\n",
    "    )\n",
    "    return backbone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "314cb4d0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-31T16:57:38.430540Z",
     "iopub.status.busy": "2024-03-31T16:57:38.430074Z",
     "iopub.status.idle": "2024-03-31T16:57:40.912946Z",
     "shell.execute_reply": "2024-03-31T16:57:40.911656Z"
    },
    "papermill": {
     "duration": 2.498091,
     "end_time": "2024-03-31T16:57:40.915960",
     "exception": false,
     "start_time": "2024-03-31T16:57:38.417869",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-03-31 16:57:39--  https://github.com/innat/VideoSwin/releases/download/v2.0/videoswin_small_kinetics400.weights.h5\r\n",
      "Resolving github.com (github.com)... 140.82.121.4\r\n",
      "Connecting to github.com (github.com)|140.82.121.4|:443... connected.\r\n",
      "HTTP request sent, awaiting response... 302 Found\r\n",
      "Location: https://objects.githubusercontent.com/github-production-release-asset-2e65be/697696973/898b24c6-f517-4b01-872b-8f19acd2c54d?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20240331%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240331T165739Z&X-Amz-Expires=300&X-Amz-Signature=ebd091cee3f64c57654966b81170827c3c667d8bab72a6b4969a987561926f0f&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=697696973&response-content-disposition=attachment%3B%20filename%3Dvideoswin_small_kinetics400.weights.h5&response-content-type=application%2Foctet-stream [following]\r\n",
      "--2024-03-31 16:57:39--  https://objects.githubusercontent.com/github-production-release-asset-2e65be/697696973/898b24c6-f517-4b01-872b-8f19acd2c54d?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20240331%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240331T165739Z&X-Amz-Expires=300&X-Amz-Signature=ebd091cee3f64c57654966b81170827c3c667d8bab72a6b4969a987561926f0f&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=697696973&response-content-disposition=attachment%3B%20filename%3Dvideoswin_small_kinetics400.weights.h5&response-content-type=application%2Foctet-stream\r\n",
      "Resolving objects.githubusercontent.com (objects.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\r\n",
      "Connecting to objects.githubusercontent.com (objects.githubusercontent.com)|185.199.108.133|:443... connected.\r\n",
      "HTTP request sent, awaiting response... 200 OK\r\n",
      "Length: 198902800 (190M) [application/octet-stream]\r\n",
      "Saving to: 'videoswin_small_kinetics400.weights.h5'\r\n",
      "\r\n",
      "videoswin_small_kin 100%[===================>] 189.69M   237MB/s    in 0.8s    \r\n",
      "\r\n",
      "2024-03-31 16:57:40 (237 MB/s) - 'videoswin_small_kinetics400.weights.h5' saved [198902800/198902800]\r\n",
      "\r\n"
     ]
    }
   ],
   "source": [
    "!wget https://github.com/innat/VideoSwin/releases/download/v2.0/videoswin_small_kinetics400.weights.h5\n",
    "\n",
    "def vswin_small():\n",
    "    backbone=VideoSwinBackbone(\n",
    "        input_shape=(32, 224, 224, 3), \n",
    "        embed_dim=96,\n",
    "        depths=[2, 2, 18, 2],\n",
    "        num_heads=[3, 6, 12, 24],\n",
    "        include_rescaling=False, \n",
    "    )\n",
    "    backbone.load_weights(\n",
    "        'videoswin_small_kinetics400.weights.h5'\n",
    "    )\n",
    "    return backbone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "30973043",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-31T16:57:40.941747Z",
     "iopub.status.busy": "2024-03-31T16:57:40.941255Z",
     "iopub.status.idle": "2024-03-31T16:57:44.047679Z",
     "shell.execute_reply": "2024-03-31T16:57:44.046374Z"
    },
    "papermill": {
     "duration": 3.123539,
     "end_time": "2024-03-31T16:57:44.050740",
     "exception": false,
     "start_time": "2024-03-31T16:57:40.927201",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-03-31 16:57:41--  https://github.com/innat/VideoSwin/releases/download/v2.0/videoswin_base_kinetics400.weights.h5\r\n",
      "Resolving github.com (github.com)... 140.82.121.4\r\n",
      "Connecting to github.com (github.com)|140.82.121.4|:443... connected.\r\n",
      "HTTP request sent, awaiting response... 302 Found\r\n",
      "Location: https://objects.githubusercontent.com/github-production-release-asset-2e65be/697696973/d5a7b9f0-78b7-4151-b1d3-ddba5c66c7c1?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20240331%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240331T165742Z&X-Amz-Expires=300&X-Amz-Signature=4dbdc130edd48a081675524290e07e70aa8d48bcd5862bb80d8f15b72903abdd&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=697696973&response-content-disposition=attachment%3B%20filename%3Dvideoswin_base_kinetics400.weights.h5&response-content-type=application%2Foctet-stream [following]\r\n",
      "--2024-03-31 16:57:42--  https://objects.githubusercontent.com/github-production-release-asset-2e65be/697696973/d5a7b9f0-78b7-4151-b1d3-ddba5c66c7c1?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20240331%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240331T165742Z&X-Amz-Expires=300&X-Amz-Signature=4dbdc130edd48a081675524290e07e70aa8d48bcd5862bb80d8f15b72903abdd&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=697696973&response-content-disposition=attachment%3B%20filename%3Dvideoswin_base_kinetics400.weights.h5&response-content-type=application%2Foctet-stream\r\n",
      "Resolving objects.githubusercontent.com (objects.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\r\n",
      "Connecting to objects.githubusercontent.com (objects.githubusercontent.com)|185.199.108.133|:443... connected.\r\n",
      "HTTP request sent, awaiting response... 200 OK\r\n",
      "Length: 351381896 (335M) [application/octet-stream]\r\n",
      "Saving to: 'videoswin_base_kinetics400.weights.h5'\r\n",
      "\r\n",
      "videoswin_base_kine 100%[===================>] 335.10M   241MB/s    in 1.4s    \r\n",
      "\r\n",
      "2024-03-31 16:57:43 (241 MB/s) - 'videoswin_base_kinetics400.weights.h5' saved [351381896/351381896]\r\n",
      "\r\n"
     ]
    }
   ],
   "source": [
    "!wget https://github.com/innat/VideoSwin/releases/download/v2.0/videoswin_base_kinetics400.weights.h5\n",
    "\n",
    "def vswin_base():\n",
    "    backbone=VideoSwinBackbone(\n",
    "        input_shape=(32, 224, 224, 3), \n",
    "        embed_dim=128,\n",
    "        depths=[2, 2, 18, 2],\n",
    "        num_heads=[4, 8, 16, 32],\n",
    "        include_rescaling=False, \n",
    "    )\n",
    "    backbone.load_weights(\n",
    "        'videoswin_base_kinetics400.weights.h5'\n",
    "    )\n",
    "    return backbone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e3bdd3ec",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-31T16:57:44.077645Z",
     "iopub.status.busy": "2024-03-31T16:57:44.077210Z",
     "iopub.status.idle": "2024-03-31T16:57:51.781985Z",
     "shell.execute_reply": "2024-03-31T16:57:51.780643Z"
    },
    "papermill": {
     "duration": 7.721169,
     "end_time": "2024-03-31T16:57:51.784182",
     "exception": false,
     "start_time": "2024-03-31T16:57:44.063013",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"video_swin_backbone\"</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1mModel: \"video_swin_backbone\"\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> Layer (type)                    </span>┃<span style=\"font-weight: bold\"> Output Shape           </span>┃<span style=\"font-weight: bold\">       Param # </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ videos (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)             │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">224</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">224</span>,   │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
       "│                                 │ <span style=\"color: #00af00; text-decoration-color: #00af00\">3</span>)                     │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ videoswin_patching_and_embeddi… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">56</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">56</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">96</span>) │         <span style=\"color: #00af00; text-decoration-color: #00af00\">9,504</span> │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">VideoSwinPatchingAndEmbedding</span>) │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ pos_drop (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>)              │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">56</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">56</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">96</span>) │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ videoswin_basic_layer_1         │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">28</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">28</span>,     │       <span style=\"color: #00af00; text-decoration-color: #00af00\">313,386</span> │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">VideoSwinBasicLayer</span>)           │ <span style=\"color: #00af00; text-decoration-color: #00af00\">192</span>)                   │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ videoswin_basic_layer_2         │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">14</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">14</span>,     │     <span style=\"color: #00af00; text-decoration-color: #00af00\">1,216,596</span> │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">VideoSwinBasicLayer</span>)           │ <span style=\"color: #00af00; text-decoration-color: #00af00\">384</span>)                   │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ videoswin_basic_layer_3         │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">7</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">7</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">768</span>)  │    <span style=\"color: #00af00; text-decoration-color: #00af00\">12,012,024</span> │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">VideoSwinBasicLayer</span>)           │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ videoswin_basic_layer_4         │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">7</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">7</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">768</span>)  │    <span style=\"color: #00af00; text-decoration-color: #00af00\">14,297,424</span> │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">VideoSwinBasicLayer</span>)           │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ videoswin_top_norm              │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">7</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">7</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">768</span>)  │         <span style=\"color: #00af00; text-decoration-color: #00af00\">1,536</span> │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">LayerNormalization</span>)            │                        │               │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                   \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape          \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ videos (\u001b[38;5;33mInputLayer\u001b[0m)             │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m32\u001b[0m, \u001b[38;5;34m224\u001b[0m, \u001b[38;5;34m224\u001b[0m,   │             \u001b[38;5;34m0\u001b[0m │\n",
       "│                                 │ \u001b[38;5;34m3\u001b[0m)                     │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ videoswin_patching_and_embeddi… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m16\u001b[0m, \u001b[38;5;34m56\u001b[0m, \u001b[38;5;34m56\u001b[0m, \u001b[38;5;34m96\u001b[0m) │         \u001b[38;5;34m9,504\u001b[0m │\n",
       "│ (\u001b[38;5;33mVideoSwinPatchingAndEmbedding\u001b[0m) │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ pos_drop (\u001b[38;5;33mDropout\u001b[0m)              │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m16\u001b[0m, \u001b[38;5;34m56\u001b[0m, \u001b[38;5;34m56\u001b[0m, \u001b[38;5;34m96\u001b[0m) │             \u001b[38;5;34m0\u001b[0m │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ videoswin_basic_layer_1         │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m16\u001b[0m, \u001b[38;5;34m28\u001b[0m, \u001b[38;5;34m28\u001b[0m,     │       \u001b[38;5;34m313,386\u001b[0m │\n",
       "│ (\u001b[38;5;33mVideoSwinBasicLayer\u001b[0m)           │ \u001b[38;5;34m192\u001b[0m)                   │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ videoswin_basic_layer_2         │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m16\u001b[0m, \u001b[38;5;34m14\u001b[0m, \u001b[38;5;34m14\u001b[0m,     │     \u001b[38;5;34m1,216,596\u001b[0m │\n",
       "│ (\u001b[38;5;33mVideoSwinBasicLayer\u001b[0m)           │ \u001b[38;5;34m384\u001b[0m)                   │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ videoswin_basic_layer_3         │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m16\u001b[0m, \u001b[38;5;34m7\u001b[0m, \u001b[38;5;34m7\u001b[0m, \u001b[38;5;34m768\u001b[0m)  │    \u001b[38;5;34m12,012,024\u001b[0m │\n",
       "│ (\u001b[38;5;33mVideoSwinBasicLayer\u001b[0m)           │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ videoswin_basic_layer_4         │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m16\u001b[0m, \u001b[38;5;34m7\u001b[0m, \u001b[38;5;34m7\u001b[0m, \u001b[38;5;34m768\u001b[0m)  │    \u001b[38;5;34m14,297,424\u001b[0m │\n",
       "│ (\u001b[38;5;33mVideoSwinBasicLayer\u001b[0m)           │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ videoswin_top_norm              │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m16\u001b[0m, \u001b[38;5;34m7\u001b[0m, \u001b[38;5;34m7\u001b[0m, \u001b[38;5;34m768\u001b[0m)  │         \u001b[38;5;34m1,536\u001b[0m │\n",
       "│ (\u001b[38;5;33mLayerNormalization\u001b[0m)            │                        │               │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">27,850,470</span> (106.24 MB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m27,850,470\u001b[0m (106.24 MB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">27,850,470</span> (106.24 MB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m27,850,470\u001b[0m (106.24 MB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "keras_models = [vswin_tiny(), vswin_small(), vswin_base()]\n",
    "keras_models[0].summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93c03854",
   "metadata": {
    "papermill": {
     "duration": 0.012956,
     "end_time": "2024-03-31T16:57:51.810513",
     "exception": false,
     "start_time": "2024-03-31T16:57:51.797557",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# TorchVision: Video Swin : Pretrained: ImageNet 1K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "649b28ba",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-31T16:57:51.839013Z",
     "iopub.status.busy": "2024-03-31T16:57:51.838285Z",
     "iopub.status.idle": "2024-03-31T16:57:52.183960Z",
     "shell.execute_reply": "2024-03-31T16:57:52.182857Z"
    },
    "papermill": {
     "duration": 0.362617,
     "end_time": "2024-03-31T16:57:52.186447",
     "exception": false,
     "start_time": "2024-03-31T16:57:51.823830",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "from torchinfo import summary\n",
    "from torchvision.models.video import Swin3D_T_Weights, Swin3D_S_Weights, Swin3D_B_Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b6cb791e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-31T16:57:52.216057Z",
     "iopub.status.busy": "2024-03-31T16:57:52.215643Z",
     "iopub.status.idle": "2024-03-31T16:57:52.224782Z",
     "shell.execute_reply": "2024-03-31T16:57:52.223537Z"
    },
    "papermill": {
     "duration": 0.02691,
     "end_time": "2024-03-31T16:57:52.227103",
     "exception": false,
     "start_time": "2024-03-31T16:57:52.200193",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def exclude_top(model):\n",
    "    backbone = torch.nn.Sequential(\n",
    "        *(list(model.children())[:-2])\n",
    "    )\n",
    "    backbone.eval()\n",
    "    return backbone\n",
    "\n",
    "def torch_vswin_tiny():\n",
    "    torch_model = torchvision.models.video.swin3d_t(\n",
    "        weights=Swin3D_T_Weights.KINETICS400_V1\n",
    "    ).eval()\n",
    "    backbone = exclude_top(torch_model)\n",
    "    return backbone\n",
    "\n",
    "def torch_vswin_small():\n",
    "    torch_model = torchvision.models.video.swin3d_s(\n",
    "        weights=Swin3D_S_Weights.KINETICS400_V1\n",
    "    ).eval()\n",
    "    backbone = exclude_top(torch_model)\n",
    "    return backbone\n",
    "\n",
    "def torch_vswin_base():\n",
    "    torch_model = torchvision.models.video.swin3d_b(\n",
    "        weights=Swin3D_B_Weights.KINETICS400_V1\n",
    "    ).eval()\n",
    "    backbone = exclude_top(torch_model)\n",
    "    return backbone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0d69b1bc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-31T16:57:52.256949Z",
     "iopub.status.busy": "2024-03-31T16:57:52.256159Z",
     "iopub.status.idle": "2024-03-31T16:58:14.269808Z",
     "shell.execute_reply": "2024-03-31T16:58:14.268431Z"
    },
    "papermill": {
     "duration": 22.031378,
     "end_time": "2024-03-31T16:58:14.272471",
     "exception": false,
     "start_time": "2024-03-31T16:57:52.241093",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://download.pytorch.org/models/swin3d_t-7615ae03.pth\" to /root/.cache/torch/hub/checkpoints/swin3d_t-7615ae03.pth\n",
      "100%|██████████| 122M/122M [00:00<00:00, 131MB/s]\n",
      "Downloading: \"https://download.pytorch.org/models/swin3d_s-da41c237.pth\" to /root/.cache/torch/hub/checkpoints/swin3d_s-da41c237.pth\n",
      "100%|██████████| 218M/218M [00:06<00:00, 37.7MB/s]\n",
      "Downloading: \"https://download.pytorch.org/models/swin3d_b_1k-24f7c7c6.pth\" to /root/.cache/torch/hub/checkpoints/swin3d_b_1k-24f7c7c6.pth\n",
      "100%|██████████| 364M/364M [00:02<00:00, 135MB/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "=========================================================================================================\n",
       "Layer (type:depth-idx)                                  Output Shape              Param #\n",
       "=========================================================================================================\n",
       "Sequential                                              [1, 16, 7, 7, 768]        --\n",
       "├─PatchEmbed3d: 1-1                                     [1, 16, 56, 56, 96]       --\n",
       "│    └─Conv3d: 2-1                                      [1, 96, 16, 56, 56]       9,312\n",
       "│    └─LayerNorm: 2-2                                   [1, 16, 56, 56, 96]       192\n",
       "├─Dropout: 1-2                                          [1, 16, 56, 56, 96]       --\n",
       "├─Sequential: 1-3                                       [1, 16, 7, 7, 768]        --\n",
       "│    └─Sequential: 2-3                                  [1, 16, 56, 56, 96]       --\n",
       "│    │    └─SwinTransformerBlock: 3-1                   [1, 16, 56, 56, 96]       119,445\n",
       "│    │    └─SwinTransformerBlock: 3-2                   [1, 16, 56, 56, 96]       119,445\n",
       "│    └─PatchMerging: 2-4                                [1, 16, 28, 28, 192]      --\n",
       "│    │    └─LayerNorm: 3-3                              [1, 16, 28, 28, 384]      768\n",
       "│    │    └─Linear: 3-4                                 [1, 16, 28, 28, 192]      73,728\n",
       "│    └─Sequential: 2-5                                  [1, 16, 28, 28, 192]      --\n",
       "│    │    └─SwinTransformerBlock: 3-5                   [1, 16, 28, 28, 192]      460,074\n",
       "│    │    └─SwinTransformerBlock: 3-6                   [1, 16, 28, 28, 192]      460,074\n",
       "│    └─PatchMerging: 2-6                                [1, 16, 14, 14, 384]      --\n",
       "│    │    └─LayerNorm: 3-7                              [1, 16, 14, 14, 768]      1,536\n",
       "│    │    └─Linear: 3-8                                 [1, 16, 14, 14, 384]      294,912\n",
       "│    └─Sequential: 2-7                                  [1, 16, 14, 14, 384]      --\n",
       "│    │    └─SwinTransformerBlock: 3-9                   [1, 16, 14, 14, 384]      1,804,884\n",
       "│    │    └─SwinTransformerBlock: 3-10                  [1, 16, 14, 14, 384]      1,804,884\n",
       "│    │    └─SwinTransformerBlock: 3-11                  [1, 16, 14, 14, 384]      1,804,884\n",
       "│    │    └─SwinTransformerBlock: 3-12                  [1, 16, 14, 14, 384]      1,804,884\n",
       "│    │    └─SwinTransformerBlock: 3-13                  [1, 16, 14, 14, 384]      1,804,884\n",
       "│    │    └─SwinTransformerBlock: 3-14                  [1, 16, 14, 14, 384]      1,804,884\n",
       "│    └─PatchMerging: 2-8                                [1, 16, 7, 7, 768]        --\n",
       "│    │    └─LayerNorm: 3-15                             [1, 16, 7, 7, 1536]       3,072\n",
       "│    │    └─Linear: 3-16                                [1, 16, 7, 7, 768]        1,179,648\n",
       "│    └─Sequential: 2-9                                  [1, 16, 7, 7, 768]        --\n",
       "│    │    └─SwinTransformerBlock: 3-17                  [1, 16, 7, 7, 768]        7,148,712\n",
       "│    │    └─SwinTransformerBlock: 3-18                  [1, 16, 7, 7, 768]        7,148,712\n",
       "├─LayerNorm: 1-4                                        [1, 16, 7, 7, 768]        1,536\n",
       "=========================================================================================================\n",
       "Total params: 27,850,470\n",
       "Trainable params: 27,850,470\n",
       "Non-trainable params: 0\n",
       "Total mult-adds (M): 486.09\n",
       "=========================================================================================================\n",
       "Input size (MB): 19.27\n",
       "Forward/backward pass size (MB): 1464.34\n",
       "Params size (MB): 75.43\n",
       "Estimated Total Size (MB): 1559.03\n",
       "========================================================================================================="
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch_models = [torch_vswin_tiny(), torch_vswin_small(), torch_vswin_base()]\n",
    "summary(\n",
    "    torch_models[0], input_size=(1, 3, 32, 224, 224)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f5f856f",
   "metadata": {
    "papermill": {
     "duration": 0.022661,
     "end_time": "2024-03-31T16:58:14.318044",
     "exception": false,
     "start_time": "2024-03-31T16:58:14.295383",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "68608e40",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-31T16:58:14.365429Z",
     "iopub.status.busy": "2024-03-31T16:58:14.364979Z",
     "iopub.status.idle": "2024-03-31T16:58:14.532780Z",
     "shell.execute_reply": "2024-03-31T16:58:14.531291Z"
    },
    "papermill": {
     "duration": 0.194807,
     "end_time": "2024-03-31T16:58:14.535575",
     "exception": false,
     "start_time": "2024-03-31T16:58:14.340768",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 32, 224, 224, 3]) torch.Size([1, 3, 32, 224, 224])\n"
     ]
    }
   ],
   "source": [
    "common_input = np.random.normal(0, 1, (1, 32, 224, 224, 3)).astype('float32')\n",
    "keras_input = ops.array(common_input)\n",
    "torch_input = torch.from_numpy(common_input.transpose(0, 4, 1, 2, 3))\n",
    "print(keras_input.shape, torch_input.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c4ade2b5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-31T16:58:14.583908Z",
     "iopub.status.busy": "2024-03-31T16:58:14.583167Z",
     "iopub.status.idle": "2024-03-31T16:58:14.589488Z",
     "shell.execute_reply": "2024-03-31T16:58:14.588707Z"
    },
    "papermill": {
     "duration": 0.033121,
     "end_time": "2024-03-31T16:58:14.591746",
     "exception": false,
     "start_time": "2024-03-31T16:58:14.558625",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def logit_checking(keras_model, torch_model):\n",
    "    # forward pass\n",
    "    keras_predict = keras_model(keras_input)\n",
    "    torch_predict = torch_model(torch_input) \n",
    "    print(keras_predict.shape, torch_predict.shape)\n",
    "    np.testing.assert_allclose(\n",
    "        keras_predict.detach().numpy(),\n",
    "        torch_predict.detach().numpy(),\n",
    "        1e-4, 1e-4\n",
    "    )\n",
    "    del keras_model \n",
    "    del torch_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8930bfc5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-31T16:58:14.640567Z",
     "iopub.status.busy": "2024-03-31T16:58:14.639913Z",
     "iopub.status.idle": "2024-03-31T16:59:23.764724Z",
     "shell.execute_reply": "2024-03-31T16:59:23.763372Z"
    },
    "papermill": {
     "duration": 69.152659,
     "end_time": "2024-03-31T16:59:23.767439",
     "exception": false,
     "start_time": "2024-03-31T16:58:14.614780",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 16, 7, 7, 768]) torch.Size([1, 16, 7, 7, 768])\n",
      "torch.Size([1, 16, 7, 7, 768]) torch.Size([1, 16, 7, 7, 768])\n",
      "torch.Size([1, 16, 7, 7, 1024]) torch.Size([1, 16, 7, 7, 1024])\n"
     ]
    }
   ],
   "source": [
    "for km, tm in zip(keras_models, torch_models):\n",
    "    logit_checking(\n",
    "        km, tm\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "68938336",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-31T16:59:23.817302Z",
     "iopub.status.busy": "2024-03-31T16:59:23.816843Z",
     "iopub.status.idle": "2024-03-31T16:59:24.215821Z",
     "shell.execute_reply": "2024-03-31T16:59:24.214493Z"
    },
    "papermill": {
     "duration": 0.42734,
     "end_time": "2024-03-31T16:59:24.218241",
     "exception": false,
     "start_time": "2024-03-31T16:59:23.790901",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55b12581",
   "metadata": {
    "papermill": {
     "duration": 0.023243,
     "end_time": "2024-03-31T16:59:24.265940",
     "exception": false,
     "start_time": "2024-03-31T16:59:24.242697",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Keras: Video Swin Base - Pretrained: ImageNet 22K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "43d4625f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-31T16:59:24.315830Z",
     "iopub.status.busy": "2024-03-31T16:59:24.315022Z",
     "iopub.status.idle": "2024-03-31T16:59:32.786393Z",
     "shell.execute_reply": "2024-03-31T16:59:32.784606Z"
    },
    "papermill": {
     "duration": 8.49938,
     "end_time": "2024-03-31T16:59:32.789273",
     "exception": false,
     "start_time": "2024-03-31T16:59:24.289893",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-03-31 16:59:25--  https://github.com/innat/VideoSwin/releases/download/v2.0/videoswin_base_kinetics400_imagenet22k.weights.h5\r\n",
      "Resolving github.com (github.com)... 140.82.121.4\r\n",
      "Connecting to github.com (github.com)|140.82.121.4|:443... connected.\r\n",
      "HTTP request sent, awaiting response... 302 Found\r\n",
      "Location: https://objects.githubusercontent.com/github-production-release-asset-2e65be/697696973/75b53567-f9ae-4739-87c1-0d5d9d423f25?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20240331%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240331T165925Z&X-Amz-Expires=300&X-Amz-Signature=b3146437a5644138b963a8e376f8cc066e3ff2c0b4bb7a05e1f705e930095453&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=697696973&response-content-disposition=attachment%3B%20filename%3Dvideoswin_base_kinetics400_imagenet22k.weights.h5&response-content-type=application%2Foctet-stream [following]\r\n",
      "--2024-03-31 16:59:25--  https://objects.githubusercontent.com/github-production-release-asset-2e65be/697696973/75b53567-f9ae-4739-87c1-0d5d9d423f25?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20240331%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240331T165925Z&X-Amz-Expires=300&X-Amz-Signature=b3146437a5644138b963a8e376f8cc066e3ff2c0b4bb7a05e1f705e930095453&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=697696973&response-content-disposition=attachment%3B%20filename%3Dvideoswin_base_kinetics400_imagenet22k.weights.h5&response-content-type=application%2Foctet-stream\r\n",
      "Resolving objects.githubusercontent.com (objects.githubusercontent.com)... 185.199.110.133, 185.199.111.133, 185.199.108.133, ...\r\n",
      "Connecting to objects.githubusercontent.com (objects.githubusercontent.com)|185.199.110.133|:443... connected.\r\n",
      "HTTP request sent, awaiting response... 200 OK\r\n",
      "Length: 351381896 (335M) [application/octet-stream]\r\n",
      "Saving to: 'videoswin_base_kinetics400_imagenet22k.weights.h5'\r\n",
      "\r\n",
      "videoswin_base_kine 100%[===================>] 335.10M  47.1MB/s    in 6.5s    \r\n",
      "\r\n",
      "2024-03-31 16:59:32 (51.9 MB/s) - 'videoswin_base_kinetics400_imagenet22k.weights.h5' saved [351381896/351381896]\r\n",
      "\r\n"
     ]
    }
   ],
   "source": [
    "!wget https://github.com/innat/VideoSwin/releases/download/v2.0/videoswin_base_kinetics400_imagenet22k.weights.h5\n",
    "\n",
    "def vswin_base():\n",
    "    backbone=VideoSwinBackbone(\n",
    "        input_shape=(32, 224, 224, 3), \n",
    "        embed_dim=128,\n",
    "        depths=[2, 2, 18, 2],\n",
    "        num_heads=[4, 8, 16, 32],\n",
    "        include_rescaling=False, \n",
    "    )\n",
    "    backbone.load_weights(\n",
    "        'videoswin_base_kinetics400_imagenet22k.weights.h5'\n",
    "    )\n",
    "    return backbone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2da61fbf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-31T16:59:32.846326Z",
     "iopub.status.busy": "2024-03-31T16:59:32.845846Z",
     "iopub.status.idle": "2024-03-31T16:59:35.458124Z",
     "shell.execute_reply": "2024-03-31T16:59:35.456891Z"
    },
    "papermill": {
     "duration": 2.644126,
     "end_time": "2024-03-31T16:59:35.461400",
     "exception": false,
     "start_time": "2024-03-31T16:59:32.817274",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "keras_models = vswin_base()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "29944adc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-31T16:59:35.518541Z",
     "iopub.status.busy": "2024-03-31T16:59:35.518114Z",
     "iopub.status.idle": "2024-03-31T16:59:58.027938Z",
     "shell.execute_reply": "2024-03-31T16:59:58.026294Z"
    },
    "papermill": {
     "duration": 22.542295,
     "end_time": "2024-03-31T16:59:58.031557",
     "exception": false,
     "start_time": "2024-03-31T16:59:35.489262",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://download.pytorch.org/models/swin3d_b_22k-7c6ae6fa.pth\" to /root/.cache/torch/hub/checkpoints/swin3d_b_22k-7c6ae6fa.pth\n",
      "100%|██████████| 364M/364M [00:19<00:00, 19.7MB/s]\n"
     ]
    }
   ],
   "source": [
    "import torchvision\n",
    "from torchvision.models.video import Swin3D_B_Weights\n",
    "\n",
    "torch_model = torchvision.models.video.swin3d_b(\n",
    "    weights=Swin3D_B_Weights.KINETICS400_IMAGENET22K_V1\n",
    ").eval()\n",
    "torch_model = exclude_top(torch_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5bb18406",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-31T16:59:58.125789Z",
     "iopub.status.busy": "2024-03-31T16:59:58.124570Z",
     "iopub.status.idle": "2024-03-31T17:00:31.735065Z",
     "shell.execute_reply": "2024-03-31T17:00:31.733788Z"
    },
    "papermill": {
     "duration": 33.660037,
     "end_time": "2024-03-31T17:00:31.738443",
     "exception": false,
     "start_time": "2024-03-31T16:59:58.078406",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 16, 7, 7, 1024]) torch.Size([1, 16, 7, 7, 1024])\n"
     ]
    }
   ],
   "source": [
    "logit_checking(\n",
    "    keras_models, torch_model\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20f490d2",
   "metadata": {
    "papermill": {
     "duration": 0.042622,
     "end_time": "2024-03-31T17:00:31.825841",
     "exception": false,
     "start_time": "2024-03-31T17:00:31.783219",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "none",
   "dataSources": [
    {
     "modelInstanceId": 17431,
     "sourceId": 21048,
     "sourceType": "modelInstanceVersion"
    },
    {
     "modelInstanceId": 17474,
     "sourceId": 21097,
     "sourceType": "modelInstanceVersion"
    },
    {
     "modelInstanceId": 17533,
     "sourceId": 21184,
     "sourceType": "modelInstanceVersion"
    }
   ],
   "dockerImageVersionId": 30673,
   "isGpuEnabled": false,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 246.289645,
   "end_time": "2024-03-31T17:00:36.170724",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-03-31T16:56:29.881079",
   "version": "2.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}