AE-4DVARNet_artcicleBoB.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RNHzBS8nvpZB"
   },
   "source": [
    "# Initialisation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 29008,
     "status": "ok",
     "timestamp": 1615217520478,
     "user": {
      "displayName": "Jean-Marie VIENT",
      "photoUrl": "",
      "userId": "16174505464901920949"
     },
     "user_tz": -60
    },
    "id": "I5iO50AvvpZD",
    "outputId": "4792202b-1a29-4bdb-d194-919d9b93a7c3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive\n",
      "[name: \"/device:CPU:0\"\n",
      "device_type: \"CPU\"\n",
      "memory_limit: 268435456\n",
      "locality {\n",
      "}\n",
      "incarnation: 2417112855465611824\n",
      ", name: \"/device:GPU:0\"\n",
      "device_type: \"GPU\"\n",
      "memory_limit: 15703311680\n",
      "locality {\n",
      "  bus_id: 1\n",
      "  links {\n",
      "  }\n",
      "}\n",
      "incarnation: 1922738279955809365\n",
      "physical_device_desc: \"device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0\"\n",
      "]\n"
     ]
    }
   ],
   "source": [
    "#Import data \n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import time\n",
    "from sklearn.decomposition import PCA\n",
    "from tqdm import tqdm\n",
    "from random import randrange\n",
    "import sklearn\n",
    "from sklearn import decomposition\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.cluster import KMeans\n",
    "from scipy.sparse import diags\n",
    "from scipy.stats import multivariate_normal\n",
    "from scipy.ndimage.morphology import distance_transform_edt as bwdist\n",
    "import scipy.ndimage as nd\n",
    "from scipy.interpolate import RegularGridInterpolator\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "import scipy.stats as ss \n",
    "from keras.constraints import Constraint\n",
    "from keras import backend as K\n",
    "from datetime import date, datetime, timedelta\n",
    "import matplotlib.pyplot as plt\n",
    "from google.colab import drive\n",
    "#drive.mount('/content/drive')\n",
    "#from tensorflow.python.client import device_lib\n",
    "print(device_lib.list_local_devices())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Kg2QP-hPvpZM"
   },
   "source": [
    "# Parameters "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "executionInfo": {
     "elapsed": 409,
     "status": "ok",
     "timestamp": 1615217539810,
     "user": {
      "displayName": "Jean-Marie VIENT",
      "photoUrl": "",
      "userId": "16174505464901920949"
     },
     "user_tz": -60
    },
    "id": "6EB_5VAtvpZP"
   },
   "outputs": [],
   "source": [
    "DirSAVE                     = '/'\n",
    "flagTrWMissingData          = 0  # Training phase with or without missing data\n",
    "flagloadOIData              = 0    # load OI: work on rough variable or anomaly\n",
    "lname_cov                   = [\"ssh_mod\"]\n",
    "lid_cov                     = [\"OI\"]\n",
    "size_tw                     = 1   # Length of the 4th dimension          \n",
    "Wsquare                     = 4     # half-width of holes\n",
    "Nsquare                     = 3     # number of holes\n",
    "DimAE                       = 20  # Dimension of the latent space\n",
    "flagLoadModel               = 0     # load pre-defined AE model or not\n",
    "sigNoise                    = 1e-1\n",
    "flagTrOuputWOMissingData    = 1\n",
    "stdMask                     = 0.\n",
    "dropout                     = 0.03 #[0.6,0.7,0.8]0.03\n",
    "wl2                         = 0.0000\n",
    "batch_size                  = 16 \n",
    "NbEpoc                      = 6\n",
    "Niter                       = 6*3\n",
    "thrMisData                  = 0.15 #noise\n",
    "N_cov                       = 0\n",
    "flagUseMaskinEncoder        = 0\n",
    "\n",
    "def createGlobParams(params):\n",
    "    return dict(((k, eval(k)) for k in params))\n",
    "list_globParams=[\n",
    "    'flagTrOuputWOMissingData','flagTrWMissingData',\\\n",
    "    'flagloadOIData','size_tw','Wsquare',\\\n",
    "    'Nsquare','DimAE','flagLoadModel',\\\n",
    "    'sigNoise',\\\n",
    "    'stdMask',\\\n",
    "    'dropout','wl2','batch_size',\\\n",
    "    'NbEpoc','Niter','DirSAVE','N_cov']\n",
    "globParams = createGlobParams(list_globParams)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4ovPNEb-vpZV"
   },
   "source": [
    "# Import data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 8685,
     "status": "ok",
     "timestamp": 1615217550336,
     "user": {
      "displayName": "Jean-Marie VIENT",
      "photoUrl": "",
      "userId": "16174505464901920949"
     },
     "user_tz": -60
    },
    "id": "PTnPawODvpZW",
    "outputId": "445c10ac-1061-431b-912a-439d0f93f698"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64,) (64,) (895,) (441,)\n",
      "(895, 64, 64) (441, 64, 64) (100, 64, 64)\n"
     ]
    }
   ],
   "source": [
    "dataset= np.load(\"*/Dataset_64_ZOI.npy\",allow_pickle='TRUE').item()\n",
    "dataH=dataset['CSED_Hourly']\n",
    "mask=dataset['Cloud_Daily']\n",
    "lat_grid=dataset['Lat_ZOI']\n",
    "lon_grid=dataset['Lon_ZOI']\n",
    "#corr lalpaciens var exp des norm\n",
    "# =============================================================================\n",
    "# Creation of the test and training dataset\n",
    "# =============================================================================\n",
    "def prepdata(xH,mask,N_Catalog=0):\n",
    "    if N_Catalog!=0:\n",
    "        test=np.zeros(mask[-365:].shape)\n",
    "        for i in range(0,len(test)):\n",
    "            test[i]=xH[26304+i*24+12].reshape(test.shape[1],test.shape[2])\n",
    "\n",
    "        train=np.zeros((N_Catalog*365,xH.shape[1],xH.shape[2]))\n",
    "\n",
    "        for i in range(N_Catalog):\n",
    "            for j in range(len(test)):\n",
    "                h=randrange(0,24)\n",
    "                train[i*365+j]=xH[i*365+j*24+h]\n",
    "    else:\n",
    "        xD=np.empty((xH.shape[0]//24,xH.shape[1],xH.shape[2]))\n",
    "        for i in range(len(xD)):\n",
    "            xD[i]=xH[12+i*24]\n",
    "    mask_train = mask[:-100]\n",
    "    mask_pred  = mask[-100:]\n",
    "    x_train    = xD[:len(mask_train)]\n",
    "    y_pred     = xD[len(mask_train):len(mask_train)+len(mask_pred)]\n",
    "    return mask_train,mask_pred,x_train,y_pred\n",
    "\n",
    "\n",
    "mask[np.where(mask==0)]=1;mask[np.where(np.isnan(mask))]=0 # O for missing data\n",
    "mask_train,mask_pred,x_train,y_pred = prepdata(dataH,mask)\n",
    "\n",
    "x_train,x_test,mask_train,mask_test=sklearn.model_selection.train_test_split(x_train,mask_train,test_size=0.33)\n",
    "# list of test dates\n",
    "indN_Tt = np.arange(len(x_train),len(x_train)+len(x_test))\n",
    "indN_Tr = np.arange(len(x_train))\n",
    "lday_test=[ datetime.strftime(datetime.strptime(\"2001-1-01\",'%Y-%m-%d')\\\n",
    "                      + timedelta(days=np.float64(i)),\"%Y-%m-%d\") for i in indN_Tt ]\n",
    "indLat     = np.arange(0,64)\n",
    "indLon     = np.arange(0,64)      \n",
    "print(indLat.shape,indLon.shape,indN_Tr.shape,indN_Tt.shape)\n",
    "print(x_train.shape,x_test.shape,y_pred.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 8123,
     "status": "ok",
     "timestamp": 1615217551459,
     "user": {
      "displayName": "Jean-Marie VIENT",
      "photoUrl": "",
      "userId": "16174505464901920949"
     },
     "user_tz": -60
    },
    "id": "_1WOy6QyvpZi",
    "outputId": "3c883ee5-8376-440a-c387-1e558b8e1882"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/sklearn/utils/extmath.py:765: RuntimeWarning: invalid value encountered in true_divide\n",
      "  updated_mean = (last_sum + new_sum) / updated_sample_count\n",
      "/usr/local/lib/python3.7/dist-packages/sklearn/utils/extmath.py:706: RuntimeWarning: Degrees of freedom <= 0 for slice.\n",
      "  result = op(x, *args, **kwargs)\n",
      "/usr/local/lib/python3.7/dist-packages/numpy/lib/nanfunctions.py:1114: RuntimeWarning: All-NaN slice encountered\n",
      "  overwrite_input=overwrite_input)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MEAN (4096,)\n",
      "False\n",
      "False\n",
      "(895, 64, 64, 1)\n",
      "False\n",
      "False\n",
      "(441, 64, 64, 1)\n",
      "False\n",
      "False\n",
      "(895, 64, 64, 1)\n",
      "False\n",
      "False\n",
      "(441, 64, 64, 1)\n",
      "False\n",
      "False\n",
      "(441, 64, 64, 1)\n",
      "False\n",
      "False\n",
      "(895, 64, 64, 1)\n",
      "False\n",
      "False\n",
      "(100, 64, 64, 1)\n",
      "False\n",
      "False\n",
      "(100, 64, 64, 1)\n",
      "False\n",
      "False\n",
      "(895, 64, 64, 1)\n",
      "False\n",
      "False\n",
      "(441, 64, 64, 1)\n",
      "False\n",
      "False\n",
      "(895, 64, 64, 1)\n",
      "... (after normalization) mean Tr = 0.095261\n",
      "... (after normalization) mean Tt = 0.094365\n",
      "... (after normalization) mean x_train = 0.095261\n",
      "... (after normalization) mean x_train_missing = 0.095261\n",
      "... (after normalization) mean x_test = 0.094365\n",
      "... (after normalization) mean x_test_missing = 0.014594\n",
      "... (after normalization) mean y_pred = -0.000000\n",
      "... (after normalization) mean y_pred_missing = -0.002209\n"
     ]
    }
   ],
   "source": [
    "gt_train= x_train\n",
    "gt_test = x_test\n",
    "gt_pred = np.copy(y_pred)\n",
    "err_train = np.random.normal(0,thrMisData*np.nanvar(x_train),(x_train.shape))\n",
    "err_test = np.random.normal(0,thrMisData*np.nanvar(x_test),(x_test.shape))\n",
    "err_pred = np.random.normal(0,thrMisData*np.nanvar(y_pred),(y_pred.shape))\n",
    "\n",
    "stdsc=sklearn.preprocessing.StandardScaler()\n",
    "stdsc.fit(y_pred.reshape(y_pred.shape[0],-1))\n",
    "mean_Tr = stdsc.mean_\n",
    "std_Tr = stdsc.var_\n",
    "\n",
    "gt_train        = gt_train.reshape(x_train.shape[0],-1)-mean_Tr\n",
    "x_train         = x_train.reshape(x_train.shape[0],-1)-mean_Tr\n",
    "gt_test         = gt_test.reshape(len(x_test),-1)-mean_Tr                       \n",
    "x_test          = x_test.reshape(len(x_test),-1)-mean_Tr\n",
    "y_pred          = y_pred.reshape(len(y_pred),-1)-mean_Tr\n",
    "\n",
    "if flagTrWMissingData == 1 :\n",
    "    x_train_missing=x_train.reshape(x_train.shape[0],indLat.shape[0],indLon.shape[0])*mask_train\n",
    "else :\n",
    "    mask_train[:]=1\n",
    "    x_train_missing=x_train.reshape(x_train.shape[0],indLat.shape[0],indLon.shape[0])*mask_train\n",
    "\n",
    "x_test_missing=x_test.reshape(x_test.shape[0],indLat.shape[0],indLon.shape[0])*mask_test\n",
    "y_pred_missing=y_pred.reshape(y_pred.shape[0],indLat.shape[0],indLon.shape[0])*mask_pred\n",
    "\n",
    "#Dimensionning                           \n",
    "gt_train        = gt_train.reshape(x_train.shape[0],indLat.shape[0],indLon.shape[0],1)\n",
    "x_train         = x_train.reshape(x_train.shape[0],indLat.shape[0],indLon.shape[0],1)\n",
    "x_train_missing = x_train_missing.reshape(x_train_missing.shape[0],indLat.shape[0],indLon.shape[0],1)\n",
    "mask_train      = mask_train.reshape(x_train.shape[0],indLat.shape[0],indLon.shape[0],1)\n",
    "\n",
    "gt_test         = gt_test.reshape(indN_Tt.shape[0],indLat.shape[0],indLon.shape[0],1)\n",
    "x_test          = x_test.reshape(indN_Tt.shape[0],indLat.shape[0],indLon.shape[0],1)\n",
    "x_test_missing  = x_test_missing.reshape(indN_Tt.shape[0],indLat.shape[0],indLon.shape[0],1)   \n",
    "mask_test       = mask_test.reshape(indN_Tt.shape[0],indLat.shape[0],indLon.shape[0],1)\n",
    "\n",
    "y_pred          = y_pred.reshape(y_pred.shape[0],indLat.shape[0],indLon.shape[0],1)\n",
    "y_pred_missing  = y_pred_missing.reshape(y_pred_missing.shape[0],indLat.shape[0],indLon.shape[0],1)\n",
    "mask_pred       = mask_pred.reshape(mask_pred.shape[0],indLat.shape[0],indLon.shape[0],1)\n",
    "\n",
    "#Change Nan to 0 Check isany Nan/inf \n",
    "def isany(x):\n",
    "    x[np.where(np.isnan(x))]=0\n",
    "    print(np.any(np.isnan(x)))\n",
    "    print(np.any(np.isinf(x)))\n",
    "    print(x.shape)\n",
    "\n",
    "isany(x_train)\n",
    "isany(x_test)\n",
    "isany(x_train_missing)\n",
    "isany(x_test_missing)\n",
    "isany(mask_test)\n",
    "isany(mask_train)\n",
    "isany(y_pred)\n",
    "isany(y_pred_missing)\n",
    "isany(x_train)\n",
    "isany(gt_test)\n",
    "isany(gt_train)\n",
    "#isany(gt_pred)\n",
    "\n",
    "\n",
    "print(\"... (after normalization) mean Tr = %f\"%(np.mean(gt_train)))\n",
    "print(\"... (after normalization) mean Tt = %f\"%(np.mean(gt_test)))\n",
    "print(\"... (after normalization) mean x_train = %f\"%(np.mean(x_train)))\n",
    "print(\"... (after normalization) mean x_train_missing = %f\"%(np.mean(x_train_missing)))\n",
    "print(\"... (after normalization) mean x_test = %f\"%(np.mean(x_test)))\n",
    "print(\"... (after normalization) mean x_test_missing = %f\"%(np.mean(x_test_missing)))\n",
    "print(\"... (after normalization) mean y_pred = %f\"%(np.mean(y_pred)))\n",
    "print(\"... (after normalization) mean y_pred_missing = %f\"%(np.mean(y_pred_missing)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "93mvfED-vpZn"
   },
   "source": [
    "### Define GENN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "jg0RKrt2vpZx"
   },
   "outputs": [],
   "source": [
    "# Constraints on kernel to to zeros\n",
    "# a specific position\n",
    "class Constraint_Zero(Constraint):\n",
    "    def __init__(self, position,kernel_shape,dw):\n",
    "        self.position = position\n",
    "        mask_array    = np.ones((kernel_shape))\n",
    "        mask_array[position[0]-dw:position[0]+dw+1,position[1]-dw:position[1]+dw+1,:,:] = 0.0\n",
    "\n",
    "        self.mask = K.variable(value=mask_array, dtype='float32', name='mask')\n",
    "\n",
    "        print(self.mask.shape)\n",
    "    def __call__(self, w):\n",
    "        print(self.mask.shape)\n",
    "        new_w = w * self.mask\n",
    "\n",
    "        return new_w\n",
    "\n",
    "def fl27(dict_global_Params,x_data,mask_data):\n",
    "\n",
    "    # import Global Parameters\n",
    "    for key,val in dict_global_Params.items():\n",
    "        exec(\"globals()['\"+key+\"']=val\")\n",
    "\n",
    "    WFilter       = 11#\n",
    "    NbResUnit     = 10#3#\n",
    "    dW            = 0\n",
    "    flagdownScale = 1 #: 0: only HR scale, 1 : only LR, 2 : HR + LR , 2 : MR, HR + LR annd LR,\n",
    "    scaleLR       = 2**2\n",
    "    NbFilter      = 1*DimAE\n",
    "    flagSRResNet  = 0\n",
    "     \n",
    "    input_layer = keras.layers.Input(shape=(x_train.shape[1],x_train.shape[2],x_train.shape[3]))\n",
    "    mask       = keras.layers.Input(shape=(x_train.shape[1],x_train.shape[2],x_train.shape[3]))\n",
    "    \n",
    "    if flagUseMaskinEncoder == 1:\n",
    "        dmask   = keras.layers.Lambda(lambda x: 0.2*x - 0.1)(mask)\n",
    "        \n",
    "        for jj in range(0,6):\n",
    "            dx    = keras.layers.Conv2D(10,(3,3),activation='relu', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(dmask)            \n",
    "            dx    = keras.layers.Conv2D(1,(3,3),activation='linear', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(dx)            \n",
    "            dmask = keras.layers.Add()([dmask,dx])\n",
    "            \n",
    "        x0       = keras.layers.Concatenate(axis=-1)([input_layer,dmask])\n",
    "    else:\n",
    "        x0  = keras.layers.Lambda(lambda x: 1. * x)(input_layer)\n",
    "\n",
    "    # coarse scale\n",
    "    if flagdownScale > 0 :\n",
    "        if flagUseMaskinEncoder == 1:\n",
    "            x = keras.layers.AveragePooling2D((scaleLR,scaleLR), padding='valid')(x0)\n",
    "            \n",
    "            x = keras.layers.Conv2D(NbFilter,(WFilter,WFilter),activation='relu', \n",
    "                padding='same',use_bias=False,\n",
    "                kernel_regularizer=keras.regularizers.l2(wl2),\n",
    "                kernel_constraint=Constraint_Zero((int(WFilter/2),int(WFilter/2)),(WFilter,WFilter,2*x_train.shape[3],NbFilter),dW))(x)\n",
    "        else:\n",
    "            x = keras.layers.AveragePooling2D((scaleLR,scaleLR), padding='valid')(x0)\n",
    "\n",
    "            x = keras.layers.Conv2D(NbFilter,(WFilter,WFilter),activation='relu', \n",
    "            padding='same',use_bias=False,\n",
    "            kernel_regularizer=keras.regularizers.l2(wl2),\n",
    "            kernel_constraint=Constraint_Zero((int(WFilter/2),int(WFilter/2)),(WFilter,WFilter,x_train.shape[3],NbFilter),dW))(x)\n",
    "        x  = keras.layers.Conv2D(DimAE,(1,1),activation='linear', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(x)\n",
    "  \n",
    "        # registration/evolution in feature space\n",
    "        scale = 0.1\n",
    "        for kk in range(0,NbResUnit):\n",
    "            if 1*1 :\n",
    "                dx = keras.layers.Conv2D(5*DimAE,(1,1),activation='relu', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(x)\n",
    "            else:\n",
    "                dx = keras.layers.Lambda(lambda x: scale * x)(x)\n",
    "      \n",
    "            dx_lin  = keras.layers.Conv2D(DimAE,(1,1),activation='linear', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(dx)\n",
    "            dx1 = keras.layers.Conv2D(DimAE,(1,1),activation='linear', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(dx)\n",
    "            dx2 = keras.layers.Conv2D(DimAE,(1,1),activation='linear', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(dx)\n",
    "      \n",
    "            dx1 = keras.layers.Multiply()([dx1,dx2])\n",
    "            \n",
    "            dx  = keras.layers.Add()([dx1,dx_lin])\n",
    "            dx  = keras.layers.Activation('tanh')(dx)\n",
    "            x  = keras.layers.Add()([x,dx])\n",
    "        x  = keras.layers.Conv2D(int(x_train.shape[3]/(N_cov+1)),(1,1),activation='linear', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(x)\n",
    "        x1 = keras.layers.Conv2DTranspose(int(x_train.shape[3]/(N_cov+1)),(scaleLR,scaleLR),strides=(scaleLR,scaleLR),use_bias=False,activation='linear',padding='same',output_padding=None,kernel_regularizer=keras.regularizers.l2(wl2))(x)\n",
    "        \n",
    "        if flagSRResNet == 1: ## postprocessing: super-resolution-like block\n",
    "            x1  = keras.layers.Conv2D(DimAE,(3,3),activation='linear', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(x1)               \n",
    "            \n",
    "            scale = 0.1\n",
    "            for kk in range(0,NbResUnit):\n",
    "                if 1*1 :\n",
    "                    dx = keras.layers.Conv2D(2*DimAE,(3,3),activation='relu', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(x1)\n",
    "                else:\n",
    "                    dx = keras.layers.Lambda(lambda x: scale * x)(x1)\n",
    "          \n",
    "                dx_lin  = keras.layers.Conv2D(DimAE,(3,3),activation='linear', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(dx)\n",
    "                dx1 = keras.layers.Conv2D(DimAE,(3,3),activation='linear', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(dx)\n",
    "                dx2 = keras.layers.Conv2D(DimAE,(3,3),activation='linear', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(dx)\n",
    "          \n",
    "                dx1 = keras.layers.Multiply()([dx1,dx2])\n",
    "        \n",
    "                dx  = keras.layers.Add()([dx1,dx_lin])\n",
    "                dx  = keras.layers.Activation('tanh')(dx_lin)\n",
    "                x1  = keras.layers.Add()([x1,dx])\n",
    "            x1  = keras.layers.Conv2D(x_train.shape[3]/(N_cov+1),(3,3),activation='linear', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(x1)\n",
    "        else:\n",
    "            x1  = keras.layers.Conv2D(2*DimAE,(3,3),activation='relu', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(x1) \n",
    "            x1  = keras.layers.Conv2D(int(x_train.shape[3]/(N_cov+1)),(3,3),activation='linear', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(x1)          \n",
    "     \n",
    "    # fine scale\n",
    "    if flagUseMaskinEncoder == 1:\n",
    "        x = keras.layers.Conv2D(NbFilter,(WFilter,WFilter),activation='relu', \n",
    "            padding='same',use_bias=False,\n",
    "            kernel_regularizer=keras.regularizers.l2(wl2),\n",
    "            kernel_constraint=Constraint_Zero((int(WFilter/2),int(WFilter/2)),(WFilter,WFilter,2*x_train.shape[3],NbFilter),dW))(x0)\n",
    "    else:\n",
    "        x = keras.layers.Conv2D(NbFilter,(WFilter,WFilter),activation='relu', \n",
    "            padding='same',use_bias=False,\n",
    "            kernel_regularizer=keras.regularizers.l2(wl2),\n",
    "            kernel_constraint=Constraint_Zero((int(WFilter/2),int(WFilter/2)),(WFilter,WFilter,x_train.shape[3],NbFilter),dW))(x0)\n",
    "\n",
    "    if flagdownScale > 0 :\n",
    "        x  = keras.layers.Concatenate(axis=-1)([x,x1])\n",
    "    x  = keras.layers.Conv2D(DimAE,(1,1),activation='linear', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(x)\n",
    "\n",
    "    for kk in range(0,NbResUnit):\n",
    "        if 1*1 :\n",
    "            dx      = keras.layers.Conv2D(5*DimAE,(1,1),activation='relu', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(x)\n",
    "        else:\n",
    "            dx  = keras.layers.Lambda(lambda x: scale * x)(x)\n",
    "          \n",
    "        dx_lin  = keras.layers.Conv2D(DimAE,(1,1),activation='linear', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(dx)\n",
    "        dx1 = keras.layers.Conv2D(DimAE,(1,1),activation='linear', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(dx)\n",
    "        dx2 = keras.layers.Conv2D(DimAE,(1,1),activation='linear', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(dx)\n",
    "          \n",
    "        dx1 = keras.layers.Multiply()([dx1,dx2])\n",
    "        \n",
    "        dx  = keras.layers.Add()([dx1,dx_lin])\n",
    "        dx  = keras.layers.Activation('tanh')(dx)\n",
    "        x  = keras.layers.Add()([x,dx])\n",
    "\n",
    "    x  = keras.layers.Conv2D(int(x_train.shape[3]/(N_cov+1)),(1,1),activation='linear', padding='same',use_bias=False,kernel_regularizer=keras.regularizers.l2(wl2))(x)\n",
    "\n",
    "    if flagdownScale == 0:\n",
    "        encoder    = keras.models.Model([input_layer,mask],x)\n",
    "    elif flagdownScale == 1:\n",
    "        encoder    = keras.models.Model([input_layer,mask],x1)\n",
    "    elif flagdownScale == 2:\n",
    "        x          = keras.layers.Add()([x,x1]) \n",
    "        encoder    = keras.models.Model([input_layer,mask],x)\n",
    "    elif flagdownScale == 3:\n",
    "        x          = keras.layers.Add()([x,x1]) \n",
    "        encoder    = keras.models.Model([input_layer,mask],[x1,x])\n",
    "   \n",
    "    if flagdownScale == 3:\n",
    "        decoder_input1 = keras.layers.Input(shape=(x_train.shape[1],x_train.shape[2],x_train.shape[3]))    \n",
    "        decoder_input2 = keras.layers.Input(shape=(x_train.shape[1],x_train.shape[2],x_train.shape[3]))    \n",
    "        x1  = keras.layers.Lambda(lambda x: 1. * x)(decoder_input1)\n",
    "        x2  = keras.layers.Lambda(lambda x: 1. * x)(decoder_input2)\n",
    "        decoder       = keras.models.Model([decoder_input1,decoder_input2],[x1,x2])\n",
    "    else:\n",
    "        decoder_input = keras.layers.Input(shape=(x_train.shape[1],x_train.shape[2],x_train.shape[3]))    \n",
    "        x  = keras.layers.Lambda(lambda x: 1. * x)(decoder_input)\n",
    "        decoder       = keras.models.Model(decoder_input,x)\n",
    "      \n",
    "    encoder.summary()\n",
    "    decoder.summary()\n",
    "\n",
    "    if flagdownScale < 3:\n",
    "        input_data = keras.layers.Input(shape=(x_train.shape[1],x_train.shape[2],x_train.shape[3]))\n",
    "        mask       = keras.layers.Input(shape=(x_train.shape[1],x_train.shape[2],x_train.shape[3]))\n",
    "  \n",
    "        x          = decoder(encoder([input_data,mask]))\n",
    "        model_AE   = keras.models.Model([input_data,mask],x)      \n",
    "    elif flagdownScale == 3:\n",
    "        \n",
    "        input_data = keras.layers.Input(shape=(x_train.shape[1],x_train.shape[2],x_train.shape[3]))\n",
    "        mask       = keras.layers.Input(shape=(x_train.shape[1],x_train.shape[2],x_train.shape[3]))\n",
    "        x,xLR      = encoder([input_data,mask])\n",
    "        \n",
    "        model_AE    = keras.models.Model([input_data,mask],x)\n",
    "        model_AE_MR = keras.models.Model([input_data,mask],[x,xLR])\n",
    "\n",
    "    size_tw = int(x_train.shape[3]/(N_cov+1))\n",
    "    #model_AE.compile(loss='mean_squared_error',optimizer=keras.optimizers.Adam(lr=1e-3))\n",
    "    model_AE.compile(loss=keras_custom_loss_function(size_tw),optimizer=keras.optimizers.Adam(lr=1e-3))\n",
    "    model_AE.summary()\n",
    "\n",
    "    DimCAE = DimAE\n",
    "\n",
    "    return encoder, decoder, model_AE, DimCAE\n",
    "\n",
    "def define_DINConvAE(NiterProjection,model_AE,shape,\\\n",
    "                    size_tw,N_cov=0):\n",
    "                     \n",
    "\n",
    "    # encoder-decoder with masked data\n",
    "    x_input         = keras.layers.Input((shape[1],shape[2],shape[3]))\n",
    "    mask            = keras.layers.Input((shape[1],shape[2],shape[3]))\n",
    "    \n",
    "    x     = keras.layers.Lambda(lambda x:1.*x)(x_input)\n",
    "    mask_ = keras.layers.Lambda(lambda x:1.-x)(mask)\n",
    "\n",
    "    # Iterations of fixed-point projection\n",
    "    index = np.arange(0,(N_cov+1)*size_tw,N_cov+1)   \n",
    "    for kk in range(0,NiterProjection):\n",
    "        x_proj   = model_AE([x,mask])\n",
    "        x_proj   = keras.layers.Multiply()([x_proj,slice_layer(index)(mask_)])\n",
    "        x        = keras.layers.Multiply()([slice_layer(index)(x),slice_layer(index)(mask)])\n",
    "        x        = keras.layers.Add()([x,x_proj])\n",
    "\n",
    "    x_proj = model_AE([x,mask]) \n",
    "    global_model_FP    = keras.models.Model([x_input,mask],[x_proj])\n",
    "\n",
    "    x_input         = keras.layers.Input((shape[1],shape[2],shape[3]))\n",
    "    mask            = keras.layers.Input((shape[1],shape[2],shape[3]))\n",
    "\n",
    "    maskg = keras.layers.Lambda(lambda x: 1.*x)(mask)\n",
    "      \n",
    "    x_proj = global_model_FP([x_input,maskg])\n",
    "\n",
    "  \n",
    "    # AE error with x_proj\n",
    "    err1 = error(x_proj,x_input,mask,size_tw,shape,1,N_cov)\n",
    "    # compute error (x_proj-x_input)**2 with full-1 mask\n",
    "    x_proj2 = model_AE([x_proj,keras.layers.Lambda(lambda x:1.-0.*x)(mask)])\n",
    "    err2    = error(x_proj,x_proj2,mask,size_tw,shape,0,N_cov)\n",
    "    # compute error (x_proj-x_input)**2 with full-1 mask\n",
    "    x_proj3 = model_AE([x_proj,keras.layers.Lambda(lambda x:0.*x)(mask)])\n",
    "    err3    = error(x_proj3,x_proj,mask,size_tw,shape,0,N_cov)\n",
    "    # add all errors\n",
    "    err    = keras.layers.Add()([err1,err2])\n",
    "    err    = keras.layers.Add()([err,err3])\n",
    "\n",
    "    # return global model\n",
    "    global_model_FP_Masked  = keras.models.Model([x_input,mask],err)\n",
    "    global_model_FP.summary()\n",
    "    global_model_FP_Masked.summary()\n",
    "  \n",
    "    return global_model_FP,global_model_FP_Masked\n",
    "\n",
    "def slice_layer(index):\n",
    "    def func(x_input):\n",
    "        return tf.gather(x_input, index, axis=3)\n",
    "    return keras.layers.Lambda(func)\n",
    "\n",
    "def assign_sliced_layer(size_tw,N_cov,x_output):\n",
    "    def func(x_input,x_output):\n",
    "        for i in range(1,(N_cov+1)):\n",
    "            index = np.arange(i,(N_cov+1)*size_tw,(N_cov+1),dtype='int32')\n",
    "            x_output = keras.layers.Concatenate()([x_output,slice_layer(index)(x_input)])\n",
    "        index  = np.stack([np.arange(i*size_tw,(i+1)*size_tw) \\\n",
    "                           for i in range(0,N_cov+1)]).T.flatten()\n",
    "        x_proj = slice_layer(index)(x_output)\n",
    "        return x_proj\n",
    "    return keras.layers.Lambda(func,arguments={'x_output':x_output})\n",
    "\n",
    "def error(x1,x2,mask,size_tw,shape,alpha,N_cov):\n",
    "    if x1.shape[3]>x2.shape[3]:\n",
    "        index = np.arange(0,(N_cov+1)*size_tw,N_cov+1,dtype='int32')\n",
    "        err   = keras.layers.Subtract()([slice_layer(index)(x1),x2])\n",
    "        err   = keras.layers.Multiply()([err,slice_layer(index)(mask)])\n",
    "    elif x2.shape[3]>x1.shape[3]:\n",
    "        index = np.arange(0,(N_cov+1)*size_tw,N_cov+1,dtype='int32')\n",
    "        err   = keras.layers.Subtract()([x1,slice_layer(index)(x2)])\n",
    "        err   = keras.layers.Multiply()([err,slice_layer(index)(mask)])\n",
    "    else:\n",
    "        err   = keras.layers.Subtract()([x1,x2])\n",
    "        err   = keras.layers.Multiply()([err,mask])\n",
    "    err   = keras.layers.Multiply()([err,err])\n",
    "    err   = keras.layers.Reshape((err.shape[-3],err.shape[-2],err.shape[-1],1))(err)\n",
    "    err   = keras.layers.GlobalAveragePooling3D()(err)\n",
    "    err   = keras.layers.Reshape((1,))(err)\n",
    "    err   = keras.layers.Lambda(lambda x: alpha*x)(err)\n",
    "    return err\n",
    "\n",
    "def keras_custom_loss_function(size_tw):\n",
    "    def insert_Sobel(size_tw,dir=\"x\"):\n",
    "        kernel_weights=np.zeros((3,3,size_tw,size_tw))\n",
    "        if dir==\"x\":\n",
    "            sobel=np.array([[-1,0,1],[-2,0,2],[-1,0,1]]).T\n",
    "        if dir==\"y\":\n",
    "            sobel=np.array([[-1,-2,-1],[0,0,0],[1,2,1]]).T\n",
    "        for i in range(size_tw):\n",
    "            kernel_weights[:,:,i,i]=sobel\n",
    "        return kernel_weights\n",
    "    def lossFunction(y_true,y_pred):\n",
    "        mae = tf.keras.losses.mean_absolute_error(y_true, y_pred)\n",
    "        filter_gx  = K.constant(insert_Sobel(size_tw,\"x\"))\n",
    "        filter_gy  = K.constant(insert_Sobel(size_tw,\"y\"))\n",
    "        Gx_true = K.conv2d(y_true, filter_gx, padding=\"same\")\n",
    "        Gy_true = K.conv2d(y_true, filter_gy, padding=\"same\")\n",
    "        Grad_true = K.sqrt(keras.layers.Add()([keras.layers.Multiply()([Gx_true,Gx_true]),\\\n",
    "                                       keras.layers.Multiply()([Gy_true,Gy_true])]))\n",
    "        Gx_pred = K.conv2d(y_pred, filter_gx, padding=\"same\")\n",
    "        Gy_pred = K.conv2d(y_pred, filter_gy, padding=\"same\")\n",
    "        Grad_pred = K.sqrt(keras.layers.Add()([keras.layers.Multiply()([Gx_pred,Gx_pred]),\\\n",
    "                                       keras.layers.Multiply()([Gy_pred,Gy_pred])]))\n",
    "        mae_grad = tf.keras.losses.mean_absolute_error(Grad_true, Grad_pred)\n",
    "        alpha= 0.5\n",
    "        loss = ((1.-alpha)*mae) + (alpha*mae_grad)\n",
    "        return loss\n",
    "    return lossFunction\n",
    "\n",
    "def root_mean_squared_error(y_true, y_pred):\n",
    "        return K.sqrt(K.mean(K.square(y_pred - y_true)))\n",
    "\n",
    "def eval_AEPerformance(x_train,rec_AE_Tr,x_test,rec_AE_Tt):\n",
    "\n",
    "    mse_AE_Tr        = np.mean( (rec_AE_Tr - x_train)**2 )\n",
    "    var_Tr           = np.mean( (x_train-np.mean(x_train,axis=0)) ** 2 )\n",
    "    exp_var_AE_Tr    = 1. - mse_AE_Tr / var_Tr\n",
    "   \n",
    "    mse_AE_Tt        = np.mean( (rec_AE_Tt - x_test)**2 )\n",
    "    var_Tt           = np.mean( (x_test-np.mean(x_train,axis=0))** 2 )\n",
    "    exp_var_AE_Tt    = 1. - mse_AE_Tt / var_Tt\n",
    "           \n",
    "    return exp_var_AE_Tr,exp_var_AE_Tt\n",
    "\n",
    "# functions for the evaluation of interpolation and auto-encoding performance\n",
    "def eval_AEPerformance(x_train,rec_AE_Tr,x_test,rec_AE_Tt):\n",
    "\n",
    "    mse_AE_Tr        = np.mean( (rec_AE_Tr - x_train)**2 )\n",
    "    var_Tr           = np.mean( (x_train-np.mean(x_train,axis=0)) ** 2 )\n",
    "    exp_var_AE_Tr    = 1. - mse_AE_Tr / var_Tr\n",
    "    \n",
    "    mse_AE_Tt        = np.mean( (rec_AE_Tt - x_test)**2 )\n",
    "    var_Tt           = np.mean( (x_test-np.mean(x_train,axis=0))** 2 )\n",
    "    exp_var_AE_Tt    = 1. - mse_AE_Tt / var_Tt\n",
    "            \n",
    "    return exp_var_AE_Tr,exp_var_AE_Tt\n",
    "\n",
    "def eval_InterpPerformance(mask_train,x_train,x_train_missing,x_train_pred,\n",
    "                           mask_test,x_test,x_test_missing,x_test_pred):\n",
    "    mse_train      = np.zeros((2))\n",
    "    mse_train[0]   = np.sum( mask_train * (x_train_pred - x_train_missing)**2 ) / np.sum( mask_train )\n",
    "    mse_train[1]   = np.mean( (x_train_pred - x_train)**2 )\n",
    "    exp_var_train  = 1. - mse_train #/ var_Tr\n",
    "            \n",
    "    mse_test        = np.zeros((2))\n",
    "    mse_test[0]     = np.sum( mask_test * (x_test_pred - x_test_missing)**2 ) / np.sum( mask_test )\n",
    "    mse_test[1]     = np.mean( (x_test_pred - x_test)**2 ) \n",
    "    exp_var_test = 1. - mse_test #/ var_Tt\n",
    "\n",
    "    mse_train_interp        = np.sum( (1.-mask_train) * (x_train_pred - x_train)**2 ) / np.sum( 1. - mask_train )\n",
    "    exp_var_train_interp    = 1. - mse_train_interp \n",
    "    \n",
    "    mse_test_interp        = np.sum( (1.-mask_test) * (x_test_pred - x_test)**2 ) / np.sum( 1. - mask_test )\n",
    "    exp_var_test_interp    = 1. - mse_test_interp\n",
    "            \n",
    "    return mse_train,exp_var_train,mse_test,exp_var_test,mse_train_interp,exp_var_train_interp,mse_test_interp,exp_var_test_interp\n",
    "\n",
    "# function to create recursive paths\n",
    "def mk_dir_recursive(dir_path):\n",
    "    if os.path.isdir(dir_path):\n",
    "        return\n",
    "    h, t = os.path.split(dir_path)  # head/tail\n",
    "    if not os.path.isdir(h):\n",
    "        mk_dir_recursive(h)\n",
    "\n",
    "    new_path = join_paths(h, t)\n",
    "    if not os.path.isdir(new_path):\n",
    "        os.mkdir(new_path)\n",
    "\n",
    "def Gradient(img, order):\n",
    "    \"\"\" calculate x, y gradient and magnitude \"\"\" \n",
    "    sobelx = cv2.Sobel(img,cv2.CV_64F,1,0,ksize=3)\n",
    "    sobelx = sobelx/8.0\n",
    "    sobely = cv2.Sobel(img,cv2.CV_64F,0,1,ksize=3)\n",
    "    sobely = sobely/8.0\n",
    "    sobel_norm = np.sqrt(sobelx*sobelx+sobely*sobely)\n",
    "    if (order==0):\n",
    "        return sobelx\n",
    "    elif (order==1):\n",
    "        return sobely\n",
    "    else:\n",
    "        return sobel_norm\n",
    "\n",
    "def insert_Sobel(size_tw,dir=\"x\"):\n",
    "    kernel_weights=np.zeros((3,3,size_tw,size_tw))\n",
    "    if dir==\"x\":\n",
    "        sobel=np.array([[-1,0,1],[-2,0,2],[-1,0,1]]).T\n",
    "    if dir==\"y\":\n",
    "        sobel=np.array([[-1,-2,-1],[0,0,0],[1,2,1]]).T\n",
    "    for i in range(size_tw):\n",
    "        kernel_weights[:,:,i,i]=sobel\n",
    "    return kernel_weights\n",
    "\n",
    "# New loss function for unsupervised setting:\n",
    "# L = MAE + ||Grad||^2\n",
    "def keras_custom_loss_function2(size_tw):\n",
    "    def insert_Sobel(size_tw,dir=\"x\"):\n",
    "        kernel_weights=np.zeros((3,3,size_tw,size_tw))\n",
    "        if dir==\"x\":\n",
    "            sobel=np.array([[-1,0,1],[-2,0,2],[-1,0,1]]).T\n",
    "        if dir==\"y\":\n",
    "            sobel=np.array([[-1,-2,-1],[0,0,0],[1,2,1]]).T\n",
    "        for i in range(size_tw):\n",
    "            kernel_weights[:,:,i,i]=sobel\n",
    "        return kernel_weights\n",
    "    def lossFunction(y_true,y_pred):\n",
    "        filter_gx  = K.constant(insert_Sobel(size_tw,\"x\"))\n",
    "        filter_gy  = K.constant(insert_Sobel(size_tw,\"y\"))\n",
    "        Gx_pred = K.conv2d(y_pred, filter_gx, padding=\"same\")\n",
    "        Gy_pred = K.conv2d(y_pred, filter_gy, padding=\"same\")\n",
    "        Grad_pred = K.sqrt(keras.layers.Add()([keras.layers.Multiply()([Gx_pred,Gx_pred]),\\\n",
    "                                       keras.layers.Multiply()([Gy_pred,Gy_pred])]))\n",
    "        loss = K.mean(keras.layers.Multiply()([Grad_pred,Grad_pred]))\n",
    "        return loss\n",
    "    return lossFunction\n",
    "\n",
    "def thresholding(x,thr):\n",
    "    greater = K.greater_equal(x,thr) #will return boolean values\n",
    "    greater = K.cast(greater, dtype=K.floatx()) #will convert bool to 0 and 1    \n",
    "    return greater\n",
    "\n",
    "def slice_layer(index):\n",
    "    def func(x_input):\n",
    "        return tf.gather(x_input, index, axis=3)\n",
    "    return keras.layers.Lambda(func)\n",
    "\n",
    "def assign_sliced_layer(size_tw,N_cov,x_output):\n",
    "    def func(x_input,x_output):\n",
    "        for i in range(1,(N_cov+1)):\n",
    "            index = np.arange(i,(N_cov+1)*size_tw,(N_cov+1),dtype='int32')\n",
    "            x_output = keras.layers.Concatenate()([x_output,slice_layer(index)(x_input)])\n",
    "        index  = np.stack([np.arange(i*size_tw,(i+1)*size_tw) \\\n",
    "                           for i in range(0,N_cov+1)]).T.flatten()\n",
    "        x_proj = slice_layer(index)(x_output)\n",
    "        return x_proj\n",
    "    return keras.layers.Lambda(func,arguments={'x_output':x_output})\n",
    "\n",
    "def error(x1,x2,mask,size_tw,shape,alpha,N_cov):\n",
    "    if x1.shape[3]>x2.shape[3]:\n",
    "        index = np.arange(0,(N_cov+1)*size_tw,N_cov+1,dtype='int32')\n",
    "        err   = keras.layers.Subtract()([slice_layer(index)(x1),x2])\n",
    "        err   = keras.layers.Multiply()([err,slice_layer(index)(mask)])\n",
    "    elif x2.shape[3]>x1.shape[3]:\n",
    "        index = np.arange(0,(N_cov+1)*size_tw,N_cov+1,dtype='int32')\n",
    "        err   = keras.layers.Subtract()([x1,slice_layer(index)(x2)])\n",
    "        err   = keras.layers.Multiply()([err,slice_layer(index)(mask)])\n",
    "    else:\n",
    "        err   = keras.layers.Subtract()([x1,x2])\n",
    "        err   = keras.layers.Multiply()([err,mask])\n",
    "    err   = keras.layers.Multiply()([err,err])\n",
    "    err   = keras.layers.Reshape((err.shape[-3],err.shape[-2],err.shape[-1],1))(err)\n",
    "    # normalize err\n",
    "    err   = keras.layers.GlobalAveragePooling3D()(err)\n",
    "    err   = keras.layers.Reshape((1,))(err)\n",
    "    err   = keras.layers.Lambda(lambda x: alpha*x)(err)\n",
    "    return err\n",
    "\n",
    "def regularize_Gradient(x_proj,size_tw):\n",
    "    filter_gx  = K.constant(insert_Sobel(size_tw,\"x\"))\n",
    "    filter_gy  = K.constant(insert_Sobel(size_tw,\"y\"))\n",
    "    def Gradient(tensors):\n",
    "        tensor = tensors[0]\n",
    "        filter = tensors[1]\n",
    "        return K.conv2d(tensor, filter, padding=\"same\")\n",
    "    Gx_pred = keras.layers.Lambda(Gradient)([x_proj, filter_gx])\n",
    "    Gy_pred = keras.layers.Lambda(Gradient)([x_proj, filter_gy])\n",
    "    #Grad_pred = K.sqrt(keras.layers.Add()([keras.layers.Multiply()([Gx_pred,Gx_pred]),\\\n",
    "    #                                    keras.layers.Multiply()([Gy_pred,Gy_pred])]))\n",
    "    Grad_pred = keras.layers.Add()([keras.layers.Multiply()([Gx_pred,Gx_pred]),\\\n",
    "                                        keras.layers.Multiply()([Gy_pred,Gy_pred])])\n",
    "    reg_Gradient   = keras.layers.Reshape((Grad_pred.shape[-3],Grad_pred.shape[-2],Grad_pred.shape[-1],1))(Grad_pred)\n",
    "    reg_Gradient   = keras.layers.GlobalAveragePooling3D()(reg_Gradient)\n",
    "    reg_Gradient   = keras.layers.Reshape((1,))(reg_Gradient)\n",
    "    reg_Gradient   = keras.layers.Lambda(lambda x: 1*x)(reg_Gradient)\n",
    "    return reg_Gradient\n",
    "\n",
    "def define_DINConvAE(NiterProjection,model_AE,shape,\\\n",
    "                     flagUseMaskinEncoder,\\\n",
    "                     size_tw,include_covariates,N_cov=0):\n",
    "\n",
    "    # encoder-decoder with masked data\n",
    "    x_input         = keras.layers.Input((shape[1],shape[2],shape[3]))\n",
    "    mask            = keras.layers.Input((shape[1],shape[2],shape[3]))\n",
    "    \n",
    "    x     = keras.layers.Lambda(lambda x:1.*x)(x_input)\n",
    "    mask_ = keras.layers.Lambda(lambda x:1.-x)(mask)\n",
    "\n",
    "    # Iterations of fixed-point projection\n",
    "    index = np.arange(0,(N_cov+1)*size_tw,N_cov+1)   \n",
    "    for kk in range(0,NiterProjection):\n",
    "        x_proj   = model_AE([x,mask])\n",
    "        x_proj   = keras.layers.Multiply()([x_proj,slice_layer(index)(mask_)])\n",
    "        x        = keras.layers.Multiply()([slice_layer(index)(x),slice_layer(index)(mask)])\n",
    "        x        = keras.layers.Add()([x,x_proj])\n",
    "\n",
    "\n",
    "    x_proj = model_AE([x,mask]) \n",
    "    global_model_FP    = keras.models.Model([x_input,mask],[x_proj])\n",
    "\n",
    "    # randomly sample an additionnal missing data mask\n",
    "    # additive noise + spatial smoothing\n",
    "    if flagUseMaskinEncoder == 1:\n",
    "        WAvFilter     = 3\n",
    "        NIterAvFilter = 3\n",
    "        thrNoise      = 1.5 * stdMask + 1e-7\n",
    "        maskg   = keras.layers.GaussianNoise(stdMask)(mask)\n",
    "        avFilter       = 1./(WAvFilter**3)*np.ones((WAvFilter,WAvFilter,WAvFilter,1,1))\n",
    "        spatialAvLayer = keras.layers.Conv3D(1,(WAvFilter,WAvFilter,WAvFilter),weights=[avFilter],\\\n",
    "                           padding='same',activation='linear',use_bias=False,name='SpatialAverage')\n",
    "        spatialAvLayer.trainable = False\n",
    "        maskg = keras.layers.Lambda(lambda x: K.permute_dimensions(x,(0,3,1,2)))(maskg) \n",
    "        maskg  = keras.layers.Reshape((shape[3],shape[1],shape[2],1))(maskg)\n",
    "        for nn in range(0,NIterAvFilter):\n",
    "            maskg  = spatialAvLayer(maskg) \n",
    "        maskg = keras.layers.Lambda(lambda x: K.permute_dimensions(x,(0,2,3,1,4)))(maskg) \n",
    "        maskg = keras.layers.Reshape((shape[1],shape[2],shape[3]))(maskg)\n",
    "        maskg = keras.layers.Lambda(lambda x: thresholding(x,thrNoise))(maskg)    \n",
    "        maskg  = keras.layers.Multiply()([mask,maskg])\n",
    "        maskg  = keras.layers.Subtract()([mask,maskg])       \n",
    "    else:\n",
    "        maskg = keras.layers.Lambda(lambda x: 1.*x)(mask)\n",
    "\n",
    "    x_proj = global_model_FP([x_input,maskg])\n",
    "    # AE error with x_proj\n",
    "    err1 = error(x_proj,x_input,mask,size_tw,shape,1,N_cov)\n",
    "    # AE error with x_proj\n",
    "    # compute error (x_proj-x_input)**2 with full-1 mask\n",
    "    x_proj_ = x_proj\n",
    "    \n",
    "    x_proj2 = model_AE([x_proj_,keras.layers.Lambda(lambda x:1.-0.*x)(mask)])\n",
    "    err2    = error(x_proj_,x_proj2,mask,size_tw,shape,0,N_cov)\n",
    "    # compute error (x_proj-x_input)**2 with full-1 mask\n",
    "    x_proj3 = model_AE([x_proj_,keras.layers.Lambda(lambda x:0.*x)(mask)])\n",
    "    err3    = error(x_proj3,x_proj_,mask,size_tw,shape,0,N_cov)\n",
    "    # add all errors\n",
    "    err    = keras.layers.Add()([err1,err2])\n",
    "    err    = keras.layers.Add()([err,err3])\n",
    "\n",
    "    # return global model\n",
    "    global_model_FP_Masked  = keras.models.Model([x_input,mask],[err,x_proj])\n",
    "\n",
    "    global_model_FP.summary()\n",
    "    global_model_FP_Masked.summary()\n",
    "    return global_model_FP,global_model_FP_Masked"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1212,
     "status": "ok",
     "timestamp": 1614841073805,
     "user": {
      "displayName": "Jean-Marie VIENT",
      "photoUrl": "",
      "userId": "16174505464901920949"
     },
     "user_tz": -60
    },
    "id": "Gcs8QZ4uvpZs",
    "outputId": "fdbfec8d-9896-4f21-ad89-a118f6162067"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11, 11, 1, 20)\n",
      "(11, 11, 1, 20)\n",
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 64, 64, 1)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lambda (Lambda)                 (None, 64, 64, 1)    0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d (AveragePooli (None, 16, 16, 1)    0           lambda[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d (Conv2D)                 (None, 16, 16, 20)   2420        average_pooling2d[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 16, 16, 20)   400         conv2d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 16, 16, 100)  2000        conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 16, 16, 20)   2000        conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, 16, 16, 20)   2000        conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply (Multiply)             (None, 16, 16, 20)   0           conv2d_4[0][0]                   \n",
      "                                                                 conv2d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 16, 16, 20)   2000        conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add (Add)                       (None, 16, 16, 20)   0           multiply[0][0]                   \n",
      "                                                                 conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation (Activation)         (None, 16, 16, 20)   0           add[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 16, 16, 20)   0           conv2d_1[0][0]                   \n",
      "                                                                 activation[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, 16, 16, 100)  2000        add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_8 (Conv2D)               (None, 16, 16, 20)   2000        conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_9 (Conv2D)               (None, 16, 16, 20)   2000        conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_1 (Multiply)           (None, 16, 16, 20)   0           conv2d_8[0][0]                   \n",
      "                                                                 conv2d_9[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_7 (Conv2D)               (None, 16, 16, 20)   2000        conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, 16, 16, 20)   0           multiply_1[0][0]                 \n",
      "                                                                 conv2d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 16, 16, 20)   0           add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, 16, 16, 20)   0           add_1[0][0]                      \n",
      "                                                                 activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_10 (Conv2D)              (None, 16, 16, 100)  2000        add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_12 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_13 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_2 (Multiply)           (None, 16, 16, 20)   0           conv2d_12[0][0]                  \n",
      "                                                                 conv2d_13[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_11 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, 16, 16, 20)   0           multiply_2[0][0]                 \n",
      "                                                                 conv2d_11[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 16, 16, 20)   0           add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, 16, 16, 20)   0           add_3[0][0]                      \n",
      "                                                                 activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_14 (Conv2D)              (None, 16, 16, 100)  2000        add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_16 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_17 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_3 (Multiply)           (None, 16, 16, 20)   0           conv2d_16[0][0]                  \n",
      "                                                                 conv2d_17[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_15 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_6 (Add)                     (None, 16, 16, 20)   0           multiply_3[0][0]                 \n",
      "                                                                 conv2d_15[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 16, 16, 20)   0           add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "add_7 (Add)                     (None, 16, 16, 20)   0           add_5[0][0]                      \n",
      "                                                                 activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_18 (Conv2D)              (None, 16, 16, 100)  2000        add_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_20 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_21 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_4 (Multiply)           (None, 16, 16, 20)   0           conv2d_20[0][0]                  \n",
      "                                                                 conv2d_21[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_19 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_8 (Add)                     (None, 16, 16, 20)   0           multiply_4[0][0]                 \n",
      "                                                                 conv2d_19[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 16, 16, 20)   0           add_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "add_9 (Add)                     (None, 16, 16, 20)   0           add_7[0][0]                      \n",
      "                                                                 activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_22 (Conv2D)              (None, 16, 16, 100)  2000        add_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_24 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_22[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_25 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_22[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_5 (Multiply)           (None, 16, 16, 20)   0           conv2d_24[0][0]                  \n",
      "                                                                 conv2d_25[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_23 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_22[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_10 (Add)                    (None, 16, 16, 20)   0           multiply_5[0][0]                 \n",
      "                                                                 conv2d_23[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 16, 16, 20)   0           add_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "add_11 (Add)                    (None, 16, 16, 20)   0           add_9[0][0]                      \n",
      "                                                                 activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_26 (Conv2D)              (None, 16, 16, 100)  2000        add_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_28 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_26[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_29 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_26[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_6 (Multiply)           (None, 16, 16, 20)   0           conv2d_28[0][0]                  \n",
      "                                                                 conv2d_29[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_27 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_26[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_12 (Add)                    (None, 16, 16, 20)   0           multiply_6[0][0]                 \n",
      "                                                                 conv2d_27[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, 16, 16, 20)   0           add_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "add_13 (Add)                    (None, 16, 16, 20)   0           add_11[0][0]                     \n",
      "                                                                 activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_30 (Conv2D)              (None, 16, 16, 100)  2000        add_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_32 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_30[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_33 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_30[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_7 (Multiply)           (None, 16, 16, 20)   0           conv2d_32[0][0]                  \n",
      "                                                                 conv2d_33[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_31 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_30[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_14 (Add)                    (None, 16, 16, 20)   0           multiply_7[0][0]                 \n",
      "                                                                 conv2d_31[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_7 (Activation)       (None, 16, 16, 20)   0           add_14[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "add_15 (Add)                    (None, 16, 16, 20)   0           add_13[0][0]                     \n",
      "                                                                 activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_34 (Conv2D)              (None, 16, 16, 100)  2000        add_15[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_36 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_34[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_37 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_34[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_8 (Multiply)           (None, 16, 16, 20)   0           conv2d_36[0][0]                  \n",
      "                                                                 conv2d_37[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_35 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_34[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_16 (Add)                    (None, 16, 16, 20)   0           multiply_8[0][0]                 \n",
      "                                                                 conv2d_35[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_8 (Activation)       (None, 16, 16, 20)   0           add_16[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "add_17 (Add)                    (None, 16, 16, 20)   0           add_15[0][0]                     \n",
      "                                                                 activation_8[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_38 (Conv2D)              (None, 16, 16, 100)  2000        add_17[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_40 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_38[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_41 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_38[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_9 (Multiply)           (None, 16, 16, 20)   0           conv2d_40[0][0]                  \n",
      "                                                                 conv2d_41[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_39 (Conv2D)              (None, 16, 16, 20)   2000        conv2d_38[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_18 (Add)                    (None, 16, 16, 20)   0           multiply_9[0][0]                 \n",
      "                                                                 conv2d_39[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_9 (Activation)       (None, 16, 16, 20)   0           add_18[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "add_19 (Add)                    (None, 16, 16, 20)   0           add_17[0][0]                     \n",
      "                                                                 activation_9[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_42 (Conv2D)              (None, 16, 16, 1)    20          add_19[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_transpose (Conv2DTranspo (None, 64, 64, 1)    16          conv2d_42[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_43 (Conv2D)              (None, 64, 64, 40)   360         conv2d_transpose[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            [(None, 64, 64, 1)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_44 (Conv2D)              (None, 64, 64, 1)    360         conv2d_43[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 83,576\n",
      "Trainable params: 83,576\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Model: \"model_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_3 (InputLayer)         [(None, 64, 64, 1)]       0         \n",
      "_________________________________________________________________\n",
      "lambda_1 (Lambda)            (None, 64, 64, 1)         0         \n",
      "=================================================================\n",
      "Total params: 0\n",
      "Trainable params: 0\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Model: \"model_2\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_4 (InputLayer)            [(None, 64, 64, 1)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_5 (InputLayer)            [(None, 64, 64, 1)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "model (Functional)              (None, 64, 64, 1)    83576       input_4[0][0]                    \n",
      "                                                                 input_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "model_1 (Functional)            (None, 64, 64, 1)    0           model[0][0]                      \n",
      "==================================================================================================\n",
      "Total params: 83,576\n",
      "Trainable params: 83,576\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#Apply model     \n",
    "encoder, decoder, model_AE, DimCAE = fl27(globParams,x_train,mask_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eOLreoecvpZw"
   },
   "source": [
    "### Train-Model - FP - Iterated projection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 521,
     "status": "ok",
     "timestamp": 1614841077126,
     "user": {
      "displayName": "Jean-Marie VIENT",
      "photoUrl": "",
      "userId": "16174505464901920949"
     },
     "user_tz": -60
    },
    "id": "JEbLPGz_vpZ1",
    "outputId": "7b445232-3fb6-4058-f6f1-f8c9d27de163"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ready\n"
     ]
    }
   ],
   "source": [
    "def flagProcess4_Optim0(dict_global_Params,x_train,x_train_missing,mask_train,gt_train,\\\n",
    "                    x_test,x_test_missing,mask_test,gt_test,lday_test,encoder,decoder,model_AE,DimCAE):\n",
    "\n",
    "    for key,val in dict_global_Params.items():\n",
    "        exec(\"globals()['\"+key+\"']=val\")\n",
    "\n",
    "    meanTr = stdsc.mean_\n",
    "    stdTr = np.mean(stdsc.var_)\n",
    "    # ***************** #\n",
    "    # model compilation #\n",
    "    # ***************** #\n",
    "\n",
    "    # model fit\n",
    "    NbProjection   = [0,0,2,2,5,5,10,15,14]\n",
    "    NbProjection   = [5,5,5,5]\n",
    "    lrUpdate       = [1e-3,1e-4,1e-5,1e-5,1e-5,1e-6,1e-6,1e-5,1e-6]\n",
    "    if flagTrOuputWOMissingData==0:\n",
    "        lrUpdate   = [1e-4,1e-5,1e-6,1e-7]\n",
    "    else:\n",
    "        lrUpdate   = [1e-3,1e-4,1e-5,1e-6]\n",
    "    IterUpdate     = [0,3,10,15,20,25,30,35,40]\n",
    "    #IterUpdate     = [0,6,15,20]\n",
    "    val_split      = 0.1\n",
    "    \n",
    "    iterInit = 0\n",
    "    IterTrainAE = 0\n",
    "    IterUpdateInit = 10000\n",
    "    \n",
    "    ## initialization\n",
    "    x_train_init = np.copy(x_train_missing)\n",
    "    x_test_init  = np.copy(x_test_missing)\n",
    "\n",
    "    comptUpdate = 0\n",
    "\n",
    "    # ******************** #\n",
    "    # Start Learning model #\n",
    "    # ******************** #\n",
    "        \n",
    "    print(\"..... Start learning AE model\")\n",
    "    for iter in tqdm(range(iterInit,Niter)):\n",
    "        if iter == IterUpdate[comptUpdate]:\n",
    "            # update DINConvAE model\n",
    "            NBProjCurrent = NbProjection[comptUpdate]\n",
    "            print(\"..... Update/initialize number of projections in DINCOnvAE model # %d\"%(NbProjection[comptUpdate]))\n",
    "            global_model_FP,global_model_FP_Masked = define_DINConvAE(NbProjection[comptUpdate],model_AE,x_train.shape,\\\n",
    "                                                                          flagUseMaskinEncoder,\\\n",
    "                                                                          size_tw,N_cov)\n",
    "            if flagTrOuputWOMissingData == 1:\n",
    "                global_model_FP.compile(loss='mean_squared_error',optimizer=keras.optimizers.Adam(lr=lrUpdate[comptUpdate]))\n",
    "                #global_model_FP.compile(loss=keras_custom_loss_function(size_tw),optimizer=keras.optimizers.Adam(lr=lrUpdate[comptUpdate]))\n",
    "            else:\n",
    "                #global_model_FP_Masked.compile(loss='mean_squared_error',optimizer=keras.optimizers.Adam(lr=lrUpdate[comptUpdate]))\n",
    "                global_model_FP_Masked.compile(loss=['mean_squared_error',keras_custom_loss_function2(size_tw)],loss_weights=[1.0,1e-10],optimizer=keras.optimizers.Adam(lr=lrUpdate[comptUpdate]))\n",
    "            if comptUpdate < len(NbProjection)-1:\n",
    "                comptUpdate += 1\n",
    "        \n",
    "        # gradient descent iteration            \n",
    "        if flagTrOuputWOMissingData == 1:\n",
    "            history = global_model_FP.fit([x_train_init,mask_train],gt_train,\n",
    "                  batch_size=batch_size,\n",
    "                  epochs = NbEpoc,\n",
    "                  verbose = 1, \n",
    "                  validation_split=val_split)\n",
    "        else:\n",
    "            history = global_model_FP_Masked.fit([x_train_init,mask_train],[np.zeros((x_train_init.shape[0],1)),gt_train],\n",
    "                  batch_size=batch_size,\n",
    "                  epochs = NbEpoc,\n",
    "                  verbose = 1, \n",
    "                  validation_split=val_split)\n",
    "\n",
    "        # *********************** #\n",
    "        # Prediction on test data #\n",
    "        # *********************** #\n",
    "\n",
    "        # trained full-model\n",
    "        x_train_pred    = global_model_FP.predict([x_train_init,mask_train])\n",
    "        x_test_pred     = global_model_FP.predict([x_test_init,mask_test])\n",
    "\n",
    "        # trained AE applied to gap-free data\n",
    "        if flagUseMaskinEncoder == 1:\n",
    "          rec_AE_Tr     = model_AE.predict([x_train,np.zeros((mask_train.shape))])\n",
    "          rec_AE_Tt     = model_AE.predict([x_test,np.zeros((mask_train.shape))])\n",
    "        else:\n",
    "          rec_AE_Tr     = model_AE.predict([x_train,np.ones((mask_train.shape))])\n",
    "          rec_AE_Tt     = model_AE.predict([x_test,np.ones((mask_test.shape))])\n",
    "\n",
    "\n",
    "        mse_train,exp_var_train,\\\n",
    "        mse_test,exp_var_test,\\\n",
    "        mse_train_interp,exp_var_train_interp,\\\n",
    "        mse_test_interp,exp_var_test_interp =\\\n",
    "        eval_InterpPerformance(mask_train,x_train,x_train_missing,x_train_pred,\\\n",
    "                               mask_test,x_test,x_test_missing,x_test_pred)\n",
    "        \n",
    "        # interpolation and reconstruction score for the center image\n",
    "        # when dealing with time series\n",
    "        if x_train_init.shape[3] > 0 :\n",
    "            \n",
    "            dWCenter    = 32  \n",
    "            \n",
    "            dT = np.floor( x_train_init.shape[3] / 2 ).astype(int)\n",
    "            mse_train_center        = np.mean( (x_train_pred[:,:,:,dT] - x_train[:,:,:,dT] )**2 )\n",
    "            mse_train_center_interp = np.sum( (x_train_pred[:,:,:,dT]  - x_train[:,:,:,dT] )**2 * (1.-mask_train[:,:,:,dT])  ) / np.sum( (1.-mask_train[:,:,:,dT]) )\n",
    "            \n",
    "            mse_test_center         = np.mean( (x_test_pred[:,:,:,dT] - x_test[:,:,:,dT] )**2 )\n",
    "            mse_test_center_interp  = np.sum( (x_test_pred[:,:,:,dT]  - x_test[:,:,:,dT] )**2 * (1.-mask_test[:,:,:,dT])  ) / np.sum( (1-mask_test[:,:,:,dT]) )\n",
    "            \n",
    "            var_train_center        = np.var(  x_train[:,:,:,dT] )\n",
    "            var_test_center         = np.var(  x_test[:,:,:,dT] )\n",
    "            \n",
    "            exp_var_train_center         = 1.0 - mse_train_center / var_train_center\n",
    "            exp_var_train_interp_center  = 1.0 - mse_train_center_interp / var_train_center\n",
    "            exp_var_test_center          = 1.0 - mse_test_center  / var_test_center\n",
    "            exp_var_test_interp_center   = 1.0 - mse_test_center_interp/ var_test_center\n",
    "        # AE performance of the trained AE applied to gap-free data\n",
    "        exp_var_AE_Tr,exp_var_AE_Tt = eval_AEPerformance(x_train,rec_AE_Tr,x_test,rec_AE_Tt)\n",
    "        \n",
    "        print(\".......... Auto-encoder performance when applied to gap-free data\")\n",
    "        print('.... explained variance AE (Tr)  : %.2f%%'%(100.*exp_var_AE_Tr))\n",
    "        print('.... explained variance AE (Tt)  : %.2f%%'%(100.*exp_var_AE_Tt))\n",
    "        \n",
    "        if flagUseMaskinEncoder == 1:\n",
    "        \n",
    "            exp_var_AE_Tr,exp_var_AE_Tt = eval_AEPerformance(x_train,rec_AE_Tr,x_test,rec_AE_Tt)\n",
    "        \n",
    "            print('.... explained variance AE (Tr) with mask  : %.2f%%'%(100.*exp_var_AE_Tr))\n",
    "            print('.... explained variance AE (Tt) with mask  : %.2f%%'%(100.*exp_var_AE_Tt))\n",
    "\n",
    "\n",
    "\n",
    "        # update training data\n",
    "        if iter > IterUpdateInit:\n",
    "            # mask = 0(missing data) ; 1(data)\n",
    "          x_train_init = mask_train * x_train_missing + (1.-mask_train) * x_train_pred\n",
    "          x_test_init  = mask_test  * x_test_missing  + (1.-mask_test)  * x_test_pred\n",
    "    return global_model_FP,global_model_FP_Masked\n",
    "print('ready')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 163496,
     "status": "ok",
     "timestamp": 1614841976094,
     "user": {
      "displayName": "Jean-Marie VIENT",
      "photoUrl": "",
      "userId": "16174505464901920949"
     },
     "user_tz": -60
    },
    "id": "WWWtqOlmvpZ5",
    "outputId": "f13a3c12-1283-4a9b-c0b9-05145e62ea13"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/6 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "..... Start learning AE model\n",
      "..... Update/initialize number of projections in DINCOnvAE model # 5\n",
      "Model: \"model_11\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_14 (InputLayer)           [(None, 64, 64, 1)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_15 (InputLayer)           [(None, 64, 64, 1)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lambda_94 (Lambda)              (None, 64, 64, 1)    0           input_14[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_95 (Lambda)              (None, 64, 64, 1)    0           input_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "model_2 (Functional)            (None, 64, 64, 1)    83576       lambda_94[0][0]                  \n",
      "                                                                 input_15[0][0]                   \n",
      "                                                                 add_68[0][0]                     \n",
      "                                                                 input_15[0][0]                   \n",
      "                                                                 add_69[0][0]                     \n",
      "                                                                 input_15[0][0]                   \n",
      "                                                                 add_70[0][0]                     \n",
      "                                                                 input_15[0][0]                   \n",
      "                                                                 add_71[0][0]                     \n",
      "                                                                 input_15[0][0]                   \n",
      "                                                                 add_72[0][0]                     \n",
      "                                                                 input_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_97 (Lambda)              (None, 64, 64, 1)    0           lambda_94[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_98 (Lambda)              (None, 64, 64, 1)    0           input_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_96 (Lambda)              (None, 64, 64, 1)    0           lambda_95[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_85 (Multiply)          (None, 64, 64, 1)    0           lambda_97[0][0]                  \n",
      "                                                                 lambda_98[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_84 (Multiply)          (None, 64, 64, 1)    0           model_2[32][0]                   \n",
      "                                                                 lambda_96[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_68 (Add)                    (None, 64, 64, 1)    0           multiply_85[0][0]                \n",
      "                                                                 multiply_84[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "lambda_100 (Lambda)             (None, 64, 64, 1)    0           add_68[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "lambda_101 (Lambda)             (None, 64, 64, 1)    0           input_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_99 (Lambda)              (None, 64, 64, 1)    0           lambda_95[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_87 (Multiply)          (None, 64, 64, 1)    0           lambda_100[0][0]                 \n",
      "                                                                 lambda_101[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_86 (Multiply)          (None, 64, 64, 1)    0           model_2[33][0]                   \n",
      "                                                                 lambda_99[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_69 (Add)                    (None, 64, 64, 1)    0           multiply_87[0][0]                \n",
      "                                                                 multiply_86[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "lambda_103 (Lambda)             (None, 64, 64, 1)    0           add_69[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "lambda_104 (Lambda)             (None, 64, 64, 1)    0           input_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_102 (Lambda)             (None, 64, 64, 1)    0           lambda_95[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_89 (Multiply)          (None, 64, 64, 1)    0           lambda_103[0][0]                 \n",
      "                                                                 lambda_104[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_88 (Multiply)          (None, 64, 64, 1)    0           model_2[34][0]                   \n",
      "                                                                 lambda_102[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_70 (Add)                    (None, 64, 64, 1)    0           multiply_89[0][0]                \n",
      "                                                                 multiply_88[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "lambda_106 (Lambda)             (None, 64, 64, 1)    0           add_70[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "lambda_107 (Lambda)             (None, 64, 64, 1)    0           input_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_105 (Lambda)             (None, 64, 64, 1)    0           lambda_95[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_91 (Multiply)          (None, 64, 64, 1)    0           lambda_106[0][0]                 \n",
      "                                                                 lambda_107[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_90 (Multiply)          (None, 64, 64, 1)    0           model_2[35][0]                   \n",
      "                                                                 lambda_105[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_71 (Add)                    (None, 64, 64, 1)    0           multiply_91[0][0]                \n",
      "                                                                 multiply_90[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "lambda_109 (Lambda)             (None, 64, 64, 1)    0           add_71[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "lambda_110 (Lambda)             (None, 64, 64, 1)    0           input_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_108 (Lambda)             (None, 64, 64, 1)    0           lambda_95[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_93 (Multiply)          (None, 64, 64, 1)    0           lambda_109[0][0]                 \n",
      "                                                                 lambda_110[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_92 (Multiply)          (None, 64, 64, 1)    0           model_2[36][0]                   \n",
      "                                                                 lambda_108[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_72 (Add)                    (None, 64, 64, 1)    0           multiply_93[0][0]                \n",
      "                                                                 multiply_92[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 83,576\n",
      "Trainable params: 83,576\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Model: \"model_12\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_15 (InputLayer)           [(None, 64, 64, 1)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_14 (InputLayer)           [(None, 64, 64, 1)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lambda_111 (Lambda)             (None, 64, 64, 1)    0           input_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "model_11 (Functional)           (None, 64, 64, 1)    83576       input_14[0][0]                   \n",
      "                                                                 lambda_111[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_113 (Lambda)             (None, 64, 64, 1)    0           input_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "model_2 (Functional)            (None, 64, 64, 1)    83576       model_11[0][0]                   \n",
      "                                                                 lambda_113[0][0]                 \n",
      "                                                                 model_11[0][0]                   \n",
      "                                                                 lambda_115[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_115 (Lambda)             (None, 64, 64, 1)    0           input_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "subtract_12 (Subtract)          (None, 64, 64, 1)    0           model_11[0][0]                   \n",
      "                                                                 input_14[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "subtract_13 (Subtract)          (None, 64, 64, 1)    0           model_11[0][0]                   \n",
      "                                                                 model_2[38][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_94 (Multiply)          (None, 64, 64, 1)    0           subtract_12[0][0]                \n",
      "                                                                 input_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_96 (Multiply)          (None, 64, 64, 1)    0           subtract_13[0][0]                \n",
      "                                                                 input_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "subtract_14 (Subtract)          (None, 64, 64, 1)    0           model_2[39][0]                   \n",
      "                                                                 model_11[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_95 (Multiply)          (None, 64, 64, 1)    0           multiply_94[0][0]                \n",
      "                                                                 multiply_94[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "multiply_97 (Multiply)          (None, 64, 64, 1)    0           multiply_96[0][0]                \n",
      "                                                                 multiply_96[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "multiply_98 (Multiply)          (None, 64, 64, 1)    0           subtract_14[0][0]                \n",
      "                                                                 input_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "reshape_24 (Reshape)            (None, 64, 64, 1, 1) 0           multiply_95[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "reshape_26 (Reshape)            (None, 64, 64, 1, 1) 0           multiply_97[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "multiply_99 (Multiply)          (None, 64, 64, 1)    0           multiply_98[0][0]                \n",
      "                                                                 multiply_98[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling3d_12 (Gl (None, 1)            0           reshape_24[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling3d_13 (Gl (None, 1)            0           reshape_26[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "reshape_28 (Reshape)            (None, 64, 64, 1, 1) 0           multiply_99[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "reshape_25 (Reshape)            (None, 1)            0           global_average_pooling3d_12[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "reshape_27 (Reshape)            (None, 1)            0           global_average_pooling3d_13[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling3d_14 (Gl (None, 1)            0           reshape_28[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_112 (Lambda)             (None, 1)            0           reshape_25[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_114 (Lambda)             (None, 1)            0           reshape_27[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "reshape_29 (Reshape)            (None, 1)            0           global_average_pooling3d_14[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "add_73 (Add)                    (None, 1)            0           lambda_112[0][0]                 \n",
      "                                                                 lambda_114[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_116 (Lambda)             (None, 1)            0           reshape_29[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_74 (Add)                    (None, 1)            0           add_73[0][0]                     \n",
      "                                                                 lambda_116[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 83,576\n",
      "Trainable params: 83,576\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Epoch 1/6\n",
      "(11, 11, 1, 20)\n",
      "(11, 11, 1, 20)\n",
      "51/51 [==============================] - 13s 97ms/step - loss: 0.0220 - val_loss: 0.0118\n",
      "Epoch 2/6\n",
      "51/51 [==============================] - 4s 73ms/step - loss: 0.0113 - val_loss: 0.0110\n",
      "Epoch 3/6\n",
      "51/51 [==============================] - 4s 74ms/step - loss: 0.0107 - val_loss: 0.0109\n",
      "Epoch 4/6\n",
      "51/51 [==============================] - 3s 68ms/step - loss: 0.0106 - val_loss: 0.0108\n",
      "Epoch 5/6\n",
      "51/51 [==============================] - 4s 74ms/step - loss: 0.0107 - val_loss: 0.0109\n",
      "Epoch 6/6\n",
      "51/51 [==============================] - 4s 71ms/step - loss: 0.0108 - val_loss: 0.0116\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:149: RuntimeWarning: invalid value encountered in double_scalars\n",
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:95: RuntimeWarning: invalid value encountered in double_scalars\n",
      " 17%|█▋        | 1/6 [00:36<03:01, 36.28s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".......... Auto-encoder performance when applied to gap-free data\n",
      ".... explained variance AE (Tr)  : 91.42%\n",
      ".... explained variance AE (Tt)  : 91.14%\n",
      "Epoch 1/6\n",
      "51/51 [==============================] - 4s 71ms/step - loss: 0.0120 - val_loss: 0.0110\n",
      "Epoch 2/6\n",
      "51/51 [==============================] - 4s 78ms/step - loss: 0.0107 - val_loss: 0.0110\n",
      "Epoch 3/6\n",
      "51/51 [==============================] - 3s 67ms/step - loss: 0.0112 - val_loss: 0.0136\n",
      "Epoch 4/6\n",
      "51/51 [==============================] - 4s 71ms/step - loss: 0.0119 - val_loss: 0.0109\n",
      "Epoch 5/6\n",
      "51/51 [==============================] - 3s 66ms/step - loss: 0.0106 - val_loss: 0.0111\n",
      "Epoch 6/6\n",
      "51/51 [==============================] - 3s 67ms/step - loss: 0.0107 - val_loss: 0.0118\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 2/6 [00:59<02:09, 32.30s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".......... Auto-encoder performance when applied to gap-free data\n",
      ".... explained variance AE (Tr)  : 91.42%\n",
      ".... explained variance AE (Tt)  : 91.16%\n",
      "Epoch 1/6\n",
      "51/51 [==============================] - 4s 69ms/step - loss: 0.0123 - val_loss: 0.0111\n",
      "Epoch 2/6\n",
      "51/51 [==============================] - 3s 66ms/step - loss: 0.0106 - val_loss: 0.0107\n",
      "Epoch 3/6\n",
      "51/51 [==============================] - 3s 67ms/step - loss: 0.0105 - val_loss: 0.0114\n",
      "Epoch 4/6\n",
      "51/51 [==============================] - 3s 65ms/step - loss: 0.0112 - val_loss: 0.0117\n",
      "Epoch 5/6\n",
      "51/51 [==============================] - 4s 69ms/step - loss: 0.0107 - val_loss: 0.0107\n",
      "Epoch 6/6\n",
      "51/51 [==============================] - 4s 71ms/step - loss: 0.0104 - val_loss: 0.0106\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 3/6 [01:21<01:28, 29.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".......... Auto-encoder performance when applied to gap-free data\n",
      ".... explained variance AE (Tr)  : 92.16%\n",
      ".... explained variance AE (Tt)  : 91.86%\n",
      "..... Update/initialize number of projections in DINCOnvAE model # 5\n",
      "Model: \"model_13\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_16 (InputLayer)           [(None, 64, 64, 1)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_17 (InputLayer)           [(None, 64, 64, 1)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lambda_117 (Lambda)             (None, 64, 64, 1)    0           input_16[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_118 (Lambda)             (None, 64, 64, 1)    0           input_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "model_2 (Functional)            (None, 64, 64, 1)    83576       lambda_117[0][0]                 \n",
      "                                                                 input_17[0][0]                   \n",
      "                                                                 add_75[0][0]                     \n",
      "                                                                 input_17[0][0]                   \n",
      "                                                                 add_76[0][0]                     \n",
      "                                                                 input_17[0][0]                   \n",
      "                                                                 add_77[0][0]                     \n",
      "                                                                 input_17[0][0]                   \n",
      "                                                                 add_78[0][0]                     \n",
      "                                                                 input_17[0][0]                   \n",
      "                                                                 add_79[0][0]                     \n",
      "                                                                 input_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_120 (Lambda)             (None, 64, 64, 1)    0           lambda_117[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_121 (Lambda)             (None, 64, 64, 1)    0           input_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_119 (Lambda)             (None, 64, 64, 1)    0           lambda_118[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_101 (Multiply)         (None, 64, 64, 1)    0           lambda_120[0][0]                 \n",
      "                                                                 lambda_121[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_100 (Multiply)         (None, 64, 64, 1)    0           model_2[40][0]                   \n",
      "                                                                 lambda_119[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_75 (Add)                    (None, 64, 64, 1)    0           multiply_101[0][0]               \n",
      "                                                                 multiply_100[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "lambda_123 (Lambda)             (None, 64, 64, 1)    0           add_75[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "lambda_124 (Lambda)             (None, 64, 64, 1)    0           input_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_122 (Lambda)             (None, 64, 64, 1)    0           lambda_118[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_103 (Multiply)         (None, 64, 64, 1)    0           lambda_123[0][0]                 \n",
      "                                                                 lambda_124[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_102 (Multiply)         (None, 64, 64, 1)    0           model_2[41][0]                   \n",
      "                                                                 lambda_122[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_76 (Add)                    (None, 64, 64, 1)    0           multiply_103[0][0]               \n",
      "                                                                 multiply_102[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "lambda_126 (Lambda)             (None, 64, 64, 1)    0           add_76[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "lambda_127 (Lambda)             (None, 64, 64, 1)    0           input_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_125 (Lambda)             (None, 64, 64, 1)    0           lambda_118[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_105 (Multiply)         (None, 64, 64, 1)    0           lambda_126[0][0]                 \n",
      "                                                                 lambda_127[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_104 (Multiply)         (None, 64, 64, 1)    0           model_2[42][0]                   \n",
      "                                                                 lambda_125[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_77 (Add)                    (None, 64, 64, 1)    0           multiply_105[0][0]               \n",
      "                                                                 multiply_104[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "lambda_129 (Lambda)             (None, 64, 64, 1)    0           add_77[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "lambda_130 (Lambda)             (None, 64, 64, 1)    0           input_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_128 (Lambda)             (None, 64, 64, 1)    0           lambda_118[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_107 (Multiply)         (None, 64, 64, 1)    0           lambda_129[0][0]                 \n",
      "                                                                 lambda_130[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_106 (Multiply)         (None, 64, 64, 1)    0           model_2[43][0]                   \n",
      "                                                                 lambda_128[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_78 (Add)                    (None, 64, 64, 1)    0           multiply_107[0][0]               \n",
      "                                                                 multiply_106[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "lambda_132 (Lambda)             (None, 64, 64, 1)    0           add_78[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "lambda_133 (Lambda)             (None, 64, 64, 1)    0           input_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_131 (Lambda)             (None, 64, 64, 1)    0           lambda_118[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_109 (Multiply)         (None, 64, 64, 1)    0           lambda_132[0][0]                 \n",
      "                                                                 lambda_133[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_108 (Multiply)         (None, 64, 64, 1)    0           model_2[44][0]                   \n",
      "                                                                 lambda_131[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_79 (Add)                    (None, 64, 64, 1)    0           multiply_109[0][0]               \n",
      "                                                                 multiply_108[0][0]               \n",
      "==================================================================================================\n",
      "Total params: 83,576\n",
      "Trainable params: 83,576\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Model: \"model_14\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_17 (InputLayer)           [(None, 64, 64, 1)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_16 (InputLayer)           [(None, 64, 64, 1)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lambda_134 (Lambda)             (None, 64, 64, 1)    0           input_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "model_13 (Functional)           (None, 64, 64, 1)    83576       input_16[0][0]                   \n",
      "                                                                 lambda_134[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_136 (Lambda)             (None, 64, 64, 1)    0           input_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "model_2 (Functional)            (None, 64, 64, 1)    83576       model_13[0][0]                   \n",
      "                                                                 lambda_136[0][0]                 \n",
      "                                                                 model_13[0][0]                   \n",
      "                                                                 lambda_138[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_138 (Lambda)             (None, 64, 64, 1)    0           input_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "subtract_15 (Subtract)          (None, 64, 64, 1)    0           model_13[0][0]                   \n",
      "                                                                 input_16[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "subtract_16 (Subtract)          (None, 64, 64, 1)    0           model_13[0][0]                   \n",
      "                                                                 model_2[46][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_110 (Multiply)         (None, 64, 64, 1)    0           subtract_15[0][0]                \n",
      "                                                                 input_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_112 (Multiply)         (None, 64, 64, 1)    0           subtract_16[0][0]                \n",
      "                                                                 input_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "subtract_17 (Subtract)          (None, 64, 64, 1)    0           model_2[47][0]                   \n",
      "                                                                 model_13[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_111 (Multiply)         (None, 64, 64, 1)    0           multiply_110[0][0]               \n",
      "                                                                 multiply_110[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "multiply_113 (Multiply)         (None, 64, 64, 1)    0           multiply_112[0][0]               \n",
      "                                                                 multiply_112[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "multiply_114 (Multiply)         (None, 64, 64, 1)    0           subtract_17[0][0]                \n",
      "                                                                 input_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "reshape_30 (Reshape)            (None, 64, 64, 1, 1) 0           multiply_111[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "reshape_32 (Reshape)            (None, 64, 64, 1, 1) 0           multiply_113[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "multiply_115 (Multiply)         (None, 64, 64, 1)    0           multiply_114[0][0]               \n",
      "                                                                 multiply_114[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling3d_15 (Gl (None, 1)            0           reshape_30[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling3d_16 (Gl (None, 1)            0           reshape_32[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "reshape_34 (Reshape)            (None, 64, 64, 1, 1) 0           multiply_115[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "reshape_31 (Reshape)            (None, 1)            0           global_average_pooling3d_15[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "reshape_33 (Reshape)            (None, 1)            0           global_average_pooling3d_16[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling3d_17 (Gl (None, 1)            0           reshape_34[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_135 (Lambda)             (None, 1)            0           reshape_31[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_137 (Lambda)             (None, 1)            0           reshape_33[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "reshape_35 (Reshape)            (None, 1)            0           global_average_pooling3d_17[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "add_80 (Add)                    (None, 1)            0           lambda_135[0][0]                 \n",
      "                                                                 lambda_137[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_139 (Lambda)             (None, 1)            0           reshape_35[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_81 (Add)                    (None, 1)            0           add_80[0][0]                     \n",
      "                                                                 lambda_139[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 83,576\n",
      "Trainable params: 83,576\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Epoch 1/6\n",
      "(11, 11, 1, 20)\n",
      "(11, 11, 1, 20)\n",
      "51/51 [==============================] - 13s 95ms/step - loss: 0.0105 - val_loss: 0.0106\n",
      "Epoch 2/6\n",
      "51/51 [==============================] - 4s 73ms/step - loss: 0.0104 - val_loss: 0.0106\n",
      "Epoch 3/6\n",
      "51/51 [==============================] - 3s 68ms/step - loss: 0.0103 - val_loss: 0.0106\n",
      "Epoch 4/6\n",
      "51/51 [==============================] - 3s 66ms/step - loss: 0.0102 - val_loss: 0.0106\n",
      "Epoch 5/6\n",
      "51/51 [==============================] - 3s 67ms/step - loss: 0.0104 - val_loss: 0.0106\n",
      "Epoch 6/6\n",
      "51/51 [==============================] - 3s 66ms/step - loss: 0.0103 - val_loss: 0.0106\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 4/6 [01:56<01:02, 31.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".......... Auto-encoder performance when applied to gap-free data\n",
      ".... explained variance AE (Tr)  : 92.23%\n",
      ".... explained variance AE (Tt)  : 91.91%\n",
      "Epoch 1/6\n",
      "51/51 [==============================] - 4s 71ms/step - loss: 0.0103 - val_loss: 0.0105\n",
      "Epoch 2/6\n",
      "51/51 [==============================] - 4s 74ms/step - loss: 0.0103 - val_loss: 0.0106\n",
      "Epoch 3/6\n",
      "51/51 [==============================] - 4s 75ms/step - loss: 0.0103 - val_loss: 0.0105\n",
      "Epoch 4/6\n",
      "51/51 [==============================] - 4s 69ms/step - loss: 0.0102 - val_loss: 0.0105\n",
      "Epoch 5/6\n",
      "51/51 [==============================] - 3s 66ms/step - loss: 0.0102 - val_loss: 0.0105\n",
      "Epoch 6/6\n",
      "51/51 [==============================] - 3s 67ms/step - loss: 0.0102 - val_loss: 0.0105\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████▎ | 5/6 [02:19<00:28, 28.69s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".......... Auto-encoder performance when applied to gap-free data\n",
      ".... explained variance AE (Tr)  : 92.27%\n",
      ".... explained variance AE (Tt)  : 91.95%\n",
      "Epoch 1/6\n",
      "51/51 [==============================] - 4s 74ms/step - loss: 0.0102 - val_loss: 0.0105\n",
      "Epoch 2/6\n",
      "51/51 [==============================] - 4s 73ms/step - loss: 0.0102 - val_loss: 0.0105\n",
      "Epoch 3/6\n",
      "51/51 [==============================] - 3s 67ms/step - loss: 0.0102 - val_loss: 0.0105\n",
      "Epoch 4/6\n",
      "51/51 [==============================] - 4s 70ms/step - loss: 0.0102 - val_loss: 0.0105\n",
      "Epoch 5/6\n",
      "51/51 [==============================] - 4s 69ms/step - loss: 0.0102 - val_loss: 0.0105\n",
      "Epoch 6/6\n",
      "51/51 [==============================] - 3s 66ms/step - loss: 0.0102 - val_loss: 0.0104\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6/6 [02:42<00:00, 27.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".......... Auto-encoder performance when applied to gap-free data\n",
      ".... explained variance AE (Tr)  : 92.31%\n",
      ".... explained variance AE (Tt)  : 91.99%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "global_model_FP,global_model_FP_Masked=flagProcess4_Optim0(globParams,x_train,x_train_missing,mask_train,gt_train,x_test,x_test_missing,mask_test,gt_test,lday_test,encoder,decoder,model_AE,DimCAE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nYGiJA8lvpZ-"
   },
   "source": [
    "### Prédiction / Interpolation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Kl3eUXPVvpZ_"
   },
   "outputs": [],
   "source": [
    "Pred_GM = global_model_FP.predict([y_pred,mask_pred]).reshape(gt_pred.shape[0],gt_pred.shape[1],gt_pred.shape[2])\n",
    "Pred_AE = model_AE.predict([y_pred,mask_pred]).reshape(gt_pred.shape[0],gt_pred.shape[1],gt_pred.shape[2])\n",
    "Pred_AE_norm = (Pred_AE.reshape(len(y_pred),-1)+mean_Tr) .reshape(Pred_AE.shape[0],Pred_AE.shape[1],Pred_AE.shape[2])\n",
    "Pred_GM_norm = (Pred_GM.reshape(len(y_pred),-1)+mean_Tr).reshape(Pred_AE.shape[0],Pred_AE.shape[1],Pred_AE.shape[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "PqM26-Gjuw3y"
   },
   "outputs": [],
   "source": [
    "Pred_AE_norm[np.where(Pred_AE_norm<np.nanmin(gt_pred))]=np.nanmin(gt_pred)\n",
    "Pred_GM_norm[np.where(Pred_GM_norm<np.nanmin(gt_pred))]=np.nanmin(gt_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ObIf6haKOSsX"
   },
   "source": [
    "# Some metrics\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 439,
     "status": "ok",
     "timestamp": 1614842323149,
     "user": {
      "displayName": "Jean-Marie VIENT",
      "photoUrl": "",
      "userId": "16174505464901920949"
     },
     "user_tz": -60
    },
    "id": "DbOIMlSGURRi",
    "outputId": "602fd401-79b2-4894-e4af-d94709d0367d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'EVS': 0.9504256427614494, 'RMSE': 0.12629229049962115, 'NRMSE': -0.03392215609313714, 'MAbsEr%': 1.973055102032464, 'MaxEr': 1.5878864467702805, 'MAbsEr': 0.003409719793125987, 'R²': 0.9503258120518892} {'EVS': 0.9657846878409643, 'RMSE': 0.10483878622165772, 'NRMSE': -0.028159736883042567, 'MAbsEr%': 1.6352918187037824, 'MaxEr': 1.600932371020317, 'MAbsEr': 0.005255662966519692, 'R²': 0.9657688834767162}\n"
     ]
    }
   ],
   "source": [
    "#from sklearn.metrics import mean_squared_error\n",
    "from math import sqrt\n",
    "import numpy as np\n",
    "from sklearn.metrics import explained_variance_score\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.metrics import max_error\n",
    "from sklearn.metrics import median_absolute_error\n",
    "import datetime as dt\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def mean_absolute_percentage_error(y_true, y_pred):\n",
    "    y_true, y_pred = np.array(y_true), np.array(y_pred)\n",
    "    return np.nanmean(np.abs((y_true - y_pred) / y_true)) * 100\n",
    "\n",
    "def RMSE(a,b):\n",
    "    \"\"\" Compute the Root Mean Square Error between 2 n-dimensional vectors. \"\"\" \n",
    "    return np.sqrt(np.nanmean((a-b)**2))\n",
    "\n",
    "\n",
    "def Getmetrics(Target,Pred):\n",
    "    Target_value = Target.reshape(Target.shape)[np.where(~np.isnan(Target))]\n",
    "    Pred_value   = Pred.reshape(Target.shape)[np.where(~np.isnan(Target))]\n",
    "    Target_flat  = Target_value.flatten()\n",
    "    Pred_flat    = Pred_value.flatten()\n",
    "    Metrics=dict()\n",
    "\n",
    "    return {'EVS':explained_variance_score(Target_flat,Pred_flat),'RMSE':RMSE(Target_flat,Pred_flat),'NRMSE':RMSE(Target_flat,Pred_flat)/np.nanmean(Target_flat),'MAbsEr%':mean_absolute_percentage_error(Target_flat,Pred_flat),'MaxEr':max_error(Target_flat,Pred_flat),'MAbsEr':median_absolute_error(Target_flat,Pred_flat),'R²':r2_score(Target_flat,Pred_flat)}\n",
    "\n",
    "MetricGM=Getmetrics(gt_pred,Pred_GM_norm)\n",
    "MetricAE=Getmetrics(gt_pred,Pred_AE_norm)\n",
    "print(MetricGM,MetricAE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "7Unvvl9mB2Bm"
   },
   "outputs": [],
   "source": [
    "np.save('/content/drive/MyDrive/GENN_GM_norm2',Pred_GM_norm)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "GENN-Dec.ipynb",
   "provenance": [
    {
     "file_id": "1Vjn9BIJHgR1zctAUBI2s3ADmzSMDq5Jq",
     "timestamp": 1608122797975
    },
    {
     "file_id": "1GQIMp6sD21eL9UNUQqyr0vQZ7Qy03ow_",
     "timestamp": 1607607804514
    }
   ],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}