chapters/data_ingestion/data_ingestion.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "data_ingestion.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "48kkPFz6Isaa"
      },
      "source": [
        "# Data Ingestion\n",
        "\n",
        "- パッケージインストール後はランタイムを再起動すること（Colab）"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pKNZQHIgI1Vg"
      },
      "source": [
        "## Setup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oNjLqwMyI_uJ"
      },
      "source": [
        "### Install the packages"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "HdVbjN8TH6AV",
        "outputId": "9720daa1-946c-4260-c75b-2d2935c9efe5"
      },
      "source": [
        "!pip install -U tensorflow==2.4.1 tfx==0.27.0 pandas==1.2.3"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting tensorflow==2.4.1\n",
            "  Downloading tensorflow-2.4.1-cp37-cp37m-manylinux2010_x86_64.whl (394.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 394.3 MB 14 kB/s \n",
            "\u001b[?25hCollecting tfx==0.27.0\n",
            "  Downloading tfx-0.27.0-py3-none-any.whl (2.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.3 MB 39.0 MB/s \n",
            "\u001b[?25hCollecting pandas==1.2.3\n",
            "  Downloading pandas-1.2.3-cp37-cp37m-manylinux1_x86_64.whl (9.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 9.9 MB 10.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: astunparse~=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (1.6.3)\n",
            "Requirement already satisfied: absl-py~=0.10 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (0.12.0)\n",
            "Collecting gast==0.3.3\n",
            "  Downloading gast-0.3.3-py2.py3-none-any.whl (9.7 kB)\n",
            "Requirement already satisfied: numpy~=1.19.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (1.19.5)\n",
            "Requirement already satisfied: termcolor~=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (1.1.0)\n",
            "Requirement already satisfied: six~=1.15.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (1.15.0)\n",
            "Requirement already satisfied: typing-extensions~=3.7.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (3.7.4.3)\n",
            "Collecting tensorflow-estimator<2.5.0,>=2.4.0\n",
            "  Downloading tensorflow_estimator-2.4.0-py2.py3-none-any.whl (462 kB)\n",
            "\u001b[K     |████████████████████████████████| 462 kB 37.5 MB/s \n",
            "\u001b[?25hCollecting grpcio~=1.32.0\n",
            "  Downloading grpcio-1.32.0-cp37-cp37m-manylinux2014_x86_64.whl (3.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.8 MB 36.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: wheel~=0.35 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (0.37.0)\n",
            "Collecting h5py~=2.10.0\n",
            "  Downloading h5py-2.10.0-cp37-cp37m-manylinux1_x86_64.whl (2.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9 MB 37.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (3.17.3)\n",
            "Requirement already satisfied: opt-einsum~=3.3.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (3.3.0)\n",
            "Requirement already satisfied: keras-preprocessing~=1.1.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (1.1.2)\n",
            "Requirement already satisfied: flatbuffers~=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (1.12)\n",
            "Requirement already satisfied: wrapt~=1.12.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (1.12.1)\n",
            "Requirement already satisfied: google-pasta~=0.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (0.2.0)\n",
            "Requirement already satisfied: tensorboard~=2.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (2.5.0)\n",
            "Collecting keras-tuner<1.0.2,>=1\n",
            "  Downloading keras-tuner-1.0.1.tar.gz (54 kB)\n",
            "\u001b[K     |████████████████████████████████| 54 kB 2.4 MB/s \n",
            "\u001b[?25hCollecting tensorflow-transform<0.28,>=0.27\n",
            "  Downloading tensorflow_transform-0.27.0-py3-none-any.whl (386 kB)\n",
            "\u001b[K     |████████████████████████████████| 386 kB 34.7 MB/s \n",
            "\u001b[?25hCollecting tensorflow-hub<0.10,>=0.9.0\n",
            "  Downloading tensorflow_hub-0.9.0-py2.py3-none-any.whl (103 kB)\n",
            "\u001b[K     |████████████████████████████████| 103 kB 46.0 MB/s \n",
            "\u001b[?25hCollecting ml-metadata<0.28,>=0.27\n",
            "  Downloading ml_metadata-0.27.0-cp37-cp37m-manylinux2010_x86_64.whl (2.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9 MB 36.6 MB/s \n",
            "\u001b[?25hCollecting kubernetes<12,>=10.0.1\n",
            "  Downloading kubernetes-11.0.0-py3-none-any.whl (1.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.5 MB 46.6 MB/s \n",
            "\u001b[?25hCollecting ml-pipelines-sdk==0.27.0\n",
            "  Downloading ml_pipelines_sdk-0.27.0-py3-none-any.whl (1.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 44.9 MB/s \n",
            "\u001b[?25hCollecting tensorflow-data-validation<0.28,>=0.27\n",
            "  Downloading tensorflow_data_validation-0.27.0-cp37-cp37m-manylinux2010_x86_64.whl (1.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.3 MB 40.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: google-api-python-client<2,>=1.7.8 in /usr/local/lib/python3.7/dist-packages (from tfx==0.27.0) (1.12.8)\n",
            "Requirement already satisfied: jinja2<3,>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from tfx==0.27.0) (2.11.3)\n",
            "Collecting absl-py~=0.10\n",
            "  Downloading absl_py-0.10.0-py3-none-any.whl (127 kB)\n",
            "\u001b[K     |████████████████████████████████| 127 kB 61.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: click<8,>=7 in /usr/local/lib/python3.7/dist-packages (from tfx==0.27.0) (7.1.2)\n",
            "Collecting attrs<21,>=19.3.0\n",
            "  Downloading attrs-20.3.0-py2.py3-none-any.whl (49 kB)\n",
            "\u001b[K     |████████████████████████████████| 49 kB 6.1 MB/s \n",
            "\u001b[?25hCollecting tfx-bsl<0.28,>=0.27\n",
            "  Downloading tfx_bsl-0.27.1-cp37-cp37m-manylinux2010_x86_64.whl (2.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.2 MB 37.0 MB/s \n",
            "\u001b[?25hCollecting pyarrow<3,>=1\n",
            "  Downloading pyarrow-2.0.0-cp37-cp37m-manylinux2014_x86_64.whl (17.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 17.7 MB 73 kB/s \n",
            "\u001b[?25hCollecting tensorflow-serving-api!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,<3,>=1.15\n",
            "  Downloading tensorflow_serving_api-2.5.1-py2.py3-none-any.whl (38 kB)\n",
            "Collecting tensorflow-model-analysis<0.28,>=0.27\n",
            "  Downloading tensorflow_model_analysis-0.27.0-py3-none-any.whl (1.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.7 MB 23.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyyaml<6,>=3.12 in /usr/local/lib/python3.7/dist-packages (from tfx==0.27.0) (3.13)\n",
            "Collecting apache-beam[gcp]<3,>=2.27\n",
            "  Downloading apache_beam-2.31.0-cp37-cp37m-manylinux2010_x86_64.whl (9.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 9.7 MB 29.8 MB/s \n",
            "\u001b[?25hCollecting docker<5,>=4.1\n",
            "  Downloading docker-4.4.4-py2.py3-none-any.whl (147 kB)\n",
            "\u001b[K     |████████████████████████████████| 147 kB 60.2 MB/s \n",
            "\u001b[?25hCollecting tensorflow-cloud<0.2,>=0.1\n",
            "  Downloading tensorflow_cloud-0.1.16-py3-none-any.whl (92 kB)\n",
            "\u001b[K     |████████████████████████████████| 92 kB 564 kB/s \n",
            "\u001b[?25hRequirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas==1.2.3) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas==1.2.3) (2.8.2)\n",
            "Collecting requests<3.0.0,>=2.24.0\n",
            "  Downloading requests-2.26.0-py2.py3-none-any.whl (62 kB)\n",
            "\u001b[K     |████████████████████████████████| 62 kB 857 kB/s \n",
            "\u001b[?25hRequirement already satisfied: oauth2client<5,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.27->tfx==0.27.0) (4.1.3)\n",
            "Requirement already satisfied: pymongo<4.0.0,>=3.8.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.27->tfx==0.27.0) (3.12.0)\n",
            "Requirement already satisfied: crcmod<2.0,>=1.7 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.27->tfx==0.27.0) (1.7)\n",
            "Collecting dill<0.3.2,>=0.3.1.1\n",
            "  Downloading dill-0.3.1.1.tar.gz (151 kB)\n",
            "\u001b[K     |████████████████████████████████| 151 kB 53.8 MB/s \n",
            "\u001b[?25hCollecting fastavro<2,>=0.21.4\n",
            "  Downloading fastavro-1.4.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.3 MB 39.2 MB/s \n",
            "\u001b[?25hCollecting hdfs<3.0.0,>=2.1.0\n",
            "  Downloading hdfs-2.6.0-py3-none-any.whl (33 kB)\n",
            "Collecting avro-python3!=1.9.2,<1.10.0,>=1.8.1\n",
            "  Downloading avro-python3-1.9.2.1.tar.gz (37 kB)\n",
            "Requirement already satisfied: pydot<2,>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.27->tfx==0.27.0) (1.3.0)\n",
            "Requirement already satisfied: httplib2<0.20.0,>=0.8 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.27->tfx==0.27.0) (0.17.4)\n",
            "Collecting future<1.0.0,>=0.18.2\n",
            "  Downloading future-0.18.2.tar.gz (829 kB)\n",
            "\u001b[K     |████████████████████████████████| 829 kB 54.3 MB/s \n",
            "\u001b[?25hCollecting google-cloud-dlp<2,>=0.12.0\n",
            "  Downloading google_cloud_dlp-1.0.0-py2.py3-none-any.whl (169 kB)\n",
            "\u001b[K     |████████████████████████████████| 169 kB 48.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: google-cloud-core<2,>=0.28.1 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.27->tfx==0.27.0) (1.0.3)\n",
            "Collecting google-cloud-bigtable<2,>=0.31.1\n",
            "  Downloading google_cloud_bigtable-1.7.0-py2.py3-none-any.whl (267 kB)\n",
            "\u001b[K     |████████████████████████████████| 267 kB 51.0 MB/s \n",
            "\u001b[?25hCollecting google-cloud-language<2,>=1.3.0\n",
            "  Downloading google_cloud_language-1.3.0-py2.py3-none-any.whl (83 kB)\n",
            "\u001b[K     |████████████████████████████████| 83 kB 1.9 MB/s \n",
            "\u001b[?25hCollecting google-cloud-pubsub<2,>=0.39.0\n",
            "  Downloading google_cloud_pubsub-1.7.0-py2.py3-none-any.whl (144 kB)\n",
            "\u001b[K     |████████████████████████████████| 144 kB 50.3 MB/s \n",
            "\u001b[?25hCollecting google-apitools<0.5.32,>=0.5.31\n",
            "  Downloading google-apitools-0.5.31.tar.gz (173 kB)\n",
            "\u001b[K     |████████████████████████████████| 173 kB 46.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: cachetools<5,>=3.1.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.27->tfx==0.27.0) (4.2.2)\n",
            "Collecting google-cloud-vision<2,>=0.38.0\n",
            "  Downloading google_cloud_vision-1.0.0-py2.py3-none-any.whl (435 kB)\n",
            "\u001b[K     |████████████████████████████████| 435 kB 51.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: google-cloud-bigquery<3,>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.27->tfx==0.27.0) (1.21.0)\n",
            "Requirement already satisfied: google-cloud-datastore<2,>=1.7.1 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.27->tfx==0.27.0) (1.8.0)\n",
            "Collecting grpcio-gcp<1,>=0.2.2\n",
            "  Downloading grpcio_gcp-0.2.2-py2.py3-none-any.whl (9.4 kB)\n",
            "Requirement already satisfied: google-auth<2,>=1.18.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.27->tfx==0.27.0) (1.34.0)\n",
            "Collecting google-cloud-spanner<2,>=1.13.0\n",
            "  Downloading google_cloud_spanner-1.19.1-py2.py3-none-any.whl (255 kB)\n",
            "\u001b[K     |████████████████████████████████| 255 kB 60.8 MB/s \n",
            "\u001b[?25hCollecting google-cloud-profiler<4,>=3.0.4\n",
            "  Downloading google-cloud-profiler-3.0.5.tar.gz (34 kB)\n",
            "Collecting google-cloud-videointelligence<2,>=1.8.0\n",
            "  Downloading google_cloud_videointelligence-1.16.1-py2.py3-none-any.whl (183 kB)\n",
            "\u001b[K     |████████████████████████████████| 183 kB 50.7 MB/s \n",
            "\u001b[?25hCollecting websocket-client>=0.32.0\n",
            "  Downloading websocket_client-1.2.1-py2.py3-none-any.whl (52 kB)\n",
            "\u001b[K     |████████████████████████████████| 52 kB 1.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: google-auth-httplib2>=0.0.3 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client<2,>=1.7.8->tfx==0.27.0) (0.0.4)\n",
            "Requirement already satisfied: google-api-core<2dev,>=1.21.0 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client<2,>=1.7.8->tfx==0.27.0) (1.26.3)\n",
            "Requirement already satisfied: uritemplate<4dev,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client<2,>=1.7.8->tfx==0.27.0) (3.0.1)\n",
            "Requirement already satisfied: setuptools>=40.3.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2dev,>=1.21.0->google-api-python-client<2,>=1.7.8->tfx==0.27.0) (57.2.0)\n",
            "Requirement already satisfied: packaging>=14.3 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2dev,>=1.21.0->google-api-python-client<2,>=1.7.8->tfx==0.27.0) (21.0)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2dev,>=1.21.0->google-api-python-client<2,>=1.7.8->tfx==0.27.0) (1.53.0)\n",
            "Collecting fasteners>=0.14\n",
            "  Downloading fasteners-0.16.3-py2.py3-none-any.whl (28 kB)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.18.0->apache-beam[gcp]<3,>=2.27->tfx==0.27.0) (4.7.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.18.0->apache-beam[gcp]<3,>=2.27->tfx==0.27.0) (0.2.8)\n",
            "Requirement already satisfied: google-resumable-media!=0.4.0,<0.5.0dev,>=0.3.1 in /usr/local/lib/python3.7/dist-packages (from google-cloud-bigquery<3,>=1.6.0->apache-beam[gcp]<3,>=2.27->tfx==0.27.0) (0.4.1)\n",
            "Collecting grpc-google-iam-v1<0.13dev,>=0.12.3\n",
            "  Downloading grpc-google-iam-v1-0.12.3.tar.gz (13 kB)\n",
            "Collecting google-cloud-core<2,>=0.28.1\n",
            "  Downloading google_cloud_core-1.7.2-py2.py3-none-any.whl (28 kB)\n",
            "Requirement already satisfied: docopt in /usr/local/lib/python3.7/dist-packages (from hdfs<3.0.0,>=2.1.0->apache-beam[gcp]<3,>=2.27->tfx==0.27.0) (0.6.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2<3,>=2.7.3->tfx==0.27.0) (2.0.1)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.7/dist-packages (from keras-tuner<1.0.2,>=1->tfx==0.27.0) (0.8.9)\n",
            "Collecting terminaltables\n",
            "  Downloading terminaltables-3.1.0.tar.gz (12 kB)\n",
            "Collecting colorama\n",
            "  Downloading colorama-0.4.4-py2.py3-none-any.whl (16 kB)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from keras-tuner<1.0.2,>=1->tfx==0.27.0) (4.62.0)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from keras-tuner<1.0.2,>=1->tfx==0.27.0) (1.4.1)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from keras-tuner<1.0.2,>=1->tfx==0.27.0) (0.22.2.post1)\n",
            "Requirement already satisfied: urllib3>=1.24.2 in /usr/local/lib/python3.7/dist-packages (from kubernetes<12,>=10.0.1->tfx==0.27.0) (1.24.3)\n",
            "Requirement already satisfied: requests-oauthlib in /usr/local/lib/python3.7/dist-packages (from kubernetes<12,>=10.0.1->tfx==0.27.0) (1.3.0)\n",
            "Requirement already satisfied: certifi>=14.05.14 in /usr/local/lib/python3.7/dist-packages (from kubernetes<12,>=10.0.1->tfx==0.27.0) (2021.5.30)\n",
            "Requirement already satisfied: pyasn1>=0.1.7 in /usr/local/lib/python3.7/dist-packages (from oauth2client<5,>=2.0.1->apache-beam[gcp]<3,>=2.27->tfx==0.27.0) (0.4.8)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=14.3->google-api-core<2dev,>=1.21.0->google-api-python-client<2,>=1.7.8->tfx==0.27.0) (2.4.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.24.0->apache-beam[gcp]<3,>=2.27->tfx==0.27.0) (2.10)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.24.0->apache-beam[gcp]<3,>=2.27->tfx==0.27.0) (2.0.4)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.1) (1.0.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.1) (3.3.4)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.1) (0.6.1)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.1) (1.8.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.1) (0.4.5)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard~=2.4->tensorflow==2.4.1) (4.6.3)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib->kubernetes<12,>=10.0.1->tfx==0.27.0) (3.1.1)\n",
            "Requirement already satisfied: tensorflow-datasets in /usr/local/lib/python3.7/dist-packages (from tensorflow-cloud<0.2,>=0.1->tfx==0.27.0) (4.0.1)\n",
            "Requirement already satisfied: google-cloud-storage in /usr/local/lib/python3.7/dist-packages (from tensorflow-cloud<0.2,>=0.1->tfx==0.27.0) (1.18.1)\n",
            "Collecting joblib<0.15,>=0.12\n",
            "  Downloading joblib-0.14.1-py2.py3-none-any.whl (294 kB)\n",
            "\u001b[K     |████████████████████████████████| 294 kB 53.4 MB/s \n",
            "\u001b[?25hCollecting tensorflow-metadata<0.28,>=0.27\n",
            "  Downloading tensorflow_metadata-0.27.0-py3-none-any.whl (47 kB)\n",
            "\u001b[K     |████████████████████████████████| 47 kB 4.4 MB/s \n",
            "\u001b[?25hCollecting ipython<8,>=7\n",
            "  Downloading ipython-7.26.0-py3-none-any.whl (786 kB)\n",
            "\u001b[K     |████████████████████████████████| 786 kB 30.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: ipywidgets<8,>=7 in /usr/local/lib/python3.7/dist-packages (from tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (7.6.3)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.7/dist-packages (from ipython<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (2.6.1)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from ipython<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (4.4.2)\n",
            "Requirement already satisfied: matplotlib-inline in /usr/local/lib/python3.7/dist-packages (from ipython<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (0.1.2)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.7/dist-packages (from ipython<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (0.2.0)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (0.7.5)\n",
            "Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.7/dist-packages (from ipython<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (4.8.0)\n",
            "Collecting prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0\n",
            "  Downloading prompt_toolkit-3.0.19-py3-none-any.whl (368 kB)\n",
            "\u001b[K     |████████████████████████████████| 368 kB 44.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.7/dist-packages (from ipython<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (0.18.0)\n",
            "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.7/dist-packages (from ipython<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (5.0.5)\n",
            "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (1.0.0)\n",
            "Requirement already satisfied: nbformat>=4.2.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (5.1.3)\n",
            "Requirement already satisfied: widgetsnbextension~=3.5.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (3.5.1)\n",
            "Requirement already satisfied: ipykernel>=4.5.1 in /usr/local/lib/python3.7/dist-packages (from ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (4.10.1)\n",
            "Requirement already satisfied: tornado>=4.0 in /usr/local/lib/python3.7/dist-packages (from ipykernel>=4.5.1->ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (5.1.1)\n",
            "Requirement already satisfied: jupyter-client in /usr/local/lib/python3.7/dist-packages (from ipykernel>=4.5.1->ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (5.3.5)\n",
            "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /usr/local/lib/python3.7/dist-packages (from jedi>=0.16->ipython<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (0.8.2)\n",
            "Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /usr/local/lib/python3.7/dist-packages (from nbformat>=4.2.0->ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (2.6.0)\n",
            "Requirement already satisfied: jupyter-core in /usr/local/lib/python3.7/dist-packages (from nbformat>=4.2.0->ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (4.7.1)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.7/dist-packages (from nbformat>=4.2.0->ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (0.2.0)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.7/dist-packages (from pexpect>4.3->ipython<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (0.7.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (0.2.5)\n",
            "Collecting tensorflow-serving-api!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,<3,>=1.15\n",
            "  Downloading tensorflow_serving_api-2.4.2-py2.py3-none-any.whl (38 kB)\n",
            "  Downloading tensorflow_serving_api-2.4.1-py2.py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: notebook>=4.4.1 in /usr/local/lib/python3.7/dist-packages (from widgetsnbextension~=3.5.0->ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (5.3.1)\n",
            "Requirement already satisfied: terminado>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (0.10.1)\n",
            "Requirement already satisfied: nbconvert in /usr/local/lib/python3.7/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (5.6.1)\n",
            "Requirement already satisfied: Send2Trash in /usr/local/lib/python3.7/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (1.8.0)\n",
            "Requirement already satisfied: pyzmq>=13 in /usr/local/lib/python3.7/dist-packages (from jupyter-client->ipykernel>=4.5.1->ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (22.2.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->markdown>=2.6.8->tensorboard~=2.4->tensorflow==2.4.1) (3.5.0)\n",
            "Requirement already satisfied: mistune<2,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (0.8.4)\n",
            "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (1.4.3)\n",
            "Requirement already satisfied: defusedxml in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (0.7.1)\n",
            "Requirement already satisfied: entrypoints>=0.2.2 in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (0.3)\n",
            "Requirement already satisfied: bleach in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (4.0.0)\n",
            "Requirement already satisfied: testpath in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (0.5.0)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.7/dist-packages (from bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets<8,>=7->tensorflow-model-analysis<0.28,>=0.27->tfx==0.27.0) (0.5.1)\n",
            "Requirement already satisfied: promise in /usr/local/lib/python3.7/dist-packages (from tensorflow-datasets->tensorflow-cloud<0.2,>=0.1->tfx==0.27.0) (2.3)\n",
            "Requirement already satisfied: dm-tree in /usr/local/lib/python3.7/dist-packages (from tensorflow-datasets->tensorflow-cloud<0.2,>=0.1->tfx==0.27.0) (0.1.6)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.7/dist-packages (from tensorflow-datasets->tensorflow-cloud<0.2,>=0.1->tfx==0.27.0) (5.2.2)\n",
            "Building wheels for collected packages: avro-python3, dill, future, google-apitools, google-cloud-profiler, grpc-google-iam-v1, keras-tuner, terminaltables\n",
            "  Building wheel for avro-python3 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for avro-python3: filename=avro_python3-1.9.2.1-py3-none-any.whl size=43512 sha256=be3cbbee7ce44866d361aa33c9d527bfa925adda7bc0f38de6be153fa35f622f\n",
            "  Stored in directory: /root/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1\n",
            "  Building wheel for dill (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for dill: filename=dill-0.3.1.1-py3-none-any.whl size=78546 sha256=164340e4eaafa2b77711cd09173cf07b0ae085a9c98fe1e53e12041431c635da\n",
            "  Stored in directory: /root/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f\n",
            "  Building wheel for future (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for future: filename=future-0.18.2-py3-none-any.whl size=491070 sha256=18227f1112b820b9511a953ef7204a39522e05f77435a9f31d93ffcaad6b76ca\n",
            "  Stored in directory: /root/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0\n",
            "  Building wheel for google-apitools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for google-apitools: filename=google_apitools-0.5.31-py3-none-any.whl size=131040 sha256=d290a0bea44bc2b2fa0587ca07827f651728410018b156ef165c2cb53e3498a9\n",
            "  Stored in directory: /root/.cache/pip/wheels/19/b5/2f/1cc3cf2b31e7a9cd1508731212526d9550271274d351c96f16\n",
            "  Building wheel for google-cloud-profiler (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for google-cloud-profiler: filename=google_cloud_profiler-3.0.5-cp37-cp37m-linux_x86_64.whl size=703454 sha256=b8da4a22c5182d51b1e9d004654fd48acf41dc3589a83820e51be45898fbc60b\n",
            "  Stored in directory: /root/.cache/pip/wheels/9f/24/e7/54a8a406d778d2d81aafc6d45b6cc9cca479373d2037f9102a\n",
            "  Building wheel for grpc-google-iam-v1 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for grpc-google-iam-v1: filename=grpc_google_iam_v1-0.12.3-py3-none-any.whl size=18515 sha256=04085f856e1dd094e1c66e65ec5fd6d2fe6ae68711d108df30253bb2ef6fac65\n",
            "  Stored in directory: /root/.cache/pip/wheels/b9/ee/67/2e444183030cb8d31ce8b34cee34a7afdbd3ba5959ea846380\n",
            "  Building wheel for keras-tuner (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-tuner: filename=keras_tuner-1.0.1-py3-none-any.whl size=73198 sha256=7c7240d8a82fd362456af1b10716c34b16bf1ba05606622dca1e5004fa78e17b\n",
            "  Stored in directory: /root/.cache/pip/wheels/0b/cf/2f/1a1749d3a3650fac3305a8d7f9237b6de7c41068e2f8520ca2\n",
            "  Building wheel for terminaltables (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for terminaltables: filename=terminaltables-3.1.0-py3-none-any.whl size=15354 sha256=3c2e1624e36717731a63951488cc8e7e9488edd2ca2373a069fa4b0e04ea41b9\n",
            "  Stored in directory: /root/.cache/pip/wheels/ba/ad/c8/2d98360791161cd3db6daf6b5e730f34021fc9367d5879f497\n",
            "Successfully built avro-python3 dill future google-apitools google-cloud-profiler grpc-google-iam-v1 keras-tuner terminaltables\n",
            "Installing collected packages: requests, grpcio, prompt-toolkit, grpcio-gcp, absl-py, tensorflow-estimator, pyarrow, ipython, hdfs, h5py, grpc-google-iam-v1, google-cloud-core, gast, future, fasteners, fastavro, dill, avro-python3, tensorflow, google-cloud-vision, google-cloud-videointelligence, google-cloud-spanner, google-cloud-pubsub, google-cloud-profiler, google-cloud-language, google-cloud-dlp, google-cloud-bigtable, google-apitools, apache-beam, tensorflow-serving-api, tensorflow-metadata, pandas, joblib, websocket-client, tfx-bsl, terminaltables, colorama, attrs, tensorflow-transform, ml-metadata, keras-tuner, docker, tensorflow-model-analysis, tensorflow-hub, tensorflow-data-validation, tensorflow-cloud, ml-pipelines-sdk, kubernetes, tfx\n",
            "  Attempting uninstall: requests\n",
            "    Found existing installation: requests 2.23.0\n",
            "    Uninstalling requests-2.23.0:\n",
            "      Successfully uninstalled requests-2.23.0\n",
            "  Attempting uninstall: grpcio\n",
            "    Found existing installation: grpcio 1.34.1\n",
            "    Uninstalling grpcio-1.34.1:\n",
            "      Successfully uninstalled grpcio-1.34.1\n",
            "  Attempting uninstall: prompt-toolkit\n",
            "    Found existing installation: prompt-toolkit 1.0.18\n",
            "    Uninstalling prompt-toolkit-1.0.18:\n",
            "      Successfully uninstalled prompt-toolkit-1.0.18\n",
            "  Attempting uninstall: absl-py\n",
            "    Found existing installation: absl-py 0.12.0\n",
            "    Uninstalling absl-py-0.12.0:\n",
            "      Successfully uninstalled absl-py-0.12.0\n",
            "  Attempting uninstall: tensorflow-estimator\n",
            "    Found existing installation: tensorflow-estimator 2.5.0\n",
            "    Uninstalling tensorflow-estimator-2.5.0:\n",
            "      Successfully uninstalled tensorflow-estimator-2.5.0\n",
            "  Attempting uninstall: pyarrow\n",
            "    Found existing installation: pyarrow 3.0.0\n",
            "    Uninstalling pyarrow-3.0.0:\n",
            "      Successfully uninstalled pyarrow-3.0.0\n",
            "  Attempting uninstall: ipython\n",
            "    Found existing installation: ipython 5.5.0\n",
            "    Uninstalling ipython-5.5.0:\n",
            "      Successfully uninstalled ipython-5.5.0\n",
            "  Attempting uninstall: h5py\n",
            "    Found existing installation: h5py 3.1.0\n",
            "    Uninstalling h5py-3.1.0:\n",
            "      Successfully uninstalled h5py-3.1.0\n",
            "  Attempting uninstall: google-cloud-core\n",
            "    Found existing installation: google-cloud-core 1.0.3\n",
            "    Uninstalling google-cloud-core-1.0.3:\n",
            "      Successfully uninstalled google-cloud-core-1.0.3\n",
            "  Attempting uninstall: gast\n",
            "    Found existing installation: gast 0.4.0\n",
            "    Uninstalling gast-0.4.0:\n",
            "      Successfully uninstalled gast-0.4.0\n",
            "  Attempting uninstall: future\n",
            "    Found existing installation: future 0.16.0\n",
            "    Uninstalling future-0.16.0:\n",
            "      Successfully uninstalled future-0.16.0\n",
            "  Attempting uninstall: dill\n",
            "    Found existing installation: dill 0.3.4\n",
            "    Uninstalling dill-0.3.4:\n",
            "      Successfully uninstalled dill-0.3.4\n",
            "  Attempting uninstall: tensorflow\n",
            "    Found existing installation: tensorflow 2.5.0\n",
            "    Uninstalling tensorflow-2.5.0:\n",
            "      Successfully uninstalled tensorflow-2.5.0\n",
            "  Attempting uninstall: google-cloud-language\n",
            "    Found existing installation: google-cloud-language 1.2.0\n",
            "    Uninstalling google-cloud-language-1.2.0:\n",
            "      Successfully uninstalled google-cloud-language-1.2.0\n",
            "  Attempting uninstall: tensorflow-metadata\n",
            "    Found existing installation: tensorflow-metadata 1.2.0\n",
            "    Uninstalling tensorflow-metadata-1.2.0:\n",
            "      Successfully uninstalled tensorflow-metadata-1.2.0\n",
            "  Attempting uninstall: pandas\n",
            "    Found existing installation: pandas 1.1.5\n",
            "    Uninstalling pandas-1.1.5:\n",
            "      Successfully uninstalled pandas-1.1.5\n",
            "  Attempting uninstall: joblib\n",
            "    Found existing installation: joblib 1.0.1\n",
            "    Uninstalling joblib-1.0.1:\n",
            "      Successfully uninstalled joblib-1.0.1\n",
            "  Attempting uninstall: attrs\n",
            "    Found existing installation: attrs 21.2.0\n",
            "    Uninstalling attrs-21.2.0:\n",
            "      Successfully uninstalled attrs-21.2.0\n",
            "  Attempting uninstall: tensorflow-hub\n",
            "    Found existing installation: tensorflow-hub 0.12.0\n",
            "    Uninstalling tensorflow-hub-0.12.0:\n",
            "      Successfully uninstalled tensorflow-hub-0.12.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "multiprocess 0.70.12.2 requires dill>=0.3.4, but you have dill 0.3.1.1 which is incompatible.\n",
            "jupyter-console 5.2.0 requires prompt-toolkit<2.0.0,>=1.0.0, but you have prompt-toolkit 3.0.19 which is incompatible.\n",
            "google-colab 1.0.0 requires ipython~=5.5.0, but you have ipython 7.26.0 which is incompatible.\n",
            "google-colab 1.0.0 requires pandas~=1.1.0; python_version >= \"3.0\", but you have pandas 1.2.3 which is incompatible.\n",
            "google-colab 1.0.0 requires requests~=2.23.0, but you have requests 2.26.0 which is incompatible.\n",
            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "Successfully installed absl-py-0.10.0 apache-beam-2.31.0 attrs-20.3.0 avro-python3-1.9.2.1 colorama-0.4.4 dill-0.3.1.1 docker-4.4.4 fastavro-1.4.4 fasteners-0.16.3 future-0.18.2 gast-0.3.3 google-apitools-0.5.31 google-cloud-bigtable-1.7.0 google-cloud-core-1.7.2 google-cloud-dlp-1.0.0 google-cloud-language-1.3.0 google-cloud-profiler-3.0.5 google-cloud-pubsub-1.7.0 google-cloud-spanner-1.19.1 google-cloud-videointelligence-1.16.1 google-cloud-vision-1.0.0 grpc-google-iam-v1-0.12.3 grpcio-1.32.0 grpcio-gcp-0.2.2 h5py-2.10.0 hdfs-2.6.0 ipython-7.26.0 joblib-0.14.1 keras-tuner-1.0.1 kubernetes-11.0.0 ml-metadata-0.27.0 ml-pipelines-sdk-0.27.0 pandas-1.2.3 prompt-toolkit-3.0.19 pyarrow-2.0.0 requests-2.26.0 tensorflow-2.4.1 tensorflow-cloud-0.1.16 tensorflow-data-validation-0.27.0 tensorflow-estimator-2.4.0 tensorflow-hub-0.9.0 tensorflow-metadata-0.27.0 tensorflow-model-analysis-0.27.0 tensorflow-serving-api-2.4.1 tensorflow-transform-0.27.0 terminaltables-3.1.0 tfx-0.27.0 tfx-bsl-0.27.1 websocket-client-1.2.1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "IPython",
                  "google",
                  "pandas",
                  "prompt_toolkit"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M2Y3N070JCEu"
      },
      "source": [
        "### Import the packages"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iR4CD1SJJISE"
      },
      "source": [
        "import logging\n",
        "import os\n",
        "import sys\n",
        "\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "from tfx.components import CsvExampleGen\n",
        "from tfx.orchestration.experimental.interactive.interactive_context import InteractiveContext\n",
        "from tfx.proto import example_gen_pb2"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nN_l7xXlJZbc"
      },
      "source": [
        "## The dataset\n",
        "\n",
        "### Download the dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hvBNrT9JIAkt"
      },
      "source": [
        "!mkdir data"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YuwHpjcFIF5x"
      },
      "source": [
        "import logging\n",
        "import pandas as pd\n",
        "\n",
        "# Initial dataset source\n",
        "DATASET_URL = \"http://bit.ly/building-ml-pipelines-dataset\"\n",
        "\n",
        "# Initial local dataset location\n",
        "LOCAL_FILE_NAME = \"data/consumer_complaints_with_narrative.csv\"\n",
        "\n",
        "\n",
        "def download_dataset(url=DATASET_URL):\n",
        "    \"\"\"download_dataset downloads the remote dataset to a local path\n",
        "\n",
        "    Keyword Arguments:\n",
        "        url {string} --\n",
        "            complete url path to the csv data source (default: {DATASET_URL})\n",
        "        local_path {string} --\n",
        "            initial local file location (default: {LOCAL_FILE_NAME})\n",
        "    Returns:\n",
        "        None\n",
        "    \"\"\"\n",
        "    df = pd.read_csv(url, index_col=0)\n",
        "    df.to_csv(LOCAL_FILE_NAME)\n",
        "    logging.info(\"Download completed.\")\n",
        "\n",
        "\n",
        "download_dataset()"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P9ARrXBQJdta"
      },
      "source": [
        "### Load the dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 530
        },
        "id": "nOq35aDJIJCU",
        "outputId": "dba9c5eb-654d-4cb5-a925-fb87a4ceee32"
      },
      "source": [
        "df = pd.read_csv(LOCAL_FILE_NAME)\n",
        "df.head()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>product</th>\n",
              "      <th>sub_product</th>\n",
              "      <th>issue</th>\n",
              "      <th>sub_issue</th>\n",
              "      <th>consumer_complaint_narrative</th>\n",
              "      <th>company</th>\n",
              "      <th>state</th>\n",
              "      <th>zip_code</th>\n",
              "      <th>company_response</th>\n",
              "      <th>timely_response</th>\n",
              "      <th>consumer_disputed</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Debt collection</td>\n",
              "      <td>I do not know</td>\n",
              "      <td>Disclosure verification of debt</td>\n",
              "      <td>Right to dispute notice not received</td>\n",
              "      <td>I was denied employment because of a judgment ...</td>\n",
              "      <td>Encore Capital Group</td>\n",
              "      <td>NY</td>\n",
              "      <td>113XX</td>\n",
              "      <td>Closed with explanation</td>\n",
              "      <td>Yes</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Credit reporting</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Improper use of my credit report</td>\n",
              "      <td>Report improperly shared by CRC</td>\n",
              "      <td>I have a credit card through XXXX XXXX and XXX...</td>\n",
              "      <td>Experian</td>\n",
              "      <td>IL</td>\n",
              "      <td>606XX</td>\n",
              "      <td>Closed with non-monetary relief</td>\n",
              "      <td>Yes</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Debt collection</td>\n",
              "      <td>I do not know</td>\n",
              "      <td>Cont'd attempts collect debt not owed</td>\n",
              "      <td>Debt is not mine</td>\n",
              "      <td>Almost daily phone calls from Stellar Recovery...</td>\n",
              "      <td>Stellar Recovery Inc.</td>\n",
              "      <td>MI</td>\n",
              "      <td>480XX</td>\n",
              "      <td>Closed with explanation</td>\n",
              "      <td>Yes</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Mortgage</td>\n",
              "      <td>Conventional fixed mortgage</td>\n",
              "      <td>Loan servicing, payments, escrow account</td>\n",
              "      <td>NaN</td>\n",
              "      <td>I submitted my monthly mortgage payment to Pri...</td>\n",
              "      <td>Primary Residential Mortgage</td>\n",
              "      <td>CT</td>\n",
              "      <td>066XX</td>\n",
              "      <td>Closed with monetary relief</td>\n",
              "      <td>Yes</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Student loan</td>\n",
              "      <td>Non-federal student loan</td>\n",
              "      <td>Dealing with my lender or servicer</td>\n",
              "      <td>Received bad information about my loan</td>\n",
              "      <td>I contacted America Education Services in XX/X...</td>\n",
              "      <td>AES/PHEAA</td>\n",
              "      <td>FL</td>\n",
              "      <td>321XX</td>\n",
              "      <td>Closed with explanation</td>\n",
              "      <td>Yes</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            product  ... consumer_disputed\n",
              "0   Debt collection  ...                 0\n",
              "1  Credit reporting  ...                 0\n",
              "2   Debt collection  ...                 1\n",
              "3          Mortgage  ...                 0\n",
              "4      Student loan  ...                 1\n",
              "\n",
              "[5 rows x 11 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tomwAUYYJsSc"
      },
      "source": [
        "## Experiments\n",
        "\n",
        "### First ingestion"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ngcKmXujIOJj",
        "outputId": "eb7644fd-5fe9-4261-fc98-104f629eac69"
      },
      "source": [
        "context = InteractiveContext(pipeline_root='tfx')"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:absl:InteractiveContext metadata_connection_config not provided: using SQLite ML Metadata database at tfx/metadata.sqlite.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 230
        },
        "id": "smaEac9yIRKE",
        "outputId": "485dd919-e148-4271-9f33-194ad48e0409"
      },
      "source": [
        "data_dir = 'data'\n",
        "example_gen = CsvExampleGen(input_base=data_dir)\n",
        "context.run(example_gen)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.\n",
            "WARNING:apache_beam.runners.interactive.interactive_environment:Dependencies required for Interactive Beam PCollection visualization are not available, please use: `pip install apache-beam[interactive]` to install necessary dependencies to enable all data visualization features.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "        if (typeof window.interactive_beam_jquery == 'undefined') {\n",
              "          var jqueryScript = document.createElement('script');\n",
              "          jqueryScript.src = 'https://code.jquery.com/jquery-3.4.1.slim.min.js';\n",
              "          jqueryScript.type = 'text/javascript';\n",
              "          jqueryScript.onload = function() {\n",
              "            var datatableScript = document.createElement('script');\n",
              "            datatableScript.src = 'https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js';\n",
              "            datatableScript.type = 'text/javascript';\n",
              "            datatableScript.onload = function() {\n",
              "              window.interactive_beam_jquery = jQuery.noConflict(true);\n",
              "              window.interactive_beam_jquery(document).ready(function($){\n",
              "                \n",
              "              });\n",
              "            }\n",
              "            document.head.appendChild(datatableScript);\n",
              "          };\n",
              "          document.head.appendChild(jqueryScript);\n",
              "        } else {\n",
              "          window.interactive_beam_jquery(document).ready(function($){\n",
              "            \n",
              "          });\n",
              "        }"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:apache_beam.io.tfrecordio:Couldn't find python-snappy so the implementation of _TFRecordUtil._masked_crc32c is not as fast as it could be.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object expanded\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">ExecutionResult</span><span class=\"deemphasize\"> at 0x7f0a50ab18d0</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.execution_id</td><td class = \"attrvalue\">1</td></tr><tr><td class=\"attr-name\">.component</td><td class = \"attrvalue\"><style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object collapsed\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">CsvExampleGen</span><span class=\"deemphasize\"> at 0x7f09dc3c6990</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.inputs</td><td class = \"attrvalue\">{}</td></tr><tr><td class=\"attr-name\">.outputs</td><td class = \"attrvalue\"><table class=\"attr-table\"><tr><td class=\"attr-name\">['examples']</td><td class = \"attrvalue\"><style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object collapsed\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">Channel</span> of type <span class=\"class-name\">'Examples'</span> (1 artifact)<span class=\"deemphasize\"> at 0x7f09dc3c6d90</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.type_name</td><td class = \"attrvalue\">Examples</td></tr><tr><td class=\"attr-name\">._artifacts</td><td class = \"attrvalue\"><table class=\"attr-table\"><tr><td class=\"attr-name\">[0]</td><td class = \"attrvalue\"><style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object collapsed\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">Artifact</span> of type <span class=\"class-name\">'Examples'</span> (uri: tfx/CsvExampleGen/examples/1)<span class=\"deemphasize\"> at 0x7f09dc3cf950</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.type</td><td class = \"attrvalue\">&lt;class &#x27;tfx.types.standard_artifacts.Examples&#x27;&gt;</td></tr><tr><td class=\"attr-name\">.uri</td><td class = \"attrvalue\">tfx/CsvExampleGen/examples/1</td></tr><tr><td class=\"attr-name\">.span</td><td class = \"attrvalue\">0</td></tr><tr><td class=\"attr-name\">.split_names</td><td class = \"attrvalue\">[&quot;train&quot;, &quot;eval&quot;]</td></tr><tr><td class=\"attr-name\">.version</td><td class = \"attrvalue\">0</td></tr></table></div></td></tr></table></td></tr></table></div></td></tr></table></td></tr><tr><td class=\"attr-name\">.exec_properties</td><td class = \"attrvalue\"><table class=\"attr-table\"><tr><td class=\"attr-name\">['input_base']</td><td class = \"attrvalue\">data</td></tr><tr><td class=\"attr-name\">['input_config']</td><td class = \"attrvalue\">{\n",
              "  &quot;splits&quot;: [\n",
              "    {\n",
              "      &quot;name&quot;: &quot;single_split&quot;,\n",
              "      &quot;pattern&quot;: &quot;*&quot;\n",
              "    }\n",
              "  ]\n",
              "}</td></tr><tr><td class=\"attr-name\">['output_config']</td><td class = \"attrvalue\">{\n",
              "  &quot;split_config&quot;: {\n",
              "    &quot;splits&quot;: [\n",
              "      {\n",
              "        &quot;hash_buckets&quot;: 2,\n",
              "        &quot;name&quot;: &quot;train&quot;\n",
              "      },\n",
              "      {\n",
              "        &quot;hash_buckets&quot;: 1,\n",
              "        &quot;name&quot;: &quot;eval&quot;\n",
              "      }\n",
              "    ]\n",
              "  }\n",
              "}</td></tr><tr><td class=\"attr-name\">['output_data_format']</td><td class = \"attrvalue\">6</td></tr><tr><td class=\"attr-name\">['custom_config']</td><td class = \"attrvalue\">None</td></tr><tr><td class=\"attr-name\">['range_config']</td><td class = \"attrvalue\">None</td></tr><tr><td class=\"attr-name\">['span']</td><td class = \"attrvalue\">0</td></tr><tr><td class=\"attr-name\">['version']</td><td class = \"attrvalue\">None</td></tr><tr><td class=\"attr-name\">['input_fingerprint']</td><td class = \"attrvalue\">split:single_split,num_files:1,total_bytes:78956236,xor_checksum:1629119443,sum_checksum:1629119443</td></tr></table></td></tr></table></div></td></tr><tr><td class=\"attr-name\">.component.inputs</td><td class = \"attrvalue\">{}</td></tr><tr><td class=\"attr-name\">.component.outputs</td><td class = \"attrvalue\"><table class=\"attr-table\"><tr><td class=\"attr-name\">['examples']</td><td class = \"attrvalue\"><style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object collapsed\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">Channel</span> of type <span class=\"class-name\">'Examples'</span> (1 artifact)<span class=\"deemphasize\"> at 0x7f09dc3c6d90</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.type_name</td><td class = \"attrvalue\">Examples</td></tr><tr><td class=\"attr-name\">._artifacts</td><td class = \"attrvalue\"><table class=\"attr-table\"><tr><td class=\"attr-name\">[0]</td><td class = \"attrvalue\"><style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object collapsed\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">Artifact</span> of type <span class=\"class-name\">'Examples'</span> (uri: tfx/CsvExampleGen/examples/1)<span class=\"deemphasize\"> at 0x7f09dc3cf950</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.type</td><td class = \"attrvalue\">&lt;class &#x27;tfx.types.standard_artifacts.Examples&#x27;&gt;</td></tr><tr><td class=\"attr-name\">.uri</td><td class = \"attrvalue\">tfx/CsvExampleGen/examples/1</td></tr><tr><td class=\"attr-name\">.span</td><td class = \"attrvalue\">0</td></tr><tr><td class=\"attr-name\">.split_names</td><td class = \"attrvalue\">[&quot;train&quot;, &quot;eval&quot;]</td></tr><tr><td class=\"attr-name\">.version</td><td class = \"attrvalue\">0</td></tr></table></div></td></tr></table></td></tr></table></div></td></tr></table></td></tr></table></div>"
            ],
            "text/plain": [
              "ExecutionResult(\n",
              "    component_id: CsvExampleGen\n",
              "    execution_id: 1\n",
              "    outputs:\n",
              "        examples: Channel(\n",
              "            type_name: Examples\n",
              "            artifacts: [Artifact(artifact: id: 1\n",
              "        type_id: 5\n",
              "        uri: \"tfx/CsvExampleGen/examples/1\"\n",
              "        properties {\n",
              "          key: \"split_names\"\n",
              "          value {\n",
              "            string_value: \"[\\\"train\\\", \\\"eval\\\"]\"\n",
              "          }\n",
              "        }\n",
              "        custom_properties {\n",
              "          key: \"input_fingerprint\"\n",
              "          value {\n",
              "            string_value: \"split:single_split,num_files:1,total_bytes:78956236,xor_checksum:1629119443,sum_checksum:1629119443\"\n",
              "          }\n",
              "        }\n",
              "        custom_properties {\n",
              "          key: \"payload_format\"\n",
              "          value {\n",
              "            string_value: \"FORMAT_TF_EXAMPLE\"\n",
              "          }\n",
              "        }\n",
              "        custom_properties {\n",
              "          key: \"span\"\n",
              "          value {\n",
              "            string_value: \"0\"\n",
              "          }\n",
              "        }\n",
              "        custom_properties {\n",
              "          key: \"state\"\n",
              "          value {\n",
              "            string_value: \"published\"\n",
              "          }\n",
              "        }\n",
              "        state: LIVE\n",
              "        , artifact_type: id: 5\n",
              "        name: \"Examples\"\n",
              "        properties {\n",
              "          key: \"span\"\n",
              "          value: INT\n",
              "        }\n",
              "        properties {\n",
              "          key: \"split_names\"\n",
              "          value: STRING\n",
              "        }\n",
              "        properties {\n",
              "          key: \"version\"\n",
              "          value: INT\n",
              "        }\n",
              "        )]\n",
              "            additional_properties: {}\n",
              "            additional_custom_properties: {}\n",
              "        ))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1DvC4pmKJwJh"
      },
      "source": [
        "### Split the dataset into subsets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 176
        },
        "id": "Tn5gLsU-ITUz",
        "outputId": "af0a7c9f-8496-43bd-c0d4-fcff34dc1d1b"
      },
      "source": [
        "output = example_gen_pb2.Output(\n",
        "    split_config=example_gen_pb2.SplitConfig(splits=[\n",
        "        example_gen_pb2.SplitConfig.Split(name='train', hash_buckets=6),\n",
        "        example_gen_pb2.SplitConfig.Split(name='eval', hash_buckets=2),\n",
        "        example_gen_pb2.SplitConfig.Split(name='test', hash_buckets=2)\n",
        "    ]))\n",
        "\n",
        "example_gen = CsvExampleGen(input_base=data_dir, output_config=output)\n",
        "\n",
        "context.run(example_gen)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object expanded\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">ExecutionResult</span><span class=\"deemphasize\"> at 0x7f09dafae390</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.execution_id</td><td class = \"attrvalue\">2</td></tr><tr><td class=\"attr-name\">.component</td><td class = \"attrvalue\"><style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object collapsed\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">CsvExampleGen</span><span class=\"deemphasize\"> at 0x7f09dafc6310</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.inputs</td><td class = \"attrvalue\">{}</td></tr><tr><td class=\"attr-name\">.outputs</td><td class = \"attrvalue\"><table class=\"attr-table\"><tr><td class=\"attr-name\">['examples']</td><td class = \"attrvalue\"><style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object collapsed\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">Channel</span> of type <span class=\"class-name\">'Examples'</span> (1 artifact)<span class=\"deemphasize\"> at 0x7f09dafc66d0</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.type_name</td><td class = \"attrvalue\">Examples</td></tr><tr><td class=\"attr-name\">._artifacts</td><td class = \"attrvalue\"><table class=\"attr-table\"><tr><td class=\"attr-name\">[0]</td><td class = \"attrvalue\"><style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object collapsed\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">Artifact</span> of type <span class=\"class-name\">'Examples'</span> (uri: tfx/CsvExampleGen/examples/2)<span class=\"deemphasize\"> at 0x7f09dafe1c90</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.type</td><td class = \"attrvalue\">&lt;class &#x27;tfx.types.standard_artifacts.Examples&#x27;&gt;</td></tr><tr><td class=\"attr-name\">.uri</td><td class = \"attrvalue\">tfx/CsvExampleGen/examples/2</td></tr><tr><td class=\"attr-name\">.span</td><td class = \"attrvalue\">0</td></tr><tr><td class=\"attr-name\">.split_names</td><td class = \"attrvalue\">[&quot;train&quot;, &quot;eval&quot;, &quot;test&quot;]</td></tr><tr><td class=\"attr-name\">.version</td><td class = \"attrvalue\">0</td></tr></table></div></td></tr></table></td></tr></table></div></td></tr></table></td></tr><tr><td class=\"attr-name\">.exec_properties</td><td class = \"attrvalue\"><table class=\"attr-table\"><tr><td class=\"attr-name\">['input_base']</td><td class = \"attrvalue\">data</td></tr><tr><td class=\"attr-name\">['input_config']</td><td class = \"attrvalue\">{\n",
              "  &quot;splits&quot;: [\n",
              "    {\n",
              "      &quot;name&quot;: &quot;single_split&quot;,\n",
              "      &quot;pattern&quot;: &quot;*&quot;\n",
              "    }\n",
              "  ]\n",
              "}</td></tr><tr><td class=\"attr-name\">['output_config']</td><td class = \"attrvalue\">{\n",
              "  &quot;split_config&quot;: {\n",
              "    &quot;splits&quot;: [\n",
              "      {\n",
              "        &quot;hash_buckets&quot;: 6,\n",
              "        &quot;name&quot;: &quot;train&quot;\n",
              "      },\n",
              "      {\n",
              "        &quot;hash_buckets&quot;: 2,\n",
              "        &quot;name&quot;: &quot;eval&quot;\n",
              "      },\n",
              "      {\n",
              "        &quot;hash_buckets&quot;: 2,\n",
              "        &quot;name&quot;: &quot;test&quot;\n",
              "      }\n",
              "    ]\n",
              "  }\n",
              "}</td></tr><tr><td class=\"attr-name\">['output_data_format']</td><td class = \"attrvalue\">6</td></tr><tr><td class=\"attr-name\">['custom_config']</td><td class = \"attrvalue\">None</td></tr><tr><td class=\"attr-name\">['range_config']</td><td class = \"attrvalue\">None</td></tr><tr><td class=\"attr-name\">['span']</td><td class = \"attrvalue\">0</td></tr><tr><td class=\"attr-name\">['version']</td><td class = \"attrvalue\">None</td></tr><tr><td class=\"attr-name\">['input_fingerprint']</td><td class = \"attrvalue\">split:single_split,num_files:1,total_bytes:78956236,xor_checksum:1629119443,sum_checksum:1629119443</td></tr></table></td></tr></table></div></td></tr><tr><td class=\"attr-name\">.component.inputs</td><td class = \"attrvalue\">{}</td></tr><tr><td class=\"attr-name\">.component.outputs</td><td class = \"attrvalue\"><table class=\"attr-table\"><tr><td class=\"attr-name\">['examples']</td><td class = \"attrvalue\"><style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object collapsed\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">Channel</span> of type <span class=\"class-name\">'Examples'</span> (1 artifact)<span class=\"deemphasize\"> at 0x7f09dafc66d0</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.type_name</td><td class = \"attrvalue\">Examples</td></tr><tr><td class=\"attr-name\">._artifacts</td><td class = \"attrvalue\"><table class=\"attr-table\"><tr><td class=\"attr-name\">[0]</td><td class = \"attrvalue\"><style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object collapsed\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">Artifact</span> of type <span class=\"class-name\">'Examples'</span> (uri: tfx/CsvExampleGen/examples/2)<span class=\"deemphasize\"> at 0x7f09dafe1c90</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.type</td><td class = \"attrvalue\">&lt;class &#x27;tfx.types.standard_artifacts.Examples&#x27;&gt;</td></tr><tr><td class=\"attr-name\">.uri</td><td class = \"attrvalue\">tfx/CsvExampleGen/examples/2</td></tr><tr><td class=\"attr-name\">.span</td><td class = \"attrvalue\">0</td></tr><tr><td class=\"attr-name\">.split_names</td><td class = \"attrvalue\">[&quot;train&quot;, &quot;eval&quot;, &quot;test&quot;]</td></tr><tr><td class=\"attr-name\">.version</td><td class = \"attrvalue\">0</td></tr></table></div></td></tr></table></td></tr></table></div></td></tr></table></td></tr></table></div>"
            ],
            "text/plain": [
              "ExecutionResult(\n",
              "    component_id: CsvExampleGen\n",
              "    execution_id: 2\n",
              "    outputs:\n",
              "        examples: Channel(\n",
              "            type_name: Examples\n",
              "            artifacts: [Artifact(artifact: id: 2\n",
              "        type_id: 5\n",
              "        uri: \"tfx/CsvExampleGen/examples/2\"\n",
              "        properties {\n",
              "          key: \"split_names\"\n",
              "          value {\n",
              "            string_value: \"[\\\"train\\\", \\\"eval\\\", \\\"test\\\"]\"\n",
              "          }\n",
              "        }\n",
              "        custom_properties {\n",
              "          key: \"input_fingerprint\"\n",
              "          value {\n",
              "            string_value: \"split:single_split,num_files:1,total_bytes:78956236,xor_checksum:1629119443,sum_checksum:1629119443\"\n",
              "          }\n",
              "        }\n",
              "        custom_properties {\n",
              "          key: \"payload_format\"\n",
              "          value {\n",
              "            string_value: \"FORMAT_TF_EXAMPLE\"\n",
              "          }\n",
              "        }\n",
              "        custom_properties {\n",
              "          key: \"span\"\n",
              "          value {\n",
              "            string_value: \"0\"\n",
              "          }\n",
              "        }\n",
              "        custom_properties {\n",
              "          key: \"state\"\n",
              "          value {\n",
              "            string_value: \"published\"\n",
              "          }\n",
              "        }\n",
              "        state: LIVE\n",
              "        , artifact_type: id: 5\n",
              "        name: \"Examples\"\n",
              "        properties {\n",
              "          key: \"span\"\n",
              "          value: INT\n",
              "        }\n",
              "        properties {\n",
              "          key: \"split_names\"\n",
              "          value: STRING\n",
              "        }\n",
              "        properties {\n",
              "          key: \"version\"\n",
              "          value: INT\n",
              "        }\n",
              "        )]\n",
              "            additional_properties: {}\n",
              "            additional_custom_properties: {}\n",
              "        ))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9PkYJOHpJ_i6"
      },
      "source": [
        "### Preserve existing splits"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1hNR5Kz-IV3u"
      },
      "source": [
        "!mkdir -p data/train\n",
        "!mkdir -p data/eval\n",
        "!mkdir -p data/test"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BHBtRSA4IYB9"
      },
      "source": [
        "df.iloc[:20000].to_csv('data/train/20k-consumer-complaints-training.csv', index=False)\n",
        "df.iloc[20000:24000].to_csv('data/eval/4k-consumer-complaints-eval.csv', index=False)\n",
        "df.iloc[24000:26000].to_csv('data/test/2k-consumer-complaints-test.csv', index=False)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 176
        },
        "id": "VBcyt6_oIaI_",
        "outputId": "498622ad-0cb1-4fbe-8cd9-9ace0b4d33d2"
      },
      "source": [
        "input = example_gen_pb2.Input(splits=[\n",
        "    example_gen_pb2.Input.Split(name='train', pattern='train/*'),\n",
        "    example_gen_pb2.Input.Split(name='eval', pattern='eval/*'),\n",
        "    example_gen_pb2.Input.Split(name='test', pattern='test/*')\n",
        "])\n",
        "\n",
        "example_gen = CsvExampleGen(input_base=data_dir, input_config=input)\n",
        "context.run(example_gen)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object expanded\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">ExecutionResult</span><span class=\"deemphasize\"> at 0x7f09db211f90</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.execution_id</td><td class = \"attrvalue\">3</td></tr><tr><td class=\"attr-name\">.component</td><td class = \"attrvalue\"><style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object collapsed\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">CsvExampleGen</span><span class=\"deemphasize\"> at 0x7f09daa5e990</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.inputs</td><td class = \"attrvalue\">{}</td></tr><tr><td class=\"attr-name\">.outputs</td><td class = \"attrvalue\"><table class=\"attr-table\"><tr><td class=\"attr-name\">['examples']</td><td class = \"attrvalue\"><style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object collapsed\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">Channel</span> of type <span class=\"class-name\">'Examples'</span> (1 artifact)<span class=\"deemphasize\"> at 0x7f09daa62e50</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.type_name</td><td class = \"attrvalue\">Examples</td></tr><tr><td class=\"attr-name\">._artifacts</td><td class = \"attrvalue\"><table class=\"attr-table\"><tr><td class=\"attr-name\">[0]</td><td class = \"attrvalue\"><style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object collapsed\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">Artifact</span> of type <span class=\"class-name\">'Examples'</span> (uri: tfx/CsvExampleGen/examples/3)<span class=\"deemphasize\"> at 0x7f09db2112d0</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.type</td><td class = \"attrvalue\">&lt;class &#x27;tfx.types.standard_artifacts.Examples&#x27;&gt;</td></tr><tr><td class=\"attr-name\">.uri</td><td class = \"attrvalue\">tfx/CsvExampleGen/examples/3</td></tr><tr><td class=\"attr-name\">.span</td><td class = \"attrvalue\">0</td></tr><tr><td class=\"attr-name\">.split_names</td><td class = \"attrvalue\">[&quot;train&quot;, &quot;eval&quot;, &quot;test&quot;]</td></tr><tr><td class=\"attr-name\">.version</td><td class = \"attrvalue\">0</td></tr></table></div></td></tr></table></td></tr></table></div></td></tr></table></td></tr><tr><td class=\"attr-name\">.exec_properties</td><td class = \"attrvalue\"><table class=\"attr-table\"><tr><td class=\"attr-name\">['input_base']</td><td class = \"attrvalue\">data</td></tr><tr><td class=\"attr-name\">['input_config']</td><td class = \"attrvalue\">{\n",
              "  &quot;splits&quot;: [\n",
              "    {\n",
              "      &quot;name&quot;: &quot;train&quot;,\n",
              "      &quot;pattern&quot;: &quot;train/*&quot;\n",
              "    },\n",
              "    {\n",
              "      &quot;name&quot;: &quot;eval&quot;,\n",
              "      &quot;pattern&quot;: &quot;eval/*&quot;\n",
              "    },\n",
              "    {\n",
              "      &quot;name&quot;: &quot;test&quot;,\n",
              "      &quot;pattern&quot;: &quot;test/*&quot;\n",
              "    }\n",
              "  ]\n",
              "}</td></tr><tr><td class=\"attr-name\">['output_config']</td><td class = \"attrvalue\">{}</td></tr><tr><td class=\"attr-name\">['output_data_format']</td><td class = \"attrvalue\">6</td></tr><tr><td class=\"attr-name\">['custom_config']</td><td class = \"attrvalue\">None</td></tr><tr><td class=\"attr-name\">['range_config']</td><td class = \"attrvalue\">None</td></tr><tr><td class=\"attr-name\">['span']</td><td class = \"attrvalue\">0</td></tr><tr><td class=\"attr-name\">['version']</td><td class = \"attrvalue\">None</td></tr><tr><td class=\"attr-name\">['input_fingerprint']</td><td class = \"attrvalue\">split:train,num_files:1,total_bytes:23608185,xor_checksum:1629119519,sum_checksum:1629119519\n",
              "split:eval,num_files:1,total_bytes:4759360,xor_checksum:1629119519,sum_checksum:1629119519\n",
              "split:test,num_files:1,total_bytes:2428892,xor_checksum:1629119519,sum_checksum:1629119519</td></tr></table></td></tr></table></div></td></tr><tr><td class=\"attr-name\">.component.inputs</td><td class = \"attrvalue\">{}</td></tr><tr><td class=\"attr-name\">.component.outputs</td><td class = \"attrvalue\"><table class=\"attr-table\"><tr><td class=\"attr-name\">['examples']</td><td class = \"attrvalue\"><style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object collapsed\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">Channel</span> of type <span class=\"class-name\">'Examples'</span> (1 artifact)<span class=\"deemphasize\"> at 0x7f09daa62e50</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.type_name</td><td class = \"attrvalue\">Examples</td></tr><tr><td class=\"attr-name\">._artifacts</td><td class = \"attrvalue\"><table class=\"attr-table\"><tr><td class=\"attr-name\">[0]</td><td class = \"attrvalue\"><style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object collapsed\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">Artifact</span> of type <span class=\"class-name\">'Examples'</span> (uri: tfx/CsvExampleGen/examples/3)<span class=\"deemphasize\"> at 0x7f09db2112d0</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.type</td><td class = \"attrvalue\">&lt;class &#x27;tfx.types.standard_artifacts.Examples&#x27;&gt;</td></tr><tr><td class=\"attr-name\">.uri</td><td class = \"attrvalue\">tfx/CsvExampleGen/examples/3</td></tr><tr><td class=\"attr-name\">.span</td><td class = \"attrvalue\">0</td></tr><tr><td class=\"attr-name\">.split_names</td><td class = \"attrvalue\">[&quot;train&quot;, &quot;eval&quot;, &quot;test&quot;]</td></tr><tr><td class=\"attr-name\">.version</td><td class = \"attrvalue\">0</td></tr></table></div></td></tr></table></td></tr></table></div></td></tr></table></td></tr></table></div>"
            ],
            "text/plain": [
              "ExecutionResult(\n",
              "    component_id: CsvExampleGen\n",
              "    execution_id: 3\n",
              "    outputs:\n",
              "        examples: Channel(\n",
              "            type_name: Examples\n",
              "            artifacts: [Artifact(artifact: id: 3\n",
              "        type_id: 5\n",
              "        uri: \"tfx/CsvExampleGen/examples/3\"\n",
              "        properties {\n",
              "          key: \"split_names\"\n",
              "          value {\n",
              "            string_value: \"[\\\"train\\\", \\\"eval\\\", \\\"test\\\"]\"\n",
              "          }\n",
              "        }\n",
              "        custom_properties {\n",
              "          key: \"input_fingerprint\"\n",
              "          value {\n",
              "            string_value: \"split:train,num_files:1,total_bytes:23608185,xor_checksum:1629119519,sum_checksum:1629119519\\nsplit:eval,num_files:1,total_bytes:4759360,xor_checksum:1629119519,sum_checksum:1629119519\\nsplit:test,num_files:1,total_bytes:2428892,xor_checksum:1629119519,sum_checksum:1629119519\"\n",
              "          }\n",
              "        }\n",
              "        custom_properties {\n",
              "          key: \"payload_format\"\n",
              "          value {\n",
              "            string_value: \"FORMAT_TF_EXAMPLE\"\n",
              "          }\n",
              "        }\n",
              "        custom_properties {\n",
              "          key: \"span\"\n",
              "          value {\n",
              "            string_value: \"0\"\n",
              "          }\n",
              "        }\n",
              "        custom_properties {\n",
              "          key: \"state\"\n",
              "          value {\n",
              "            string_value: \"published\"\n",
              "          }\n",
              "        }\n",
              "        state: LIVE\n",
              "        , artifact_type: id: 5\n",
              "        name: \"Examples\"\n",
              "        properties {\n",
              "          key: \"span\"\n",
              "          value: INT\n",
              "        }\n",
              "        properties {\n",
              "          key: \"split_names\"\n",
              "          value: STRING\n",
              "        }\n",
              "        properties {\n",
              "          key: \"version\"\n",
              "          value: INT\n",
              "        }\n",
              "        )]\n",
              "            additional_properties: {}\n",
              "            additional_custom_properties: {}\n",
              "        ))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uPXwfEpvKGkN"
      },
      "source": [
        "### Spanning Datasets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gJqHE8zlIcIj"
      },
      "source": [
        "!mkdir -p data/export-0\n",
        "!mkdir -p data/export-1\n",
        "!mkdir -p data/export-2"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NXbGrO-HIeck"
      },
      "source": [
        "df.iloc[:20000].to_csv('data/export-0/20k-consumer-complaints.csv', index=False)\n",
        "df.iloc[24000].to_csv('data/export-1/24k-consumer-complaints.csv', index=False)\n",
        "df.iloc[26000].to_csv('data/export-2/26k-consumer-complaints.csv', index=False)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 176
        },
        "id": "wxCyU3MpIjDk",
        "outputId": "cb54b1c8-9f24-412d-af8c-6e3f04f05163"
      },
      "source": [
        "input = example_gen_pb2.Input(splits=[\n",
        "    example_gen_pb2.Input.Split(pattern='export-{SPAN}/*')\n",
        "])\n",
        "\n",
        "example_gen = CsvExampleGen(input_base=data_dir, input_config=input)\n",
        "context.run(example_gen)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object expanded\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">ExecutionResult</span><span class=\"deemphasize\"> at 0x7f09dbb71f90</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.execution_id</td><td class = \"attrvalue\">4</td></tr><tr><td class=\"attr-name\">.component</td><td class = \"attrvalue\"><style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object collapsed\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">CsvExampleGen</span><span class=\"deemphasize\"> at 0x7f09dbb17fd0</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.inputs</td><td class = \"attrvalue\">{}</td></tr><tr><td class=\"attr-name\">.outputs</td><td class = \"attrvalue\"><table class=\"attr-table\"><tr><td class=\"attr-name\">['examples']</td><td class = \"attrvalue\"><style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object collapsed\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">Channel</span> of type <span class=\"class-name\">'Examples'</span> (1 artifact)<span class=\"deemphasize\"> at 0x7f09dbb17bd0</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.type_name</td><td class = \"attrvalue\">Examples</td></tr><tr><td class=\"attr-name\">._artifacts</td><td class = \"attrvalue\"><table class=\"attr-table\"><tr><td class=\"attr-name\">[0]</td><td class = \"attrvalue\"><style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object collapsed\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">Artifact</span> of type <span class=\"class-name\">'Examples'</span> (uri: tfx/CsvExampleGen/examples/4)<span class=\"deemphasize\"> at 0x7f09db254910</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.type</td><td class = \"attrvalue\">&lt;class &#x27;tfx.types.standard_artifacts.Examples&#x27;&gt;</td></tr><tr><td class=\"attr-name\">.uri</td><td class = \"attrvalue\">tfx/CsvExampleGen/examples/4</td></tr><tr><td class=\"attr-name\">.span</td><td class = \"attrvalue\">0</td></tr><tr><td class=\"attr-name\">.split_names</td><td class = \"attrvalue\">[&quot;train&quot;, &quot;eval&quot;]</td></tr><tr><td class=\"attr-name\">.version</td><td class = \"attrvalue\">0</td></tr></table></div></td></tr></table></td></tr></table></div></td></tr></table></td></tr><tr><td class=\"attr-name\">.exec_properties</td><td class = \"attrvalue\"><table class=\"attr-table\"><tr><td class=\"attr-name\">['input_base']</td><td class = \"attrvalue\">data</td></tr><tr><td class=\"attr-name\">['input_config']</td><td class = \"attrvalue\">{\n",
              "  &quot;splits&quot;: [\n",
              "    {\n",
              "      &quot;pattern&quot;: &quot;export-2/*&quot;\n",
              "    }\n",
              "  ]\n",
              "}</td></tr><tr><td class=\"attr-name\">['output_config']</td><td class = \"attrvalue\">{\n",
              "  &quot;split_config&quot;: {\n",
              "    &quot;splits&quot;: [\n",
              "      {\n",
              "        &quot;hash_buckets&quot;: 2,\n",
              "        &quot;name&quot;: &quot;train&quot;\n",
              "      },\n",
              "      {\n",
              "        &quot;hash_buckets&quot;: 1,\n",
              "        &quot;name&quot;: &quot;eval&quot;\n",
              "      }\n",
              "    ]\n",
              "  }\n",
              "}</td></tr><tr><td class=\"attr-name\">['output_data_format']</td><td class = \"attrvalue\">6</td></tr><tr><td class=\"attr-name\">['custom_config']</td><td class = \"attrvalue\">None</td></tr><tr><td class=\"attr-name\">['range_config']</td><td class = \"attrvalue\">None</td></tr><tr><td class=\"attr-name\">['span']</td><td class = \"attrvalue\">2</td></tr><tr><td class=\"attr-name\">['version']</td><td class = \"attrvalue\">None</td></tr><tr><td class=\"attr-name\">['input_fingerprint']</td><td class = \"attrvalue\">split:,num_files:1,total_bytes:412,xor_checksum:1629119534,sum_checksum:1629119534</td></tr></table></td></tr></table></div></td></tr><tr><td class=\"attr-name\">.component.inputs</td><td class = \"attrvalue\">{}</td></tr><tr><td class=\"attr-name\">.component.outputs</td><td class = \"attrvalue\"><table class=\"attr-table\"><tr><td class=\"attr-name\">['examples']</td><td class = \"attrvalue\"><style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object collapsed\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">Channel</span> of type <span class=\"class-name\">'Examples'</span> (1 artifact)<span class=\"deemphasize\"> at 0x7f09dbb17bd0</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.type_name</td><td class = \"attrvalue\">Examples</td></tr><tr><td class=\"attr-name\">._artifacts</td><td class = \"attrvalue\"><table class=\"attr-table\"><tr><td class=\"attr-name\">[0]</td><td class = \"attrvalue\"><style>\n",
              ".tfx-object.expanded {\n",
              "  padding: 4px 8px 4px 8px;\n",
              "  background: white;\n",
              "  border: 1px solid #bbbbbb;\n",
              "  box-shadow: 4px 4px 2px rgba(0,0,0,0.05);\n",
              "}\n",
              ".tfx-object, .tfx-object * {\n",
              "  font-size: 11pt;\n",
              "}\n",
              ".tfx-object > .title {\n",
              "  cursor: pointer;\n",
              "}\n",
              ".tfx-object .expansion-marker {\n",
              "  color: #999999;\n",
              "}\n",
              ".tfx-object.expanded > .title > .expansion-marker:before {\n",
              "  content: '▼';\n",
              "}\n",
              ".tfx-object.collapsed > .title > .expansion-marker:before {\n",
              "  content: '▶';\n",
              "}\n",
              ".tfx-object .class-name {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object .deemphasize {\n",
              "  opacity: 0.5;\n",
              "}\n",
              ".tfx-object.collapsed > table.attr-table {\n",
              "  display: none;\n",
              "}\n",
              ".tfx-object.expanded > table.attr-table {\n",
              "  display: block;\n",
              "}\n",
              ".tfx-object table.attr-table {\n",
              "  border: 2px solid white;\n",
              "  margin-top: 5px;\n",
              "}\n",
              ".tfx-object table.attr-table td.attr-name {\n",
              "  vertical-align: top;\n",
              "  font-weight: bold;\n",
              "}\n",
              ".tfx-object table.attr-table td.attrvalue {\n",
              "  text-align: left;\n",
              "}\n",
              "</style>\n",
              "<script>\n",
              "function toggleTfxObject(element) {\n",
              "  var objElement = element.parentElement;\n",
              "  if (objElement.classList.contains('collapsed')) {\n",
              "    objElement.classList.remove('collapsed');\n",
              "    objElement.classList.add('expanded');\n",
              "  } else {\n",
              "    objElement.classList.add('collapsed');\n",
              "    objElement.classList.remove('expanded');\n",
              "  }\n",
              "}\n",
              "</script>\n",
              "<div class=\"tfx-object collapsed\"><div class = \"title\" onclick=\"toggleTfxObject(this)\"><span class=\"expansion-marker\"></span><span class=\"class-name\">Artifact</span> of type <span class=\"class-name\">'Examples'</span> (uri: tfx/CsvExampleGen/examples/4)<span class=\"deemphasize\"> at 0x7f09db254910</span></div><table class=\"attr-table\"><tr><td class=\"attr-name\">.type</td><td class = \"attrvalue\">&lt;class &#x27;tfx.types.standard_artifacts.Examples&#x27;&gt;</td></tr><tr><td class=\"attr-name\">.uri</td><td class = \"attrvalue\">tfx/CsvExampleGen/examples/4</td></tr><tr><td class=\"attr-name\">.span</td><td class = \"attrvalue\">0</td></tr><tr><td class=\"attr-name\">.split_names</td><td class = \"attrvalue\">[&quot;train&quot;, &quot;eval&quot;]</td></tr><tr><td class=\"attr-name\">.version</td><td class = \"attrvalue\">0</td></tr></table></div></td></tr></table></td></tr></table></div></td></tr></table></td></tr></table></div>"
            ],
            "text/plain": [
              "ExecutionResult(\n",
              "    component_id: CsvExampleGen\n",
              "    execution_id: 4\n",
              "    outputs:\n",
              "        examples: Channel(\n",
              "            type_name: Examples\n",
              "            artifacts: [Artifact(artifact: id: 4\n",
              "        type_id: 5\n",
              "        uri: \"tfx/CsvExampleGen/examples/4\"\n",
              "        properties {\n",
              "          key: \"split_names\"\n",
              "          value {\n",
              "            string_value: \"[\\\"train\\\", \\\"eval\\\"]\"\n",
              "          }\n",
              "        }\n",
              "        custom_properties {\n",
              "          key: \"input_fingerprint\"\n",
              "          value {\n",
              "            string_value: \"split:,num_files:1,total_bytes:412,xor_checksum:1629119534,sum_checksum:1629119534\"\n",
              "          }\n",
              "        }\n",
              "        custom_properties {\n",
              "          key: \"payload_format\"\n",
              "          value {\n",
              "            string_value: \"FORMAT_TF_EXAMPLE\"\n",
              "          }\n",
              "        }\n",
              "        custom_properties {\n",
              "          key: \"span\"\n",
              "          value {\n",
              "            string_value: \"2\"\n",
              "          }\n",
              "        }\n",
              "        custom_properties {\n",
              "          key: \"state\"\n",
              "          value {\n",
              "            string_value: \"published\"\n",
              "          }\n",
              "        }\n",
              "        state: LIVE\n",
              "        , artifact_type: id: 5\n",
              "        name: \"Examples\"\n",
              "        properties {\n",
              "          key: \"span\"\n",
              "          value: INT\n",
              "        }\n",
              "        properties {\n",
              "          key: \"split_names\"\n",
              "          value: STRING\n",
              "        }\n",
              "        properties {\n",
              "          key: \"version\"\n",
              "          value: INT\n",
              "        }\n",
              "        )]\n",
              "            additional_properties: {}\n",
              "            additional_custom_properties: {}\n",
              "        ))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    }
  ]
}