Spark ML Scala.ipynb

{
    "nbformat_minor": 1, 
    "cells": [
        {
            "source": "import org.apache.spark.ml.clustering.KMeans\nimport org.apache.spark.ml.linalg.Vector\nimport org.apache.spark.ml.linalg.Vectors\nimport org.apache.spark.ml.feature.VectorAssembler", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": null
        }, 
        {
            "source": "//use insert to code option in data panel", 
            "cell_type": "code", 
            "metadata": {
                "scrolled": true
            }, 
            "outputs": [], 
            "execution_count": null
        }, 
        {
            "source": "taxifile.createOrReplaceTempView(\"taxifile\")\n\nval taxifence = spark.sql(\"\"\"select Dropoff_latitude,Dropoff_longitude from taxifile where\n   Dropoff_latitude > 40.70 and \n   Dropoff_latitude < 40.86 and \n   Dropoff_longitude > -74.02 and \n   Dropoff_longitude < -73.93\"\"\")\n\ntaxifence.count", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": null
        }, 
        {
            "source": "val assembler = (new VectorAssembler()\n    .setInputCols(Array(\"Dropoff_latitude\", \"Dropoff_longitude\"))\n    .setOutputCol(\"features\"))\nval taxivector = assembler.transform(taxifence)\nval taxifeat = taxivector.drop(\"Dropoff_latitude\",\"Dropoff_longitude\")", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": null
        }, 
        {
            "source": "val kmeans = new KMeans().setK(2).setSeed(1L)\nval model = kmeans.fit(taxifeat)\nprintln(\"Cluster Centers: \")\nmodel.clusterCenters.foreach(println)", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": null
        }, 
        {
            "source": "", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": null
        }
    ], 
    "metadata": {
        "kernelspec": {
            "display_name": "Scala 2.11 with Spark 2.1", 
            "name": "scala-spark21", 
            "language": "scala"
        }, 
        "language_info": {
            "mimetype": "text/x-scala", 
            "version": "2.11.8", 
            "name": "scala", 
            "pygments_lexer": "scala", 
            "file_extension": ".scala", 
            "codemirror_mode": "text/x-scala"
        }
    }, 
    "nbformat": 4
}