Risk Profile Monitoring.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [],
   "source": [
    "from geopy.geocoders import GoogleV3\n",
    "from shapely.geometry import Point\n",
    "from shapely import wkt\n",
    "import urllib.request\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import bs4 as bs\n",
    "import warnings\n",
    "import random\n",
    "import gmplot \n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Source: https://www.doogal.co.uk/polylines.php\n",
    "polylines_df = pd.read_csv('polylines.csv')\n",
    "polylines_list = polylines_df.values.tolist()\n",
    "polylines_string = json.dumps(polylines_list).replace(\",\", \"\").replace(\"]]\", \"\").replace(\"[\", \"\").replace(\"]\", \",\")\n",
    "\n",
    "multipolygon_df = pd.DataFrame({'geom': ['MULTIPOLYGON ((('+polylines_string+')))']})\n",
    "multipolygon_df['geom'] = multipolygon_df['geom'].apply(wkt.loads)\n",
    "multipolygon = multipolygon_df.iloc[0]['geom']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_random(number, polygon):\n",
    "    points = []\n",
    "    minx, miny, maxx, maxy = polygon.bounds\n",
    "    while len(points) < number:\n",
    "        #Florida International University Latitude\n",
    "        x = np.random.normal(25.7574, 0.20, 1)\n",
    "        #Florida International University Longitude\n",
    "        y = np.random.normal(-80.3733, 0.15, 1)\n",
    "        pt = x, y\n",
    "        pnt = Point(pt)\n",
    "        if polygon.contains(pnt):\n",
    "            points.append(pt)\n",
    "    return(points)\n",
    "\n",
    "# calls function to generate a specified number of (Latitude, Longitude) coordinates bounded by polygon  \n",
    "coordinates = generate_random(20000, multipolygon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save coordinate data to csv\n",
    "df = pd.DataFrame(coordinates)\n",
    "\n",
    "df.to_csv(\"coordinates.csv\", sep = ',', index = False, header = ['Latitude', 'Longitude'])\n",
    "\n",
    "# Read and Load Data\n",
    "data = pd.read_csv('coordinates.csv', sep = ',') \n",
    "\n",
    "# Formatting of Dataframe (remove [] and string -> float)\n",
    "data['Latitude'] = data['Latitude'].str.strip('[]').astype(float)\n",
    "data['Longitude'] = data['Longitude'].str.strip('[]').astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot Central Location (FIU MMC CAMPUS)\n",
    "gmap = gmplot.GoogleMapPlotter(25.7574, -80.3733, 15)\n",
    "gmap.marker(25.7574, -80.3733, color = 'red')\n",
    "\n",
    "# Loop over and Mark Lattitude and Longitude Coordinates in Map\n",
    "for latitude, longitude in data.itertuples(index = False):\n",
    "     gmap.marker(latitude, longitude, color = 'cornflowerblue')\n",
    "\n",
    "# Google API Key\n",
    "googlekey = \"AIzaSyB8Qm6takzPBAxsfqUXclRf5Bc81ZDLuuU\"        \n",
    "        \n",
    "# Google API Key    \n",
    "gmap.apikey = googlekey\n",
    "\n",
    "# Save Generated Map to a Given Directory \n",
    "gmap.draw('map.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "geolocator = GoogleV3(api_key = googlekey)\n",
    "\n",
    "# Converts coordinates to nearest address calling Google's Geocode API\n",
    "addresses = []\n",
    "for i in range(len(coordinates)):\n",
    "    location = geolocator.reverse(coordinates[i])\n",
    "    string = json.dumps(location.raw)\n",
    "    jsondata = json.loads(string)\n",
    "    addresses.append(jsondata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filters the different locations\n",
    "filter_types = []\n",
    "\n",
    "for i in range(len(addresses)):\n",
    "    if addresses[i]['types'][0] in ['street_address', 'premise']:\n",
    "        filter_types.append(addresses[i])  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parses the json structure to format \n",
    "formatted_data = []\n",
    "\n",
    "for i in range(len(filter_types)):\n",
    "    formatting = {\n",
    "                'place_id': filter_types[i]['place_id'],\n",
    "                'formatted_address': filter_types[i]['formatted_address'],\n",
    "                'latitude': filter_types[i]['geometry']['location']['lat'],\n",
    "                'longitude': filter_types[i]['geometry']['location']['lng']\n",
    "                 }\n",
    "    formatted_data.append(formatting)\n",
    "\n",
    "df = pd.DataFrame(formatted_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This block eliminates duplicate addresses\n",
    "placeid_list = []\n",
    "for i in range(len(filter_types)):\n",
    "    placeid_list.append(filter_types[i]['place_id'])\n",
    "    \n",
    "unique_list = [] \n",
    "for i in placeid_list:   \n",
    "    if i not in unique_list: \n",
    "        unique_list.append(i) \n",
    "\n",
    "unique_df = pd.DataFrame(unique_list)\n",
    "unique_df.rename(columns={0:'place_id'}, inplace = True)\n",
    "\n",
    "df = pd.concat([df, unique_df], axis=1, join='inner')\n",
    "df = df.iloc[:, : 4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Web Scrape current values and demographics for FIU \n",
    "source = urllib.request.urlopen('https://www.collegetuitioncompare.com/edu/133951/florida-international-university/enrollment/').read()\n",
    "soup = bs.BeautifulSoup(source, 'lxml')\n",
    "\n",
    "table = soup.table\n",
    "table_rows = table.find_all('tr')\n",
    "\n",
    "total = int((table_rows[1].find_all('td')[0].text).replace(\",\", \"\"))\n",
    "women = int((table_rows[3].find_all('td')[0].text).replace(\",\", \"\"))\n",
    "\n",
    "frac_women = women / total\n",
    "# This split handles the women\n",
    "first_split = df.sample(frac = frac_women, random_state = 200)\n",
    "\n",
    "# This split handles the remaining (men)\n",
    "second_split = df.drop(first_split.index)\n",
    "\n",
    "# Assign genders based on proportion\n",
    "first_split['gender'] = 'F'\n",
    "second_split['gender'] = 'M'\n",
    "\n",
    "df = first_split.append(second_split, sort = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add index as column to dataframe (mimic of a student id)\n",
    "df['sid'] = df.index\n",
    "\n",
    "# hypothesized parameters\n",
    "frac_student = 0.55\n",
    "frac_faculty = 0.05\n",
    "frac_support = 0.50\n",
    "\n",
    "first_split_student = df.sample(frac = frac_student, random_state = 200)\n",
    "second_split_faculty = df.drop(first_split_student.index).sample(frac = frac_faculty, random_state = 200)\n",
    "third_split_support = df.drop(first_split_student.index).drop(second_split_faculty.index).sample(frac = frac_support, random_state = 200)\n",
    "fourth_split_admin = df.drop(first_split_student.index).drop(second_split_faculty.index).drop(third_split_support.index)\n",
    "\n",
    "# Define split with values\n",
    "first_split_student['classification'] = 'Student'\n",
    "second_split_faculty['classification'] = 'Faculty'\n",
    "third_split_support['classification'] = 'Support'\n",
    "fourth_split_admin['classification'] = 'Administrative'\n",
    "\n",
    "# combine dataframe\n",
    "df = first_split_student.append([second_split_faculty,third_split_support,fourth_split_admin],sort = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Student Proportion: 0.55\n",
      "Faculty Proportion: 0.02\n",
      "Support Proportion: 0.21\n",
      "Administrative Proportion: 0.21\n"
     ]
    }
   ],
   "source": [
    "# Validation of hypothesized proportions\n",
    "print('Student Proportion: {}'.format(round((len(df.loc[df['classification'] == 'Student']) / len(df)), 2))),\n",
    "print('Faculty Proportion: {}'.format(round((len(df.loc[df['classification'] == 'Faculty']) / len(df)), 2))),\n",
    "print('Support Proportion: {}'.format(round((len(df.loc[df['classification'] == 'Support']) / len(df)), 2))),\n",
    "print('Administrative Proportion: {}'.format(round((len(df.loc[df['classification'] == 'Administrative']) / len(df)), 2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Suppress FutureDeprecation Warning (coming from helper function blocks.py in lib of pandas)\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "# Repeating ID's indicates household size\n",
    "filtered = pd.DataFrame(df.groupby('place_id').filter(lambda g: len(g) > 1).groupby('place_id').size().sort_values(ascending=False))\n",
    "filtered.columns = ['count']\n",
    "filtered = filtered.reset_index()\n",
    "\n",
    "# add household to revised_df and initialize to 0\n",
    "df['household'] = 0\n",
    "\n",
    "for index in range(len(filtered)):\n",
    "    df.iloc[[df['place_id'] == filtered.iloc[index]['place_id']], [7]] = filtered.iloc[index]['count']        \n",
    "\n",
    "# Remainder will be between [1, 4]\n",
    "for index in range(len(df)):\n",
    "    if df.iloc[index]['household'] == 0:\n",
    "        df.iloc[[index], [7]] = random.randint(1, 4)\n",
    "    else: \n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initiate/create age column\n",
    "df['age'] = 0    \n",
    "\n",
    "# students ages between [18, 25] \n",
    "# all others ages between [35, 60] \n",
    "\n",
    "for index in range(len(df)):\n",
    "    # the 80% of students\n",
    "    if df.loc[df.index[index]]['classification'] == 'Student':\n",
    "        df.iloc[[index], [8]] = random.randint(18, 25)\n",
    "    \n",
    "    # everyone else (faculty, support, admin)       \n",
    "    elif df.loc[df.index[index]]['classification'] == 'Faculty':\n",
    "        df.iloc[[index], [8]] = random.randint(35, 60) \n",
    "    \n",
    "    elif df.loc[df.index[index]]['classification'] == 'Support':\n",
    "        df.iloc[[index], [8]] = random.randint(35, 60)     \n",
    "    \n",
    "    elif df.loc[df.index[index]]['classification'] == 'Administrative':\n",
    "        df.iloc[[index], [8]] = random.randint(35, 60)   \n",
    "    \n",
    "    else:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Customized .csv containing subject of courses and course abbreviations\n",
    "coursedf = pd.read_csv('offeredcourses.csv').values.tolist()\n",
    "\n",
    "# Generates 4 levels for each course abbreviation\n",
    "catalog = []\n",
    "for i in range(len(coursedf)):\n",
    "    for j in range(1,5):\n",
    "        if (i % 2) != 0:\n",
    "            k=j+4\n",
    "            generate = {'subject': coursedf[i][0],\n",
    "                        'version': str(k),\n",
    "                        'course':  coursedf[i][1]+str(j)}\n",
    "            catalog.append(generate)\n",
    "        else: \n",
    "            generate = {'subject': coursedf[i][0],\n",
    "                        'version': str(j),\n",
    "                        'course':  coursedf[i][1]+str(j)}\n",
    "            catalog.append(generate)\n",
    "# catalog of courses\n",
    "catalogdf = pd.DataFrame(catalog)\n",
    "\n",
    "# dataframe of unique subjects\n",
    "subjectdf = pd.DataFrame(catalogdf['subject'].unique()).reset_index()\n",
    "subjectdf.rename(columns = {'index':'ID'}, inplace = True)\n",
    "subjectdf.ID = subjectdf['ID'] + 1\n",
    "subjectdf.rename(columns={0:'subject'}, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [],
   "source": [
    "# isolate faculty data from dataframe \n",
    "facultydf = df.loc[df['classification'] == 'Faculty']\n",
    "facultydf = facultydf.reset_index()\n",
    "\n",
    "# assigns random subject id for each faculty\n",
    "randomdf = pd.Series(np.random.randint(1,20, size=(len(facultydf))))  \n",
    "randomdf = randomdf.rename('ID')\n",
    "\n",
    "# retrieves the subject name by linking subject id\n",
    "result = pd.concat([facultydf, randomdf], axis=1)\n",
    "mergedf = pd.merge(result, subjectdf, on='ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [],
   "source": [
    "# assign courses to faculty based on subject\n",
    "teach = []\n",
    "for i in range(len(mergedf)):\n",
    "    add = catalogdf['course'].loc[(catalogdf['subject'] == mergedf.iloc[i]['subject']) & (catalogdf['version'] == str(random.randrange(1,8)))]\n",
    "    teach.append(add.values)\n",
    "    \n",
    "teachdf = pd.DataFrame(teach) \n",
    "teachdf.rename(columns={0:'course'}, inplace = True)\n",
    "\n",
    "# new faculty dataframe with subject and courses they instruct\n",
    "facultydf = pd.concat([mergedf, teachdf], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creates composite class column by combining teacher's id and the course they teach \n",
    "teachcoursedf = facultydf[['sid','subject' ,'course']]\n",
    "teachcoursedf = teachcoursedf.reset_index()\n",
    "teachcoursedf.rename(columns = {'index':'registerid'}, inplace = True)\n",
    "teachcoursedf.registerid = teachcoursedf['registerid'] + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [],
   "source": [
    "#creates a matrix of classes to assign to student as \"registration\"\n",
    "registrymatrix = []\n",
    "i = 0\n",
    "j = 0\n",
    "k = 1\n",
    "x = 2\n",
    "y = 3\n",
    "while i != len(df.loc[df['classification'] == 'Student']):\n",
    "    if j == len(teachcoursedf):\n",
    "        j = 0 \n",
    "    if k == len(teachcoursedf):\n",
    "        k = 0\n",
    "    if x == len(teachcoursedf):\n",
    "        x = 0\n",
    "    if y == len(teachcoursedf):\n",
    "        y = 0\n",
    "    registrymatrix.append([j + 1,k + 1, x + 1, y + 1])\n",
    "    i = i + 1\n",
    "    j = j + 1\n",
    "    k = k + 1\n",
    "    x = x + 1\n",
    "    y = y + 1\n",
    "\n",
    "registrymatrixdf = pd.DataFrame(registrymatrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [],
   "source": [
    "studentdf = df.loc[df['classification'] == 'Student']\n",
    "studentdf = studentdf.reset_index()    \n",
    "    \n",
    "registrymatrixdf = pd.DataFrame(registrymatrix)\n",
    "\n",
    "frame1 = pd.concat([studentdf[['sid']], registrymatrixdf[[0]]], axis=1)\n",
    "frame1.rename(columns={0:'courseid'}, inplace = True) \n",
    "\n",
    "frame2 = pd.concat([studentdf[['sid']], registrymatrixdf[[1]]], axis=1)\n",
    "frame2.rename(columns={1:'courseid'}, inplace = True) \n",
    "\n",
    "frame3 = pd.concat([studentdf[['sid']], registrymatrixdf[[2]]], axis=1)\n",
    "frame3.rename(columns={2:'courseid'}, inplace = True) \n",
    "\n",
    "frame4 = pd.concat([studentdf[['sid']], registrymatrixdf[[3]]], axis=1)\n",
    "frame4.rename(columns={3:'courseid'}, inplace = True) \n",
    "\n",
    "# Puts the matrix in a tabular format to represent the classes students are registered for\n",
    "finalframe = pd.concat([frame1, frame2, frame3, frame4], ignore_index=True, sort=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract Zip Codes from Full Address\n",
    "zipcodes = [i[-1] for i in df.formatted_address.str.split('FL ')]\n",
    "for i in range(len(zipcodes)):\n",
    "    zipcodes[i] = zipcodes[i].split(\",\")[0]\n",
    "    \n",
    "zipcodes = pd.DataFrame(zipcodes)\n",
    "zipcodes.columns = ['zipcode']\n",
    "\n",
    "# reset index and preserve structure\n",
    "copy = df.reset_index().drop(['index'], axis=1)\n",
    "# Append to full dataframe\n",
    "df = pd.concat([copy, zipcodes], axis=1)\n",
    "\n",
    "# Finally, Save the Results to csv file!\n",
    "df.to_csv('people.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}