Import_Census_1870-CHUNK.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Notebook for census data 1870: CHUNK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_location = '/Volumes/Seagate Expansion Drive/ipums/bocconi_1870_3-0/1870_3-0/us1870c_usa.dat'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = pd.DataFrame()\n",
    "for chunk in pd.read_fwf(\n",
    "    data_location, header=None, chunksize=10**6,\n",
    "    colspecs=[\n",
    "        (0, 1),\n",
    "        (1, 5),\n",
    "        (5, 11),\n",
    "        (11, 19),\n",
    "        (19, 23),\n",
    "        (31, 33),\n",
    "        (33, 34),\n",
    "        (34, 35),\n",
    "        (35, 37),\n",
    "        (37, 38),\n",
    "        (38, 39),\n",
    "        (39, 41),\n",
    "        (41, 42),\n",
    "        (42, 44),\n",
    "        (44, 45),\n",
    "        (45, 46),\n",
    "        (46, 48),\n",
    "        (48, 50),\n",
    "        (50, 52),\n",
    "        (52, 53),\n",
    "        (53, 57),\n",
    "        (57, 60),\n",
    "        (60, 61),\n",
    "        (61, 64),\n",
    "        (64, 65),\n",
    "        (72, 77),\n",
    "        (85, 86),\n",
    "        (86, 87),\n",
    "        (87, 90),\n",
    "        (109, 110),\n",
    "        (113, 114),\n",
    "        (117, 118),\n",
    "        (118, 121),\n",
    "        (121, 123),\n",
    "        (123, 125),\n",
    "        (125, 128),\n",
    "        (147, 151),\n",
    "        (187, 193),\n",
    "        (250, 252),\n",
    "        (252, 254),\n",
    "        (264, 265),\n",
    "        (268, 269),\n",
    "        (299, 300),\n",
    "        (301, 302),\n",
    "        (308, 309),\n",
    "        (322, 323),\n",
    "        (323, 324),\n",
    "        (324, 325),\n",
    "        (325, 326),\n",
    "        (327, 328),\n",
    "        (328, 329),\n",
    "        (382, 385),\n",
    "        (500, 502),\n",
    "        (519, 520),\n",
    "        (600, 602),\n",
    "        (619, 622),\n",
    "        (625, 629),\n",
    "        (633, 637),\n",
    "        (641, 645),\n",
    "        (649, 701),\n",
    "        (1227, 1228),\n",
    "        (1422, 1432),\n",
    "        (1432, 1442),\n",
    "        (1442, 1446),\n",
    "        (1449, 1453),\n",
    "        (1652, 1654),\n",
    "        (1654, 1655),\n",
    "        (1655, 1656),\n",
    "        (1656, 1658),\n",
    "        (1658, 1659),\n",
    "        (1659, 1660),\n",
    "        (1660, 1662),\n",
    "        (1662, 1663),\n",
    "        (1663, 1665),\n",
    "        (1665, 1666),\n",
    "        (1666, 1667),\n",
    "        (1667, 1669),\n",
    "        (1669, 1671),\n",
    "        (1671, 1673),\n",
    "        (1673, 1674),\n",
    "        (1696, 1712),\n",
    "        (1712, 1728),\n",
    "        (1772, 1822),\n",
    "        (2084, 2085),\n",
    "        (2145, 2151),\n",
    "        (2173, 2175),\n",
    "        (2199, 2201),\n",
    "        (2218, 2219),\n",
    "        (2347, 2348),\n",
    "        (2348, 2384),\n",
    "        (2537, 2539),\n",
    "        (2539, 2543),\n",
    "        (2543, 2546),\n",
    "        (2546, 2548),\n",
    "        (2548, 2549),\n",
    "        (2549, 2552),\n",
    "        (2552, 2555),\n",
    "        (2555, 2558),\n",
    "        (2558, 2561),\n",
    "        (2561, 2566),\n",
    "        (2566, 2567),\n",
    "        (2567, 2568),\n",
    "        (2568, 2569),\n",
    "        (2569, 2570),\n",
    "        (2570, 2578),\n",
    "        (2578, 2586),\n",
    "        (2586, 2587),\n",
    "        (2587, 2588),\n",
    "        (2588, 2589),\n",
    "        (2589, 2639),\n",
    "        (2639, 2689),\n",
    "        (2689, 2791),\n",
    "        (2791, 2866),\n",
    "        (2866, 2874),\n",
    "        (2874, 2877),\n",
    "        (2877, 2884),\n",
    "        (2884, 2889),\n",
    "        (2889, 2925),\n",
    "        (2925, 2929),\n",
    "        (2929, 2955),\n",
    "        (2955, 2956),\n",
    "        (2956, 2957),\n",
    "        (2957, 2971),\n",
    "        (2971, 2972),\n",
    "        (2972, 2986),\n",
    "        (2986, 2987),\n",
    "        (2987, 2988),\n",
    "        (2988, 2989),\n",
    "        (2989, 2990),\n",
    "        (2990, 3003),\n",
    "        (3003, 3034),\n",
    "        (3034, 3035),\n",
    "        (3035, 3043),\n",
    "        (3043, 3046),\n",
    "        (3046, 3054),\n",
    "        (3054, 3057),\n",
    "        (3057, 3060),\n",
    "        (3060, 3065),\n",
    "        (3065, 3070),\n",
    "        (3070, 3071),\n",
    "        (3071, 3072),\n",
    "        (3072, 3073),\n",
    "    ],\n",
    "    names=[\n",
    "        'rectype',\n",
    "        'yearp',\n",
    "        'samplep',\n",
    "        'serialp',\n",
    "        'pernum',\n",
    "        'momloc',\n",
    "        'stepmom',\n",
    "        'momrule_hist',\n",
    "        'poploc',\n",
    "        'steppop',\n",
    "        'poprule_hist',\n",
    "        'sploc',\n",
    "        'sprule_hist',\n",
    "        'famsize',\n",
    "        'nchild',\n",
    "        'nchlt5',\n",
    "        'famunit',\n",
    "        'eldch',\n",
    "        'yngch',\n",
    "        'nsibs',\n",
    "        'relate',\n",
    "        'age',\n",
    "        'sex',\n",
    "        'race',\n",
    "        'marst',\n",
    "        'bpl',\n",
    "        'nativity',\n",
    "        'citizen',\n",
    "        'hispan',\n",
    "        'school',\n",
    "        'lit',\n",
    "        'labforce',\n",
    "        'occ1950',\n",
    "        'occscore',\n",
    "        'sei',\n",
    "        'ind1950',\n",
    "        'detocc',\n",
    "        'realprop',\n",
    "        'impsp',\n",
    "        'imprel',\n",
    "        'qage',\n",
    "        'qbpl',\n",
    "        'qocc',\n",
    "        'qrace',\n",
    "        'qsex',\n",
    "        'racamind',\n",
    "        'racasian',\n",
    "        'racblk',\n",
    "        'racpacis',\n",
    "        'racother',\n",
    "        'racwht',\n",
    "        'agediff',\n",
    "        'racesing',\n",
    "        'hisprule',\n",
    "        'relflag',\n",
    "        'presgl',\n",
    "        'erscor50',\n",
    "        'edscor50',\n",
    "        'npboss50',\n",
    "        'occstr',\n",
    "        'isrelate',\n",
    "        'slwt',\n",
    "        'perwt',\n",
    "        'birthyr',\n",
    "        'occ',\n",
    "        'momlocimp',\n",
    "        'stepmomimp',\n",
    "        'momruleimp',\n",
    "        'poplocimp',\n",
    "        'steppopimp',\n",
    "        'popruleimp',\n",
    "        'splocimp',\n",
    "        'spruleimp',\n",
    "        'famsizeimp',\n",
    "        'nchildimp',\n",
    "        'nchlt5imp',\n",
    "        'famunitimp',\n",
    "        'eldchimp',\n",
    "        'yngchimp',\n",
    "        'nsibsimp',\n",
    "        'namelast',\n",
    "        'namefrst',\n",
    "        'bplstr',\n",
    "        'vote',\n",
    "        'persprop',\n",
    "        'agemonth',\n",
    "        'sursim',\n",
    "        'qbirthmo',\n",
    "        'anyallocation',\n",
    "        'histid',\n",
    "        'versionhist',\n",
    "        'us1870c_1001',\n",
    "        'us1870c_1002',\n",
    "        'us1870c_1003',\n",
    "        'us1870c_1004',\n",
    "        'us1870c_1005',\n",
    "        'us1870c_1006',\n",
    "        'us1870c_1007',\n",
    "        'us1870c_1008',\n",
    "        'us1870c_1010',\n",
    "        'us1870c_1011',\n",
    "        'us1870c_1012',\n",
    "        'us1870c_1013',\n",
    "        'us1870c_1014',\n",
    "        'us1870c_1015',\n",
    "        'us1870c_1016',\n",
    "        'us1870c_1017',\n",
    "        'us1870c_1018',\n",
    "        'us1870c_1019',\n",
    "        'us1870c_1020',\n",
    "        'us1870c_1021',\n",
    "        'us1870c_1022',\n",
    "        'us1870c_1023',\n",
    "        'us1870c_1024',\n",
    "        'us1870c_1025',\n",
    "        'us1870c_1026',\n",
    "        'us1870c_1027',\n",
    "        'us1870c_1028',\n",
    "        'us1870c_1029',\n",
    "        'us1870c_1030',\n",
    "        'us1870c_1031',\n",
    "        'us1870c_1032',\n",
    "        'us1870c_1033',\n",
    "        'us1870c_1034',\n",
    "        'us1870c_1035',\n",
    "        'us1870c_1036',\n",
    "        'us1870c_1037',\n",
    "        'us1870c_1038',\n",
    "        'us1870c_1039',\n",
    "        'us1870c_1040',\n",
    "        'us1870c_1041',\n",
    "        'us1870c_1042',\n",
    "        'us1870c_1043',\n",
    "        'us1870c_1044',\n",
    "        'us1870c_1045',\n",
    "        'us1870c_1046',\n",
    "        'us1870c_1047',\n",
    "        'us1870c_1048',\n",
    "        'us1870c_1049',\n",
    "        'us1870c_1050',\n",
    "        'us1870c_1051',\n",
    "        'us1870c_1052',\n",
    "    ],\n",
    "    dtype={\n",
    "        'rectypep': object,\n",
    "        'yearp': object,\n",
    "        'samplep': object,\n",
    "        'serialp': object,\n",
    "        'pernum': object,\n",
    "        'momloc': object,\n",
    "        'stepmom': object,\n",
    "        'momrule_hist': object,\n",
    "        'poploc': object,\n",
    "        'steppop': object,\n",
    "        'poprule_hist': object,\n",
    "        'sploc': object,\n",
    "        'sprule_hist': object,\n",
    "        'famsize': object,\n",
    "        'nchild': object,\n",
    "        'nchlt5': object,\n",
    "        'famunit': object,\n",
    "        'eldch': object,\n",
    "        'yngch': object,\n",
    "        'nsibs': object,\n",
    "        'relate': object,\n",
    "        'age': object,\n",
    "        'sex': object,\n",
    "        'race': object,\n",
    "        'marst': object,\n",
    "        'bpl': object,\n",
    "        'nativity': object,\n",
    "        'citizen': object,\n",
    "        'hispan': object,\n",
    "        'school': object,\n",
    "        'lit': object,\n",
    "        'labforce': object,\n",
    "        'occ1950': object,\n",
    "        'occscore': object,\n",
    "        'sei': object,\n",
    "        'ind1950': object,\n",
    "        'detocc': object,\n",
    "        'realprop': object,\n",
    "        'impsp': object,\n",
    "        'imprel': object,\n",
    "        'qage': object,\n",
    "        'qbpl': object,\n",
    "        'qocc': object,\n",
    "        'qrace': object,\n",
    "        'qsex': object,\n",
    "        'racamind': object,\n",
    "        'racasian': object,\n",
    "        'racblk': object,\n",
    "        'racpacis': object,\n",
    "        'racother': object,\n",
    "        'racwht': object,\n",
    "        'agediff': object,\n",
    "        'racesing': object,\n",
    "        'hisprule': object,\n",
    "        'relflag': object,\n",
    "        'presgl': object,\n",
    "        'erscor50': object,\n",
    "        'edscor50': object,\n",
    "        'npboss50': object,\n",
    "        'occstr': object,\n",
    "        'isrelate': object,\n",
    "        'slwt': object,\n",
    "        'perwt': object,\n",
    "        'birthyr': object,\n",
    "        'occ': object,\n",
    "        'momlocimp': object,\n",
    "        'stepmomimp': object,\n",
    "        'momruleimp': object,\n",
    "        'poplocimp': object,\n",
    "        'steppopimp': object,\n",
    "        'popruleimp': object,\n",
    "        'splocimp': object,\n",
    "        'spruleimp': object,\n",
    "        'famsizeimp': object,\n",
    "        'nchildimp': object,\n",
    "        'nchlt5imp': object,\n",
    "        'famunitimp': object,\n",
    "        'eldchimp': object,\n",
    "        'yngchimp': object,\n",
    "        'nsibsimp': object,\n",
    "        'namelast': object,\n",
    "        'namefrst': object,\n",
    "        'bplstr': object,\n",
    "        'vote': object,\n",
    "        'persprop': object,\n",
    "        'agemonth': object,\n",
    "        'sursim': object,\n",
    "        'qbirthmo': object,\n",
    "        'anyallocation': object,\n",
    "        'histid': object,\n",
    "        'versionhist': object,\n",
    "        'us1870c_1001': object,\n",
    "        'us1870c_1002': object,\n",
    "        'us1870c_1003': object,\n",
    "        'us1870c_1004': object,\n",
    "        'us1870c_1005': object,\n",
    "        'us1870c_1006': object,\n",
    "        'us1870c_1007': object,\n",
    "        'us1870c_1008': object,\n",
    "        'us1870c_1010': object,\n",
    "        'us1870c_1011': object,\n",
    "        'us1870c_1012': object,\n",
    "        'us1870c_1013': object,\n",
    "        'us1870c_1014': object,\n",
    "        'us1870c_1015': object,\n",
    "        'us1870c_1016': object,\n",
    "        'us1870c_1017': object,\n",
    "        'us1870c_1018': object,\n",
    "        'us1870c_1019': object,\n",
    "        'us1870c_1020': object,\n",
    "        'us1870c_1021': object,\n",
    "        'us1870c_1022': object,\n",
    "        'us1870c_1023': object,\n",
    "        'us1870c_1024': object,\n",
    "        'us1870c_1025': object,\n",
    "        'us1870c_1026': object,\n",
    "        'us1870c_1027': object,\n",
    "        'us1870c_1028': object,\n",
    "        'us1870c_1029': object,\n",
    "        'us1870c_1030': object,\n",
    "        'us1870c_1031': object,\n",
    "        'us1870c_1032': object,\n",
    "        'us1870c_1033': object,\n",
    "        'us1870c_1034': object,\n",
    "        'us1870c_1035': object,\n",
    "        'us1870c_1036': object,\n",
    "        'us1870c_1037': object,\n",
    "        'us1870c_1038': object,\n",
    "        'us1870c_1039': object,\n",
    "        'us1870c_1040': object,\n",
    "        'us1870c_1041': object,\n",
    "        'us1870c_1042': object,\n",
    "        'us1870c_1043': object,\n",
    "        'us1870c_1044': object,\n",
    "        'us1870c_1045': object,\n",
    "        'us1870c_1046': object,\n",
    "        'us1870c_1047': object,\n",
    "        'us1870c_1048': object,\n",
    "        'us1870c_1049': object,\n",
    "        'us1870c_1050': object,\n",
    "        'us1870c_1051': object,\n",
    "        'us1870c_1052': object,       \n",
    "    },\n",
    "    usecols=['rectype', 'histid', 'namefrst', 'namelast']):\n",
    "    chunk = chunk[chunk['rectype']=='P']\n",
    "    chunk = chunk.drop('rectype', 1)\n",
    "    output = output.append(chunk)\n",
    "    del chunk\n",
    "    gc.collect()\n",
    "\n",
    "output.to_csv('/Users/bcornejocostas/Desktop/OneDrive_1_07-06-2021/1870_typeP_names.txt', sep='\\t', index=False, encoding='utf8')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-1-8f663355d3f4>, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-1-8f663355d3f4>\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m    git remote add origin https://github.com/bjcs39/Import-Data.git\u001b[0m\n\u001b[0m             ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "git remote add origin https://github.com/bjcs39/Import-Data.git\n",
    "git branch -M main\n",
    "git push -u origin main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}