scraping_archives.go.kr.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 데이터 수집\n",
    "\n",
    "* archives.go.kr\n",
    "* http://theme.archives.go.kr/next/closed/page/ReadClosedCommon.do?yr=2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup as bs\n",
    "import pandas as pd\n",
    "import re\n",
    "import time\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"http://theme.archives.go.kr/next/closed/page/ReadClosedCommon.do?yr=2019\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "response = requests.get(url)\n",
    "html = bs(response.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "207"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table = pd.read_html(url)\n",
    "len(table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# body > table > tbody > tr:nth-child(2) > td:nth-child(2) > table > tbody > tr:nth-child(4) > td > table\n",
    "url = \"http://theme.archives.go.kr/next/closed/page/orgListClosedCommon.do\"\n",
    "query = \"bookmark=&pageParam=page&page=2&searchResultCount=&gubun=80&orgnz_id=108001&searchType=&keyword=&keywordGubun=&viewPage=ClosedList2017\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bookmark': '',\n",
       " 'pageParam': 'page',\n",
       " 'page': '2',\n",
       " 'searchResultCount': '',\n",
       " 'gubun': '80',\n",
       " 'orgnz_id': '108001',\n",
       " 'searchType': '',\n",
       " 'keyword': '',\n",
       " 'keywordGubun': '',\n",
       " 'viewPage': 'ClosedList2017'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_split = query.split(\"&\")\n",
    "params = {}\n",
    "for q in q_split:\n",
    "    q_key = q.split(\"=\")[0]\n",
    "    q_val = q.split(\"=\")[1]\n",
    "    params[q_key] = q_val\n",
    "    \n",
    "params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def requests_page(url, params):\n",
    "    response = requests.post(url, params)\n",
    "    response.raise_for_status()\n",
    "    html = bs(response.text)\n",
    "    return html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_last_page_no(html):\n",
    "    find_last_page = html.select(\"a\")[-1][\"href\"]\n",
    "    last_page_no = re.sub(\"[^0-9]\", \"\", find_last_page)\n",
    "    last_page_no = int(last_page_no)\n",
    "    return last_page_no"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_table(html):\n",
    "    table = html.select(\"table\")\n",
    "    doc = table[1].select(\"table\")[2]\n",
    "    df_doc = pd.read_html(str(doc))[0]\n",
    "    cols = df_doc.loc[0].tolist()\n",
    "    df_doc.columns = cols\n",
    "    df_doc = df_doc.drop(0)\n",
    "    return df_doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_table(url, params):\n",
    "    \"\"\"\n",
    "    url, params 를 넘기면 기록물 목록을 반환\n",
    "    \"\"\"\n",
    "    html = requests_page(url, params)\n",
    "    table = parse_table(html)\n",
    "    return table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34988"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "html = requests_page(url, params)\n",
    "last_page_no = find_last_page_no(html)\n",
    "last_page_no"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bookmark': '',\n",
       " 'pageParam': 'page',\n",
       " 'page': '2',\n",
       " 'searchResultCount': '',\n",
       " 'gubun': '80',\n",
       " 'orgnz_id': '108001',\n",
       " 'searchType': '',\n",
       " 'keyword': '',\n",
       " 'keywordGubun': '',\n",
       " 'viewPage': 'ClosedList2017'}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "last_page_no = 3\n",
    "page_list = []\n",
    "for page_no in range(1, last_page_no+1):\n",
    "    \"\"\"\n",
    "    0~1초 휴지시간을 갖고 페이지마다 돌며 수집\n",
    "    \"\"\"\n",
    "    random_time = np.random.uniform(0, 1)\n",
    "    time.sleep(random_time)\n",
    "    \n",
    "    params[\"page\"] = page_no\n",
    "    df_page = get_table(url, params)\n",
    "    page_list.append(df_page)\n",
    "    df = pd.concat(page_list)\n",
    "    df.to_csv(\"archives_go_kr.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(30, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>번호</th>\n",
       "      <th>기록물건명</th>\n",
       "      <th>생산부서</th>\n",
       "      <th>기록물철명</th>\n",
       "      <th>생산년도</th>\n",
       "      <th>관리번호</th>\n",
       "      <th>재분류결과</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>사건송치(경계침범,명예훼손)</td>\n",
       "      <td>경찰청 강원도지방경찰청 고성경찰서 수사과</td>\n",
       "      <td>수사사건종결철 2 (1980)</td>\n",
       "      <td>1980</td>\n",
       "      <td>CA0319141</td>\n",
       "      <td>부분공개</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>사건송치(사기)</td>\n",
       "      <td>경찰청 강원도지방경찰청 고성경찰서 수사과</td>\n",
       "      <td>수사사건종결철 2 (1980)</td>\n",
       "      <td>1980</td>\n",
       "      <td>CA0319141</td>\n",
       "      <td>부분공개</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>사건송치(폭행)</td>\n",
       "      <td>경찰청 강원도지방경찰청 고성경찰서 수사과</td>\n",
       "      <td>수사사건종결철 2 (1980)</td>\n",
       "      <td>1980</td>\n",
       "      <td>CA0319141</td>\n",
       "      <td>부분공개</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>사건송치(명예훼손,폭행)</td>\n",
       "      <td>경찰청 강원도지방경찰청 고성경찰서 수사과</td>\n",
       "      <td>수사사건종결철 2 (1980)</td>\n",
       "      <td>1980</td>\n",
       "      <td>CA0319141</td>\n",
       "      <td>부분공개</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>사건송치(사기,배임,강제집행면탈)</td>\n",
       "      <td>경찰청 강원도지방경찰청 고성경찰서 수사과</td>\n",
       "      <td>수사사건종결철 2 (1980)</td>\n",
       "      <td>1980</td>\n",
       "      <td>CA0319141</td>\n",
       "      <td>부분공개</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   번호               기록물건명                    생산부서             기록물철명  생산년도  \\\n",
       "0   1     사건송치(경계침범,명예훼손)  경찰청 강원도지방경찰청 고성경찰서 수사과  수사사건종결철 2 (1980)  1980   \n",
       "1   2            사건송치(사기)  경찰청 강원도지방경찰청 고성경찰서 수사과  수사사건종결철 2 (1980)  1980   \n",
       "2   3            사건송치(폭행)  경찰청 강원도지방경찰청 고성경찰서 수사과  수사사건종결철 2 (1980)  1980   \n",
       "3   4       사건송치(명예훼손,폭행)  경찰청 강원도지방경찰청 고성경찰서 수사과  수사사건종결철 2 (1980)  1980   \n",
       "4   5  사건송치(사기,배임,강제집행면탈)  경찰청 강원도지방경찰청 고성경찰서 수사과  수사사건종결철 2 (1980)  1980   \n",
       "\n",
       "        관리번호 재분류결과  \n",
       "0  CA0319141  부분공개  \n",
       "1  CA0319141  부분공개  \n",
       "2  CA0319141  부분공개  \n",
       "3  CA0319141  부분공개  \n",
       "4  CA0319141  부분공개  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(df.shape)\n",
    "pd.read_csv(\"archives_go_kr.csv\").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}