package.json

{
  "name": "crawler-analyzer",
  "version": "1.0.0",
  "description": "The goal of this project is to build a distributed, decentralized crawler to scrape 1 billion web pages using a couple hundred dollars  of commodity AWS hardware. The trick is using AWS spot instances, which allow compute on-demand with Amazon's idle resources. When demand is low, these spot instances can cost anywhere from 10-60% of their original price.",
  "main": "index.js",
  "scripts": {
    "prod": "cross-env NODE_ENV=production node --max-old-space-size=4096 src/index.js",
    "dev": "cross-env NODE_ENV=development node --max-old-space-size=2056 src/index.js",
    "stat-prod": "cross-env NODE_ENV=production node statistics/startServer.js 0.0.0.0 80",
    "debug": "cross-env NODE_ENV=development node --inspect-brk src/index.js",
    "test-unit": "cross-env NODE_ENV=test mocha tests/unit-tests/**/*test.js -w --reporter spec",
    "test-integration": "cross-env NODE_ENV=test mocha tests/integration-tests/**/*test.js -w --reporter spec",
    "test-e2e": "cross-env NODE_ENV=test mocha tests/end-to-end-tests/**/*test.js -w --reporter spec",
    "test": "cross-env NODE_ENV=test mocha tests/**/*test.js -w --reporter spec",
    "lint": "eslint .",
    "profile": "cross-env NODE_ENV=development node --prof src/index.js",
    "postinstall": "node -e \"var s='../',d='node_modules/APP',fs=require('fs');fs.exists(d,function(e){e||fs.symlinkSync(s,d,'dir')});\""
  },
  "author": "",
  "license": "ISC",
  "dependencies": {
    "aws-sdk": "^2.176.0",
    "axios": "^0.16.2",
    "bloomd": "^0.2.6",
    "bloomfilter": "^0.0.16",
    "bluebird": "^3.5.1",
    "body-parser": "^1.18.2",
    "bunyan": "^1.8.12",
    "cross-env": "^5.1.1",
    "dateformat": "^3.0.2",
    "death": "^1.1.0",
    "express": "^4.16.2",
    "htmlparser2": "^3.9.2",
    "lodash": "^4.17.5",
    "lru-cache": "^4.1.1",
    "minimist": "^1.2.0",
    "mkdirp": "^0.5.1",
    "posix": "^4.1.1",
    "request": "^2.83.0",
    "robots": "^0.10.1",
    "robots-parser": "^1.0.1",
    "s3-upload-stream": "^1.0.7",
    "sha1": "^1.1.1",
    "through2": "^2.0.3",
    "through2-concurrent": "^1.1.1",
    "tldjs": "^2.2.0"
  },
  "devDependencies": {
    "chai": "^4.1.2",
    "eslint": "^4.13.1",
    "eslint-config-airbnb-base": "^12.1.0",
    "eslint-plugin-import": "^2.8.0",
    "klaw": "^2.1.1",
    "mocha": "^4.0.1",
    "rimraf": "^2.6.2",
    "sinon": "^4.1.2",
    "stream-array": "^1.1.2",
    "string-to-stream": "^1.1.0",
    "supertest": "^3.0.0"
  },
  "directories": {
    "test": "tests"
  },
  "repository": {
    "type": "git",
    "url": "git+https://github.com/nadrane/crawler.git"
  },
  "bugs": {
    "url": "https://github.com/nadrane/crawler/issues"
  },
  "homepage": "https://github.com/nadrane/crawler#readme"
}