Provide a setuptools-based build infrastructure

.gitignore

@@ -1,6 +1,10 @@
 *.pyc
 *.swp
 *.log
-settings.py
 dump.rdb
 nohup.out
+/build
+/dist
+*.egg
+/nosetests.xml
+*.egg-info

.travis.yml

@@ -8,10 +8,9 @@ install:
   - PYTHONPATH= PATH=/home/travis/anaconda/bin:$PATH pip install -r requirements.txt --use-mirrors
   - PYTHONPATH= PATH=/home/travis/anaconda/bin:$PATH pip install patsy --use-mirrors
   - PYTHONPATH= PATH=/home/travis/anaconda/bin:$PATH pip install msgpack_python --use-mirrors
-  - cp src/settings.py.example src/settings.py
   - pip install pep8 --use-mirrors
 script:
-  - PYTHONPATH= PATH=/home/travis/anaconda/bin:$PATH nosetests -v --nocapture
+  - PYTHONPATH= python setup.py test
   - pep8 --exclude=migrations --ignore=E501,E251,E265 ./ 
 notifications:
   email: false

MANIFEST.in

@@ -0,0 +1,2 @@
+recursive-include src/skyline/webapp/static *
+recursive-include src/skyline/webapp/templates *

bin/analyzer.d

@@ -5,9 +5,17 @@
 BASEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."
 RETVAL=0
 
+# Determine the script to run depending on whether this script is in the source
+# tree or installed
+SCRIPT="${BASEDIR}/bin/analyzer-agent"
+if [ -e "${BASEDIR}/setup.py" ]
+then
+    SCRIPT="${BASEDIR}/src/skyline/analyzer/agent.py"
+fi
+
 start () {
     rm -f $BASEDIR/src/analyzer/*.pyc
-    /usr/bin/env python $BASEDIR/src/analyzer/analyzer-agent.py start
+    /usr/bin/env python $SCRIPT start
         RETVAL=$?
         if [[ $RETVAL -eq 0 ]]; then
             echo "started analyzer-agent"
@@ -20,7 +28,7 @@ start () {
 stop () {
     # TODO: write a real kill script
     ps aux | grep 'analyzer-agent.py start' | grep -v grep | awk '{print $2 }' | xargs sudo kill -9
-    /usr/bin/env python $BASEDIR/src/analyzer/analyzer-agent.py stop
+    /usr/bin/env python $SCRIPT stop
         RETVAL=$?
         if [[ $RETVAL -eq 0 ]]; then
             echo "stopped analyzer-agent"
@@ -32,7 +40,7 @@ stop () {
 
 run () {
     echo "running analyzer"
-    /usr/bin/env python $BASEDIR/src/analyzer/analyzer-agent.py run
+    /usr/bin/env python $SCRIPT run
 }
 
 # See how we were called.

bin/horizon.d

@@ -5,9 +5,17 @@
 BASEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."
 RETVAL=0
 
+# Determine the script to run depending on whether this script is in the source
+# tree or installed
+SCRIPT="${BASEDIR}/bin/horizon-agent"
+if [ -e "${BASEDIR}/setup.py" ]
+then
+    SCRIPT="${BASEDIR}/src/skyline/horizon/agent.py"
+fi
+
 start () {
     rm $BASEDIR/src/horizon/*.pyc
-    /usr/bin/env python $BASEDIR/src/horizon/horizon-agent.py start
+    /usr/bin/env python $SCRIPT start
         RETVAL=$?
         if [[ $RETVAL -eq 0 ]]; then
             echo "started horizon-agent"
@@ -20,7 +28,7 @@ start () {
 stop () {
     # TODO: write a real kill script
     ps aux | grep 'horizon-agent.py start' | grep -v grep | awk '{print $2 }' | xargs sudo kill -9
-    /usr/bin/env python $BASEDIR/src/horizon/horizon-agent.py stop
+    /usr/bin/env python $SCRIPT stop
         RETVAL=$?
         if [[ $RETVAL -eq 0 ]]; then
             echo "stopped horizon-agent"
@@ -32,7 +40,7 @@ stop () {
 
 run () {
     echo "running horizon"
-    /usr/bin/env python $BASEDIR/src/horizon/horizon-agent.py run
+    /usr/bin/env python $SCRIPT run
 }
 
 # See how we were called.

bin/webapp.d

@@ -5,9 +5,17 @@
 BASEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."
 RETVAL=0
 
+# Determine the script to run depending on whether this script is in the source
+# tree or installed
+SCRIPT="${BASEDIR}/bin/skyline-webapp"
+if [ -e "${BASEDIR}/setup.py" ]
+then
+    SCRIPT="${BASEDIR}/src/skyline/webapp/webapp.py"
+fi
+
 start () {
     rm -f $BASEDIR/src/webapp/*.pyc
-    /usr/bin/env python $BASEDIR/src/webapp/webapp.py start
+    /usr/bin/env python $SCRIPT start
         RETVAL=$?
         if [[ $RETVAL -eq 0 ]]; then
             echo "started webapp"
@@ -18,7 +26,7 @@ start () {
 }
 
 stop () {
-    /usr/bin/env python $BASEDIR/src/webapp/webapp.py stop
+    /usr/bin/env python $SCRIPT stop
         RETVAL=$?
         if [[ $RETVAL -eq 0 ]]; then
             echo "stopped webapp"
@@ -30,7 +38,7 @@ stop () {
 
 restart () {
     rm -f $BASEDIR/src/webapp/*.pyc
-    /usr/bin/env python $BASEDIR/src/webapp/webapp.py restart
+    /usr/bin/env python $SCRIPT restart
         RETVAL=$?
         if [[ $RETVAL -eq 0 ]]; then
             echo "restarted webapp"
@@ -42,7 +50,7 @@ restart () {
 
 run () {
     echo "running webapp"
-    /usr/bin/env python $BASEDIR/src/webapp/webapp.py run
+    /usr/bin/env python $SCRIPT run
 }
 
 # See how we were called.

setup.cfg

@@ -0,0 +1,2 @@
+[nosetests]
+tests = tests

setup.py

@@ -0,0 +1,49 @@
+from setuptools import setup, find_packages
+
+# work around a nose test bug: http://bugs.python.org/issue15881#msg170215
+try:
+    import multiprocessing
+except ImportError:
+    pass
+
+setup(name='skyline',
+      version='0.1.0',
+      description='''
+                  It'll detect your anomalies!
+                  ''',
+      author='etsy',
+      author_email='',
+      license='MIT',
+      url='https://github.com/etsy/skyline',
+      keywords=['anomaly detection', 'timeseries', 'monitoring'],
+      classifiers=['Programming Language :: Python'],
+
+      setup_requires=['nose>=1.0', 'unittest2', 'mock'],
+      install_requires=['redis==2.7.2',
+                        'hiredis==0.1.1',
+                        'python-daemon==1.6',
+                        'flask==0.9',
+                        'simplejson==2.0.9',
+                        'numpy',
+                        'scipy',
+                        'pandas',
+                        'patsy',
+                        'statsmodels',
+                        'msgpack_python'],
+
+      packages=find_packages('src'),
+      package_dir={'': 'src'},
+      include_package_data=True,
+
+      entry_points={
+          'console_scripts': [
+              'analyzer-agent = skyline.analyzer.agent:run',
+              'horizon-agent = skyline.horizon.agent:run',
+              'skyline-webapp = skyline.webapp.webapp:run'
+              ]
+      },
+
+      scripts=['bin/analyzer.d', 'bin/horizon.d', 'bin/webapp.d'],
+
+      test_suite='nose.collector',
+      )

src/analyzer/analyzer-agent.py → src/skyline/analyzer/agent.py

@@ -2,15 +2,14 @@
 import sys
 import traceback
 from os import getpid
-from os.path import dirname, abspath, isdir
+from os.path import isdir
 from daemon import runner
 from time import sleep, time
 
-# add the shared settings file to namespace
-sys.path.insert(0, dirname(dirname(abspath(__file__))))
-import settings
+from skyline import settings
+from skyline.analyzer.analyzer import Analyzer
 
-from analyzer import Analyzer
+logger = logging.getLogger("AnalyzerLog")
 
 
 class AnalyzerAgent():
@@ -28,7 +27,8 @@ def run(self):
         while 1:
             sleep(100)
 
-if __name__ == "__main__":
+
+def run():
     """
     Start the Analyzer agent.
     """
@@ -42,9 +42,9 @@ def run(self):
 
     # Make sure we can run all the algorithms
     try:
-        from algorithms import *
+        from skyline.analyzer import algorithms
         timeseries = map(list, zip(map(float, range(int(time()) - 86400, int(time()) + 1)), [1] * 86401))
-        ensemble = [globals()[algorithm](timeseries) for algorithm in settings.ALGORITHMS]
+        ensemble = [getattr(algorithms, algorithm)(timeseries) for algorithm in settings.ALGORITHMS]
     except KeyError as e:
         print "Algorithm %s deprecated or not defined; check settings.ALGORITHMS" % e
         sys.exit(1)
@@ -55,7 +55,6 @@ def run(self):
 
     analyzer = AnalyzerAgent()
 
-    logger = logging.getLogger("AnalyzerLog")
     logger.setLevel(logging.DEBUG)
     formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
     handler = logging.FileHandler(settings.LOG_PATH + '/analyzer.log')
@@ -68,3 +67,6 @@ def run(self):
         daemon_runner = runner.DaemonRunner(analyzer)
         daemon_runner.daemon_context.files_preserve = [handler.stream]
         daemon_runner.do_action()
+
+if __name__ == "__main__":
+    run()

src/analyzer/alerters.py → src/skyline/analyzer/alerters.py

@@ -3,7 +3,7 @@
 from email.MIMEImage import MIMEImage
 from smtplib import SMTP
 import alerters
-import settings
+from skyline import settings
 
 
 """

src/analyzer/algorithms.py → src/skyline/analyzer/algorithms.py

@@ -8,7 +8,7 @@
 from msgpack import unpackb, packb
 from redis import StrictRedis
 
-from settings import (
+from skyline.settings import (
     ALGORITHMS,
     CONSENSUS,
     FULL_DURATION,

src/analyzer/analyzer.py → src/skyline/analyzer/analyzer.py

@@ -11,7 +11,7 @@
 import traceback
 import operator
 import socket
-import settings
+from skyline import settings
 
 from alerters import trigger_alert
 from algorithms import run_selected_algorithm

src/horizon/horizon-agent.py → src/skyline/horizon/agent.py

@@ -6,16 +6,15 @@
 from multiprocessing import Queue
 from daemon import runner
 
-# add the shared settings file to namespace
-sys.path.insert(0, dirname(dirname(abspath(__file__))))
-import settings
+from skyline import settings
 
 from listen import Listen
 from roomba import Roomba
 from worker import Worker
 
 # TODO: http://stackoverflow.com/questions/6728236/exception-thrown-in-multiprocessing-pool-not-detected
 
+logger = logging.getLogger("HorizonLog")
 
 class Horizon():
     def __init__(self):
@@ -60,7 +59,8 @@ def run(self):
         while 1:
             time.sleep(100)
 
-if __name__ == "__main__":
+
+def run():
     """
     Start the Horizon agent.
     """
@@ -74,7 +74,6 @@ def run(self):
 
     horizon = Horizon()
 
-    logger = logging.getLogger("HorizonLog")
     logger.setLevel(logging.DEBUG)
     formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
     handler = logging.FileHandler(settings.LOG_PATH + '/horizon.log')
@@ -87,3 +86,6 @@ def run(self):
         daemon_runner = runner.DaemonRunner(horizon)
         daemon_runner.daemon_context.files_preserve = [handler.stream]
         daemon_runner.do_action()
+
+if __name__ == "__main__":
+    run()

src/horizon/listen.py → src/skyline/horizon/listen.py

@@ -6,7 +6,7 @@
 from msgpack import unpackb
 
 import logging
-import settings
+from skyline import settings
 
 logger = logging.getLogger("HorizonLog")
 
@@ -18,6 +18,11 @@
   import pickle
   USING_CPICKLE = False
 
+try:
+  from cStringIO import StringIO
+except:
+  from StringIO import StringIO
+
 # This whole song & dance is due to pickle being insecure
 # yet performance critical for carbon. We leave the insecure
 # mode (which is faster) as an option (USE_INSECURE_UNPICKLER).

src/horizon/roomba.py → src/skyline/horizon/roomba.py

@@ -7,7 +7,7 @@
 from time import time, sleep
 
 import logging
-import settings
+from skyline import settings
 
 logger = logging.getLogger("HorizonLog")
 

src/horizon/worker.py → src/skyline/horizon/worker.py

@@ -7,7 +7,7 @@
 
 import logging
 import socket
-import settings
+from skyline import settings
 
 logger = logging.getLogger("HorizonLog")
 

src/webapp/static/js/skyline.js → src/skyline/webapp/static/js/skyline.js

@@ -38,7 +38,7 @@ var handle_data = function(data) {
 // The callback to this function is handle_data()
 var pull_data = function() {
     $.ajax({
-        url: "/static/dump/anomalies.json",
+        url: "/anomalies.json",
         dataType: 'jsonp'
     });
 }