Edge.couchdb 2.x support

bootstrap.py

@@ -25,10 +25,7 @@
 
 from optparse import OptionParser
 
-__version__ = '2015-07-01'
-# See zc.buildout's changelog if this version is up to date.
-
-tmpeggs = tempfile.mkdtemp(prefix='bootstrap-')
+tmpeggs = tempfile.mkdtemp()
 
 usage = '''\
 [DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
@@ -43,9 +40,8 @@
 '''
 
 parser = OptionParser(usage=usage)
-parser.add_option("--version",
-                  action="store_true", default=False,
-                  help=("Return bootstrap.py version."))
+parser.add_option("-v", "--version", help="use a specific zc.buildout version")
+
 parser.add_option("-t", "--accept-buildout-test-releases",
                   dest='accept_buildout_test_releases',
                   action="store_true", default=False,
@@ -63,33 +59,25 @@
 parser.add_option("--allow-site-packages",
                   action="store_true", default=False,
                   help=("Let bootstrap.py use existing site packages"))
-parser.add_option("--buildout-version",
-                  help="Use a specific zc.buildout version")
 parser.add_option("--setuptools-version",
-                  help="Use a specific setuptools version")
-parser.add_option("--setuptools-to-dir",
-                  help=("Allow for re-use of existing directory of "
-                        "setuptools versions"))
+                  help="use a specific setuptools version")
 
-options, args = parser.parse_args()
-if options.version:
-    print("bootstrap.py version %s" % __version__)
-    sys.exit(0)
 
+options, args = parser.parse_args()
 
 ######################################################################
 # load/install setuptools
 
 try:
+    if options.allow_site_packages:
+        import setuptools
+        import pkg_resources
     from urllib.request import urlopen
 except ImportError:
     from urllib2 import urlopen
 
 ez = {}
-if os.path.exists('ez_setup.py'):
-    exec(open('ez_setup.py').read(), ez)
-else:
-    exec(urlopen('https://bootstrap.pypa.io/ez_setup.py').read(), ez)
+exec(urlopen('https://bootstrap.pypa.io/ez_setup.py').read(), ez)
 
 if not options.allow_site_packages:
     # ez_setup imports site, which adds site packages
@@ -100,21 +88,12 @@
     # We can't remove these reliably
     if hasattr(site, 'getsitepackages'):
         for sitepackage_path in site.getsitepackages():
-            # Strip all site-packages directories from sys.path that
-            # are not sys.prefix; this is because on Windows
-            # sys.prefix is a site-package directory.
-            if sitepackage_path != sys.prefix:
-                sys.path[:] = [x for x in sys.path
-                               if sitepackage_path not in x]
+            sys.path[:] = [x for x in sys.path if sitepackage_path not in x]
 
 setup_args = dict(to_dir=tmpeggs, download_delay=0)
 
 if options.setuptools_version is not None:
     setup_args['version'] = options.setuptools_version
-else:
-    setup_args['version'] = "7.0"
-if options.setuptools_to_dir is not None:
-    setup_args['to_dir'] = options.setuptools_to_dir
 
 ez['use_setuptools'](**setup_args)
 import setuptools
@@ -131,12 +110,7 @@
 
 ws = pkg_resources.working_set
 
-setuptools_path = ws.find(
-    pkg_resources.Requirement.parse('setuptools')).location
-
-# Fix sys.path here as easy_install.pth added before PYTHONPATH
 cmd = [sys.executable, '-c',
-       'import sys; sys.path[0:0] = [%r]; ' % setuptools_path +
        'from setuptools.command.easy_install import main; main()',
        '-mZqNxd', tmpeggs]
 
@@ -149,8 +123,11 @@
 if find_links:
     cmd.extend(['-f', find_links])
 
+setuptools_path = ws.find(
+    pkg_resources.Requirement.parse('setuptools')).location
+
 requirement = 'zc.buildout'
-version = options.buildout_version or "2.2.5"
+version = options.version
 if version is None and not options.accept_buildout_test_releases:
     # Figure out the most recent final version of zc.buildout.
     import setuptools.package_index
@@ -190,7 +167,7 @@ def _final_version(parsed_version):
 cmd.append(requirement)
 
 import subprocess
-if subprocess.call(cmd) != 0:
+if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=setuptools_path)) != 0:
     raise Exception(
         "Failed to execute command:\n%s" % repr(cmd)[1:-1])
 

bootstrap.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+virtualenv --clear --never-download .
+./bin/pip install -r requirements.txt

profiles/auctions.cfg

@@ -8,3 +8,6 @@ edge_data_bridge_auctions = True
 [edge_data_bridge_auctions.yaml]
 <= edge_data_bridge_defaults
 resources = auctions
+
+[nginx.conf]
+auctions = 1

profiles/base.cfg

@@ -3,6 +3,7 @@ package-name =
     openprocurement.edge
     openprocurement_client
 parts =
+    mkdirs
     autostart
     scripts
     couchpy
@@ -16,6 +17,7 @@ parts =
     openprocurement.edge.ini
     edge_data_bridge
     nginx.conf
+    couchdb.sh
 eggs =
     request_id_middleware
     server_cookie_middleware
@@ -29,12 +31,25 @@ recipe = collective.recipe.template[genshi]:genshi
 input = ${buildout:directory}/templates/${:_buildout_section_name_}
 output = ${buildout:directory}/etc/${:_buildout_section_name_}
 
+[couchdb.sh]
+recipe = sact.recipe.jinjatemplate
+template-file = templates/${:_buildout_section_name_}
+target-file = bin/${:_buildout_section_name_}
+target-executable = true
+
+[mkdirs]
+recipe =  z3c.recipe.mkdir
+paths =
+    ${buildout:directory}/var/log
+    ${buildout:directory}/var/couchdb
+    ${buildout:directory}/var/log/nginx
+
 [scripts]
 recipe = zc.recipe.egg
 eggs =
     chaussette
     ${buildout:eggs}
-interpreter = python
+interpreter = python_interpreter
 scripts = chaussette
 
 [circus]
@@ -79,7 +94,7 @@ edge_data_bridge_tenders =
 statsd = True
 edge = True
 edge2 = True
-nginx = True
+nginx = False
 statsd = False
 migration_skip = True
 sandbox_mode = True
@@ -91,11 +106,13 @@ edge_host = 0.0.0.0
 endpoint = ipc://${buildout:directory}/var/circus_endpoint
 pubsub_endpoint = ipc://${buildout:directory}/var/circus_pubsub
 timezone = Europe/Kiev
+couchdb_ver = ${couchdb.ini:couchdb_ver}
 
 [couchdb.ini]
 <= config-from-template
 port = 5984
 host = 127.0.0.1
+couchdb_ver = 1
 compaction_start_time = 00:00
 compaction_end_time = 04:00
 username =
@@ -134,9 +151,10 @@ scripts = edge_data_bridge
 [edge_data_bridge_defaults]
 <= config-from-template
 input = ${buildout:directory}/templates/edge_data_bridge.yaml
-user_agent = edge.${:resources}
+resources =
 resources_api_server =
 resources_api_version = 2.3
+user_agent = edge.${:resources}
 couch_url = ${openprocurement.edge.ini:couchdb_url}
 public_db = ${openprocurement.edge.ini:couchdb_db}
 log_db_name = ${openprocurement.edge.ini:logs_db}
@@ -168,6 +186,26 @@ bulk_save_interval = 3
 bulk_query_interval = 3
 bulk_query_limit = 5000
 
+[config-from-template-jinja]
+recipe = sact.recipe.jinjatemplate
+template-file = templates/${:_buildout_section_name_}
+target-file = etc/${:_buildout_section_name_}
+
+[nginx.conf]
+<= config-from-template-jinja
+server_name = "edge"
+port = 8090
+static_path = static
+auctions =
+contracts =
+plans =
+tenders =
+couchdb_host = ${couchdb.ini:host}
+couchdb_port = ${couchdb.ini:port}
+couchdb_db = ${openprocurement.edge.ini:couchdb_db}
+edge_port = ${circus.ini:edge_port}
+api_version = ${edge_data_bridge_defaults:resources_api_version}
+
 [logrotate.conf]
 <= config-from-template
 
@@ -176,7 +214,6 @@ recipe = z3c.recipe.usercrontab
 times = @hourly
 command = /usr/sbin/logrotate --state ${buildout:directory}/var/logrotate.status ${logrotate.conf:output} >>${buildout:directory}/var/log/logrotate.log 2>&1
 
-
 [autostart]
 recipe = z3c.recipe.usercrontab
 times = @reboot

profiles/contracts.cfg

@@ -9,3 +9,6 @@ edge_data_bridge_contracts = True
 [edge_data_bridge_contracts.yaml]
 <= edge_data_bridge_defaults
 resources = contracts
+
+[nginx.conf]
+contracts = 1

profiles/devel.cfg

@@ -6,3 +6,16 @@ extends =
     versions.cfg
 show-picked-versions = true
 update-versions-file = profiles/versions.cfg
+parts +=
+    nginx.conf
+    couchdb.sh
+
+[circus.ini]
+nginx = True
+
+[couchdb.sh]
+recipe = sact.recipe.jinjatemplate
+template-file = templates/${:_buildout_section_name_}
+target-file = bin/${:_buildout_section_name_}
+
+target-executable = true

profiles/plans.cfg

@@ -8,3 +8,6 @@ edge_data_bridge_plans = True
 [edge_data_bridge_plans.yaml]
 <= edge_data_bridge_defaults
 resources = plans
+
+[nginx.conf]
+plans = 1

profiles/sources.cfg

@@ -8,5 +8,6 @@ opgh = ${remotes:gh}openprocurement/openprocurement
 opgh_push = ${remotes:gh_push}openprocurement/openprocurement
 
 [sources]
+statsdhandler = git ${remotes:gh}openprocurement/statsdhandler.git pushurl=${remotes:gh_push}openprocurement/statsdhandler.git
 openprocurement.edge = git ${remotes:opgh}.edge.git pushurl=${remotes:opgh}.edge.git branch=master
 openprocurement_client = git ${remotes:opgh}.client.python.git pushurl=${remotes:opgh_push}.client.python.git branch=use_requests

profiles/statsd.cfg

@@ -0,0 +1,6 @@
+[buildout]
+package-name += statsdhandler
+
+[statsd_handler_defaults]
+<= config-from-template
+input = ${buildout:directory}/templates/statsd_handler.yaml

profiles/statsd_auctions.cfg

@@ -0,0 +1,17 @@
+[buildout]
+parts +=
+    edge_data_bridge_auctions.yaml
+    statsd_auctions_handler.yaml
+
+[circus.ini]
+edge_data_bridge_auctions = True
+
+[edge_data_bridge_auctions.yaml]
+<= edge_data_bridge_defaults
+resources = auctions
+statsdhandler = 1
+handler_config_path = ${buildout:directory}/etc/statsd_${:resources}_handler.yaml
+
+[statsd_auctions_handler.yaml]
+<= statsd_handler_defaults
+app_key = edge_auctions

profiles/statsd_contracts.cfg

@@ -0,0 +1,17 @@
+[buildout]
+parts +=
+    edge_data_bridge_contracts.yaml
+    statsd_contracts_handler.yaml
+
+[circus.ini]
+edge_data_bridge_contracts = True
+
+[edge_data_bridge_contracts.yaml]
+<= edge_data_bridge_defaults
+resources = contracts
+statsdhandler = 1
+handler_config_path = ${buildout:directory}/etc/statsd_${:resources}_handler.yaml
+
+[statsd_contracts_handler.yaml]
+<= statsd_handler_defaults
+app_key = edge_contracts

profiles/statsd_plans.cfg

@@ -0,0 +1,17 @@
+[buildout]
+parts +=
+    edge_data_bridge_plans.yaml
+    statsd_plans_handler.yaml
+
+[circus.ini]
+edge_data_bridge_plans = True
+
+[edge_data_bridge_plans.yaml]
+<= edge_data_bridge_defaults
+resources = plans
+statsdhandler = 1
+handler_config_path = ${buildout:directory}/etc/statsd_${:resources}_handler.yaml
+
+[statsd_plans_handler.yaml]
+<= statsd_handler_defaults
+app_key = edge_plans

profiles/statsd_tenders.cfg

@@ -0,0 +1,17 @@
+[buildout]
+parts +=
+    edge_data_bridge_tenders.yaml
+    statsd_tenders_handler.yaml
+
+[circus.ini]
+edge_data_bridge_tenders = True
+
+[edge_data_bridge_tenders.yaml]
+<= edge_data_bridge_defaults
+resources = tenders
+statsdhandler = 1
+handler_config_path = ${buildout:directory}/etc/statsd_${:resources}_handler.yaml
+
+[statsd_tenders_handler.yaml]
+<= statsd_handler_defaults
+app_key = edge_tenders

profiles/tenders.cfg

@@ -8,3 +8,6 @@ edge_data_bridge_tenders = True
 [edge_data_bridge_tenders.yaml]
 <= edge_data_bridge_defaults
 resources = tenders
+
+[nginx.conf]
+tenders = 1