buildout.cfg

[buildout]
extensions = mr.developer
develop = .
unzip = true
parts = var karl supervisord.conf postoffice.ini

find-links = ./karlhosting/karl4

# override any eggs-directory or download-cache settings
# that might be in ~/.buildout/default.cfg. We have to do this because
# we're pinning versions using a custom package index instead of
# using versions.cfg.

customization-packages =

# version pinning:
show-picked-versions = true
update-versions-file = versions.cfg
extends = versions.cfg

[var]
recipe = z3c.recipe.mkdir
paths =
    var/chameleon_cache
    var/log

[sources]
karl = git git@github.com:karlproject/karl.git
relstorage = git git@github.com:zodb/relstorage.git
repoze.catalog = git git@github.com:repoze/repoze.catalog.git
repoze.pgtextindex = git git@github.com:repoze/repoze.pgtextindex.git
repoze.profile = git git@github.com:cguardia/repoze.profile.git
repoze.postoffice = git git@github.com:repoze/repoze.postoffice.git
repoze.sendmail = git git@github.com:karlproject/repoze.sendmail.git
repoze.urchin = svn svn+ssh://repoze@svn.repoze.org/svn/repoze.urchin/trunk
repozitory = git git@github.com:Pylons/repozitory.git
pyramid_formish = git git@github.com:Pylons/pyramid_formish.git
chameleon = git git@github.com:chrisrossi/chameleon.git
appendonly = svn svn+ssh://svn.zope.org/repos/main/appendonly/trunk
external_link_ticket = git git@github.com:karlproject/karl.external_link_ticket.git

[karl]
recipe = zc.recipe.egg
unzip = true
dependent-scripts = true
eggs =
    karl [tests]
    persistent
    ${buildout:customization-packages}
    repoze.debug
    repoze.profile
    supervisor
    superlance
    guillotine
    RelStorage [postgresql]
    repoze.pgtextindex
    repoze.postoffice
    repozitory
    pyramid_debugtoolbar
    slowlog
    waitress
    repoze.profile
    repoze.vhm
    repoze.urchin
dependent-scripts = true
interpreter = py

[postoffice.ini]
recipe = zc.recipe.deployment:configuration
directory = ${buildout:directory}/etc
name = ${:directory}/${:_buildout_section_name_}
maildir = ${buildout:directory}/var/po_mail
mail_filters = to_hostname:pg.example.com
text =
  [post office]
  zodb_uri = zconfig://${postoffice_zodb.conf:location}
  maildir = ${:maildir}
  max_message_size = 100m

  # 1 message every 4 minutes, using the To header field as a discriminator
  ooo_loop_frequency = 0.25
  ooo_loop_headers = To

  # Five minutes
  ooo_throttle_period = 300

  reject_filters =
      header_regexp_file: ${:directory}/header_checks.txt
      body_regexp_file: ${:directory}/body_checks.txt

  [queue:osi]
  filters = ${:mail_filters}

[postoffice_zodb.conf]
recipe = zc.recipe.deployment:configuration
directory = ${buildout:directory}/etc
name = ${:directory}/${:_buildout_section_name_}
dsn = postgresql://localhost/karltestpo
text =
  %import relstorage
  %define INSTANCE ${buildout:directory}
  <zodb main>
    cache-size 100000
    pool-size 1
    <relstorage>
      <postgresql>
        dsn ${:dsn}
      </postgresql>
      shared-blob-dir False
      blob-dir $INSTANCE/var/postoffice_blob_cache
      keep-history false
      cache-local-mb 500
      cache-local-object-max 20000
    </relstorage>
  </zodb>

[karldb]
dsn = postgresql://karltest:test@localhost:5432/karltest

[karl.ini]
recipe = zc.recipe.deployment:configuration
directory = ${buildout:directory}/etc
name = ${:directory}/${:_buildout_section_name_}
var = ${buildout:directory}/var

who_secret = secret
who_cookie = pnutbtrd
login_cookie_max_age = 36000
mail_white_list =
static_rev =
redis = redislog = false
sentry =
sentry_dsn =
sentry_handler =
graylog = graylog = false
graylog_handler =
graylog_host =
graylog_port =
statsd_uri =
  statsd_uri = statsd://localhost:8125
statsd =
framestats = true
slowlog = slowlog = false
timezone =
connection_stats_threshhold =
  connection_stats_filename = ${:var}/log/connection_stats.csv
  connection_stats_threshhold = 0.0
pipeline_debug =
  debug = true
  jquery_dev_mode = false
  js_devel_mode = false

mailout_throttle = 30
mailout_stats_dir = ${:var}/mailout_stats_dir
offline_app_url = karl.example.org
system_email_domain = example.org
system_list_subdomain =
mailin_trace_file =
browser_upgrade_url =
  browser_upgrade_url = 'http://mozilla.org'
box.client_id =
script_app_url = https://karlstaging.gocept.com
statsd_uri =
urchin_filter =
browserid_secret = ssshhhwabbits
statsd_uri = statsd_uri = statsd://localhost:8125
http_port = 6543
threadpool_workers = 1
redislog =
handler_redislog =
postoffice.queue = karl
admin =
  admin_only = false
  admin_userids =
      nborland
      plonepaul
      cguardia
      obell
      ajoseph
      cwinston
admin_email = admin@example.org
kaltura =
gsa_sync =
main-pipeline =
  [pipeline:main]
  pipeline =
      egg:guillotine#guillotine
      egg:Paste#evalerror
      egg:karl#timeit
      browserid
      karl
zodbconn.uri.postoffice =
saml =

text =
  [app:karl]
  use = egg:karl
  var = ${:var}
  who_secret = ${:who_secret}
  who_cookie = ${:who_cookie}
  login_cookie_max_age = ${:login_cookie_max_age}
  ${:admin}
  ${:mail_white_list}
  update.timestamp = ./karlhosting/TIMESTAMP.txt
  error_monitor_dir = ${:var}/errors
  static_rev = ${:static_rev}
  postoffice.queue = ${:postoffice.queue}
  debugtoolbar = false
  zodbconn.uri = zconfig:${pg_zodb.conf:location}
  ${:zodbconn.uri.postoffice}
  tm.attempts = 5
  mailin_trace_file = ${:var}/mailin_trace

  ${:redis}
  ${:sentry}

  ${:statsd_uri}
  framestats = ${:framestats}
  ${:slowlog}

  ${:timezone}

  ${:connection_stats_threshhold}
  ${:pipeline_debug}

  # cutnpasted from karlserve.instance python code ("global" config)
  reload_templates = false
  static_postfix = static
  # XXX ???
  upload_limit = 0  
  min_pw_length = 8
  selectable_groups = group.KarlStaff group.KarlUserAdmin group.KarlAdmin group.KarlCommunications
  aspell_executable = aspell
  aspell_max_words = 5000
  aspell_languages = en

  # cutnpasted from karlserve.instance ("per instance" config)
  system_name = KARL

  error_monitor_subsystems =
                           karl
                           mailin
                           mailout
                           ${:gsa_sync}
                           digest
                           update_feeds

  # mode can be NORMAL, READONLY, or MAINTENANCE
  mode = NORMAL

  # cutnpasted from instances.ini (uh.. another "per instance" config)
  repozitory_db_string = ${karldb:dsn}
  pgtextindex.dsn = ${karldb:dsn}
  pgtextindex.maxlen = 70000

  # We can hard code this here, since this is only used by OSI
  intranet_search_paths = 
      /profiles
      /offices
      /communities/budget/view.html
      /communities/global-training-calendar
      /communities/legal-policies
      /communities/resource-center
      /communities/netsuite-go-live
      /communities/password-help
      /communities/osf-dc-ospc
      /communities/strategy-budget
      /communities/strategy-unit
      /communities/the-president-s-office

  # Formerly persistent config on OSF, no need to configure
  system_name = KARL
  admin_email = ${:admin_email}
  forgot_password_url = https://businesscenter.soros.org/gsa/forgotpassword.aspx
  ${:kaltura}
  package = osi
  postoffice.bounce_from_email = bounces@osf.karlstaging.gocept.com
  reload_templates = false
  staff_change_password_url = https://businesscenter.soros.org/gsa/changepassword.aspx
  statistics_folder = var/stats/osf
  mailout_throttle = ${:mailout_throttle}
  mailout_stats_dir = ${:mailout_stats_dir}

  # Formerly persistent config on OSF that should be configured
  offline_app_url = ${:offline_app_url}
  system_email_domain = ${:system_email_domain}
  ${:system_list_subdomain}
  ${:mailin_trace_file}
  ${:browser_upgrade_url}
  ${:box.client_id}

  script_app_url = ${:script_app_url}

  ${:saml}

  ${:urchin_filter}

  [filter:browserid]
  use = egg:repoze.browserid#browserid
  secret_key = ${:browserid_secret}

  [filter:responselogger]
  use = egg:repoze.debug#responselogger
  trace_log = ${:var}/log/trace.log

  ${:statsd}

  [filter:profile]
  use = egg:repoze.profile
  log_filename = ${:var}/log/karl.profile
  discard_first_request = true
  path = /__profile__
  #flush_at_shutdown = true

  ${:main-pipeline}

  [server:main]
  use = egg:waitress#main
  host = 0.0.0.0
  port = ${:http_port}
  threads = ${:threadpool_workers}

  [loggers]
  keys = root, karl, waitress, osi, QueueProcessor, repoze.postoffice

  [handlers]
  keys = console${:redislog}${:sentry_handler}${:graylog_handler}

  [formatters]
  keys = generic

  [logger_root]
  level = INFO
  handlers = console${:sentry_handler}${:graylog_handler}

  [logger_karl]
  level = DEBUG
  handlers = console${:redislog}${:graylog_handler}
  qualname = karl

  [logger_osi]
  level = DEBUG
  handlers = console${:redislog}${:graylog_handler}
  qualname = osi

  [logger_QueueProcessor]
  level = INFO
  handlers = console${:redislog}${:graylog_handler}
  qualName = QueueProcessor

  [logger_repoze.postoffice]
  level = INFO
  handlers = console${:redislog}${:graylog_handler}
  qualName = repoze.postoffice

  [logger_waitress]
  level = INFO
  handlers = 
  qualname = waitress

  [handler_console]
  class = StreamHandler
  args = (sys.stderr,)
  level = NOTSET
  formatter = generic

  ${:handler_redislog}

  [handler_sentry]
  class = getattr(__import__('raven.handlers.logging', {}, {}, ['*']), 'SentryHandler')
  args = ('${:sentry_dsn}',)
  level = ERROR

  [handler_graylog]
  class = getattr(__import__('pygelf', {}, {}, ['*']), 'GelfTcpHandler')
  args = ('${:graylog_host}', ${:graylog_port})
  level = NOTSET
  formatter = generic

  [formatter_generic]
  format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s

[pg_zodb.conf]
recipe = zc.recipe.deployment:configuration
directory = ${buildout:directory}/etc
name = ${:directory}/${:_buildout_section_name_}
zodb_cache_size = 99999999
zodb_cache_size_bytes =
zodb_pool_size = 4
relstorage_cache_servers =
relstorage_cache_prefix =
cache_local_mb = cache-local-mb 100
cache_local_object_max =
cache-local-dir = ${buildout:directory}/var/pickle-cache
blob-cache-local-dir = ${buildout:directory}/var/blob-cache
text =
  %import newt.db
  %define INSTANCE ${buildout:directory}

  <zodb>
    cache-size ${:zodb_cache_size}
    ${:zodb_cache_size_bytes}
    pool-size ${:zodb_pool_size}
    <relstorage>
      <newt>
        transform karl.models.newttransform.transform
        auxiliary-tables karlex
        <postgresql>
          dsn ${karldb:dsn}
        </postgresql>
      </newt>
      shared-blob-dir False
      blob-dir ${:blob-cache-local-dir}
      blob-cache-size 1gb
      keep-history False
      read-only False
      commit-lock-timeout 60
      ${:relstorage_cache_servers}
      ${:relstorage_cache_prefix}
      ${:cache_local_mb}
      ${:cache_local_object_max}
    </relstorage>
  </zodb>

[supervisord.conf]
recipe = zc.recipe.deployment:configuration
directory = ${buildout:directory}/etc
name = ${:directory}/${:_buildout_section_name_}
var = ${buildout:directory}/var
supervisord_port = 8889
postgres =
  [program:postgres]
  command = %(here)s/../bin/postgres -D %(here)s/../var/postgresql
  redirect_stderr = true
  stdout_logfile = %(here)s/../var/log/postgres.log
  autostart = False

environment = CHAMELEON_CACHE='${:var}/chameleon_cache',

prod =

memmon_karl = 1GB

text =
  [unix_http_server]
  file = ${buildout:directory}/var/supervisord.sock

  [supervisord]
  logfile=${:var}/log/supervisord.log
  logfile_maxbytes=50MB
  logfile_backups=10
  loglevel=info
  pidfile=${:var}/supervisord.pid
  nodaemon=false

  [rpcinterface:supervisor]
  supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

  [supervisorctl]
  serverurl=unix://${buildout:directory}/var/supervisord.sock

  ${:postgres}

  [program:karl]
  command = ${buildout:bin-directory}/paster serve ${karl.ini:location}
  redirect_stderr = true
  stdout_logfile = ${:var}/log/karl.log
  environment =
      ${:environment}

  ${:prod}

  # Restart karl when it's using too much memory
  [eventlistener:memmon-karl]
  command=%(here)s/../bin/memmon -p karl=${:memmon_karl}
  events=TICK_60