master.cfg

# ex: set ft=python:
##############################################################################
# Name:       master.cfg
# Purpose:    Buildbot configuration for wxWidgets
# Created:    Mar 15 2007
# Copyright:  (c) 2007 Mike Wetherell
#             (c) 2015 Vadim Zeitlin
# Licence:    wxWidgets licence
##############################################################################

# Define the dictionary used by buildmaster and its usual alias.
c = BuildmasterConfig = {}

buildmaster_email = "vadim@wxwidgets.org"

####### BUILD SLAVES

from buildbot.buildslave import BuildSlave
c["slaves"] = []

# The passwords are in a separate file to keep them secret.
try:
    execfile("passwd.cfg")
except:
    raise RuntimeError("Error reading slave table")

for s in slaves.items():
    c["slaves"].append(BuildSlave(s[0], s[1],
                       notify_on_missing=buildmaster_email))

# Only listen on localhost interface as all connections go through stunnel.
c["slavePortnum"] = "tcp:9989:interface=127.0.0.1"

####### CHANGE SOURCES

# Currently none, Github notifies us using a web hook, see status target
# configuration below.

####### BUILDERS

from buildbot.process import factory
from buildbot.steps.shell import ShellCommand,Compile
c["builders"] = []

quick_builders = {"master": [], "WX_3_0_BRANCH": []}
full_builders = {"master": [], "WX_3_0_BRANCH": []}

# Helpers for constructing the build steps used below.

# m must be either "incremental" or "full"
def checkout_step(br, m):
    from buildbot.steps.source.git import Git
    return Git(
        repourl="https://github.com/wxWidgets/wxWidgets.git",
        branch=br,
        mode=m,
        submodules=True,
    )

def configure_step(config_args=[]):
    from buildbot.steps.shell import Configure
    return Configure(
        command="./configure %s" % ' '.join(config_args)
    )

def build_step(what, **kwargs):
    class GCCCompile(Compile):
        def __init__(self, **kwargs):
            # Match gcc/clang warning format:
            #
            #   file.c:123: warning: whatever
            Compile.__init__(
                self,
                warningPattern=r"^(.+?):([0-9]+): warning: (.*)$",
                warningExtractor=Compile.warnExtractFromRegexpGroups,
                **kwargs
            )

            self.addSuppression([
                # In OS X and GTK+ 3 builds there are a lot of warnings about
                # using deprecated toolkit functions, and there are also a
                # couple of them in our own tests which check that deprecated
                # functionality still works.
                (None, ".*deprecated.*", None, None),

                # We're not interested in warnings in 3rd party libraries at all.
                (r"^.*src/tiff/.*$", None, None, None),
            ])

    return GCCCompile(
        name="build " + what,
        description=["building", what],
        descriptionDone=[what],
        **kwargs
    )

def build_libraries_step():
    return build_step("libraries")

def build_programs_step():
    return build_step("programs",
        command="(cd samples && make) && (cd utils && make)",
        haltOnFailure=False,
    )

def build_tests_step():
    return build_step("tests",
        workdir="build/tests",
    )

def run_tests_step(cmd, **kwargs):
    return ShellCommand(
        name="test",
        workdir="build/tests",
        command=cmd,
        warnOnFailure=True,
        sigtermTime=10,
        description=["running", "tests"],
        descriptionDone=["tests"],
        **kwargs
    )

# cmd is the command used to actually run the GUI test.
def run_gui_tests_step(cmd, **kwargs):
    return ShellCommand(
        name="GUI test",
        workdir="build/tests",
        command=cmd,
        flunkOnFailure=False, # GUI tests are flaky under buildbot
        warnOnFailure=True,
        sigtermTime=10,
        description=["running", "GUI", "tests"],
        descriptionDone=["GUI", "tests"],
        **kwargs
    )

# Build a full shell script running the specified command and checking its
# core, if it crashed and created one.
def run_and_check_core(cmd):
    return """\
ulimit -c unlimited
ERR=0

run_and_check_core() {
    rm -f core
    trap 'kill -TERM $child_pid' TERM
    "$@" &
    child_pid=$!
    wait $child_pid
    ERR=$?
    if [ -f core ]; then
        echo Command crashed.
        if [ -x "`which gdb`" ]; then
            echo Attempting to display backtrace:
            gdb -batch -c core -ex 'set pagination off' -ex bt "$1"
        fi
        rm -f core
    fi
    echo
}

run_and_check_core %s
exit $ERR
""" % cmd

def do_make_unix_like_factory(branch, config_args, mode, run_gui_test_cmd):
    return factory.BuildFactory([
        # This needs to be done whenever this file changes in Git, as it's
        # modified locally after running configure in the build directory.
        #ShellCommand(
        #    name="clean up",
        #    command="rm -f src/tiff/libtiff/tif_config.h",
        #),
        checkout_step(branch, mode),
        configure_step(config_args),
        build_libraries_step(),
        build_programs_step(),
        build_tests_step(),
        run_tests_step(run_and_check_core("./test")),
        run_gui_tests_step(run_and_check_core(run_gui_test_cmd)),
    ])

def make_unix_factory(branch="master", config_args=[], mode="incremental"):
    return do_make_unix_like_factory(
        branch, config_args, mode, "./test_gui --sync"
    )

def make_osx_factory(branch="master", config_args=[], mode="incremental"):
    return do_make_unix_like_factory(
        branch, config_args, mode, "./test_gui.app/Contents/MacOS/test_gui"
    )

def make_ios_factory(branch="master", mode="incremental"):
    config_args=["--with-osx_iphone",
                 "--enable-monolithic",
                 "--with-macosx-version-min=7.1",
                 "--with-macosx-sdk=$(xcrun --sdk iphonesimulator --show-sdk-path)",
                ]

    # Don't build any programs for iOS, this doesn't work currently.
    return factory.BuildFactory([
        checkout_step(branch, mode),
        configure_step(config_args),
        build_libraries_step(),
    ])

def build_nmake_step(what, makedir, **kwargs):
    class MSVCCompile(Compile):
        def __init__(self, **kwargs):
            # Match MSVC warning format:
            #
            #   file.c(123) : warning C1234: whatever
            Compile.__init__(
                self,
                warningPattern=r"^(.+?)\(([0-9]+)\) : warning ([A-Z]{1,3}[0-9]{4}: .*)" + "\r?$",
                warningExtractor=Compile.warnExtractFromRegexpGroups,
                **kwargs
            )

            self.addSuppression([
                # Avoid warnings about deprecated standard library functions
                # in third party code.
                (r"^.*\\src\\zlib\\.*\.c$", r"^C4996:.*", None, None),

                # With VC8 (only?) there are also warnings about [x]sgetn()
                # being deprecated in wxStdInputStreamBuffer declaration and
                # implementation.
                (r"^.*\\wx/stdstream\.h$", r"^C4996:.*", None, None),
                (r"^.*\\streams\\stdstream\.cpp$", r"^C4996:.*", None, None),
            ])

    return MSVCCompile(
        name=what,
        workdir=r"build\%s" % makedir,
        command=["nmake",
                 "/nologo",
                 "/f", "makefile.vc",
                 "BUILD=release",
                 "SHARED=1",
                ],
        description=["building", what],
        descriptionDone=[what],
        **kwargs
    )

# We have mode parameter here for consistency with the other factories, but
# this one is only really available in full mode as nmake is incapable of
# detecting header dependencies and so can't be used for quick rebuilds.
def make_nmake_factory(branch="master", mode="full"):
    if mode != "full":
        raise RuntimeError("nmake factory doesn't support incremental builds")

    # Make sure the DLLs are found when running tests.
    tests_env={"PATH": [r"..\lib\vc_dll", "${PATH}"]}

    return factory.BuildFactory([
        checkout_step(branch, mode),
        build_nmake_step("libraries", r"build\msw"),
        build_nmake_step("samples", r"samples",
            haltOnFailure=False,
        ),
        build_nmake_step("tests", r"tests"),
        run_tests_step(r"vc_mswudll\test.exe", env=tests_env),
        run_gui_tests_step(r"vc_mswudll\test_gui.exe", env=tests_env)
    ])


# This function should be used instead of adding builders directly to
# c["builders"] as it ensures that all builders use consistent names and build
# directories and also adds each of them to one of xxx_builders arrays so that
# it can be associated with the right scheduler later.
#
# Optional keyword arguments are passed to the factory function.
def add_builder(slave,
                branch,
                port,
                factory,
                full=False,
                platform="",
                **kwargs):
    n = ""
    d = "builds/"

    if branch == "master":
        d += "master"
    elif branch == "WX_3_0_BRANCH":
        d += "wx30"
        n += "3.0-"
    else:
        raise "Unknown branch " + branch

    n += port
    d += "_" + port[2:].replace("-", "_").lower() # Discard "wx" prefix.

    if platform != "":
        n += "-" + platform
        d += "_" + platform.replace("-", "_").lower()

    if full:
        n += "-full"
        d += "_full"
        kwargs.update({"mode": "full"})

        full_builders[branch].append(n)
    else:
        quick_builders[branch].append(n)

    c["builders"].append({
            "name": n,
            "builddir": d,
            "slavename": slave,
            "factory": factory(branch, **kwargs)
        })

# Convenience wrapper for add_builder to add an unstable master build.
def add_master_builder(slave,
                       port,
                       factory,
                       full=False,
                       platform="",
                       **kwargs):
    add_builder(slave, "master", port, factory, full, platform, **kwargs)

# Convenience wrapper for add_builder to add a stable branch build.
def add_wx30_builder(slave,
                     port,
                     factory,
                     full=False,
                     platform="",
                     **kwargs):
    add_builder(slave, "WX_3_0_BRANCH", port, factory, full, platform, **kwargs)


# Map platforms to the slaves using them.
slave_for = {
        "Linux-i686":    "brandt32",
        "Linux-x86_64":  "brandt64",
    }

# Set up Linux builds done on both master and 3.0 branches.
for br in ["master", "WX_3_0_BRANCH"]:
    # The same wxGTK builds are done on a couple of different platforms.
    for pl in ["Linux-i686", "Linux-x86_64"]:
        if br == "master":
            disable_compat_arg="--disable-compat30"
        elif br == "WX_3_0_BRANCH":
            disable_compat_arg="--disable-compat28"
        else:
            raise "Unknown branch " + br

        add_builder(
            slave=slave_for[pl],
            branch=br,
            port="wxGTK-STL",
            platform=pl,
            factory=make_unix_factory,
            config_args=["--enable-stl", disable_compat_arg]
        )

        add_builder(
            slave=slave_for[pl],
            branch=br,
            port="wxGTK",
            platform=pl,
            full=True,
            factory=make_unix_factory,
        )

# Set up OS X builds: things are more complicated here as they don't use the
# same options with master and 3.0.

osx_args_32_only = ["--enable-macosx_arch=i386"]
osx_args_64_only = ["--enable-macosx_arch=x86_64"]
osx_args_min_10_5 = ["--with-macosx-version-min=10.5",
                     "--with-macosx-sdk=/Developer/SDKs/MacOSX10.6.sdk",
                     "--with-cppunit-prefix=/Developer/libstdc++variants/cppunit",
                    ] + osx_args_32_only
osx_args_min_10_7 = ["--with-macosx-version-min=10.7",
                     "--with-cppunit-prefix=/Developer/libstdc++variants/cppunit",
                    ]
osx_args_min_10_9 = ["--with-macosx-version-min=10.9",
                     "--with-cppunit-prefix=/Developer/libc++variants/cppunit",
                    ]

add_master_builder(
    slave="csleobuild",
    port="wxOSX-Cocoa",
    factory=make_osx_factory,
    config_args=osx_args_min_10_9 + osx_args_64_only
)

add_master_builder(
    slave="csleobuild",
    port="wxOSX-Cocoa-10.7",
    factory=make_osx_factory,
    full=True,
    config_args=osx_args_min_10_7
)

add_master_builder(
    slave="csleobuild",
    port="wxiOS",
    factory=make_ios_factory,
    full=True,
)

add_wx30_builder(
    slave="csleobuild",
    port="wxOSX-Cocoa-10.5",
    factory=make_osx_factory,
    config_args=osx_args_min_10_5
)

add_wx30_builder(
    slave="csleobuild",
    port="wxOSX-Cocoa-10.5",
    full=True,
    factory=make_osx_factory,
    config_args=osx_args_min_10_5
)

add_wx30_builder(
    slave="csleobuild",
    port="wxOSX-Carbon",
    factory=make_osx_factory,
    config_args=["--with-osx_carbon"] + osx_args_min_10_5
)

# Set up MSW builds.
add_master_builder(
    slave="disc-xp",
    port="wxMSW-VC8",
    factory=make_nmake_factory,
    full=True,
)

add_wx30_builder(
    slave="disc-xp",
    port="wxMSW-VC8",
    full=True,
    factory=make_nmake_factory
)

####### SCHEDULERS

c["schedulers"] = []

from buildbot.schedulers.basic import SingleBranchScheduler
c["schedulers"].append(SingleBranchScheduler(
        name="quick_master",
        branch="master",
        treeStableTimer=2*60,
        builderNames=quick_builders["master"],
    ))

c["schedulers"].append(SingleBranchScheduler(
        name="quick_wx30",
        branch="WX_3_0_BRANCH",
        treeStableTimer=2*60,
        builderNames=quick_builders["WX_3_0_BRANCH"],
    ))

from buildbot.schedulers import timed
c["schedulers"].append(timed.Nightly(
        name="full_master",
        branch="master",
        builderNames=full_builders["master"],
        hour=3, minute=17,
        onlyIfChanged=True,
    ))

c["schedulers"].append(timed.Nightly(
        name="full_wx30",
        branch="WX_3_0_BRANCH",
        builderNames=full_builders["WX_3_0_BRANCH"],
        hour=5, minute=43,
        onlyIfChanged=True,
    ))

from buildbot.schedulers.forcesched import *
c["schedulers"].append(ForceScheduler(
        name="force",
        builderNames=quick_builders["master"] +
                     quick_builders["WX_3_0_BRANCH"] +
                     full_builders["master"] +
                     full_builders["WX_3_0_BRANCH"],
    ))

####### STATUS TARGETS

c["status"] = []

# Define public read-only web status endpoint that also supports a change hook
# used by Github to report the changes to the code.
from buildbot.status import html
c["status"].append(
        html.WebStatus(
            http_port=8010,
            allowForce=False,
            change_hook_dialects={
                "github": {
                    "strict": True,
                    "secret": github_secret,
                }
            },
        )
    )

# And also a protected (by HTTP authentication) endpoint allowing to force
# things.
c["status"].append(html.WebStatus(http_port=8011, allowForce=True))

# This allows to run "buildbot statuslog --master localhost:8012" to view
# buildbot activity on stdout.
from buildbot.status.client import PBListener
c["status"].append(PBListener(8012))

from buildbot.status import mail
c["status"].append(mail.MailNotifier(fromaddr="buildbot@wxwidgets.org",
                                     extraRecipients=[buildmaster_email],
                                     mode="problem"))

####### PROJECT IDENTITY

c["projectName"] = "wxWidgets"
c["projectURL"] = "http://www.wxwidgets.org"
c["buildbotURL"] = "http://buildbot.tt-solutions.com/wx/"