Fixes for travic CI build (facebookarchive#3)

Summary:
This should fix the Travis CI builds. It adds rust toolchain support inside docker and sets the required THRIFT env variable.
Pull Request resolved: facebookexperimental/rust-shed#3

Reviewed By: krallin

Differential Revision: D18905608

Pulled By: lukaspiatkowski

fbshipit-source-id: 5db1eff6f215a6617d8acaa0c99a62d45225956b

build/fbcode_builder/docker_builder.py

@@ -42,16 +42,20 @@ def setup(self):
         # To allow exercising non-root installs -- we change users after the
         # system packages are installed.  TODO: For users not defined in the
         # image, we should probably `useradd`.
-        return self.step('Setup', [
-            # Docker's FROM does not understand shell quoting.
-            ShellQuoted('FROM {}'.format(self.option('os_image'))),
-            # /bin/sh syntax is a pain
-            ShellQuoted('SHELL ["/bin/bash", "-c"]'),
-        ]
-            + self.install_debian_deps() + [self._change_user()]
-            + [self.workdir(self.option('prefix'))]
+        return self.step(
+            "Setup",
+            [
+                # Docker's FROM does not understand shell quoting.
+                ShellQuoted("FROM {}".format(self.option("os_image"))),
+                # /bin/sh syntax is a pain
+                ShellQuoted('SHELL ["/bin/bash", "-c"]'),
+            ]
+            + self.install_debian_deps()
+            + [self._change_user()]
+            + [self.workdir(self.option("prefix"))]
             + self.create_python_venv()
             + self.python_venv()
+            + self.rust_toolchain(),
         )
 
     def python_venv(self):
@@ -71,6 +75,9 @@ def step(self, name, actions):
     def run(self, shell_cmd):
         return ShellQuoted('RUN {cmd}').format(cmd=shell_cmd)
 
+    def set_env(self, key, value):
+        return ShellQuoted("ENV {key}={val}").format(key=key, val=value)
+
     def workdir(self, dir):
         return [
             # As late as Docker 1.12.5, this results in `build` being owned

build/fbcode_builder/facebook_legocastle_builder.py

@@ -44,6 +44,7 @@ def __init__(self, message):
 
 LegocastleFBCheckout = namedtuple('LegocastleFBCheckout', ['project_and_path'])
 LegocastleWorkdir = namedtuple('LegocastleWorkdir', ['dir'])
+LegocastleSetEnv = namedtuple("LegocastleSetEnv", ["key", "value"])
 
 
 class LegocastleStep(namedtuple('LegocastleStepBase', ('name', 'actions'))):
@@ -60,9 +61,13 @@ def __new__(cls, name, actions):
 class LegocastleFBCodeBuilder(FBCodeBuilder):
 
     def setup(self):
-        return self.step('Setup',
-            self.create_python_venv() + [
-            self.run(ShellQuoted("""
+        return self.step(
+            "Setup",
+            self.create_python_venv()
+            + [
+                self.run(
+                    ShellQuoted(
+                        """
 case "$OSTYPE" in
   darwin*)
     http_proxy= https_proxy= ./tools/lfs/lfs.py \\
@@ -72,17 +77,44 @@ def setup(self):
     tar xzf homebrew.tar.gz -C /var/tmp
   ;;
 esac
-"""))])
+"""
+                    )
+                )
+            ]
+            + self.rust_toolchain(),
+        )
 
     def step(self, name, actions):
         return LegocastleStep(name=name, actions=actions)
 
     def run(self, shell_cmd):
         return shell_cmd
 
+    def set_env(self, key, value):
+        return LegocastleSetEnv(key, value)
+
     def workdir(self, dir):
         return LegocastleWorkdir(dir)
 
+    def rust_toolchain(self):
+        actions = []
+        if self.option("rust_toolchain", False):
+            (toolchain, is_bootstrap) = self.option("rust_toolchain")
+            assert toolchain == "stable", (
+                "Only stable toolchain is supported in legocastle,"
+                + " but {0} provided"
+            ).format(toolchain)
+            actions = [
+                self.set_env(
+                    "PATH", ShellQuoted("$BOX_DIR/xplat/rust/toolchain/current:$PATH")
+                ),
+                self.set_env("RUSTC_BOOTSTRAP", "1" if is_bootstrap else "0"),
+                self.run(ShellQuoted("rustc --version")),
+                self.run(ShellQuoted("rustdoc --version")),
+                self.run(ShellQuoted("cargo --version")),
+            ]
+        return actions
+
     def fb_github_project_workdir(self, project_and_path, github_org='facebook'):
         return LegocastleFBCheckout(project_and_path)
 
@@ -106,6 +138,7 @@ def _render_impl(self, actions):
         #
         # If we just replayed the last workdir('baz'), we would not end up
         # in '<github_prefix>/foo/bar/baz', but rather in `<unknown>/baz`.
+        next_set_env = []
         next_step_workdirs = []
         shipit_projects = []
         build_steps = []
@@ -124,6 +157,8 @@ def _render_impl(self, actions):
                 shipit_projects.append(
                     action.project_and_path.split('/', 1)[0]
                 )
+            elif isinstance(action, LegocastleSetEnv):
+                next_set_env.append(action)
             elif isinstance(action, LegocastleStep):
                 pre_actions = [
                     ShellQuoted('set -ex')
@@ -157,7 +192,10 @@ def _render_impl(self, actions):
                 )
                 pre_actions.extend(self.workdir(w) for w in next_step_workdirs)
 
-                shell_steps = []
+                shell_steps = [
+                    ShellQuoted("export {k}={v}").format(k=a.key, v=a.value)
+                    for a in next_set_env
+                ]
                 for a in itertools.chain(pre_actions, action.actions):
                     if isinstance(a, LegocastleWorkdir):
                         next_step_workdirs.append(a.dir)
@@ -167,6 +205,11 @@ def _render_impl(self, actions):
                                 '_wd={d} ; mkdir -p "$_wd" && cd "$_wd"'
                             ).format(d=a.dir)
                         )
+                    elif isinstance(a, LegocastleSetEnv):
+                        next_set_env.append(a)
+                        shell_steps.append(
+                            ShellQuoted("export {k}={v}").format(k=a.key, v=a.value)
+                        )
                     else:
                         shell_steps.append(a)
 

build/fbcode_builder/fbcode_builder.py

@@ -172,6 +172,10 @@ def run(self, shell_cmd):
         'Run this bash command'
         raise NotImplementedError
 
+    def set_env(self, key, value):
+        'Set the environment "key" to value "value"'
+        raise NotImplementedError
+
     def workdir(self, dir):
         'Create this directory if it does not exist, and change into it'
         raise NotImplementedError
@@ -276,6 +280,52 @@ def python_venv(self):
                         self.python_deps())))))
         return(actions)
 
+    def enable_rust_toolchain(self, toolchain="stable", is_bootstrap=True):
+        choices = set(["stable", "beta", "nightly"])
+
+        assert toolchain in choices, (
+            "while enabling rust toolchain: {} is not in {}"
+        ).format(toolchain, choices)
+
+        rust_toolchain_opt = (toolchain, is_bootstrap)
+        prev_opt = self.option("rust_toolchain", rust_toolchain_opt)
+        assert prev_opt == rust_toolchain_opt, (
+            "while enabling rust toolchain: previous toolchain already set to"
+            " {}, but trying to set it to {} now"
+        ).format(prev_opt, rust_toolchain_opt)
+
+        self.add_option("rust_toolchain", rust_toolchain_opt)
+
+    def rust_toolchain(self):
+        actions = []
+        if self.option("rust_toolchain", False):
+            (toolchain, is_bootstrap) = self.option("rust_toolchain")
+            rust_dir = path_join(self.option("prefix"), "rust")
+            actions = [
+                self.set_env("CARGO_HOME", rust_dir),
+                self.set_env("RUSTUP_HOME", rust_dir),
+                self.set_env("RUSTC_BOOTSTRAP", "1" if is_bootstrap else "0"),
+                self.run(
+                    ShellQuoted(
+                        "curl -sSf https://build.travis-ci.com/files/rustup-init.sh"
+                        " | sh -s --"
+                        "   --default-toolchain={r} "
+                        "   --profile=minimal"
+                        "   --no-modify-path"
+                        "   -y"
+                    ).format(p=rust_dir, r=toolchain)
+                ),
+                self.set_env(
+                    "PATH",
+                    ShellQuoted("{p}:$PATH").format(p=path_join(rust_dir, "bin")),
+                ),
+                self.run(ShellQuoted("rustup update")),
+                self.run(ShellQuoted("rustc --version")),
+                self.run(ShellQuoted("rustup --version")),
+                self.run(ShellQuoted("cargo --version")),
+            ]
+        return actions
+
     def debian_ccache_setup_steps(self):
         return []  # It's ok to ship a renderer without ccache support.
 
@@ -388,6 +438,18 @@ def cmake_install(self, name, cmake_path='..'):
             self.cmake_configure(name, cmake_path) + self.make_and_install()
         )
 
+    def cargo_build(self, name):
+        return self.step(
+            "Build {0}".format(name),
+            [
+                self.run(
+                    ShellQuoted("cargo build -j {n}").format(
+                        n=self.option("make_parallelism")
+                    )
+                )
+            ],
+        )
+
     def fb_github_autoconf_install(self, project_and_path, github_org='facebook'):
         return [
             self.fb_github_project_workdir(project_and_path, github_org),
@@ -399,3 +461,9 @@ def fb_github_cmake_install(self, project_and_path, cmake_path='..', github_org=
             self.fb_github_project_workdir(project_and_path, github_org),
             self.cmake_install(project_and_path, cmake_path),
         ]
+
+    def fb_github_cargo_build(self, project_and_path, github_org="facebook"):
+        return [
+            self.fb_github_project_workdir(project_and_path, github_org),
+            self.cargo_build(project_and_path),
+        ]

build/fbcode_builder/shell_builder.py

@@ -33,6 +33,9 @@ class ShellFBCodeBuilder(FBCodeBuilder):
     def _render_impl(self, steps):
         return raw_shell(shell_join('\n', recursively_flatten_list(steps)))
 
+    def set_env(self, key, value):
+        return ShellQuoted("export {key}={val}").format(key=key, val=value)
+
     def workdir(self, dir):
         return [
             ShellQuoted('mkdir -p {d} && cd {d}').format(

build/fbcode_builder/specs/rust_shed.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+# Copyright (c) Facebook, Inc. and its affiliates.
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from shell_quoting import path_join
+import specs.fbthrift as fbthrift
+
+
+def fbcode_builder_spec(builder):
+    builder.enable_rust_toolchain()
+    return {
+        "depends_on": [fbthrift],
+        "steps": [
+            builder.set_env(
+                "THRIFT", path_join(builder.option("prefix"), "bin", "thrift1")
+            ),
+            builder.fb_github_cargo_build(
+                "rust-shed/", github_org="facebookexperimental"
+            ),
+        ],
+    }