Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move ensure_rust_toolchain to buildall #83

Merged
merged 6 commits into from
Jan 15, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions pyodide_build/build_env.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,7 @@
from pyodide_build.recipe import load_all_recipes

RUST_BUILD_PRELUDE = """
rustup toolchain install ${RUST_TOOLCHAIN} && rustup default ${RUST_TOOLCHAIN}
rustup target add wasm32-unknown-emscripten --toolchain ${RUST_TOOLCHAIN}
rustup default ${RUST_TOOLCHAIN}
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I suppose we could also do this in _ensure_rust_toolchain() and remove RUST_BUILD_PRELUDE entirely.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Then , will pyodide build-recipes-no-deps work?

"""


Expand Down
31 changes: 31 additions & 0 deletions pyodide_build/buildall.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
from pyodide_build.build_env import BuildArgs
from pyodide_build.buildpkg import needs_rebuild
from pyodide_build.common import (
exit_with_stdio,
extract_wheel_metadata_file,
find_matching_wheels,
find_missing_executables,
Expand Down Expand Up @@ -158,6 +159,8 @@ def build(self, build_args: BuildArgs, build_dir: Path) -> None:
# been updated and should be rebuilt even though its own
# files haven't been updated.
"--force-rebuild",
# We already did the rust setup in buildall
"--skip-rust-setup",
],
check=False,
stdout=subprocess.DEVNULL,
Expand Down Expand Up @@ -645,6 +648,28 @@ def run(self, n_jobs: int, already_built: set[str]) -> None:
self.build_queue.put((job_priority(dependent), dependent))


def _ensure_rust_toolchain():
rust_toolchain = build_env.get_build_flag("RUST_TOOLCHAIN")
result = subprocess.run(
["rustup", "toolchain", "install", rust_toolchain], check=False
)
if result.returncode == 0:
result = subprocess.run(
[
"rustup",
"target",
"add",
"wasm32-unknown-emscripten",
"--toolchain",
rust_toolchain,
],
check=False,
)
if result.returncode != 0:
logger.error("ERROR: rustup toolchain install failed")
exit_with_stdio(result)


def build_from_graph(
pkg_map: dict[str, BasePackage],
build_args: BuildArgs,
Expand Down Expand Up @@ -686,6 +711,12 @@ def build_from_graph(
for pkg_name in needs_build:
pkg_map[pkg_name].unbuilt_host_dependencies.difference_update(already_built)

needs_rust = any(
pkg_map[pkg_name].meta.is_rust_package() for pkg_name in needs_build
)
if needs_rust:
_ensure_rust_toolchain()

if already_built:
logger.info(
"The following packages are already built: [bold]%s[/bold]",
Expand Down
20 changes: 19 additions & 1 deletion pyodide_build/cli/build_recipes.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import typer

from pyodide_build import build_env, buildall
from pyodide_build import build_env, buildall, recipe
from pyodide_build.build_env import BuildArgs, init_environment
from pyodide_build.buildpkg import RecipeBuilder
from pyodide_build.common import get_num_cores
Expand All @@ -17,6 +17,7 @@ class Args:
build_dir: Path
install_dir: Path
build_args: BuildArgs
skip_rust_setup: bool
force_rebuild: bool
n_jobs: int

Expand All @@ -28,6 +29,7 @@ def __init__(
install_dir: Path | str | None = None,
build_args: BuildArgs,
force_rebuild: bool,
skip_rust_setup: bool = False,
n_jobs: int | None = None,
):
cwd = Path.cwd()
Expand All @@ -41,6 +43,7 @@ def __init__(
)
self.build_args = build_args
self.force_rebuild = force_rebuild
self.skip_rust_setup = skip_rust_setup
self.n_jobs = n_jobs or get_num_cores()
if not self.recipe_dir.is_dir():
raise FileNotFoundError(f"Recipe directory {self.recipe_dir} not found")
Expand Down Expand Up @@ -93,6 +96,11 @@ def build_recipes_no_deps(
"--continue",
help="Continue a build from the middle. For debugging. Implies '--force-rebuild'",
),
skip_rust_setup: bool = typer.Option(
False,
"--skip-rust-setup",
help="Don't setup rust environment when building a rust package",
),
) -> None:
"""Build packages using yaml recipes but don't try to resolve dependencies"""
init_environment()
Expand All @@ -113,15 +121,25 @@ def build_recipes_no_deps(
build_dir=build_dir,
recipe_dir=recipe_dir,
force_rebuild=force_rebuild,
skip_rust_setup=skip_rust_setup,
)

return build_recipes_no_deps_impl(packages, args, continue_)


def _rust_setup(recipe_dir: Path, packages: list[str]):
recipes = recipe.load_recipes(recipe_dir, packages, False)
if any(recipe.is_rust_package() for recipe in recipes.values()):
buildall._ensure_rust_toolchain()


def build_recipes_no_deps_impl(
packages: list[str], args: Args, continue_: bool
) -> None:
# TODO: use multiprocessing?
if not args.skip_rust_setup:
_rust_setup(args.recipe_dir, packages)

for package in packages:
package_path = args.recipe_dir / package
package_build_dir = args.build_dir / package / "build"
Expand Down
Loading