Skip to content

Commit

Permalink
Uses a pull step to install Python dependencies (#57)
Browse files Browse the repository at this point in the history
If we use the `pip_packages` job variable, that will end up being passed
as `EXTRA_PIP_PACKAGES`, which are installed much earlier in the Docker
entrypoint rather than in Python after we have logging established.  By
moving these to a pull step, we will be able to get better visibility
into failures with the installation.

Part of ENG-1050
  • Loading branch information
chrisguidry authored Feb 20, 2025
1 parent 0f138cc commit 9bedd5e
Show file tree
Hide file tree
Showing 2 changed files with 75 additions and 2 deletions.
14 changes: 12 additions & 2 deletions src/prefect_cloud/cli/root.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,6 @@ async def deploy(
task = progress.add_task("Inspecting code...", total=None)

# Pre-process CLI arguments
pip_packages = get_dependencies(dependencies)
env_vars = process_key_value_pairs(env, progress=progress)

# Get repository info and file contents
Expand Down Expand Up @@ -155,6 +154,18 @@ async def deploy(
progress.update(task, description="Deploying code...")

pull_steps = [github_ref.to_pull_step(credentials_name)]
if dependencies:
quoted_dependencies = [
f"'{dependency}'" for dependency in get_dependencies(dependencies)
]
pull_steps.append(
{
"prefect.deployments.steps.run_shell_script": {
"directory": "{{ git-clone.directory }}",
"script": f"uv pip install {' '.join(quoted_dependencies)}",
}
}
)
if with_requirements:
pull_steps.append(
{
Expand All @@ -174,7 +185,6 @@ async def deploy(
pull_steps=pull_steps,
parameter_schema=parameter_schema,
job_variables={
"pip_packages": pip_packages,
"env": {"PREFECT_CLOUD_API_URL": api_url} | env_vars,
},
)
Expand Down
63 changes: 63 additions & 0 deletions tests/test_cli/test_root.py
Original file line number Diff line number Diff line change
Expand Up @@ -491,6 +491,69 @@ def test_run():
)


def test_deploy_with_dependencies():
"""Test deployment with python dependencies"""
with patch("prefect_cloud.auth.get_prefect_cloud_client") as mock_client:
client = AsyncMock()
mock_client.return_value.__aenter__.return_value = client

client.ensure_managed_work_pool = AsyncMock(
return_value=WorkPool(
type="prefect:managed", name="test-pool", is_paused=False
)
)
client.create_managed_deployment = AsyncMock(return_value="test-deployment-id")

with patch("prefect_cloud.cli.root.auth.get_cloud_urls_or_login") as mock_urls:
mock_urls.return_value = ("https://ui.url", "https://api.url", "test-key")

with patch(
"prefect_cloud.github.GitHubRepo.get_file_contents"
) as mock_content:
mock_content.return_value = textwrap.dedent("""
def test_function():
pass
""").lstrip()

invoke_and_assert(
command=[
"deploy",
"test.py:test_function",
"--from",
"github.com/owner/repo",
"--with",
"requests>=2",
"--with",
"pandas>=1",
],
expected_code=0,
expected_output_contains=[
"Deployed test_function! 🎉",
"Run: prefect-cloud run test_function/test_function",
"Schedule: prefect-cloud schedule test_function/test_function <SCHEDULE>",
"View: https://ui.url/deployments/deployment/test-deployment-id",
],
)

# Verify deployment was created with correct pull steps
client.create_managed_deployment.assert_called_once()
deployment = client.create_managed_deployment.call_args[1]

# We don't want to pass them as EXTRA_PIP_PACKAGES, because that happens
# before logging can be set up, so any problems with the dependencies
# will be obscured to users.
assert "pip_packages" not in deployment["job_variables"]

pull_steps = deployment["pull_steps"]
assert len(pull_steps) == 2
assert pull_steps[1] == {
"prefect.deployments.steps.run_shell_script": {
"directory": "{{ git-clone.directory }}",
"script": "uv pip install 'requests>=2' 'pandas>=1'",
}
}


def test_deploy_with_requirements_file():
"""Test deployment with a requirements file"""
with patch("prefect_cloud.auth.get_prefect_cloud_client") as mock_client:
Expand Down

0 comments on commit 9bedd5e

Please sign in to comment.