Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add skip_failed flag #121

Closed
wants to merge 2 commits into from
Closed

add skip_failed flag #121

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e046fb4
add skip_failed flag
FoamyGuy Sep 23, 2024
9bceb91
use skip_failed flag for community bundle task
FoamyGuy Sep 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ jobs:
# Use the community bundle because it's smaller and faster
git clone --recurse-submodules https://github.com/adafruit/CircuitPython_Community_Bundle.git
cd CircuitPython_Community_Bundle
circuitpython-build-bundles --filename_prefix test-bundle --library_location libraries --library_depth 2
circuitpython-build-bundles --filename_prefix test-bundle --library_location libraries --library_depth 2 --skip_failed
Copy link
Member

@jepler jepler Sep 24, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I added more notes to the related issue but I want to note here that I'm uneasy about adding skip_failed during the test of circuitpython-build-tools: If a bug is introduced into the code here, --skip_failed might cover it up.

Instead, consider using the communty bundle at a specific ref/tag that is known to build properly, and remove --skip_failed.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What do you think about having this task use the main Adafruit Bundle instead of the community bundle?

The comment here notes that community bundle was chosen because it's smaller and thus faster. It is still true that the community bundle is smaller, but it's also gaining ground at ~158 libraries right now. I think the main bundle is ~340ish but am not certain. I imagine that the disparity was greater when this task was initially set up.

I am not so sure that the increased speed is that big of a benefit. As I understand it this is always run automatically inside of github actions, there isn't typically a person sitting and waiting for it to complete.

In my mind pointing it to the main Adafruit Bundle would also be nice because it will give another chance to discover ASAP if any changes to the build-tool could cause any issues for the Adafruit libraries or bundle.

- name: Build Python package
run: |
pip install --upgrade setuptools wheel twine readme_renderer testresources
Expand Down
30 changes: 20 additions & 10 deletions circuitpython_build_tools/scripts/build_bundles.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ def get_bundle_requirements(directory, package_list):
pypi_reqs.add(original_name)
return sorted(dependencies), sorted(pypi_reqs)

def build_bundle_json(libs, bundle_version, output_filename, package_folder_prefix, remote_name="origin"):
def build_bundle_json(libs, bundle_version, output_filename, package_folder_prefix, remote_name="origin", skip_failed=False):
"""
Generate a JSON file of all the libraries in libs
"""
Expand All @@ -124,7 +124,14 @@ def build_bundle_json(libs, bundle_version, output_filename, package_folder_pref
# otherwise it's just shuffling info around
for library_path in libs:
package = {}
package_info = build.get_package_info(library_path, package_folder_prefix)
try:
package_info = build.get_package_info(library_path, package_folder_prefix)
except ValueError as e:
if not skip_failed:
raise e
else:
continue

module_name, repo = get_module_name(library_path, remote_name)
if package_info["module_name"] is not None:
package["module_name"] = package_info["module_name"]
Expand Down Expand Up @@ -152,7 +159,7 @@ def build_bundle_json(libs, bundle_version, output_filename, package_folder_pref
out_file.close()

def build_bundle(libs, bundle_version, output_filename, package_folder_prefix,
build_tools_version="devel", mpy_cross=None, example_bundle=False, remote_name="origin"):
build_tools_version="devel", mpy_cross=None, example_bundle=False, remote_name="origin", skip_failed=False):
build_dir = "build-" + os.path.basename(output_filename)
top_folder = os.path.basename(output_filename).replace(".zip", "")
build_lib_dir = os.path.join(build_dir, top_folder, "lib")
Expand All @@ -177,7 +184,8 @@ def build_bundle(libs, bundle_version, output_filename, package_folder_prefix,
except ValueError as e:
print("build.library failure:", library_path)
print(e)
success = False
if not skip_failed:
success = False

print()
print("Generating VERSIONS")
Expand All @@ -188,7 +196,8 @@ def build_bundle(libs, bundle_version, output_filename, package_folder_prefix,
if versions.returncode != 0:
print("Failed to generate versions file. Its likely a library hasn't been "
"released yet.")
success = False
if not skip_failed:
success = False

repo = None
for line in versions.stdout.split(b"\n"):
Expand Down Expand Up @@ -243,9 +252,10 @@ def _find_libraries(current_path, depth):
@click.option('--library_depth', default=0, help="Depth of library folders. This is useful when multiple libraries are bundled together but are initially in separate subfolders.")
@click.option('--package_folder_prefix', default="adafruit_", help="Prefix string used to determine package folders to bundle.")
@click.option('--remote_name', default="origin", help="Git remote name to use during building")
@click.option('--skip_failed', default=False, is_flag=True, help="Skip over any libraries that fail to build, and continue on with the rest of the bundle.")
@click.option('--ignore', "-i", multiple=True, type=click.Choice(all_modules), help="Bundles to ignore building")
@click.option('--only', "-o", multiple=True, type=click.Choice(all_modules), help="Bundles to build building")
def build_bundles(filename_prefix, output_directory, library_location, library_depth, package_folder_prefix, remote_name, ignore, only):
def build_bundles(filename_prefix, output_directory, library_location, library_depth, package_folder_prefix, remote_name, skip_failed, ignore, only):
os.makedirs(output_directory, exist_ok=True)

package_folder_prefix = package_folder_prefix.split(", ")
Expand Down Expand Up @@ -276,7 +286,7 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d
filename_prefix + '-py-{VERSION}.zip'.format(
VERSION=bundle_version))
build_bundle(libs, bundle_version, zip_filename, package_folder_prefix,
build_tools_version=build_tools_version, remote_name=remote_name)
build_tools_version=build_tools_version, remote_name=remote_name, skip_failed=skip_failed)

# Build .mpy bundle(s)
if "mpy" not in ignore:
Expand All @@ -289,19 +299,19 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d
TAG=version["name"],
VERSION=bundle_version))
build_bundle(libs, bundle_version, zip_filename, package_folder_prefix,
mpy_cross=mpy_cross, build_tools_version=build_tools_version, remote_name=remote_name)
mpy_cross=mpy_cross, build_tools_version=build_tools_version, remote_name=remote_name, skip_failed=skip_failed)

# Build example bundle
if "example" not in ignore:
zip_filename = os.path.join(output_directory,
filename_prefix + '-examples-{VERSION}.zip'.format(
VERSION=bundle_version))
build_bundle(libs, bundle_version, zip_filename, package_folder_prefix,
build_tools_version=build_tools_version, example_bundle=True, remote_name=remote_name)
build_tools_version=build_tools_version, example_bundle=True, remote_name=remote_name, skip_failed=skip_failed)

# Build Bundle JSON
if "json" not in ignore:
json_filename = os.path.join(output_directory,
filename_prefix + '-{VERSION}.json'.format(
VERSION=bundle_version))
build_bundle_json(libs, bundle_version, json_filename, package_folder_prefix, remote_name=remote_name)
build_bundle_json(libs, bundle_version, json_filename, package_folder_prefix, remote_name=remote_name, skip_failed=skip_failed)
Loading