diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index b6c2196..4094c8d 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -4,10 +4,10 @@ This code is licensed under the MPL 2.0 [LICENSE](LICENSE). # Contributing -When contributing to Singularity Registry Global Client, it is -important to properly communicate the gist of the contribution. -If it is a simple code or editorial fix, simply explaining this -within the GitHub Pull Request (PR) will suffice. But if this is a larger +When contributing to Singularity Registry Global Client, it is +important to properly communicate the gist of the contribution. +If it is a simple code or editorial fix, simply explaining this +within the GitHub Pull Request (PR) will suffice. But if this is a larger fix or Enhancement, it should be first discussed with the project leader or developers. @@ -29,8 +29,8 @@ all your interactions with the project members and users. 4. The project's default copyright and header have been included in any new source files. 5. All (major) changes to Singularity Registry must be documented in - [docs](docs). If your PR changes a core functionality, please - include clear description of the changes in your PR so that the docs + [docs](docs). If your PR changes a core functionality, please + include clear description of the changes in your PR so that the docs can be updated, or better, submit another PR to update the docs directly. 6. If necessary, update the README.md. 7. The pull request will be reviewed by others, and the final merge must be @@ -102,7 +102,7 @@ an incident. Further details of specific enforcement policies may be posted separately. Project maintainers, contributors and users who do not follow or enforce the -Code of Conduct in good faith may face temporary or permanent repercussions +Code of Conduct in good faith may face temporary or permanent repercussions with their involvement in the project as determined by the project's leader(s). ## Attribution diff --git a/.github/dev-requirements.txt b/.github/dev-requirements.txt new file mode 100644 index 0000000..6116de3 --- /dev/null +++ b/.github/dev-requirements.txt @@ -0,0 +1,4 @@ +pre-commit +black==23.3.0 +isort +flake8 diff --git a/.github/workflows/generate.yaml b/.github/workflows/generate.yaml index c652fb4..2b3e8e0 100644 --- a/.github/workflows/generate.yaml +++ b/.github/workflows/generate.yaml @@ -36,7 +36,7 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false - matrix: + matrix: image: ["ubuntu", "centos", "rockylinux:9.0", "alpine", "busybox"] name: Generate Matrix @@ -62,12 +62,12 @@ jobs: outfile: ${{ matrix.image }}.json - name: View Output run: cat ${{ matrix.image }}.json - + test-diffs: runs-on: ubuntu-latest strategy: fail-fast: false - matrix: + matrix: image: [["vanessa/salad", "vanessa-salad.json"]] name: Generate Matrix diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c6fa2ba..d30cf78 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,19 +21,9 @@ jobs: with: files: ./docs/getting_started/ ./docs/index.rst - - name: Lint python code + - name: Lint and format Python code run: | export PATH="/usr/share/miniconda/bin:$PATH" source activate black - pip install black - black --check container_guts - - - name: Check imports with pyflakes - run: | - export PATH="/usr/share/miniconda/bin:$PATH" - source activate black - pyflakes container_guts/*.py - pyflakes container_guts/client - pyflakes container_guts/main/client.py - pyflakes container_guts/main/templates.py - pyflakes container_guts/main/container/docker.py + pip install -r .github/dev-requirements.txt + pre-commit run --all-files diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..135ea85 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,25 @@ +exclude: ".all-contributorsrc" +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-docstring-first + - id: end-of-file-fixer + - id: trailing-whitespace + - id: mixed-line-ending + + - repo: local + hooks: + - id: black + name: black + language: python + types: [python] + entry: black + + - id: flake8 + name: flake8 + language: python + types: [python] + entry: flake8 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b1f451..6aecebd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,10 +14,10 @@ and **Merged pull requests**. Critical items to know are: The versions coincide with releases on pip. Only major versions will be released as tags on Github. ## [0.0.x](https://github.com/singularityhub/guts/tree/main) (0.0.x) + - add function for most similar (0.0.16) - skip /dev in tar (0.0.15) - diff action and support (0.0.14) - adding fs (filesystem) extraction support (0.0.13) - tag should be own file (0.0.12) - Support for output directory (so path prepared by guts) (0.0.11) - Initial creation of project (0.0.1) - diff --git a/action/diff/action.yaml b/action/diff/action.yaml index 91baa0c..152697b 100644 --- a/action/diff/action.yaml +++ b/action/diff/action.yaml @@ -29,11 +29,11 @@ runs: - name: Guts Diff for ${{ inputs.image }} id: guts_run env: - image: ${{ inputs.image }} - outfile: ${{ inputs.outfile }} - outdir: ${{ inputs.outdir }} - database: ${{ inputs.database }} - run: | + image: ${{ inputs.image }} + outfile: ${{ inputs.outfile }} + outdir: ${{ inputs.outdir }} + database: ${{ inputs.database }} + run: | cmd="guts diff" if [ "${outfile}" != "" ]; then cmd="${cmd} --outfile ${outfile}" @@ -44,7 +44,7 @@ runs: if [ "${database}" != "" ]; then cmd="${cmd} --db ${database}" fi - cmd="${cmd} ${image}" + cmd="${cmd} ${image}" printf "${cmd}\n" ${cmd} shell: bash diff --git a/action/manifest/action.yaml b/action/manifest/action.yaml index a9d2f66..5ace1ee 100644 --- a/action/manifest/action.yaml +++ b/action/manifest/action.yaml @@ -30,11 +30,11 @@ runs: - name: Guts for ${{ inputs.image }} id: guts_run env: - image: ${{ inputs.image }} - outfile: ${{ inputs.outfile }} - outdir: ${{ inputs.outdir }} + image: ${{ inputs.image }} + outfile: ${{ inputs.outfile }} + outdir: ${{ inputs.outdir }} includes: ${{ inputs.include }} - run: | + run: | cmd="guts manifest" if [ "${outfile}" != "" ]; then cmd="${cmd} --outfile ${outfile}" @@ -45,7 +45,7 @@ runs: for include in ${includes}; do cmd="${cmd} --include ${include}" done - cmd="${cmd} ${image}" + cmd="${cmd} ${image}" printf "${cmd}\n" ${cmd} shell: bash diff --git a/container_guts/client/__init__.py b/container_guts/client/__init__.py index 54445a2..dbd5e86 100644 --- a/container_guts/client/__init__.py +++ b/container_guts/client/__init__.py @@ -1,7 +1,7 @@ #!/usr/bin/env python __author__ = "Vanessa Sochat" -__copyright__ = "Copyright 2021-2022, Vanessa Sochat" +__copyright__ = "Copyright 2021-2024, Vanessa Sochat" __license__ = "MPL 2.0" import argparse @@ -69,8 +69,20 @@ def get_parser(): help="Database root (of json files) to use, either filesystem or git URL to clone", dest="database", ) + similar = subparsers.add_parser( + "similar", + description="calculate similarity of your container against a guts database.", + formatter_class=argparse.RawTextHelpFormatter, + ) + for command in [diff, similar]: + command.add_argument( + "--db", + "--database", + help="Database root (of json files) to use, either filesystem or git URL to clone", + dest="database", + ) - for command in manifest, diff: + for command in manifest, diff, similar: command.add_argument( "-i", "--include", @@ -159,6 +171,8 @@ def help(return_code=0): from .manifest import main elif args.command == "diff": from .diff import main + elif args.command == "similar": + from .similar import main # Pass on to the correct parser return_code = 0 diff --git a/container_guts/client/diff.py b/container_guts/client/diff.py index 77472e0..dc76bd9 100644 --- a/container_guts/client/diff.py +++ b/container_guts/client/diff.py @@ -1,16 +1,16 @@ __author__ = "Vanessa Sochat" -__copyright__ = "Copyright 2022, Vanessa Sochat" +__copyright__ = "Copyright 2022-2024, Vanessa Sochat" __license__ = "MPL 2.0" import json import os import container_guts.utils as utils + from ..main import ManifestGenerator def main(args, parser, extra, subparser): - # Show args to the user print(" image: %s" % args.image) print(" outfile: %s" % args.outfile) diff --git a/container_guts/client/manifest.py b/container_guts/client/manifest.py index be4251d..c6ca94c 100644 --- a/container_guts/client/manifest.py +++ b/container_guts/client/manifest.py @@ -1,16 +1,16 @@ __author__ = "Vanessa Sochat" -__copyright__ = "Copyright 2022, Vanessa Sochat" +__copyright__ = "Copyright 2022-2024, Vanessa Sochat" __license__ = "MPL 2.0" import json import os import container_guts.utils as utils + from ..main import ManifestGenerator def main(args, parser, extra, subparser): - # Show args to the user print(" image: %s" % args.image) print(" outfile: %s" % args.outfile) diff --git a/container_guts/client/similar.py b/container_guts/client/similar.py new file mode 100644 index 0000000..92bb798 --- /dev/null +++ b/container_guts/client/similar.py @@ -0,0 +1,41 @@ +__author__ = "Vanessa Sochat" +__copyright__ = "Copyright 2022-2024, Vanessa Sochat" +__license__ = "MPL 2.0" + +import json +import os + +import container_guts.utils as utils + +from ..main import ManifestGenerator + + +def main(args, parser, extra, subparser): + # Show args to the user + print(" image: %s" % args.image) + print(" outfile: %s" % args.outfile) + print(" outdir: %s" % args.outdir) + print("container tech: %s" % args.container_tech) + print(" database: %s" % args.database) + + # Derive an initial manifest + cli = ManifestGenerator(tech=args.container_tech) + manifests = cli.similar(args.image, database=args.database) + outfile = None + + # Default to using outfile first, then outdir if defined + if args.outfile: + outfile = args.outfile + elif args.outdir: + outfile = os.path.join(args.outdir, "%s.json" % cli.save_path(args.image)) + dirname = os.path.dirname(outfile) + if not os.path.exists(dirname): + os.makedirs(dirname) + + # If we have an output file, make sure to set step output + if outfile: + print(f"Saving to {outfile}...") + print(f"::set-output name=outfile::{outfile}") + utils.write_json(manifests, outfile) + else: + print(json.dumps(manifests, indent=4)) diff --git a/container_guts/defaults.py b/container_guts/defaults.py index f7e683c..d2c20bb 100644 --- a/container_guts/defaults.py +++ b/container_guts/defaults.py @@ -1,10 +1,11 @@ __author__ = "Vanessa Sochat" -__copyright__ = "Copyright 2021-2022, Vanessa Sochat" +__copyright__ = "Copyright 2021-2024, Vanessa Sochat" __license__ = "MPL 2.0" -import container_guts.utils as utils import os +import container_guts.utils as utils + # Default database for base image database = "https://github.com/singularityhub/shpc-guts" diff --git a/container_guts/logger.py b/container_guts/logger.py index 5e2a74a..84b7def 100644 --- a/container_guts/logger.py +++ b/container_guts/logger.py @@ -1,5 +1,5 @@ __author__ = "Vanessa Sochat" -__copyright__ = "Copyright 2021-2022, Vanessa Sochat" +__copyright__ = "Copyright 2021-2024, Vanessa Sochat" __license__ = "MPL 2.0" import inspect @@ -37,7 +37,6 @@ def add_prefix(msg, char=">>"): class ColorizingStreamHandler(_logging.StreamHandler): - BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) RESET_SEQ = LogColors.ENDC COLOR_SEQ = "\033[%dm" diff --git a/container_guts/main/client.py b/container_guts/main/client.py index 1d3f571..8fc4608 100644 --- a/container_guts/main/client.py +++ b/container_guts/main/client.py @@ -1,5 +1,5 @@ __author__ = "Vanessa Sochat" -__copyright__ = "Copyright 2021-2022, Vanessa Sochat" +__copyright__ = "Copyright 2021-2024, Vanessa Sochat" __license__ = "MPL 2.0" @@ -8,10 +8,9 @@ from .. import utils from ..logger import logger - -from .database import Database -from .container.decorator import ensure_container from .container.base import ContainerName +from .container.decorator import ensure_container +from .database import Database class ManifestGenerator: @@ -60,7 +59,7 @@ def diff(self, image, database=None): # Catch the error so we clean up the running container try: result = db.diff(self.manifests[image.uri]) - except: + except Exception: self.container.cleanup(image) return @@ -70,6 +69,28 @@ def diff(self, image, database=None): self.container.cleanup(image) return {image.uri: {"diff": result}} + @ensure_container + def similar(self, image, database=None): + """ + Generate a manifest for an image and similarity scores + """ + print(f"Generating similarity for {image}") + tmpdir = self.extract(image, cleanup=False, includes=["paths", "fs"]) + db = Database(database) + + # Catch the error so we clean up the running container + try: + result = db.similar(self.manifests[image.uri]) + except Exception: + self.container.cleanup(image) + return + + # Only cleans up if was cloned + db.cleanup() + shutil.rmtree(tmpdir, ignore_errors=True) + self.container.cleanup(image) + return {image.uri: {"similar": result}} + def extract_filesystem(self, root): """ List all contents of the filesystem @@ -181,7 +202,6 @@ def get_manifests(self, root): for jsonfile in utils.recursive_find(root, "json$"): data = utils.read_json(jsonfile) if "manifest" in jsonfile: - continue print("Found layer config %s" % jsonfile) diff --git a/container_guts/main/container/base.py b/container_guts/main/container/base.py index 549541e..78b9c9d 100644 --- a/container_guts/main/container/base.py +++ b/container_guts/main/container/base.py @@ -1,11 +1,11 @@ __author__ = "Vanessa Sochat" -__copyright__ = "Copyright 2021-2022, Vanessa Sochat" +__copyright__ = "Copyright 2021-2024, Vanessa Sochat" __license__ = "MPL 2.0" import os import re - import shutil + import container_guts.main.templates as templates import container_guts.utils as utils from container_guts.logger import logger diff --git a/container_guts/main/container/decorator.py b/container_guts/main/container/decorator.py index 122563f..cb0957d 100644 --- a/container_guts/main/container/decorator.py +++ b/container_guts/main/container/decorator.py @@ -1,6 +1,6 @@ __author__ = "Vanessa Sochat" -__copyright__ = "Copyright The ORAS Authors." -__license__ = "Apache-2.0" +__copyright__ = "Copyright 2021-2024, Vanessa Sochat" +__license__ = "MPL 2.0" from functools import partial, update_wrapper diff --git a/container_guts/main/container/docker.py b/container_guts/main/container/docker.py index dc8a61f..c066a33 100644 --- a/container_guts/main/container/docker.py +++ b/container_guts/main/container/docker.py @@ -1,5 +1,5 @@ __author__ = "Vanessa Sochat" -__copyright__ = "Copyright 2022, Vanessa Sochat" +__copyright__ = "Copyright 2022-2024, Vanessa Sochat" __license__ = "MPL 2.0" @@ -8,9 +8,9 @@ import sys import container_guts.utils as utils -from .decorator import ensure_container -from .base import ContainerTechnology, ContainerName +from .base import ContainerName, ContainerTechnology +from .decorator import ensure_container class DockerContainer(ContainerTechnology): @@ -93,7 +93,7 @@ def export(self, image, tmpdir=None, cleanup=True): try: self.call(["tar", "--ignore-failed-read", "-xf", export, "-C", export_dir]) self.call(["tar", "--ignore-failed-read", "-xf", save, "-C", save_dir]) - except: + except Exception: self.call(["tar", "-xf", export, "-C", export_dir]) self.call(["tar", "-xf", save, "-C", save_dir]) diff --git a/container_guts/main/database.py b/container_guts/main/database.py index e1477d8..0d52cd0 100644 --- a/container_guts/main/database.py +++ b/container_guts/main/database.py @@ -1,5 +1,5 @@ __author__ = "Vanessa Sochat" -__copyright__ = "Copyright 2022, Vanessa Sochat" +__copyright__ = "Copyright 2024, Vanessa Sochat" __license__ = "MPL 2.0" @@ -8,6 +8,7 @@ import subprocess as sp import container_guts.utils as utils + from ..defaults import database as default_database @@ -46,6 +47,40 @@ def diff(self, manifest): "unique_fs": sorted(list(fs)), } + def similar(self, manifest): + """ + Find similar images against the database. + """ + # Get and subtract all paths that are unique to the image + diff = self.diff(manifest) + + # This is the image, minus all unique / different paths + fs = {x for x in manifest["fs"] if x not in diff["unique_fs"]} + + scores = {} + count = len(fs) + top_score = None + most_similar_image = None + for data in self.iter_containers(): + base_image = list(data.keys())[0] + base_fs = set(list(data.values())[0]["fs"]) + # Total that are in common / total + intersection = fs.intersection(base_fs) + score = len(intersection) / count + if top_score is None or score > top_score: + top_score = score + most_similar_image = base_image + scores[base_image] = { + "intersection_div_total_score": score, + "intersection": len(intersection), + "total_non_unique_image": count, + "base_image": base_image, + } + + # Add the most similar image + scores["most_similar"] = {"base_image": most_similar_image, "score": top_score} + return scores + def set_database(self): """ Ensure we have a database on the local filesystem. diff --git a/container_guts/main/templates.py b/container_guts/main/templates.py index ddc21f9..00f0d02 100644 --- a/container_guts/main/templates.py +++ b/container_guts/main/templates.py @@ -1,10 +1,9 @@ __author__ = "Vanessa Sochat" -__copyright__ = "Copyright 2021-2022, Vanessa Sochat" +__copyright__ = "Copyright 2021-2024, Vanessa Sochat" __license__ = "MPL 2.0" import re - docker_regex = re.compile( "(?:(?P[^/@]+[.:][^/@]*)/)?" "(?P(?:[^:@/]+/)+)?" diff --git a/container_guts/utils/fileio.py b/container_guts/utils/fileio.py index 2598ff9..07c1063 100644 --- a/container_guts/utils/fileio.py +++ b/container_guts/utils/fileio.py @@ -1,5 +1,5 @@ __author__ = "Vanessa Sochat" -__copyright__ = "Copyright 2021-2022, Vanessa Sochat" +__copyright__ = "Copyright 2021-2024, Vanessa Sochat" __license__ = "MPL 2.0" import errno diff --git a/container_guts/utils/terminal.py b/container_guts/utils/terminal.py index 3d4bede..9dfa040 100644 --- a/container_guts/utils/terminal.py +++ b/container_guts/utils/terminal.py @@ -1,10 +1,11 @@ __author__ = "Vanessa Sochat" -__copyright__ = "Copyright 2021-2022, Vanessa Sochat" +__copyright__ = "Copyright 2021-2024, Vanessa Sochat" __license__ = "MPL 2.0" import os from subprocess import PIPE, STDOUT, Popen + from container_guts.logger import logger @@ -31,7 +32,7 @@ def get_userhome(): import pwd return pwd.getpwuid(os.getuid())[5] - except: + except Exception: # Fallback to envar for Windows, etc. return os.environ.get("HOME") diff --git a/container_guts/version.py b/container_guts/version.py index a26f59a..94c8e98 100644 --- a/container_guts/version.py +++ b/container_guts/version.py @@ -1,8 +1,8 @@ __author__ = "Vanessa Sochat" -__copyright__ = "Copyright 2021-2022, Vanessa Sochat" +__copyright__ = "Copyright 2021-2024, Vanessa Sochat" __license__ = "MPL 2.0" -__version__ = "0.0.15" +__version__ = "0.0.16" AUTHOR = "Vanessa Sochat" NAME = "container-guts" EMAIL = "vsoch@users.noreply.github.com" diff --git a/docs/_static/theme.css b/docs/_static/theme.css index dd220b1..94cc142 100644 --- a/docs/_static/theme.css +++ b/docs/_static/theme.css @@ -24,76 +24,75 @@ .wy-side-nav-search a { visibility: hidden; } - + /* Vertical navigation ==================================================== */ .wy-menu-vertical header, .wy-menu-vertical p.caption, .rst-versions a{ - color: darkred; + color: darkred; } - + .wy-menu-vertical a.reference:hover, .wy-menu-vertical a.reference.internal:hover{ - background: #3486bc; + background: #3486bc; color: #fff; } - + .wy-nav-side { - background: rgb(240,240,240); + background: rgb(240,240,240); } - + .wy-menu-vertical a.reference { color: #000; } - + .rst-versions .rst-current-version, .wy-nav-top, .wy-menu-vertical li.toctree-l2.current li.toctree-l3>a:hover { background: #f0001e; color: #fee; } - + /* Main content ==================================================== */ - + .wy-nav-content .highlight, .wy-nav-content .rst-content .warning { background: #FEE; } - + .wy-nav-content .rst-content .warning .admonition-title { - background: #44CC11; + background: #44CC11; } - + .wy-nav-content .highlight .kn { - color: #bea2bf; + color: #bea2bf; } - + .wy-nav-content .highlight .nn, .wy-nav-content a.fa, .rst-content dl:not(.docutils) dt, .wy-nav-content .admonition.warning a { - color: darkred; + color: darkred; } - + .wy-nav-content a.reference { color: darkred; text-decoration: underline; } - + .wy-nav-content .rst-content dl:not(.docutils) dt { background: lightblue; border-top: solid 3px #3486bc;; } - + /* Footer ==================================================== */ .wy-nav-content footer a { - color: darkred; + color: darkred; } .keep-us-sustainable { background-color: #333333; } - diff --git a/docs/conf.py b/docs/conf.py index 1a3c1fd..b0b9d0b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,18 +13,18 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys import os -from recommonmark.parser import CommonMarkParser - -source_parsers = {".md": CommonMarkParser} +import sys +from recommonmark.parser import CommonMarkParser # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath("../")) +source_parsers = {".md": CommonMarkParser} + # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. @@ -60,7 +60,7 @@ project = "Container Guts 'Guts'" copyright = "2022-2023, Vanessa Sochat" -from container_guts import version +from container_guts import version # noqa # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/docs/getting_started/index.rst b/docs/getting_started/index.rst index 2293967..043ad70 100644 --- a/docs/getting_started/index.rst +++ b/docs/getting_started/index.rst @@ -12,7 +12,7 @@ first draft, we: And since we want to find "special" executables in a container, the intended workflow will be to have a set of core bases (e.g., ubuntu) that we can subtract -one. +one. If you have any questions or issues, please `let us know `_ .. toctree:: diff --git a/docs/getting_started/user-guide.rst b/docs/getting_started/user-guide.rst index 2f257d1..190d02f 100644 --- a/docs/getting_started/user-guide.rst +++ b/docs/getting_started/user-guide.rst @@ -10,7 +10,7 @@ If you haven't installed Guts yet, you should read :ref:`getting_started-install Commands ======== -Guts currently has just one client command, "manifest" to +Guts currently has just one client command, "manifest" to get a manifest of executables on the ``PATH``! -------- @@ -43,7 +43,7 @@ Or to get fs and paths: .. code-block:: console - $ guts manifest --include paths --include fs ubuntu --outfile ubuntu-guts.json + $ guts manifest --include paths --include fs ubuntu --outfile ubuntu-guts.json This generic "manifest" command is the main entrypoint to extract guts. @@ -54,7 +54,7 @@ Diff **under development** -A diff will take your container and compares it against a set of base images, +A diff will take your container and compares it against a set of base images, and only reveals the diff output (the executables in PATH that are special to your container). If you don't provide a database (repository or path on the filesystem) we use the default at ``singularityhub/shpc-guts``. @@ -80,7 +80,7 @@ For a single image (e.g., on dispatch) name: Generate Container Guts on: - workflow_dispatch: + workflow_dispatch: inputs: docker_uri: description: 'Docker identifier to generate recipe for' @@ -108,7 +108,7 @@ Matrix Images Manifest or for a matrix! E.g., you might want to save them nested in their directory location. - + .. code-block:: yaml name: Generate Container Guts @@ -204,5 +204,3 @@ The above would be the same as doing: Note that for all of the above, by default guts will be installed for you, unless you install a custom version in a previous step. - - diff --git a/examples/README.md b/examples/README.md index 0a47a33..040f776 100644 --- a/examples/README.md +++ b/examples/README.md @@ -3,5 +3,3 @@ These are some example (fun) analyses using the library. - [OS Diff](os_diff): look at differences between container bases. - - diff --git a/examples/os_diffs/README.md b/examples/os_diffs/README.md index cfa87ba..78bb4d7 100644 --- a/examples/os_diffs/README.md +++ b/examples/os_diffs/README.md @@ -14,7 +14,7 @@ The output will generate: - png/svg of a heatmap that compares base OSs (below) - a [unique-paths.json](unique-paths.json) to show paths unique to each base image - a [os-diffs.csv](os-diffs.csv) with the raw data. - + ![os-diffs.png](os-diffs.png) ## What I Learned diff --git a/examples/os_diffs/diff.py b/examples/os_diffs/diff.py index 34db883..323f78f 100644 --- a/examples/os_diffs/diff.py +++ b/examples/os_diffs/diff.py @@ -1,15 +1,13 @@ #!/usr/bin/env python3 import argparse -import pandas import json import os import re -import shutil import sys -import tempfile import matplotlib.pyplot as plt +import pandas import seaborn as sns sns.set_theme(style="white") @@ -46,7 +44,7 @@ def plot_heatmap(df, save_tos=None): cmap = sns.diverging_palette(230, 3, as_cmap=True) # Draw the heatmap with the mask and correct aspect ratio - p = sns.heatmap(df, linewidths=0.5, cbar=False, cmap=cmap) + sns.heatmap(df, linewidths=0.5, cbar=False, cmap=cmap) # used for heatmap # p.tick_params(labelsize=5) plt.tight_layout() @@ -77,7 +75,7 @@ def uniques(self): x.replace(self.database, "").strip("/").replace(".json", ""): x for x in bases } - df = pandas.DataFrame(columns=list(names), index=list(names), data=0) + # df = pandas.DataFrame(columns=list(names), index=list(names), data=0) uniques = {name: set() for name in names} for A in names: Apaths = self.read_fs(names[A]) @@ -165,7 +163,6 @@ def recursive_find(base, pattern=None): def main(): - parser = get_parser() # If an error occurs while parsing the arguments, the interpreter will exit with value 2 diff --git a/examples/os_diffs/os-diffs.svg b/examples/os_diffs/os-diffs.svg index a1867b2..12bcc08 100644 --- a/examples/os_diffs/os-diffs.svg +++ b/examples/os_diffs/os-diffs.svg @@ -21,19 +21,19 @@ - - @@ -43,328 +43,328 @@ z - - - - - - - - - - - - - - - - @@ -403,37 +403,37 @@ z - @@ -472,29 +472,29 @@ z - @@ -533,117 +533,117 @@ z - - - - - @@ -685,18 +685,18 @@ z - @@ -849,73 +849,73 @@ z - - - @@ -994,44 +994,44 @@ z - @@ -1172,39 +1172,39 @@ z - @@ -1287,32 +1287,32 @@ z - @@ -2245,2651 +2245,2651 @@ z - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/examples/os_diffs/unique-paths.json b/examples/os_diffs/unique-paths.json index 5c405ad..d6e4d64 100644 --- a/examples/os_diffs/unique-paths.json +++ b/examples/os_diffs/unique-paths.json @@ -11824,4 +11824,4 @@ "/usr/lib/libtls.so.17", "/usr/lib/libtls.so.17.0.1" ] -} \ No newline at end of file +} diff --git a/examples/test_diff/autamus-clingo-diff.json b/examples/test_diff/autamus-clingo-diff.json index c0a00ca..691ce7b 100644 --- a/examples/test_diff/autamus-clingo-diff.json +++ b/examples/test_diff/autamus-clingo-diff.json @@ -29058,4 +29058,4 @@ ] } } -} \ No newline at end of file +} diff --git a/examples/test_diff/biocontainers-samtools.json b/examples/test_diff/biocontainers-samtools.json index efa92fa..55a24a6 100644 --- a/examples/test_diff/biocontainers-samtools.json +++ b/examples/test_diff/biocontainers-samtools.json @@ -3021,4 +3021,4 @@ ] } } -} \ No newline at end of file +} diff --git a/examples/test_diff/vanessa-salad-diff.json b/examples/test_diff/vanessa-salad-diff.json index 4c6e0fb..0804cec 100644 --- a/examples/test_diff/vanessa-salad-diff.json +++ b/examples/test_diff/vanessa-salad-diff.json @@ -9,4 +9,4 @@ ] } } -} \ No newline at end of file +} diff --git a/examples/ubuntu-guts.json b/examples/ubuntu-guts.json index 4fc06e8..5df9837 100644 --- a/examples/ubuntu-guts.json +++ b/examples/ubuntu-guts.json @@ -3671,4 +3671,4 @@ "/etc/ld.so.conf.d/x86_64-linux-gnu.conf" ] } -} \ No newline at end of file +} diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..151029f --- /dev/null +++ b/setup.cfg @@ -0,0 +1,8 @@ +[flake8] +exclude = benchmarks docs +max-line-length = 100 +ignore = E1 E2 E5 W5 +per-file-ignores = + container_guts/utils/__init__.py:F401 + container_guts/main/container/__init__.py:F401 + container_guts/main/__init__.py:F401 diff --git a/setup.py b/setup.py index 5304899..cad17fc 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ -from setuptools import setup, find_packages -import codecs import os +from setuptools import find_packages, setup + def get_lookup(): lookup = dict() @@ -13,7 +13,7 @@ def get_lookup(): # Read in requirements def get_reqs(lookup=None, key="INSTALL_REQUIRES"): - if lookup == None: + if lookup is None: lookup = get_lookup() install_requires = [] @@ -23,7 +23,7 @@ def get_reqs(lookup=None, key="INSTALL_REQUIRES"): if "exact_version" in module_meta: dependency = "%s==%s" % (module_name, module_meta["exact_version"]) elif "min_version" in module_meta: - if module_meta["min_version"] == None: + if module_meta["min_version"] is None: dependency = module_name else: dependency = "%s>=%s" % (module_name, module_meta["min_version"]) @@ -50,7 +50,7 @@ def get_reqs(lookup=None, key="INSTALL_REQUIRES"): try: with open("README.md") as filey: LONG_DESCRIPTION = filey.read() -except: +except Exception: LONG_DESCRIPTION = DESCRIPTION ################################################################################ @@ -58,7 +58,6 @@ def get_reqs(lookup=None, key="INSTALL_REQUIRES"): ################################################################################ if __name__ == "__main__": - INSTALL_REQUIRES = get_reqs(lookup) TESTS_REQUIRES = get_reqs(lookup, "TESTS_REQUIRES") INSTALL_REQUIRES_ALL = get_reqs(lookup, "INSTALL_REQUIRES_ALL") @@ -93,7 +92,7 @@ def get_reqs(lookup=None, key="INSTALL_REQUIRES"): "Topic :: Software Development", "Topic :: Scientific/Engineering", "Operating System :: Unix", - "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.9", ], entry_points={"console_scripts": ["guts=container_guts.client:run"]}, )