Skip to content

Commit

Permalink
Add a script to easily compare failed snapshot tests (#8346)
Browse files Browse the repository at this point in the history
### Related

- Related to #8182 

### What

Introduce a new helper to manage and visualise failed kittest snapshot
tests using Rerun.

```
pixi run snapshots --help

# view all failed snapshot
pixi run snapshots

# remove all failed snapshot temp files
pixi run snapshots --clean

# only view (or clean) snapshot for that crate
pixi run snapshots -p re_time_panel
```


https://github.com/user-attachments/assets/c2ad63be-ff85-4235-a161-a359ff540d89
  • Loading branch information
abey79 authored Feb 7, 2025
1 parent 6122c48 commit f618298
Show file tree
Hide file tree
Showing 2 changed files with 130 additions and 0 deletions.
2 changes: 2 additions & 0 deletions pixi.toml
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,8 @@ py-test = { cmd = "python -m pytest -vv rerun_py/tests/unit", depends-on = [
"py-build",
] }

snapshots = "python scripts/snapshots.py"

[feature.python-docs.tasks]
# Build the documentation search index.
# See `pixi run search-index --help` for more information.
Expand Down
128 changes: 128 additions & 0 deletions scripts/snapshots.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
"""
View or clean failed kittest snapshot tests.
Usage:
```
pixi run snapshot --help
```
"""

from __future__ import annotations

import argparse
from pathlib import Path
from sys import stderr
from typing import Iterator

import numpy as np
import PIL.Image as Image
import rerun as rr
import rerun.blueprint as rrb

CRATES_DIR = Path(__file__).parent.parent / "crates"


def find_failed_snapshot_tests(package: str | None) -> Iterator[tuple[Path, Path, Path]]:
for diff_path in CRATES_DIR.rglob("**/*.diff.png"):
if package is not None:
if not any(package == str(part) for part in diff_path.parts):
continue

original_path = diff_path.parent / diff_path.name.replace(".diff.png", ".png")
new_path = diff_path.parent / diff_path.name.replace(".diff.png", ".new.png")

if original_path.exists() and new_path.exists():
yield original_path, new_path, diff_path


def blueprint(path: Path) -> rrb.Blueprint:
test_name = path.stem
crate_name = path.relative_to(CRATES_DIR).parts[1]

return rrb.Blueprint(
rrb.Tabs(
rrb.Horizontal(
rrb.Spatial2DView(origin="original", name="Original"),
rrb.Spatial2DView(origin="new", name="New"),
rrb.Spatial2DView(origin="diff", name="Diff"),
name="Side-by-side",
),
rrb.Tabs(
rrb.Spatial2DView(origin="original", name="Original"),
rrb.Spatial2DView(origin="new", name="New"),
rrb.Spatial2DView(origin="diff", name="Diff"),
),
rrb.Tabs(
rrb.Vertical(
rrb.Spatial2DView(
contents=["/original", "/new"],
name="Overlay (opacity)",
overrides={
"/new": [rr.components.Opacity(0.5)],
},
),
name='NOTE: Select the "new" entity visualizer and play with the "Opacity" component',
),
name="Overlay (tab)",
),
name=f"{crate_name}/{test_name}",
),
rrb.TimePanel(expanded=False),
)


def log_failed_snapshot_tests(original_path: Path, new_path: Path, diff_path: Path, args: argparse.Namespace) -> None:
recording = rr.new_recording(f"rerun_example_{original_path.stem}")

with recording:
default_blueprint = blueprint(original_path)

if args.stdout:
rr.stdout(default_blueprint=default_blueprint)
elif args.serve:
rr.serve(default_blueprint=default_blueprint)
elif args.connect:
rr.connect(args.addr, default_blueprint=default_blueprint)
elif args.save is not None:
rr.save(args.save, default_blueprint=default_blueprint)
elif not args.headless:
rr.spawn(default_blueprint=default_blueprint)

rr.log("original", rr.Image(np.array(Image.open(original_path))), static=True)
rr.log("new", rr.Image(np.array(Image.open(new_path))), static=True)
rr.log("diff", rr.Image(np.array(Image.open(diff_path))), static=True)

rr.log(
"doc/tabs",
rr.TextDocument(
"### Click on one of the tabs below to show the Original/New/Diff images.", media_type="text/markdown"
),
)


def main():
parser = argparse.ArgumentParser(description="Logs all failed snapshot tests for comparison in rerun")
parser.add_argument("-p", "--package", type=str, help="Only consider the provided package")
parser.add_argument("--clean", action="store_true", help="Clean snapshot files instead of displaying them")
rr.script_add_args(parser)

args = parser.parse_args()

none_found = True
for original_path, new_path, diff_path in find_failed_snapshot_tests(args.package):
none_found = False

if args.clean:
print(f"Removing {new_path}", file=stderr)
new_path.unlink(missing_ok=True)
print(f"Removing {diff_path}", file=stderr)
diff_path.unlink(missing_ok=True)
else:
log_failed_snapshot_tests(original_path, new_path, diff_path, args)

if none_found:
print("No failed snapshot found", file=stderr)


if __name__ == "__main__":
main()

0 comments on commit f618298

Please sign in to comment.