Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[PIMO] speed up #2379

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
+414 −357
Open

[PIMO] speed up #2379

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
34c1e78
initial draft
jpcbertoldo Oct 20, 2024
966a268
implement speed up
jpcbertoldo Oct 21, 2024
f5405eb
minor corrections and updates
jpcbertoldo Oct 22, 2024
61b529b
update notebooks
jpcbertoldo Oct 22, 2024
c4f4ac3
Merge branch 'main' into aupimo/speedup
samet-akcay Oct 24, 2024
b9b30f1
Merge branch 'main' into aupimo/speedup
samet-akcay Oct 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 51 additions & 30 deletions notebooks/700_metrics/701a_aupimo.ipynb
View file
Open in desktop

Large diffs are not rendered by default.

38 changes: 19 additions & 19 deletions notebooks/700_metrics/701b_aupimo_advanced_i.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,9 +254,9 @@
"output_type": "stream",
"text": [
"MEAN\n",
"aupimo_result.aupimos[labels == 1].mean().item()=0.742841961578308\n",
"aupimo_result.aupimos[labels == 1].mean().item()=0.7428374946357311\n",
"OTHER STATISTICS\n",
"DescribeResult(nobs=92, minmax=(0.0, 1.0), mean=0.742841961578308, variance=0.08757792704451817, skewness=-0.9285678601866055, kurtosis=-0.3299211772047075)\n"
"DescribeResult(nobs=92, minmax=(0.0, 1.0), mean=0.7428374946357313, variance=0.08757776807097678, skewness=-0.9284572154639179, kurtosis=-0.3300816832805764)\n"
]
},
{
Expand Down Expand Up @@ -396,7 +396,7 @@
" statistic value image_index\n",
"0 whislo 0.00 65\n",
"1 q1 0.53 58\n",
"2 med 0.89 63\n",
"2 med 0.89 9\n",
"3 q3 1.00 22\n",
"4 whishi 1.00 0\n"
]
Expand Down Expand Up @@ -660,7 +660,7 @@
"Lower bound: 0.00001\n",
"Upper bound: 0.00010\n",
"Thresholds corresponding to the FPR bounds\n",
"Lower threshold: 0.504\n",
"Lower threshold: 0.505\n",
"Upper threshold: 0.553\n"
]
}
Expand Down Expand Up @@ -1002,7 +1002,7 @@
},
{
"cell_type": "code",
"execution_count": 20,
"execution_count": 19,
"metadata": {},
"outputs": [
{
Expand All @@ -1013,7 +1013,7 @@
"0 whislo 0.00 0.00 65 1\n",
"1 q1 0.53 0.53 58 1\n",
"2 mean 0.74 0.75 7 1\n",
"3 med 0.89 0.89 63 1\n",
"3 med 0.89 0.90 9 1\n",
"4 q3 1.00 1.00 22 1\n",
"5 whishi 1.00 1.00 0 1\n"
]
Expand All @@ -1035,7 +1035,7 @@
},
{
"cell_type": "code",
"execution_count": 21,
"execution_count": 20,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -1070,7 +1070,7 @@
},
{
"cell_type": "code",
"execution_count": 22,
"execution_count": 21,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -1101,7 +1101,7 @@
},
{
"cell_type": "code",
"execution_count": 23,
"execution_count": 22,
"metadata": {},
"outputs": [
{
Expand All @@ -1111,7 +1111,7 @@
"<Figure size 700x200 with 1 Axes>"
]
},
"execution_count": 23,
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -1177,7 +1177,7 @@
},
{
"cell_type": "code",
"execution_count": 24,
"execution_count": 23,
"metadata": {},
"outputs": [
{
Expand All @@ -1201,7 +1201,7 @@
},
{
"cell_type": "code",
"execution_count": 25,
"execution_count": 24,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -1249,7 +1249,7 @@
},
{
"cell_type": "code",
"execution_count": 26,
"execution_count": 25,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -1282,7 +1282,7 @@
},
{
"cell_type": "code",
"execution_count": 27,
"execution_count": 26,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -1318,7 +1318,7 @@
},
{
"cell_type": "code",
"execution_count": 28,
"execution_count": 27,
"metadata": {},
"outputs": [
{
Expand All @@ -1328,7 +1328,7 @@
" statistic value nearest index label\n",
"0 whislo 0.42 0.42 90 0\n",
"1 q1 0.43 0.43 80 0\n",
"2 med 0.45 0.45 105 0\n",
"2 med 0.45 0.46 79 0\n",
"3 mean 0.46 0.46 89 0\n",
"4 q3 0.48 0.48 75 0\n",
"5 whishi 0.52 0.52 95 0\n"
Expand All @@ -1344,7 +1344,7 @@
},
{
"cell_type": "code",
"execution_count": 29,
"execution_count": 28,
"metadata": {},
"outputs": [
{
Expand All @@ -1354,7 +1354,7 @@
" statistic value nearest index label\n",
"0 whislo 0.42 0.42 90 0\n",
"1 q1 0.52 0.52 95 0\n",
"2 med 0.65 0.65 17 1\n",
"2 med 0.65 0.65 62 1\n",
"3 mean 0.66 0.66 45 1\n",
"4 q3 0.77 0.77 108 1\n",
"5 whishi 1.00 1.00 22 1\n"
Expand Down Expand Up @@ -1406,7 +1406,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.14"
"version": "3.10.15"
},
"orig_nbformat": 4
},
Expand Down
41 changes: 17 additions & 24 deletions notebooks/700_metrics/701c_aupimo_advanced_ii.ipynb
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions notebooks/700_metrics/701d_aupimo_advanced_iii.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -354,7 +354,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.14"
"version": "3.10.15"
},
"orig_nbformat": 4
},
Expand Down
4 changes: 2 additions & 2 deletions src/anomalib/metrics/pimo/binary_classification_curve.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ def binary_classification_curve(
return torch.from_numpy(result).to(scores_batch.device)


def _get_linspaced_thresholds(anomaly_maps: torch.Tensor, num_thresholds: int) -> torch.Tensor:
def _get_minmax_linspaced_thresholds(anomaly_maps: torch.Tensor, num_thresholds: int) -> torch.Tensor:
"""Get thresholds linearly spaced between the min and max of the anomaly maps."""
_validate.is_num_thresholds_gte2(num_thresholds)
# this operation can be a bit expensive
Expand Down Expand Up @@ -241,7 +241,7 @@ def threshold_and_binary_classification_curve(
f"but it is ignored because `thresholds_choice` is '{threshold_choice.value}'.",
)
# `num_thresholds` is validated in the function below
thresholds = _get_linspaced_thresholds(anomaly_maps, num_thresholds)
thresholds = _get_minmax_linspaced_thresholds(anomaly_maps, num_thresholds)

elif threshold_choice == ThresholdMethod.MEAN_FPR_OPTIMIZED:
raise NotImplementedError(f"TODO implement {threshold_choice.value}") # noqa: EM102
Expand Down
38 changes: 28 additions & 10 deletions src/anomalib/metrics/pimo/dataclasses.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ class PIMOResult:
thresholds
"""

# metadata
fpr_lower_bound: float
fpr_upper_bound: float

# data
thresholds: torch.Tensor = field(repr=False) # shape => (K,)
shared_fpr: torch.Tensor = field(repr=False) # shape => (K,)
Expand Down Expand Up @@ -80,6 +84,25 @@ def __post_init__(self) -> None:
)
raise TypeError(msg)

first_shared_fpr = self.shared_fpr[0]
last_shared_fpr = self.shared_fpr[-1]

if not torch.isclose(first_shared_fpr, torch.tensor(self.fpr_upper_bound, dtype=torch.float64), rtol=1e-2):
msg = (
f"Invalid {self.__class__.__name__} object. "
"The first shared FPR value is not equal to the upper bound: "
f"{first_shared_fpr=} != {self.fpr_upper_bound=}."
)
raise ValueError(msg)

if not torch.isclose(last_shared_fpr, torch.tensor(self.fpr_lower_bound, dtype=torch.float64), rtol=1e-2):
msg = (
f"Invalid {self.__class__.__name__} object. "
"The last shared FPR value is not equal to the lower bound: "
f"{last_shared_fpr=} != {self.fpr_lower_bound=}."
)
raise ValueError(msg)

def thresh_at(self, fpr_level: float) -> tuple[int, float, float]:
"""Return the threshold at the given shared FPR.

Expand Down Expand Up @@ -183,7 +206,6 @@ def __post_init__(self) -> None:
def from_pimo_result(
cls: type["AUPIMOResult"],
pimo_result: PIMOResult,
fpr_bounds: tuple[float, float],
num_thresholds_auc: int,
aupimos: torch.Tensor,
) -> "AUPIMOResult":
Expand Down Expand Up @@ -211,16 +233,12 @@ def from_pimo_result(
msg = "Expected all anomalous images to have valid AUPIMOs (not nan), but some have NaN values."
raise TypeError(msg)

fpr_lower_bound, fpr_upper_bound = fpr_bounds
# recall: fpr upper/lower bounds are the same as the thresh lower/upper bounds
_, thresh_lower_bound, __ = pimo_result.thresh_at(fpr_upper_bound)
_, thresh_upper_bound, __ = pimo_result.thresh_at(fpr_lower_bound)
# `_` is the threshold's index, `__` is the actual fpr value
return cls(
fpr_lower_bound=fpr_lower_bound,
fpr_upper_bound=fpr_upper_bound,
fpr_lower_bound=pimo_result.fpr_lower_bound,
fpr_upper_bound=pimo_result.fpr_upper_bound,
num_thresholds=num_thresholds_auc,
thresh_lower_bound=float(thresh_lower_bound),
thresh_upper_bound=float(thresh_upper_bound),
# recall: fpr upper/lower bounds are the same as the thresh lower/upper bounds
thresh_lower_bound=float(pimo_result.thresholds[0].item()),
thresh_upper_bound=float(pimo_result.thresholds[-1].item()),
aupimos=aupimos,
)
Loading
Loading