Adding feature direction rules (#1358)

* add feature direction Signed-off-by: Amit Galitzky <[email protected]> * changed condition Signed-off-by: Amit Galitzky <[email protected]> --------- Signed-off-by: Amit Galitzky <[email protected]> (cherry picked from commit 1a3b8c9) Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
opensearch-project · Nov 26, 2024 · 31f3f1d · 31f3f1d
1 parent 8bea041
commit 31f3f1d
Show file tree

Hide file tree

Showing 15 changed files with 387 additions and 50 deletions.
diff --git a/src/main/java/org/opensearch/ad/ml/IgnoreSimilarExtractor.java b/src/main/java/org/opensearch/ad/ml/IgnoreSimilarExtractor.java
@@ -53,15 +53,18 @@ public static ThresholdArrays processDetectorRules(AnomalyDetector detector) {
         if (rules != null) {
             for (Rule rule : rules) {
                 for (Condition condition : rule.getConditions()) {
-                    processCondition(
-                        condition,
-                        featureNames,
-                        baseDimension,
-                        ignoreSimilarFromAbove,
-                        ignoreSimilarFromBelow,
-                        ignoreSimilarFromAboveByRatio,
-                        ignoreSimilarFromBelowByRatio
-                    );
+                    if (condition.getThresholdType() != ThresholdType.ACTUAL_IS_BELOW_EXPECTED
+                        && condition.getThresholdType() != ThresholdType.ACTUAL_IS_OVER_EXPECTED) {
+                        processCondition(
+                            condition,
+                            featureNames,
+                            baseDimension,
+                            ignoreSimilarFromAbove,
+                            ignoreSimilarFromBelow,
+                            ignoreSimilarFromAboveByRatio,
+                            ignoreSimilarFromBelowByRatio
+                        );
+                    }
                 }
             }
         }
@@ -100,7 +103,10 @@ private static void processCondition(
         int featureIndex = featureNames.indexOf(featureName);
 
         ThresholdType thresholdType = condition.getThresholdType();
-        double value = condition.getValue();
+        Double value = condition.getValue();
+        if (value == null) {
+            value = 0d;
+        }
 
         switch (thresholdType) {
             case ACTUAL_OVER_EXPECTED_MARGIN:

diff --git a/src/main/java/org/opensearch/ad/ml/ThresholdingResult.java b/src/main/java/org/opensearch/ad/ml/ThresholdingResult.java
@@ -12,14 +12,17 @@
 package org.opensearch.ad.ml;
 
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 
 import org.apache.commons.lang.builder.ToStringBuilder;
+import org.opensearch.ad.model.AnomalyDetector;
 import org.opensearch.ad.model.AnomalyResult;
+import org.opensearch.ad.model.Rule;
 import org.opensearch.timeseries.ml.IntermediateResult;
 import org.opensearch.timeseries.model.Config;
 import org.opensearch.timeseries.model.Entity;
@@ -331,6 +334,12 @@ public List<AnomalyResult> toIndexableResults(
         String taskId,
         String error
     ) {
+        List<Rule> rules = new ArrayList<>();
+        if (detector instanceof AnomalyDetector) {
+            AnomalyDetector detectorConfig = (AnomalyDetector) detector;
+            rules = detectorConfig.getRules();
+        }
+
         return Collections
             .singletonList(
                 AnomalyResult
@@ -358,7 +367,8 @@ public List<AnomalyResult> toIndexableResults(
                         likelihoodOfValues,
                         threshold,
                         currentData,
-                        featureImputed
+                        featureImputed,
+                        rules
                     )
             );
     }

diff --git a/src/main/java/org/opensearch/ad/model/AnomalyDetector.java b/src/main/java/org/opensearch/ad/model/AnomalyDetector.java
@@ -835,6 +835,17 @@ private void validateRules(List<Feature> features, List<Rule> rules) {
                         this.issueType = ValidationIssueType.RULE;
                         return;
                     }
+                } else if (thresholdType == ThresholdType.ACTUAL_IS_BELOW_EXPECTED
+                    || thresholdType == ThresholdType.ACTUAL_IS_OVER_EXPECTED) {
+                    // Check if both operator and value are null
+                    if (condition.getOperator() != null || condition.getValue() != null) {
+                        this.errorMessage = SUPPRESSION_RULE_ISSUE_PREFIX
+                            + "For threshold type \""
+                            + thresholdType
+                            + "\", both operator and value must be empty or null, as this rule compares actual to expected values directly";
+                        this.issueType = ValidationIssueType.RULE;
+                        return;
+                    }
                 }
             }
         }

diff --git a/src/main/java/org/opensearch/ad/model/AnomalyResult.java b/src/main/java/org/opensearch/ad/model/AnomalyResult.java
@@ -17,6 +17,7 @@
 import java.io.IOException;
 import java.time.Instant;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
 
@@ -312,6 +313,7 @@ public AnomalyResult(
      * @param threshold Current threshold
      * @param currentData imputed data if any
      * @param featureImputed whether feature is imputed or not
+     * @param rules rules we apply on anomaly grade based on condition
      * @return the converted AnomalyResult instance
      */
     public static AnomalyResult fromRawTRCFResult(
@@ -338,15 +340,20 @@ public static AnomalyResult fromRawTRCFResult(
         double[] likelihoodOfValues,
         Double threshold,
         double[] currentData,
-        boolean[] featureImputed
+        boolean[] featureImputed,
+        List<Rule> rules
     ) {
         List<DataByFeatureId> convertedRelevantAttribution = null;
         List<DataByFeatureId> convertedPastValuesList = null;
         List<ExpectedValueList> convertedExpectedValues = null;
+        List<FeatureData> featuresForComparison = null;
 
         int featureSize = featureData == null ? 0 : featureData.size();
 
         if (grade > 0) {
+            // Get the top feature names based on the relevant attribution criteria
+            featuresForComparison = getTopFeatureNames(featureData, relevantAttribution);
+
             if (relevantAttribution != null) {
                 if (relevantAttribution.length == featureSize) {
                     convertedRelevantAttribution = new ArrayList<>(featureSize);
@@ -425,6 +432,28 @@ public static AnomalyResult fromRawTRCFResult(
                         );
                 }
             }
+
+            for (FeatureData feature : featuresForComparison) {
+                Double valueToCompare = getValueToCompare(feature, convertedPastValuesList, featureData);
+                Double expectedValue = getExpectedValue(feature, convertedExpectedValues);
+                if (valueToCompare == null || expectedValue == null) {
+                    continue; // Skip if either valueToCompare or expectedValue is missing
+                }
+                for (Rule rule : rules) {
+                    for (Condition condition : rule.getConditions()) {
+                        if (condition.getFeatureName().equals(feature.getFeatureName())) {
+                            ThresholdType thresholdType = condition.getThresholdType();
+                            if (thresholdType == ThresholdType.ACTUAL_IS_BELOW_EXPECTED && valueToCompare < expectedValue) {
+                                LOG.info("changed anomaly grade from: " + grade + " to 0d for detector: " + detectorId);
+                                grade = 0d;
+                            } else if (thresholdType == ThresholdType.ACTUAL_IS_OVER_EXPECTED && valueToCompare > expectedValue) {
+                                LOG.info("changed anomaly grade from: " + grade + " to 0d for detector: " + detectorId);
+                                grade = 0d;
+                            }
+                        }
+                    }
+                }
+            }
         }
 
         List<FeatureImputed> featureImputedList = new ArrayList<>();
@@ -468,6 +497,69 @@ public static AnomalyResult fromRawTRCFResult(
         );
     }
 
+    private static Double getValueToCompare(
+        FeatureData feature,
+        List<DataByFeatureId> convertedPastValuesList,
+        List<FeatureData> featureData
+    ) {
+        String featureId = feature.getFeatureId();
+        if (convertedPastValuesList != null) {
+            for (DataByFeatureId data : convertedPastValuesList) {
+                if (data.getFeatureId().equals(featureId)) {
+                    return data.getData();
+                }
+            }
+        } else {
+            for (FeatureData data : featureData) {
+                if (data.getFeatureId().equals(featureId)) {
+                    return data.getData();
+                }
+            }
+        }
+        return 0d;
+    }
+
+    private static Double getExpectedValue(FeatureData feature, List<ExpectedValueList> convertedExpectedValues) {
+        Double expectedValue = 0d;
+        if (convertedExpectedValues != null) {
+            for (ExpectedValueList expectedValueList : convertedExpectedValues) {
+                if (expectedValueList != null && expectedValueList.getValueList() != null) {
+                    for (var data : expectedValueList.getValueList()) {
+                        if (data.getFeatureId().equals(feature.getFeatureId())) {
+                            expectedValue = data.getData();
+                        }
+                    }
+                }
+            }
+        }
+        return expectedValue;
+    }
+
+    private static List<FeatureData> getTopFeatureNames(List<FeatureData> featureData, double[] relevantAttribution) {
+        List<FeatureData> topFeatureNames = new ArrayList<>();
+
+        if (relevantAttribution == null || relevantAttribution.length == 0 || (relevantAttribution.length != featureData.size())) {
+            topFeatureNames.addAll(featureData);
+            return topFeatureNames;
+        }
+
+        // Find the maximum rounded value in a single pass and add corresponding feature names
+        double maxRoundedAttribution = Arrays
+            .stream(relevantAttribution)
+            .map(value -> Math.round(value * 100.0) / 100.0)
+            .max()
+            .orElse(Double.NaN);
+
+        // Collect feature names with values that match the max rounded value
+        for (int i = 0; i < relevantAttribution.length; i++) {
+            if (Math.round(relevantAttribution[i] * 100.0) / 100.0 == maxRoundedAttribution) {
+                topFeatureNames.add(featureData.get(i));
+            }
+        }
+
+        return topFeatureNames;
+    }
+
     public AnomalyResult(StreamInput input) throws IOException {
         super(input);
         this.modelId = input.readOptionalString();

diff --git a/src/main/java/org/opensearch/ad/model/Condition.java b/src/main/java/org/opensearch/ad/model/Condition.java
@@ -29,9 +29,9 @@ public class Condition implements Writeable, ToXContentObject {
     private String featureName;
     private ThresholdType thresholdType;
     private Operator operator;
-    private double value;
+    private Double value;
 
-    public Condition(String featureName, ThresholdType thresholdType, Operator operator, double value) {
+    public Condition(String featureName, ThresholdType thresholdType, Operator operator, Double value) {
         this.featureName = featureName;
         this.thresholdType = thresholdType;
         this.operator = operator;
@@ -42,7 +42,7 @@ public Condition(StreamInput input) throws IOException {
         this.featureName = input.readString();
         this.thresholdType = input.readEnum(ThresholdType.class);
         this.operator = input.readEnum(Operator.class);
-        this.value = input.readDouble();
+        this.value = input.readBoolean() ? input.readDouble() : null;
     }
 
     /**
@@ -56,7 +56,7 @@ public static Condition parse(XContentParser parser) throws IOException {
         String featureName = null;
         ThresholdType thresholdType = null;
         Operator operator = null;
-        Double value = 0d;
+        Double value = null;
 
         ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
         while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
@@ -71,10 +71,18 @@ public static Condition parse(XContentParser parser) throws IOException {
                     thresholdType = ThresholdType.valueOf(parser.text().toUpperCase(Locale.ROOT));
                     break;
                 case OPERATOR_FIELD:
-                    operator = Operator.valueOf(parser.text().toUpperCase(Locale.ROOT));
+                    if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
+                        operator = null; // Set operator to null if the field is missing
+                    } else {
+                        operator = Operator.valueOf(parser.text().toUpperCase(Locale.ROOT));
+                    }
                     break;
                 case VALUE_FIELD:
-                    value = parser.doubleValue();
+                    if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
+                        value = null; // Set value to null if the field is missing
+                    } else {
+                        value = parser.doubleValue();
+                    }
                     break;
                 default:
                     break;
@@ -89,8 +97,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             .startObject()
             .field(FEATURE_NAME_FIELD, featureName)
             .field(THRESHOLD_TYPE_FIELD, thresholdType)
-            .field(OPERATOR_FIELD, operator)
-            .field(VALUE_FIELD, value);
+            .field(OPERATOR_FIELD, operator);
+        if (value != null) {
+            builder.field("value", value);
+        }
         return xContentBuilder.endObject();
     }
 
@@ -99,7 +109,10 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(featureName);
         out.writeEnum(thresholdType);
         out.writeEnum(operator);
-        out.writeDouble(value);
+        out.writeBoolean(value != null);
+        if (value != null) {
+            out.writeDouble(value);
+        }
     }
 
     public String getFeatureName() {
@@ -114,7 +127,7 @@ public Operator getOperator() {
         return operator;
     }
 
-    public double getValue() {
+    public Double getValue() {
         return value;
     }
 

diff --git a/src/main/java/org/opensearch/ad/model/ThresholdType.java b/src/main/java/org/opensearch/ad/model/ThresholdType.java
@@ -57,7 +57,19 @@ public enum ThresholdType {
      * should be ignored if the ratio of the deviation from the expected to the actual
      * (b-a)/|a| is less than or equal to ignoreNearExpectedFromBelowByRatio.
      */
-    EXPECTED_OVER_ACTUAL_RATIO("the ratio of the expected value over the actual value");
+    EXPECTED_OVER_ACTUAL_RATIO("the ratio of the expected value over the actual value"),
+
+    /**
+      * Specifies a threshold for ignoring anomalies based on whether the actual value
+      * is over the expected value returned from the model.
+      */
+    ACTUAL_IS_OVER_EXPECTED("the actual value is over the expected value"),
+
+    /**
+    * Specifies a threshold for ignoring anomalies based on whether the actual value
+    * is below the expected value returned from the model.
+     * */
+    ACTUAL_IS_BELOW_EXPECTED("the actual value is below the expected value");
 
     private final String description;