Support IN filter expressions in OAL (#5390)

docs/en/concepts-and-designs/oal.md

@@ -34,7 +34,7 @@ Read [Scope Definitions](scope-definitions.md), you can find all existing Scopes
 Use filter to build the conditions for the value of fields, by using field name and expression. 
 
 The expressions support to link by `and`, `or` and `(...)`. 
-The OPs support `==`, `!=`, `>`, `<`, `>=`, `<=`, `like %...`, `like ...%` and `like %...%`, with type detection based of field type. Trigger compile
+The OPs support `==`, `!=`, `>`, `<`, `>=`, `<=`, `in [...]` ,`like %...`, `like ...%` and `like %...%`, with type detection based of field type. Trigger compile
  or code generation error if incompatible. 
 
 ## Aggregation Function
@@ -54,12 +54,12 @@ In this case, input are request of each ServiceInstanceJVMCPU scope, avg is base
 
 In this case, all input are requests of each endpoint, condition is `endpoint.status == true`.
 - `sum`. The sum calls per scope entity.
-> Service_Calls_Sum = from(Service.*).sum();
+> service_calls_sum = from(Service.*).sum();
 
 In this case, calls of each service. 
 
 - `histogram`. Read [Heatmap in WIKI](https://en.wikipedia.org/wiki/Heat_map)
-> All_heatmap = from(All.latency).histogram(100, 20);
+> all_heatmap = from(All.latency).histogram(100, 20);
 
 In this case, thermodynamic heatmap of all incoming requests. 
 The parameter (1) is the precision of latency calculation, such as in above case, 113ms and 193ms are considered same in the 101-200ms group.
@@ -78,7 +78,7 @@ The parameter (2) is the status of this request. The status(success/failure) eff
 **percentile** is the first multiple value metrics, introduced since 7.0.0. As having multiple values, it could be query through `getMultipleLinearIntValues` GraphQL query.
 In this case, `p99`, `p95`, `p90`, `p75`, `p50` of all incoming request. The parameter is the precision of p99 latency calculation, such as in above case, 120ms and 124 are considered same.
 Before 7.0.0, use `p99`, `p95`, `p90`, `p75`, `p50` func(s) to calculate metrics separately. Still supported in 7.x, but don't be recommended, and don't be included in official OAL script. 
-> All_p99 = from(All.latency).p99(10);
+> all_p99 = from(All.latency).p99(10);
 
 In this case, p99 value of all incoming requests. The parameter is the precision of p99 latency calculation, such as in above case, 120ms and 124 are considered same.
 
@@ -100,28 +100,37 @@ In default, no one is being disable.
 ## Examples
 ```
 // Caculate p99 of both Endpoint1 and Endpoint2
-Endpoint_p99 = from(Endpoint.latency).filter(name in ("Endpoint1", "Endpoint2")).summary(0.99)
+endpoint_p99 = from(Endpoint.latency).filter(name in ("Endpoint1", "Endpoint2")).summary(0.99)
 
 // Caculate p99 of Endpoint name started with `serv`
 serv_Endpoint_p99 = from(Endpoint.latency).filter(name like "serv%").summary(0.99)
 
 // Caculate the avg response time of each Endpoint
-Endpoint_avg = from(Endpoint.latency).avg()
+endpoint_avg = from(Endpoint.latency).avg()
 
 // Caculate the p50, p75, p90, p95 and p99 of each Endpoint by 50 ms steps.
-Endpoint_percentile = from(Endpoint.latency).percentile(10)
+endpoint_percentile = from(Endpoint.latency).percentile(10)
 
 // Caculate the percent of response status is true, for each service.
-Endpoint_success = from(Endpoint.*).filter(status == true).percent()
+endpoint_success = from(Endpoint.*).filter(status == true).percent()
 
 // Caculate the percent of response code in [200, 299], for each service.
-Endpoint_200 = from(Endpoint.*).filter(responseCode like "2%").percent()
+endpoint_200 = from(Endpoint.*).filter(responseCode like "2%").percent()
 
 // Caculate the percent of response code in [500, 599], for each service.
-Endpoint_500 = from(Endpoint.*).filter(responseCode like "5%").percent()
+endpoint_500 = from(Endpoint.*).filter(responseCode like "5%").percent()
+
+// Caculate the sum of response code in [404, 500, 503], for each service.
+endpoint_abnormal = from(Endpoint.*).filter(responseCode in [404, 500, 503]).sum()
+
+// Caculate the sum of request type in [RequestType.PRC, RequestType.gRPC], for each service.
+endpoint_rpc_calls_sum = from(Endpoint.*).filter(type in [RequestType.PRC, RequestType.gRPC]).sum()
+
+// Caculate the sum of endpoint name in ["/v1", "/v2"], for each service.
+endpoint_url_sum = from(Endpoint.*).filter(endpointName in ["/v1", "/v2"]).sum()
 
 // Caculate the sum of calls for each service.
-EndpointCalls = from(Endpoint.*).sum()
+endpoint_calls = from(Endpoint.*).sum()
 
 disable(segment);
 disable(endpoint_relation_server_side);

oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4

@@ -61,6 +61,26 @@ SRC_PROFILE_TASK: 'profile_task';
 SRC_PROFILE_TASK_LOG: 'profile_task_log';
 SRC_PROFILE_THREAD_SHANPSHOT: 'profile_task_segment_snapshot';
 
+// Constructors symbols
+
+DOT:                                 '.';
+LR_BRACKET:                          '(';
+RR_BRACKET:                          ')';
+LS_BRACKET:                          '[';
+RS_BRACKET:                          ']';
+COMMA:                               ',';
+SEMI:                                ';';
+EQUAL:                               '=';
+DUALEQUALS:                          '==';
+ALL:                                 '*';
+GREATER:                             '>';
+LESS:                                '<';
+GREATER_EQUAL:                       '>=';
+LESS_EQUAL:                          '<=';
+NOT_EQUAL:                           '!=';
+LIKE:                                'like';
+IN:                                  'in';
+
 // Literals
 
 BOOL_LITERAL:       'true'
@@ -119,20 +139,3 @@ fragment Letter
     | ~[\u0000-\u007F\uD800-\uDBFF] // covers all characters above 0x7F which are not a surrogate
     | [\uD800-\uDBFF] [\uDC00-\uDFFF] // covers UTF-16 surrogate pairs encodings for U+10000 to U+10FFFF
     ;
-
-// Constructors symbols
-
-DOT:                                 '.';
-LR_BRACKET:                          '(';
-RR_BRACKET:                          ')';
-COMMA:                               ',';
-SEMI:                                ';';
-EQUAL:                               '=';
-DUALEQUALS:                          '==';
-ALL:                                 '*';
-GREATER:                             '>';
-LESS:                                '<';
-GREATER_EQUAL:                       '>=';
-LESS_EQUAL:                          '<=';
-NOT_EQUAL:                           '!=';
-LIKE:                                'like';

oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4

@@ -88,7 +88,7 @@ literalExpression
     ;
 
 expression
-    : booleanMatch | stringMatch | greaterMatch | lessMatch | greaterEqualMatch | lessEqualMatch | notEqualMatch | booleanNotEqualMatch | likeMatch
+    : booleanMatch | stringMatch | greaterMatch | lessMatch | greaterEqualMatch | lessEqualMatch | notEqualMatch | booleanNotEqualMatch | likeMatch | inMatch
     ;
 
 booleanMatch
@@ -127,6 +127,14 @@ likeMatch
     :  conditionAttribute LIKE stringConditionValue
     ;
 
+inMatch
+    :  conditionAttribute IN multiConditionValue
+    ;
+
+multiConditionValue
+    : LS_BRACKET (numberConditionValue ((COMMA numberConditionValue)*) | stringConditionValue ((COMMA stringConditionValue)*) | enumConditionValue ((COMMA enumConditionValue)*)) RS_BRACKET
+    ;
+
 conditionAttribute
     : IDENTIFIER
     ;

oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/ConditionExpression.java

@@ -18,7 +18,8 @@
 
 package org.apache.skywalking.oal.rt.parser;
 
-import lombok.AllArgsConstructor;
+import java.util.LinkedList;
+import java.util.List;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
@@ -26,10 +27,32 @@ import lombok.Setter;
 @Getter
 @Setter
 @NoArgsConstructor
-@AllArgsConstructor
 public class ConditionExpression {
     // original from script
     private String expressionType;
     private String attribute;
     private String value;
+    private List<String> values;
+
+    public ConditionExpression(final String expressionType, final String attribute, final String value) {
+        this.expressionType = expressionType;
+        this.attribute = attribute;
+        this.value = value;
+    }
+
+    public void addValue(String value) {
+        if (values != null) {
+            values.add(value);
+        } else {
+            this.value = value;
+        }
+    }
+
+    public void enterMultiConditionValue() {
+        values = new LinkedList<>();
+    }
+
+    public void exitMultiConditionValue() {
+        value = "new Object[]{" + String.join(",", values) + "}";
+    }
 }

oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/OALListener.java

@@ -153,6 +153,21 @@ public class OALListener extends OALParserBaseListener {
         conditionExpression.setExpressionType("likeMatch");
     }
 
+    @Override
+    public void enterInMatch(final OALParser.InMatchContext ctx) {
+        conditionExpression.setExpressionType("inMatch");
+    }
+
+    @Override
+    public void enterMultiConditionValue(final OALParser.MultiConditionValueContext ctx) {
+        conditionExpression.enterMultiConditionValue();
+    }
+
+    @Override
+    public void exitMultiConditionValue(final OALParser.MultiConditionValueContext ctx) {
+        conditionExpression.exitMultiConditionValue();
+    }
+
     @Override
     public void enterBooleanConditionValue(OALParser.BooleanConditionValueContext ctx) {
         enterConditionValue(ctx.getText());
@@ -178,7 +193,7 @@ public class OALListener extends OALParserBaseListener {
             // Value is an enum.
             value = sourcePackage + value;
         }
-        conditionExpression.setValue(value);
+        conditionExpression.addValue(value);
     }
 
     /////////////

oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/ScriptParserTest.java

@@ -167,6 +167,46 @@ public class ScriptParserTest {
         Assert.assertEquals("lessEqualMatch", booleanMatchExp.getExpressionType());
     }
 
+    @Test
+    public void testParse6() throws IOException {
+        ScriptParser parser = ScriptParser.createFromScriptText(
+            "service_response_s4_summary = from(Service.latency).filter(latency like \"%a\").sum();",
+            TEST_SOURCE_PACKAGE
+        );
+        List<AnalysisResult> results = parser.parse().getMetricsStmts();
+        Assert.assertEquals(1, results.size());
+        AnalysisResult result = results.get(0);
+        List<Expression> expressions = result.getFilterExpressions();
+        Assert.assertEquals(1, expressions.size());
+        Expression expression = expressions.get(0);
+        Assert.assertEquals("source.getLatency()", expression.getLeft());
+        Assert.assertEquals(
+            "org.apache.skywalking.oap.server.core.analysis.metrics.expression.LikeMatch",
+            expression.getExpressionObject()
+        );
+        Assert.assertEquals("\"%a\"", expression.getRight());
+    }
+
+    @Test
+    public void testParse7() throws IOException {
+        ScriptParser parser = ScriptParser.createFromScriptText(
+            "service_response_s4_summary = from(Service.latency).filter(latency != 1).filter(latency in [1,2, 3]).sum();",
+            TEST_SOURCE_PACKAGE
+        );
+        List<AnalysisResult> results = parser.parse().getMetricsStmts();
+        Assert.assertEquals(1, results.size());
+        AnalysisResult result = results.get(0);
+        List<Expression> expressions = result.getFilterExpressions();
+        Assert.assertEquals(2, expressions.size());
+        Expression expression = expressions.get(1);
+        Assert.assertEquals("source.getLatency()", expression.getLeft());
+        Assert.assertEquals(
+            "org.apache.skywalking.oap.server.core.analysis.metrics.expression.InMatch",
+            expression.getExpressionObject()
+        );
+        Assert.assertEquals("new Object[]{1,2,3}", expression.getRight());
+    }
+
     @Test
     public void testDisable() throws IOException {
         ScriptParser parser = ScriptParser.createFromScriptText("disable(segment);", TEST_SOURCE_PACKAGE);

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/EqualMatch.java

@@ -23,6 +23,19 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.FilterM
 
 @FilterMatcher("stringMatch")
 public class EqualMatch {
+
+    public boolean match(String left, String right) {
+        if (left.startsWith("\"") && left.endsWith("\"")) {
+            left = left.substring(1, left.length() - 1);
+        }
+
+        if (right.startsWith("\"") && right.endsWith("\"")) {
+            right = left.substring(1, right.length() - 1);
+        }
+
+        return Objects.equals(left, right);
+    }
+
     public boolean match(Object left, Object right) {
         return Objects.equals(left, right);
     }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/InMatch.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.skywalking.oap.server.core.analysis.metrics.expression;
+
+import java.util.Objects;
+import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.FilterMatcher;
+
+@FilterMatcher
+public class InMatch {
+
+    public boolean match(Object left, Object[] rights) {
+        for (Object right : rights) {
+            if (right instanceof String) {
+                String r = (String) right;
+                if (r.startsWith("\"") && r.endsWith("\"")) {
+                    right = r.substring(1, r.length() - 1);
+                }
+            }
+            if (Objects.equals(left, right)) {
+                return true;
+            }
+        }
+        return false;
+    }
+}

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/LikeMatch.java

@@ -23,14 +23,18 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.FilterM
 @FilterMatcher
 public class LikeMatch {
     public boolean match(String left, String right) {
-        if (left == null || right == null) {
+        if (right == null || left == null) {
             return false;
         }
-        if (left.startsWith("%") && left.endsWith("%")) { // %keyword%
-            return right.contains(left.substring(1, left.length() - 1));
+        if (right.startsWith("\"") && right.endsWith("\"")) {
+            right = right.substring(1, right.length() - 1);
         }
-        return (left.startsWith("%") && right.endsWith(left.substring(1)))  // %suffix
-            || (left.endsWith("%") && right.startsWith(left.substring(0, left.length() - 1))) // prefix%
+
+        if (right.startsWith("%") && right.endsWith("%")) { // %keyword%
+            return left.contains(right.substring(1, right.length() - 1));
+        }
+        return (right.startsWith("%") && left.endsWith(right.substring(1)))  // %suffix
+            || (right.endsWith("%") && left.startsWith(right.substring(0, right.length() - 1))) // prefix%
             ;
     }
 }

oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/EqualMatchTest.java

@@ -20,6 +20,7 @@ package org.apache.skywalking.oap.server.core.analysis.metrics.expression;
 
 import org.junit.Test;
 
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 public class EqualMatchTest {
@@ -55,4 +56,11 @@ public class EqualMatchTest {
         boolean match = new EqualMatch().match(a, b);
         assertTrue(match);
     }
+
+    @Test
+    public void stringShouldEqual() {
+        assertTrue(new EqualMatch().match("\"a\"", "a"));
+        assertTrue(new EqualMatch().match("a", "a"));
+        assertFalse(new EqualMatch().match("\"a\"", "ab"));
+    }
 }

oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/InMatchTest.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.skywalking.oap.server.core.analysis.metrics.expression;
+
+import org.apache.skywalking.oap.server.core.source.RequestType;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class InMatchTest {
+
+    @Test
+    public void testIn() {
+        Assert.assertTrue(new InMatch().match("a", new Object[] {
+            "\"a\"",
+            "\"b\""
+        }));
+        Assert.assertFalse(new InMatch().match("c", new Object[] {
+            "\"a\"",
+            "\"b\""
+        }));
+
+        Assert.assertTrue(
+            new InMatch().match(RequestType.RPC, new Object[] {
+                RequestType.HTTP,
+                RequestType.DATABASE,
+                RequestType.RPC
+            }));
+        Assert.assertFalse(
+            new InMatch().match(RequestType.gRPC, new Object[] {
+                                    RequestType.HTTP,
+                                    RequestType.DATABASE,
+                                    RequestType.RPC
+                                }
+            ));
+
+        Assert.assertTrue(new InMatch().match(1, new Object[] {
+            1,
+            2,
+            3
+        }));
+        Assert.assertFalse(new InMatch().match(4, new Object[] {
+            1,
+            2,
+            3
+        }));
+
+        Assert.assertTrue(new InMatch().match(1.0D, new Object[] {
+            1.0D,
+            2.0D,
+            3.0D
+        }));
+        Assert.assertFalse(new InMatch().match(4.0D, new Object[] {
+            1.0D,
+            2.0D,
+            3.0D
+        }));
+    }
+}

oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/LikeMatchTest.java

@@ -20,17 +20,20 @@ package org.apache.skywalking.oap.server.core.analysis.metrics.expression;
 
 import org.junit.Test;
 
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 public class LikeMatchTest {
     @Test
     public void testLike() {
-        assertTrue(new LikeMatch().match("%Black", "MaxBlack"));
-        assertTrue(new LikeMatch().match("Max%", "MaxBlack"));
-        assertTrue(new LikeMatch().match("%axBl%", "MaxBlack"));
+        assertTrue(new LikeMatch().match("MaxBlack", "%Black"));
+        assertTrue(new LikeMatch().match("MaxBlack", "Max%"));
+        assertTrue(new LikeMatch().match("MaxBlack", "%axBl%"));
 
-        assertFalse(new LikeMatch().match("Max%", "CarolineChanning"));
-        assertFalse(new LikeMatch().match("%Max", "CarolineChanning"));
+        assertFalse(new LikeMatch().match("CarolineChanning", "Max%"));
+        assertFalse(new LikeMatch().match("CarolineChanning", "%Max"));
+
+        assertTrue(new LikeMatch().match("MaxBlack", "\"%Black\""));
+        assertFalse(new LikeMatch().match("CarolineChanning", "\"Max%\""));
     }
 }