diff --git a/docs/en/concepts-and-designs/oal.md b/docs/en/concepts-and-designs/oal.md index 62aef850c..195b5f7fe 100644 --- a/docs/en/concepts-and-designs/oal.md +++ b/docs/en/concepts-and-designs/oal.md @@ -34,7 +34,7 @@ Read [Scope Definitions](scope-definitions.md), you can find all existing Scopes Use filter to build the conditions for the value of fields, by using field name and expression. The expressions support to link by `and`, `or` and `(...)`. -The OPs support `==`, `!=`, `>`, `<`, `>=`, `<=`, `like %...`, `like ...%` and `like %...%`, with type detection based of field type. Trigger compile +The OPs support `==`, `!=`, `>`, `<`, `>=`, `<=`, `in [...]` ,`like %...`, `like ...%` and `like %...%`, with type detection based of field type. Trigger compile or code generation error if incompatible. ## Aggregation Function @@ -54,12 +54,12 @@ In this case, input are request of each ServiceInstanceJVMCPU scope, avg is base In this case, all input are requests of each endpoint, condition is `endpoint.status == true`. - `sum`. The sum calls per scope entity. -> Service_Calls_Sum = from(Service.*).sum(); +> service_calls_sum = from(Service.*).sum(); In this case, calls of each service. - `histogram`. Read [Heatmap in WIKI](https://en.wikipedia.org/wiki/Heat_map) -> All_heatmap = from(All.latency).histogram(100, 20); +> all_heatmap = from(All.latency).histogram(100, 20); In this case, thermodynamic heatmap of all incoming requests. The parameter (1) is the precision of latency calculation, such as in above case, 113ms and 193ms are considered same in the 101-200ms group. @@ -78,7 +78,7 @@ The parameter (2) is the status of this request. The status(success/failure) eff **percentile** is the first multiple value metrics, introduced since 7.0.0. As having multiple values, it could be query through `getMultipleLinearIntValues` GraphQL query. In this case, `p99`, `p95`, `p90`, `p75`, `p50` of all incoming request. The parameter is the precision of p99 latency calculation, such as in above case, 120ms and 124 are considered same. Before 7.0.0, use `p99`, `p95`, `p90`, `p75`, `p50` func(s) to calculate metrics separately. Still supported in 7.x, but don't be recommended, and don't be included in official OAL script. -> All_p99 = from(All.latency).p99(10); +> all_p99 = from(All.latency).p99(10); In this case, p99 value of all incoming requests. The parameter is the precision of p99 latency calculation, such as in above case, 120ms and 124 are considered same. @@ -100,28 +100,37 @@ In default, no one is being disable. ## Examples ``` // Caculate p99 of both Endpoint1 and Endpoint2 -Endpoint_p99 = from(Endpoint.latency).filter(name in ("Endpoint1", "Endpoint2")).summary(0.99) +endpoint_p99 = from(Endpoint.latency).filter(name in ("Endpoint1", "Endpoint2")).summary(0.99) // Caculate p99 of Endpoint name started with `serv` serv_Endpoint_p99 = from(Endpoint.latency).filter(name like "serv%").summary(0.99) // Caculate the avg response time of each Endpoint -Endpoint_avg = from(Endpoint.latency).avg() +endpoint_avg = from(Endpoint.latency).avg() // Caculate the p50, p75, p90, p95 and p99 of each Endpoint by 50 ms steps. -Endpoint_percentile = from(Endpoint.latency).percentile(10) +endpoint_percentile = from(Endpoint.latency).percentile(10) // Caculate the percent of response status is true, for each service. -Endpoint_success = from(Endpoint.*).filter(status == true).percent() +endpoint_success = from(Endpoint.*).filter(status == true).percent() // Caculate the percent of response code in [200, 299], for each service. -Endpoint_200 = from(Endpoint.*).filter(responseCode like "2%").percent() +endpoint_200 = from(Endpoint.*).filter(responseCode like "2%").percent() // Caculate the percent of response code in [500, 599], for each service. -Endpoint_500 = from(Endpoint.*).filter(responseCode like "5%").percent() +endpoint_500 = from(Endpoint.*).filter(responseCode like "5%").percent() + +// Caculate the sum of response code in [404, 500, 503], for each service. +endpoint_abnormal = from(Endpoint.*).filter(responseCode in [404, 500, 503]).sum() + +// Caculate the sum of request type in [RequestType.PRC, RequestType.gRPC], for each service. +endpoint_rpc_calls_sum = from(Endpoint.*).filter(type in [RequestType.PRC, RequestType.gRPC]).sum() + +// Caculate the sum of endpoint name in ["/v1", "/v2"], for each service. +endpoint_url_sum = from(Endpoint.*).filter(endpointName in ["/v1", "/v2"]).sum() // Caculate the sum of calls for each service. -EndpointCalls = from(Endpoint.*).sum() +endpoint_calls = from(Endpoint.*).sum() disable(segment); disable(endpoint_relation_server_side); diff --git a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 index ddf90b50d..aa598c248 100644 --- a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 +++ b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 @@ -61,6 +61,26 @@ SRC_PROFILE_TASK: 'profile_task'; SRC_PROFILE_TASK_LOG: 'profile_task_log'; SRC_PROFILE_THREAD_SHANPSHOT: 'profile_task_segment_snapshot'; +// Constructors symbols + +DOT: '.'; +LR_BRACKET: '('; +RR_BRACKET: ')'; +LS_BRACKET: '['; +RS_BRACKET: ']'; +COMMA: ','; +SEMI: ';'; +EQUAL: '='; +DUALEQUALS: '=='; +ALL: '*'; +GREATER: '>'; +LESS: '<'; +GREATER_EQUAL: '>='; +LESS_EQUAL: '<='; +NOT_EQUAL: '!='; +LIKE: 'like'; +IN: 'in'; + // Literals BOOL_LITERAL: 'true' @@ -119,20 +139,3 @@ fragment Letter | ~[\u0000-\u007F\uD800-\uDBFF] // covers all characters above 0x7F which are not a surrogate | [\uD800-\uDBFF] [\uDC00-\uDFFF] // covers UTF-16 surrogate pairs encodings for U+10000 to U+10FFFF ; - -// Constructors symbols - -DOT: '.'; -LR_BRACKET: '('; -RR_BRACKET: ')'; -COMMA: ','; -SEMI: ';'; -EQUAL: '='; -DUALEQUALS: '=='; -ALL: '*'; -GREATER: '>'; -LESS: '<'; -GREATER_EQUAL: '>='; -LESS_EQUAL: '<='; -NOT_EQUAL: '!='; -LIKE: 'like'; diff --git a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 index 12cc62826..cc0fd5085 100644 --- a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 +++ b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 @@ -88,7 +88,7 @@ literalExpression ; expression - : booleanMatch | stringMatch | greaterMatch | lessMatch | greaterEqualMatch | lessEqualMatch | notEqualMatch | booleanNotEqualMatch | likeMatch + : booleanMatch | stringMatch | greaterMatch | lessMatch | greaterEqualMatch | lessEqualMatch | notEqualMatch | booleanNotEqualMatch | likeMatch | inMatch ; booleanMatch @@ -127,6 +127,14 @@ likeMatch : conditionAttribute LIKE stringConditionValue ; +inMatch + : conditionAttribute IN multiConditionValue + ; + +multiConditionValue + : LS_BRACKET (numberConditionValue ((COMMA numberConditionValue)*) | stringConditionValue ((COMMA stringConditionValue)*) | enumConditionValue ((COMMA enumConditionValue)*)) RS_BRACKET + ; + conditionAttribute : IDENTIFIER ; diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/ConditionExpression.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/ConditionExpression.java index 60e90dcda..8f7bc1015 100644 --- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/ConditionExpression.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/ConditionExpression.java @@ -18,7 +18,8 @@ package org.apache.skywalking.oal.rt.parser; -import lombok.AllArgsConstructor; +import java.util.LinkedList; +import java.util.List; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -26,10 +27,32 @@ import lombok.Setter; @Getter @Setter @NoArgsConstructor -@AllArgsConstructor public class ConditionExpression { // original from script private String expressionType; private String attribute; private String value; + private List values; + + public ConditionExpression(final String expressionType, final String attribute, final String value) { + this.expressionType = expressionType; + this.attribute = attribute; + this.value = value; + } + + public void addValue(String value) { + if (values != null) { + values.add(value); + } else { + this.value = value; + } + } + + public void enterMultiConditionValue() { + values = new LinkedList<>(); + } + + public void exitMultiConditionValue() { + value = "new Object[]{" + String.join(",", values) + "}"; + } } diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/OALListener.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/OALListener.java index c146bf7f8..be94ccab0 100644 --- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/OALListener.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/OALListener.java @@ -153,6 +153,21 @@ public class OALListener extends OALParserBaseListener { conditionExpression.setExpressionType("likeMatch"); } + @Override + public void enterInMatch(final OALParser.InMatchContext ctx) { + conditionExpression.setExpressionType("inMatch"); + } + + @Override + public void enterMultiConditionValue(final OALParser.MultiConditionValueContext ctx) { + conditionExpression.enterMultiConditionValue(); + } + + @Override + public void exitMultiConditionValue(final OALParser.MultiConditionValueContext ctx) { + conditionExpression.exitMultiConditionValue(); + } + @Override public void enterBooleanConditionValue(OALParser.BooleanConditionValueContext ctx) { enterConditionValue(ctx.getText()); @@ -178,7 +193,7 @@ public class OALListener extends OALParserBaseListener { // Value is an enum. value = sourcePackage + value; } - conditionExpression.setValue(value); + conditionExpression.addValue(value); } ///////////// diff --git a/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/ScriptParserTest.java b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/ScriptParserTest.java index a066547fe..fd91dd2d5 100644 --- a/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/ScriptParserTest.java +++ b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/ScriptParserTest.java @@ -167,6 +167,46 @@ public class ScriptParserTest { Assert.assertEquals("lessEqualMatch", booleanMatchExp.getExpressionType()); } + @Test + public void testParse6() throws IOException { + ScriptParser parser = ScriptParser.createFromScriptText( + "service_response_s4_summary = from(Service.latency).filter(latency like \"%a\").sum();", + TEST_SOURCE_PACKAGE + ); + List results = parser.parse().getMetricsStmts(); + Assert.assertEquals(1, results.size()); + AnalysisResult result = results.get(0); + List expressions = result.getFilterExpressions(); + Assert.assertEquals(1, expressions.size()); + Expression expression = expressions.get(0); + Assert.assertEquals("source.getLatency()", expression.getLeft()); + Assert.assertEquals( + "org.apache.skywalking.oap.server.core.analysis.metrics.expression.LikeMatch", + expression.getExpressionObject() + ); + Assert.assertEquals("\"%a\"", expression.getRight()); + } + + @Test + public void testParse7() throws IOException { + ScriptParser parser = ScriptParser.createFromScriptText( + "service_response_s4_summary = from(Service.latency).filter(latency != 1).filter(latency in [1,2, 3]).sum();", + TEST_SOURCE_PACKAGE + ); + List results = parser.parse().getMetricsStmts(); + Assert.assertEquals(1, results.size()); + AnalysisResult result = results.get(0); + List expressions = result.getFilterExpressions(); + Assert.assertEquals(2, expressions.size()); + Expression expression = expressions.get(1); + Assert.assertEquals("source.getLatency()", expression.getLeft()); + Assert.assertEquals( + "org.apache.skywalking.oap.server.core.analysis.metrics.expression.InMatch", + expression.getExpressionObject() + ); + Assert.assertEquals("new Object[]{1,2,3}", expression.getRight()); + } + @Test public void testDisable() throws IOException { ScriptParser parser = ScriptParser.createFromScriptText("disable(segment);", TEST_SOURCE_PACKAGE); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/EqualMatch.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/EqualMatch.java index 46ce2ed29..32c8a0091 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/EqualMatch.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/EqualMatch.java @@ -23,6 +23,19 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.FilterM @FilterMatcher("stringMatch") public class EqualMatch { + + public boolean match(String left, String right) { + if (left.startsWith("\"") && left.endsWith("\"")) { + left = left.substring(1, left.length() - 1); + } + + if (right.startsWith("\"") && right.endsWith("\"")) { + right = left.substring(1, right.length() - 1); + } + + return Objects.equals(left, right); + } + public boolean match(Object left, Object right) { return Objects.equals(left, right); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/InMatch.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/InMatch.java new file mode 100644 index 000000000..2873f3d27 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/InMatch.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.analysis.metrics.expression; + +import java.util.Objects; +import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.FilterMatcher; + +@FilterMatcher +public class InMatch { + + public boolean match(Object left, Object[] rights) { + for (Object right : rights) { + if (right instanceof String) { + String r = (String) right; + if (r.startsWith("\"") && r.endsWith("\"")) { + right = r.substring(1, r.length() - 1); + } + } + if (Objects.equals(left, right)) { + return true; + } + } + return false; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/LikeMatch.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/LikeMatch.java index 30fbfc3e8..ab416cb37 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/LikeMatch.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/LikeMatch.java @@ -23,14 +23,18 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.FilterM @FilterMatcher public class LikeMatch { public boolean match(String left, String right) { - if (left == null || right == null) { + if (right == null || left == null) { return false; } - if (left.startsWith("%") && left.endsWith("%")) { // %keyword% - return right.contains(left.substring(1, left.length() - 1)); + if (right.startsWith("\"") && right.endsWith("\"")) { + right = right.substring(1, right.length() - 1); } - return (left.startsWith("%") && right.endsWith(left.substring(1))) // %suffix - || (left.endsWith("%") && right.startsWith(left.substring(0, left.length() - 1))) // prefix% + + if (right.startsWith("%") && right.endsWith("%")) { // %keyword% + return left.contains(right.substring(1, right.length() - 1)); + } + return (right.startsWith("%") && left.endsWith(right.substring(1))) // %suffix + || (right.endsWith("%") && left.startsWith(right.substring(0, right.length() - 1))) // prefix% ; } } diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/EqualMatchTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/EqualMatchTest.java index b0ccf85ca..bce02419b 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/EqualMatchTest.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/EqualMatchTest.java @@ -20,6 +20,7 @@ package org.apache.skywalking.oap.server.core.analysis.metrics.expression; import org.junit.Test; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class EqualMatchTest { @@ -55,4 +56,11 @@ public class EqualMatchTest { boolean match = new EqualMatch().match(a, b); assertTrue(match); } + + @Test + public void stringShouldEqual() { + assertTrue(new EqualMatch().match("\"a\"", "a")); + assertTrue(new EqualMatch().match("a", "a")); + assertFalse(new EqualMatch().match("\"a\"", "ab")); + } } \ No newline at end of file diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/InMatchTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/InMatchTest.java new file mode 100644 index 000000000..60530dfd9 --- /dev/null +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/InMatchTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.analysis.metrics.expression; + +import org.apache.skywalking.oap.server.core.source.RequestType; +import org.junit.Assert; +import org.junit.Test; + +public class InMatchTest { + + @Test + public void testIn() { + Assert.assertTrue(new InMatch().match("a", new Object[] { + "\"a\"", + "\"b\"" + })); + Assert.assertFalse(new InMatch().match("c", new Object[] { + "\"a\"", + "\"b\"" + })); + + Assert.assertTrue( + new InMatch().match(RequestType.RPC, new Object[] { + RequestType.HTTP, + RequestType.DATABASE, + RequestType.RPC + })); + Assert.assertFalse( + new InMatch().match(RequestType.gRPC, new Object[] { + RequestType.HTTP, + RequestType.DATABASE, + RequestType.RPC + } + )); + + Assert.assertTrue(new InMatch().match(1, new Object[] { + 1, + 2, + 3 + })); + Assert.assertFalse(new InMatch().match(4, new Object[] { + 1, + 2, + 3 + })); + + Assert.assertTrue(new InMatch().match(1.0D, new Object[] { + 1.0D, + 2.0D, + 3.0D + })); + Assert.assertFalse(new InMatch().match(4.0D, new Object[] { + 1.0D, + 2.0D, + 3.0D + })); + } +} diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/LikeMatchTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/LikeMatchTest.java index ebd7a85c8..0c09d26cc 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/LikeMatchTest.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/LikeMatchTest.java @@ -20,17 +20,20 @@ package org.apache.skywalking.oap.server.core.analysis.metrics.expression; import org.junit.Test; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class LikeMatchTest { @Test public void testLike() { - assertTrue(new LikeMatch().match("%Black", "MaxBlack")); - assertTrue(new LikeMatch().match("Max%", "MaxBlack")); - assertTrue(new LikeMatch().match("%axBl%", "MaxBlack")); + assertTrue(new LikeMatch().match("MaxBlack", "%Black")); + assertTrue(new LikeMatch().match("MaxBlack", "Max%")); + assertTrue(new LikeMatch().match("MaxBlack", "%axBl%")); - assertFalse(new LikeMatch().match("Max%", "CarolineChanning")); - assertFalse(new LikeMatch().match("%Max", "CarolineChanning")); + assertFalse(new LikeMatch().match("CarolineChanning", "Max%")); + assertFalse(new LikeMatch().match("CarolineChanning", "%Max")); + + assertTrue(new LikeMatch().match("MaxBlack", "\"%Black\"")); + assertFalse(new LikeMatch().match("CarolineChanning", "\"Max%\"")); } }