kazusa
/
skywalking
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* MQE query: make metadata not return `null`. (#12142) 

* MQE labeled metrics Binary Operation: return empty value if the labels not match rather than report error.
This commit is contained in:
Wan Kai 2024-04-22 12:25:12 +08:00 committed by GitHub
parent d6fc848e8b
commit 5808627227
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 43 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
docs/en/api/metrics-query-expression.md
View File

@ -82,7 +82,8 @@ For the result type of the expression, please refer to the following table.
### Binary Operation Rules
The following table lists if the different result types of the input expressions could do this operation and the result type after the operation.
The expression could be on the left or right side of the operator.
**Note**: If the expressions on both sides of the operator are the `TIME_SERIES_VALUES with labels`, they should have the same labels for calculation.
**Note**: If the expressions result on both sides of the operator are `with labels`, they should have the same labels for calculation.
If the labels match, will reserve left expression result labels and the calculated value. Otherwise, will return empty value.
| Expression | Expression | Yes/No | ExpressionResultType |
|-------------------------|---------------------------|--------|--------------------------|

2
docs/en/changes/changes.md
View File

@ -111,6 +111,8 @@
* Add Service Hierarchy auto matching layer relationships (upper -> lower) as following:
- ACTIVEMQ -> K8S_SERVICE
* Calculate Nginx service HTTP Latency by MQE.
* MQE query: make metadata not return `null`.
* MQE labeled metrics Binary Operation: return empty value if the labels not match rather than report error.
#### UI

10
oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/MQEVisitorBase.java
View File

@ -149,6 +149,7 @@ public abstract class MQEVisitorBase extends MQEParserBaseVisitor<ExpressionResu
MQEParser.LabelNameListContext labelNameListContext = ctx.aggregateLabelsFunc().labelNameList();
if (null != labelNameListContext) {
for (MQEParser.LabelNameContext labelNameContext : labelNameListContext.labelName()) {
// ignore the label name that does not exist in the result
if (expResult.getResults()
.get(0)
.getMetric()
@ -156,10 +157,6 @@ public abstract class MQEVisitorBase extends MQEParserBaseVisitor<ExpressionResu
.stream()
.anyMatch(label -> label.getKey().equals(labelNameContext.getText()))) {
labelNames.add(labelNameContext.getText());
} else {
expResult.setError(
"The label [" + labelNameContext.getText() + "] does not exist in the result.");
return expResult;
}
}
}
@ -221,11 +218,6 @@ public abstract class MQEVisitorBase extends MQEParserBaseVisitor<ExpressionResu
}
KeyValue targetLabel = buildLabel(ctx.label());
KeyValue replaceLabel = buildLabel(ctx.replaceLabel().label());
// if (!targetLabel.getKey().equals(replaceLabel.getKey())) {
// result.setError("The target label name and replace label name must be the same.");
// return result;
// }
List<KeyValue> targetLabels = parseLabelValue(targetLabel, Const.COMMA);
List<KeyValue> replaceLabels = parseLabelValue(replaceLabel, Const.COMMA);

9
oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/operation/AggregateLabelsOp.java
View File

@ -31,7 +31,6 @@ import org.apache.skywalking.mqe.rt.operation.aggregatelabels.AvgAggregateLabels
import org.apache.skywalking.mqe.rt.operation.aggregatelabels.MaxAggregateLabelsFunc;
import org.apache.skywalking.mqe.rt.operation.aggregatelabels.MinAggregateLabelsFunc;
import org.apache.skywalking.mqe.rt.operation.aggregatelabels.SumAggregateLabelsFunc;
import org.apache.skywalking.mqe.rt.type.Metadata;
import org.apache.skywalking.oap.server.core.query.type.KeyValue;
import org.apache.skywalking.oap.server.library.util.CollectionUtils;
@ -98,9 +97,7 @@ public class AggregateLabelsOp {
}
}
MQEValues mqeValues = new MQEValues();
Metadata metadata = new Metadata();
metadata.setLabels(labels);
mqeValues.setMetric(metadata);
mqeValues.getMetric().setLabels(labels);
mqeValues.setValues(combineTo);
expResult.getResults().add(mqeValues);
});
@ -108,10 +105,8 @@ public class AggregateLabelsOp {
}
private static List<KeyValue> getLabels(final List<String> labelNames, final MQEValues mqeValues) {
List<KeyValue> a =
labelNames.stream().map(labelName -> mqeValues.getMetric().getLabels().stream().filter(label -> labelName.equals(label.getKey()))
return labelNames.stream().map(labelName -> mqeValues.getMetric().getLabels().stream().filter(label -> labelName.equals(label.getKey()))
.findAny().orElseGet(() -> new KeyValue(labelName, ""))).collect(toList());
return a;
}
}

51
oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/operation/LROp.java
View File

@ -107,7 +107,6 @@ public interface LROp {
} else {
double value = calculate.apply(left.getDoubleValue(), right.getDoubleValue(), opType);
mqeValue.setDoubleValue(value);
mqeValue.setEmptyValue(false);
}
return result;
}
@ -115,31 +114,24 @@ public interface LROp {
private static ExpressionResult single2SingleLabeled(ExpressionResult singleLeft,
ExpressionResult singleRight,
int opType, LROp calculate) throws IllegalExpressionException {
Map<KeyValue, List<MQEValue>> labelMapR = new HashMap<>();
if (singleLeft.getResults().size() != singleRight.getResults().size()) {
throw new IllegalExpressionException(
"Operation between labeled metrics should have the same label.");
}
Map<Set<KeyValue>, List<MQEValue>> labelMapR = new HashMap<>();
singleRight.getResults().forEach(mqeValuesR -> {
// For now, we only have a single anonymous label named `_`
labelMapR.put(mqeValuesR.getMetric().getLabels().get(0), mqeValuesR.getValues());
labelMapR.put(new HashSet<>(mqeValuesR.getMetric().getLabels()), mqeValuesR.getValues());
});
for (MQEValues mqeValuesL : singleLeft.getResults()) {
//reserve left metric info
MQEValue valueL = mqeValuesL.getValues().get(0);
List<MQEValue> mqeValuesR = labelMapR.get(mqeValuesL.getMetric().getLabels().get(0));
List<MQEValue> mqeValuesR = labelMapR.get(new HashSet<>(mqeValuesL.getMetric().getLabels()));
if (mqeValuesR == null) {
throw new IllegalExpressionException(
"Operation between labeled metrics should have the same label.");
}
MQEValue valueR = mqeValuesR.get(0);
if (valueL.isEmptyValue() || valueR.isEmptyValue()) {
valueL.setEmptyValue(true);
valueL.setDoubleValue(0);
} else {
MQEValue valueR = mqeValuesR.get(0);
if (valueL.isEmptyValue() || valueR.isEmptyValue()) {
valueL.setEmptyValue(true);
continue;
}
double value = calculate.apply(valueL.getDoubleValue(), valueR.getDoubleValue(), opType);
valueL.setDoubleValue(value);
valueL.setEmptyValue(false);
}
}
@ -193,14 +185,12 @@ public interface LROp {
int opType, LROp calculate) {
MQEValues mqeValuesL = seriesLeft.getResults().get(0);
MQEValues mqeValuesR = seriesRight.getResults().get(0);
mqeValuesL.setMetric(null);
for (int i = 0; i < mqeValuesL.getValues().size(); i++) {
//clean metric info
MQEValue valueL = mqeValuesL.getValues().get(i);
MQEValue valueR = mqeValuesR.getValues().get(i);
if (valueL.isEmptyValue() || valueR.isEmptyValue()) {
valueL.setEmptyValue(true);
valueL.setDoubleValue(0);
continue;
}
//time should be mapped
@ -222,7 +212,6 @@ public interface LROp {
MQEValue valueR = mqeValuesR.getValues().get(i);
if (valueL.isEmptyValue() || valueR.isEmptyValue()) {
valueL.setEmptyValue(true);
valueL.setDoubleValue(0);
continue;
}
double newValue = calculate.apply(valueL.getDoubleValue(), valueR.getDoubleValue(), opType);
@ -244,7 +233,6 @@ public interface LROp {
MQEValue valueR = mqeValuesR.getValues().get(i);
if (valueL.isEmptyValue() || valueR.isEmptyValue()) {
valueL.setEmptyValue(true);
valueL.setDoubleValue(0);
continue;
}
double newValue = calculate.apply(valueL.getDoubleValue(), valueR.getDoubleValue(), opType);
@ -259,30 +247,25 @@ public interface LROp {
ExpressionResult seriesRight,
int opType, LROp calculate) throws IllegalExpressionException {
Map<Set<KeyValue>, List<MQEValue>> labelMapR = new HashMap<>();
if (seriesLeft.getResults().size() != seriesRight.getResults().size()) {
throw new IllegalExpressionException(
"Operation between labeled metrics should have the same label.");
}
seriesRight.getResults().forEach(mqeValuesR -> {
labelMapR.put(new HashSet<>(mqeValuesR.getMetric().getLabels()), mqeValuesR.getValues());
});
for (MQEValues mqeValuesL : seriesLeft.getResults()) {
for (int i = 0; i < mqeValuesL.getValues().size(); i++) {
//reserve left metric info
//reserve left metric info, if right metric not exist, set empty value
MQEValue valueL = mqeValuesL.getValues().get(i);
List<MQEValue> mqeValuesR = labelMapR.get(new HashSet<>(mqeValuesL.getMetric().getLabels()));
if (mqeValuesR == null) {
throw new IllegalExpressionException(
"Operation between labeled metrics should have the same label.");
}
MQEValue valueR = mqeValuesR.get(i);
if (valueL.isEmptyValue() || valueR.isEmptyValue()) {
valueL.setEmptyValue(true);
valueL.setDoubleValue(0);
continue;
} else {
MQEValue valueR = mqeValuesR.get(i);
if (valueL.isEmptyValue() || valueR.isEmptyValue()) {
valueL.setEmptyValue(true);
continue;
}
double newValue = calculate.apply(valueL.getDoubleValue(), valueR.getDoubleValue(), opType);
mqeValuesL.getValues().get(i).setDoubleValue(newValue);
}
double newValue = calculate.apply(valueL.getDoubleValue(), valueR.getDoubleValue(), opType);
mqeValuesL.getValues().get(i).setDoubleValue(newValue);
}
}

2
oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/operation/LogicalFunctionOp.java
View File

@ -25,7 +25,6 @@ import org.apache.skywalking.mqe.rt.type.ExpressionResult;
import org.apache.skywalking.mqe.rt.type.ExpressionResultType;
import org.apache.skywalking.mqe.rt.type.MQEValue;
import org.apache.skywalking.mqe.rt.type.MQEValues;
import org.apache.skywalking.mqe.rt.type.Metadata;
import org.apache.skywalking.oap.server.library.util.CollectionUtils;
import java.util.Objects;
@ -85,7 +84,6 @@ public class LogicalFunctionOp {
MQEValue mqeValue = new MQEValue();
MQEValues mqeValues = new MQEValues();
mqeValues.setMetric(new Metadata());
mqeValues.getValues().add(mqeValue);
result.getResults().add(mqeValues);
mqeValue.setDoubleValue(present ? 1 : 0);

2
oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/type/MQEValues.java
View File

@ -25,7 +25,7 @@ import lombok.Data;
@Data
public class MQEValues {
private Metadata metric;
private Metadata metric = new Metadata();
private List<MQEValue> values = new ArrayList<>();
}

5
oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/type/Metadata.java
View File

@ -20,6 +20,7 @@
package org.apache.skywalking.mqe.rt.type;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import lombok.Data;
import org.apache.skywalking.oap.server.core.query.type.KeyValue;
@ -27,4 +28,8 @@ import org.apache.skywalking.oap.server.core.query.type.KeyValue;
@Data
public class Metadata {
private List<KeyValue> labels = new ArrayList<>();
public void sortLabelsByKey(Comparator<String> comparator) {
labels.sort(Comparator.comparing(KeyValue::getKey, comparator));
}
}

29
oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/expr/rt/AlarmMQEVerifyVisitor.java
View File

@ -18,9 +18,7 @@
package org.apache.skywalking.oap.server.core.alarm.provider.expr.rt;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import lombok.Getter;
@ -31,14 +29,9 @@ import org.apache.skywalking.mqe.rt.type.ExpressionResult;
import org.apache.skywalking.mqe.rt.type.ExpressionResultType;
import org.apache.skywalking.mqe.rt.type.MQEValue;
import org.apache.skywalking.mqe.rt.type.MQEValues;
import org.apache.skywalking.mqe.rt.type.Metadata;
import org.apache.skywalking.oap.server.core.query.enumeration.Step;
import org.apache.skywalking.oap.server.core.query.type.KeyValue;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
import org.apache.skywalking.oap.server.library.util.CollectionUtils;
import static org.apache.skywalking.oap.server.core.analysis.metrics.DataLabel.GENERAL_LABEL_NAME;
/**
* Used for verify the alarm expression and get the metrics name when read the alarm rules.
@ -78,29 +71,11 @@ public class AlarmMQEVerifyVisitor extends MQEVisitorBase {
MQEValue mqeValue = new MQEValue();
mqeValue.setEmptyValue(true);
mockMqeValues.getValues().add(mqeValue);
result.getResults().add(mockMqeValues);
result.setType(ExpressionResultType.TIME_SERIES_VALUES);
if (dataType == Column.ValueDataType.COMMON_VALUE) {
result.getResults().add(mockMqeValues);
result.setType(ExpressionResultType.TIME_SERIES_VALUES);
return result;
} else if (dataType == Column.ValueDataType.LABELED_VALUE) {
List<KeyValue> queryLabels = buildLabels(ctx.labelList());
ArrayList<MQEValues> mqeValuesList = new ArrayList<>();
if (CollectionUtils.isEmpty(queryLabels)) {
KeyValue label = new KeyValue(GENERAL_LABEL_NAME, GENERAL_LABEL_NAME);
Metadata metadata = new Metadata();
metadata.getLabels().add(label);
mockMqeValues.setMetric(metadata);
mqeValuesList.add(mockMqeValues);
} else {
for (KeyValue queryLabel : queryLabels) {
Metadata metadata = new Metadata();
metadata.getLabels().add(queryLabel);
mockMqeValues.setMetric(metadata);
mqeValuesList.add(mockMqeValues);
}
}
result.setType(ExpressionResultType.TIME_SERIES_VALUES);
result.setResults(mqeValuesList);
result.setLabeledResult(true);
return result;
} else {

34
oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/expr/rt/AlarmMQEVisitor.java
View File

@ -19,6 +19,7 @@
package org.apache.skywalking.oap.server.core.alarm.provider.expr.rt;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@ -30,7 +31,6 @@ import org.apache.skywalking.mqe.rt.type.ExpressionResult;
import org.apache.skywalking.mqe.rt.type.ExpressionResultType;
import org.apache.skywalking.mqe.rt.type.MQEValue;
import org.apache.skywalking.mqe.rt.type.MQEValues;
import org.apache.skywalking.mqe.rt.type.Metadata;
import org.apache.skywalking.oap.server.core.analysis.metrics.DataLabel;
import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
import org.apache.skywalking.oap.server.core.analysis.metrics.DoubleValueHolder;
@ -46,7 +46,6 @@ import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetad
import org.apache.skywalking.oap.server.library.util.CollectionUtils;
import org.joda.time.LocalDateTime;
import static org.apache.skywalking.oap.server.core.analysis.metrics.DataLabel.GENERAL_LABEL_NAME;
import static org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO.Util.composeLabelConditions;
@Slf4j
@ -99,7 +98,7 @@ public class AlarmMQEVisitor extends MQEVisitorBase {
List<KeyValue> queryLabels = buildLabels(ctx.labelList());
Map<String, DataTable> timeValues = labeledValuesMap.get(metricName);
if (CollectionUtils.isEmpty(timeValues)) {
mqeValuesList = buildEmptyLabeledMQEValuesList(queryLabels);
mqeValuesList = buildEmptyMQEValuesList();
} else {
mqeValuesList = buildLabledMqeValuesList(timeValues, queryLabels);
}
@ -191,7 +190,6 @@ public class AlarmMQEVisitor extends MQEVisitorBase {
List<MQEValues> mqeValuesList = new ArrayList<>();
MQEValues mqeValues = new MQEValues();
for (String time : windowTimes) {
Metadata metadata = new Metadata();
Double metricValue = timeValues.get(time);
MQEValue mqeValue = new MQEValue();
//use timeBucket as id here
@ -201,7 +199,6 @@ public class AlarmMQEVisitor extends MQEVisitorBase {
} else {
mqeValue.setEmptyValue(true);
}
mqeValues.setMetric(metadata);
mqeValues.getValues().add(mqeValue);
}
mqeValuesList.add(mqeValues);
@ -215,7 +212,6 @@ public class AlarmMQEVisitor extends MQEVisitorBase {
for (String labelCondition : labelConditions) {
MQEValues mqeValues = new MQEValues();
for (String time : windowTimes) {
Metadata metadata = new Metadata();
DataTable dataTable = timeValues.getOrDefault(time, new DataTable());
Long metricValue = dataTable.get(labelCondition);
MQEValue mqeValue = new MQEValue();
@ -229,11 +225,12 @@ public class AlarmMQEVisitor extends MQEVisitorBase {
DataLabel dataLabel = new DataLabel();
dataLabel.put(labelCondition);
for (Map.Entry<String, String> label : dataLabel.entrySet()) {
metadata.getLabels().add(new KeyValue(label.getKey(), label.getValue()));
mqeValues.getMetric().getLabels().add(new KeyValue(label.getKey(), label.getValue()));
}
mqeValues.setMetric(metadata);
mqeValues.getValues().add(mqeValue);
}
//Sort labels by key in natural order by default
mqeValues.getMetric().sortLabelsByKey(Comparator.naturalOrder());
mqeValuesList.add(mqeValues);
}
return mqeValuesList;
@ -256,25 +253,4 @@ public class AlarmMQEVisitor extends MQEVisitorBase {
mqeValuesList.add(initMQEValues());
return mqeValuesList;
}
private ArrayList<MQEValues> buildEmptyLabeledMQEValuesList(List<KeyValue> queryLabels) {
ArrayList<MQEValues> mqeValuesList = new ArrayList<>();
if (CollectionUtils.isEmpty(queryLabels)) {
MQEValues mqeValues = initMQEValues();
KeyValue label = new KeyValue(GENERAL_LABEL_NAME, GENERAL_LABEL_NAME);
Metadata metadata = new Metadata();
metadata.getLabels().add(label);
mqeValues.setMetric(metadata);
mqeValuesList.add(mqeValues);
} else {
for (KeyValue label : queryLabels) {
MQEValues mqeValues = initMQEValues();
Metadata metadata = new Metadata();
metadata.getLabels().add(label);
mqeValues.setMetric(metadata);
mqeValuesList.add(mqeValues);
}
}
return mqeValuesList;
}
}

16
oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/mqe/rt/MQEVisitor.java
View File

@ -20,6 +20,7 @@ package org.apache.skywalking.oap.query.graphql.mqe.rt;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@ -28,7 +29,6 @@ import org.apache.skywalking.mqe.rt.exception.IllegalExpressionException;
import org.apache.skywalking.mqe.rt.grammar.MQEParser;
import org.apache.skywalking.mqe.rt.type.MQEValue;
import org.apache.skywalking.mqe.rt.type.MQEValues;
import org.apache.skywalking.mqe.rt.type.Metadata;
import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.analysis.metrics.DataLabel;
import org.apache.skywalking.oap.server.core.query.AggregationQueryService;
@ -195,10 +195,8 @@ public class MQEVisitor extends MQEVisitorBase {
mqeValue.setDoubleValue(Double.parseDouble(selectedRecord.getValue()));
mqeValueList.add(mqeValue);
});
Metadata metadata = new Metadata();
MQEValues mqeValues = new MQEValues();
mqeValues.setValues(mqeValueList);
mqeValues.setMetric(metadata);
result.getResults().add(mqeValues);
result.setType(ExpressionResultType.SORTED_LIST);
}
@ -220,10 +218,8 @@ public class MQEVisitor extends MQEVisitorBase {
mqeValue.setTraceID(record.getRefId());
mqeValueList.add(mqeValue);
});
Metadata metadata = new Metadata();
MQEValues mqeValues = new MQEValues();
mqeValues.setValues(mqeValueList);
mqeValues.setMetric(metadata);
result.getResults().add(mqeValues);
result.setType(ExpressionResultType.RECORD_LIST);
}
@ -250,10 +246,8 @@ public class MQEVisitor extends MQEVisitorBase {
mqeValue.setDoubleValue(kvInt.getValue());
mqeValueList.add(mqeValue);
}
Metadata metadata = new Metadata();
MQEValues mqeValues = new MQEValues();
mqeValues.setValues(mqeValueList);
mqeValues.setMetric(metadata);
result.getResults().add(mqeValues);
result.setType(ExpressionResultType.TIME_SERIES_VALUES);
}
@ -288,15 +282,15 @@ public class MQEVisitor extends MQEVisitorBase {
}
}
Metadata metadata = new Metadata();
MQEValues mqeValues = new MQEValues();
DataLabel dataLabel = new DataLabel();
dataLabel.put(metricsValues.getLabel());
for (Map.Entry<String, String> label : dataLabel.entrySet()) {
metadata.getLabels().add(new KeyValue(label.getKey(), label.getValue()));
mqeValues.getMetric().getLabels().add(new KeyValue(label.getKey(), label.getValue()));
}
MQEValues mqeValues = new MQEValues();
//Sort labels by key in natural order by default
mqeValues.getMetric().sortLabelsByKey(Comparator.naturalOrder());
mqeValues.setValues(mqeValueList);
mqeValues.setMetric(metadata);
result.getResults().add(mqeValues);
});
result.setType(ExpressionResultType.TIME_SERIES_VALUES);

2
oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol

@ -1 +1 @@
Subproject commit f0bdba688afecff2167d018ecdaf509d2c8598f7
Subproject commit f74557b29dc7bef2830786c4fe3e39e4a0919c98