From b0bb7cf3beb9caa0c8647227c38598b946d8e0c0 Mon Sep 17 00:00:00 2001 From: Gao Hongtao Date: Sat, 8 Aug 2020 22:26:06 +0800 Subject: [PATCH] Query all of labels once selected labels condition is absent. (#5202) --- .../core/storage/query/IMetricsQueryDAO.java | 61 +++++++++ .../storage/query/MetricsQueryUtilTest.java | 123 ++++++++++++++++++ .../query/MetricsQueryEsDAO.java | 40 +----- .../plugin/influxdb/query/MetricsQuery.java | 30 +---- .../plugin/jdbc/h2/dao/H2MetricsQueryDAO.java | 30 +---- 5 files changed, 195 insertions(+), 89 deletions(-) create mode 100644 oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/query/MetricsQueryUtilTest.java diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java index 79be8994c..112a1ea14 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java @@ -19,7 +19,14 @@ package org.apache.skywalking.oap.server.core.storage.query; import java.io.IOException; +import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.TreeSet; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable; import org.apache.skywalking.oap.server.core.query.input.Duration; import org.apache.skywalking.oap.server.core.query.input.MetricsCondition; import org.apache.skywalking.oap.server.core.query.type.HeatMap; @@ -27,6 +34,9 @@ import org.apache.skywalking.oap.server.core.query.type.IntValues; import org.apache.skywalking.oap.server.core.query.type.KVInt; import org.apache.skywalking.oap.server.core.query.type.MetricsValues; import org.apache.skywalking.oap.server.core.storage.DAO; +import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata; + +import static java.util.stream.Collectors.toList; /** * Query metrics values in different ways. @@ -76,5 +86,56 @@ public interface IMetricsQueryDAO extends DAO { } return origin; } + + /** + * Compose the multiple metric result based on conditions. + */ + public static List composeLabelValue(final MetricsCondition condition, + final List labels, + final List ids, + final Map idMap) { + List allLabels; + if (Objects.isNull(labels) || labels.size() < 1) { + allLabels = idMap.values().stream() + .flatMap(dataTable -> dataTable.keys().stream()) + .distinct().collect(Collectors.toList()); + } else { + allLabels = labels; + } + final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName()); + List labeledValues = new TreeSet<>(allLabels).stream() + .flatMap(label -> ids.stream().map(id -> + new LabeledValue( + label, + id, + Optional.ofNullable(idMap.getOrDefault(id, new DataTable()).get(label)).orElse((long) defaultValue)))) + .collect(toList()); + MetricsValues current = new MetricsValues(); + List result = new ArrayList<>(); + for (LabeledValue each : labeledValues) { + if (Objects.equals(current.getLabel(), each.label)) { + current.getValues().addKVInt(each.kv); + } else { + current = new MetricsValues(); + current.setLabel(each.label); + current.getValues().addKVInt(each.kv); + result.add(current); + } + } + return result; + } + } + + class LabeledValue { + private final String label; + private final KVInt kv; + + public LabeledValue(String label, String id, long value) { + this.label = label; + KVInt kv = new KVInt(); + kv.setId(id); + kv.setValue(value); + this.kv = kv; + } } } diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/query/MetricsQueryUtilTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/query/MetricsQueryUtilTest.java new file mode 100644 index 000000000..683db2084 --- /dev/null +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/query/MetricsQueryUtilTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.storage.query; + +import com.google.gson.Gson; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import lombok.RequiredArgsConstructor; +import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable; +import org.apache.skywalking.oap.server.core.query.input.MetricsCondition; +import org.apache.skywalking.oap.server.core.query.sql.Function; +import org.apache.skywalking.oap.server.core.query.type.MetricsValues; +import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import static com.google.common.collect.ImmutableMap.of; +import static java.util.Arrays.asList; +import static org.apache.skywalking.oap.server.core.storage.annotation.Column.ValueDataType.LABELED_VALUE; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; + +@RunWith(Parameterized.class) +@RequiredArgsConstructor +public class MetricsQueryUtilTest { + + private static final int DEFAULT_VALUE = -1; + + private static final String MODULE_NAME = "meter-test"; + + private final List queryConditionLabels; + + private final List datePoints; + + private final Map valueColumnData; + + private final String expectedResult; + + @Parameterized.Parameters + public static Collection data() { + return Arrays.asList(new Object[][] { + { + asList("200", "400"), + asList("202007291425", "202007291426"), + of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")), + "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," + + "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}]" + }, + { + asList("400", "200"), + asList("202007291425", "202007291426"), + of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")), + "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," + + "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}]" + }, + { + Collections.emptyList(), + asList("202007291425", "202007291426"), + of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")), + "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," + + "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}]" + }, + { + Collections.singletonList("200"), + asList("202007291425", "202007291426"), + of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")), + "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}]" + }, + { + asList("200", "400", "500"), + asList("202007291425", "202007291426"), + of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")), + "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," + + "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}," + + "{\"label\":\"500\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":" + DEFAULT_VALUE + "},{\"id\":\"202007291426\",\"value\":" + DEFAULT_VALUE + "}]}}]" + }, + { + asList("200", "400"), + asList("202007291425", "202007291426"), + of("202007291425", new DataTable("200,1|400,2")), + "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":" + DEFAULT_VALUE + "}]}}," + + "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":" + DEFAULT_VALUE + "}]}}]" + }, + }); + } + + @Before + public void setup() { + ValueColumnMetadata.INSTANCE.putIfAbsent( + MODULE_NAME, "value", LABELED_VALUE, Function.None, DEFAULT_VALUE + ); + } + + @Test + public void testComposeLabelValue() { + MetricsCondition condition = new MetricsCondition(); + condition.setName(MODULE_NAME); + List result = IMetricsQueryDAO.Util.composeLabelValue(condition, queryConditionLabels, datePoints, valueColumnData); + assertThat(new Gson().toJson(result), is(expectedResult)); + } + +} \ No newline at end of file diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java index c49ca9aa8..9e43af3b0 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java @@ -138,42 +138,12 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO { }); SearchResponse response = getClient().ids(condition.getName(), ids.toArray(new String[0])); - Map> idMap = toMap(response); - - Map labeledValues = new HashMap<>(labels.size()); - labels.forEach(label -> { - MetricsValues labelValue = new MetricsValues(); - labelValue.setLabel(label); - - labeledValues.put(label, labelValue); - }); - - final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName()); - for (String id : ids) { - if (idMap.containsKey(id)) { - Map source = idMap.get(id); - DataTable multipleValues = new DataTable((String) source.getOrDefault(valueColumnName, "")); - - labels.forEach(label -> { - final IntValues values = labeledValues.get(label).getValues(); - Long data = multipleValues.get(label); - if (data == null) { - data = (long) defaultValue; - } - KVInt kv = new KVInt(); - kv.setId(id); - kv.setValue(data); - values.addKVInt(kv); - }); - } - + Map idMap = new HashMap<>(); + SearchHit[] hits = response.getHits().getHits(); + for (SearchHit hit : hits) { + idMap.put(hit.getId(), new DataTable((String) hit.getSourceAsMap().getOrDefault(valueColumnName, ""))); } - - return Util.sortValues( - new ArrayList<>(labeledValues.values()), - ids, - defaultValue - ); + return Util.composeLabelValue(condition, labels, ids, idMap); } @Override diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java index f3eeb6117..5fa583218 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java @@ -175,40 +175,16 @@ public class MetricsQuery implements IMetricsQueryDAO { log.debug("SQL: {} result set: {}", query.getCommand(), series); } - Map labeledValues = new HashMap<>(labels.size()); - labels.forEach(label -> { - MetricsValues labelValue = new MetricsValues(); - labelValue.setLabel(label); - - labeledValues.put(label, labelValue); - }); - - final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName()); + Map idMap = new HashMap<>(); if (!CollectionUtils.isEmpty(series)) { series.get(0).getValues().forEach(values -> { final String id = (String) values.get(1); DataTable multipleValues = new DataTable(5); multipleValues.toObject((String) values.get(2)); - - labels.forEach(label -> { - Long data = multipleValues.get(label); - if (data == null) { - data = (long) defaultValue; - } - final IntValues intValues = labeledValues.get(label).getValues(); - KVInt kv = new KVInt(); - kv.setId(id); - kv.setValue(data); - intValues.addKVInt(kv); - }); + idMap.put(id, multipleValues); }); } - - return Util.sortValues( - new ArrayList<>(labeledValues.values()), - ids, - defaultValue - ); + return Util.composeLabelValue(condition, labels, ids, idMap); } @Override diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java index 84d0ca08b..f2b0f323b 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java @@ -162,16 +162,7 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO } sql.append(")"); - Map labeledValues = new HashMap<>(labels.size()); - labels.forEach(label -> { - MetricsValues labelValue = new MetricsValues(); - labelValue.setLabel(label); - - labeledValues.put(label, labelValue); - }); - - final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName()); - + Map idMap = new HashMap<>(); try (Connection connection = h2Client.getConnection()) { try (ResultSet resultSet = h2Client.executeQuery( connection, sql.toString(), parameters.toArray(new Object[0]))) { @@ -181,28 +172,13 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO DataTable multipleValues = new DataTable(5); multipleValues.toObject(resultSet.getString(valueColumnName)); - labels.forEach(label -> { - Long data = multipleValues.get(label); - if (data == null) { - data = (long) defaultValue; - } - final IntValues values = labeledValues.get(label).getValues(); - KVInt kv = new KVInt(); - kv.setId(id); - kv.setValue(data); - values.addKVInt(kv); - }); + idMap.put(id, multipleValues); } } } catch (SQLException e) { throw new IOException(e); } - - return Util.sortValues( - new ArrayList<>(labeledValues.values()), - ids, - defaultValue - ); + return Util.composeLabelValue(condition, labels, ids, idMap); } @Override