kazusa
/
skywalking-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Query all of labels once selected labels condition is absent. (#5202)

This commit is contained in:
Gao Hongtao 2020-08-08 22:26:06 +08:00 committed by GitHub
parent 9b6a4ed28c
commit b0bb7cf3be
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 195 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java
View File

@ -19,7 +19,14 @@
package org.apache.skywalking.oap.server.core.storage.query;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.TreeSet;
import java.util.stream.Collectors;
import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
import org.apache.skywalking.oap.server.core.query.input.Duration;
import org.apache.skywalking.oap.server.core.query.input.MetricsCondition;
import org.apache.skywalking.oap.server.core.query.type.HeatMap;
@ -27,6 +34,9 @@ import org.apache.skywalking.oap.server.core.query.type.IntValues;
import org.apache.skywalking.oap.server.core.query.type.KVInt;
import org.apache.skywalking.oap.server.core.query.type.MetricsValues;
import org.apache.skywalking.oap.server.core.storage.DAO;
import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
import static java.util.stream.Collectors.toList;
/**
* Query metrics values in different ways.
@ -76,5 +86,56 @@ public interface IMetricsQueryDAO extends DAO {
}
return origin;
}
/**
* Compose the multiple metric result based on conditions.
*/
public static List<MetricsValues> composeLabelValue(final MetricsCondition condition,
final List<String> labels,
final List<String> ids,
final Map<String, DataTable> idMap) {
List<String> allLabels;
if (Objects.isNull(labels) || labels.size() < 1) {
allLabels = idMap.values().stream()
.flatMap(dataTable -> dataTable.keys().stream())
.distinct().collect(Collectors.toList());
} else {
allLabels = labels;
}
final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
List<LabeledValue> labeledValues = new TreeSet<>(allLabels).stream()
.flatMap(label -> ids.stream().map(id ->
new LabeledValue(
label,
id,
Optional.ofNullable(idMap.getOrDefault(id, new DataTable()).get(label)).orElse((long) defaultValue))))
.collect(toList());
MetricsValues current = new MetricsValues();
List<MetricsValues> result = new ArrayList<>();
for (LabeledValue each : labeledValues) {
if (Objects.equals(current.getLabel(), each.label)) {
current.getValues().addKVInt(each.kv);
} else {
current = new MetricsValues();
current.setLabel(each.label);
current.getValues().addKVInt(each.kv);
result.add(current);
}
}
return result;
}
}
class LabeledValue {
private final String label;
private final KVInt kv;
public LabeledValue(String label, String id, long value) {
this.label = label;
KVInt kv = new KVInt();
kv.setId(id);
kv.setValue(value);
this.kv = kv;
}
}
}

123
oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/query/MetricsQueryUtilTest.java Normal file
View File

@ -0,0 +1,123 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.storage.query;
import com.google.gson.Gson;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import lombok.RequiredArgsConstructor;
import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
import org.apache.skywalking.oap.server.core.query.input.MetricsCondition;
import org.apache.skywalking.oap.server.core.query.sql.Function;
import org.apache.skywalking.oap.server.core.query.type.MetricsValues;
import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import static com.google.common.collect.ImmutableMap.of;
import static java.util.Arrays.asList;
import static org.apache.skywalking.oap.server.core.storage.annotation.Column.ValueDataType.LABELED_VALUE;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.Is.is;
@RunWith(Parameterized.class)
@RequiredArgsConstructor
public class MetricsQueryUtilTest {
private static final int DEFAULT_VALUE = -1;
private static final String MODULE_NAME = "meter-test";
private final List<String> queryConditionLabels;
private final List<String> datePoints;
private final Map<String, DataTable> valueColumnData;
private final String expectedResult;
@Parameterized.Parameters
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][] {
{
asList("200", "400"),
asList("202007291425", "202007291426"),
of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")),
"[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," +
"{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}]"
},
{
asList("400", "200"),
asList("202007291425", "202007291426"),
of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")),
"[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," +
"{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}]"
},
{
Collections.emptyList(),
asList("202007291425", "202007291426"),
of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")),
"[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," +
"{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}]"
},
{
Collections.singletonList("200"),
asList("202007291425", "202007291426"),
of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")),
"[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}]"
},
{
asList("200", "400", "500"),
asList("202007291425", "202007291426"),
of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")),
"[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," +
"{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}," +
"{\"label\":\"500\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":" + DEFAULT_VALUE + "},{\"id\":\"202007291426\",\"value\":" + DEFAULT_VALUE + "}]}}]"
},
{
asList("200", "400"),
asList("202007291425", "202007291426"),
of("202007291425", new DataTable("200,1|400,2")),
"[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":" + DEFAULT_VALUE + "}]}}," +
"{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":" + DEFAULT_VALUE + "}]}}]"
},
});
}
@Before
public void setup() {
ValueColumnMetadata.INSTANCE.putIfAbsent(
MODULE_NAME, "value", LABELED_VALUE, Function.None, DEFAULT_VALUE
);
}
@Test
public void testComposeLabelValue() {
MetricsCondition condition = new MetricsCondition();
condition.setName(MODULE_NAME);
List<MetricsValues> result = IMetricsQueryDAO.Util.composeLabelValue(condition, queryConditionLabels, datePoints, valueColumnData);
assertThat(new Gson().toJson(result), is(expectedResult));
}
}

40
oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java
View File

@ -138,42 +138,12 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO {
});
SearchResponse response = getClient().ids(condition.getName(), ids.toArray(new String[0]));
Map<String, Map<String, Object>> idMap = toMap(response);
Map<String, MetricsValues> labeledValues = new HashMap<>(labels.size());
labels.forEach(label -> {
MetricsValues labelValue = new MetricsValues();
labelValue.setLabel(label);
labeledValues.put(label, labelValue);
});
final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
for (String id : ids) {
if (idMap.containsKey(id)) {
Map<String, Object> source = idMap.get(id);
DataTable multipleValues = new DataTable((String) source.getOrDefault(valueColumnName, ""));
labels.forEach(label -> {
final IntValues values = labeledValues.get(label).getValues();
Long data = multipleValues.get(label);
if (data == null) {
data = (long) defaultValue;
}
KVInt kv = new KVInt();
kv.setId(id);
kv.setValue(data);
values.addKVInt(kv);
});
}
Map<String, DataTable> idMap = new HashMap<>();
SearchHit[] hits = response.getHits().getHits();
for (SearchHit hit : hits) {
idMap.put(hit.getId(), new DataTable((String) hit.getSourceAsMap().getOrDefault(valueColumnName, "")));
}
return Util.sortValues(
new ArrayList<>(labeledValues.values()),
ids,
defaultValue
);
return Util.composeLabelValue(condition, labels, ids, idMap);
}
@Override

30
oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java
View File

@ -175,40 +175,16 @@ public class MetricsQuery implements IMetricsQueryDAO {
log.debug("SQL: {} result set: {}", query.getCommand(), series);
}
Map<String, MetricsValues> labeledValues = new HashMap<>(labels.size());
labels.forEach(label -> {
MetricsValues labelValue = new MetricsValues();
labelValue.setLabel(label);
labeledValues.put(label, labelValue);
});
final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
Map<String, DataTable> idMap = new HashMap<>();
if (!CollectionUtils.isEmpty(series)) {
series.get(0).getValues().forEach(values -> {
final String id = (String) values.get(1);
DataTable multipleValues = new DataTable(5);
multipleValues.toObject((String) values.get(2));
labels.forEach(label -> {
Long data = multipleValues.get(label);
if (data == null) {
data = (long) defaultValue;
}
final IntValues intValues = labeledValues.get(label).getValues();
KVInt kv = new KVInt();
kv.setId(id);
kv.setValue(data);
intValues.addKVInt(kv);
});
idMap.put(id, multipleValues);
});
}
return Util.sortValues(
new ArrayList<>(labeledValues.values()),
ids,
defaultValue
);
return Util.composeLabelValue(condition, labels, ids, idMap);
}
@Override

30
oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java
View File

@ -162,16 +162,7 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO
}
sql.append(")");
Map<String, MetricsValues> labeledValues = new HashMap<>(labels.size());
labels.forEach(label -> {
MetricsValues labelValue = new MetricsValues();
labelValue.setLabel(label);
labeledValues.put(label, labelValue);
});
final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
Map<String, DataTable> idMap = new HashMap<>();
try (Connection connection = h2Client.getConnection()) {
try (ResultSet resultSet = h2Client.executeQuery(
connection, sql.toString(), parameters.toArray(new Object[0]))) {
@ -181,28 +172,13 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO
DataTable multipleValues = new DataTable(5);
multipleValues.toObject(resultSet.getString(valueColumnName));
labels.forEach(label -> {
Long data = multipleValues.get(label);
if (data == null) {
data = (long) defaultValue;
}
final IntValues values = labeledValues.get(label).getValues();
KVInt kv = new KVInt();
kv.setId(id);
kv.setValue(data);
values.addKVInt(kv);
});
idMap.put(id, multipleValues);
}
}
} catch (SQLException e) {
throw new IOException(e);
}
return Util.sortValues(
new ArrayList<>(labeledValues.values()),
ids,
defaultValue
);
return Util.composeLabelValue(condition, labels, ids, idMap);
}
@Override