kazusa
/
skywalking-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use the ids query method instead of get method. (#2887) 

#2886
This commit is contained in:
彭勇升 pengys 2019-06-17 12:05:21 +08:00 committed by 吴晟 Wu Sheng
parent a70a2b236e
commit 70064f47cb
6 changed files with 43 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/ID.java
View File

@ -1,49 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.skywalking.oap.server.core.query;
import lombok.Getter;
import org.apache.skywalking.oap.server.core.Const;
/**
* @author peng-yongsheng
*/
@Getter
public class ID {
private final long timeBucket;
private final String entityId;
public ID(long timeBucket, String entityId) {
this.timeBucket = timeBucket;
this.entityId = entityId;
}
public ID(long timeBucket) {
this.timeBucket = timeBucket;
this.entityId = Const.EMPTY_STRING;
}
@Override public String toString() {
if (entityId.length() > 0) {
return timeBucket + Const.ID_SPLIT + entityId;
} else {
return String.valueOf(timeBucket);
}
}
}

13
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetricQueryService.java
View File

@ -22,6 +22,7 @@ import java.io.IOException;
import java.text.ParseException;
import java.util.*;
import org.apache.skywalking.apm.util.StringUtil;
import org.apache.skywalking.oap.server.core.Const;
import org.apache.skywalking.oap.server.core.analysis.Downsampling;
import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.query.entity.*;
@ -70,11 +71,11 @@ public class MetricQueryService implements Service {
public IntValues getLinearIntValues(final String indName, final String id, final Downsampling downsampling, final long startTB,
final long endTB) throws IOException, ParseException {
List<DurationPoint> durationPoints = DurationUtils.INSTANCE.getDurationPoints(downsampling, startTB, endTB);
List<ID> ids = new ArrayList<>();
List<String> ids = new ArrayList<>();
if (StringUtil.isEmpty(id)) {
durationPoints.forEach(durationPoint -> ids.add(new ID(durationPoint.getPoint())));
durationPoints.forEach(durationPoint -> ids.add(String.valueOf(durationPoint.getPoint())));
} else {
durationPoints.forEach(durationPoint -> ids.add(new ID(durationPoint.getPoint(), id)));
durationPoints.forEach(durationPoint -> ids.add(durationPoint.getPoint() + Const.ID_SPLIT + id));
}
return getMetricQueryDAO().getLinearIntValues(indName, downsampling, ids, ValueColumnIds.INSTANCE.getValueCName(indName));
@ -83,12 +84,12 @@ public class MetricQueryService implements Service {
public Thermodynamic getThermodynamic(final String indName, final String id, final Downsampling downsampling, final long startTB,
final long endTB) throws IOException, ParseException {
List<DurationPoint> durationPoints = DurationUtils.INSTANCE.getDurationPoints(downsampling, startTB, endTB);
List<ID> ids = new ArrayList<>();
List<String> ids = new ArrayList<>();
durationPoints.forEach(durationPoint -> {
if (id == null) {
ids.add(new ID(durationPoint.getPoint()));
ids.add(String.valueOf(durationPoint.getPoint()));
} else {
ids.add(new ID(durationPoint.getPoint(), id));
ids.add(durationPoint.getPoint() + Const.ID_SPLIT + id);
}
});

5
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java
View File

@ -21,7 +21,6 @@ package org.apache.skywalking.oap.server.core.storage.query;
import java.io.IOException;
import java.util.List;
import org.apache.skywalking.oap.server.core.analysis.Downsampling;
import org.apache.skywalking.oap.server.core.query.ID;
import org.apache.skywalking.oap.server.core.query.entity.*;
import org.apache.skywalking.oap.server.core.query.sql.*;
import org.apache.skywalking.oap.server.core.storage.DAO;
@ -33,7 +32,7 @@ public interface IMetricsQueryDAO extends DAO {
IntValues getValues(String indName, Downsampling downsampling, long startTB, long endTB, Where where, String valueCName, Function function) throws IOException;
IntValues getLinearIntValues(String indName, Downsampling downsampling, List<ID> ids, String valueCName) throws IOException;
IntValues getLinearIntValues(String indName, Downsampling downsampling, List<String> ids, String valueCName) throws IOException;
Thermodynamic getThermodynamic(String indName, Downsampling downsampling, List<ID> ids, String valueCName) throws IOException;
Thermodynamic getThermodynamic(String indName, Downsampling downsampling, List<String> ids, String valueCName) throws IOException;
}

18
oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java
View File

@ -42,6 +42,8 @@ import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.*;
import org.elasticsearch.common.unit.*;
import org.elasticsearch.common.xcontent.*;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.slf4j.*;
@ -216,6 +218,22 @@ public class ElasticSearchClient implements Client {
return client.get(request);
}
public Map<String, Map<String, Object>> ids(String indexName, String... ids) throws IOException {
indexName = formatIndexName(indexName);
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.types(TYPE);
searchRequest.source().query(QueryBuilders.idsQuery().addIds(ids));
SearchResponse response = client.search(searchRequest);
Map<String, Map<String, Object>> result = new HashMap<>();
SearchHit[] hits = response.getHits().getHits();
for (SearchHit hit : hits) {
result.put(hit.getId(), hit.getSourceAsMap());
}
return result;
}
public void forceInsert(String indexName, String id, XContentBuilder source) throws IOException {
IndexRequest request = prepareInsert(indexName, id, source);
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);

30
oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java
View File

@ -22,14 +22,12 @@ import java.io.IOException;
import java.util.*;
import org.apache.skywalking.oap.server.core.analysis.Downsampling;
import org.apache.skywalking.oap.server.core.analysis.metrics.*;
import org.apache.skywalking.oap.server.core.query.ID;
import org.apache.skywalking.oap.server.core.query.entity.*;
import org.apache.skywalking.oap.server.core.query.sql.*;
import org.apache.skywalking.oap.server.core.storage.model.ModelName;
import org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO;
import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient;
import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.*;
import org.elasticsearch.action.get.GetResponse;
import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.*;
@ -101,19 +99,18 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO {
}
}
@Override public IntValues getLinearIntValues(String indName, Downsampling downsampling, List<ID> ids, String valueCName) throws IOException {
@Override public IntValues getLinearIntValues(String indName, Downsampling downsampling, List<String> ids, String valueCName) throws IOException {
String indexName = ModelName.build(downsampling, indName);
IntValues intValues = new IntValues();
for (ID id : ids) {
String modelName = TimeSeriesUtils.timeSeries(indexName, id.getTimeBucket(), downsampling);
GetResponse response = getClient().get(modelName, id.toString());
Map<String, Map<String, Object>> response = getClient().ids(indexName, ids.toArray(new String[0]));
IntValues intValues = new IntValues();
for (String id : ids) {
KVInt kvInt = new KVInt();
kvInt.setId(response.getId());
kvInt.setId(id);
kvInt.setValue(0);
Map<String, Object> source = response.getSource();
if (source != null) {
if (response.containsKey(id)) {
Map<String, Object> source = response.get(id);
kvInt.setValue(((Number)source.getOrDefault(valueCName, 0)).longValue());
}
intValues.getValues().add(kvInt);
@ -122,18 +119,17 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO {
return intValues;
}
@Override public Thermodynamic getThermodynamic(String indName, Downsampling downsampling, List<ID> ids, String valueCName) throws IOException {
@Override public Thermodynamic getThermodynamic(String indName, Downsampling downsampling, List<String> ids, String valueCName) throws IOException {
String indexName = ModelName.build(downsampling, indName);
Thermodynamic thermodynamic = new Thermodynamic();
List<List<Long>> thermodynamicValueMatrix = new ArrayList<>();
int numOfSteps = 0;
for (ID id : ids) {
String modelName = TimeSeriesUtils.timeSeries(indexName, id.getTimeBucket(), downsampling);
GetResponse response = getClient().get(modelName, id.toString());
Map<String, Map<String, Object>> response = getClient().ids(indexName, ids.toArray(new String[0]));
Map<String, Object> source = response.getSource();
int numOfSteps = 0;
for (String id : ids) {
Map<String, Object> source = response.get(id);
if (source == null) {
// add empty list to represent no data exist for this time bucket
thermodynamicValueMatrix.add(new ArrayList<>());

11
oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java
View File

@ -23,7 +23,6 @@ import java.sql.*;
import java.util.*;
import org.apache.skywalking.oap.server.core.analysis.Downsampling;
import org.apache.skywalking.oap.server.core.analysis.metrics.*;
import org.apache.skywalking.oap.server.core.query.ID;
import org.apache.skywalking.oap.server.core.query.entity.*;
import org.apache.skywalking.oap.server.core.query.sql.*;
import org.apache.skywalking.oap.server.core.storage.model.ModelName;
@ -101,7 +100,7 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO
return orderWithDefault0(intValues, ids);
}
@Override public IntValues getLinearIntValues(String indName, Downsampling downsampling, List<ID> ids, String valueCName) throws IOException {
@Override public IntValues getLinearIntValues(String indName, Downsampling downsampling, List<String> ids, String valueCName) throws IOException {
String tableName = ModelName.build(downsampling, indName);
StringBuilder idValues = new StringBuilder();
@ -127,11 +126,7 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO
throw new IOException(e);
}
List<String> idList = new ArrayList<>();
for (ID id : ids) {
idList.add(id.toString());
}
return orderWithDefault0(intValues, idList);
return orderWithDefault0(intValues, ids);
}
/**
@ -154,7 +149,7 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO
return intValues;
}
@Override public Thermodynamic getThermodynamic(String indName, Downsampling downsampling, List<ID> ids, String valueCName) throws IOException {
@Override public Thermodynamic getThermodynamic(String indName, Downsampling downsampling, List<String> ids, String valueCName) throws IOException {
String tableName = ModelName.build(downsampling, indName);
StringBuilder idValues = new StringBuilder();