From 5b0419b7c2d5cbc4f913bdc5e4dc013c82eaafba Mon Sep 17 00:00:00 2001 From: Jared Tan Date: Fri, 26 Apr 2019 23:39:40 +0800 Subject: [PATCH] support es query segment max size. (#2540) * support es query span max size. * refactoring segmentQueryMaxSize. * fix * fix --- docker/config/application.yml | 1 + .../src/main/assembly/application.yml | 1 + .../src/main/resources/application.yml | 1 + .../StorageModuleElasticsearchConfig.java | 12 +++++++++- .../StorageModuleElasticsearchProvider.java | 2 +- .../elasticsearch/query/TraceQueryEsDAO.java | 23 +++++++++++++++---- 6 files changed, 33 insertions(+), 7 deletions(-) diff --git a/docker/config/application.yml b/docker/config/application.yml index 2e145a5cda..d1c9090923 100644 --- a/docker/config/application.yml +++ b/docker/config/application.yml @@ -68,6 +68,7 @@ storage: flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests metadataQueryMaxSize: ${SW_STORAGE_ES_QUERY_MAX_SIZE:5000} + segmentQueryMaxSize: ${SW_STORAGE_ES_QUERY_SEGMENT_SIZE:200} # h2: # driver: ${SW_STORAGE_H2_DRIVER:org.h2.jdbcx.JdbcDataSource} # url: ${SW_STORAGE_H2_URL:jdbc:h2:mem:skywalking-oap-db} diff --git a/oap-server/server-starter/src/main/assembly/application.yml b/oap-server/server-starter/src/main/assembly/application.yml index 6ef2025bed..1f7a8cfe7a 100644 --- a/oap-server/server-starter/src/main/assembly/application.yml +++ b/oap-server/server-starter/src/main/assembly/application.yml @@ -68,6 +68,7 @@ storage: # flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests # concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests # metadataQueryMaxSize: ${SW_STORAGE_ES_QUERY_MAX_SIZE:5000} +# segmentQueryMaxSize: ${SW_STORAGE_ES_QUERY_SEGMENT_SIZE:200} h2: driver: ${SW_STORAGE_H2_DRIVER:org.h2.jdbcx.JdbcDataSource} url: ${SW_STORAGE_H2_URL:jdbc:h2:mem:skywalking-oap-db} diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 460d36816b..4c6d2325c2 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -68,6 +68,7 @@ storage: flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests metadataQueryMaxSize: ${SW_STORAGE_ES_QUERY_MAX_SIZE:5000} + segmentQueryMaxSize: ${SW_STORAGE_ES_QUERY_SEGMENT_SIZE:200} # h2: # driver: ${SW_STORAGE_H2_DRIVER:org.h2.jdbcx.JdbcDataSource} # url: ${SW_STORAGE_H2_URL:jdbc:h2:mem:skywalking-oap-db} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java index 8bc741d9fd..dffce36b1b 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java @@ -18,7 +18,8 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch; -import lombok.*; +import lombok.Getter; +import lombok.Setter; import org.apache.skywalking.oap.server.library.module.ModuleConfig; /** @@ -43,6 +44,7 @@ public class StorageModuleElasticsearchConfig extends ModuleConfig { private String user; private String password; private int metadataQueryMaxSize = 5000; + private int segmentQueryMaxSize = 200; public String getUser() { return user; @@ -163,4 +165,12 @@ public class StorageModuleElasticsearchConfig extends ModuleConfig { public void setMetadataQueryMaxSize(int metadataQueryMaxSize) { this.metadataQueryMaxSize = metadataQueryMaxSize; } + + public int getSegmentQueryMaxSize() { + return segmentQueryMaxSize; + } + + public void setSegmentQueryMaxSize(int segmentQueryMaxSize) { + this.segmentQueryMaxSize = segmentQueryMaxSize; + } } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java index f65fdeaeba..a779921154 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java @@ -115,7 +115,7 @@ public class StorageModuleElasticsearchProvider extends ModuleProvider { this.registerServiceImplementation(ITopologyQueryDAO.class, new TopologyQueryEsDAO(elasticSearchClient)); this.registerServiceImplementation(IMetricQueryDAO.class, new MetricQueryEsDAO(elasticSearchClient)); - this.registerServiceImplementation(ITraceQueryDAO.class, new TraceQueryEsDAO(elasticSearchClient)); + this.registerServiceImplementation(ITraceQueryDAO.class, new TraceQueryEsDAO(elasticSearchClient, config.getSegmentQueryMaxSize())); this.registerServiceImplementation(IMetadataQueryDAO.class, new MetadataQueryEsDAO(elasticSearchClient, config.getMetadataQueryMaxSize())); this.registerServiceImplementation(IAggregationQueryDAO.class, new AggregationQueryEsDAO(elasticSearchClient)); this.registerServiceImplementation(IAlarmQueryDAO.class, new AlarmQueryEsDAO(elasticSearchClient)); diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TraceQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TraceQueryEsDAO.java index 4528189f1c..8a31bf7f2c 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TraceQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TraceQueryEsDAO.java @@ -20,15 +20,25 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; import com.google.common.base.Strings; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.List; import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord; -import org.apache.skywalking.oap.server.core.query.entity.*; +import org.apache.skywalking.oap.server.core.query.entity.BasicTrace; +import org.apache.skywalking.oap.server.core.query.entity.QueryOrder; +import org.apache.skywalking.oap.server.core.query.entity.Span; +import org.apache.skywalking.oap.server.core.query.entity.TraceBrief; +import org.apache.skywalking.oap.server.core.query.entity.TraceState; import org.apache.skywalking.oap.server.core.storage.query.ITraceQueryDAO; import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; import org.apache.skywalking.oap.server.library.util.BooleanUtils; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.index.query.*; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; @@ -38,8 +48,11 @@ import org.elasticsearch.search.sort.SortOrder; */ public class TraceQueryEsDAO extends EsDAO implements ITraceQueryDAO { - public TraceQueryEsDAO(ElasticSearchClient client) { + private int segmentQueryMaxSize; + + public TraceQueryEsDAO(ElasticSearchClient client, int segmentQueryMaxSize) { super(client); + this.segmentQueryMaxSize = segmentQueryMaxSize; } @Override @@ -123,7 +136,7 @@ public class TraceQueryEsDAO extends EsDAO implements ITraceQueryDAO { @Override public List queryByTraceId(String traceId) throws IOException { SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource(); sourceBuilder.query(QueryBuilders.termQuery(SegmentRecord.TRACE_ID, traceId)); - sourceBuilder.size(20); + sourceBuilder.size(segmentQueryMaxSize); SearchResponse response = getClient().search(SegmentRecord.INDEX_NAME, sourceBuilder);