From 96b5b20398405438bcccef8aa039b020ec6a5bad Mon Sep 17 00:00:00 2001 From: peng-yongsheng <8082209@qq.com> Date: Wed, 28 Feb 2018 22:23:19 +0800 Subject: [PATCH 1/4] searchService: add the src span type condition by the value of SpanType.Entry_VALUE. --- .../storage/es/dao/ui/ServiceNameServiceEsUIDAO.java | 7 ++++++- .../storage/h2/dao/ui/ServiceNameServiceH2UIDAO.java | 6 +++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/apm-collector/apm-collector-storage/collector-storage-es-provider/src/main/java/org/apache/skywalking/apm/collector/storage/es/dao/ui/ServiceNameServiceEsUIDAO.java b/apm-collector/apm-collector-storage/collector-storage-es-provider/src/main/java/org/apache/skywalking/apm/collector/storage/es/dao/ui/ServiceNameServiceEsUIDAO.java index 3775bf281..341c2ea4b 100644 --- a/apm-collector/apm-collector-storage/collector-storage-es-provider/src/main/java/org/apache/skywalking/apm/collector/storage/es/dao/ui/ServiceNameServiceEsUIDAO.java +++ b/apm-collector/apm-collector-storage/collector-storage-es-provider/src/main/java/org/apache/skywalking/apm/collector/storage/es/dao/ui/ServiceNameServiceEsUIDAO.java @@ -29,6 +29,7 @@ import org.apache.skywalking.apm.network.proto.SpanType; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; @@ -58,7 +59,11 @@ public class ServiceNameServiceEsUIDAO extends EsDAO implements IServiceNameServ searchRequestBuilder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); searchRequestBuilder.setSize(topN); - searchRequestBuilder.setQuery(QueryBuilders.matchQuery(ServiceNameTable.COLUMN_SERVICE_NAME, keyword)); + BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); + boolQuery.must().add(QueryBuilders.matchQuery(ServiceNameTable.COLUMN_SERVICE_NAME, keyword)); + boolQuery.must().add(QueryBuilders.termQuery(ServiceNameTable.COLUMN_SRC_SPAN_TYPE, SpanType.Entry_VALUE)); + + searchRequestBuilder.setQuery(boolQuery); SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); SearchHit[] searchHits = searchResponse.getHits().getHits(); diff --git a/apm-collector/apm-collector-storage/collector-storage-h2-provider/src/main/java/org/apache/skywalking/apm/collector/storage/h2/dao/ui/ServiceNameServiceH2UIDAO.java b/apm-collector/apm-collector-storage/collector-storage-h2-provider/src/main/java/org/apache/skywalking/apm/collector/storage/h2/dao/ui/ServiceNameServiceH2UIDAO.java index 55c520b72..7bc2471dc 100644 --- a/apm-collector/apm-collector-storage/collector-storage-h2-provider/src/main/java/org/apache/skywalking/apm/collector/storage/h2/dao/ui/ServiceNameServiceH2UIDAO.java +++ b/apm-collector/apm-collector-storage/collector-storage-h2-provider/src/main/java/org/apache/skywalking/apm/collector/storage/h2/dao/ui/ServiceNameServiceH2UIDAO.java @@ -60,9 +60,9 @@ public class ServiceNameServiceH2UIDAO extends H2DAO implements IServiceNameServ } @Override public List searchService(String keyword, int topN) { - String dynamicSql = "select {0},{1} from {2} where {3} like ? limit ?"; - String sql = SqlBuilder.buildSql(dynamicSql, ServiceNameTable.COLUMN_SERVICE_ID, ServiceNameTable.COLUMN_SERVICE_NAME, ServiceNameTable.TABLE, ServiceNameTable.COLUMN_SERVICE_NAME); - Object[] params = new Object[] {keyword, topN}; + String dynamicSql = "select {0},{1} from {2} where {3} like ? and {4} = ? limit ?"; + String sql = SqlBuilder.buildSql(dynamicSql, ServiceNameTable.COLUMN_SERVICE_ID, ServiceNameTable.COLUMN_SERVICE_NAME, ServiceNameTable.TABLE, ServiceNameTable.COLUMN_SERVICE_NAME, ServiceNameTable.COLUMN_SRC_SPAN_TYPE); + Object[] params = new Object[] {keyword, SpanType.Entry_VALUE, topN}; List serviceInfos = new LinkedList<>(); try (ResultSet rs = getClient().executeQuery(sql, params)) { From c33ac107e21c1dad492384574d050aa0deb8bdfd Mon Sep 17 00:00:00 2001 From: peng-yongsheng <8082209@qq.com> Date: Wed, 28 Feb 2018 22:40:31 +0800 Subject: [PATCH 2/4] getServiceTPSTrend: 1. Change the parameter name: Start -> startTimeBucket End -> endTimeBucket 2. Calculate callsPerSec metric. --- .../apm/collector/ui/query/ServiceQuery.java | 15 +++++++++------ .../collector/ui/service/ServiceNameService.java | 16 +++++++++------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/query/ServiceQuery.java b/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/query/ServiceQuery.java index 09c17cccf..39b546731 100644 --- a/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/query/ServiceQuery.java +++ b/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/query/ServiceQuery.java @@ -65,16 +65,19 @@ public class ServiceQuery implements Query { } public ResponseTimeTrend getServiceResponseTimeTrend(int serviceId, Duration duration) throws ParseException { - long start = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart()); - long end = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd()); - return getServiceNameService().getServiceResponseTimeTrend(serviceId, duration.getStep(), start, end); + long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart()); + long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd()); + return getServiceNameService().getServiceResponseTimeTrend(serviceId, duration.getStep(), startTimeBucket, endTimeBucket); } public ThroughputTrend getServiceTPSTrend(int serviceId, Duration duration) throws ParseException { - long start = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart()); - long end = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd()); + long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart()); + long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd()); - return getServiceNameService().getServiceTPSTrend(serviceId, duration.getStep(), start, end); + long startSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getStart()); + long endSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getEnd()); + + return getServiceNameService().getServiceTPSTrend(serviceId, duration.getStep(), startTimeBucket, endTimeBucket, startSecondTimeBucket, endSecondTimeBucket); } public SLATrend getServiceSLATrend(int serviceId, Duration duration) throws ParseException { diff --git a/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/service/ServiceNameService.java b/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/service/ServiceNameService.java index 30cb8b6b6..c7a58c362 100644 --- a/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/service/ServiceNameService.java +++ b/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/service/ServiceNameService.java @@ -66,21 +66,23 @@ public class ServiceNameService { return serviceNameServiceUIDAO.searchService(keyword, topN); } - public ThroughputTrend getServiceTPSTrend(int serviceId, Step step, long start, long end) throws ParseException { + public ThroughputTrend getServiceTPSTrend(int serviceId, Step step, long startTimeBucket, + long endTimeBucket, long startSecondTimeBucket, long endSecondTimeBucket) throws ParseException { ThroughputTrend throughputTrend = new ThroughputTrend(); - List durationPoints = DurationUtils.INSTANCE.getDurationPoints(step, start, end); + List durationPoints = DurationUtils.INSTANCE.getDurationPoints(step, startTimeBucket, endTimeBucket); List callsTrends = serviceMetricUIDAO.getServiceTPSTrend(serviceId, step, durationPoints); - //TODO - callsTrends.forEach(calls -> throughputTrend.getTrendList().add(calls / 1000)); + ServiceName serviceName = serviceNameCacheService.get(serviceId); + int secondBetween = secondBetweenService.calculate(serviceName.getApplicationId(), startSecondTimeBucket, endSecondTimeBucket); + callsTrends.forEach(calls -> throughputTrend.getTrendList().add(calls / secondBetween)); return throughputTrend; } - public ResponseTimeTrend getServiceResponseTimeTrend(int serviceId, Step step, long start, - long end) throws ParseException { + public ResponseTimeTrend getServiceResponseTimeTrend(int serviceId, Step step, long startTimeBucket, + long endTimeBucket) throws ParseException { ResponseTimeTrend responseTimeTrend = new ResponseTimeTrend(); - List durationPoints = DurationUtils.INSTANCE.getDurationPoints(step, start, end); + List durationPoints = DurationUtils.INSTANCE.getDurationPoints(step, startTimeBucket, endTimeBucket); responseTimeTrend.setTrendList(serviceMetricUIDAO.getServiceResponseTimeTrend(serviceId, step, durationPoints)); return responseTimeTrend; } From 5af1c91500edb83bc8211063abc912be74ece514 Mon Sep 17 00:00:00 2001 From: peng-yongsheng <8082209@qq.com> Date: Wed, 28 Feb 2018 22:45:03 +0800 Subject: [PATCH 3/4] getServiceSLATrend: 1. Change the parameter name: Start -> startTimeBucket End -> endTimeBucket --- .../skywalking/apm/collector/ui/query/ServiceQuery.java | 6 +++--- .../apm/collector/ui/service/ServiceNameService.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/query/ServiceQuery.java b/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/query/ServiceQuery.java index 39b546731..00c79eb58 100644 --- a/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/query/ServiceQuery.java +++ b/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/query/ServiceQuery.java @@ -81,9 +81,9 @@ public class ServiceQuery implements Query { } public SLATrend getServiceSLATrend(int serviceId, Duration duration) throws ParseException { - long start = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart()); - long end = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd()); - return getServiceNameService().getServiceSLATrend(serviceId, duration.getStep(), start, end); + long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart()); + long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd()); + return getServiceNameService().getServiceSLATrend(serviceId, duration.getStep(), startTimeBucket, endTimeBucket); } public Topology getServiceTopology(int serviceId, Duration duration) throws ParseException { diff --git a/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/service/ServiceNameService.java b/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/service/ServiceNameService.java index c7a58c362..d587b1bc8 100644 --- a/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/service/ServiceNameService.java +++ b/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/service/ServiceNameService.java @@ -87,9 +87,9 @@ public class ServiceNameService { return responseTimeTrend; } - public SLATrend getServiceSLATrend(int serviceId, Step step, long start, long end) throws ParseException { + public SLATrend getServiceSLATrend(int serviceId, Step step, long startTimeBucket, long endTimeBucket) throws ParseException { SLATrend slaTrend = new SLATrend(); - List durationPoints = DurationUtils.INSTANCE.getDurationPoints(step, start, end); + List durationPoints = DurationUtils.INSTANCE.getDurationPoints(step, startTimeBucket, endTimeBucket); slaTrend.setTrendList(serviceMetricUIDAO.getServiceSLATrend(serviceId, step, durationPoints)); return slaTrend; } From 573291ef5458d6c718d72b47d3158218c95f615a Mon Sep 17 00:00:00 2001 From: peng-yongsheng <8082209@qq.com> Date: Thu, 1 Mar 2018 00:30:13 +0800 Subject: [PATCH 4/4] Refactor service topology build logic. --- .../dao/ui/IServiceReferenceMetricUIDAO.java | 64 ++++++++- .../dao/ui/ServiceReferenceEsMetricUIDAO.java | 132 ++++++------------ .../dao/ui/ServiceReferenceH2MetricUIDAO.java | 17 +-- .../apm/collector/ui/query/ServiceQuery.java | 10 +- .../ui/service/ServiceTopologyService.java | 61 ++++++-- 5 files changed, 162 insertions(+), 122 deletions(-) diff --git a/apm-collector/apm-collector-storage/collector-storage-define/src/main/java/org/apache/skywalking/apm/collector/storage/dao/ui/IServiceReferenceMetricUIDAO.java b/apm-collector/apm-collector-storage/collector-storage-define/src/main/java/org/apache/skywalking/apm/collector/storage/dao/ui/IServiceReferenceMetricUIDAO.java index 8aba521d2..ee4602391 100644 --- a/apm-collector/apm-collector-storage/collector-storage-define/src/main/java/org/apache/skywalking/apm/collector/storage/dao/ui/IServiceReferenceMetricUIDAO.java +++ b/apm-collector/apm-collector-storage/collector-storage-define/src/main/java/org/apache/skywalking/apm/collector/storage/dao/ui/IServiceReferenceMetricUIDAO.java @@ -21,7 +21,6 @@ package org.apache.skywalking.apm.collector.storage.dao.ui; import java.util.List; import org.apache.skywalking.apm.collector.storage.base.dao.DAO; import org.apache.skywalking.apm.collector.storage.table.MetricSource; -import org.apache.skywalking.apm.collector.storage.ui.common.Call; import org.apache.skywalking.apm.collector.storage.ui.common.Step; /** @@ -29,15 +28,66 @@ import org.apache.skywalking.apm.collector.storage.ui.common.Step; */ public interface IServiceReferenceMetricUIDAO extends DAO { - List getFrontServices(Step step, long startTime, long endTime, + List getFrontServices(Step step, long startTimeBucket, long endTimeBucket, MetricSource metricSource, int behindServiceId); - List getBehindServices(Step step, long startTime, long endTime, + List getBehindServices(Step step, long startTimeBucket, long endTimeBucket, MetricSource metricSource, int frontServiceId); - List getFrontServices(Step step, long startTime, long endTime, - MetricSource metricSource, List behindServiceIds); + class ServiceReferenceMetric { + private int source; + private int target; + private long calls; + private long errorCalls; + private long durations; + private long errorDurations; - List getBehindServices(Step step, long startTime, long endTime, - MetricSource metricSource, List frontServiceIds); + public int getSource() { + return source; + } + + public void setSource(int source) { + this.source = source; + } + + public int getTarget() { + return target; + } + + public void setTarget(int target) { + this.target = target; + } + + public long getCalls() { + return calls; + } + + public void setCalls(long calls) { + this.calls = calls; + } + + public long getErrorCalls() { + return errorCalls; + } + + public void setErrorCalls(long errorCalls) { + this.errorCalls = errorCalls; + } + + public long getDurations() { + return durations; + } + + public void setDurations(long durations) { + this.durations = durations; + } + + public long getErrorDurations() { + return errorDurations; + } + + public void setErrorDurations(long errorDurations) { + this.errorDurations = errorDurations; + } + } } diff --git a/apm-collector/apm-collector-storage/collector-storage-es-provider/src/main/java/org/apache/skywalking/apm/collector/storage/es/dao/ui/ServiceReferenceEsMetricUIDAO.java b/apm-collector/apm-collector-storage/collector-storage-es-provider/src/main/java/org/apache/skywalking/apm/collector/storage/es/dao/ui/ServiceReferenceEsMetricUIDAO.java index 73da95432..bfc6dfff4 100644 --- a/apm-collector/apm-collector-storage/collector-storage-es-provider/src/main/java/org/apache/skywalking/apm/collector/storage/es/dao/ui/ServiceReferenceEsMetricUIDAO.java +++ b/apm-collector/apm-collector-storage/collector-storage-es-provider/src/main/java/org/apache/skywalking/apm/collector/storage/es/dao/ui/ServiceReferenceEsMetricUIDAO.java @@ -25,7 +25,6 @@ import org.apache.skywalking.apm.collector.storage.dao.ui.IServiceReferenceMetri import org.apache.skywalking.apm.collector.storage.es.base.dao.EsDAO; import org.apache.skywalking.apm.collector.storage.table.MetricSource; import org.apache.skywalking.apm.collector.storage.table.service.ServiceReferenceMetricTable; -import org.apache.skywalking.apm.collector.storage.ui.common.Call; import org.apache.skywalking.apm.collector.storage.ui.common.Step; import org.apache.skywalking.apm.collector.storage.utils.TimePyramidTableNameBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; @@ -47,7 +46,8 @@ public class ServiceReferenceEsMetricUIDAO extends EsDAO implements IServiceRefe super(client); } - @Override public List getFrontServices(Step step, long startTime, long endTime, MetricSource metricSource, + @Override public List getFrontServices(Step step, long startTimeBucket, long endTimeBucket, + MetricSource metricSource, int behindServiceId) { String tableName = TimePyramidTableNameBuilder.build(step, ServiceReferenceMetricTable.TABLE); @@ -56,7 +56,7 @@ public class ServiceReferenceEsMetricUIDAO extends EsDAO implements IServiceRefe searchRequestBuilder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); - boolQuery.must().add(QueryBuilders.rangeQuery(ServiceReferenceMetricTable.COLUMN_TIME_BUCKET).gte(startTime).lte(endTime)); + boolQuery.must().add(QueryBuilders.rangeQuery(ServiceReferenceMetricTable.COLUMN_TIME_BUCKET).gte(startTimeBucket).lte(endTimeBucket)); boolQuery.must().add(QueryBuilders.termQuery(ServiceReferenceMetricTable.COLUMN_BEHIND_SERVICE_ID, behindServiceId)); boolQuery.must().add(QueryBuilders.termQuery(ServiceReferenceMetricTable.COLUMN_SOURCE_VALUE, metricSource.getValue())); @@ -65,19 +65,22 @@ public class ServiceReferenceEsMetricUIDAO extends EsDAO implements IServiceRefe TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms(ServiceReferenceMetricTable.COLUMN_FRONT_SERVICE_ID).field(ServiceReferenceMetricTable.COLUMN_FRONT_SERVICE_ID).size(100); aggregationBuilder.subAggregation(AggregationBuilders.sum(ServiceReferenceMetricTable.COLUMN_TRANSACTION_CALLS).field(ServiceReferenceMetricTable.COLUMN_TRANSACTION_CALLS)); + aggregationBuilder.subAggregation(AggregationBuilders.sum(ServiceReferenceMetricTable.COLUMN_TRANSACTION_ERROR_CALLS).field(ServiceReferenceMetricTable.COLUMN_TRANSACTION_ERROR_CALLS)); aggregationBuilder.subAggregation(AggregationBuilders.sum(ServiceReferenceMetricTable.COLUMN_TRANSACTION_DURATION_SUM).field(ServiceReferenceMetricTable.COLUMN_TRANSACTION_DURATION_SUM)); + aggregationBuilder.subAggregation(AggregationBuilders.sum(ServiceReferenceMetricTable.COLUMN_TRANSACTION_ERROR_DURATION_SUM).field(ServiceReferenceMetricTable.COLUMN_TRANSACTION_ERROR_DURATION_SUM)); searchRequestBuilder.addAggregation(aggregationBuilder); SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); - List calls = new LinkedList<>(); + List referenceMetrics = new LinkedList<>(); Terms frontServiceIdTerms = searchResponse.getAggregations().get(ServiceReferenceMetricTable.COLUMN_FRONT_SERVICE_ID); - buildNodeByBehindServiceId(calls, frontServiceIdTerms, behindServiceId); + buildNodeByBehindServiceId(referenceMetrics, frontServiceIdTerms, behindServiceId); - return calls; + return referenceMetrics; } - @Override public List getBehindServices(Step step, long startTime, long endTime, MetricSource metricSource, + @Override public List getBehindServices(Step step, long startTimeBucket, long endTimeBucket, + MetricSource metricSource, int frontServiceId) { String tableName = TimePyramidTableNameBuilder.build(step, ServiceReferenceMetricTable.TABLE); @@ -86,7 +89,7 @@ public class ServiceReferenceEsMetricUIDAO extends EsDAO implements IServiceRefe searchRequestBuilder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); - boolQuery.must().add(QueryBuilders.rangeQuery(ServiceReferenceMetricTable.COLUMN_TIME_BUCKET).gte(startTime).lte(endTime)); + boolQuery.must().add(QueryBuilders.rangeQuery(ServiceReferenceMetricTable.COLUMN_TIME_BUCKET).gte(startTimeBucket).lte(endTimeBucket)); boolQuery.must().add(QueryBuilders.termQuery(ServiceReferenceMetricTable.COLUMN_FRONT_SERVICE_ID, frontServiceId)); boolQuery.must().add(QueryBuilders.termQuery(ServiceReferenceMetricTable.COLUMN_SOURCE_VALUE, metricSource.getValue())); @@ -95,106 +98,59 @@ public class ServiceReferenceEsMetricUIDAO extends EsDAO implements IServiceRefe TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms(ServiceReferenceMetricTable.COLUMN_BEHIND_SERVICE_ID).field(ServiceReferenceMetricTable.COLUMN_BEHIND_SERVICE_ID).size(100); aggregationBuilder.subAggregation(AggregationBuilders.sum(ServiceReferenceMetricTable.COLUMN_TRANSACTION_CALLS).field(ServiceReferenceMetricTable.COLUMN_TRANSACTION_CALLS)); + aggregationBuilder.subAggregation(AggregationBuilders.sum(ServiceReferenceMetricTable.COLUMN_TRANSACTION_ERROR_CALLS).field(ServiceReferenceMetricTable.COLUMN_TRANSACTION_ERROR_CALLS)); aggregationBuilder.subAggregation(AggregationBuilders.sum(ServiceReferenceMetricTable.COLUMN_TRANSACTION_DURATION_SUM).field(ServiceReferenceMetricTable.COLUMN_TRANSACTION_DURATION_SUM)); + aggregationBuilder.subAggregation(AggregationBuilders.sum(ServiceReferenceMetricTable.COLUMN_TRANSACTION_ERROR_DURATION_SUM).field(ServiceReferenceMetricTable.COLUMN_TRANSACTION_ERROR_DURATION_SUM)); searchRequestBuilder.addAggregation(aggregationBuilder); SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); - List calls = new LinkedList<>(); + List referenceMetrics = new LinkedList<>(); Terms behindServiceIdTerms = searchResponse.getAggregations().get(ServiceReferenceMetricTable.COLUMN_BEHIND_SERVICE_ID); - buildNodeByFrontServiceId(calls, behindServiceIdTerms, frontServiceId); + buildNodeByFrontServiceId(referenceMetrics, behindServiceIdTerms, frontServiceId); - return calls; + return referenceMetrics; } - @Override public List getFrontServices(Step step, long startTime, long endTime, MetricSource metricSource, - List behindServiceIds) { - String tableName = TimePyramidTableNameBuilder.build(step, ServiceReferenceMetricTable.TABLE); - - SearchRequestBuilder searchRequestBuilder = getClient().prepareSearch(tableName); - searchRequestBuilder.setTypes(ServiceReferenceMetricTable.TABLE_TYPE); - searchRequestBuilder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); - - BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); - boolQuery.must().add(QueryBuilders.rangeQuery(ServiceReferenceMetricTable.COLUMN_TIME_BUCKET).gte(startTime).lte(endTime)); - boolQuery.must().add(QueryBuilders.termsQuery(ServiceReferenceMetricTable.COLUMN_BEHIND_SERVICE_ID, behindServiceIds)); - boolQuery.must().add(QueryBuilders.termQuery(ServiceReferenceMetricTable.COLUMN_SOURCE_VALUE, metricSource.getValue())); - - searchRequestBuilder.setQuery(boolQuery); - searchRequestBuilder.setSize(0); - - return executeAggregation(searchRequestBuilder); - } - - @Override public List getBehindServices(Step step, long startTime, long endTime, MetricSource metricSource, - List frontServiceIds) { - String tableName = TimePyramidTableNameBuilder.build(step, ServiceReferenceMetricTable.TABLE); - - SearchRequestBuilder searchRequestBuilder = getClient().prepareSearch(tableName); - searchRequestBuilder.setTypes(ServiceReferenceMetricTable.TABLE_TYPE); - searchRequestBuilder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); - - BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); - boolQuery.must().add(QueryBuilders.rangeQuery(ServiceReferenceMetricTable.COLUMN_TIME_BUCKET).gte(startTime).lte(endTime)); - boolQuery.must().add(QueryBuilders.termsQuery(ServiceReferenceMetricTable.COLUMN_FRONT_SERVICE_ID, frontServiceIds)); - boolQuery.must().add(QueryBuilders.termQuery(ServiceReferenceMetricTable.COLUMN_SOURCE_VALUE, metricSource.getValue())); - - searchRequestBuilder.setQuery(boolQuery); - searchRequestBuilder.setSize(0); - - return executeAggregation(searchRequestBuilder); - } - - private List executeAggregation(SearchRequestBuilder searchRequestBuilder) { - TermsAggregationBuilder frontAggregationBuilder = AggregationBuilders.terms(ServiceReferenceMetricTable.COLUMN_FRONT_SERVICE_ID).field(ServiceReferenceMetricTable.COLUMN_FRONT_SERVICE_ID).size(100); - TermsAggregationBuilder behindAggregationBuilder = AggregationBuilders.terms(ServiceReferenceMetricTable.COLUMN_BEHIND_SERVICE_ID).field(ServiceReferenceMetricTable.COLUMN_BEHIND_SERVICE_ID).size(100); - behindAggregationBuilder.subAggregation(AggregationBuilders.sum(ServiceReferenceMetricTable.COLUMN_TRANSACTION_CALLS).field(ServiceReferenceMetricTable.COLUMN_TRANSACTION_CALLS)); - behindAggregationBuilder.subAggregation(AggregationBuilders.sum(ServiceReferenceMetricTable.COLUMN_TRANSACTION_DURATION_SUM).field(ServiceReferenceMetricTable.COLUMN_TRANSACTION_DURATION_SUM)); - frontAggregationBuilder.subAggregation(behindAggregationBuilder); - - searchRequestBuilder.addAggregation(frontAggregationBuilder); - SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); - - List nodes = new LinkedList<>(); - Terms frontServiceIdTerms = searchResponse.getAggregations().get(ServiceReferenceMetricTable.COLUMN_FRONT_SERVICE_ID); - frontServiceIdTerms.getBuckets().forEach(frontServiceIdBucket -> { - int frontServiceId = frontServiceIdBucket.getKeyAsNumber().intValue(); - - Terms behindServiceIdTerms = frontServiceIdBucket.getAggregations().get(ServiceReferenceMetricTable.COLUMN_BEHIND_SERVICE_ID); - buildNodeByFrontServiceId(nodes, behindServiceIdTerms, frontServiceId); - }); - - return nodes; - } - - private void buildNodeByFrontServiceId(List calls, Terms behindServiceIdTerms, int frontServiceId) { + private void buildNodeByFrontServiceId(List referenceMetrics, Terms behindServiceIdTerms, + int frontServiceId) { behindServiceIdTerms.getBuckets().forEach(behindServiceIdBucket -> { int behindServiceId = behindServiceIdBucket.getKeyAsNumber().intValue(); Sum callsSum = behindServiceIdBucket.getAggregations().get(ServiceReferenceMetricTable.COLUMN_TRANSACTION_CALLS); - Sum responseTimes = behindServiceIdBucket.getAggregations().get(ServiceReferenceMetricTable.COLUMN_TRANSACTION_DURATION_SUM); + Sum errorCallsSum = behindServiceIdBucket.getAggregations().get(ServiceReferenceMetricTable.COLUMN_TRANSACTION_ERROR_CALLS); + Sum durationSum = behindServiceIdBucket.getAggregations().get(ServiceReferenceMetricTable.COLUMN_TRANSACTION_DURATION_SUM); + Sum errorDurationSum = behindServiceIdBucket.getAggregations().get(ServiceReferenceMetricTable.COLUMN_TRANSACTION_ERROR_DURATION_SUM); - Call call = new Call(); - call.setSource(frontServiceId); - call.setTarget(behindServiceId); -// call.setCalls((int)callsSum.getValue()); -// call.setResponseTimes((int)responseTimes.getValue()); - calls.add(call); + ServiceReferenceMetric referenceMetric = new ServiceReferenceMetric(); + referenceMetric.setSource(frontServiceId); + referenceMetric.setTarget(behindServiceId); + referenceMetric.setCalls((long)callsSum.getValue()); + referenceMetric.setErrorCalls((long)errorCallsSum.getValue()); + referenceMetric.setDurations((long)durationSum.getValue()); + referenceMetric.setErrorDurations((long)errorDurationSum.getValue()); + referenceMetrics.add(referenceMetric); }); } - private void buildNodeByBehindServiceId(List calls, Terms frontServiceIdTerms, int behindServiceId) { + private void buildNodeByBehindServiceId(List referenceMetrics, Terms frontServiceIdTerms, + int behindServiceId) { frontServiceIdTerms.getBuckets().forEach(frontServiceIdBucket -> { int frontServiceId = frontServiceIdBucket.getKeyAsNumber().intValue(); - Sum callsSum = frontServiceIdBucket.getAggregations().get(ServiceReferenceMetricTable.COLUMN_TRANSACTION_CALLS); - Sum responseTimes = frontServiceIdBucket.getAggregations().get(ServiceReferenceMetricTable.COLUMN_TRANSACTION_DURATION_SUM); - Call call = new Call(); - call.setTarget(behindServiceId); - call.setSource(frontServiceId); -// call.setCalls((int)callsSum.getValue()); -// call.setResponseTimes((int)responseTimes.getValue()); - calls.add(call); + Sum callsSum = frontServiceIdBucket.getAggregations().get(ServiceReferenceMetricTable.COLUMN_TRANSACTION_CALLS); + Sum errorCallsSum = frontServiceIdBucket.getAggregations().get(ServiceReferenceMetricTable.COLUMN_TRANSACTION_ERROR_CALLS); + Sum durationSum = frontServiceIdBucket.getAggregations().get(ServiceReferenceMetricTable.COLUMN_TRANSACTION_DURATION_SUM); + Sum errorDurationSum = frontServiceIdBucket.getAggregations().get(ServiceReferenceMetricTable.COLUMN_TRANSACTION_ERROR_DURATION_SUM); + + ServiceReferenceMetric referenceMetric = new ServiceReferenceMetric(); + referenceMetric.setTarget(behindServiceId); + referenceMetric.setSource(frontServiceId); + referenceMetric.setCalls((long)callsSum.getValue()); + referenceMetric.setErrorCalls((long)errorCallsSum.getValue()); + referenceMetric.setDurations((long)durationSum.getValue()); + referenceMetric.setErrorDurations((long)errorDurationSum.getValue()); + referenceMetrics.add(referenceMetric); }); } } diff --git a/apm-collector/apm-collector-storage/collector-storage-h2-provider/src/main/java/org/apache/skywalking/apm/collector/storage/h2/dao/ui/ServiceReferenceH2MetricUIDAO.java b/apm-collector/apm-collector-storage/collector-storage-h2-provider/src/main/java/org/apache/skywalking/apm/collector/storage/h2/dao/ui/ServiceReferenceH2MetricUIDAO.java index c5f27f97f..435664df4 100644 --- a/apm-collector/apm-collector-storage/collector-storage-h2-provider/src/main/java/org/apache/skywalking/apm/collector/storage/h2/dao/ui/ServiceReferenceH2MetricUIDAO.java +++ b/apm-collector/apm-collector-storage/collector-storage-h2-provider/src/main/java/org/apache/skywalking/apm/collector/storage/h2/dao/ui/ServiceReferenceH2MetricUIDAO.java @@ -23,7 +23,6 @@ import org.apache.skywalking.apm.collector.client.h2.H2Client; import org.apache.skywalking.apm.collector.storage.dao.ui.IServiceReferenceMetricUIDAO; import org.apache.skywalking.apm.collector.storage.h2.base.dao.H2DAO; import org.apache.skywalking.apm.collector.storage.table.MetricSource; -import org.apache.skywalking.apm.collector.storage.ui.common.Call; import org.apache.skywalking.apm.collector.storage.ui.common.Step; /** @@ -35,23 +34,15 @@ public class ServiceReferenceH2MetricUIDAO extends H2DAO implements IServiceRefe super(client); } - @Override public List getFrontServices(Step step, long startTime, long endTime, MetricSource metricSource, + @Override public List getFrontServices(Step step, long startTimeBucket, long endTimeBucket, + MetricSource metricSource, int behindServiceId) { return null; } - @Override public List getBehindServices(Step step, long startTime, long endTime, MetricSource metricSource, + @Override public List getBehindServices(Step step, long startTimeBucket, long endTimeBucket, + MetricSource metricSource, int frontServiceId) { return null; } - - @Override public List getFrontServices(Step step, long startTime, long endTime, MetricSource metricSource, - List behindServiceIds) { - return null; - } - - @Override public List getBehindServices(Step step, long startTime, long endTime, MetricSource metricSource, - List frontServiceIds) { - return null; - } } diff --git a/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/query/ServiceQuery.java b/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/query/ServiceQuery.java index 00c79eb58..ba7c6433a 100644 --- a/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/query/ServiceQuery.java +++ b/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/query/ServiceQuery.java @@ -87,8 +87,12 @@ public class ServiceQuery implements Query { } public Topology getServiceTopology(int serviceId, Duration duration) throws ParseException { - long start = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart()); - long end = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd()); - return getServiceTopologyService().getServiceTopology(duration.getStep(), serviceId, start, end); + long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart()); + long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd()); + + long startSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getStart()); + long endSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getEnd()); + + return getServiceTopologyService().getServiceTopology(duration.getStep(), serviceId, startTimeBucket, endTimeBucket, startSecondTimeBucket, endSecondTimeBucket); } } diff --git a/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/service/ServiceTopologyService.java b/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/service/ServiceTopologyService.java index 5ce4340be..f932d0969 100644 --- a/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/service/ServiceTopologyService.java +++ b/apm-collector/apm-collector-ui/collector-ui-jetty-provider/src/main/java/org/apache/skywalking/apm/collector/ui/service/ServiceTopologyService.java @@ -21,6 +21,7 @@ package org.apache.skywalking.apm.collector.ui.service; import java.text.ParseException; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -38,6 +39,7 @@ import org.apache.skywalking.apm.collector.storage.ui.common.Call; import org.apache.skywalking.apm.collector.storage.ui.common.Node; import org.apache.skywalking.apm.collector.storage.ui.common.Step; import org.apache.skywalking.apm.collector.storage.ui.common.Topology; +import org.apache.skywalking.apm.collector.storage.ui.common.VisualUserNode; import org.apache.skywalking.apm.network.trace.component.ComponentsDefine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,40 +55,77 @@ public class ServiceTopologyService { private final IServiceMetricUIDAO serviceMetricUIDAO; private final IServiceReferenceMetricUIDAO serviceReferenceMetricUIDAO; private final ServiceNameCacheService serviceNameCacheService; + private final SecondBetweenService secondBetweenService; public ServiceTopologyService(ModuleManager moduleManager) { this.serviceMetricUIDAO = moduleManager.find(StorageModule.NAME).getService(IServiceMetricUIDAO.class); this.serviceReferenceMetricUIDAO = moduleManager.find(StorageModule.NAME).getService(IServiceReferenceMetricUIDAO.class); this.applicationComponentUIDAO = moduleManager.find(StorageModule.NAME).getService(IApplicationComponentUIDAO.class); this.serviceNameCacheService = moduleManager.find(CacheModule.NAME).getService(ServiceNameCacheService.class); + this.secondBetweenService = new SecondBetweenService(moduleManager); } - public Topology getServiceTopology(Step step, int serviceId, long start, long end) throws ParseException { - logger.debug("start: {}, end: {}", start, end); - List applicationComponents = applicationComponentUIDAO.load(step, start, end); + public Topology getServiceTopology(Step step, int serviceId, long startTimeBucket, + long endTimeBucket, long startSecondTimeBucket, long endSecondTimeBucket) throws ParseException { + logger.debug("startTimeBucket: {}, endTimeBucket: {}", startTimeBucket, endTimeBucket); + List applicationComponents = applicationComponentUIDAO.load(step, startTimeBucket, endTimeBucket); Map components = new HashMap<>(); applicationComponents.forEach(component -> components.put(component.getApplicationId(), ComponentsDefine.getInstance().getComponentName(component.getComponentId()))); - List calleeCalls = serviceReferenceMetricUIDAO.getFrontServices(step, start, end, MetricSource.Callee, serviceId); - calleeCalls.addAll(serviceReferenceMetricUIDAO.getBehindServices(step, start, end, MetricSource.Caller, serviceId)); + List referenceMetrics = serviceReferenceMetricUIDAO.getFrontServices(step, startTimeBucket, endTimeBucket, MetricSource.Callee, serviceId); + referenceMetrics.addAll(serviceReferenceMetricUIDAO.getBehindServices(step, startTimeBucket, endTimeBucket, MetricSource.Caller, serviceId)); Set nodeIds = new HashSet<>(); - calleeCalls.forEach(call -> { - call.setCallType(Const.EMPTY_STRING); - nodeIds.add(call.getSource()); - nodeIds.add(call.getTarget()); + + List calls = new LinkedList<>(); + referenceMetrics.forEach(referenceMetric -> { + nodeIds.add(referenceMetric.getSource()); + nodeIds.add(referenceMetric.getTarget()); + + Call call = new Call(); + call.setSource(referenceMetric.getSource()); + call.setTarget(referenceMetric.getTarget()); + call.setAvgResponseTime((referenceMetric.getDurations() - referenceMetric.getErrorDurations()) / (referenceMetric.getCalls() - referenceMetric.getErrorCalls())); + call.setCallType(components.getOrDefault(serviceNameCacheService.get(referenceMetric.getTarget()).getApplicationId(), Const.UNKNOWN)); + try { + int applicationId = serviceNameCacheService.get(referenceMetric.getTarget()).getApplicationId(); + call.setCallsPerSec(referenceMetric.getCalls() / secondBetweenService.calculate(applicationId, startSecondTimeBucket, endSecondTimeBucket)); + } catch (ParseException e) { + logger.error(e.getMessage(), e); + } + calls.add(call); }); - List serviceNodes = serviceMetricUIDAO.getServicesMetric(step, start, end, MetricSource.Callee, nodeIds); + List serviceNodes = serviceMetricUIDAO.getServicesMetric(step, startTimeBucket, endTimeBucket, MetricSource.Callee, nodeIds); + + Set gotNodes = new HashSet<>(); + serviceNodes.forEach(serviceNode -> gotNodes.add(serviceNode.getId())); + + Set callerNodeIds = new HashSet<>(); + nodeIds.forEach(nodeId -> { + if (!gotNodes.contains(nodeId)) { + callerNodeIds.add(nodeId); + } + }); + + serviceNodes.addAll(serviceMetricUIDAO.getServicesMetric(step, startTimeBucket, endTimeBucket, MetricSource.Caller, callerNodeIds)); serviceNodes.forEach(serviceNode -> { ServiceName serviceName = serviceNameCacheService.get(serviceNode.getId()); serviceNode.setName(serviceName.getServiceName()); }); + if (callerNodeIds.contains(Const.NONE_SERVICE_ID)) { + VisualUserNode userNode = new VisualUserNode(); + userNode.setId(Const.NONE_SERVICE_ID); + userNode.setName(Const.USER_CODE); + userNode.setType(Const.USER_CODE.toUpperCase()); + serviceNodes.add(userNode); + } + Topology topology = new Topology(); - topology.setCalls(calleeCalls); + topology.setCalls(calls); topology.setNodes(serviceNodes); return topology; }