Provide profile analyze query (#4335)

* provide profile analyze query

Co-authored-by: 吴晟 Wu Sheng <wu.sheng@foxmail.com>

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java

@@ -72,6 +72,16 @@ public class CoreModuleConfig extends ModuleConfig {
      */
     private long maxSizeOfProfileTask = 10_000L;
 
+    /**
+     * Analyze profile snapshots paging size.
+     */
+    private int maxPageSizeOfQueryProfileSnapshot = 500;
+
+    /**
+     * Analyze profile snapshots max size.
+     */
+    private int maxSizeOfAnalyzeProfileSnapshot = 12000;
+
     CoreModuleConfig() {
         this.downsampling = new ArrayList<>();
     }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java

@@ -169,7 +169,7 @@ public class CoreModuleProvider extends ModuleProvider {
 
         // add profile service implementations
         this.registerServiceImplementation(ProfileTaskMutationService.class, new ProfileTaskMutationService(getManager()));
-        this.registerServiceImplementation(ProfileTaskQueryService.class, new ProfileTaskQueryService(getManager()));
+        this.registerServiceImplementation(ProfileTaskQueryService.class, new ProfileTaskQueryService(getManager(), moduleConfig));
         this.registerServiceImplementation(ProfileTaskCache.class, new ProfileTaskCache(getManager(), moduleConfig));
 
         this.registerServiceImplementation(CommandService.class, new CommandService(getManager()));

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profile/analyze/ProfileAnalyzer.java

@@ -18,10 +18,17 @@
 
 package org.apache.skywalking.oap.server.core.profile.analyze;
 
+import org.apache.skywalking.oap.server.core.profile.ProfileThreadSnapshotRecord;
 import org.apache.skywalking.oap.server.core.query.entity.ProfileAnalyzation;
 import org.apache.skywalking.oap.server.core.query.entity.ProfileStackTree;
+import org.apache.skywalking.oap.server.core.storage.StorageModule;
+import org.apache.skywalking.oap.server.core.storage.profile.IProfileThreadSnapshotQueryDAO;
+import org.apache.skywalking.oap.server.library.module.ModuleManager;
 import org.apache.skywalking.oap.server.library.util.CollectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -32,14 +39,89 @@ import java.util.stream.Collectors;
  */
 public class ProfileAnalyzer {
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(ProfileAnalyzer.class);
+
     private static final ProfileAnalyzeCollector ANALYZE_COLLECTOR = new ProfileAnalyzeCollector();
 
+    private final int threadSnapshotAnalyzeBatchSize;
+    private final int analyzeSnapshotMaxSize;
+
+    private final ModuleManager moduleManager;
+    private IProfileThreadSnapshotQueryDAO profileThreadSnapshotQueryDAO;
+
+    public ProfileAnalyzer(ModuleManager moduleManager, int snapshotAnalyzeBatchSize, int analyzeSnapshotMaxSize) {
+        this.moduleManager = moduleManager;
+        this.threadSnapshotAnalyzeBatchSize = snapshotAnalyzeBatchSize;
+        this.analyzeSnapshotMaxSize = analyzeSnapshotMaxSize;
+    }
+
+    /**
+     * search snapshots and analyze
+     * @param segmentId
+     * @param start
+     * @param end
+     * @return
+     */
+    public ProfileAnalyzation analyze(String segmentId, long start, long end) throws IOException {
+        ProfileAnalyzation analyzation = new ProfileAnalyzation();
+
+        // query sequence range list
+        SequenceSearch sequenceSearch = getAllSequenceRange(segmentId, start, end);
+        if (sequenceSearch == null) {
+            analyzation.setTip("Data not found");
+            return analyzation;
+        }
+        if (sequenceSearch.totalSequenceCount > analyzeSnapshotMaxSize) {
+            analyzation.setTip("Out of snapshot analyze limit, " + sequenceSearch.totalSequenceCount + " snapshots found, but analysis first " + analyzeSnapshotMaxSize + " snapshots only.");
+        }
+
+        // query snapshots
+        List<ProfileStack> stacks = sequenceSearch.getRanges().parallelStream().map(r -> {
+            try {
+                return getProfileThreadSnapshotQueryDAO().queryRecords(segmentId, r.getMinSequence(), r.getMaxSequence());
+            } catch (IOException e) {
+                LOGGER.warn(e.getMessage(), e);
+                return Collections.<ProfileThreadSnapshotRecord>emptyList();
+            }
+        }).flatMap(t -> t.stream()).map(ProfileStack::deserialize).collect(Collectors.toList());
+
+        // analyze
+        analyzation.setTrees(analyze(stacks));
+
+        return analyzation;
+    }
+
+    private SequenceSearch getAllSequenceRange(String segmentId, long start, long end) throws IOException {
+        // query min and max sequence
+        int minSequence = getProfileThreadSnapshotQueryDAO().queryMinSequence(segmentId, start, end);
+        int maxSequence = getProfileThreadSnapshotQueryDAO().queryMaxSequence(segmentId, start, end);
+
+        // data not found
+        if (maxSequence <= 0) {
+            return null;
+        }
+
+        SequenceSearch sequenceSearch = new SequenceSearch(maxSequence - minSequence);
+        maxSequence = Math.min(maxSequence, minSequence + analyzeSnapshotMaxSize);
+
+        do {
+            int batchMax = Math.min(minSequence + threadSnapshotAnalyzeBatchSize, maxSequence);
+            sequenceSearch.getRanges().add(new SequenceRange(minSequence, batchMax));
+            minSequence = batchMax + 1;
+        } while (minSequence < maxSequence);
+
+        // increase last range max sequence, need to include last sequence data
+        sequenceSearch.getRanges().getLast().increaseMaxSequence();
+
+        return sequenceSearch;
+    }
+
     /**
      * Analyze records
      * @param stacks
      * @return
      */
-    public static ProfileAnalyzation analyze(List<ProfileStack> stacks) {
+    protected List<ProfileStackTree> analyze(List<ProfileStack> stacks) {
         if (CollectionUtils.isEmpty(stacks)) {
             return null;
         }
@@ -50,9 +132,52 @@ public class ProfileAnalyzer {
                 .filter(s -> CollectionUtils.isNotEmpty(s.getStack()))
                 .collect(Collectors.groupingBy(s -> s.getStack().get(0), ANALYZE_COLLECTOR));
 
-        ProfileAnalyzation analyzer = new ProfileAnalyzation();
-        analyzer.setTrees(new ArrayList<>(stackTrees.values()));
-        return analyzer;
+        return new ArrayList<>(stackTrees.values());
     }
 
+    private IProfileThreadSnapshotQueryDAO getProfileThreadSnapshotQueryDAO() {
+        if (profileThreadSnapshotQueryDAO == null) {
+            profileThreadSnapshotQueryDAO = moduleManager.find(StorageModule.NAME).provider().getService(IProfileThreadSnapshotQueryDAO.class);
+        }
+        return profileThreadSnapshotQueryDAO;
+    }
+
+    private static class SequenceSearch {
+        private LinkedList<SequenceRange> ranges = new LinkedList<>();
+        private int totalSequenceCount;
+
+        public SequenceSearch(int totalSequenceCount) {
+            this.totalSequenceCount = totalSequenceCount;
+        }
+
+        public LinkedList<SequenceRange> getRanges() {
+            return ranges;
+        }
+
+        public int getTotalSequenceCount() {
+            return totalSequenceCount;
+        }
+    }
+
+    private static class SequenceRange {
+        private int minSequence;
+        private int maxSequence;
+
+        public SequenceRange(int minSequence, int maxSequence) {
+            this.minSequence = minSequence;
+            this.maxSequence = maxSequence;
+        }
+
+        public int getMinSequence() {
+            return minSequence;
+        }
+
+        public int getMaxSequence() {
+            return maxSequence;
+        }
+
+        public void increaseMaxSequence() {
+            this.maxSequence++;
+        }
+    }
 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/ProfileTaskQueryService.java

@@ -19,9 +19,12 @@ package org.apache.skywalking.oap.server.core.query;
 
 import com.google.common.base.Objects;
 import org.apache.skywalking.oap.server.core.CoreModule;
+import org.apache.skywalking.oap.server.core.CoreModuleConfig;
 import org.apache.skywalking.oap.server.core.cache.ServiceInstanceInventoryCache;
 import org.apache.skywalking.oap.server.core.cache.ServiceInventoryCache;
+import org.apache.skywalking.oap.server.core.profile.analyze.ProfileAnalyzer;
 import org.apache.skywalking.oap.server.core.query.entity.BasicTrace;
+import org.apache.skywalking.oap.server.core.query.entity.ProfileAnalyzation;
 import org.apache.skywalking.oap.server.core.query.entity.ProfileTask;
 import org.apache.skywalking.oap.server.core.query.entity.ProfileTaskLog;
 import org.apache.skywalking.oap.server.core.register.ServiceInstanceInventory;
@@ -54,8 +57,11 @@ public class ProfileTaskQueryService implements Service {
     private ServiceInventoryCache serviceInventoryCache;
     private ServiceInstanceInventoryCache serviceInstanceInventoryCache;
 
-    public ProfileTaskQueryService(ModuleManager moduleManager) {
+    private final ProfileAnalyzer profileAnalyzer;
+
+    public ProfileTaskQueryService(ModuleManager moduleManager, CoreModuleConfig moduleConfig) {
         this.moduleManager = moduleManager;
+        this.profileAnalyzer = new ProfileAnalyzer(moduleManager, moduleConfig.getMaxPageSizeOfQueryProfileSnapshot(), moduleConfig.getMaxSizeOfAnalyzeProfileSnapshot());
     }
 
     private IProfileTaskQueryDAO getProfileTaskDAO() {
@@ -140,4 +146,8 @@ public class ProfileTaskQueryService implements Service {
         return getProfileThreadSnapshotQueryDAO().queryProfiledSegments(taskId);
     }
 
+    public ProfileAnalyzation getProfileAnalyze(final String segmentId, final long start, final long end) throws IOException {
+        return profileAnalyzer.analyze(segmentId, start, end);
+    }
+
 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profile/IProfileThreadSnapshotQueryDAO.java

@@ -38,4 +38,24 @@ public interface IProfileThreadSnapshotQueryDAO extends DAO {
      */
     List<BasicTrace> queryProfiledSegments(String taskId) throws IOException;
 
+    /**
+     * search snapshots min sequence
+     * @return min sequence, return -1 if not found data
+     */
+    int queryMinSequence(String segmentId, long start, long end) throws IOException;
+
+    /**
+     * search snapshots max sequence
+     * @return max sequence, return -1 if not found data
+     */
+    int queryMaxSequence(String segmentId, long start, long end) throws IOException;
+
+    /**
+     * search snapshots with sequence range
+     * @param minSequence min sequence, include self
+     * @param maxSequence max sequence, exclude self
+     * @return snapshots
+     */
+    List<ProfileThreadSnapshotRecord> queryRecords(String segmentId, int minSequence, int maxSequence) throws IOException;
+
 }

oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profile/ProfileAnalyzerTest.java

@@ -18,8 +18,8 @@
 
 package org.apache.skywalking.oap.server.core.profile;
 
-import org.apache.skywalking.oap.server.core.profile.bean.ProfileStackAnalyze;
-import org.apache.skywalking.oap.server.core.profile.bean.ProfileStackAnalyzeHolder;
+import org.apache.skywalking.oap.server.core.profile.analyze.ProfileStackAnalyze;
+import org.apache.skywalking.oap.server.core.profile.analyze.ProfileStackAnalyzeHolder;
 import org.junit.Test;
 import org.yaml.snakeyaml.Yaml;
 

oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profile/analyze/ProfileStackAnalyze.java

@@ -15,16 +15,14 @@
  * limitations under the License.
  *
  */
-package org.apache.skywalking.oap.server.core.profile.bean;
+package org.apache.skywalking.oap.server.core.profile.analyze;
 
 import lombok.Data;
-import org.apache.skywalking.oap.server.core.profile.analyze.ProfileAnalyzer;
-import org.apache.skywalking.oap.server.core.profile.analyze.ProfileStack;
-import org.apache.skywalking.oap.server.core.query.entity.ProfileAnalyzation;
+import org.apache.skywalking.oap.server.core.query.entity.ProfileStackTree;
 
 import java.util.List;
 
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
 
 @Data
 public class ProfileStackAnalyze {
@@ -34,11 +32,12 @@ public class ProfileStackAnalyze {
 
     public void analyzeAndAssert() {
         List<ProfileStack> stacks = data.transform();
-        ProfileAnalyzation analyze = ProfileAnalyzer.analyze(stacks);
+        List<ProfileStackTree> trees = new ProfileAnalyzer(null, 100, 500).analyze(stacks);
 
-        assertEquals(analyze.getTrees().size(), expected.size());
-        for (int i = 0; i < analyze.getTrees().size(); i++) {
-            expected.get(i).verify(analyze.getTrees().get(i));
+        assertNotNull(trees);
+        assertEquals(trees.size(), expected.size());
+        for (int i = 0; i < trees.size(); i++) {
+            expected.get(i).verify(trees.get(i));
         }
     }
 

oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profile/analyze/ProfileStackAnalyzeHolder.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  *
  */
-package org.apache.skywalking.oap.server.core.profile.bean;
+package org.apache.skywalking.oap.server.core.profile.analyze;
 
 import lombok.Data;
 

oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profile/analyze/ProfileStackData.java

@@ -15,11 +15,10 @@
  * limitations under the License.
  *
  */
-package org.apache.skywalking.oap.server.core.profile.bean;
+package org.apache.skywalking.oap.server.core.profile.analyze;
 
 import com.google.common.base.Splitter;
 import lombok.Data;
-import org.apache.skywalking.oap.server.core.profile.analyze.ProfileStack;
 
 import java.util.ArrayList;
 import java.util.List;

oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profile/analyze/ProfileStackElementMatcher.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  *
  */
-package org.apache.skywalking.oap.server.core.profile.bean;
+package org.apache.skywalking.oap.server.core.profile.analyze;
 
 import lombok.Data;
 import org.apache.skywalking.oap.server.core.query.entity.ProfileStackElement;

oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/ProfileQuery.java

@@ -27,7 +27,6 @@ import org.apache.skywalking.oap.server.core.query.entity.ProfileTask;
 import org.apache.skywalking.oap.server.library.module.ModuleManager;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.List;
 
 /**
@@ -59,10 +58,8 @@ public class ProfileQuery implements GraphQLQueryResolver {
         return getProfileTaskQueryService().getTaskTraces(taskID);
     }
 
-    public ProfileAnalyzation getProfileAnalyze(final String segmentId, final long start, final long end) {
-        ProfileAnalyzation analyzation = new ProfileAnalyzation();
-        analyzation.setTrees(Collections.emptyList());
-        return analyzation;
+    public ProfileAnalyzation getProfileAnalyze(final String segmentId, final long start, final long end) throws IOException {
+        return getProfileTaskQueryService().getProfileAnalyze(segmentId, start, end);
     }
 
 }

oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ProfileThreadSnapshotQueryEsDAO.java

@@ -31,6 +31,9 @@ import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.sort.SortOrder;
 
@@ -44,6 +47,8 @@ public class ProfileThreadSnapshotQueryEsDAO extends EsDAO implements IProfileTh
 
     private final int querySegemntMaxSize;
 
+    protected final ProfileThreadSnapshotRecord.Builder builder = new ProfileThreadSnapshotRecord.Builder();
+
     public ProfileThreadSnapshotQueryEsDAO(ElasticSearchClient client, int profileTaskQueryMaxSize) {
         super(client);
         this.querySegemntMaxSize = profileTaskQueryMaxSize;
@@ -105,4 +110,56 @@ public class ProfileThreadSnapshotQueryEsDAO extends EsDAO implements IProfileTh
 
         return result;
     }
+
+    @Override
+    public int queryMinSequence(String segmentId, long start, long end) throws IOException {
+        return querySequenceWithAgg(AggregationBuilders.min(ProfileThreadSnapshotRecord.SEQUENCE).field(ProfileThreadSnapshotRecord.SEQUENCE), segmentId, start, end);
+    }
+
+    @Override
+    public int queryMaxSequence(String segmentId, long start, long end) throws IOException {
+        return querySequenceWithAgg(AggregationBuilders.max(ProfileThreadSnapshotRecord.SEQUENCE).field(ProfileThreadSnapshotRecord.SEQUENCE), segmentId, start, end);
+    }
+
+    @Override
+    public List<ProfileThreadSnapshotRecord> queryRecords(String segmentId, int minSequence, int maxSequence) throws IOException {
+        // search traces
+        SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();
+
+        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+        sourceBuilder.query(boolQueryBuilder);
+        List<QueryBuilder> mustQueryList = boolQueryBuilder.must();
+
+        mustQueryList.add(QueryBuilders.termQuery(ProfileThreadSnapshotRecord.SEGMENT_ID, segmentId));
+        mustQueryList.add(QueryBuilders.rangeQuery(ProfileThreadSnapshotRecord.SEQUENCE).gte(minSequence).lt(maxSequence));
+        sourceBuilder.size(maxSequence - minSequence);
+
+        SearchResponse response = getClient().search(ProfileThreadSnapshotRecord.INDEX_NAME, sourceBuilder);
+
+        List<ProfileThreadSnapshotRecord> result = new ArrayList<>(maxSequence - minSequence);
+        for (SearchHit searchHit : response.getHits().getHits()) {
+            ProfileThreadSnapshotRecord record = builder.map2Data(searchHit.getSourceAsMap());
+
+            result.add(record);
+        }
+        return result;
+    }
+
+    protected int querySequenceWithAgg(AbstractAggregationBuilder aggregationBuilder, String segmentId, long start, long end) throws IOException {
+        SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();
+
+        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+        sourceBuilder.query(boolQueryBuilder);
+        List<QueryBuilder> mustQueryList = boolQueryBuilder.must();
+
+        mustQueryList.add(QueryBuilders.termQuery(ProfileThreadSnapshotRecord.SEGMENT_ID, segmentId));
+        mustQueryList.add(QueryBuilders.rangeQuery(ProfileThreadSnapshotRecord.DUMP_TIME).gte(start).lte(end));
+        sourceBuilder.size(0);
+
+        sourceBuilder.aggregation(aggregationBuilder);
+        SearchResponse response = getClient().search(ProfileThreadSnapshotRecord.INDEX_NAME, sourceBuilder);
+        NumericMetricsAggregation.SingleValue agg = response.getAggregations().get(ProfileThreadSnapshotRecord.SEQUENCE);
+
+        return (int) agg.value();
+    }
 }

oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/StorageModuleElasticsearch7Provider.java

@@ -125,7 +125,7 @@ public class StorageModuleElasticsearch7Provider extends ModuleProvider {
 
         this.registerServiceImplementation(IProfileTaskQueryDAO.class, new ProfileTaskQueryEsDAO(elasticSearch7Client, config.getProfileTaskQueryMaxSize()));
         this.registerServiceImplementation(IProfileTaskLogQueryDAO.class, new ProfileTaskLogEsDAO(elasticSearch7Client, config.getProfileTaskQueryMaxSize()));
-        this.registerServiceImplementation(IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQueryEsDAO(elasticSearch7Client, config.getProfileTaskQueryMaxSize()));
+        this.registerServiceImplementation(IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQueryEs7DAO(elasticSearch7Client, config.getProfileTaskQueryMaxSize()));
     }
 
     @Override

oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/query/ProfileThreadSnapshotQueryEs7DAO.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.skywalking.oap.server.storage.plugin.elasticsearch7.query;
+
+import org.apache.skywalking.oap.server.core.profile.ProfileThreadSnapshotRecord;
+import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient;
+import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ProfileThreadSnapshotQueryEsDAO;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+
+import java.io.IOException;
+
+/**
+ * Following aggregations APIs of ElasticsSearch 6 and 7 have different return type, need to re-compile based on different client jars.
+ *
+ * @see org.elasticsearch.search.aggregations.AggregationBuilders#min(java.lang.String)
+ * @see org.elasticsearch.search.aggregations.AggregationBuilders#max(java.lang.String)
+ */
+public class ProfileThreadSnapshotQueryEs7DAO extends ProfileThreadSnapshotQueryEsDAO {
+    public ProfileThreadSnapshotQueryEs7DAO(ElasticSearchClient client, int profileTaskQueryMaxSize) {
+        super(client, profileTaskQueryMaxSize);
+    }
+
+    @Override
+    public int queryMinSequence(String segmentId, long start, long end) throws IOException {
+        return querySequenceWithAgg(AggregationBuilders.min(ProfileThreadSnapshotRecord.SEQUENCE).field(ProfileThreadSnapshotRecord.SEQUENCE), segmentId, start, end);
+    }
+
+    @Override
+    public int queryMaxSequence(String segmentId, long start, long end) throws IOException {
+        return querySequenceWithAgg(AggregationBuilders.max(ProfileThreadSnapshotRecord.SEQUENCE).field(ProfileThreadSnapshotRecord.SEQUENCE), segmentId, start, end);
+    }
+}

oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2ProfileThreadSnapshotQueryDAO.java

@@ -18,6 +18,7 @@
 
 package org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao;
 
+import org.apache.skywalking.apm.util.StringUtil;
 import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord;
 import org.apache.skywalking.oap.server.core.profile.ProfileThreadSnapshotRecord;
 import org.apache.skywalking.oap.server.core.query.entity.BasicTrace;
@@ -31,10 +32,7 @@ import java.io.IOException;
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
+import java.util.*;
 
 public class H2ProfileThreadSnapshotQueryDAO implements IProfileThreadSnapshotQueryDAO {
     private JDBCHikariCPClient h2Client;
@@ -98,4 +96,75 @@ public class H2ProfileThreadSnapshotQueryDAO implements IProfileThreadSnapshotQu
         }
         return result;
     }
+
+    @Override
+    public int queryMinSequence(String segmentId, long start, long end) throws IOException {
+        return querySequenceWithAgg("min", segmentId, start, end);
+    }
+
+    @Override
+    public int queryMaxSequence(String segmentId, long start, long end) throws IOException {
+        return querySequenceWithAgg("max", segmentId, start, end);
+    }
+
+    @Override
+    public List<ProfileThreadSnapshotRecord> queryRecords(String segmentId, int minSequence, int maxSequence) throws IOException {
+        StringBuilder sql = new StringBuilder();
+        sql.append("select * from ").append(ProfileThreadSnapshotRecord.INDEX_NAME).append(" where ");
+        sql.append(" 1=1 ");
+        sql.append(" and ").append(ProfileThreadSnapshotRecord.SEGMENT_ID).append(" = ? ");
+        sql.append(" and ").append(ProfileThreadSnapshotRecord.SEQUENCE).append(" >= ? ");
+        sql.append(" and ").append(ProfileThreadSnapshotRecord.SEQUENCE).append(" < ? ");
+
+        Object[] params = new Object[] {segmentId, minSequence, maxSequence};
+
+        ArrayList<ProfileThreadSnapshotRecord> result = new ArrayList<>(maxSequence - minSequence);
+        try (Connection connection = h2Client.getConnection()) {
+
+            try (ResultSet resultSet = h2Client.executeQuery(connection, sql.toString(), params)) {
+                while (resultSet.next()) {
+                    ProfileThreadSnapshotRecord record = new ProfileThreadSnapshotRecord();
+
+                    record.setTaskId(resultSet.getString(ProfileThreadSnapshotRecord.TASK_ID));
+                    record.setSegmentId(resultSet.getString(ProfileThreadSnapshotRecord.SEGMENT_ID));
+                    record.setDumpTime(resultSet.getLong(ProfileThreadSnapshotRecord.DUMP_TIME));
+                    record.setSequence(resultSet.getInt(ProfileThreadSnapshotRecord.SEQUENCE));
+                    String dataBinaryBase64 = resultSet.getString(ProfileThreadSnapshotRecord.STACK_BINARY);
+                    if (StringUtil.isNotEmpty(dataBinaryBase64)) {
+                        record.setStackBinary(Base64.getDecoder().decode(dataBinaryBase64));
+                    }
+
+                    result.add(record);
+                }
+            }
+        } catch (SQLException e) {
+            throw new IOException(e);
+        }
+
+        return result;
+    }
+
+    private int querySequenceWithAgg(String aggType, String segmentId, long start, long end) throws IOException {
+        StringBuilder sql = new StringBuilder();
+        sql.append("select ").append(aggType).append("(").append(ProfileThreadSnapshotRecord.SEQUENCE).append(") from ").append(ProfileThreadSnapshotRecord.INDEX_NAME).append(" where ");
+        sql.append(" 1=1 ");
+        sql.append(" and ").append(ProfileThreadSnapshotRecord.SEGMENT_ID).append(" = ? ");
+        sql.append(" and ").append(ProfileThreadSnapshotRecord.DUMP_TIME).append(" >= ? ");
+        sql.append(" and ").append(ProfileThreadSnapshotRecord.DUMP_TIME).append(" <= ? ");
+
+        Object[] params = new Object[] {segmentId, start, end};
+
+        try (Connection connection = h2Client.getConnection()) {
+
+            try (ResultSet resultSet = h2Client.executeQuery(connection, sql.toString(), params)) {
+                while (resultSet.next()) {
+                    return resultSet.getInt(1);
+                }
+            }
+        } catch (SQLException e) {
+            throw new IOException(e);
+        }
+        return -1;
+    }
+
 }

oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageProvider.java

@@ -31,6 +31,7 @@ import org.apache.skywalking.oap.server.core.storage.cache.IServiceInstanceInven
 import org.apache.skywalking.oap.server.core.storage.cache.IServiceInventoryCacheDAO;
 import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskLogQueryDAO;
 import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskQueryDAO;
+import org.apache.skywalking.oap.server.core.storage.profile.IProfileThreadSnapshotQueryDAO;
 import org.apache.skywalking.oap.server.core.storage.query.IAggregationQueryDAO;
 import org.apache.skywalking.oap.server.core.storage.query.IAlarmQueryDAO;
 import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO;
@@ -109,6 +110,7 @@ public class MySQLStorageProvider extends ModuleProvider {
 
         this.registerServiceImplementation(IProfileTaskQueryDAO.class, new H2ProfileTaskQueryDAO(mysqlClient));
         this.registerServiceImplementation(IProfileTaskLogQueryDAO.class, new H2ProfileTaskLogQueryDAO(mysqlClient));
+        this.registerServiceImplementation(IProfileThreadSnapshotQueryDAO.class, new H2ProfileThreadSnapshotQueryDAO(mysqlClient));
     }
     
     @Override public void start() throws ServiceNotProvidedException, ModuleStartException {

test/e2e/e2e-mysql/src/docker/rc.d/rc0-prepare.sh

@@ -15,12 +15,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-MYSQL_URL="https://central.maven.org/maven2/mysql/mysql-connector-java/8.0.13/mysql-connector-java-8.0.13.jar"
+MYSQL_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.13/mysql-connector-java-8.0.13.jar"
 MYSQL_DRIVER="mysql-connector-java-8.0.13.jar"
 
 echo "MySQL database is storage provider..."   
 # Download MySQL connector.
-curl ${MYSQL_URL} > "${SW_HOME}/oap-libs/${MYSQL_DRIVER}"
+curl -L -o "${SW_HOME}/oap-libs/${MYSQL_DRIVER}" ${MYSQL_URL}
 [[ $? -ne 0 ]] && echo "Fail to download ${MYSQL_DRIVER}." && exit 1
 
 # Modify application.yml to set MySQL as storage provider.

test/e2e/e2e-profile/e2e-profile-service/src/main/java/org/apache/skywalking/e2e/profile/TestController.java

@@ -45,8 +45,8 @@ public class TestController {
         if (!createUser.getEnableProfiling()) {
             return user;
         } else {
-            // sleep 10 second
-            TimeUnit.SECONDS.sleep(10);
+            // sleep 6200 milliseconds
+            TimeUnit.MILLISECONDS.sleep(6200);
             return user;
         }
     }

test/e2e/e2e-profile/e2e-profile-test-runner/src/docker/rc.d/rc0-prepare.sh

@@ -21,12 +21,12 @@ original_wd=$(pwd)
 
 
 if test "${STORAGE}" = "mysql"; then
-  MYSQL_URL="https://central.maven.org/maven2/mysql/mysql-connector-java/8.0.13/mysql-connector-java-8.0.13.jar"
+  MYSQL_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.13/mysql-connector-java-8.0.13.jar"
   MYSQL_DRIVER="mysql-connector-java-8.0.13.jar"
 
   echo "MySQL database is storage provider..."
   # Download MySQL connector.
-  curl ${MYSQL_URL} > "${SW_HOME}/oap-libs/${MYSQL_DRIVER}"
+  curl -L -o "${SW_HOME}/oap-libs/${MYSQL_DRIVER}" ${MYSQL_URL}
   [[ $? -ne 0 ]] && echo "Fail to download ${MYSQL_DRIVER}." && exit 1
 fi
 

test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/ProfileClient.java

@@ -24,9 +24,7 @@ import org.apache.skywalking.e2e.SimpleQueryClient;
 import org.apache.skywalking.e2e.profile.creation.ProfileTaskCreationRequest;
 import org.apache.skywalking.e2e.profile.creation.ProfileTaskCreationResult;
 import org.apache.skywalking.e2e.profile.creation.ProfileTaskCreationResultWrapper;
-import org.apache.skywalking.e2e.profile.query.ProfileTaskQuery;
-import org.apache.skywalking.e2e.profile.query.ProfileTasks;
-import org.apache.skywalking.e2e.profile.query.Traces;
+import org.apache.skywalking.e2e.profile.query.*;
 import org.apache.skywalking.e2e.trace.Trace;
 import org.springframework.core.ParameterizedTypeReference;
 import org.springframework.http.HttpMethod;
@@ -119,6 +117,28 @@ public class ProfileClient extends SimpleQueryClient {
         return Objects.requireNonNull(responseEntity.getBody()).getData().getTraces();
     }
 
+    public ProfileAnalyzation getProfileAnalyzation(final String segmentId, long start, long end) throws IOException {
+        final URL queryFileUrl = Resources.getResource("getProfileAnalyzation.gql");
+        final String queryString = Resources.readLines(queryFileUrl, Charset.forName("UTF8"))
+                .stream()
+                .filter(it -> !it.startsWith("#"))
+                .collect(Collectors.joining())
+                .replace("{segmentId}", segmentId)
+                .replace("{start}", String.valueOf(start))
+                .replace("{end}", String.valueOf(end));
+        final ResponseEntity<GQLResponse<ProfileAnalyzation>> responseEntity = restTemplate.exchange(
+                new RequestEntity<>(queryString, HttpMethod.POST, URI.create(endpointUrl)),
+                new ParameterizedTypeReference<GQLResponse<ProfileAnalyzation>>() {
+                }
+        );
+
+        if (responseEntity.getStatusCode() != HttpStatus.OK) {
+            throw new RuntimeException("Response status != 200, actual: " + responseEntity.getStatusCode());
+        }
+
+        return Objects.requireNonNull(responseEntity.getBody()).getData();
+    }
+
 
 
 }

test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/query/ProfileAnalyzation.java

@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.skywalking.e2e.profile.query;
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class ProfileAnalyzation {
+
+    private ProfileStackTrees data;
+
+    @Data
+    public static class ProfileStackTrees {
+        private List<ProfileStackTree> trees;
+
+        @Override
+        public String toString() {
+            return "ProfileStackTrees{" +
+                    "trees=" + trees +
+                    '}';
+        }
+    }
+
+    @Data
+    public static class ProfileStackTree {
+        private List<ProfileStackElement> elements;
+
+        @Override
+        public String toString() {
+            return "ProfileStackTree{" +
+                    "elements=" + elements +
+                    '}';
+        }
+    }
+
+    @Data
+    public static class ProfileStackElement {
+        private String id;
+        private String parentId;
+        private String codeSignature;
+        private String duration;
+        private String durationChildExcluded;
+        private String count;
+
+        @Override
+        public String toString() {
+            return "ProfileStackElement{" +
+                    "id='" + id + '\'' +
+                    ", parentId='" + parentId + '\'' +
+                    ", codeSignature='" + codeSignature + '\'' +
+                    ", duration='" + duration + '\'' +
+                    ", durationChildExcluded='" + durationChildExcluded + '\'' +
+                    ", count='" + count + '\'' +
+                    '}';
+        }
+    }
+
+    @Override
+    public String toString() {
+        return "ProfileAnalyzation{" +
+                "data=" + data +
+                '}';
+    }
+}

test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/query/ProfileStackElementMatcher.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.skywalking.e2e.profile.query;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.skywalking.e2e.verification.AbstractMatcher;
+
+@Setter
+@Getter
+public class ProfileStackElementMatcher extends AbstractMatcher<ProfileAnalyzation.ProfileStackElement> {
+
+    private String id;
+    private String parentId;
+    private String codeSignature;
+    private String duration;
+    private String durationChildExcluded;
+    private String count;
+
+    @Override
+    public void verify(ProfileAnalyzation.ProfileStackElement element) {
+        doVerify(id, element.getId());
+        doVerify(parentId, element.getParentId());
+        doVerify(codeSignature, element.getCodeSignature());
+        doVerify(duration, element.getDuration());
+        doVerify(durationChildExcluded, element.getDurationChildExcluded());
+        doVerify(count, element.getCount());
+    }
+}

test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/query/ProfileStackTreeMatcher.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.skywalking.e2e.profile.query;
+
+import lombok.Data;
+import org.apache.skywalking.e2e.verification.AbstractMatcher;
+import org.assertj.core.api.Assertions;
+
+import java.util.List;
+
+@Data
+public class ProfileStackTreeMatcher extends AbstractMatcher<ProfileAnalyzation.ProfileStackTree> {
+
+    private List<ProfileStackElementMatcher> elements;
+
+    @Override
+    public void verify(ProfileAnalyzation.ProfileStackTree profileStackTree) {
+        Assertions.assertThat(profileStackTree.getElements()).hasSameSizeAs(this.elements);
+
+        int size = this.elements.size();
+
+        for (int i = 0; i < size; i++) {
+            elements.get(i).verify(profileStackTree.getElements().get(i));
+        }
+    }
+}

test/e2e/e2e-profile/e2e-profile-test-runner/src/main/resources/expected-data/org.apache.skywalking.e2e.ProfileVerificationITCase.profileAnayzation.yml

@@ -0,0 +1,389 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+elements:
+  - id: 1
+    parentId: 0
+    codeSignature: not null
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 2
+    parentId: 1
+    codeSignature: not null
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 3
+    parentId: 2
+    codeSignature: not null
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 4
+    parentId: 3
+    codeSignature: not null
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 5
+    parentId: 4
+    codeSignature: org.apache.tomcat.util.net.SocketProcessorBase.run:49
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 6
+    parentId: 5
+    codeSignature: org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun:1747
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 7
+    parentId: 6
+    codeSignature: org.apache.coyote.AbstractProtocol$ConnectionHandler.process:836
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 8
+    parentId: 7
+    codeSignature: org.apache.coyote.AbstractProcessorLight.process:66
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 9
+    parentId: 8
+    codeSignature: org.apache.coyote.http11.Http11Processor.service:408
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 10
+    parentId: 9
+    codeSignature: org.apache.catalina.connector.CoyoteAdapter.service:343
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 11
+    parentId: 10
+    codeSignature: org.apache.catalina.core.StandardEngineValve.invoke:74
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 12
+    parentId: 11
+    codeSignature: org.apache.catalina.valves.ErrorReportValve.invoke:92
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 13
+    parentId: 12
+    codeSignature: org.apache.catalina.core.StandardHostValve.invoke:-1
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 14
+    parentId: 13
+    codeSignature: org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.InstMethodsInter.intercept:93
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 15
+    parentId: 14
+    codeSignature: not null
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 16
+    parentId: 15
+    codeSignature: not null
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 17
+    parentId: 16
+    codeSignature: not null
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 18
+    parentId: 17
+    codeSignature: org.apache.catalina.authenticator.AuthenticatorBase.invoke:490
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 19
+    parentId: 18
+    codeSignature: org.apache.catalina.core.StandardContextValve.invoke:96
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 20
+    parentId: 19
+    codeSignature: org.apache.catalina.core.StandardWrapperValve.invoke:200
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 21
+    parentId: 20
+    codeSignature: org.apache.catalina.core.ApplicationFilterChain.doFilter:166
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 22
+    parentId: 21
+    codeSignature: org.apache.catalina.core.ApplicationFilterChain.internalDoFilter:193
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 23
+    parentId: 22
+    codeSignature: org.springframework.web.filter.OncePerRequestFilter.doFilter:107
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 24
+    parentId: 23
+    codeSignature: org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal:200
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 25
+    parentId: 24
+    codeSignature: org.apache.catalina.core.ApplicationFilterChain.doFilter:166
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 26
+    parentId: 25
+    codeSignature: org.apache.catalina.core.ApplicationFilterChain.internalDoFilter:193
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 27
+    parentId: 26
+    codeSignature: org.springframework.web.filter.OncePerRequestFilter.doFilter:107
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 28
+    parentId: 27
+    codeSignature: org.springframework.web.filter.HiddenHttpMethodFilter.doFilterInternal:93
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 29
+    parentId: 28
+    codeSignature: org.apache.catalina.core.ApplicationFilterChain.doFilter:166
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 30
+    parentId: 29
+    codeSignature: org.apache.catalina.core.ApplicationFilterChain.internalDoFilter:193
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 31
+    parentId: 30
+    codeSignature: org.springframework.web.filter.OncePerRequestFilter.doFilter:107
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 32
+    parentId: 31
+    codeSignature: org.springframework.web.filter.FormContentFilter.doFilterInternal:92
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 33
+    parentId: 32
+    codeSignature: org.apache.catalina.core.ApplicationFilterChain.doFilter:166
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 34
+    parentId: 33
+    codeSignature: org.apache.catalina.core.ApplicationFilterChain.internalDoFilter:193
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 35
+    parentId: 34
+    codeSignature: org.springframework.web.filter.OncePerRequestFilter.doFilter:107
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 36
+    parentId: 35
+    codeSignature: org.springframework.web.filter.RequestContextFilter.doFilterInternal:99
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 37
+    parentId: 36
+    codeSignature: org.apache.catalina.core.ApplicationFilterChain.doFilter:166
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 38
+    parentId: 37
+    codeSignature: org.apache.catalina.core.ApplicationFilterChain.internalDoFilter:193
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 39
+    parentId: 38
+    codeSignature: org.apache.tomcat.websocket.server.WsFilter.doFilter:53
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 40
+    parentId: 39
+    codeSignature: org.apache.catalina.core.ApplicationFilterChain.doFilter:166
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 41
+    parentId: 40
+    codeSignature: org.apache.catalina.core.ApplicationFilterChain.internalDoFilter:231
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 42
+    parentId: 41
+    codeSignature: javax.servlet.http.HttpServlet.service:741
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 43
+    parentId: 42
+    codeSignature: org.springframework.web.servlet.FrameworkServlet.service:882
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 44
+    parentId: 43
+    codeSignature: javax.servlet.http.HttpServlet.service:660
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 45
+    parentId: 44
+    codeSignature: org.springframework.web.servlet.FrameworkServlet.doPost:908
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 46
+    parentId: 45
+    codeSignature: org.springframework.web.servlet.FrameworkServlet.processRequest:1005
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 47
+    parentId: 46
+    codeSignature: org.springframework.web.servlet.DispatcherServlet.doService:942
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 48
+    parentId: 47
+    codeSignature: org.springframework.web.servlet.DispatcherServlet.doDispatch:1039
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 49
+    parentId: 48
+    codeSignature: org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle:87
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 50
+    parentId: 49
+    codeSignature: org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal:797
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 51
+    parentId: 50
+    codeSignature: org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod:892
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 52
+    parentId: 51
+    codeSignature: org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle:104
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 53
+    parentId: 52
+    codeSignature: org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest:138
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 54
+    parentId: 53
+    codeSignature: org.springframework.web.method.support.InvocableHandlerMethod.doInvoke:190
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 55
+    parentId: 54
+    codeSignature: not null
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 56
+    parentId: 55
+    codeSignature: sun.reflect.DelegatingMethodAccessorImpl.invoke:43
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 57
+    parentId: 56
+    codeSignature: sun.reflect.NativeMethodAccessorImpl.invoke:62
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 58
+    parentId: 57
+    codeSignature: sun.reflect.NativeMethodAccessorImpl.invoke0:-2
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 59
+    parentId: 58
+    codeSignature: org.apache.skywalking.e2e.profile.TestController.createAuthor:49
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 60
+    parentId: 59
+    codeSignature: java.util.concurrent.TimeUnit.sleep:386
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 61
+    parentId: 60
+    codeSignature: java.lang.Thread.sleep:340
+    duration: gt 0
+    durationChildExcluded: 0
+    count: gt 0
+  - id: 62
+    parentId: 61
+    codeSignature: java.lang.Thread.sleep:-2
+    duration: gt 0
+    durationChildExcluded: gt 0
+    count: gt 0

test/e2e/e2e-profile/e2e-profile-test-runner/src/main/resources/getProfileAnalyzation.gql

@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+  "query":"query getProfileAnalyze($segmentId: String!, $start: Long!, $end: Long!) {
+    data: getProfileAnalyze(segmentId: $segmentId, start: $start, end: $end) {
+      trees {
+		elements {
+          id
+          parentId
+          codeSignature
+          duration
+          durationChildExcluded
+          count
+		}
+	  }
+    }
+  }",
+  "variables": {
+    "segmentId": "{segmentId}",
+	"start": "{start}",
+	"end": "{end}"
+  }
+}

test/e2e/e2e-profile/e2e-profile-test-runner/src/test/java/org/apache/skywalking/e2e/ProfileVerificationITCase.java

@@ -22,9 +22,7 @@ import org.apache.skywalking.e2e.profile.ProfileClient;
 import org.apache.skywalking.e2e.profile.creation.ProfileTaskCreationRequest;
 import org.apache.skywalking.e2e.profile.creation.ProfileTaskCreationResult;
 import org.apache.skywalking.e2e.profile.creation.ProfileTaskCreationResultMatcher;
-import org.apache.skywalking.e2e.profile.query.ProfileTaskQuery;
-import org.apache.skywalking.e2e.profile.query.ProfileTasks;
-import org.apache.skywalking.e2e.profile.query.ProfilesTasksMatcher;
+import org.apache.skywalking.e2e.profile.query.*;
 import org.apache.skywalking.e2e.service.Service;
 import org.apache.skywalking.e2e.service.ServicesMatcher;
 import org.apache.skywalking.e2e.service.ServicesQuery;
@@ -50,6 +48,7 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
 import org.springframework.web.client.RestTemplate;
 import org.yaml.snakeyaml.Yaml;
 
+import java.io.IOException;
 import java.io.InputStream;
 import java.time.LocalDateTime;
 import java.time.ZoneOffset;
@@ -99,6 +98,7 @@ public class ProfileVerificationITCase {
                                 .end(LocalDateTime.now())
                                 .orderByDuration()
                 );
+                LOGGER.info("query traces: {}", traces);
                 if (!traces.isEmpty()) {
                     break;
                 }
@@ -108,20 +108,16 @@ public class ProfileVerificationITCase {
         }
 
         // verify basic info
-        int verifyServiceCount = 3;
-        for (int i = 1; i <= verifyServiceCount; i++) {
+        doRetryableVerification(() -> {
             try {
                 verifyServices(minutesAgo);
             } catch (Exception e) {
-                if (i == verifyServiceCount) {
-                    throw new IllegalStateException("match services fail!", e);
-                }
+                LOGGER.warn(e.getMessage(), e);
             }
-        }
+        });
 
         // create profile task
         verifyCreateProfileTask(minutesAgo);
-
     }
 
     private ResponseEntity<String> sendRequest(boolean needProfiling) {
@@ -148,8 +144,8 @@ public class ProfileVerificationITCase {
                 .endpointName("/e2e/users")
                 .duration(1)
                 .startTime(-1)
-                .minDurationThreshold(1000)
-                .dumpPeriod(50)
+                .minDurationThreshold(1500)
+                .dumpPeriod(500)
                 .maxSamplingCount(5).build();
 
         // verify create task
@@ -172,9 +168,9 @@ public class ProfileVerificationITCase {
         verifyProfiledSegment(creationResult.getId());
     }
 
-    private void verifyProfiledSegment(String taskId) throws InterruptedException {
-        // found segment id
-        String foundedSegmentId = null;
+    private void verifyProfiledSegment(String taskId) throws InterruptedException, IOException {
+        // found trace
+        Trace foundedTrace = null;
         for (int i = 0; i < 10; i++) {
             try {
                 List<Trace> traces = profileClient.getProfiledTraces(taskId);
@@ -183,7 +179,7 @@ public class ProfileVerificationITCase {
                 InputStream expectedInputStream = new ClassPathResource("expected-data/org.apache.skywalking.e2e.ProfileVerificationITCase.profileSegments.yml").getInputStream();
                 final TracesMatcher tracesMatcher = new Yaml().loadAs(expectedInputStream, TracesMatcher.class);
                 tracesMatcher.verifyLoosely(traces);
-                foundedSegmentId = traces.get(0).getKey();
+                foundedTrace = traces.get(0);
                 break;
 
             } catch (Exception e) {
@@ -193,6 +189,17 @@ public class ProfileVerificationITCase {
                 TimeUnit.SECONDS.sleep(retryInterval);
             }
         }
+
+        String segmentId = foundedTrace.getKey();
+        long start = Long.parseLong(foundedTrace.getStart());
+        long end = start + foundedTrace.getDuration();
+        ProfileAnalyzation analyzation = profileClient.getProfileAnalyzation(segmentId, start, end);
+        LOGGER.info("get profile analyzation : {}", analyzation);
+        InputStream expectedInputStream = new ClassPathResource("expected-data/org.apache.skywalking.e2e.ProfileVerificationITCase.profileAnayzation.yml").getInputStream();
+        final ProfileStackTreeMatcher servicesMatcher = new Yaml().loadAs(expectedInputStream, ProfileStackTreeMatcher.class);
+        servicesMatcher.verify(analyzation.getData().getTrees().get(0));
+
+
     }
 
     private void verifyProfileTask(int serviceId, String verifyResources) throws InterruptedException {
@@ -278,4 +285,15 @@ public class ProfileVerificationITCase {
         return instances;
     }
 
+    private void doRetryableVerification(Runnable runnable) throws InterruptedException {
+        while (true) {
+            try {
+                runnable.run();
+                break;
+            } catch (Throwable ignored) {
+                Thread.sleep(retryInterval);
+            }
+        }
+    }
+
 }