Including the original OAP config into profile exporter tool (#13101)

.github/workflows/skywalking.yaml

@@ -329,7 +329,7 @@ jobs:
       ( always() && ! cancelled() ) &&
       ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true')
     name: E2E test
-    needs: [docker]
+    needs: [docker, dist-tar]
     runs-on: ${{ matrix.test.runs-on || 'ubuntu-latest' }}
     timeout-minutes: 60
     env:
@@ -715,6 +715,11 @@ jobs:
         run: |
           find docker-images -name "*.tar" -exec docker load -i {} \;
           find docker-images -name "*.tar" -exec rm {} \;
+      - uses: actions/download-artifact@v4
+        name: Download distribution tar
+        with:
+          name: dist
+          path: dist
       - name: Login to ghcr
         uses: docker/login-action@v3
         with:

apm-dist/src/main/assembly/binary.xml

@@ -84,7 +84,14 @@
         </fileSet>
 
         <fileSet>
-            <directory>${project.basedir}/../oap-server/server-tools/profile-exporter/tool-profile-snapshot-exporter/target/oap-libs</directory>
+            <directory>${project.basedir}/../oap-server/server-tools/profile-exporter/tool-profile-snapshot-exporter/target/</directory>
+            <includes>
+                <include>tool-profile-snapshot-exporter-${project.version}.jar</include>
+            </includes>
+            <outputDirectory>oap-libs</outputDirectory>
+        </fileSet>
+        <fileSet>
+            <directory>${project.basedir}/../oap-server/server-tools/profile-exporter/tool-profile-snapshot-exporter/target/oap-libs/</directory>
             <outputDirectory>oap-libs</outputDirectory>
         </fileSet>
 

oap-server/server-tools/profile-exporter/tool-profile-snapshot-server-mock/src/main/java/org/apache/skywalking/oap/server/tool/profile/core/MockCoreModuleProvider.java

@@ -23,26 +23,41 @@ import org.apache.skywalking.oap.server.core.CoreModuleConfig;
 import org.apache.skywalking.oap.server.core.CoreModuleProvider;
 import org.apache.skywalking.oap.server.core.analysis.meter.MeterSystem;
 import org.apache.skywalking.oap.server.core.annotation.AnnotationScan;
+import org.apache.skywalking.oap.server.core.cache.AsyncProfilerTaskCache;
 import org.apache.skywalking.oap.server.core.cache.NetworkAddressAliasCache;
 import org.apache.skywalking.oap.server.core.cache.ProfileTaskCache;
 import org.apache.skywalking.oap.server.core.command.CommandService;
 import org.apache.skywalking.oap.server.core.config.ConfigService;
 import org.apache.skywalking.oap.server.core.config.DownSamplingConfigService;
+import org.apache.skywalking.oap.server.core.config.HierarchyDefinitionService;
 import org.apache.skywalking.oap.server.core.config.IComponentLibraryCatalogService;
 import org.apache.skywalking.oap.server.core.config.NamingControl;
+import org.apache.skywalking.oap.server.core.config.group.EndpointNameGroupService;
 import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping;
+import org.apache.skywalking.oap.server.core.hierarchy.HierarchyService;
+import org.apache.skywalking.oap.server.core.management.ui.menu.UIMenuManagementService;
 import org.apache.skywalking.oap.server.core.management.ui.template.UITemplateManagementService;
 import org.apache.skywalking.oap.server.core.oal.rt.OALEngineLoaderService;
+import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.AsyncProfilerMutationService;
+import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.AsyncProfilerQueryService;
+import org.apache.skywalking.oap.server.core.profiling.continuous.ContinuousProfilingMutationService;
+import org.apache.skywalking.oap.server.core.profiling.continuous.ContinuousProfilingQueryService;
+import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingMutationService;
+import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingQueryService;
 import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskMutationService;
 import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskQueryService;
 import org.apache.skywalking.oap.server.core.query.AggregationQueryService;
 import org.apache.skywalking.oap.server.core.query.AlarmQueryService;
 import org.apache.skywalking.oap.server.core.query.BrowserLogQueryService;
 import org.apache.skywalking.oap.server.core.query.EventQueryService;
+import org.apache.skywalking.oap.server.core.query.HierarchyQueryService;
 import org.apache.skywalking.oap.server.core.query.LogQueryService;
 import org.apache.skywalking.oap.server.core.query.MetadataQueryService;
 import org.apache.skywalking.oap.server.core.query.MetricsMetadataQueryService;
 import org.apache.skywalking.oap.server.core.query.MetricsQueryService;
+import org.apache.skywalking.oap.server.core.query.RecordQueryService;
+import org.apache.skywalking.oap.server.core.query.TTLStatusQuery;
+import org.apache.skywalking.oap.server.core.query.TagAutoCompleteQueryService;
 import org.apache.skywalking.oap.server.core.query.TopNRecordsQueryService;
 import org.apache.skywalking.oap.server.core.query.TopologyQueryService;
 import org.apache.skywalking.oap.server.core.query.TraceQueryService;
@@ -52,6 +67,7 @@ import org.apache.skywalking.oap.server.core.server.GRPCHandlerRegister;
 import org.apache.skywalking.oap.server.core.server.HTTPHandlerRegister;
 import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
 import org.apache.skywalking.oap.server.core.source.SourceReceiver;
+import org.apache.skywalking.oap.server.core.status.ServerStatusService;
 import org.apache.skywalking.oap.server.core.storage.StorageException;
 import org.apache.skywalking.oap.server.core.storage.model.IModelManager;
 import org.apache.skywalking.oap.server.core.storage.model.ModelCreator;
@@ -121,6 +137,10 @@ public class MockCoreModuleProvider extends CoreModuleProvider {
 
         CoreModuleConfig moduleConfig = new CoreModuleConfig();
         this.registerServiceImplementation(ConfigService.class, new ConfigService(moduleConfig, this));
+        this.registerServiceImplementation(ServerStatusService.class, new ServerStatusService(getManager(), moduleConfig));
+        moduleConfig.setEnableHierarchy(false);
+        this.registerServiceImplementation(HierarchyDefinitionService.class, new HierarchyDefinitionService(moduleConfig));
+        this.registerServiceImplementation(HierarchyService.class, new HierarchyService(getManager(), moduleConfig));
         this.registerServiceImplementation(
                 DownSamplingConfigService.class, new DownSamplingConfigService(Collections.emptyList()));
 
@@ -171,8 +191,39 @@ public class MockCoreModuleProvider extends CoreModuleProvider {
 
         // Management
         this.registerServiceImplementation(UITemplateManagementService.class, new UITemplateManagementService(getManager()));
+        this.registerServiceImplementation(
+            UIMenuManagementService.class, new UIMenuManagementService(getManager(), moduleConfig));
 
         this.registerServiceImplementation(EventQueryService.class, new EventQueryService(getManager()));
+        this.registerServiceImplementation(RecordQueryService.class, new RecordQueryService(getManager()));
+        this.registerServiceImplementation(HierarchyQueryService.class, new HierarchyQueryService(getManager(), moduleConfig));
+        this.registerServiceImplementation(
+            TTLStatusQuery.class, new TTLStatusQuery(
+                getManager(),
+                moduleConfig.getMetricsDataTTL(),
+                moduleConfig.getRecordDataTTL()
+            )
+        );
+        this.registerServiceImplementation(
+            TagAutoCompleteQueryService.class, new TagAutoCompleteQueryService(getManager(), moduleConfig));
+        this.registerServiceImplementation(
+            AsyncProfilerMutationService.class, new AsyncProfilerMutationService(getManager()));
+        this.registerServiceImplementation(
+            AsyncProfilerQueryService.class, new AsyncProfilerQueryService(getManager()));
+        this.registerServiceImplementation(
+            AsyncProfilerTaskCache.class, new AsyncProfilerTaskCache(getManager(), moduleConfig));
+        this.registerServiceImplementation(
+            EBPFProfilingMutationService.class, new EBPFProfilingMutationService(getManager()));
+        this.registerServiceImplementation(
+            EBPFProfilingQueryService.class,
+            new EBPFProfilingQueryService(getManager(), moduleConfig, this.storageModels)
+        );
+        this.registerServiceImplementation(
+            ContinuousProfilingMutationService.class, new ContinuousProfilingMutationService(getManager()));
+        this.registerServiceImplementation(
+            ContinuousProfilingQueryService.class, new ContinuousProfilingQueryService(getManager()));
+        this.registerServiceImplementation(EndpointNameGroupService.class, new EndpointNameGrouping());
+
     }
 
     @Override

test/e2e-v2/cases/profiling/trace/es/docker-compose.yml

@@ -20,6 +20,8 @@ services:
     image: elastic/elasticsearch:7.15.0
     expose:
       - 9200
+    ports:
+      - 9200:9200
     networks:
       - e2e
     environment:

test/e2e-v2/cases/profiling/trace/es/e2e.yaml

@@ -35,3 +35,4 @@ verify:
   cases:
     - includes:
         - ../profiling-cases.yaml
+        - ../profiling-cases-es.yaml

test/e2e-v2/cases/profiling/trace/expected/profile-dump-process.yml

@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log_output:
+  {{- range .log_output }}
+    - "{{ notEmpty . }}"
+  {{- end }}

test/e2e-v2/cases/profiling/trace/expected/profile-dump-result.yml

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+success

test/e2e-v2/cases/profiling/trace/profiling-cases-es.yaml

@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is used to show how to write configuration files and can be used to test.
+
+  cases:
+    # dump the profiling data
+    - query: | 
+        cd dist && tar -zxf *.tar.gz && cd apache-skywalking-apm-bin && \
+        mkdir -p export && \
+        SW_STORAGE=elasticsearch bash tools/profile-exporter/profile_exporter.sh \
+          --taskid=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling trace list --service-name=e2e-service-provider --endpoint-name=POST:/profile/{name} | yq e '.[0].id' -) \
+          --traceid=$( \
+            swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling trace segment-list --task-id=$( \
+              swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling trace list --service-name=e2e-service-provider --endpoint-name=POST:/profile/{name} | yq e '.[0].id' - \
+            ) | yq e '.[0] | .traceid' - \
+          ) \
+          ./export > log.txt && \
+          yq -n '{"log_output": load_str("log.txt") | split("\n") | .[:-1]}'
+      expected: expected/profile-dump-process.yml
+    - query: |
+        cd dist/apache-skywalking-apm-bin && \
+        taskid=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling trace list --service-name=e2e-service-provider --endpoint-name=POST:/profile/{name} | yq e '.[0].id' -) \
+        traceid=$(swctl --display yaml --base-url="http://${oap_host}:${oap_12800}/graphql" profiling trace segment-list --task-id="$taskid" | yq e '.[0] | .traceid' -); \
+        [ -d "./export/$traceid/work" ] && [ "$(ls -A "./export/$traceid/work" 2>/dev/null)" ] && echo "success" || echo "failure"
+      expected: expected/profile-dump-result.yml
+
+

tools/profile-exporter/profile_exporter.sh

@@ -44,6 +44,7 @@ done
 
 # prepare paths
 oap_libs_dir="${exporter_dir}/../../oap-libs"
+oap_config="${exporter_dir}/../../config"
 exporter_log_file="${exporter_dir}/profile_exporter_log4j2.xml"
 tool_application_config="${exporter_dir}/application.yml"
 [[ ! -f ${tool_application_config} ]] \
@@ -70,7 +71,7 @@ JAVA_OPTS=" -Xms256M -Xmx512M"
 _RUNJAVA=${JAVA_HOME}/bin/java
 [ -z "$JAVA_HOME" ] && _RUNJAVA=java
 
-CLASSPATH="${work_dir}/config:$CLASSPATH"
+CLASSPATH="${work_dir}/config:${oap_config}:$CLASSPATH"
 for i in "${oap_libs_dir}"/*.jar
 do
     CLASSPATH="$i:$CLASSPATH"