From 5038595bbb613466bff6a90b2fa4286f0aa66955 Mon Sep 17 00:00:00 2001 From: mrproliu <741550557@qq.com> Date: Fri, 14 Mar 2025 16:53:34 +0800 Subject: [PATCH] Including the original OAP config into profile exporter tool (#13101) --- .github/workflows/skywalking.yaml | 7 ++- apm-dist/src/main/assembly/binary.xml | 9 +++- .../profile/core/MockCoreModuleProvider.java | 51 +++++++++++++++++++ .../profiling/trace/es/docker-compose.yml | 2 + test/e2e-v2/cases/profiling/trace/es/e2e.yaml | 1 + .../trace/expected/profile-dump-process.yml | 19 +++++++ .../trace/expected/profile-dump-result.yml | 16 ++++++ .../profiling/trace/profiling-cases-es.yaml | 40 +++++++++++++++ tools/profile-exporter/profile_exporter.sh | 3 +- 9 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 test/e2e-v2/cases/profiling/trace/expected/profile-dump-process.yml create mode 100644 test/e2e-v2/cases/profiling/trace/expected/profile-dump-result.yml create mode 100644 test/e2e-v2/cases/profiling/trace/profiling-cases-es.yaml diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index 84fd4c5416..b264ea9611 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -329,7 +329,7 @@ jobs: ( always() && ! cancelled() ) && ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test - needs: [docker] + needs: [docker, dist-tar] runs-on: ${{ matrix.test.runs-on || 'ubuntu-latest' }} timeout-minutes: 60 env: @@ -715,6 +715,11 @@ jobs: run: | find docker-images -name "*.tar" -exec docker load -i {} \; find docker-images -name "*.tar" -exec rm {} \; + - uses: actions/download-artifact@v4 + name: Download distribution tar + with: + name: dist + path: dist - name: Login to ghcr uses: docker/login-action@v3 with: diff --git a/apm-dist/src/main/assembly/binary.xml b/apm-dist/src/main/assembly/binary.xml index f03d91ae46..7848d357dc 100644 --- a/apm-dist/src/main/assembly/binary.xml +++ b/apm-dist/src/main/assembly/binary.xml @@ -84,7 +84,14 @@ - ${project.basedir}/../oap-server/server-tools/profile-exporter/tool-profile-snapshot-exporter/target/oap-libs + ${project.basedir}/../oap-server/server-tools/profile-exporter/tool-profile-snapshot-exporter/target/ + + tool-profile-snapshot-exporter-${project.version}.jar + + oap-libs + + + ${project.basedir}/../oap-server/server-tools/profile-exporter/tool-profile-snapshot-exporter/target/oap-libs/ oap-libs diff --git a/oap-server/server-tools/profile-exporter/tool-profile-snapshot-server-mock/src/main/java/org/apache/skywalking/oap/server/tool/profile/core/MockCoreModuleProvider.java b/oap-server/server-tools/profile-exporter/tool-profile-snapshot-server-mock/src/main/java/org/apache/skywalking/oap/server/tool/profile/core/MockCoreModuleProvider.java index 2296f1113b..4639dfe334 100755 --- a/oap-server/server-tools/profile-exporter/tool-profile-snapshot-server-mock/src/main/java/org/apache/skywalking/oap/server/tool/profile/core/MockCoreModuleProvider.java +++ b/oap-server/server-tools/profile-exporter/tool-profile-snapshot-server-mock/src/main/java/org/apache/skywalking/oap/server/tool/profile/core/MockCoreModuleProvider.java @@ -23,26 +23,41 @@ import org.apache.skywalking.oap.server.core.CoreModuleConfig; import org.apache.skywalking.oap.server.core.CoreModuleProvider; import org.apache.skywalking.oap.server.core.analysis.meter.MeterSystem; import org.apache.skywalking.oap.server.core.annotation.AnnotationScan; +import org.apache.skywalking.oap.server.core.cache.AsyncProfilerTaskCache; import org.apache.skywalking.oap.server.core.cache.NetworkAddressAliasCache; import org.apache.skywalking.oap.server.core.cache.ProfileTaskCache; import org.apache.skywalking.oap.server.core.command.CommandService; import org.apache.skywalking.oap.server.core.config.ConfigService; import org.apache.skywalking.oap.server.core.config.DownSamplingConfigService; +import org.apache.skywalking.oap.server.core.config.HierarchyDefinitionService; import org.apache.skywalking.oap.server.core.config.IComponentLibraryCatalogService; import org.apache.skywalking.oap.server.core.config.NamingControl; +import org.apache.skywalking.oap.server.core.config.group.EndpointNameGroupService; import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping; +import org.apache.skywalking.oap.server.core.hierarchy.HierarchyService; +import org.apache.skywalking.oap.server.core.management.ui.menu.UIMenuManagementService; import org.apache.skywalking.oap.server.core.management.ui.template.UITemplateManagementService; import org.apache.skywalking.oap.server.core.oal.rt.OALEngineLoaderService; +import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.AsyncProfilerMutationService; +import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.AsyncProfilerQueryService; +import org.apache.skywalking.oap.server.core.profiling.continuous.ContinuousProfilingMutationService; +import org.apache.skywalking.oap.server.core.profiling.continuous.ContinuousProfilingQueryService; +import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingMutationService; +import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingQueryService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskMutationService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskQueryService; import org.apache.skywalking.oap.server.core.query.AggregationQueryService; import org.apache.skywalking.oap.server.core.query.AlarmQueryService; import org.apache.skywalking.oap.server.core.query.BrowserLogQueryService; import org.apache.skywalking.oap.server.core.query.EventQueryService; +import org.apache.skywalking.oap.server.core.query.HierarchyQueryService; import org.apache.skywalking.oap.server.core.query.LogQueryService; import org.apache.skywalking.oap.server.core.query.MetadataQueryService; import org.apache.skywalking.oap.server.core.query.MetricsMetadataQueryService; import org.apache.skywalking.oap.server.core.query.MetricsQueryService; +import org.apache.skywalking.oap.server.core.query.RecordQueryService; +import org.apache.skywalking.oap.server.core.query.TTLStatusQuery; +import org.apache.skywalking.oap.server.core.query.TagAutoCompleteQueryService; import org.apache.skywalking.oap.server.core.query.TopNRecordsQueryService; import org.apache.skywalking.oap.server.core.query.TopologyQueryService; import org.apache.skywalking.oap.server.core.query.TraceQueryService; @@ -52,6 +67,7 @@ import org.apache.skywalking.oap.server.core.server.GRPCHandlerRegister; import org.apache.skywalking.oap.server.core.server.HTTPHandlerRegister; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.core.status.ServerStatusService; import org.apache.skywalking.oap.server.core.storage.StorageException; import org.apache.skywalking.oap.server.core.storage.model.IModelManager; import org.apache.skywalking.oap.server.core.storage.model.ModelCreator; @@ -121,6 +137,10 @@ public class MockCoreModuleProvider extends CoreModuleProvider { CoreModuleConfig moduleConfig = new CoreModuleConfig(); this.registerServiceImplementation(ConfigService.class, new ConfigService(moduleConfig, this)); + this.registerServiceImplementation(ServerStatusService.class, new ServerStatusService(getManager(), moduleConfig)); + moduleConfig.setEnableHierarchy(false); + this.registerServiceImplementation(HierarchyDefinitionService.class, new HierarchyDefinitionService(moduleConfig)); + this.registerServiceImplementation(HierarchyService.class, new HierarchyService(getManager(), moduleConfig)); this.registerServiceImplementation( DownSamplingConfigService.class, new DownSamplingConfigService(Collections.emptyList())); @@ -171,8 +191,39 @@ public class MockCoreModuleProvider extends CoreModuleProvider { // Management this.registerServiceImplementation(UITemplateManagementService.class, new UITemplateManagementService(getManager())); + this.registerServiceImplementation( + UIMenuManagementService.class, new UIMenuManagementService(getManager(), moduleConfig)); this.registerServiceImplementation(EventQueryService.class, new EventQueryService(getManager())); + this.registerServiceImplementation(RecordQueryService.class, new RecordQueryService(getManager())); + this.registerServiceImplementation(HierarchyQueryService.class, new HierarchyQueryService(getManager(), moduleConfig)); + this.registerServiceImplementation( + TTLStatusQuery.class, new TTLStatusQuery( + getManager(), + moduleConfig.getMetricsDataTTL(), + moduleConfig.getRecordDataTTL() + ) + ); + this.registerServiceImplementation( + TagAutoCompleteQueryService.class, new TagAutoCompleteQueryService(getManager(), moduleConfig)); + this.registerServiceImplementation( + AsyncProfilerMutationService.class, new AsyncProfilerMutationService(getManager())); + this.registerServiceImplementation( + AsyncProfilerQueryService.class, new AsyncProfilerQueryService(getManager())); + this.registerServiceImplementation( + AsyncProfilerTaskCache.class, new AsyncProfilerTaskCache(getManager(), moduleConfig)); + this.registerServiceImplementation( + EBPFProfilingMutationService.class, new EBPFProfilingMutationService(getManager())); + this.registerServiceImplementation( + EBPFProfilingQueryService.class, + new EBPFProfilingQueryService(getManager(), moduleConfig, this.storageModels) + ); + this.registerServiceImplementation( + ContinuousProfilingMutationService.class, new ContinuousProfilingMutationService(getManager())); + this.registerServiceImplementation( + ContinuousProfilingQueryService.class, new ContinuousProfilingQueryService(getManager())); + this.registerServiceImplementation(EndpointNameGroupService.class, new EndpointNameGrouping()); + } @Override diff --git a/test/e2e-v2/cases/profiling/trace/es/docker-compose.yml b/test/e2e-v2/cases/profiling/trace/es/docker-compose.yml index 6bfae23cc8..cb777c2f35 100644 --- a/test/e2e-v2/cases/profiling/trace/es/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/trace/es/docker-compose.yml @@ -20,6 +20,8 @@ services: image: elastic/elasticsearch:7.15.0 expose: - 9200 + ports: + - 9200:9200 networks: - e2e environment: diff --git a/test/e2e-v2/cases/profiling/trace/es/e2e.yaml b/test/e2e-v2/cases/profiling/trace/es/e2e.yaml index 8a4fdf3a3e..e11265db92 100644 --- a/test/e2e-v2/cases/profiling/trace/es/e2e.yaml +++ b/test/e2e-v2/cases/profiling/trace/es/e2e.yaml @@ -35,3 +35,4 @@ verify: cases: - includes: - ../profiling-cases.yaml + - ../profiling-cases-es.yaml diff --git a/test/e2e-v2/cases/profiling/trace/expected/profile-dump-process.yml b/test/e2e-v2/cases/profiling/trace/expected/profile-dump-process.yml new file mode 100644 index 0000000000..673693ef73 --- /dev/null +++ b/test/e2e-v2/cases/profiling/trace/expected/profile-dump-process.yml @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log_output: + {{- range .log_output }} + - "{{ notEmpty . }}" + {{- end }} diff --git a/test/e2e-v2/cases/profiling/trace/expected/profile-dump-result.yml b/test/e2e-v2/cases/profiling/trace/expected/profile-dump-result.yml new file mode 100644 index 0000000000..edf76fee2c --- /dev/null +++ b/test/e2e-v2/cases/profiling/trace/expected/profile-dump-result.yml @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +success diff --git a/test/e2e-v2/cases/profiling/trace/profiling-cases-es.yaml b/test/e2e-v2/cases/profiling/trace/profiling-cases-es.yaml new file mode 100644 index 0000000000..8aacfe65b2 --- /dev/null +++ b/test/e2e-v2/cases/profiling/trace/profiling-cases-es.yaml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + + cases: + # dump the profiling data + - query: | + cd dist && tar -zxf *.tar.gz && cd apache-skywalking-apm-bin && \ + mkdir -p export && \ + SW_STORAGE=elasticsearch bash tools/profile-exporter/profile_exporter.sh \ + --taskid=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling trace list --service-name=e2e-service-provider --endpoint-name=POST:/profile/{name} | yq e '.[0].id' -) \ + --traceid=$( \ + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling trace segment-list --task-id=$( \ + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling trace list --service-name=e2e-service-provider --endpoint-name=POST:/profile/{name} | yq e '.[0].id' - \ + ) | yq e '.[0] | .traceid' - \ + ) \ + ./export > log.txt && \ + yq -n '{"log_output": load_str("log.txt") | split("\n") | .[:-1]}' + expected: expected/profile-dump-process.yml + - query: | + cd dist/apache-skywalking-apm-bin && \ + taskid=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling trace list --service-name=e2e-service-provider --endpoint-name=POST:/profile/{name} | yq e '.[0].id' -) \ + traceid=$(swctl --display yaml --base-url="http://${oap_host}:${oap_12800}/graphql" profiling trace segment-list --task-id="$taskid" | yq e '.[0] | .traceid' -); \ + [ -d "./export/$traceid/work" ] && [ "$(ls -A "./export/$traceid/work" 2>/dev/null)" ] && echo "success" || echo "failure" + expected: expected/profile-dump-result.yml + + diff --git a/tools/profile-exporter/profile_exporter.sh b/tools/profile-exporter/profile_exporter.sh index 98a5b56100..fb04d4228c 100755 --- a/tools/profile-exporter/profile_exporter.sh +++ b/tools/profile-exporter/profile_exporter.sh @@ -44,6 +44,7 @@ done # prepare paths oap_libs_dir="${exporter_dir}/../../oap-libs" +oap_config="${exporter_dir}/../../config" exporter_log_file="${exporter_dir}/profile_exporter_log4j2.xml" tool_application_config="${exporter_dir}/application.yml" [[ ! -f ${tool_application_config} ]] \ @@ -70,7 +71,7 @@ JAVA_OPTS=" -Xms256M -Xmx512M" _RUNJAVA=${JAVA_HOME}/bin/java [ -z "$JAVA_HOME" ] && _RUNJAVA=java -CLASSPATH="${work_dir}/config:$CLASSPATH" +CLASSPATH="${work_dir}/config:${oap_config}:$CLASSPATH" for i in "${oap_libs_dir}"/*.jar do CLASSPATH="$i:$CLASSPATH"