From f9257f5274f6a55e8bf8424444707aecf7a1f2c1 Mon Sep 17 00:00:00 2001 From: Daming Date: Thu, 30 Jul 2020 11:55:49 +0800 Subject: [PATCH] Provide kafka as collector/reporter (#4847) --- .github/workflows/e2e.kafka.yaml | 97 +++++ apm-protocol/apm-network/src/main/proto | 2 +- .../apm/agent/core/conf/Config.java | 1 + .../apm/agent/core/jvm/JVMMetricsSender.java | 109 ++++++ .../apm/agent/core/jvm/JVMService.java | 78 +--- .../apm/agent/core/meter/MeterSender.java | 154 ++++++++ .../apm/agent/core/meter/MeterService.java | 121 +----- .../core/profile/ProfileSnapshotSender.java | 126 +++++++ .../profile/ProfileTaskChannelService.java | 81 +--- ...skywalking.apm.agent.core.boot.BootService | 5 +- .../agent/core/boot/ServiceManagerTest.java | 4 +- .../agent/core/meter/MeterServiceTest.java | 9 +- apm-sniffer/config/agent.config | 3 + .../kafka-reporter-plugin/pom.xml | 162 ++++++++ .../KafkaContextManagerExtendService.java | 36 ++ .../core/kafka/KafkaJVMMetricsSender.java | 99 +++++ .../agent/core/kafka/KafkaMeterSender.java | 69 ++++ .../core/kafka/KafkaProducerManager.java | 104 +++++ .../kafka/KafkaProfileSnapshotSender.java | 72 ++++ .../core/kafka/KafkaReporterPluginConfig.java | 48 +++ .../KafkaServiceManagementServiceClient.java | 121 ++++++ .../kafka/KafkaTraceSegmentServiceClient.java | 84 +++++ ...skywalking.apm.agent.core.boot.BootService | 23 ++ apm-sniffer/optional-reporter-plugins/pom.xml | 70 ++++ apm-sniffer/pom.xml | 1 + dist-material/release-docs/LICENSE | 5 + dist-material/release-docs/NOTICE | 17 + .../licenses/LICENSE-zstd-jni.txt | 26 ++ .../guides/Java-Plugin-Development-Guide.md | 4 +- docs/en/setup/backend/backend-fetcher.md | 50 ++- docs/en/setup/backend/backend-meter.md | 12 + docs/en/setup/backend/backend-receivers.md | 5 +- .../setup/backend/configuration-vocabulary.md | 18 +- docs/en/setup/backend/dynamic-config.md | 6 +- docs/en/setup/backend/trace-sampling.md | 4 +- .../How-to-enable-kafka-reporter.md | 18 + .../setup/service-agent/java-agent/README.md | 7 + oap-server/analyzer/agent-analyzer/pom.xml | 47 +++ .../analyzer/module/AnalyzerModule.java | 39 ++ .../provider/AnalyzerModuleConfig.java} | 10 +- .../provider/AnalyzerModuleProvider.java | 149 ++++++++ .../provider/jvm}/JVMSourceDispatcher.java | 6 +- .../provider/meter}/config/MeterConfig.java | 2 +- .../provider/meter}/config/MeterConfigs.java | 2 +- .../meter}/config/MeterDataConfig.java | 2 +- .../provider/meter}/config/Scope.java | 2 +- .../provider/meter}/process/EvalData.java | 2 +- .../meter}/process/EvalHistogramData.java | 2 +- .../meter}/process/EvalMultipleData.java | 2 +- .../meter}/process/EvalSingleData.java | 2 +- .../meter/process/IMeterProcessService.java} | 16 +- .../provider/meter}/process/MeterBuilder.java | 6 +- .../meter}/process/MeterEvalOperation.java | 2 +- .../meter/process/MeterProcessService.java} | 37 +- .../meter}/process/MeterProcessor.java | 10 +- .../provider/meter}/process/Window.java | 2 +- .../trace}/DBLatencyThresholdsAndWatcher.java | 9 +- .../trace}/TraceSampleRateWatcher.java | 12 +- .../trace}/UninstrumentedGatewaysConfig.java | 9 +- .../parser/ISegmentParserListenerManager.java | 4 +- .../trace}/parser/ISegmentParserService.java | 2 +- .../parser/SegmentParserListenerManager.java | 5 +- .../parser/SegmentParserServiceImpl.java | 6 +- .../provider/trace}/parser/SpanTags.java | 2 +- .../provider/trace}/parser/TraceAnalyzer.java | 18 +- .../parser/listener/AnalysisListener.java | 2 +- .../listener/AnalysisListenerFactory.java | 6 +- .../listener/EntryAnalysisListener.java | 2 +- .../parser/listener/ExitAnalysisListener.java | 2 +- .../listener/FirstAnalysisListener.java | 2 +- .../listener/LocalAnalysisListener.java | 2 +- .../listener/MultiScopesAnalysisListener.java | 14 +- .../NetworkAddressAliasMappingListener.java | 8 +- .../listener/SegmentAnalysisListener.java | 8 +- .../parser/listener/SegmentListener.java | 2 +- .../trace}/parser/listener/SourceBuilder.java | 2 +- .../parser/listener/TraceSegmentSampler.java | 4 +- ...ing.oap.server.library.module.ModuleDefine | 19 + ...g.oap.server.library.module.ModuleProvider | 19 + .../meter}/process/EvalDataBaseTest.java | 2 +- .../provider/meter}/process/EvalDataTest.java | 2 +- .../meter}/process/EvalHistogramDataTest.java | 2 +- .../meter}/process/EvalMultipleDataTest.java | 7 +- .../meter}/process/EvalSingleDataTest.java | 2 +- .../meter}/process/MeterBaseTest.java | 20 +- .../meter}/process/MeterBuilderTest.java | 10 +- .../process/MeterEvalOperationTest.java | 5 +- .../process/MeterProcessContextTest.java | 29 +- .../meter}/process/MeterProcessorTest.java | 4 +- .../trace}/TraceSampleRateWatcherTest.java | 34 +- .../meter-receive-config/config.yaml | 39 ++ oap-server/analyzer/pom.xml | 52 +++ oap-server/pom.xml | 4 +- oap-server/server-bootstrap/pom.xml | 5 + .../src/main/resources/application.yml | 20 +- .../zookeeper/ZookeeperCoordinator.java | 10 +- .../kafka-fetcher-plugin/pom.xml | 50 +++ .../kafka/KafkaFetcherHandlerRegister.java | 146 +++++++ .../kafka/module/KafkaFetcherConfig.java | 82 ++++ .../kafka/module/KafkaFetcherModule.java | 34 ++ .../kafka/provider/KafkaFetcherProvider.java | 99 +++++ .../provider/handler/JVMMetricsHandler.java | 84 +++++ .../kafka/provider/handler/KafkaHandler.java | 65 ++++ .../provider/handler/MeterServiceHandler.java | 68 ++++ .../provider/handler/ProfileTaskHandler.java | 79 ++++ .../handler/ServiceManagementHandler.java | 135 +++++++ .../provider/handler/TraceSegmentHandler.java | 106 ++++++ ...ing.oap.server.library.module.ModuleDefine | 19 + ...g.oap.server.library.module.ModuleProvider | 19 + .../agent/kafka/mock/MockModuleManager.java | 50 +++ .../agent/kafka/mock/MockModuleProvider.java | 46 +++ .../provider/KafkaFetcherProviderTest.java | 52 +++ .../handler/JVMMetricsHandlerTest.java | 110 ++++++ .../handler/ServiceManagementHandlerTest.java | 102 +++++ .../provider/handler/SourceReceiverRule.java | 47 +++ oap-server/server-fetcher-plugin/pom.xml | 1 + .../skywalking-jvm-receiver-plugin/pom.xml | 5 + .../JVMMetricReportServiceHandler.java | 1 + .../skywalking-meter-receiver-plugin/pom.xml | 5 +- .../meter/provider/MeterReceiverProvider.java | 39 +- .../provider/handler/MeterServiceHandler.java | 12 +- .../provider/config/MeterConfigsTest.java | 6 +- .../skywalking-trace-receiver-plugin/pom.xml | 5 + .../receiver/trace/module/TraceModule.java | 3 +- .../trace/provider/TraceModuleProvider.java | 71 +--- .../TraceSegmentReportServiceHandler.java | 23 +- .../TraceSegmentReportBaseServletHandler.java | 20 +- .../TraceSegmentReportListServletHandler.java | 15 +- ...raceSegmentReportSingleServletHandler.java | 6 +- .../TraceSegmentReportServletHandlerTest.java | 30 +- .../MultiScopesAnalysisListenerTest.java | 12 +- .../zipkin/ZipkinReceiverProvider.java | 2 +- .../analysis/Receiver2AnalysisBridge.java | 2 +- .../e2e-test/docker/kafka/Dockerfile.provider | 37 ++ .../docker/kafka/docker-compose.base.yml | 69 ++++ .../docker/kafka/docker-compose.meter.yml | 68 ++++ .../docker/kafka/docker-compose.profiling.yml | 82 ++++ .../e2e-test/docker/kafka/docker-compose.yml | 83 ++++ .../apache/skywalking/e2e/kafka/KafkaE2E.java | 355 ++++++++++++++++++ .../skywalking/e2e/kafka/KafkaMeterE2E.java | 118 ++++++ .../skywalking/e2e/kafka/KafkaProfileE2E.java | 276 ++++++++++++++ .../apache/skywalking/e2e/meter/MeterE2E.java | 4 +- .../skywalking/e2e/profile/ProfileE2E.java | 4 +- .../skywalking/e2e/simple/SimpleE2E.java | 4 +- .../known-oap-backend-dependencies-es7.txt | 4 + .../known-oap-backend-dependencies.txt | 4 + 146 files changed, 4769 insertions(+), 587 deletions(-) create mode 100644 .github/workflows/e2e.kafka.yaml create mode 100644 apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/jvm/JVMMetricsSender.java create mode 100644 apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/meter/MeterSender.java create mode 100644 apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ProfileSnapshotSender.java create mode 100644 apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/pom.xml create mode 100644 apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaContextManagerExtendService.java create mode 100644 apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaJVMMetricsSender.java create mode 100644 apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaMeterSender.java create mode 100644 apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaProducerManager.java create mode 100644 apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaProfileSnapshotSender.java create mode 100644 apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaReporterPluginConfig.java create mode 100644 apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaServiceManagementServiceClient.java create mode 100644 apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaTraceSegmentServiceClient.java create mode 100644 apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService create mode 100644 apm-sniffer/optional-reporter-plugins/pom.xml create mode 100644 dist-material/release-docs/licenses/LICENSE-zstd-jni.txt create mode 100644 docs/en/setup/service-agent/java-agent/How-to-enable-kafka-reporter.md create mode 100644 oap-server/analyzer/agent-analyzer/pom.xml create mode 100644 oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/module/AnalyzerModule.java rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/TraceServiceModuleConfig.java => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/AnalyzerModuleConfig.java} (83%) mode change 100755 => 100644 create mode 100644 oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/AnalyzerModuleProvider.java rename oap-server/{server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/handler => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/jvm}/JVMSourceDispatcher.java (98%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/config/MeterConfig.java (93%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/config/MeterConfigs.java (97%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/config/MeterDataConfig.java (95%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/config/Scope.java (94%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/process/EvalData.java (97%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/process/EvalHistogramData.java (98%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/process/EvalMultipleData.java (98%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/process/EvalSingleData.java (98%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/MeterReceiverConfig.java => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/IMeterProcessService.java} (71%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/process/MeterBuilder.java (97%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/process/MeterEvalOperation.java (97%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterProcessContext.java => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessService.java} (64%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/process/MeterProcessor.java (95%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/process/Window.java (98%) rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/DBLatencyThresholdsAndWatcher.java (87%) rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/TraceSampleRateWatcher.java (80%) rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/UninstrumentedGatewaysConfig.java (93%) mode change 100755 => 100644 rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/parser/ISegmentParserListenerManager.java (88%) rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/parser/ISegmentParserService.java (94%) rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/parser/SegmentParserListenerManager.java (91%) rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/parser/SegmentParserServiceImpl.java (87%) rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/parser/SpanTags.java (96%) rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/parser/TraceAnalyzer.java (89%) mode change 100755 => 100644 rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/parser/listener/AnalysisListener.java (95%) rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/parser/listener/AnalysisListenerFactory.java (81%) rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/parser/listener/EntryAnalysisListener.java (94%) rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/parser/listener/ExitAnalysisListener.java (94%) rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/parser/listener/FirstAnalysisListener.java (94%) rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/parser/listener/LocalAnalysisListener.java (94%) rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/parser/listener/MultiScopesAnalysisListener.java (97%) mode change 100755 => 100644 rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/parser/listener/NetworkAddressAliasMappingListener.java (95%) rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/parser/listener/SegmentAnalysisListener.java (96%) rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/parser/listener/SegmentListener.java (94%) rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/parser/listener/SourceBuilder.java (99%) rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/parser/listener/TraceSegmentSampler.java (91%) create mode 100644 oap-server/analyzer/agent-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine create mode 100644 oap-server/analyzer/agent-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/process/EvalDataBaseTest.java (97%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/process/EvalDataTest.java (97%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/process/EvalHistogramDataTest.java (98%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/process/EvalMultipleDataTest.java (98%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/process/EvalSingleDataTest.java (98%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/process/MeterBaseTest.java (90%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/process/MeterBuilderTest.java (96%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/process/MeterEvalOperationTest.java (97%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/process/MeterProcessContextTest.java (69%) rename oap-server/{server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider => analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter}/process/MeterProcessorTest.java (96%) rename oap-server/{server-receiver-plugin/skywalking-trace-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/trace/provider => analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/trace}/TraceSampleRateWatcherTest.java (81%) create mode 100644 oap-server/analyzer/agent-analyzer/src/test/resources/meter-receive-config/config.yaml create mode 100644 oap-server/analyzer/pom.xml create mode 100644 oap-server/server-fetcher-plugin/kafka-fetcher-plugin/pom.xml create mode 100644 oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/KafkaFetcherHandlerRegister.java create mode 100644 oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/module/KafkaFetcherConfig.java create mode 100644 oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/module/KafkaFetcherModule.java create mode 100644 oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/KafkaFetcherProvider.java create mode 100644 oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/JVMMetricsHandler.java create mode 100644 oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/KafkaHandler.java create mode 100644 oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/MeterServiceHandler.java create mode 100644 oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/ProfileTaskHandler.java create mode 100644 oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/ServiceManagementHandler.java create mode 100644 oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/TraceSegmentHandler.java create mode 100644 oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine create mode 100644 oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider create mode 100644 oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleManager.java create mode 100644 oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleProvider.java create mode 100644 oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/KafkaFetcherProviderTest.java create mode 100644 oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/JVMMetricsHandlerTest.java create mode 100644 oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/ServiceManagementHandlerTest.java create mode 100644 oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/SourceReceiverRule.java create mode 100644 test/e2e/e2e-test/docker/kafka/Dockerfile.provider create mode 100644 test/e2e/e2e-test/docker/kafka/docker-compose.base.yml create mode 100644 test/e2e/e2e-test/docker/kafka/docker-compose.meter.yml create mode 100644 test/e2e/e2e-test/docker/kafka/docker-compose.profiling.yml create mode 100644 test/e2e/e2e-test/docker/kafka/docker-compose.yml create mode 100644 test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaE2E.java create mode 100644 test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaMeterE2E.java create mode 100644 test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaProfileE2E.java diff --git a/.github/workflows/e2e.kafka.yaml b/.github/workflows/e2e.kafka.yaml new file mode 100644 index 000000000..840a1168c --- /dev/null +++ b/.github/workflows/e2e.kafka.yaml @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: E2E + +on: + pull_request: + push: + branches: + - master + tags: + - 'v*' + +env: + SKIP_TEST: true + SW_AGENT_JDK_VERSION: 8 + CODECOV_TOKEN: d2065307-8f01-4637-9715-2781ef096db7 + +jobs: + Kafka: + name: Kafka + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Compile and Build + run: make docker + - name: Copy dist package + run: cp -R dist test/e2e/ + - name: Kafka + run: ./mvnw --batch-mode -f test/e2e/pom.xml -am -DfailIfNoTests=false verify -Dit.test=org.apache.skywalking.e2e.kafka.KafkaE2E + - name: Report Coverage + run: bash -x tools/coverage/report.sh + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: logs + path: logs + profile: + name: Kafka(profiling) + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + SW_STORAGE: influxdb + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Compile and Build + run: make docker + - name: Copy dist package + run: cp -R dist test/e2e/ + - name: Kafka Profiling + run: ./mvnw --batch-mode -f test/e2e/pom.xml -am -DfailIfNoTests=false verify -Dit.test=org.apache.skywalking.e2e.kafka.KafkaProfileE2E + - name: Report Coverage + run: bash -x tools/coverage/report.sh + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: logs + path: logs + meter: + name: Kafka(meter) + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Compile and Build + run: make docker + - name: Copy dist package + run: cp -R dist test/e2e/ + - name: Kafka Meter System + run: ./mvnw --batch-mode -f test/e2e/pom.xml -am -DfailIfNoTests=false verify -Dit.test=org.apache.skywalking.e2e.kafka.KafkaMeterE2E + - name: Report Coverage + run: bash -x tools/coverage/report.sh + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: logs + path: logs \ No newline at end of file diff --git a/apm-protocol/apm-network/src/main/proto b/apm-protocol/apm-network/src/main/proto index e3ec161a5..4be71766f 160000 --- a/apm-protocol/apm-network/src/main/proto +++ b/apm-protocol/apm-network/src/main/proto @@ -1 +1 @@ -Subproject commit e3ec161a5edbc06f05f847a0f6d7033cb9d95632 +Subproject commit 4be71766f60b943915d3ac8f91c996a5ff680ca0 diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java index 1d89ec567..694d03649 100755 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java @@ -152,6 +152,7 @@ public class Config { * Get profile task list interval */ public static int GET_PROFILE_TASK_INTERVAL = 20; + } public static class Profile { diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/jvm/JVMMetricsSender.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/jvm/JVMMetricsSender.java new file mode 100644 index 000000000..501f01ac2 --- /dev/null +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/jvm/JVMMetricsSender.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.agent.core.jvm; + +import io.grpc.Channel; +import java.util.LinkedList; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import org.apache.skywalking.apm.agent.core.boot.BootService; +import org.apache.skywalking.apm.agent.core.boot.DefaultImplementor; +import org.apache.skywalking.apm.agent.core.boot.ServiceManager; +import org.apache.skywalking.apm.agent.core.commands.CommandService; +import org.apache.skywalking.apm.agent.core.conf.Config; +import org.apache.skywalking.apm.agent.core.logging.api.ILog; +import org.apache.skywalking.apm.agent.core.logging.api.LogManager; +import org.apache.skywalking.apm.agent.core.remote.GRPCChannelListener; +import org.apache.skywalking.apm.agent.core.remote.GRPCChannelManager; +import org.apache.skywalking.apm.agent.core.remote.GRPCChannelStatus; +import org.apache.skywalking.apm.network.common.v3.Commands; +import org.apache.skywalking.apm.network.language.agent.v3.JVMMetric; +import org.apache.skywalking.apm.network.language.agent.v3.JVMMetricCollection; +import org.apache.skywalking.apm.network.language.agent.v3.JVMMetricReportServiceGrpc; + +import static org.apache.skywalking.apm.agent.core.conf.Config.Collector.GRPC_UPSTREAM_TIMEOUT; + +@DefaultImplementor +public class JVMMetricsSender implements BootService, Runnable, GRPCChannelListener { + private static final ILog logger = LogManager.getLogger(JVMMetricsSender.class); + + private volatile GRPCChannelStatus status = GRPCChannelStatus.DISCONNECT; + private volatile JVMMetricReportServiceGrpc.JVMMetricReportServiceBlockingStub stub = null; + + private LinkedBlockingQueue queue; + + @Override + public void prepare() { + queue = new LinkedBlockingQueue<>(Config.Jvm.BUFFER_SIZE); + ServiceManager.INSTANCE.findService(GRPCChannelManager.class).addChannelListener(this); + } + + @Override + public void boot() { + + } + + public void offer(JVMMetric metric) { + // drop last message and re-deliver + if (!queue.offer(metric)) { + queue.poll(); + queue.offer(metric); + } + } + + @Override + public void run() { + if (status == GRPCChannelStatus.CONNECTED) { + try { + JVMMetricCollection.Builder builder = JVMMetricCollection.newBuilder(); + LinkedList buffer = new LinkedList<>(); + queue.drainTo(buffer); + if (buffer.size() > 0) { + builder.addAllMetrics(buffer); + builder.setService(Config.Agent.SERVICE_NAME); + builder.setServiceInstance(Config.Agent.INSTANCE_NAME); + Commands commands = stub.withDeadlineAfter(GRPC_UPSTREAM_TIMEOUT, TimeUnit.SECONDS) + .collect(builder.build()); + ServiceManager.INSTANCE.findService(CommandService.class).receiveCommand(commands); + } + } catch (Throwable t) { + logger.error(t, "send JVM metrics to Collector fail."); + } + } + } + + @Override + public void statusChanged(GRPCChannelStatus status) { + if (GRPCChannelStatus.CONNECTED.equals(status)) { + Channel channel = ServiceManager.INSTANCE.findService(GRPCChannelManager.class).getChannel(); + stub = JVMMetricReportServiceGrpc.newBlockingStub(channel); + } + this.status = status; + } + + @Override + public void onComplete() { + + } + + @Override + public void shutdown() { + + } +} diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/jvm/JVMService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/jvm/JVMService.java index 02b116c49..542d7f8d4 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/jvm/JVMService.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/jvm/JVMService.java @@ -18,18 +18,13 @@ package org.apache.skywalking.apm.agent.core.jvm; -import io.grpc.Channel; -import java.util.LinkedList; import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import org.apache.skywalking.apm.agent.core.boot.BootService; import org.apache.skywalking.apm.agent.core.boot.DefaultImplementor; import org.apache.skywalking.apm.agent.core.boot.DefaultNamedThreadFactory; import org.apache.skywalking.apm.agent.core.boot.ServiceManager; -import org.apache.skywalking.apm.agent.core.commands.CommandService; -import org.apache.skywalking.apm.agent.core.conf.Config; import org.apache.skywalking.apm.agent.core.jvm.cpu.CPUProvider; import org.apache.skywalking.apm.agent.core.jvm.gc.GCProvider; import org.apache.skywalking.apm.agent.core.jvm.memory.MemoryProvider; @@ -37,17 +32,10 @@ import org.apache.skywalking.apm.agent.core.jvm.memorypool.MemoryPoolProvider; import org.apache.skywalking.apm.agent.core.jvm.thread.ThreadProvider; import org.apache.skywalking.apm.agent.core.logging.api.ILog; import org.apache.skywalking.apm.agent.core.logging.api.LogManager; -import org.apache.skywalking.apm.agent.core.remote.GRPCChannelListener; import org.apache.skywalking.apm.agent.core.remote.GRPCChannelManager; -import org.apache.skywalking.apm.agent.core.remote.GRPCChannelStatus; -import org.apache.skywalking.apm.network.common.v3.Commands; import org.apache.skywalking.apm.network.language.agent.v3.JVMMetric; -import org.apache.skywalking.apm.network.language.agent.v3.JVMMetricCollection; -import org.apache.skywalking.apm.network.language.agent.v3.JVMMetricReportServiceGrpc; import org.apache.skywalking.apm.util.RunnableWithExceptionProtection; -import static org.apache.skywalking.apm.agent.core.conf.Config.Collector.GRPC_UPSTREAM_TIMEOUT; - /** * The JVMService represents a timer, which collectors JVM cpu, memory, memorypool and gc info, and send * the collected info to Collector through the channel provided by {@link GRPCChannelManager} @@ -55,16 +43,13 @@ import static org.apache.skywalking.apm.agent.core.conf.Config.Collector.GRPC_UP @DefaultImplementor public class JVMService implements BootService, Runnable { private static final ILog logger = LogManager.getLogger(JVMService.class); - private LinkedBlockingQueue queue; private volatile ScheduledFuture collectMetricFuture; private volatile ScheduledFuture sendMetricFuture; - private Sender sender; + private JVMMetricsSender sender; @Override public void prepare() throws Throwable { - queue = new LinkedBlockingQueue(Config.Jvm.BUFFER_SIZE); - sender = new Sender(); - ServiceManager.INSTANCE.findService(GRPCChannelManager.class).addChannelListener(sender); + sender = ServiceManager.INSTANCE.findService(JVMMetricsSender.class); } @Override @@ -75,12 +60,8 @@ public class JVMService implements BootService, Runnable { this, new RunnableWithExceptionProtection.CallbackWhenException() { @Override - public void handle( - Throwable t) { - logger.error( - "JVMService produces metrics failure.", - t - ); + public void handle(Throwable t) { + logger.error("JVMService produces metrics failure.", t); } } ), 0, 1, TimeUnit.SECONDS); @@ -90,12 +71,8 @@ public class JVMService implements BootService, Runnable { sender, new RunnableWithExceptionProtection.CallbackWhenException() { @Override - public void handle( - Throwable t) { - logger.error( - "JVMService consumes and upload failure.", - t - ); + public void handle(Throwable t) { + logger.error("JVMService consumes and upload failure.", t); } } ), 0, 1, TimeUnit.SECONDS); @@ -124,50 +101,9 @@ public class JVMService implements BootService, Runnable { jvmBuilder.addAllGc(GCProvider.INSTANCE.getGCList()); jvmBuilder.setThread(ThreadProvider.INSTANCE.getThreadMetrics()); - JVMMetric jvmMetric = jvmBuilder.build(); - if (!queue.offer(jvmMetric)) { - queue.poll(); - queue.offer(jvmMetric); - } + sender.offer(jvmBuilder.build()); } catch (Exception e) { logger.error(e, "Collect JVM info fail."); } - - } - - private class Sender implements Runnable, GRPCChannelListener { - private volatile GRPCChannelStatus status = GRPCChannelStatus.DISCONNECT; - private volatile JVMMetricReportServiceGrpc.JVMMetricReportServiceBlockingStub stub = null; - - @Override - public void run() { - - if (status == GRPCChannelStatus.CONNECTED) { - try { - JVMMetricCollection.Builder builder = JVMMetricCollection.newBuilder(); - LinkedList buffer = new LinkedList(); - queue.drainTo(buffer); - if (buffer.size() > 0) { - builder.addAllMetrics(buffer); - builder.setService(Config.Agent.SERVICE_NAME); - builder.setServiceInstance(Config.Agent.INSTANCE_NAME); - Commands commands = stub.withDeadlineAfter(GRPC_UPSTREAM_TIMEOUT, TimeUnit.SECONDS) - .collect(builder.build()); - ServiceManager.INSTANCE.findService(CommandService.class).receiveCommand(commands); - } - } catch (Throwable t) { - logger.error(t, "send JVM metrics to Collector fail."); - } - } - } - - @Override - public void statusChanged(GRPCChannelStatus status) { - if (GRPCChannelStatus.CONNECTED.equals(status)) { - Channel channel = ServiceManager.INSTANCE.findService(GRPCChannelManager.class).getChannel(); - stub = JVMMetricReportServiceGrpc.newBlockingStub(channel); - } - this.status = status; - } } } diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/meter/MeterSender.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/meter/MeterSender.java new file mode 100644 index 000000000..b0fa3d63f --- /dev/null +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/meter/MeterSender.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.agent.core.meter; + +import io.grpc.Channel; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.StreamObserver; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import org.apache.skywalking.apm.agent.core.boot.BootService; +import org.apache.skywalking.apm.agent.core.boot.DefaultImplementor; +import org.apache.skywalking.apm.agent.core.boot.ServiceManager; +import org.apache.skywalking.apm.agent.core.conf.Config; +import org.apache.skywalking.apm.agent.core.logging.api.ILog; +import org.apache.skywalking.apm.agent.core.logging.api.LogManager; +import org.apache.skywalking.apm.agent.core.meter.transform.MeterTransformer; +import org.apache.skywalking.apm.agent.core.remote.GRPCChannelListener; +import org.apache.skywalking.apm.agent.core.remote.GRPCChannelManager; +import org.apache.skywalking.apm.agent.core.remote.GRPCChannelStatus; +import org.apache.skywalking.apm.agent.core.remote.GRPCStreamServiceStatus; +import org.apache.skywalking.apm.network.common.v3.Commands; +import org.apache.skywalking.apm.network.language.agent.v3.MeterData; +import org.apache.skywalking.apm.network.language.agent.v3.MeterReportServiceGrpc; + +import static org.apache.skywalking.apm.agent.core.conf.Config.Collector.GRPC_UPSTREAM_TIMEOUT; + +@DefaultImplementor +public class MeterSender implements BootService, GRPCChannelListener { + private static final ILog logger = LogManager.getLogger(MeterSender.class); + + private volatile GRPCChannelStatus status = GRPCChannelStatus.DISCONNECT; + + private volatile MeterReportServiceGrpc.MeterReportServiceStub meterReportServiceStub; + + @Override + public void prepare() { + ServiceManager.INSTANCE.findService(GRPCChannelManager.class).addChannelListener(this); + } + + @Override + public void boot() { + + } + + public void send(Map meterMap, MeterService meterService) { + if (status == GRPCChannelStatus.CONNECTED) { + StreamObserver reportStreamObserver = null; + final GRPCStreamServiceStatus status = new GRPCStreamServiceStatus(false); + try { + reportStreamObserver = meterReportServiceStub.withDeadlineAfter( + GRPC_UPSTREAM_TIMEOUT, TimeUnit.SECONDS + ).collect(new StreamObserver() { + @Override + public void onNext(Commands commands) { + } + + @Override + public void onError(Throwable throwable) { + status.finished(); + if (logger.isErrorEnable()) { + logger.error(throwable, "Send meters to collector fail with a grpc internal exception."); + } + ServiceManager.INSTANCE.findService(GRPCChannelManager.class).reportError(throwable); + } + + @Override + public void onCompleted() { + status.finished(); + } + }); + + final StreamObserver reporter = reportStreamObserver; + transform(meterMap, meterData -> reporter.onNext(meterData)); + } catch (Throwable e) { + if (!(e instanceof StatusRuntimeException)) { + logger.error(e, "Report meters to backend fail."); + return; + } + final StatusRuntimeException statusRuntimeException = (StatusRuntimeException) e; + if (statusRuntimeException.getStatus().getCode() == Status.Code.UNIMPLEMENTED) { + logger.warn("Backend doesn't support meter, it will be disabled"); + + meterService.shutdown(); + } + } finally { + if (reportStreamObserver != null) { + reportStreamObserver.onCompleted(); + } + status.wait4Finish(); + } + } + } + + protected void transform(final Map meterMap, + final Consumer consumer) { + // build and report meters + boolean hasSendMachineInfo = false; + for (MeterTransformer meterTransformer : meterMap.values()) { + final MeterData.Builder dataBuilder = meterTransformer.transform(); + if (dataBuilder == null) { + continue; + } + + // only send the service base info at the first data + if (!hasSendMachineInfo) { + dataBuilder.setService(Config.Agent.SERVICE_NAME); + dataBuilder.setServiceInstance(Config.Agent.INSTANCE_NAME); + dataBuilder.setTimestamp(System.currentTimeMillis()); + hasSendMachineInfo = true; + } + + consumer.accept(dataBuilder.build()); + } + } + + @Override + public void onComplete() { + + } + + @Override + public void shutdown() { + + } + + @Override + public void statusChanged(final GRPCChannelStatus status) { + if (GRPCChannelStatus.CONNECTED.equals(status)) { + Channel channel = ServiceManager.INSTANCE.findService(GRPCChannelManager.class).getChannel(); + meterReportServiceStub = MeterReportServiceGrpc.newStub(channel); + } else { + meterReportServiceStub = null; + } + this.status = status; + } +} diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/meter/MeterService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/meter/MeterService.java index 6e9ba5431..fa1b8e937 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/meter/MeterService.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/meter/MeterService.java @@ -18,10 +18,10 @@ package org.apache.skywalking.apm.agent.core.meter; -import io.grpc.Channel; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import io.grpc.stub.StreamObserver; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import org.apache.skywalking.apm.agent.core.boot.BootService; import org.apache.skywalking.apm.agent.core.boot.DefaultImplementor; import org.apache.skywalking.apm.agent.core.boot.DefaultNamedThreadFactory; @@ -30,38 +30,20 @@ import org.apache.skywalking.apm.agent.core.conf.Config; import org.apache.skywalking.apm.agent.core.logging.api.ILog; import org.apache.skywalking.apm.agent.core.logging.api.LogManager; import org.apache.skywalking.apm.agent.core.meter.transform.MeterTransformer; -import org.apache.skywalking.apm.agent.core.remote.GRPCChannelListener; -import org.apache.skywalking.apm.agent.core.remote.GRPCChannelManager; -import org.apache.skywalking.apm.agent.core.remote.GRPCChannelStatus; -import org.apache.skywalking.apm.agent.core.remote.GRPCStreamServiceStatus; -import org.apache.skywalking.apm.network.common.v3.Commands; -import org.apache.skywalking.apm.network.language.agent.v3.MeterData; -import org.apache.skywalking.apm.network.language.agent.v3.MeterReportServiceGrpc; import org.apache.skywalking.apm.util.RunnableWithExceptionProtection; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -import static org.apache.skywalking.apm.agent.core.conf.Config.Collector.GRPC_UPSTREAM_TIMEOUT; - @DefaultImplementor -public class MeterService implements BootService, Runnable, GRPCChannelListener { +public class MeterService implements BootService, Runnable { private static final ILog logger = LogManager.getLogger(MeterService.class); // all meters private final ConcurrentHashMap meterMap = new ConcurrentHashMap<>(); - // channel status - private volatile GRPCChannelStatus status = GRPCChannelStatus.DISCONNECT; - - // gRPC stub - private volatile MeterReportServiceGrpc.MeterReportServiceStub meterReportServiceStub; - // report meters private volatile ScheduledFuture reportMeterFuture; + private MeterSender sender; + /** * Register the meterTransformer */ @@ -70,7 +52,8 @@ public class MeterService implements BootService, Runnable, GRPCChannelListener return; } if (meterMap.size() >= Config.Meter.MAX_METER_SIZE) { - logger.warn("Already out of the meter system max size, will not report. meter name:{}", meterTransformer.getName()); + logger.warn( + "Already out of the meter system max size, will not report. meter name:{}", meterTransformer.getName()); return; } @@ -78,12 +61,12 @@ public class MeterService implements BootService, Runnable, GRPCChannelListener } @Override - public void prepare() throws Throwable { - ServiceManager.INSTANCE.findService(GRPCChannelManager.class).addChannelListener(this); + public void prepare() { + sender = ServiceManager.INSTANCE.findService(MeterSender.class); } @Override - public void boot() throws Throwable { + public void boot() { if (Config.Meter.ACTIVE) { reportMeterFuture = Executors.newSingleThreadScheduledExecutor( new DefaultNamedThreadFactory("MeterReportService") @@ -95,11 +78,11 @@ public class MeterService implements BootService, Runnable, GRPCChannelListener } @Override - public void onComplete() throws Throwable { + public void onComplete() { } @Override - public void shutdown() throws Throwable { + public void shutdown() { if (reportMeterFuture != null) { reportMeterFuture.cancel(true); } @@ -109,82 +92,10 @@ public class MeterService implements BootService, Runnable, GRPCChannelListener @Override public void run() { - if (status != GRPCChannelStatus.CONNECTED || meterMap.isEmpty()) { + if (meterMap.isEmpty()) { return; } - StreamObserver reportStreamObserver = null; - final GRPCStreamServiceStatus status = new GRPCStreamServiceStatus(false); - try { - reportStreamObserver = meterReportServiceStub.withDeadlineAfter( - GRPC_UPSTREAM_TIMEOUT, TimeUnit.SECONDS - ).collect(new StreamObserver() { - @Override - public void onNext(Commands commands) { - } - - @Override - public void onError(Throwable throwable) { - status.finished(); - if (logger.isErrorEnable()) { - logger.error(throwable, "Send meters to collector fail with a grpc internal exception."); - } - ServiceManager.INSTANCE.findService(GRPCChannelManager.class).reportError(throwable); - } - - @Override - public void onCompleted() { - status.finished(); - } - }); - - // build and report meters - boolean hasSendMachineInfo = false; - for (MeterTransformer meterTransformer : meterMap.values()) { - final MeterData.Builder dataBuilder = meterTransformer.transform(); - if (dataBuilder == null) { - continue; - } - - // only send the service base info at the first data - if (!hasSendMachineInfo) { - dataBuilder.setService(Config.Agent.SERVICE_NAME); - dataBuilder.setServiceInstance(Config.Agent.INSTANCE_NAME); - dataBuilder.setTimestamp(System.currentTimeMillis()); - hasSendMachineInfo = true; - } - - reportStreamObserver.onNext(dataBuilder.build()); - } - } catch (Throwable e) { - if (!(e instanceof StatusRuntimeException)) { - logger.error(e, "Report meters to backend fail."); - return; - } - final StatusRuntimeException statusRuntimeException = (StatusRuntimeException) e; - if (statusRuntimeException.getStatus().getCode() == Status.Code.UNIMPLEMENTED) { - logger.warn("Backend doesn't support meter, it will be disabled"); - if (reportMeterFuture != null) { - reportMeterFuture.cancel(true); - } - } - } finally { - if (reportStreamObserver != null) { - reportStreamObserver.onCompleted(); - } - status.wait4Finish(); - } - - } - - @Override - public void statusChanged(GRPCChannelStatus status) { - if (GRPCChannelStatus.CONNECTED.equals(status)) { - Channel channel = ServiceManager.INSTANCE.findService(GRPCChannelManager.class).getChannel(); - meterReportServiceStub = MeterReportServiceGrpc.newStub(channel); - } else { - meterReportServiceStub = null; - } - this.status = status; + sender.send(meterMap, this); } } diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ProfileSnapshotSender.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ProfileSnapshotSender.java new file mode 100644 index 000000000..45c18321b --- /dev/null +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ProfileSnapshotSender.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.agent.core.profile; + +import io.grpc.Channel; +import io.grpc.stub.StreamObserver; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.apache.skywalking.apm.agent.core.boot.BootService; +import org.apache.skywalking.apm.agent.core.boot.DefaultImplementor; +import org.apache.skywalking.apm.agent.core.boot.ServiceManager; +import org.apache.skywalking.apm.agent.core.logging.api.ILog; +import org.apache.skywalking.apm.agent.core.logging.api.LogManager; +import org.apache.skywalking.apm.agent.core.remote.GRPCChannelListener; +import org.apache.skywalking.apm.agent.core.remote.GRPCChannelManager; +import org.apache.skywalking.apm.agent.core.remote.GRPCChannelStatus; +import org.apache.skywalking.apm.agent.core.remote.GRPCStreamServiceStatus; +import org.apache.skywalking.apm.network.common.v3.Commands; +import org.apache.skywalking.apm.network.language.profile.v3.ProfileTaskGrpc; +import org.apache.skywalking.apm.network.language.profile.v3.ThreadSnapshot; + +import static org.apache.skywalking.apm.agent.core.conf.Config.Collector.GRPC_UPSTREAM_TIMEOUT; + +/** + * send segment snapshot + */ +@DefaultImplementor +public class ProfileSnapshotSender implements BootService, GRPCChannelListener { + private static final ILog logger = LogManager.getLogger(ProfileSnapshotSender.class); + + private volatile GRPCChannelStatus status = GRPCChannelStatus.DISCONNECT; + + private volatile ProfileTaskGrpc.ProfileTaskStub profileTaskStub; + + @Override + public void prepare() throws Throwable { + ServiceManager.INSTANCE.findService(GRPCChannelManager.class).addChannelListener(this); + } + + @Override + public void boot() throws Throwable { + + } + + @Override + public void statusChanged(final GRPCChannelStatus status) { + if (GRPCChannelStatus.CONNECTED.equals(status)) { + Channel channel = ServiceManager.INSTANCE.findService(GRPCChannelManager.class).getChannel(); + profileTaskStub = ProfileTaskGrpc.newStub(channel); + } else { + profileTaskStub = null; + } + this.status = status; + } + + public void send(List buffer) { + if (status == GRPCChannelStatus.CONNECTED) { + try { + final GRPCStreamServiceStatus status = new GRPCStreamServiceStatus(false); + StreamObserver snapshotStreamObserver = profileTaskStub.withDeadlineAfter( + GRPC_UPSTREAM_TIMEOUT, TimeUnit.SECONDS + ).collectSnapshot( + new StreamObserver() { + @Override + public void onNext( + Commands commands) { + } + + @Override + public void onError( + Throwable throwable) { + status.finished(); + if (logger.isErrorEnable()) { + logger.error( + throwable, + "Send profile segment snapshot to collector fail with a grpc internal exception." + ); + } + ServiceManager.INSTANCE.findService(GRPCChannelManager.class).reportError(throwable); + } + + @Override + public void onCompleted() { + status.finished(); + } + } + ); + for (TracingThreadSnapshot snapshot : buffer) { + final ThreadSnapshot transformSnapshot = snapshot.transform(); + snapshotStreamObserver.onNext(transformSnapshot); + } + + snapshotStreamObserver.onCompleted(); + status.wait4Finish(); + } catch (Throwable t) { + logger.error(t, "Send profile segment snapshot to backend fail."); + } + } + } + + @Override + public void onComplete() throws Throwable { + + } + + @Override + public void shutdown() throws Throwable { + + } +} \ No newline at end of file diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ProfileTaskChannelService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ProfileTaskChannelService.java index 7d78a44ba..6bcd8f291 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ProfileTaskChannelService.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ProfileTaskChannelService.java @@ -21,8 +21,8 @@ package org.apache.skywalking.apm.agent.core.profile; import io.grpc.Channel; import io.grpc.Status; import io.grpc.StatusRuntimeException; -import io.grpc.stub.StreamObserver; import java.util.ArrayList; +import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; @@ -39,12 +39,10 @@ import org.apache.skywalking.apm.agent.core.logging.api.LogManager; import org.apache.skywalking.apm.agent.core.remote.GRPCChannelListener; import org.apache.skywalking.apm.agent.core.remote.GRPCChannelManager; import org.apache.skywalking.apm.agent.core.remote.GRPCChannelStatus; -import org.apache.skywalking.apm.agent.core.remote.GRPCStreamServiceStatus; import org.apache.skywalking.apm.network.common.v3.Commands; import org.apache.skywalking.apm.network.language.profile.v3.ProfileTaskCommandQuery; import org.apache.skywalking.apm.network.language.profile.v3.ProfileTaskFinishReport; import org.apache.skywalking.apm.network.language.profile.v3.ProfileTaskGrpc; -import org.apache.skywalking.apm.network.language.profile.v3.ThreadSnapshot; import org.apache.skywalking.apm.util.RunnableWithExceptionProtection; import static org.apache.skywalking.apm.agent.core.conf.Config.Collector.GRPC_UPSTREAM_TIMEOUT; @@ -64,7 +62,7 @@ public class ProfileTaskChannelService implements BootService, Runnable, GRPCCha // gRPC stub private volatile ProfileTaskGrpc.ProfileTaskBlockingStub profileTaskBlockingStub; - private volatile ProfileTaskGrpc.ProfileTaskStub profileTaskStub; + // segment snapshot sender private final BlockingQueue snapshotQueue = new LinkedBlockingQueue<>( @@ -74,6 +72,8 @@ public class ProfileTaskChannelService implements BootService, Runnable, GRPCCha // query task list schedule private volatile ScheduledFuture getTaskListFuture; + private ProfileSnapshotSender sender; + @Override public void run() { if (status == GRPCChannelStatus.CONNECTED) { @@ -81,8 +81,7 @@ public class ProfileTaskChannelService implements BootService, Runnable, GRPCCha ProfileTaskCommandQuery.Builder builder = ProfileTaskCommandQuery.newBuilder(); // sniffer info - builder.setService(Config.Agent.SERVICE_NAME) - .setServiceInstance(Config.Agent.INSTANCE_NAME); + builder.setService(Config.Agent.SERVICE_NAME).setServiceInstance(Config.Agent.INSTANCE_NAME); // last command create time builder.setLastCommandTime(ServiceManager.INSTANCE.findService(ProfileTaskExecutionService.class) @@ -120,6 +119,8 @@ public class ProfileTaskChannelService implements BootService, Runnable, GRPCCha @Override public void boot() { + sender = ServiceManager.INSTANCE.findService(ProfileSnapshotSender.class); + if (Config.Profile.ACTIVE) { // query task list getTaskListFuture = Executors.newSingleThreadScheduledExecutor( @@ -135,7 +136,13 @@ public class ProfileTaskChannelService implements BootService, Runnable, GRPCCha new DefaultNamedThreadFactory("ProfileSendSnapshotService") ).scheduleWithFixedDelay( new RunnableWithExceptionProtection( - new SnapshotSender(), + () -> { + List buffer = new ArrayList<>(Config.Profile.SNAPSHOT_TRANSPORT_BUFFER_SIZE); + snapshotQueue.drainTo(buffer); + if (!buffer.isEmpty()) { + sender.send(buffer); + } + }, t -> logger.error("Profile segment snapshot upload failure.", t) ), 0, 500, TimeUnit.MILLISECONDS ); @@ -162,10 +169,8 @@ public class ProfileTaskChannelService implements BootService, Runnable, GRPCCha if (GRPCChannelStatus.CONNECTED.equals(status)) { Channel channel = ServiceManager.INSTANCE.findService(GRPCChannelManager.class).getChannel(); profileTaskBlockingStub = ProfileTaskGrpc.newBlockingStub(channel); - profileTaskStub = ProfileTaskGrpc.newStub(channel); } else { profileTaskBlockingStub = null; - profileTaskStub = null; } this.status = status; } @@ -197,62 +202,4 @@ public class ProfileTaskChannelService implements BootService, Runnable, GRPCCha } } - /** - * send segment snapshot - */ - private class SnapshotSender implements Runnable { - - @Override - public void run() { - if (status == GRPCChannelStatus.CONNECTED) { - try { - ArrayList buffer = new ArrayList<>( - Config.Profile.SNAPSHOT_TRANSPORT_BUFFER_SIZE); - snapshotQueue.drainTo(buffer); - if (buffer.size() > 0) { - final GRPCStreamServiceStatus status = new GRPCStreamServiceStatus(false); - StreamObserver snapshotStreamObserver = profileTaskStub.withDeadlineAfter( - GRPC_UPSTREAM_TIMEOUT, TimeUnit.SECONDS - ).collectSnapshot( - new StreamObserver() { - @Override - public void onNext( - Commands commands) { - } - - @Override - public void onError( - Throwable throwable) { - status.finished(); - if (logger.isErrorEnable()) { - logger.error( - throwable, - "Send profile segment snapshot to collector fail with a grpc internal exception." - ); - } - ServiceManager.INSTANCE.findService(GRPCChannelManager.class) - .reportError(throwable); - } - - @Override - public void onCompleted() { - status.finished(); - } - } - ); - for (TracingThreadSnapshot snapshot : buffer) { - final ThreadSnapshot transformSnapshot = snapshot.transform(); - snapshotStreamObserver.onNext(transformSnapshot); - } - - snapshotStreamObserver.onCompleted(); - status.wait4Finish(); - } - } catch (Throwable t) { - logger.error(t, "Send profile segment snapshot to backend fail."); - } - } - } - - } } diff --git a/apm-sniffer/apm-agent-core/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService b/apm-sniffer/apm-agent-core/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService index eece96b46..e1d84b9f6 100644 --- a/apm-sniffer/apm-agent-core/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService +++ b/apm-sniffer/apm-agent-core/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService @@ -20,11 +20,14 @@ org.apache.skywalking.apm.agent.core.remote.TraceSegmentServiceClient org.apache.skywalking.apm.agent.core.context.ContextManager org.apache.skywalking.apm.agent.core.sampling.SamplingService org.apache.skywalking.apm.agent.core.remote.GRPCChannelManager +org.apache.skywalking.apm.agent.core.jvm.JVMMetricsSender org.apache.skywalking.apm.agent.core.jvm.JVMService org.apache.skywalking.apm.agent.core.remote.ServiceManagementClient org.apache.skywalking.apm.agent.core.context.ContextManagerExtendService org.apache.skywalking.apm.agent.core.commands.CommandService org.apache.skywalking.apm.agent.core.commands.CommandExecutorService org.apache.skywalking.apm.agent.core.profile.ProfileTaskChannelService +org.apache.skywalking.apm.agent.core.profile.ProfileSnapshotSender org.apache.skywalking.apm.agent.core.profile.ProfileTaskExecutionService -org.apache.skywalking.apm.agent.core.meter.MeterService \ No newline at end of file +org.apache.skywalking.apm.agent.core.meter.MeterService +org.apache.skywalking.apm.agent.core.meter.MeterSender \ No newline at end of file diff --git a/apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/boot/ServiceManagerTest.java b/apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/boot/ServiceManagerTest.java index 355d55a96..829f10e23 100644 --- a/apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/boot/ServiceManagerTest.java +++ b/apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/boot/ServiceManagerTest.java @@ -58,7 +58,7 @@ public class ServiceManagerTest { public void testServiceDependencies() throws Exception { HashMap registryService = getFieldValue(ServiceManager.INSTANCE, "bootedServices"); - assertThat(registryService.size(), is(12)); + assertThat(registryService.size(), is(15)); assertTraceSegmentServiceClient(ServiceManager.INSTANCE.findService(TraceSegmentServiceClient.class)); assertContextManager(ServiceManager.INSTANCE.findService(ContextManager.class)); @@ -107,7 +107,7 @@ public class ServiceManagerTest { assertNotNull(service); List listeners = getFieldValue(service, "listeners"); - assertEquals(listeners.size(), 6); + assertEquals(listeners.size(), 7); } private void assertSamplingService(SamplingService service) { diff --git a/apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/meter/MeterServiceTest.java b/apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/meter/MeterServiceTest.java index 6af871241..7fdac30ec 100644 --- a/apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/meter/MeterServiceTest.java +++ b/apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/meter/MeterServiceTest.java @@ -67,6 +67,8 @@ public class MeterServiceTest { private MeterService registryService = new MeterService(); private List upstreamMeters; + private MeterSender sender = new MeterSender(); + private MeterReportServiceGrpc.MeterReportServiceImplBase serviceImplBase = new MeterReportServiceGrpc.MeterReportServiceImplBase() { @Override public StreamObserver collect(final StreamObserver responseObserver) { @@ -104,11 +106,14 @@ public class MeterServiceTest { @Before public void setUp() throws Throwable { + spy(sender); spy(registryService); Whitebox.setInternalState( - registryService, "meterReportServiceStub", MeterReportServiceGrpc.newStub(grpcServerRule.getChannel())); - Whitebox.setInternalState(registryService, "status", GRPCChannelStatus.CONNECTED); + sender, "meterReportServiceStub", MeterReportServiceGrpc.newStub(grpcServerRule.getChannel())); + Whitebox.setInternalState(sender, "status", GRPCChannelStatus.CONNECTED); + + Whitebox.setInternalState(registryService, "sender", sender); upstreamMeters = new ArrayList<>(); } diff --git a/apm-sniffer/config/agent.config b/apm-sniffer/config/agent.config index 058c4b97b..dcec546ac 100644 --- a/apm-sniffer/config/agent.config +++ b/apm-sniffer/config/agent.config @@ -87,3 +87,6 @@ logging.level=${SW_LOGGING_LEVEL:INFO} # mysql plugin configuration # plugin.mysql.trace_sql_parameters=${SW_MYSQL_TRACE_SQL_PARAMETERS:false} + +# Kafka producer configuration +plugin.kafka.bootstrap_servers=${SW_KAFKA_BOOTSTRAP_SERVERS:localhost:9092} diff --git a/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/pom.xml b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/pom.xml new file mode 100644 index 000000000..d1bff4324 --- /dev/null +++ b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/pom.xml @@ -0,0 +1,162 @@ + + + + + + optional-reporter-plugins + org.apache.skywalking + 8.1.0-SNAPSHOT + + 4.0.0 + + kafka-reporter-plugin + jar + + + + org.apache.kafka + kafka-clients + ${kafk-clients.version} + + + + + + + + maven-shade-plugin + + + package + + shade + + + true + true + true + true + + + org.apache.kafka + ${shade.package}/org.apache.kafka + + + common.message + ${shade.package}/common.message + + + kafka + ${shade.package}/kafka + + + org.slf4j + ${shade.package}/org.slf4j + + + org.xerial + ${shade.package}/org.xerial + + + net.jpountz + ${shade.package}/net.jpountz + + + aix.ppc64 + ${shade.package}/aix.ppc64 + + + com.github.luben + ${shade.package}/com.github.luben + + + darwin + ${shade.package}/darwin + + + include + ${shade.package}/include + + + linux + ${shade.package}/linux + + + win + ${shade.package}/win + + + + + + + + + + + maven-antrun-plugin + + + package + + run + + + + + + + + + + + + + + + + + + ant-contrib + ant-contrib + ${ant-contrib.version} + + + ant + ant + + + + + org.apache.ant + ant-nodeps + ${ant-nodeps.version} + + + + + + + \ No newline at end of file diff --git a/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaContextManagerExtendService.java b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaContextManagerExtendService.java new file mode 100644 index 000000000..8cab8e705 --- /dev/null +++ b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaContextManagerExtendService.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.agent.core.kafka; + +import org.apache.skywalking.apm.agent.core.boot.OverrideImplementor; +import org.apache.skywalking.apm.agent.core.context.ContextManagerExtendService; +import org.apache.skywalking.apm.agent.core.remote.GRPCChannelStatus; + +/** + * For compatible with {@link ContextManagerExtendService}, don't need to manage connection status by self. + */ +@OverrideImplementor(ContextManagerExtendService.class) +public class KafkaContextManagerExtendService extends ContextManagerExtendService { + + @Override + public void prepare() { + statusChanged(GRPCChannelStatus.CONNECTED); + } + +} diff --git a/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaJVMMetricsSender.java b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaJVMMetricsSender.java new file mode 100644 index 000000000..f09af4a01 --- /dev/null +++ b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaJVMMetricsSender.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.agent.core.kafka; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.utils.Bytes; +import org.apache.skywalking.apm.agent.core.boot.OverrideImplementor; +import org.apache.skywalking.apm.agent.core.boot.ServiceManager; +import org.apache.skywalking.apm.agent.core.conf.Config; +import org.apache.skywalking.apm.agent.core.jvm.JVMMetricsSender; +import org.apache.skywalking.apm.agent.core.logging.api.ILog; +import org.apache.skywalking.apm.agent.core.logging.api.LogManager; +import org.apache.skywalking.apm.network.language.agent.v3.JVMMetric; +import org.apache.skywalking.apm.network.language.agent.v3.JVMMetricCollection; + +/** + * A report to send JVM Metrics data to Kafka Broker. + */ +@OverrideImplementor(JVMMetricsSender.class) +public class KafkaJVMMetricsSender extends JVMMetricsSender { + private static final ILog logger = LogManager.getLogger(KafkaJVMMetricsSender.class); + private KafkaProducer producer; + private String topic; + + private BlockingQueue queue; + private volatile boolean running = false; + + @Override + public void run() { + if (!queue.isEmpty()) { + List buffer = new ArrayList<>(); + queue.drainTo(buffer); + + if (running) { + JVMMetricCollection metrics = JVMMetricCollection.newBuilder() + .addAllMetrics(buffer) + .setService(Config.Agent.SERVICE_NAME) + .setServiceInstance(Config.Agent.INSTANCE_NAME) + .build(); + + if (logger.isDebugEnable()) { + logger.debug( + "JVM metrics reporting, topic: {}, key: {}, length: {}", topic, metrics.getServiceInstance(), + buffer.size() + ); + } + + producer.send(new ProducerRecord<>( + topic, + metrics.getServiceInstance(), + Bytes.wrap(metrics.toByteArray()) + )); + producer.flush(); + } + } + } + + @Override + public void prepare() { + queue = new LinkedBlockingQueue<>(Config.Jvm.BUFFER_SIZE); + topic = KafkaReporterPluginConfig.Plugin.Kafka.TOPIC_METRICS; + } + + @Override + public void boot() { + producer = ServiceManager.INSTANCE.findService(KafkaProducerManager.class).getProducer(); + running = true; + + } + + @Override + public void offer(final JVMMetric metric) { + if (!queue.offer(metric)) { + queue.poll(); + queue.offer(metric); + } + } +} diff --git a/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaMeterSender.java b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaMeterSender.java new file mode 100644 index 000000000..df1eb513f --- /dev/null +++ b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaMeterSender.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.agent.core.kafka; + +import java.util.Map; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.utils.Bytes; +import org.apache.skywalking.apm.agent.core.boot.OverrideImplementor; +import org.apache.skywalking.apm.agent.core.boot.ServiceManager; +import org.apache.skywalking.apm.agent.core.conf.Config; +import org.apache.skywalking.apm.agent.core.logging.api.ILog; +import org.apache.skywalking.apm.agent.core.logging.api.LogManager; +import org.apache.skywalking.apm.agent.core.meter.MeterId; +import org.apache.skywalking.apm.agent.core.meter.MeterSender; +import org.apache.skywalking.apm.agent.core.meter.MeterService; +import org.apache.skywalking.apm.agent.core.meter.transform.MeterTransformer; +import org.apache.skywalking.apm.network.language.agent.v3.MeterDataCollection; + +/** + * A report to send JVM Metrics data to Kafka Broker. + */ +@OverrideImplementor(MeterSender.class) +public class KafkaMeterSender extends MeterSender { + private static final ILog logger = LogManager.getLogger(KafkaTraceSegmentServiceClient.class); + + private String topic; + private KafkaProducer producer; + + @Override + public void prepare() { + topic = KafkaReporterPluginConfig.Plugin.Kafka.TOPIC_METER; + } + + @Override + public void boot() { + producer = ServiceManager.INSTANCE.findService(KafkaProducerManager.class).getProducer(); + } + + public void send(Map meterMap, MeterService meterService) { + MeterDataCollection.Builder builder = MeterDataCollection.newBuilder(); + transform(meterMap, meterData -> { + if (logger.isDebugEnable()) { + logger.debug("Meter data reporting, instance: {}", meterData.getServiceInstance()); + } + builder.addMeterData(meterData); + }); + producer.send( + new ProducerRecord<>(topic, Config.Agent.INSTANCE_NAME, Bytes.wrap(builder.build().toByteArray()))); + + producer.flush(); + } +} diff --git a/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaProducerManager.java b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaProducerManager.java new file mode 100644 index 000000000..135abec9c --- /dev/null +++ b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaProducerManager.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.agent.core.kafka; + +import java.util.Arrays; +import java.util.Objects; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.DescribeTopicsResult; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.BytesSerializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.common.utils.Bytes; +import org.apache.skywalking.apm.agent.core.boot.BootService; +import org.apache.skywalking.apm.agent.core.boot.DefaultImplementor; + +/** + * Configuring, initializing and holding a KafkaProducer instance for reporters. + */ +@DefaultImplementor +public class KafkaProducerManager implements BootService, Runnable { + + private KafkaProducer producer; + + @Override + public void prepare() throws Throwable { + Properties properties = new Properties(); + properties.setProperty( + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, KafkaReporterPluginConfig.Plugin.Kafka.BOOTSTRAP_SERVERS); + KafkaReporterPluginConfig.Plugin.Kafka.PRODUCER_CONFIG.forEach((k, v) -> properties.setProperty(k, v)); + + AdminClient adminClient = AdminClient.create(properties); + DescribeTopicsResult topicsResult = adminClient.describeTopics(Arrays.asList( + KafkaReporterPluginConfig.Plugin.Kafka.TOPIC_MANAGEMENT, + KafkaReporterPluginConfig.Plugin.Kafka.TOPIC_METRICS, + KafkaReporterPluginConfig.Plugin.Kafka.TOPIC_PROFILING, + KafkaReporterPluginConfig.Plugin.Kafka.TOPIC_SEGMENT, + KafkaReporterPluginConfig.Plugin.Kafka.TOPIC_METER + )); + Set topics = topicsResult.values().entrySet().stream() + .map(entry -> { + try { + entry.getValue().get(); + return null; + } catch (InterruptedException | ExecutionException e) { + } + return entry.getKey(); + }) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + if (!topics.isEmpty()) { + throw new Exception("These topics" + topics + " don't exist."); + } + + producer = new KafkaProducer<>(properties, new StringSerializer(), new BytesSerializer()); + } + + @Override + public void boot() { + + } + + @Override + public void onComplete() { + } + + @Override + public void run() { + + } + + /** + * Get the KafkaProducer instance to send data to Kafka broker. + */ + public final KafkaProducer getProducer() { + return producer; + } + + @Override + public void shutdown() { + producer.flush(); + producer.close(); + } +} diff --git a/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaProfileSnapshotSender.java b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaProfileSnapshotSender.java new file mode 100644 index 000000000..13170d0df --- /dev/null +++ b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaProfileSnapshotSender.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.agent.core.kafka; + +import java.util.List; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.utils.Bytes; +import org.apache.skywalking.apm.agent.core.boot.OverrideImplementor; +import org.apache.skywalking.apm.agent.core.boot.ServiceManager; +import org.apache.skywalking.apm.agent.core.logging.api.ILog; +import org.apache.skywalking.apm.agent.core.logging.api.LogManager; +import org.apache.skywalking.apm.agent.core.profile.ProfileSnapshotSender; +import org.apache.skywalking.apm.agent.core.profile.TracingThreadSnapshot; +import org.apache.skywalking.apm.network.language.profile.v3.ThreadSnapshot; + +/** + * To transport profiling tasks between OAP Server and agent with gRPC. This is why we still have to configure gRPC. But + * to report the tracing profile snapshot data by Kafka Producer. + */ +@OverrideImplementor(ProfileSnapshotSender.class) +public class KafkaProfileSnapshotSender extends ProfileSnapshotSender { + private static final ILog logger = LogManager.getLogger(ProfileSnapshotSender.class); + + private String topic; + private KafkaProducer producer; + + @Override + public void prepare() { + topic = KafkaReporterPluginConfig.Plugin.Kafka.TOPIC_PROFILING; + } + + @Override + public void boot() { + producer = ServiceManager.INSTANCE.findService(KafkaProducerManager.class).getProducer(); + } + + @Override + public void send(final List buffer) { + for (TracingThreadSnapshot snapshot : buffer) { + final ThreadSnapshot object = snapshot.transform(); + if (logger.isDebugEnable()) { + logger.debug("Thread snapshot reporting, topic: {}, taskId: {}, sequence:{}, traceId: {}", + object.getTaskId(), object.getSequence(), object.getTraceSegmentId() + ); + } + + producer.send(new ProducerRecord<>( + topic, + object.getTaskId() + object.getSequence(), + Bytes.wrap(object.toByteArray()) + )); + } + } + +} diff --git a/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaReporterPluginConfig.java b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaReporterPluginConfig.java new file mode 100644 index 000000000..7dd693f05 --- /dev/null +++ b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaReporterPluginConfig.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.agent.core.kafka; + +import java.util.HashMap; +import java.util.Map; +import org.apache.skywalking.apm.agent.core.boot.PluginConfig; + +public class KafkaReporterPluginConfig { + public static class Plugin { + @PluginConfig(root = KafkaReporterPluginConfig.class) + public static class Kafka { + /** + * bootstrap.servers: A list of host/port pairs to use for establishing the initial connection to the Kafka cluster. + * This list should be in the form host1:port1,host2:port2,... + */ + public static String BOOTSTRAP_SERVERS = "localhost:9092"; + + public static String TOPIC_METRICS = "skywalking-metrics"; + + public static String TOPIC_PROFILING = "skywalking-profilings"; + + public static String TOPIC_SEGMENT = "skywalking-segments"; + + public static String TOPIC_MANAGEMENT = "skywalking-managements"; + + public static String TOPIC_METER = "skywalking-meters"; + + public static Map PRODUCER_CONFIG = new HashMap<>(); + } + } +} diff --git a/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaServiceManagementServiceClient.java b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaServiceManagementServiceClient.java new file mode 100644 index 000000000..2f6791192 --- /dev/null +++ b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaServiceManagementServiceClient.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.agent.core.kafka; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.utils.Bytes; +import org.apache.skywalking.apm.agent.core.boot.BootService; +import org.apache.skywalking.apm.agent.core.boot.DefaultNamedThreadFactory; +import org.apache.skywalking.apm.agent.core.boot.OverrideImplementor; +import org.apache.skywalking.apm.agent.core.boot.ServiceManager; +import org.apache.skywalking.apm.agent.core.conf.Config; +import org.apache.skywalking.apm.agent.core.logging.api.ILog; +import org.apache.skywalking.apm.agent.core.logging.api.LogManager; +import org.apache.skywalking.apm.agent.core.os.OSUtil; +import org.apache.skywalking.apm.agent.core.remote.ServiceManagementClient; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import org.apache.skywalking.apm.network.management.v3.InstancePingPkg; +import org.apache.skywalking.apm.network.management.v3.InstanceProperties; +import org.apache.skywalking.apm.util.RunnableWithExceptionProtection; +import org.apache.skywalking.apm.util.StringUtil; + +/** + * A service management data(Instance registering properties and Instance pinging) reporter. + */ +@OverrideImplementor(ServiceManagementClient.class) +public class KafkaServiceManagementServiceClient implements BootService, Runnable { + private static final ILog logger = LogManager.getLogger(KafkaServiceManagementServiceClient.class); + + private static List SERVICE_INSTANCE_PROPERTIES; + + private static final String TOPIC_KEY_REGISTER = "register-"; + + private ScheduledFuture heartbeatFuture; + private KafkaProducer producer; + + private String topic; + + @Override + public void prepare() { + topic = KafkaReporterPluginConfig.Plugin.Kafka.TOPIC_MANAGEMENT; + + SERVICE_INSTANCE_PROPERTIES = new ArrayList<>(); + for (String key : Config.Agent.INSTANCE_PROPERTIES.keySet()) { + SERVICE_INSTANCE_PROPERTIES.add(KeyStringValuePair.newBuilder() + .setKey(key) + .setValue(Config.Agent.INSTANCE_PROPERTIES.get(key)) + .build()); + } + + Config.Agent.INSTANCE_NAME = StringUtil.isEmpty(Config.Agent.INSTANCE_NAME) + ? UUID.randomUUID().toString().replaceAll("-", "") + "@" + OSUtil.getIPV4() + : Config.Agent.INSTANCE_NAME; + } + + @Override + public void boot() { + producer = ServiceManager.INSTANCE.findService(KafkaProducerManager.class).getProducer(); + + heartbeatFuture = Executors.newSingleThreadScheduledExecutor( + new DefaultNamedThreadFactory("ServiceManagementClientKafkaProducer") + ).scheduleAtFixedRate(new RunnableWithExceptionProtection( + this, + t -> logger.error("unexpected exception.", t) + ), 0, Config.Collector.HEARTBEAT_PERIOD, TimeUnit.SECONDS); + + InstanceProperties instance = InstanceProperties.newBuilder() + .setService(Config.Agent.SERVICE_NAME) + .setServiceInstance(Config.Agent.INSTANCE_NAME) + .addAllProperties(OSUtil.buildOSInfo( + Config.OsInfo.IPV4_LIST_SIZE)) + .addAllProperties(SERVICE_INSTANCE_PROPERTIES) + .build(); + producer.send(new ProducerRecord<>(topic, TOPIC_KEY_REGISTER + instance.getServiceInstance(), Bytes.wrap(instance.toByteArray()))); + producer.flush(); + } + + @Override + public void run() { + InstancePingPkg ping = InstancePingPkg.newBuilder() + .setService(Config.Agent.SERVICE_NAME) + .setServiceInstance(Config.Agent.INSTANCE_NAME) + .build(); + if (logger.isDebugEnable()) { + logger.debug("Heartbeat reporting, instance: {}", ping.getServiceInstance()); + } + producer.send(new ProducerRecord<>(ping.getServiceInstance(), Bytes.wrap(ping.toByteArray()))); + } + + @Override + public void onComplete() { + + } + + @Override + public void shutdown() { + heartbeatFuture.cancel(true); + } +} diff --git a/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaTraceSegmentServiceClient.java b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaTraceSegmentServiceClient.java new file mode 100644 index 000000000..322f7c3ad --- /dev/null +++ b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaTraceSegmentServiceClient.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.agent.core.kafka; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.utils.Bytes; +import org.apache.skywalking.apm.agent.core.boot.BootService; +import org.apache.skywalking.apm.agent.core.boot.OverrideImplementor; +import org.apache.skywalking.apm.agent.core.boot.ServiceManager; +import org.apache.skywalking.apm.agent.core.context.TracingContext; +import org.apache.skywalking.apm.agent.core.context.TracingContextListener; +import org.apache.skywalking.apm.agent.core.context.trace.TraceSegment; +import org.apache.skywalking.apm.agent.core.logging.api.ILog; +import org.apache.skywalking.apm.agent.core.logging.api.LogManager; +import org.apache.skywalking.apm.agent.core.remote.TraceSegmentServiceClient; +import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; + +/** + * A tracing segment data reporter. + */ +@OverrideImplementor(TraceSegmentServiceClient.class) +public class KafkaTraceSegmentServiceClient implements BootService, TracingContextListener { + private static final ILog logger = LogManager.getLogger(KafkaTraceSegmentServiceClient.class); + + private String topic; + private KafkaProducer producer; + + @Override + public void prepare() { + topic = KafkaReporterPluginConfig.Plugin.Kafka.TOPIC_SEGMENT; + } + + @Override + public void boot() { + producer = ServiceManager.INSTANCE.findService(KafkaProducerManager.class).getProducer(); + } + + @Override + public void onComplete() { + TracingContext.ListenerManager.add(this); + } + + @Override + public void shutdown() { + TracingContext.ListenerManager.remove(this); + } + + @Override + public void afterFinished(final TraceSegment traceSegment) { + if (logger.isDebugEnable()) { + logger.debug("Trace segment reporting, traceId: {}", traceSegment.getTraceSegmentId()); + } + + if (traceSegment.isIgnore()) { + logger.debug("Trace[TraceId={}] is ignored.", traceSegment.getTraceSegmentId()); + return; + } + SegmentObject upstreamSegment = traceSegment.transform(); + ProducerRecord record = new ProducerRecord<>( + topic, + upstreamSegment.getTraceSegmentId(), + Bytes.wrap(upstreamSegment.toByteArray()) + ); + producer.send(record); + } + +} \ No newline at end of file diff --git a/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService new file mode 100644 index 000000000..8e5fafef0 --- /dev/null +++ b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +org.apache.skywalking.apm.agent.core.kafka.KafkaProducerManager +org.apache.skywalking.apm.agent.core.kafka.KafkaContextManagerExtendService +org.apache.skywalking.apm.agent.core.kafka.KafkaServiceManagementServiceClient +org.apache.skywalking.apm.agent.core.kafka.KafkaJVMMetricsSender +org.apache.skywalking.apm.agent.core.kafka.KafkaTraceSegmentServiceClient +org.apache.skywalking.apm.agent.core.kafka.KafkaProfileSnapshotSender +org.apache.skywalking.apm.agent.core.kafka.KafkaMeterSender \ No newline at end of file diff --git a/apm-sniffer/optional-reporter-plugins/pom.xml b/apm-sniffer/optional-reporter-plugins/pom.xml new file mode 100644 index 000000000..fd3888ac2 --- /dev/null +++ b/apm-sniffer/optional-reporter-plugins/pom.xml @@ -0,0 +1,70 @@ + + + + + + apm-sniffer + org.apache.skywalking + 8.1.0-SNAPSHOT + + 4.0.0 + + optional-reporter-plugins + pom + + + kafka-reporter-plugin + + + + UTF-8 + + ${project.build.directory}/../../../../skywalking-agent + ${agent.package.dest.dir}/optional-reporter-plugins + + 1.0b3 + 1.8.1 + + 2.4.1 + 2.4.6.RELEASE + + + + + org.apache.skywalking + apm-agent-core + ${project.version} + provided + + + org.apache.skywalking + apm-util + ${project.version} + provided + + + org.apache.skywalking + apm-test-tools + ${project.version} + test + + + + \ No newline at end of file diff --git a/apm-sniffer/pom.xml b/apm-sniffer/pom.xml index ef0dc4b28..64f7182a6 100644 --- a/apm-sniffer/pom.xml +++ b/apm-sniffer/pom.xml @@ -36,6 +36,7 @@ apm-test-tools bootstrap-plugins optional-plugins + optional-reporter-plugins diff --git a/dist-material/release-docs/LICENSE b/dist-material/release-docs/LICENSE index 90139ab93..3e31eabe8 100755 --- a/dist-material/release-docs/LICENSE +++ b/dist-material/release-docs/LICENSE @@ -336,6 +336,10 @@ The text of each license is the standard Apache 2.0 license. jose4j 0.7.0: https://mvnrepository.com/artifact/org.bitbucket.b_c/jose4j, Apache 2.0 converter-moshi 2.5.0: https://mvnrepository.com/artifact/com.squareup.retrofit2/converter-moshi, Apache 2.0 vavr 0.10.3: https://github.com/vavr-io/vavr, Apache 2.0 + kafka-clients 2.4.1: https://github.com/apache/kafka, Apache 2.0 + lz4-java 1.6.0: https://github.com/jpountz/lz4-java, Apache 2.0 + snappy-java 1.1.7.3: https://github.com/xerial/snappy-java, Apache 2.0 + slf4j-api 1.7.28: http://www.slf4j.org, Apache 2.0 ======================================================================== MIT licenses @@ -380,6 +384,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. proto files from gogo: https://github.com/gogo/protobuf BSD-2 proto files from grpc-gateway, https://github.com/grpc-ecosystem/grpc-gateway/tree/master/protoc-gen-swagger/options BSD-3 + zstd-jni 1.4.3-1: https://github.com/luben/zstd-jni, BSD-3-Clause ======================================================================== MPL 2.0 licenses diff --git a/dist-material/release-docs/NOTICE b/dist-material/release-docs/NOTICE index d3ecdc613..3d23e0a25 100755 --- a/dist-material/release-docs/NOTICE +++ b/dist-material/release-docs/NOTICE @@ -879,3 +879,20 @@ This product includes the software developed by third-party: * sbt-extras: https://github.com/paulp/sbt-extras (BSD) (LICENSE.sbt-extras.txt) ======================================================================== + +------ + +=========================================================================== +Apache Kafka Notice +=========================================================================== + +Apache Kafka +Copyright 2020 The Apache Software Foundation. + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +This distribution has a binary dependency on jersey, which is available under the CDDL +License. The source code of jersey can be found at https://github.com/jersey/jersey/. + +======================================================================== \ No newline at end of file diff --git a/dist-material/release-docs/licenses/LICENSE-zstd-jni.txt b/dist-material/release-docs/licenses/LICENSE-zstd-jni.txt new file mode 100644 index 000000000..aed81eb31 --- /dev/null +++ b/dist-material/release-docs/licenses/LICENSE-zstd-jni.txt @@ -0,0 +1,26 @@ +Zstd-jni: JNI bindings to Zstd Library + +Copyright (c) 2015-present, Luben Karavelov/ All rights reserved. + +BSD License + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/docs/en/guides/Java-Plugin-Development-Guide.md b/docs/en/guides/Java-Plugin-Development-Guide.md index 6c6aa5843..c4d589b7d 100644 --- a/docs/en/guides/Java-Plugin-Development-Guide.md +++ b/docs/en/guides/Java-Plugin-Development-Guide.md @@ -160,8 +160,8 @@ The value should be an integer. The response code of OAL entities is according t #### Tag key `db.statement` and `db.type`. The value of `db.statement` should be a String, representing the Database statement, such as SQL, or `[No statement]/`+span#operationName if value is empty. -When exit span has this tag, OAP samples the slow statements based on `receiver-trace/default/maxSlowSQLLength`. -The threshold of slow statement is defined by following [`receiver-trace/default/slowDBAccessThreshold`](../setup/backend/slow-db-statement.md) +When exit span has this tag, OAP samples the slow statements based on `agent-analyzer/default/maxSlowSQLLength`. +The threshold of slow statement is defined by following [`agent-analyzer/default/slowDBAccessThreshold`](../setup/backend/slow-db-statement.md) #### Extension logic endpoint. Tag key `x-le` Logic endpoint is a concept, which doesn't represent a real RPC call, but requires the statistic. diff --git a/docs/en/setup/backend/backend-fetcher.md b/docs/en/setup/backend/backend-fetcher.md index ab9c33ed1..e821391be 100644 --- a/docs/en/setup/backend/backend-fetcher.md +++ b/docs/en/setup/backend/backend-fetcher.md @@ -25,7 +25,7 @@ Generic placeholders are defined as follows: * ``: a duration This will parse a textual representation of a duration. The formats accepted are based on the ISO-8601 duration format `PnDTnHnMn.nS` with days considered to be exactly 24 hours. - * ``: a string matching the regular expression [a-zA-Z_][a-zA-Z0-9_]* + * ``: a string matching the regular expression \[a-zA-Z_\]\[a-zA-Z0-9_\]* * ``: a string of unicode characters * ``: a valid string consisting of a hostname or IP followed by an optional port number * ``: a valid URL path @@ -85,3 +85,51 @@ that adds the procedure from raw data to minute rate. When you specify `avgHistogram` and `avgHistogramPercentile`, the source should be the type of `histogram`. A counterFunction is also needed due to the `bucket`, `sum` and `count` of histogram are counters. + +## Kafka Fetcher + +Kafka Fetcher pulls messages from Kafka Broker(s) what is the Agent delivered. Check the agent documentation about the details. Typically Tracing Segments, Service/Instance properties, JVM Metrics, and Meter system data are supported. Kafka Fetcher can work with gRPC/HTTP Receivers at the same time for adopting different transport protocols. + +Kafka Fetcher is disabled in default, and we configure as following to enable. + +```yaml +kafka-fetcher: + selector: ${SW_KAFKA_FETCHER:default} + default: + bootstrapServers: ${SW_KAFKA_FETCHER_SERVERS:localhost:9092} +``` + +`skywalking-segments`, `skywalking-metrics`, `skywalking-profile`, `skywalking-managements` and `skywalking-meters` topics are required by `kafka-fetcher`. +If they do not exist, Kafka Fetcher will create them in default. Also, you can create them by yourself before the OAP server started. + +When using the OAP server automatical creation mechanism, you could modify the number of partitions and replications of the topics through the following configurations: + +```yaml +kafka-fetcher: + selector: ${SW_KAFKA_FETCHER:default} + default: + bootstrapServers: ${SW_KAFKA_FETCHER_SERVERS:localhost:9092} + partitions: ${SW_KAFKA_FETCHER_PARTITIONS:3} + replicationFactor: ${SW_KAFKA_FETCHER_PARTITIONS_FACTOR:2} + enableMeterSystem: ${SW_KAFKA_FETCHER_ENABLE_METER_SYSTEM:false} + isSharding: ${SW_KAFKA_FETCHER_IS_SHARDING:false} + consumePartitions: ${SW_KAFKA_FETCHER_CONSUME_PARTITIONS:""} +``` + +In cluster mode, all topics have the same number of partitions. Then we have to set `"isSharding"` to `"true"` and assign the partitions to consume for OAP server. The OAP server can use commas to separate multiple partitions. + +Kafka Fetcher allows to configure all the Kafka producers listed [here](http://kafka.apache.org/24/documentation.html#consumerconfigs) in property `kafkaConsumerConfig`. Such as: +```yaml +kafka-fetcher: + selector: ${SW_KAFKA_FETCHER:-} + default: + bootstrapServers: ${SW_KAFKA_FETCHER_SERVERS:localhost:9092} + partitions: ${SW_KAFKA_FETCHER_PARTITIONS:3} + replicationFactor: ${SW_KAFKA_FETCHER_PARTITIONS_FACTOR:2} + enableMeterSystem: ${SW_KAFKA_FETCHER_ENABLE_METER_SYSTEM:false} + isSharding: ${SW_KAFKA_FETCHER_IS_SHARDING:true} + consumePartitions: ${SW_KAFKA_FETCHER_CONSUME_PARTITIONS:1,3,5} + kafkaConsumerConfig: + enable.auto.commit: true + ... +``` diff --git a/docs/en/setup/backend/backend-meter.md b/docs/en/setup/backend/backend-meter.md index e6f439aca..6d7b85a18 100644 --- a/docs/en/setup/backend/backend-meter.md +++ b/docs/en/setup/backend/backend-meter.md @@ -2,10 +2,22 @@ Meter receiver is accepting the metrics of [meter protocol](https://github.com/apache/skywalking-data-collect-protocol/blob/master/language-agent/Meter.proto) format into the [Meter System](./../../concepts-and-designs/meter.md). ## Module define +```yaml receiver-meter: selector: ${SW_RECEIVER_METER:default} default: +``` + +In Kafka Fetcher, we need to follow the configuration to enable it. +```yaml +kafka-fetcher: + selector: ${SW_KAFKA_FETCHER:default} + default: + bootstrapServers: ${SW_KAFKA_FETCHER_SERVERS:localhost:9092} + enableMeterSystem: ${SW_KAFKA_FETCHER_ENABLE_METER_SYSTEM:true} +``` + ## Configuration file Meter receiver is configured via a configuration file. The configuration file defines everything related to receiving from agents, as well as which rule files to load. diff --git a/docs/en/setup/backend/backend-receivers.md b/docs/en/setup/backend/backend-receivers.md index 893eac740..8db39ac8d 100644 --- a/docs/en/setup/backend/backend-receivers.md +++ b/docs/en/setup/backend/backend-receivers.md @@ -1,3 +1,4 @@ + # Choose receiver Receiver is a concept in SkyWalking backend. All modules, which are responsible for receiving telemetry or tracing data from other being monitored system, are all being called **Receiver**. If you are looking for the pull mode, @@ -13,7 +14,7 @@ We have following receivers, and `default` implementors are provided in our Apac 1. **receiver-profile**. gRPC services accept profile task status and snapshot reporter. 1. **receiver_zipkin**. See [details](#zipkin-receiver). 1. **receiver_jaeger**. See [details](#jaeger-receiver). -1. **receiver-oc**. See [details](#oc-receiver). +1. **receiver-oc**. See [details](#opencensus-receiver). 1. **receiver-meter**. See [details](backend-meter.md). The sample settings of these receivers should be already in default `application.yml`, and also list here @@ -25,8 +26,6 @@ receiver-register: receiver-trace: selector: ${SW_RECEIVER_TRACE:default} default: - sampleRate: ${SW_TRACE_SAMPLE_RATE:10000} # The sample rate precision is 1/10000. 10000 means 100% sample in default. - slowDBAccessThreshold: ${SW_SLOW_DB_THRESHOLD:default:200,mongodb:100} # The slow database access thresholds. Unit ms. receiver-jvm: selector: ${SW_RECEIVER_JVM:default} diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md index 0b55920d4..e217ab59b 100644 --- a/docs/en/setup/backend/configuration-vocabulary.md +++ b/docs/en/setup/backend/configuration-vocabulary.md @@ -125,6 +125,9 @@ core|default|role|Option values, `Mixed/Receiver/Aggregator`. **Receiver** mode | - | - | actions | The number of actions to collect. | SW_STORAGE_INFLUXDB_ACTIONS | 1000 | | - | - | duration | The time to wait at most (milliseconds). | SW_STORAGE_INFLUXDB_DURATION | 1000| | - | - | fetchTaskLogMaxSize | The max number of fetch task log in a request. | SW_STORAGE_INFLUXDB_FETCH_TASK_LOG_MAX_SIZE | 5000| +| agent-analyzer | default | Agent Analyzer. | SW_AGENT_ANALYZER | default | +| - | -| sampleRate|Sampling rate for receiving trace. The precision is 1/10000. 10000 means 100% sample in default.|SW_TRACE_SAMPLE_RATE|10000| +| - | - |slowDBAccessThreshold|The slow database access thresholds. Unit ms.|SW_SLOW_DB_THRESHOLD|default:200,mongodb:100| | receiver-sharing-server|default| Sharing server provides new gRPC and restful servers for data collection. Ana make the servers in the core module working for internal communication only.| - | - | | - | - | restHost| Binding IP of restful service. Services include GraphQL query and HTTP data report| - | - | | - | - | restPort | Binding port of restful service | - | - | @@ -139,8 +142,6 @@ core|default|role|Option values, `Mixed/Receiver/Aggregator`. **Receiver** mode | - | - | maxConcurrentCallsPerConnection | The maximum number of concurrent calls permitted for each incoming connection. Defaults to no limit. | - | - | | receiver-register|default| Read [receiver doc](backend-receivers.md) for more details | - | - | | receiver-trace|default| Read [receiver doc](backend-receivers.md) for more details | - | - | -| - | - |sampleRate|Sampling rate for receiving trace. The precision is 1/10000. 10000 means 100% sample in default.|SW_TRACE_SAMPLE_RATE|10000| -| - | - |slowDBAccessThreshold|The slow database access thresholds. Unit ms.|SW_SLOW_DB_THRESHOLD|default:200,mongodb:100| | receiver-jvm| default| Read [receiver doc](backend-receivers.md) for more details | - | - | | receiver-clr| default| Read [receiver doc](backend-receivers.md) for more details | - | - | | receiver-profile| default| Read [receiver doc](backend-receivers.md) for more details | - | - | @@ -172,6 +173,19 @@ core|default|role|Option values, `Mixed/Receiver/Aggregator`. **Receiver** mode | - | - | maxMessageSize | Sets the maximum message size allowed to be received on the server. Empty means 4 MiB | - | 4M(based on Netty) | | prometheus-fetcher | default | Read [fetcher doc](backend-fetcher.md) for more details | - | - | | - | - | active | Activate the Prometheus fetcher. | SW_PROMETHEUS_FETCHER_ACTIVE | false | +| kafka-fetcher | default | Read [fetcher doc](backend-fetcher.md) for more details | - | - | +| - | - | bootstrapServers | A list of host/port pairs to use for establishing the initial connection to the Kafka cluster. | SW_KAFKA_FETCHER_SERVERS | localhost:9092 | +| - | - | groupId | A unique string that identifies the consumer group this consumer belongs to.| - | skywalking-consumer | +| - | - | consumePartitions | Which PartitionId(s) of the topics assign to the OAP server. If more than one, is separated by commas. | SW_KAFKA_FETCHER_CONSUME_PARTITIONS | - | +| - | - | isSharding | it was true when OAP Server in cluster. | SW_KAFKA_FETCHER_IS_SHARDING | false | +| - | - | createTopicIfNotExist | If true, create the Kafka topic when it does not exist. | - | true | +| - | - | partitions | The number of partitions for the topic being created. | SW_KAFKA_FETCHER_PARTITIONS | 3 | +| - | - | enableMeterSystem | To enable to fetch and handle [Meter System](backend-meter.md) data. | SW_KAFKA_FETCHER_ENABLE_METER_SYSTEM | false +| - | - | replicationFactor | The replication factor for each partition in the topic being created. | SW_KAFKA_FETCHER_PARTITIONS_FACTOR | 2 | +| - | - | topicNameOfMetrics | Specifying Kafka topic name for Metrics data. | - | skywalking-metrics | +| - | - | topicNameOfProfiling | Specifying Kafka topic name for Profiling data. | - | skywalking-metrics | +| - | - | topicNameOfTracingSegments | Specifying Kafka topic name for Tracing data. | - | skywalking-metrics | +| - | - | topicNameOfManagements | Specifying Kafka topic name for service instance reporting and registering. | - | skywalking-metrics | | query | graphql | - | GraphQL query implementation | - | | - | - | path | Root path of GraphQL query and mutation. | SW_QUERY_GRAPHQL_PATH | /graphql| | alarm | default | - | Read [alarm doc](backend-alarm.md) for more details. | - | diff --git a/docs/en/setup/backend/dynamic-config.md b/docs/en/setup/backend/dynamic-config.md index 47284f7fd..ea053b8d5 100755 --- a/docs/en/setup/backend/dynamic-config.md +++ b/docs/en/setup/backend/dynamic-config.md @@ -6,12 +6,12 @@ Right now, SkyWalking supports following dynamic configurations. | Config Key | Value Description | Value Format Example | |:----:|:----:|:----:| -|receiver-trace.default.slowDBAccessThreshold| Thresholds of slow Database statement, override `receiver-trace/default/slowDBAccessThreshold` of `applciation.yml`. | default:200,mongodb:50| -|receiver-trace.default.uninstrumentedGateways| The uninstrumented gateways, override `gateways.yml`. | same as [`gateways.yml`](uninstrumented-gateways.md#configuration-format) | +|agent-analyzer.default.slowDBAccessThreshold| Thresholds of slow Database statement, override `receiver-trace/default/slowDBAccessThreshold` of `applciation.yml`. | default:200,mongodb:50| +|agent-analyzer.default.uninstrumentedGateways| The uninstrumented gateways, override `gateways.yml`. | same as [`gateways.yml`](uninstrumented-gateways.md#configuration-format) | |alarm.default.alarm-settings| The alarm settings, will override `alarm-settings.yml`. | same as [`alarm-settings.yml`](backend-alarm.md) | |core.default.apdexThreshold| The apdex threshold settings, will override `service-apdex-threshold.yml`. | same as [`service-apdex-threshold.yml`](apdex-threshold.md) | |core.default.endpoint-name-grouping| The endpoint name grouping setting, will override `endpoint-name-grouping.yml`. | same as [`endpoint-name-grouping.yml`](endpoint-grouping-rules.md) | -|receiver-trace.default.sampleRate| Trace sampling , override `receiver-trace/default/sampleRate` of `applciation.yml`. | 10000 | +|agent-analyzer.default.sampleRate| Trace sampling , override `receiver-trace/default/sampleRate` of `applciation.yml`. | 10000 | This feature depends on upstream service, so it is **DISABLED** by default. diff --git a/docs/en/setup/backend/trace-sampling.md b/docs/en/setup/backend/trace-sampling.md index facb0ae11..6a8ace66a 100644 --- a/docs/en/setup/backend/trace-sampling.md +++ b/docs/en/setup/backend/trace-sampling.md @@ -8,10 +8,10 @@ segments have been collected and reported by agents, the backend would do their to understand why we called it `as consistent as possible` and `do their best to don't break the trace`. ## Set the sample rate -In **receiver-trace** receiver, you will find `sampleRate` setting. +In **agent-analyzer** module, you will find `sampleRate` setting. ```yaml -receiver-trace: +agent-analyzer: default: ... sampleRate: ${SW_TRACE_SAMPLE_RATE:1000} # The sample rate precision is 1/10000. 10000 means 100% sample in default. diff --git a/docs/en/setup/service-agent/java-agent/How-to-enable-kafka-reporter.md b/docs/en/setup/service-agent/java-agent/How-to-enable-kafka-reporter.md new file mode 100644 index 000000000..162c4bcc6 --- /dev/null +++ b/docs/en/setup/service-agent/java-agent/How-to-enable-kafka-reporter.md @@ -0,0 +1,18 @@ +# How to enable Kafka Reporter + +The Kafka reporter plugin support report traces, JVM metrics, Instance Properties, and profiled snapshots to Kafka cluster, which is disabled in default. Move the jar of the plugin from `optional-reporter-plugins` to `reporter-plugins` for activating. + +Notice, currently, the agent still needs to configure GRPC receiver for delivering the task of profiling. In other words, the following configure cannot be omitted. + +```properties +# Backend service addresses. +collector.backend_service=${SW_AGENT_COLLECTOR_BACKEND_SERVICES:127.0.0.1:11800} + +# Kafka producer configuration +plugin.kafka.bootstrap_servers=${SW_KAFKA_BOOTSTRAP_SERVERS:localhost:9092} +plugin.kafka.producer_config[delivery.timeout.ms]=12000 +``` + +Kafka reporter plugin support to customize all configurations of listed in [here](http://kafka.apache.org/24/documentation.html#producerconfigs). + +After you activated the Kafka reporter, you need to open [Kafka fetcher](../../backend/backend-fetcher.md#kafka-fetcher) at the OAP side. \ No newline at end of file diff --git a/docs/en/setup/service-agent/java-agent/README.md b/docs/en/setup/service-agent/java-agent/README.md index ca725b55b..6a165199e 100755 --- a/docs/en/setup/service-agent/java-agent/README.md +++ b/docs/en/setup/service-agent/java-agent/README.md @@ -134,6 +134,8 @@ property key | Description | Default | `plugin.influxdb.trace_influxql`|If true, trace all the influxql(query and write) in InfluxDB access, default is true.|`true`| `correlation.element_max_number`|Max element count of the correlation context.|`3`| `correlation.value_max_length`|Max value length of correlation context element.|`128`| +`plugin.kafka.bootstrap_servers`| A list of host/port pairs to use for establishing the initial connection to the Kafka cluster. | `localhost:9092` +`plugin.kafka.consumer_config`| Kafka producer configuration. | ## Optional Plugins Java agent plugins are all pluggable. Optional plugins could be provided in `optional-plugins` folder under agent or 3rd party repositories. @@ -173,6 +175,11 @@ Now, we have the following known bootstrap plugins. * If you want to use OpenTracing Java APIs, try [SkyWalking OpenTracing compatible tracer](Opentracing.md). More details you could find at http://opentracing.io * If you want to specify the path of your agent.config file. Read [set config file through system properties](Specified-agent-config.md) +## Advanced Reporters +The advanced report provides an alternative way to submit the agent collected data to the backend. All of them are in the `optional-reporter-plugins` folder, move the one you needed into the `reporter-plugins` folder for the activation. **Notice, don't try to activate multiple reporters, that could cause unexpected fatal errors.** + +* Use Kafka to transport the traces, JVM metrics, instance properties, and profiled snapshots to the backend. Read the [How to enable Kafka Reporter](How-to-enable-kafka-reporter.md) for more details. + ## Plugin Development Guide SkyWalking java agent supports plugin to extend [the supported list](Supported-list.md). Please follow our [Plugin Development Guide](../../../guides/Java-Plugin-Development-Guide.md). diff --git a/oap-server/analyzer/agent-analyzer/pom.xml b/oap-server/analyzer/agent-analyzer/pom.xml new file mode 100644 index 000000000..b3e87a04f --- /dev/null +++ b/oap-server/analyzer/agent-analyzer/pom.xml @@ -0,0 +1,47 @@ + + + + + + analyzer + org.apache.skywalking + 8.1.0-SNAPSHOT + + 4.0.0 + + agent-analyzer + jar + + + org.apache.skywalking + configuration-api + ${project.version} + + + org.apache.skywalking + server-core + ${project.version} + + + org.codehaus.groovy + groovy + + + + \ No newline at end of file diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/module/AnalyzerModule.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/module/AnalyzerModule.java new file mode 100644 index 000000000..a406b4b2f --- /dev/null +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/module/AnalyzerModule.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.analyzer.module; + +import org.apache.skywalking.oap.server.analyzer.provider.meter.process.IMeterProcessService; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.ISegmentParserService; +import org.apache.skywalking.oap.server.library.module.ModuleDefine; + +public class AnalyzerModule extends ModuleDefine { + public static final String NAME = "agent-analyzer"; + + public AnalyzerModule() { + super(NAME); + } + + @Override + public Class[] services() { + return new Class[] { + ISegmentParserService.class, + IMeterProcessService.class + }; + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/TraceServiceModuleConfig.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/AnalyzerModuleConfig.java old mode 100755 new mode 100644 similarity index 83% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/TraceServiceModuleConfig.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/AnalyzerModuleConfig.java index 98c9255ba..0b005f3f3 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/TraceServiceModuleConfig.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/AnalyzerModuleConfig.java @@ -16,15 +16,18 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider; +package org.apache.skywalking.oap.server.analyzer.provider; import java.util.Collections; import java.util.List; import lombok.Getter; import lombok.Setter; +import org.apache.skywalking.oap.server.analyzer.provider.trace.DBLatencyThresholdsAndWatcher; +import org.apache.skywalking.oap.server.analyzer.provider.trace.TraceSampleRateWatcher; +import org.apache.skywalking.oap.server.analyzer.provider.trace.UninstrumentedGatewaysConfig; import org.apache.skywalking.oap.server.library.module.ModuleConfig; -public class TraceServiceModuleConfig extends ModuleConfig { +public class AnalyzerModuleConfig extends ModuleConfig { /** * The sample rate precision is 1/10000. 10000 means 100% sample in default. */ @@ -70,4 +73,7 @@ public class TraceServiceModuleConfig extends ModuleConfig { @Setter @Getter private int maxSlowSQLLength = 2000; + + @Getter + private final String configPath = "meter-receive-config"; } diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/AnalyzerModuleProvider.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/AnalyzerModuleProvider.java new file mode 100644 index 000000000..22bf19f24 --- /dev/null +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/AnalyzerModuleProvider.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.analyzer.provider; + +import java.util.List; +import lombok.Getter; +import org.apache.skywalking.oap.server.analyzer.module.AnalyzerModule; +import org.apache.skywalking.oap.server.analyzer.provider.meter.config.MeterConfig; +import org.apache.skywalking.oap.server.analyzer.provider.meter.config.MeterConfigs; +import org.apache.skywalking.oap.server.analyzer.provider.meter.process.IMeterProcessService; +import org.apache.skywalking.oap.server.analyzer.provider.meter.process.MeterProcessService; +import org.apache.skywalking.oap.server.analyzer.provider.trace.DBLatencyThresholdsAndWatcher; +import org.apache.skywalking.oap.server.analyzer.provider.trace.TraceSampleRateWatcher; +import org.apache.skywalking.oap.server.analyzer.provider.trace.UninstrumentedGatewaysConfig; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.ISegmentParserService; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.SegmentParserListenerManager; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.SegmentParserServiceImpl; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.MultiScopesAnalysisListener; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.NetworkAddressAliasMappingListener; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.SegmentAnalysisListener; +import org.apache.skywalking.oap.server.configuration.api.ConfigurationModule; +import org.apache.skywalking.oap.server.configuration.api.DynamicConfigurationService; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.oal.rt.CoreOALDefine; +import org.apache.skywalking.oap.server.core.oal.rt.OALEngineLoaderService; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; +import org.apache.skywalking.oap.server.library.module.ModuleDefine; +import org.apache.skywalking.oap.server.library.module.ModuleProvider; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; +import org.apache.skywalking.oap.server.telemetry.TelemetryModule; + +public class AnalyzerModuleProvider extends ModuleProvider { + @Getter + private final AnalyzerModuleConfig moduleConfig; + @Getter + private DBLatencyThresholdsAndWatcher thresholds; + @Getter + private UninstrumentedGatewaysConfig uninstrumentedGatewaysConfig; + @Getter + private SegmentParserServiceImpl segmentParserService; + @Getter + private TraceSampleRateWatcher traceSampleRateWatcher; + + private List meterConfigs; + @Getter + private MeterProcessService processService; + + public AnalyzerModuleProvider() { + this.moduleConfig = new AnalyzerModuleConfig(); + } + + @Override + public String name() { + return "default"; + } + + @Override + public Class module() { + return AnalyzerModule.class; + } + + @Override + public ModuleConfig createConfigBeanIfAbsent() { + return moduleConfig; + } + + @Override + public void prepare() throws ServiceNotProvidedException, ModuleStartException { + thresholds = new DBLatencyThresholdsAndWatcher(moduleConfig.getSlowDBAccessThreshold(), this); + + uninstrumentedGatewaysConfig = new UninstrumentedGatewaysConfig(this); + + traceSampleRateWatcher = new TraceSampleRateWatcher(this); + + moduleConfig.setDbLatencyThresholdsAndWatcher(thresholds); + moduleConfig.setUninstrumentedGatewaysConfig(uninstrumentedGatewaysConfig); + moduleConfig.setTraceSampleRateWatcher(traceSampleRateWatcher); + + segmentParserService = new SegmentParserServiceImpl(getManager(), moduleConfig); + this.registerServiceImplementation(ISegmentParserService.class, segmentParserService); + + meterConfigs = MeterConfigs.loadConfig(moduleConfig.getConfigPath()); + processService = new MeterProcessService(getManager()); + this.registerServiceImplementation(IMeterProcessService.class, processService); + } + + @Override + public void start() throws ModuleStartException { + // load official analysis + getManager().find(CoreModule.NAME) + .provider() + .getService(OALEngineLoaderService.class) + .load(CoreOALDefine.INSTANCE); + + DynamicConfigurationService dynamicConfigurationService = getManager().find(ConfigurationModule.NAME) + .provider() + .getService( + DynamicConfigurationService.class); + dynamicConfigurationService.registerConfigChangeWatcher(thresholds); + dynamicConfigurationService.registerConfigChangeWatcher(uninstrumentedGatewaysConfig); + dynamicConfigurationService.registerConfigChangeWatcher(traceSampleRateWatcher); + + segmentParserService.setListenerManager(listenerManager()); + + processService.start(meterConfigs); + } + + @Override + public void notifyAfterCompleted() { + + } + + @Override + public String[] requiredModules() { + return new String[] { + TelemetryModule.NAME, + CoreModule.NAME, + ConfigurationModule.NAME + }; + } + + private SegmentParserListenerManager listenerManager() { + SegmentParserListenerManager listenerManager = new SegmentParserListenerManager(); + if (moduleConfig.isTraceAnalysis()) { + listenerManager.add(new MultiScopesAnalysisListener.Factory(getManager())); + listenerManager.add(new NetworkAddressAliasMappingListener.Factory(getManager())); + } + listenerManager.add(new SegmentAnalysisListener.Factory(getManager(), moduleConfig)); + + return listenerManager; + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/handler/JVMSourceDispatcher.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/jvm/JVMSourceDispatcher.java similarity index 98% rename from oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/handler/JVMSourceDispatcher.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/jvm/JVMSourceDispatcher.java index e847a14cc..97f0ab424 100644 --- a/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/handler/JVMSourceDispatcher.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/jvm/JVMSourceDispatcher.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.jvm.provider.handler; +package org.apache.skywalking.oap.server.analyzer.provider.jvm; import java.util.List; import org.apache.skywalking.apm.network.common.v3.CPU; @@ -27,10 +27,10 @@ import org.apache.skywalking.apm.network.language.agent.v3.MemoryPool; import org.apache.skywalking.apm.network.language.agent.v3.Thread; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.NodeType; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.source.GCPhrase; import org.apache.skywalking.oap.server.core.source.MemoryPoolType; -import org.apache.skywalking.oap.server.core.analysis.NodeType; import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMCPU; import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMGC; import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMMemory; @@ -49,7 +49,7 @@ public class JVMSourceDispatcher { this.sourceReceiver = moduleManager.find(CoreModule.NAME).provider().getService(SourceReceiver.class); } - void sendMetric(String service, String serviceInstance, JVMMetric metrics) { + public void sendMetric(String service, String serviceInstance, JVMMetric metrics) { long minuteTimeBucket = TimeBucket.getMinuteTimeBucket(metrics.getTime()); final String serviceId = IDManager.ServiceID.buildId(service, NodeType.Normal); diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/config/MeterConfig.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/MeterConfig.java similarity index 93% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/config/MeterConfig.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/MeterConfig.java index acb017389..926f335af 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/config/MeterConfig.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/MeterConfig.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.config; +package org.apache.skywalking.oap.server.analyzer.provider.meter.config; import lombok.Data; diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/config/MeterConfigs.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/MeterConfigs.java similarity index 97% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/config/MeterConfigs.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/MeterConfigs.java index 597a76362..1c0745b5e 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/config/MeterConfigs.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/MeterConfigs.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.config; +package org.apache.skywalking.oap.server.analyzer.provider.meter.config; import lombok.Data; import lombok.extern.slf4j.Slf4j; diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/config/MeterDataConfig.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/MeterDataConfig.java similarity index 95% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/config/MeterDataConfig.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/MeterDataConfig.java index f5bde2c27..828d1348d 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/config/MeterDataConfig.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/MeterDataConfig.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.config; +package org.apache.skywalking.oap.server.analyzer.provider.meter.config; import lombok.Data; import org.apache.skywalking.oap.server.core.analysis.meter.function.MeterFunction; diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/config/Scope.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/Scope.java similarity index 94% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/config/Scope.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/Scope.java index 59ddb99ca..f146f99c3 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/config/Scope.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/Scope.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.config; +package org.apache.skywalking.oap.server.analyzer.provider.meter.config; import lombok.Data; import org.apache.skywalking.oap.server.core.analysis.meter.ScopeType; diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalData.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalData.java similarity index 97% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalData.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalData.java index 08af7df76..59d660e14 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalData.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalData.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.process; +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; import lombok.Data; diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalHistogramData.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalHistogramData.java similarity index 98% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalHistogramData.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalHistogramData.java index e75e2ea72..0933e54f4 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalHistogramData.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalHistogramData.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.process; +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; import io.vavr.Tuple; import io.vavr.Tuple2; diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalMultipleData.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalMultipleData.java similarity index 98% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalMultipleData.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalMultipleData.java index fdbe75c36..8ae47a501 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalMultipleData.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalMultipleData.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.process; +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; import io.vavr.Function2; diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalSingleData.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalSingleData.java similarity index 98% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalSingleData.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalSingleData.java index 5cca9b7c8..40d13973b 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalSingleData.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalSingleData.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.process; +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; import lombok.Data; import org.apache.skywalking.apm.network.language.agent.v3.Label; diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/MeterReceiverConfig.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/IMeterProcessService.java similarity index 71% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/MeterReceiverConfig.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/IMeterProcessService.java index 05e566af1..cfdc5845d 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/MeterReceiverConfig.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/IMeterProcessService.java @@ -16,13 +16,17 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider; +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; -import lombok.Getter; -import org.apache.skywalking.oap.server.library.module.ModuleConfig; +import java.util.List; +import org.apache.skywalking.oap.server.library.module.Service; -@Getter -public class MeterReceiverConfig extends ModuleConfig { +public interface IMeterProcessService extends Service { + + void initMeters(); + + MeterProcessor createProcessor(); + + List enabledBuilders(); - private final String configPath = "meter-receive-config"; } diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterBuilder.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterBuilder.java similarity index 97% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterBuilder.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterBuilder.java index 5df5e5484..b8c61fc0b 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterBuilder.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterBuilder.java @@ -16,18 +16,18 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.process; +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; import groovy.lang.GroovyShell; import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.analyzer.provider.meter.config.MeterConfig; +import org.apache.skywalking.oap.server.analyzer.provider.meter.config.Scope; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.meter.MeterEntity; import org.apache.skywalking.oap.server.core.analysis.meter.MeterSystem; import org.apache.skywalking.oap.server.core.analysis.meter.function.AcceptableValue; import org.apache.skywalking.oap.server.core.analysis.meter.function.AvgHistogramPercentileFunction; import org.apache.skywalking.oap.server.core.analysis.meter.function.BucketedValues; -import org.apache.skywalking.oap.server.receiver.meter.provider.config.MeterConfig; -import org.apache.skywalking.oap.server.receiver.meter.provider.config.Scope; import java.util.Map; import java.util.StringJoiner; diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterEvalOperation.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterEvalOperation.java similarity index 97% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterEvalOperation.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterEvalOperation.java index f5b82faf0..63ca6908c 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterEvalOperation.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterEvalOperation.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.process; +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; /** * Meter support calculate operation diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterProcessContext.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessService.java similarity index 64% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterProcessContext.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessService.java index fc6340f96..712463890 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterProcessContext.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessService.java @@ -16,26 +16,34 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.process; - -import org.apache.skywalking.oap.server.core.CoreModule; -import org.apache.skywalking.oap.server.core.analysis.meter.MeterSystem; -import org.apache.skywalking.oap.server.library.module.ModuleManager; -import org.apache.skywalking.oap.server.receiver.meter.provider.config.MeterConfig; +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; import java.util.List; import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.analyzer.provider.meter.config.MeterConfig; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.analysis.meter.MeterSystem; +import org.apache.skywalking.oap.server.library.module.ModuleManager; /** * Management all of the meter builders. */ -public class MeterProcessContext { +public class MeterProcessService implements IMeterProcessService { - private final List meterBuilders; + private List meterBuilders; + private final ModuleManager manager; - public MeterProcessContext(List meterBuilders, ModuleManager manager) { + private volatile boolean started = false; + + public MeterProcessService(ModuleManager manager) { + this.manager = manager; + } + + public void start(List meterBuilders) { final MeterSystem meterSystem = manager.find(CoreModule.NAME).provider().getService(MeterSystem.class); - this.meterBuilders = meterBuilders.stream().map(c -> new MeterBuilder(c, meterSystem)).collect(Collectors.toList()); + this.meterBuilders = meterBuilders.stream() + .map(c -> new MeterBuilder(c, meterSystem)) + .collect(Collectors.toList()); } /** @@ -48,14 +56,17 @@ public class MeterProcessContext { /** * Init all meters. */ - public void initMeters() { - meterBuilders.stream().forEach(MeterBuilder::initMeter); + public synchronized void initMeters() { + if (!started) { + meterBuilders.stream().forEach(MeterBuilder::initMeter); + started = true; + } } /** * Getting enabled builders. */ - List enabledBuilders() { + public List enabledBuilders() { return meterBuilders.stream().filter(MeterBuilder::hasInit).collect(Collectors.toList()); } diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterProcessor.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessor.java similarity index 95% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterProcessor.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessor.java index 5e0d2068d..eddc9e484 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterProcessor.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessor.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.process; +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; import groovy.lang.Binding; import groovy.lang.GroovyShell; @@ -41,7 +41,7 @@ public class MeterProcessor { /** * Process context. */ - private final MeterProcessContext context; + private final MeterProcessService processService; /** * All of meters has been read. Using it to process groovy script. @@ -63,8 +63,8 @@ public class MeterProcessor { */ private Long timestamp; - public MeterProcessor(MeterProcessContext context) { - this.context = context; + public MeterProcessor(MeterProcessService processService) { + this.processService = processService; } public void read(MeterData data) { @@ -107,7 +107,7 @@ public class MeterProcessor { } // Get all meter builders. - final List enabledBuilders = context.enabledBuilders(); + final List enabledBuilders = processService.enabledBuilders(); if (CollectionUtils.isEmpty(enabledBuilders)) { return; } diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/Window.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/Window.java similarity index 98% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/Window.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/Window.java index 035b28c61..d965c7931 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/Window.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/Window.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.process; +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/DBLatencyThresholdsAndWatcher.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/DBLatencyThresholdsAndWatcher.java similarity index 87% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/DBLatencyThresholdsAndWatcher.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/DBLatencyThresholdsAndWatcher.java index 2c25d0992..596a322d6 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/DBLatencyThresholdsAndWatcher.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/DBLatencyThresholdsAndWatcher.java @@ -16,21 +16,22 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider; +package org.apache.skywalking.oap.server.analyzer.provider.trace; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import org.apache.skywalking.oap.server.analyzer.module.AnalyzerModule; import org.apache.skywalking.oap.server.configuration.api.ConfigChangeWatcher; import org.apache.skywalking.oap.server.core.Const; -import org.apache.skywalking.oap.server.receiver.trace.module.TraceModule; +import org.apache.skywalking.oap.server.library.module.ModuleProvider; public class DBLatencyThresholdsAndWatcher extends ConfigChangeWatcher { private AtomicReference> thresholds; private AtomicReference settingsString; - public DBLatencyThresholdsAndWatcher(String config, TraceModuleProvider provider) { - super(TraceModule.NAME, provider, "slowDBAccessThreshold"); + public DBLatencyThresholdsAndWatcher(String config, ModuleProvider provider) { + super(AnalyzerModule.NAME, provider, "slowDBAccessThreshold"); thresholds = new AtomicReference<>(new HashMap<>()); settingsString = new AtomicReference<>(Const.EMPTY_STRING); diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/TraceSampleRateWatcher.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/TraceSampleRateWatcher.java similarity index 80% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/TraceSampleRateWatcher.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/TraceSampleRateWatcher.java index c199650a3..72c9c9ba1 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/TraceSampleRateWatcher.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/TraceSampleRateWatcher.java @@ -16,11 +16,13 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider; +package org.apache.skywalking.oap.server.analyzer.provider.trace; import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.analyzer.module.AnalyzerModule; +import org.apache.skywalking.oap.server.analyzer.provider.AnalyzerModuleConfig; import org.apache.skywalking.oap.server.configuration.api.ConfigChangeWatcher; -import org.apache.skywalking.oap.server.receiver.trace.module.TraceModule; +import org.apache.skywalking.oap.server.library.module.ModuleProvider; import java.util.concurrent.atomic.AtomicReference; @@ -28,8 +30,8 @@ import java.util.concurrent.atomic.AtomicReference; public class TraceSampleRateWatcher extends ConfigChangeWatcher { private AtomicReference sampleRate; - public TraceSampleRateWatcher(TraceModuleProvider provider) { - super(TraceModule.NAME, provider, "sampleRate"); + public TraceSampleRateWatcher(ModuleProvider provider) { + super(AnalyzerModule.NAME, provider, "sampleRate"); sampleRate = new AtomicReference<>(); sampleRate.set(getDefaultValue()); } @@ -60,7 +62,7 @@ public class TraceSampleRateWatcher extends ConfigChangeWatcher { } private int getDefaultValue() { - return ((TraceModuleProvider) this.getProvider()).getModuleConfig().getSampleRate(); + return ((AnalyzerModuleConfig) this.getProvider().createConfigBeanIfAbsent()).getSampleRate(); } public int getSampleRate() { diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/UninstrumentedGatewaysConfig.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/UninstrumentedGatewaysConfig.java old mode 100755 new mode 100644 similarity index 93% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/UninstrumentedGatewaysConfig.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/UninstrumentedGatewaysConfig.java index f1c13ee2c..6a03d4426 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/UninstrumentedGatewaysConfig.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/UninstrumentedGatewaysConfig.java @@ -15,16 +15,17 @@ * limitations under the License. */ -package org.apache.skywalking.oap.server.receiver.trace.provider; +package org.apache.skywalking.oap.server.analyzer.provider.trace; import lombok.Getter; import lombok.Setter; import lombok.ToString; import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.analyzer.module.AnalyzerModule; import org.apache.skywalking.oap.server.configuration.api.ConfigChangeWatcher; import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.library.module.ModuleProvider; import org.apache.skywalking.oap.server.library.util.ResourceUtils; -import org.apache.skywalking.oap.server.receiver.trace.module.TraceModule; import org.yaml.snakeyaml.Yaml; import java.io.FileNotFoundException; @@ -47,8 +48,8 @@ public class UninstrumentedGatewaysConfig extends ConfigChangeWatcher { private final AtomicReference settingsString; private volatile Map gatewayInstanceKeyedByAddress = Collections.emptyMap(); - UninstrumentedGatewaysConfig(TraceModuleProvider provider) { - super(TraceModule.NAME, provider, "uninstrumentedGateways"); + public UninstrumentedGatewaysConfig(ModuleProvider provider) { + super(AnalyzerModule.NAME, provider, "uninstrumentedGateways"); this.settingsString = new AtomicReference<>(Const.EMPTY_STRING); final GatewayInfos defaultGateways = parseGatewaysFromFile("gateways.yml"); log.info("Default configured gateways: {}", defaultGateways); diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/ISegmentParserListenerManager.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/ISegmentParserListenerManager.java similarity index 88% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/ISegmentParserListenerManager.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/ISegmentParserListenerManager.java index 0692f0037..e6aaed491 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/ISegmentParserListenerManager.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/ISegmentParserListenerManager.java @@ -16,9 +16,9 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider.parser; +package org.apache.skywalking.oap.server.analyzer.provider.trace.parser; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener.AnalysisListenerFactory; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.AnalysisListenerFactory; public interface ISegmentParserListenerManager { void add(AnalysisListenerFactory analysisListenerFactory); diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/ISegmentParserService.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/ISegmentParserService.java similarity index 94% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/ISegmentParserService.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/ISegmentParserService.java index 38558e8a7..6bd9c3f9a 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/ISegmentParserService.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/ISegmentParserService.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider.parser; +package org.apache.skywalking.oap.server.analyzer.provider.trace.parser; import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; import org.apache.skywalking.oap.server.library.module.Service; diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/SegmentParserListenerManager.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/SegmentParserListenerManager.java similarity index 91% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/SegmentParserListenerManager.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/SegmentParserListenerManager.java index 6c6431b19..ef1f07a76 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/SegmentParserListenerManager.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/SegmentParserListenerManager.java @@ -16,11 +16,12 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider.parser; +package org.apache.skywalking.oap.server.analyzer.provider.trace.parser; import java.util.LinkedList; import java.util.List; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener.AnalysisListenerFactory; + +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.AnalysisListenerFactory; public class SegmentParserListenerManager implements ISegmentParserListenerManager { diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/SegmentParserServiceImpl.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/SegmentParserServiceImpl.java similarity index 87% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/SegmentParserServiceImpl.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/SegmentParserServiceImpl.java index fe0ba123e..631f84f1b 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/SegmentParserServiceImpl.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/SegmentParserServiceImpl.java @@ -16,13 +16,13 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider.parser; +package org.apache.skywalking.oap.server.analyzer.provider.trace.parser; import lombok.RequiredArgsConstructor; import lombok.Setter; import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; import org.apache.skywalking.oap.server.library.module.ModuleManager; -import org.apache.skywalking.oap.server.receiver.trace.provider.TraceServiceModuleConfig; +import org.apache.skywalking.oap.server.analyzer.provider.AnalyzerModuleConfig; /** * The open service to the receivers. @@ -30,7 +30,7 @@ import org.apache.skywalking.oap.server.receiver.trace.provider.TraceServiceModu @RequiredArgsConstructor public class SegmentParserServiceImpl implements ISegmentParserService { private final ModuleManager moduleManager; - private final TraceServiceModuleConfig config; + private final AnalyzerModuleConfig config; @Setter private SegmentParserListenerManager listenerManager; diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/SpanTags.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/SpanTags.java similarity index 96% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/SpanTags.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/SpanTags.java index a605306c2..982d5e8ea 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/SpanTags.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/SpanTags.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider.parser; +package org.apache.skywalking.oap.server.analyzer.provider.trace.parser; /** * Reserved keys of the span. The backend analysis the metrics according the existed tags. diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/TraceAnalyzer.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/TraceAnalyzer.java old mode 100755 new mode 100644 similarity index 89% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/TraceAnalyzer.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/TraceAnalyzer.java index cd12ea75d..d87742fa6 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/TraceAnalyzer.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/TraceAnalyzer.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider.parser; +package org.apache.skywalking.oap.server.analyzer.provider.trace.parser; import java.util.ArrayList; import java.util.List; @@ -26,20 +26,20 @@ import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; import org.apache.skywalking.apm.network.language.agent.v3.SpanType; import org.apache.skywalking.oap.server.library.module.ModuleManager; -import org.apache.skywalking.oap.server.receiver.trace.provider.TraceServiceModuleConfig; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener.AnalysisListener; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener.EntryAnalysisListener; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener.ExitAnalysisListener; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener.FirstAnalysisListener; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener.LocalAnalysisListener; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener.SegmentListener; +import org.apache.skywalking.oap.server.analyzer.provider.AnalyzerModuleConfig; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.AnalysisListener; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.EntryAnalysisListener; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.ExitAnalysisListener; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.FirstAnalysisListener; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.LocalAnalysisListener; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.SegmentListener; @Slf4j @RequiredArgsConstructor public class TraceAnalyzer { private final ModuleManager moduleManager; private final SegmentParserListenerManager listenerManager; - private final TraceServiceModuleConfig config; + private final AnalyzerModuleConfig config; private List analysisListeners = new ArrayList<>(); public void doAnalysis(SegmentObject segmentObject) { diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/AnalysisListener.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/AnalysisListener.java similarity index 95% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/AnalysisListener.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/AnalysisListener.java index 30a8dac49..cc18e88c6 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/AnalysisListener.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/AnalysisListener.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener; +package org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener; /** * AnalysisListener represents the callback when OAP does the trace segment analysis. diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/AnalysisListenerFactory.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/AnalysisListenerFactory.java similarity index 81% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/AnalysisListenerFactory.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/AnalysisListenerFactory.java index 15fafbba8..c29154ad4 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/AnalysisListenerFactory.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/AnalysisListenerFactory.java @@ -16,15 +16,15 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener; +package org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener; import org.apache.skywalking.oap.server.library.module.ModuleManager; -import org.apache.skywalking.oap.server.receiver.trace.provider.TraceServiceModuleConfig; +import org.apache.skywalking.oap.server.analyzer.provider.AnalyzerModuleConfig; /** * AnalysisListenerFactory implementation creates the listener instances when required. Every AnalysisListener could * have its own creation factory. */ public interface AnalysisListenerFactory { - AnalysisListener create(ModuleManager moduleManager, TraceServiceModuleConfig config); + AnalysisListener create(ModuleManager moduleManager, AnalyzerModuleConfig config); } diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/EntryAnalysisListener.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/EntryAnalysisListener.java similarity index 94% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/EntryAnalysisListener.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/EntryAnalysisListener.java index 5550141a8..93c828fc2 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/EntryAnalysisListener.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/EntryAnalysisListener.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener; +package org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener; import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/ExitAnalysisListener.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/ExitAnalysisListener.java similarity index 94% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/ExitAnalysisListener.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/ExitAnalysisListener.java index f7b86ae5f..c95fbb9e3 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/ExitAnalysisListener.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/ExitAnalysisListener.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener; +package org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener; import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/FirstAnalysisListener.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/FirstAnalysisListener.java similarity index 94% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/FirstAnalysisListener.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/FirstAnalysisListener.java index 1f1bb71a9..af438760d 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/FirstAnalysisListener.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/FirstAnalysisListener.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener; +package org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener; import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/LocalAnalysisListener.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/LocalAnalysisListener.java similarity index 94% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/LocalAnalysisListener.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/LocalAnalysisListener.java index cbaa530be..cb5ff1867 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/LocalAnalysisListener.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/LocalAnalysisListener.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener; +package org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener; import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/MultiScopesAnalysisListener.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/MultiScopesAnalysisListener.java old mode 100755 new mode 100644 similarity index 97% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/MultiScopesAnalysisListener.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/MultiScopesAnalysisListener.java index 7b7b07509..96a8a8028 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/MultiScopesAnalysisListener.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/MultiScopesAnalysisListener.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener; +package org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener; import com.google.gson.Gson; import com.google.gson.JsonObject; @@ -31,6 +31,7 @@ import org.apache.skywalking.apm.network.language.agent.v3.SpanLayer; import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; import org.apache.skywalking.apm.network.language.agent.v3.SpanType; import org.apache.skywalking.apm.util.StringUtil; +import org.apache.skywalking.oap.server.analyzer.provider.trace.DBLatencyThresholdsAndWatcher; import org.apache.skywalking.oap.server.core.Const; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.analysis.IDManager; @@ -46,11 +47,10 @@ import org.apache.skywalking.oap.server.core.source.RequestType; import org.apache.skywalking.oap.server.core.source.ServiceInstanceRelation; import org.apache.skywalking.oap.server.core.source.SourceReceiver; import org.apache.skywalking.oap.server.library.module.ModuleManager; -import org.apache.skywalking.oap.server.receiver.trace.provider.DBLatencyThresholdsAndWatcher; -import org.apache.skywalking.oap.server.receiver.trace.provider.TraceServiceModuleConfig; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.SpanTags; +import org.apache.skywalking.oap.server.analyzer.provider.AnalyzerModuleConfig; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.SpanTags; -import static org.apache.skywalking.oap.server.receiver.trace.provider.parser.SpanTags.LOGIC_ENDPOINT; +import static org.apache.skywalking.oap.server.analyzer.provider.trace.parser.SpanTags.LOGIC_ENDPOINT; /** * MultiScopesSpanListener includes the most segment to source(s) logic. @@ -66,7 +66,7 @@ public class MultiScopesAnalysisListener implements EntryAnalysisListener, ExitA private final List logicEndpointBuilders = new ArrayList<>(10); private final Gson gson = new Gson(); private final SourceReceiver sourceReceiver; - private final TraceServiceModuleConfig config; + private final AnalyzerModuleConfig config; private final NetworkAddressAliasCache networkAddressAliasCache; private final NamingControl namingControl; @@ -371,7 +371,7 @@ public class MultiScopesAnalysisListener implements EntryAnalysisListener, ExitA } @Override - public AnalysisListener create(ModuleManager moduleManager, TraceServiceModuleConfig config) { + public AnalysisListener create(ModuleManager moduleManager, AnalyzerModuleConfig config) { return new MultiScopesAnalysisListener( sourceReceiver, config, networkAddressAliasCache, namingControl); } diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/NetworkAddressAliasMappingListener.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/NetworkAddressAliasMappingListener.java similarity index 95% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/NetworkAddressAliasMappingListener.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/NetworkAddressAliasMappingListener.java index d949b9d3e..3622e3e4a 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/NetworkAddressAliasMappingListener.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/NetworkAddressAliasMappingListener.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener; +package org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -31,7 +31,7 @@ import org.apache.skywalking.oap.server.core.config.NamingControl; import org.apache.skywalking.oap.server.core.source.NetworkAddressAliasSetup; import org.apache.skywalking.oap.server.core.source.SourceReceiver; import org.apache.skywalking.oap.server.library.module.ModuleManager; -import org.apache.skywalking.oap.server.receiver.trace.provider.TraceServiceModuleConfig; +import org.apache.skywalking.oap.server.analyzer.provider.AnalyzerModuleConfig; /** * NetworkAddressAliasMappingListener use the propagated data in the segment reference, set up the alias relationship @@ -45,7 +45,7 @@ import org.apache.skywalking.oap.server.receiver.trace.provider.TraceServiceModu @RequiredArgsConstructor public class NetworkAddressAliasMappingListener implements EntryAnalysisListener { private final SourceReceiver sourceReceiver; - private final TraceServiceModuleConfig config; + private final AnalyzerModuleConfig config; private final NamingControl namingControl; @Override @@ -107,7 +107,7 @@ public class NetworkAddressAliasMappingListener implements EntryAnalysisListener } @Override - public AnalysisListener create(ModuleManager moduleManager, TraceServiceModuleConfig config) { + public AnalysisListener create(ModuleManager moduleManager, AnalyzerModuleConfig config) { return new NetworkAddressAliasMappingListener(sourceReceiver, config, namingControl); } } diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/SegmentAnalysisListener.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/SegmentAnalysisListener.java similarity index 96% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/SegmentAnalysisListener.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/SegmentAnalysisListener.java index 1cc6a3583..b0f34613d 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/SegmentAnalysisListener.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/SegmentAnalysisListener.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener; +package org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -33,7 +33,7 @@ import org.apache.skywalking.oap.server.core.source.Segment; import org.apache.skywalking.oap.server.core.source.SourceReceiver; import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.library.util.BooleanUtils; -import org.apache.skywalking.oap.server.receiver.trace.provider.TraceServiceModuleConfig; +import org.apache.skywalking.oap.server.analyzer.provider.AnalyzerModuleConfig; /** * SegmentSpanListener forwards the segment raw data to the persistence layer with the query required conditions. @@ -174,7 +174,7 @@ public class SegmentAnalysisListener implements FirstAnalysisListener, EntryAnal private final TraceSegmentSampler sampler; private final NamingControl namingControl; - public Factory(ModuleManager moduleManager, TraceServiceModuleConfig config) { + public Factory(ModuleManager moduleManager, AnalyzerModuleConfig config) { this.sourceReceiver = moduleManager.find(CoreModule.NAME).provider().getService(SourceReceiver.class); this.sampler = new TraceSegmentSampler(config.getTraceSampleRateWatcher()); this.namingControl = moduleManager.find(CoreModule.NAME) @@ -183,7 +183,7 @@ public class SegmentAnalysisListener implements FirstAnalysisListener, EntryAnal } @Override - public AnalysisListener create(ModuleManager moduleManager, TraceServiceModuleConfig config) { + public AnalysisListener create(ModuleManager moduleManager, AnalyzerModuleConfig config) { return new SegmentAnalysisListener(sourceReceiver, sampler, namingControl); } } diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/SegmentListener.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/SegmentListener.java similarity index 94% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/SegmentListener.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/SegmentListener.java index 20fc5df8f..5897853ec 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/SegmentListener.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/SegmentListener.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener; +package org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener; import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/SourceBuilder.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/SourceBuilder.java similarity index 99% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/SourceBuilder.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/SourceBuilder.java index 689c51f91..c985eb3b9 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/SourceBuilder.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/SourceBuilder.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener; +package org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/TraceSegmentSampler.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/TraceSegmentSampler.java similarity index 91% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/TraceSegmentSampler.java rename to oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/TraceSegmentSampler.java index 0ad582169..39cf88aa7 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/TraceSegmentSampler.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/TraceSegmentSampler.java @@ -16,9 +16,9 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener; +package org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener; -import org.apache.skywalking.oap.server.receiver.trace.provider.TraceSampleRateWatcher; +import org.apache.skywalking.oap.server.analyzer.provider.trace.TraceSampleRateWatcher; /** * The sampler makes the sampling mechanism works at backend side. Sample result: [0,sampleRate) sampled, (sampleRate,~) diff --git a/oap-server/analyzer/agent-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine b/oap-server/analyzer/agent-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine new file mode 100644 index 000000000..f4df4e65f --- /dev/null +++ b/oap-server/analyzer/agent-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.oap.server.analyzer.module.AnalyzerModule diff --git a/oap-server/analyzer/agent-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider b/oap-server/analyzer/agent-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider new file mode 100644 index 000000000..a303114c4 --- /dev/null +++ b/oap-server/analyzer/agent-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.oap.server.analyzer.provider.AnalyzerModuleProvider \ No newline at end of file diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalDataBaseTest.java b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalDataBaseTest.java similarity index 97% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalDataBaseTest.java rename to oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalDataBaseTest.java index e2c3f4114..6d63bd122 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalDataBaseTest.java +++ b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalDataBaseTest.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.process; +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; import org.apache.skywalking.apm.network.language.agent.v3.Label; import org.apache.skywalking.apm.network.language.agent.v3.MeterBucketValue; diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalDataTest.java b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalDataTest.java similarity index 97% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalDataTest.java rename to oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalDataTest.java index cf15a78d1..0e7a62531 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalDataTest.java +++ b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalDataTest.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.process; +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; import org.junit.Assert; import org.junit.Test; diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalHistogramDataTest.java b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalHistogramDataTest.java similarity index 98% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalHistogramDataTest.java rename to oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalHistogramDataTest.java index b3d7dfcd2..65610f19a 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalHistogramDataTest.java +++ b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalHistogramDataTest.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.process; +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; import io.vavr.Function2; import org.apache.skywalking.apm.network.language.agent.v3.Label; diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalMultipleDataTest.java b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalMultipleDataTest.java similarity index 98% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalMultipleDataTest.java rename to oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalMultipleDataTest.java index 17ebf7335..5e1965ee6 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalMultipleDataTest.java +++ b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalMultipleDataTest.java @@ -16,17 +16,16 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.process; +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; import io.vavr.Function2; +import java.util.Arrays; +import java.util.List; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.powermock.reflect.Whitebox; -import java.util.Arrays; -import java.util.List; - public class EvalMultipleDataTest extends EvalDataBaseTest { private EvalMultipleData singleMultiple; diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalSingleDataTest.java b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalSingleDataTest.java similarity index 98% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalSingleDataTest.java rename to oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalSingleDataTest.java index 7c6658ffc..3405aa44f 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/EvalSingleDataTest.java +++ b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/EvalSingleDataTest.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.process; +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; import io.vavr.Function2; import org.junit.Assert; diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterBaseTest.java b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterBaseTest.java similarity index 90% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterBaseTest.java rename to oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterBaseTest.java index 1dcf890ce..c3e392dc8 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterBaseTest.java +++ b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterBaseTest.java @@ -16,14 +16,18 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.process; +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; import com.google.common.collect.Maps; +import java.util.HashMap; +import java.util.List; import org.apache.skywalking.apm.network.language.agent.v3.Label; import org.apache.skywalking.apm.network.language.agent.v3.MeterBucketValue; import org.apache.skywalking.apm.network.language.agent.v3.MeterData; import org.apache.skywalking.apm.network.language.agent.v3.MeterHistogram; import org.apache.skywalking.apm.network.language.agent.v3.MeterSingleValue; +import org.apache.skywalking.oap.server.analyzer.provider.meter.config.MeterConfig; +import org.apache.skywalking.oap.server.analyzer.provider.meter.config.MeterConfigs; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.CoreModuleProvider; import org.apache.skywalking.oap.server.core.analysis.DisableRegister; @@ -32,20 +36,15 @@ import org.apache.skywalking.oap.server.core.analysis.meter.function.AvgFunction import org.apache.skywalking.oap.server.core.analysis.meter.function.AvgHistogramFunction; import org.apache.skywalking.oap.server.core.analysis.meter.function.AvgHistogramPercentileFunction; import org.apache.skywalking.oap.server.library.module.ModuleManager; -import org.apache.skywalking.oap.server.receiver.meter.provider.MeterReceiverConfig; -import org.apache.skywalking.oap.server.receiver.meter.provider.config.MeterConfig; -import org.apache.skywalking.oap.server.receiver.meter.provider.config.MeterConfigs; import org.junit.Before; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.internal.util.reflection.Whitebox; -import java.util.HashMap; -import java.util.List; - import static org.mockito.Mockito.when; public abstract class MeterBaseTest { + private static final String CONFIG_PATH = "meter-receive-config"; @Mock protected CoreModuleProvider moduleProvider; @@ -81,11 +80,12 @@ public abstract class MeterBaseTest { Whitebox.setInternalState(meterSystem, "functionRegister", map); // load context - List meterConfigs = MeterConfigs.loadConfig(new MeterReceiverConfig().getConfigPath()); - final MeterProcessContext context = new MeterProcessContext(meterConfigs, moduleManager); + List meterConfigs = MeterConfigs.loadConfig(CONFIG_PATH); + final MeterProcessService service = new MeterProcessService(moduleManager); + service.start(meterConfigs); // create process and read meters - processor = context.createProcessor(); + processor = service.createProcessor(); timestamp = System.currentTimeMillis(); // single value diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterBuilderTest.java b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterBuilderTest.java similarity index 96% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterBuilderTest.java rename to oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterBuilderTest.java index 8ea38381a..b92fd4030 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterBuilderTest.java +++ b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterBuilderTest.java @@ -16,8 +16,10 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.process; +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; +import java.util.ArrayList; +import java.util.List; import org.apache.skywalking.oap.server.core.analysis.IDManager; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.meter.function.AcceptableValue; @@ -33,9 +35,6 @@ import org.mockito.internal.util.reflection.Whitebox; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.modules.junit4.PowerMockRunner; -import java.util.ArrayList; -import java.util.List; - import static org.mockito.Matchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doCallRealMethod; @@ -51,7 +50,8 @@ public class MeterBuilderTest extends MeterBaseTest { values.add(invocationOnMock.getArgumentAt(0, AcceptableValue.class)); return null; }).when(meterSystem).doStreamingCalculation(any()); - final MeterProcessContext context = (MeterProcessContext) Whitebox.getInternalState(processor, "context"); + + final MeterProcessService context = (MeterProcessService) Whitebox.getInternalState(processor, "processService"); context.enabledBuilders().stream().peek(b -> doCallRealMethod().when(b).buildAndSend(any(), any())); context.initMeters(); diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterEvalOperationTest.java b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterEvalOperationTest.java similarity index 97% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterEvalOperationTest.java rename to oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterEvalOperationTest.java index c13f69dd1..abc12bd3c 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterEvalOperationTest.java +++ b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterEvalOperationTest.java @@ -16,11 +16,10 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.process; - -import org.junit.Test; +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; import java.util.function.Consumer; +import org.junit.Test; public class MeterEvalOperationTest { diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterProcessContextTest.java b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessContextTest.java similarity index 69% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterProcessContextTest.java rename to oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessContextTest.java index 98849ba36..a53eb33d4 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterProcessContextTest.java +++ b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessContextTest.java @@ -16,12 +16,12 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.process; +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; +import java.util.List; +import org.apache.skywalking.oap.server.analyzer.provider.meter.config.MeterConfig; +import org.apache.skywalking.oap.server.analyzer.provider.meter.config.MeterConfigs; import org.apache.skywalking.oap.server.library.module.ModuleStartException; -import org.apache.skywalking.oap.server.receiver.meter.provider.MeterReceiverConfig; -import org.apache.skywalking.oap.server.receiver.meter.provider.config.MeterConfig; -import org.apache.skywalking.oap.server.receiver.meter.provider.config.MeterConfigs; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -29,8 +29,6 @@ import org.mockito.internal.util.reflection.Whitebox; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.modules.junit4.PowerMockRunner; -import java.util.List; - import static org.mockito.Matchers.any; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -38,23 +36,26 @@ import static org.mockito.Mockito.verify; @RunWith(PowerMockRunner.class) @PowerMockIgnore("javax.management.*") public class MeterProcessContextTest extends MeterBaseTest { + private static final String CONFIG_PATH = "meter-receive-config"; @Test public void testInitMeter() throws ModuleStartException { - List meterConfigs = MeterConfigs.loadConfig(new MeterReceiverConfig().getConfigPath()); - final MeterProcessContext context = new MeterProcessContext(meterConfigs, moduleManager); + List meterConfigs = MeterConfigs.loadConfig(CONFIG_PATH); + final MeterProcessService service = new MeterProcessService(moduleManager); + service.start(meterConfigs); - context.initMeters(); + service.initMeters(); verify(meterSystem, times(3)).create(any(), any(), any()); - Assert.assertEquals(3, context.enabledBuilders().size()); + Assert.assertEquals(3, service.enabledBuilders().size()); } @Test public void testCreateNewProcessor() throws ModuleStartException { - List meterConfigs = MeterConfigs.loadConfig(new MeterReceiverConfig().getConfigPath()); - final MeterProcessContext context = new MeterProcessContext(meterConfigs, moduleManager); + List meterConfigs = MeterConfigs.loadConfig(CONFIG_PATH); + final MeterProcessService service = new MeterProcessService(moduleManager); + service.start(meterConfigs); - final MeterProcessor processor = context.createProcessor(); - Assert.assertEquals(context, Whitebox.getInternalState(processor, "context")); + final MeterProcessor processor = service.createProcessor(); + Assert.assertEquals(service, Whitebox.getInternalState(processor, "processService")); } } diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterProcessorTest.java b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessorTest.java similarity index 96% rename from oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterProcessorTest.java rename to oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessorTest.java index 1ccfd83f3..6a042aed0 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/process/MeterProcessorTest.java +++ b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessorTest.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.server.receiver.meter.provider.process; +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; import org.junit.Assert; import org.junit.Test; @@ -67,7 +67,7 @@ public class MeterProcessorTest extends MeterBaseTest { @Test public void testProcess() { // each builder has build and send - MeterProcessContext context = (MeterProcessContext) Whitebox.getInternalState(processor, "context"); + MeterProcessService context = (MeterProcessService) Whitebox.getInternalState(processor, "processService"); List builders = context.enabledBuilders().stream().map(Mockito::spy) .peek(builder -> doNothing().when(builder).buildAndSend(any(), any())).collect(Collectors.toList()); Whitebox.setInternalState(context, "meterBuilders", builders); diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/trace/provider/TraceSampleRateWatcherTest.java b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/trace/TraceSampleRateWatcherTest.java similarity index 81% rename from oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/trace/provider/TraceSampleRateWatcherTest.java rename to oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/trace/TraceSampleRateWatcherTest.java index 5e621a8b7..a1584b373 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/trace/provider/TraceSampleRateWatcherTest.java +++ b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/trace/TraceSampleRateWatcherTest.java @@ -16,8 +16,11 @@ * */ -package org.apache.skywalking.oap.server.receiver.trace.provider; +package org.apache.skywalking.oap.server.analyzer.provider.trace; +import java.util.Optional; +import java.util.Set; +import org.apache.skywalking.oap.server.analyzer.provider.AnalyzerModuleProvider; import org.apache.skywalking.oap.server.configuration.api.ConfigChangeWatcher; import org.apache.skywalking.oap.server.configuration.api.ConfigTable; import org.apache.skywalking.oap.server.configuration.api.ConfigWatcherRegister; @@ -27,24 +30,21 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.runners.MockitoJUnitRunner; -import java.util.Optional; -import java.util.Set; - import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; @RunWith(MockitoJUnitRunner.class) public class TraceSampleRateWatcherTest { - private TraceModuleProvider traceModuleProvider; + private AnalyzerModuleProvider provider; @Before public void init() { - traceModuleProvider = new TraceModuleProvider(); + provider = new AnalyzerModuleProvider(); } @Test public void testInit() { - TraceSampleRateWatcher traceSampleRateWatcher = new TraceSampleRateWatcher(traceModuleProvider); + TraceSampleRateWatcher traceSampleRateWatcher = new TraceSampleRateWatcher(provider); Assert.assertEquals(traceSampleRateWatcher.getSampleRate(), 10000); Assert.assertEquals(traceSampleRateWatcher.value(), "10000"); } @@ -53,7 +53,7 @@ public class TraceSampleRateWatcherTest { public void testDynamicUpdate() throws InterruptedException { ConfigWatcherRegister register = new MockConfigWatcherRegister(3); - TraceSampleRateWatcher watcher = new TraceSampleRateWatcher(traceModuleProvider); + TraceSampleRateWatcher watcher = new TraceSampleRateWatcher(provider); register.registerConfigChangeWatcher(watcher); register.start(); @@ -61,31 +61,35 @@ public class TraceSampleRateWatcherTest { Thread.sleep(2000); } assertThat(watcher.getSampleRate(), is(9000)); - assertThat(traceModuleProvider.getModuleConfig().getSampleRate(), is(10000)); + assertThat(provider.getModuleConfig().getSampleRate(), is(10000)); } @Test public void testNotify() { - TraceSampleRateWatcher traceSampleRateWatcher = new TraceSampleRateWatcher(traceModuleProvider); - ConfigChangeWatcher.ConfigChangeEvent value1 = new ConfigChangeWatcher.ConfigChangeEvent("8000", ConfigChangeWatcher.EventType.MODIFY); + TraceSampleRateWatcher traceSampleRateWatcher = new TraceSampleRateWatcher(provider); + ConfigChangeWatcher.ConfigChangeEvent value1 = new ConfigChangeWatcher.ConfigChangeEvent( + "8000", ConfigChangeWatcher.EventType.MODIFY); traceSampleRateWatcher.notify(value1); Assert.assertEquals(traceSampleRateWatcher.getSampleRate(), 8000); Assert.assertEquals(traceSampleRateWatcher.value(), "8000"); - ConfigChangeWatcher.ConfigChangeEvent value2 = new ConfigChangeWatcher.ConfigChangeEvent("8000", ConfigChangeWatcher.EventType.DELETE); + ConfigChangeWatcher.ConfigChangeEvent value2 = new ConfigChangeWatcher.ConfigChangeEvent( + "8000", ConfigChangeWatcher.EventType.DELETE); traceSampleRateWatcher.notify(value2); Assert.assertEquals(traceSampleRateWatcher.getSampleRate(), 10000); Assert.assertEquals(traceSampleRateWatcher.value(), "10000"); - ConfigChangeWatcher.ConfigChangeEvent value3 = new ConfigChangeWatcher.ConfigChangeEvent("500", ConfigChangeWatcher.EventType.ADD); + ConfigChangeWatcher.ConfigChangeEvent value3 = new ConfigChangeWatcher.ConfigChangeEvent( + "500", ConfigChangeWatcher.EventType.ADD); traceSampleRateWatcher.notify(value3); Assert.assertEquals(traceSampleRateWatcher.getSampleRate(), 500); Assert.assertEquals(traceSampleRateWatcher.value(), "500"); - ConfigChangeWatcher.ConfigChangeEvent value4 = new ConfigChangeWatcher.ConfigChangeEvent("abc", ConfigChangeWatcher.EventType.MODIFY); + ConfigChangeWatcher.ConfigChangeEvent value4 = new ConfigChangeWatcher.ConfigChangeEvent( + "abc", ConfigChangeWatcher.EventType.MODIFY); traceSampleRateWatcher.notify(value4); Assert.assertEquals(traceSampleRateWatcher.getSampleRate(), 500); @@ -101,7 +105,7 @@ public class TraceSampleRateWatcherTest { @Override public Optional readConfig(Set keys) { ConfigTable table = new ConfigTable(); - table.add(new ConfigTable.ConfigItem("receiver-trace.default.sampleRate", "9000")); + table.add(new ConfigTable.ConfigItem("agent-analyzer.default.sampleRate", "9000")); return Optional.of(table); } } diff --git a/oap-server/analyzer/agent-analyzer/src/test/resources/meter-receive-config/config.yaml b/oap-server/analyzer/agent-analyzer/src/test/resources/meter-receive-config/config.yaml new file mode 100644 index 000000000..61fc45103 --- /dev/null +++ b/oap-server/analyzer/agent-analyzer/src/test/resources/meter-receive-config/config.yaml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meters: + - name: build_test1 + scope: + type: SERVICE + meter: + operation: avg + value: meter["test_count1"].tagFilter("k1", "v1").scale(2) + - name: build_test2 + scope: + type: SERVICE_INSTANCE + meter: + operation: avgHistogram + value: meter["test_histogram"] + - name: build_test3 + scope: + type: ENDPOINT + endpoint: test_endpoint + meter: + operation: avgHistogramPercentile + value: meter["test_histogram"] + percentile: + - 50 + - 90 + - 99 diff --git a/oap-server/analyzer/pom.xml b/oap-server/analyzer/pom.xml new file mode 100644 index 000000000..756752515 --- /dev/null +++ b/oap-server/analyzer/pom.xml @@ -0,0 +1,52 @@ + + + + + + oap-server + org.apache.skywalking + 8.1.0-SNAPSHOT + + 4.0.0 + + analyzer + pom + + + agent-analyzer + + + + + org.apache.skywalking + apm-network + ${project.version} + + + org.apache.skywalking + library-module + ${project.version} + + + org.apache.skywalking + library-util + ${project.version} + + + \ No newline at end of file diff --git a/oap-server/pom.xml b/oap-server/pom.xml index 3bd45a2de..fc7ce2401 100755 --- a/oap-server/pom.xml +++ b/oap-server/pom.xml @@ -29,6 +29,7 @@ pom server-core + analyzer server-receiver-plugin server-cluster-plugin server-storage-plugin @@ -95,7 +96,8 @@ 3.5 3.11.4 - + 2.4.1 + 2.4.6.RELEASE diff --git a/oap-server/server-bootstrap/pom.xml b/oap-server/server-bootstrap/pom.xml index eeab352cd..69a5235a0 100644 --- a/oap-server/server-bootstrap/pom.xml +++ b/oap-server/server-bootstrap/pom.xml @@ -134,6 +134,11 @@ prometheus-fetcher-plugin ${project.version} + + org.apache.skywalking + kafka-fetcher-plugin + ${project.version} + diff --git a/oap-server/server-bootstrap/src/main/resources/application.yml b/oap-server/server-bootstrap/src/main/resources/application.yml index bda4de745..1bda6a9e8 100755 --- a/oap-server/server-bootstrap/src/main/resources/application.yml +++ b/oap-server/server-bootstrap/src/main/resources/application.yml @@ -158,6 +158,12 @@ storage: duration: ${SW_STORAGE_INFLUXDB_DURATION:1000} # the time to wait at most (milliseconds) fetchTaskLogMaxSize: ${SW_STORAGE_INFLUXDB_FETCH_TASK_LOG_MAX_SIZE:5000} # the max number of fetch task log in a request +agent-analyzer: + selector: ${SW_AGENT_ANALYZER:default} + default: + sampleRate: ${SW_TRACE_SAMPLE_RATE:10000} # The sample rate precision is 1/10000. 10000 means 100% sample in default. + slowDBAccessThreshold: ${SW_SLOW_DB_THRESHOLD:default:200,mongodb:100} # The slow database access thresholds. Unit ms. + receiver-sharing-server: selector: ${SW_RECEIVER_SHARING_SERVER:default} default: @@ -176,8 +182,6 @@ receiver-register: receiver-trace: selector: ${SW_RECEIVER_TRACE:default} default: - sampleRate: ${SW_TRACE_SAMPLE_RATE:10000} # The sample rate precision is 1/10000. 10000 means 100% sample in default. - slowDBAccessThreshold: ${SW_SLOW_DB_THRESHOLD:default:200,mongodb:100} # The slow database access thresholds. Unit ms. receiver-jvm: selector: ${SW_RECEIVER_JVM:default} @@ -210,10 +214,20 @@ prometheus-fetcher: default: active: ${SW_PROMETHEUS_FETCHER_ACTIVE:false} +kafka-fetcher: + selector: ${SW_KAFKA_FETCHER:-} + default: + bootstrapServers: ${SW_KAFKA_FETCHER_SERVERS:localhost:9092} + partitions: ${SW_KAFKA_FETCHER_PARTITIONS:3} + replicationFactor: ${SW_KAFKA_FETCHER_PARTITIONS_FACTOR:2} + enableMeterSystem: ${SW_KAFKA_FETCHER_ENABLE_METER_SYSTEM:false} + isSharding: ${SW_KAFKA_FETCHER_IS_SHARDING:false} + consumePartitions: ${SW_KAFKA_FETCHER_CONSUME_PARTITIONS:""} + receiver-meter: selector: ${SW_RECEIVER_METER:-} default: - + receiver-oc: selector: ${SW_OC_RECEIVER:-} default: diff --git a/oap-server/server-cluster-plugin/cluster-zookeeper-plugin/src/main/java/org/apache/skywalking/oap/server/cluster/plugin/zookeeper/ZookeeperCoordinator.java b/oap-server/server-cluster-plugin/cluster-zookeeper-plugin/src/main/java/org/apache/skywalking/oap/server/cluster/plugin/zookeeper/ZookeeperCoordinator.java index 51f7f67f4..2fbdf0746 100644 --- a/oap-server/server-cluster-plugin/cluster-zookeeper-plugin/src/main/java/org/apache/skywalking/oap/server/cluster/plugin/zookeeper/ZookeeperCoordinator.java +++ b/oap-server/server-cluster-plugin/cluster-zookeeper-plugin/src/main/java/org/apache/skywalking/oap/server/cluster/plugin/zookeeper/ZookeeperCoordinator.java @@ -44,7 +44,7 @@ public class ZookeeperCoordinator implements ClusterRegister, ClusterNodesQuery private volatile Address selfAddress; ZookeeperCoordinator(ClusterModuleZookeeperConfig config, - ServiceDiscovery serviceDiscovery) throws Exception { + ServiceDiscovery serviceDiscovery) throws Exception { this.config = config; this.serviceDiscovery = serviceDiscovery; this.serviceCache = serviceDiscovery.serviceCacheBuilder().name(REMOTE_NAME_PATH).build(); @@ -62,11 +62,11 @@ public class ZookeeperCoordinator implements ClusterRegister, ClusterNodesQuery .id(UUID.randomUUID() .toString()) .address(remoteInstance - .getAddress() - .getHost()) + .getAddress() + .getHost()) .port(remoteInstance - .getAddress() - .getPort()) + .getAddress() + .getPort()) .payload(remoteInstance) .build(); diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/pom.xml b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/pom.xml new file mode 100644 index 000000000..f6fdfcf3b --- /dev/null +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/pom.xml @@ -0,0 +1,50 @@ + + + + + + server-fetcher-plugin + org.apache.skywalking + 8.1.0-SNAPSHOT + + 4.0.0 + + kafka-fetcher-plugin + jar + + + + org.apache.skywalking + agent-analyzer + ${project.version} + + + org.apache.kafka + kafka-clients + ${kafka-clients.version} + + + org.springframework.kafka + spring-kafka-test + ${spring-kafka-test.version} + test + + + + \ No newline at end of file diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/KafkaFetcherHandlerRegister.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/KafkaFetcherHandlerRegister.java new file mode 100644 index 000000000..b2860f6b1 --- /dev/null +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/KafkaFetcherHandlerRegister.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.analyzer.agent.kafka; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import io.netty.util.concurrent.DefaultThreadFactory; +import java.time.Duration; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.serialization.BytesDeserializer; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.utils.Bytes; +import org.apache.skywalking.apm.util.StringUtil; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.provider.handler.KafkaHandler; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.module.KafkaFetcherConfig; + +/** + * Configuring and initializing a KafkaConsumer client as a dispatcher to delivery Kafka Message to registered handler by topic. + */ +@Slf4j +public class KafkaFetcherHandlerRegister implements Runnable { + + private ImmutableMap.Builder builder = ImmutableMap.builder(); + private ImmutableMap handlerMap; + + private List topicPartitions = Lists.newArrayList(); + private KafkaConsumer consumer = null; + private final KafkaFetcherConfig config; + private final boolean isSharding; + + public KafkaFetcherHandlerRegister(KafkaFetcherConfig config) throws ModuleStartException { + this.config = config; + Properties properties = new Properties(config.getKafkaConsumerConfig()); + properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, config.getGroupId()); + properties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, config.getBootstrapServers()); + + AdminClient adminClient = AdminClient.create(properties); + Set missedTopics = adminClient.describeTopics(Lists.newArrayList( + config.getTopicNameOfManagements(), + config.getTopicNameOfMetrics(), + config.getTopicNameOfProfiling(), + config.getTopicNameOfTracingSegments(), + config.getTopicNameOfMeters() + )) + .values() + .entrySet() + .stream() + .map(entry -> { + try { + entry.getValue().get(); + return null; + } catch (InterruptedException | ExecutionException e) { + } + return entry.getKey(); + }) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + + if (!missedTopics.isEmpty()) { + log.info("Topics" + missedTopics.toString() + " not exist."); + List newTopicList = missedTopics.stream() + .map(topic -> new NewTopic( + topic, + config.getPartitions(), + (short) config.getReplicationFactor() + )).collect(Collectors.toList()); + + try { + adminClient.createTopics(newTopicList).all().get(); + } catch (Exception e) { + throw new ModuleStartException("Failed to create Kafka Topics" + missedTopics + ".", e); + } + } + + if (config.isSharding() && StringUtil.isNotEmpty(config.getConsumePartitions())) { + isSharding = true; + } else { + isSharding = false; + } + consumer = new KafkaConsumer<>(properties, new StringDeserializer(), new BytesDeserializer()); + } + + public void register(KafkaHandler handler) { + builder.put(handler.getTopic(), handler); + topicPartitions.addAll(handler.getTopicPartitions()); + } + + public void start() { + handlerMap = builder.build(); + if (isSharding) { + consumer.assign(topicPartitions); + } else { + consumer.subscribe(handlerMap.keySet()); + } + consumer.seekToEnd(consumer.assignment()); + Executors.newSingleThreadExecutor(new DefaultThreadFactory("KafkaConsumer")).submit(this); + } + + @Override + public void run() { + while (true) { + ConsumerRecords consumerRecords = consumer.poll(Duration.ofMillis(500L)); + if (!consumerRecords.isEmpty()) { + Iterator> iterator = consumerRecords.iterator(); + while (iterator.hasNext()) { + ConsumerRecord record = iterator.next(); + handlerMap.get(record.topic()).handle(record); + } + consumer.commitAsync(); + } + } + } + +} diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/module/KafkaFetcherConfig.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/module/KafkaFetcherConfig.java new file mode 100644 index 000000000..f5de711cd --- /dev/null +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/module/KafkaFetcherConfig.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.analyzer.agent.kafka.module; + +import java.util.Properties; +import lombok.Data; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; + +@Data +public class KafkaFetcherConfig extends ModuleConfig { + + /** + * Kafka consumer config. + */ + private Properties kafkaConsumerConfig = new Properties(); + + /** + * bootstrap.servers: A list of host/port pairs to use for establishing the initial connection to the Kafka cluster. + * A list of host/port pairs to use for establishing the initial connection to the Kafka cluster. + */ + private String bootstrapServers; + + /** + * group.id: A unique string that identifies the consumer group this consumer belongs to. + */ + private String groupId = "skywalking-consumer"; + + /** + * Which PartitionId(s) of the topics assign to the OAP server. If more than one, is separated by commas. + */ + private String consumePartitions = ""; + + /** + * isSharding was true when OAP Server in cluster. + */ + private boolean isSharding = false; + + /** + * If true, create the Kafka topic when it does not exist. + */ + private boolean createTopicIfNotExist = true; + + /** + * The number of partitions for the topic being created. + */ + private int partitions = 3; + + /** + * The replication factor for each partition in the topic being created. + */ + private int replicationFactor = 2; + + private boolean enableMeterSystem = false; + + private String configPath = "meter-receive-config"; + + private String topicNameOfMetrics = "skywalking-metrics"; + + private String topicNameOfProfiling = "skywalking-profilings"; + + private String topicNameOfTracingSegments = "skywalking-segments"; + + private String topicNameOfManagements = "skywalking-managements"; + + private String topicNameOfMeters = "skywalking-meters"; +} diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/module/KafkaFetcherModule.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/module/KafkaFetcherModule.java new file mode 100644 index 000000000..f456d1b75 --- /dev/null +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/module/KafkaFetcherModule.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.analyzer.agent.kafka.module; + +import org.apache.skywalking.oap.server.library.module.ModuleDefine; + +public class KafkaFetcherModule extends ModuleDefine { + public static final String NAME = "kafka-fetcher"; + + public KafkaFetcherModule() { + super(NAME); + } + + @Override + public Class[] services() { + return new Class[] { }; + } +} diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/KafkaFetcherProvider.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/KafkaFetcherProvider.java new file mode 100644 index 000000000..ef03fef27 --- /dev/null +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/KafkaFetcherProvider.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.analyzer.agent.kafka.provider; + +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.KafkaFetcherHandlerRegister; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.module.KafkaFetcherConfig; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.module.KafkaFetcherModule; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.provider.handler.JVMMetricsHandler; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.provider.handler.MeterServiceHandler; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.provider.handler.ProfileTaskHandler; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.provider.handler.ServiceManagementHandler; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.provider.handler.TraceSegmentHandler; +import org.apache.skywalking.oap.server.analyzer.module.AnalyzerModule; +import org.apache.skywalking.oap.server.analyzer.provider.meter.process.IMeterProcessService; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; +import org.apache.skywalking.oap.server.library.module.ModuleDefine; +import org.apache.skywalking.oap.server.library.module.ModuleProvider; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; + +@Slf4j +public class KafkaFetcherProvider extends ModuleProvider { + private KafkaFetcherHandlerRegister handlerRegister; + private KafkaFetcherConfig config; + + private IMeterProcessService processService; + + public KafkaFetcherProvider() { + config = new KafkaFetcherConfig(); + } + + @Override + public String name() { + return "default"; + } + + @Override + public Class module() { + return KafkaFetcherModule.class; + } + + @Override + public ModuleConfig createConfigBeanIfAbsent() { + return config; + } + + @Override + public void prepare() throws ServiceNotProvidedException, ModuleStartException { + handlerRegister = new KafkaFetcherHandlerRegister(config); + } + + @Override + public void start() throws ServiceNotProvidedException { + handlerRegister.register(new JVMMetricsHandler(getManager(), config)); + handlerRegister.register(new ServiceManagementHandler(getManager(), config)); + handlerRegister.register(new TraceSegmentHandler(getManager(), config)); + handlerRegister.register(new ProfileTaskHandler(getManager(), config)); + + if (config.isEnableMeterSystem()) { + processService = getManager().find(AnalyzerModule.NAME).provider().getService(IMeterProcessService.class); + handlerRegister.register(new MeterServiceHandler(getManager(), config)); + } + handlerRegister.start(); + } + + @Override + public void notifyAfterCompleted() throws ServiceNotProvidedException { + if (config.isEnableMeterSystem()) { + processService.initMeters(); + } + } + + @Override + public String[] requiredModules() { + return new String[] { + AnalyzerModule.NAME, + CoreModule.NAME + }; + } + +} diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/JVMMetricsHandler.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/JVMMetricsHandler.java new file mode 100644 index 000000000..c518dfb4e --- /dev/null +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/JVMMetricsHandler.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.analyzer.agent.kafka.provider.handler; + +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.utils.Bytes; +import org.apache.skywalking.apm.network.language.agent.v3.JVMMetricCollection; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.module.KafkaFetcherConfig; +import org.apache.skywalking.oap.server.analyzer.provider.jvm.JVMSourceDispatcher; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.config.NamingControl; +import org.apache.skywalking.oap.server.library.module.ModuleManager; + +/** + * A handler deserializes the message of JVM Metrics and pushes it to downstream. + */ +@Slf4j +public class JVMMetricsHandler implements KafkaHandler { + + private final NamingControl namingLengthControl; + private final JVMSourceDispatcher jvmSourceDispatcher; + + private final KafkaFetcherConfig config; + + public JVMMetricsHandler(ModuleManager moduleManager, KafkaFetcherConfig config) { + this.jvmSourceDispatcher = new JVMSourceDispatcher(moduleManager); + this.namingLengthControl = moduleManager.find(CoreModule.NAME) + .provider() + .getService(NamingControl.class); + this.config = config; + } + + @Override + public void handle(final ConsumerRecord record) { + try { + JVMMetricCollection metrics = JVMMetricCollection.parseFrom(record.value().get()); + + if (log.isDebugEnabled()) { + log.debug( + "Fetched JVM metrics from service[{}] instance[{}] reported.", + metrics.getService(), + metrics.getServiceInstance() + ); + } + JVMMetricCollection.Builder builder = metrics.toBuilder(); + builder.setService(namingLengthControl.formatServiceName(builder.getService())); + builder.setServiceInstance(namingLengthControl.formatInstanceName(builder.getServiceInstance())); + + builder.getMetricsList().forEach(jvmMetric -> { + jvmSourceDispatcher.sendMetric(builder.getService(), builder.getServiceInstance(), jvmMetric); + }); + } catch (InvalidProtocolBufferException e) { + log.error("", e); + } + } + + @Override + public String getTopic() { + return config.getTopicNameOfMetrics(); + } + + @Override + public String getConsumePartitions() { + return config.getConsumePartitions(); + } +} diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/KafkaHandler.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/KafkaHandler.java new file mode 100644 index 000000000..c0d8c4604 --- /dev/null +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/KafkaHandler.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.analyzer.agent.kafka.provider.handler; + +import com.google.common.collect.Lists; +import java.util.Collections; +import java.util.List; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.utils.Bytes; +import org.apache.skywalking.apm.util.StringUtil; + +/** + * A Handler for dealing Message reported by agent. It is binding to a topic of Kafka, and deserialize. + */ +public interface KafkaHandler { + + /** + * Which partition(s) of the topic is handled in cluster mode. Currently, we have to specify the handler working for + * partition(s). + */ + default List getTopicPartitions() { + if (StringUtil.isEmpty(getConsumePartitions())) { + return Collections.EMPTY_LIST; + } + + List topicPartitions = Lists.newArrayList(); + for (final String partition : getConsumePartitions().trim().split("\\s*,\\s*")) { + topicPartitions.add(new TopicPartition(getTopic(), Integer.parseInt(partition))); + } + return topicPartitions; + } + + /** + * + */ + String getConsumePartitions(); + + /** + * A topic of Kafka is handled. + */ + String getTopic(); + + /** + * Deserialize and push it to downstream. + */ + void handle(ConsumerRecord record); + +} diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/MeterServiceHandler.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/MeterServiceHandler.java new file mode 100644 index 000000000..a68c0c500 --- /dev/null +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/MeterServiceHandler.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.analyzer.agent.kafka.provider.handler; + +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.utils.Bytes; +import org.apache.skywalking.apm.network.language.agent.v3.MeterDataCollection; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.module.KafkaFetcherConfig; +import org.apache.skywalking.oap.server.analyzer.module.AnalyzerModule; +import org.apache.skywalking.oap.server.analyzer.provider.meter.process.IMeterProcessService; +import org.apache.skywalking.oap.server.analyzer.provider.meter.process.MeterProcessor; +import org.apache.skywalking.oap.server.library.module.ModuleManager; + +/** + * A handler deserializes the message of meter system data and pushes it to downstream. + */ +@Slf4j +public class MeterServiceHandler implements KafkaHandler { + private KafkaFetcherConfig config; + private IMeterProcessService processService; + + public MeterServiceHandler(ModuleManager manager, KafkaFetcherConfig config) { + this.config = config; + this.processService = manager.find(AnalyzerModule.NAME).provider().getService(IMeterProcessService.class); + } + + @Override + public void handle(final ConsumerRecord record) { + try { + MeterDataCollection meterDataCollection = MeterDataCollection.parseFrom(record.value().get()); + + MeterProcessor processor = processService.createProcessor(); + meterDataCollection.getMeterDataList().forEach(meterData -> processor.read(meterData)); + processor.process(); + + } catch (InvalidProtocolBufferException e) { + log.error("", e); + } + } + + @Override + public String getTopic() { + return config.getTopicNameOfMeters(); + } + + @Override + public String getConsumePartitions() { + return config.getConsumePartitions(); + } +} diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/ProfileTaskHandler.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/ProfileTaskHandler.java new file mode 100644 index 000000000..7d0746cca --- /dev/null +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/ProfileTaskHandler.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.analyzer.agent.kafka.provider.handler; + +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.utils.Bytes; +import org.apache.skywalking.apm.network.language.profile.v3.ThreadSnapshot; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.module.KafkaFetcherConfig; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.profile.ProfileThreadSnapshotRecord; +import org.apache.skywalking.oap.server.library.module.ModuleManager; + +/** + * A handler deserializes the message of profiling snapshot and pushes it to downstream. + */ +@Slf4j +public class ProfileTaskHandler implements KafkaHandler { + + private final KafkaFetcherConfig config; + + public ProfileTaskHandler(ModuleManager manager, KafkaFetcherConfig config) { + this.config = config; + } + + @Override + public void handle(final ConsumerRecord record) { + try { + ThreadSnapshot snapshot = ThreadSnapshot.parseFrom(record.value().get()); + if (log.isDebugEnabled()) { + log.debug( + "Fetched a thread snapshot[{}] from task[{}] reported", + snapshot.getTraceSegmentId(), + snapshot.getTaskId() + ); + } + + final ProfileThreadSnapshotRecord snapshotRecord = new ProfileThreadSnapshotRecord(); + snapshotRecord.setTaskId(snapshot.getTaskId()); + snapshotRecord.setSegmentId(snapshot.getTraceSegmentId()); + snapshotRecord.setDumpTime(snapshot.getTime()); + snapshotRecord.setSequence(snapshot.getSequence()); + snapshotRecord.setStackBinary(snapshot.getStack().toByteArray()); + snapshotRecord.setTimeBucket(TimeBucket.getRecordTimeBucket(snapshot.getTime())); + + RecordStreamProcessor.getInstance().in(snapshotRecord); + } catch (InvalidProtocolBufferException e) { + log.error(e.getMessage(), e); + } + } + + @Override + public String getTopic() { + return config.getTopicNameOfProfiling(); + } + + @Override + public String getConsumePartitions() { + return config.getConsumePartitions(); + } +} diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/ServiceManagementHandler.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/ServiceManagementHandler.java new file mode 100644 index 000000000..ef613f8ea --- /dev/null +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/ServiceManagementHandler.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.analyzer.agent.kafka.provider.handler; + +import com.google.gson.JsonObject; +import com.google.protobuf.InvalidProtocolBufferException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.utils.Bytes; +import org.apache.skywalking.apm.network.management.v3.InstancePingPkg; +import org.apache.skywalking.apm.network.management.v3.InstanceProperties; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.module.KafkaFetcherConfig; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.analysis.DownSampling; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.NodeType; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.instance.InstanceTraffic; +import org.apache.skywalking.oap.server.core.config.NamingControl; +import org.apache.skywalking.oap.server.core.source.ServiceInstanceUpdate; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.library.module.ModuleManager; + +/** + * A handler deserializes the message of Service Management and pushes it to downstream. + */ +@Slf4j +public class ServiceManagementHandler implements KafkaHandler { + + private final SourceReceiver sourceReceiver; + private final NamingControl namingLengthControl; + + private final KafkaFetcherConfig config; + + public ServiceManagementHandler(ModuleManager moduleManager, KafkaFetcherConfig config) { + this.sourceReceiver = moduleManager.find(CoreModule.NAME).provider().getService(SourceReceiver.class); + this.namingLengthControl = moduleManager.find(CoreModule.NAME) + .provider() + .getService(NamingControl.class); + this.config = config; + } + + @Override + public void handle(final ConsumerRecord record) { + try { + if (record.key().startsWith("register-")) { + serviceReportProperties(InstanceProperties.parseFrom(record.value().get())); + } else { + keepAlive(InstancePingPkg.parseFrom(record.value().get())); + } + } catch (InvalidProtocolBufferException e) { + log.error("", e); + } + } + + private final void serviceReportProperties(InstanceProperties request) { + ServiceInstanceUpdate serviceInstanceUpdate = new ServiceInstanceUpdate(); + final String serviceName = namingLengthControl.formatServiceName(request.getService()); + final String instanceName = namingLengthControl.formatInstanceName(request.getServiceInstance()); + serviceInstanceUpdate.setServiceId(IDManager.ServiceID.buildId(serviceName, NodeType.Normal)); + serviceInstanceUpdate.setName(instanceName); + + if (log.isDebugEnabled()) { + log.debug( + "Service[{}] instance[{}] registered.", + serviceName, + instanceName + ); + } + + JsonObject properties = new JsonObject(); + List ipv4List = new ArrayList<>(); + request.getPropertiesList().forEach(prop -> { + if (InstanceTraffic.PropertyUtil.IPV4.equals(prop.getKey())) { + ipv4List.add(prop.getValue()); + } else { + properties.addProperty(prop.getKey(), prop.getValue()); + } + }); + properties.addProperty(InstanceTraffic.PropertyUtil.IPV4S, ipv4List.stream().collect(Collectors.joining(","))); + serviceInstanceUpdate.setProperties(properties); + serviceInstanceUpdate.setTimeBucket( + TimeBucket.getTimeBucket(System.currentTimeMillis(), DownSampling.Minute)); + sourceReceiver.receive(serviceInstanceUpdate); + } + + private final void keepAlive(InstancePingPkg request) { + final long timeBucket = TimeBucket.getTimeBucket(System.currentTimeMillis(), DownSampling.Minute); + final String serviceName = namingLengthControl.formatServiceName(request.getService()); + final String instanceName = namingLengthControl.formatInstanceName(request.getServiceInstance()); + + if (log.isDebugEnabled()) { + log.debug( + "A ping of Service[{}] instance[{}].", + serviceName, + instanceName + ); + } + + ServiceInstanceUpdate serviceInstanceUpdate = new ServiceInstanceUpdate(); + serviceInstanceUpdate.setServiceId(IDManager.ServiceID.buildId(serviceName, NodeType.Normal)); + serviceInstanceUpdate.setName(instanceName); + serviceInstanceUpdate.setTimeBucket(timeBucket); + sourceReceiver.receive(serviceInstanceUpdate); + } + + @Override + public String getTopic() { + return config.getTopicNameOfManagements(); + } + + @Override + public String getConsumePartitions() { + return config.getConsumePartitions(); + } +} diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/TraceSegmentHandler.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/TraceSegmentHandler.java new file mode 100644 index 000000000..d8bb0ac4b --- /dev/null +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/TraceSegmentHandler.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.analyzer.agent.kafka.provider.handler; + +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.utils.Bytes; +import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.module.KafkaFetcherConfig; +import org.apache.skywalking.oap.server.analyzer.module.AnalyzerModule; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.ISegmentParserService; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.telemetry.TelemetryModule; +import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics; +import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics; +import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator; +import org.apache.skywalking.oap.server.telemetry.api.MetricsTag; + +/** + * A handler deserializes the message of the trace segment data and pushes it to downstream. + */ +@Slf4j +public class TraceSegmentHandler implements KafkaHandler { + + private final KafkaFetcherConfig config; + private final ISegmentParserService segmentParserService; + + private HistogramMetrics histogram; + private CounterMetrics errorCounter; + + public TraceSegmentHandler(ModuleManager moduleManager, + KafkaFetcherConfig config) { + this.config = config; + this.segmentParserService = moduleManager.find(AnalyzerModule.NAME) + .provider() + .getService(ISegmentParserService.class); + + MetricsCreator metricsCreator = moduleManager.find(TelemetryModule.NAME) + .provider().getService(MetricsCreator.class); + + histogram = metricsCreator.createHistogramMetric( + "trace_in_latency", + "The process latency of trace data", + new MetricsTag.Keys("protocol"), + new MetricsTag.Values("kafka-fetcher") + ); + errorCounter = metricsCreator.createCounter( + "trace_analysis_error_count", + "The error number of trace analysis", + new MetricsTag.Keys("protocol"), + new MetricsTag.Values("kafka-fetcher") + ); + } + + @Override + public void handle(final ConsumerRecord record) { + try { + SegmentObject segment = SegmentObject.parseFrom(record.value().get()); + if (log.isDebugEnabled()) { + log.debug( + "Fetched a tracing segment[{}] from service instance[{}].", + segment.getTraceSegmentId(), + segment.getServiceInstance() + ); + } + + HistogramMetrics.Timer timer = histogram.createTimer(); + try { + segmentParserService.send(segment); + } catch (Exception e) { + errorCounter.inc(); + } finally { + timer.finish(); + } + } catch (InvalidProtocolBufferException e) { + log.error(e.getMessage(), e); + } + } + + @Override + public String getTopic() { + return config.getTopicNameOfTracingSegments(); + } + + @Override + public String getConsumePartitions() { + return config.getConsumePartitions(); + } +} diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine new file mode 100644 index 000000000..fb0a9c415 --- /dev/null +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.oap.server.analyzer.agent.kafka.module.KafkaFetcherModule diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider new file mode 100644 index 000000000..501c7dd7c --- /dev/null +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.oap.server.analyzer.agent.kafka.provider.KafkaFetcherProvider \ No newline at end of file diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleManager.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleManager.java new file mode 100644 index 000000000..8d5ee8aa3 --- /dev/null +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleManager.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.analyzer.agent.kafka.mock; + +import com.google.common.collect.Maps; +import java.util.Map; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleNotFoundRuntimeException; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; + +public abstract class MockModuleManager extends ModuleManager { + private final Map moduleProviderHolderMap = Maps.newHashMap(); + + public MockModuleManager() { + init(); + } + + protected abstract void init(); + + protected void register(String name, ModuleProviderHolder provider) { + moduleProviderHolderMap.put(name, provider); + } + + public boolean has(String moduleName) { + return moduleProviderHolderMap.containsKey(moduleName); + } + + public ModuleProviderHolder find(String moduleName) throws ModuleNotFoundRuntimeException { + if (!moduleProviderHolderMap.containsKey(moduleName)) { + throw new ModuleNotFoundRuntimeException("ModuleProviderHolder[" + moduleName + "] cannot found in MOCK."); + } + return moduleProviderHolderMap.get(moduleName); + } +} diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleProvider.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleProvider.java new file mode 100644 index 000000000..9ffd479ae --- /dev/null +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleProvider.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.analyzer.agent.kafka.mock; + +import com.google.common.collect.Maps; +import java.util.Map; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.apache.skywalking.oap.server.library.module.Service; +import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; + +public abstract class MockModuleProvider implements ModuleServiceHolder { + protected Map, Service> serviceMap = Maps.newHashMap(); + + public MockModuleProvider() { + register(); + } + + protected abstract void register(); + + @Override + public void registerServiceImplementation(final Class serviceType, + final Service service) throws ServiceNotProvidedException { + serviceMap.put(serviceType, service); + } + + @Override + public T getService(final Class serviceType) throws ServiceNotProvidedException { + return (T) serviceMap.get(serviceType); + } +} diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/KafkaFetcherProviderTest.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/KafkaFetcherProviderTest.java new file mode 100644 index 000000000..0adc982e8 --- /dev/null +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/KafkaFetcherProviderTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.analyzer.agent.kafka.provider; + +import org.apache.skywalking.oap.server.library.module.ModuleConfig; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.module.KafkaFetcherConfig; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.module.KafkaFetcherModule; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class KafkaFetcherProviderTest { + KafkaFetcherProvider provider = new KafkaFetcherProvider(); + + @Test + public void name() { + assertEquals("default", provider.name()); + } + + @Test + public void module() { + assertEquals(KafkaFetcherModule.class, provider.module()); + } + + @Test + public void createConfigBeanIfAbsent() { + ModuleConfig moduleConfig = provider.createConfigBeanIfAbsent(); + assertTrue(moduleConfig instanceof KafkaFetcherConfig); + } + + @Test + public void requiredModules() { + + } +} diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/JVMMetricsHandlerTest.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/JVMMetricsHandlerTest.java new file mode 100644 index 000000000..4ea6f7380 --- /dev/null +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/JVMMetricsHandlerTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.analyzer.agent.kafka.provider.handler; + +import com.google.common.collect.Lists; +import java.util.List; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.utils.Bytes; +import org.apache.skywalking.apm.network.common.v3.CPU; +import org.apache.skywalking.apm.network.language.agent.v3.GC; +import org.apache.skywalking.apm.network.language.agent.v3.JVMMetric; +import org.apache.skywalking.apm.network.language.agent.v3.JVMMetricCollection; +import org.apache.skywalking.apm.network.language.agent.v3.Memory; +import org.apache.skywalking.apm.network.language.agent.v3.MemoryPool; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.config.NamingControl; +import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping; +import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMCPU; +import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMGC; +import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMMemory; +import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMMemoryPool; +import org.apache.skywalking.oap.server.core.source.Source; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.module.KafkaFetcherConfig; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.mock.MockModuleManager; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.mock.MockModuleProvider; +import org.junit.Assert; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; + +public class JVMMetricsHandlerTest { + private static final String TOPIC_NAME = "skywalking-metrics"; + private JVMMetricsHandler handler = null; + private KafkaFetcherConfig config = new KafkaFetcherConfig(); + + private ModuleManager manager; + + @ClassRule + public static SourceReceiverRule SOURCE_RECEIVER = new SourceReceiverRule() { + + @Override + protected void verify(final List sourceList) throws Throwable { + Assert.assertTrue(sourceList.get(0) instanceof ServiceInstanceJVMCPU); + Assert.assertTrue(sourceList.get(1) instanceof ServiceInstanceJVMMemory); + Assert.assertTrue(sourceList.get(2) instanceof ServiceInstanceJVMMemoryPool); + Assert.assertTrue(sourceList.get(3) instanceof ServiceInstanceJVMGC); + } + }; + + @Before + public void setup() { + manager = new MockModuleManager() { + @Override + protected void init() { + register(CoreModule.NAME, () -> new MockModuleProvider() { + @Override + protected void register() { + registerServiceImplementation(NamingControl.class, new NamingControl( + 512, 512, 512, new EndpointNameGrouping())); + registerServiceImplementation(SourceReceiver.class, SOURCE_RECEIVER); + } + }); + } + }; + handler = new JVMMetricsHandler(manager, config); + } + + @Test + public void testTopicName() { + Assert.assertEquals(handler.getTopic(), TOPIC_NAME); + } + + @Test + public void testHandler() { + long currentTimeMillis = System.currentTimeMillis(); + + JVMMetric.Builder jvmBuilder = JVMMetric.newBuilder(); + jvmBuilder.setTime(currentTimeMillis); + jvmBuilder.setCpu(CPU.newBuilder().setUsagePercent(0.98d).build()); + jvmBuilder.addAllMemory(Lists.newArrayList(Memory.newBuilder().setInit(10).setUsed(100).setIsHeap(false).build())); + jvmBuilder.addAllMemoryPool(Lists.newArrayList(MemoryPool.newBuilder().build())); + jvmBuilder.addAllGc(Lists.newArrayList(GC.newBuilder().build())); + + JVMMetricCollection metrics = JVMMetricCollection.newBuilder() + .setService("service") + .setServiceInstance("service-instance") + .addMetrics(jvmBuilder.build()) + .build(); + + handler.handle(new ConsumerRecord<>(TOPIC_NAME, 0, 0, "", Bytes.wrap(metrics.toByteArray()))); + } +} diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/ServiceManagementHandlerTest.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/ServiceManagementHandlerTest.java new file mode 100644 index 000000000..c23b00457 --- /dev/null +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/ServiceManagementHandlerTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.analyzer.agent.kafka.provider.handler; + +import java.util.List; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.utils.Bytes; +import org.apache.skywalking.apm.network.management.v3.InstancePingPkg; +import org.apache.skywalking.apm.network.management.v3.InstanceProperties; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.config.NamingControl; +import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping; +import org.apache.skywalking.oap.server.core.source.ServiceInstanceUpdate; +import org.apache.skywalking.oap.server.core.source.Source; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.mock.MockModuleManager; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.mock.MockModuleProvider; +import org.apache.skywalking.oap.server.analyzer.agent.kafka.module.KafkaFetcherConfig; +import org.junit.Assert; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; + +public class ServiceManagementHandlerTest { + private static final String TOPIC_NAME = "skywalking-managements"; + + private static final String SERVICE = "MOCK_SERVER"; + private static final String SERVICE_INSTANCE = "MOCK_SERVICE_INSTANCE"; + private KafkaHandler handler = null; + private KafkaFetcherConfig config = new KafkaFetcherConfig(); + + private ModuleManager manager; + + @ClassRule + public static SourceReceiverRule SOURCE_RECEIVER = new SourceReceiverRule() { + + @Override + protected void verify(final List sourceList) throws Throwable { + ServiceInstanceUpdate instanceUpdate = (ServiceInstanceUpdate) sourceList.get(0); + Assert.assertEquals(instanceUpdate.getName(), SERVICE_INSTANCE); + + ServiceInstanceUpdate instanceUpdate1 = (ServiceInstanceUpdate) sourceList.get(1); + Assert.assertEquals(instanceUpdate1.getName(), SERVICE_INSTANCE); + } + }; + + @Before + public void setup() { + manager = new MockModuleManager() { + @Override + protected void init() { + register(CoreModule.NAME, () -> new MockModuleProvider() { + @Override + protected void register() { + registerServiceImplementation(NamingControl.class, new NamingControl( + 512, 512, 512, new EndpointNameGrouping())); + registerServiceImplementation(SourceReceiver.class, SOURCE_RECEIVER); + } + }); + } + }; + handler = new ServiceManagementHandler(manager, config); + } + + @Test + public void testTopicName() { + Assert.assertEquals(handler.getTopic(), TOPIC_NAME); + } + + @Test + public void testHandler() { + InstanceProperties properties = InstanceProperties.newBuilder() + .setService(SERVICE) + .setServiceInstance(SERVICE_INSTANCE) + .build(); + InstancePingPkg ping = InstancePingPkg.newBuilder() + .setService(SERVICE) + .setServiceInstance(SERVICE_INSTANCE) + .build(); + + handler.handle(new ConsumerRecord<>(TOPIC_NAME, 0, 0, "register", Bytes.wrap(properties.toByteArray()))); + handler.handle( + new ConsumerRecord<>(TOPIC_NAME, 0, 0, ping.getServiceInstance(), Bytes.wrap(ping.toByteArray()))); + } +} diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/SourceReceiverRule.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/SourceReceiverRule.java new file mode 100644 index 000000000..aa80b09c4 --- /dev/null +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/SourceReceiverRule.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.analyzer.agent.kafka.provider.handler; + +import com.google.common.collect.Lists; +import java.util.List; +import org.apache.skywalking.oap.server.core.analysis.DispatcherDetectorListener; +import org.apache.skywalking.oap.server.core.source.Source; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.junit.rules.Verifier; + +public abstract class SourceReceiverRule extends Verifier implements SourceReceiver { + private final List sourceList = Lists.newArrayList(); + + @Override + public void receive(final Source source) { + sourceList.add(source); + } + + @Override + public DispatcherDetectorListener getDispatcherDetectorListener() { + return null; + } + + protected void verify() throws Throwable { + verify(sourceList); + } + + protected abstract void verify(List sourceList) throws Throwable; + +} diff --git a/oap-server/server-fetcher-plugin/pom.xml b/oap-server/server-fetcher-plugin/pom.xml index 8fa403697..d488eb9bf 100644 --- a/oap-server/server-fetcher-plugin/pom.xml +++ b/oap-server/server-fetcher-plugin/pom.xml @@ -30,6 +30,7 @@ prometheus-fetcher-plugin + kafka-fetcher-plugin diff --git a/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/pom.xml b/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/pom.xml index 33b5022dd..e87c03a7a 100644 --- a/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/pom.xml +++ b/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/pom.xml @@ -29,6 +29,11 @@ jar + + org.apache.skywalking + agent-analyzer + ${project.version} + org.apache.skywalking skywalking-sharing-server-plugin diff --git a/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/handler/JVMMetricReportServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/handler/JVMMetricReportServiceHandler.java index ebcb5afb8..f39d0057b 100644 --- a/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/handler/JVMMetricReportServiceHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/handler/JVMMetricReportServiceHandler.java @@ -23,6 +23,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.apm.network.common.v3.Commands; import org.apache.skywalking.apm.network.language.agent.v3.JVMMetricCollection; import org.apache.skywalking.apm.network.language.agent.v3.JVMMetricReportServiceGrpc; +import org.apache.skywalking.oap.server.analyzer.provider.jvm.JVMSourceDispatcher; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.config.NamingControl; import org.apache.skywalking.oap.server.library.module.ModuleManager; diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/pom.xml b/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/pom.xml index 60d497633..db732d4d5 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/pom.xml +++ b/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/pom.xml @@ -38,8 +38,9 @@ vavr - org.codehaus.groovy - groovy + org.apache.skywalking + agent-analyzer + ${project.version} diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/MeterReceiverProvider.java b/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/MeterReceiverProvider.java index a4d5af6d1..7c25f7ad3 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/MeterReceiverProvider.java +++ b/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/MeterReceiverProvider.java @@ -18,32 +18,21 @@ package org.apache.skywalking.oap.server.receiver.meter.provider; +import org.apache.skywalking.oap.server.analyzer.module.AnalyzerModule; +import org.apache.skywalking.oap.server.analyzer.provider.meter.process.IMeterProcessService; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.server.GRPCHandlerRegister; import org.apache.skywalking.oap.server.library.module.ModuleConfig; import org.apache.skywalking.oap.server.library.module.ModuleDefine; import org.apache.skywalking.oap.server.library.module.ModuleProvider; -import org.apache.skywalking.oap.server.library.module.ModuleStartException; import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; import org.apache.skywalking.oap.server.receiver.meter.module.MeterReceiverModule; -import org.apache.skywalking.oap.server.receiver.meter.provider.config.MeterConfig; -import org.apache.skywalking.oap.server.receiver.meter.provider.config.MeterConfigs; import org.apache.skywalking.oap.server.receiver.meter.provider.handler.MeterServiceHandler; -import org.apache.skywalking.oap.server.receiver.meter.provider.process.MeterProcessContext; import org.apache.skywalking.oap.server.receiver.sharing.server.SharingServerModule; -import java.util.List; - public class MeterReceiverProvider extends ModuleProvider { - private final MeterReceiverConfig config; - - private List meterConfigs; - private MeterProcessContext processContext; - - public MeterReceiverProvider() { - config = new MeterReceiverConfig(); - } + private IMeterProcessService processService; @Override public String name() { @@ -57,32 +46,34 @@ public class MeterReceiverProvider extends ModuleProvider { @Override public ModuleConfig createConfigBeanIfAbsent() { - return config; + return null; } @Override - public void prepare() throws ServiceNotProvidedException, ModuleStartException { - meterConfigs = MeterConfigs.loadConfig(config.getConfigPath()); + public void prepare() throws ServiceNotProvidedException { } @Override - public void start() throws ServiceNotProvidedException, ModuleStartException { - processContext = new MeterProcessContext(meterConfigs, getManager()); + public void start() throws ServiceNotProvidedException { + processService = getManager().find(AnalyzerModule.NAME) + .provider() + .getService(IMeterProcessService.class); GRPCHandlerRegister grpcHandlerRegister = getManager().find(SharingServerModule.NAME) - .provider() - .getService(GRPCHandlerRegister.class); - grpcHandlerRegister.addHandler(new MeterServiceHandler(processContext)); + .provider() + .getService(GRPCHandlerRegister.class); + grpcHandlerRegister.addHandler(new MeterServiceHandler(processService)); } @Override - public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException { - processContext.initMeters(); + public void notifyAfterCompleted() throws ServiceNotProvidedException { + processService.initMeters(); } @Override public String[] requiredModules() { return new String[] { CoreModule.NAME, + AnalyzerModule.NAME, SharingServerModule.NAME }; } diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/handler/MeterServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/handler/MeterServiceHandler.java index f65927725..c458a143e 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/handler/MeterServiceHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/meter/provider/handler/MeterServiceHandler.java @@ -23,9 +23,9 @@ import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.apm.network.common.v3.Commands; import org.apache.skywalking.apm.network.language.agent.v3.MeterData; import org.apache.skywalking.apm.network.language.agent.v3.MeterReportServiceGrpc; +import org.apache.skywalking.oap.server.analyzer.provider.meter.process.IMeterProcessService; +import org.apache.skywalking.oap.server.analyzer.provider.meter.process.MeterProcessor; import org.apache.skywalking.oap.server.library.server.grpc.GRPCHandler; -import org.apache.skywalking.oap.server.receiver.meter.provider.process.MeterProcessContext; -import org.apache.skywalking.oap.server.receiver.meter.provider.process.MeterProcessor; /** * Meter protocol receiver, collect and process the meters. @@ -33,15 +33,15 @@ import org.apache.skywalking.oap.server.receiver.meter.provider.process.MeterPro @Slf4j public class MeterServiceHandler extends MeterReportServiceGrpc.MeterReportServiceImplBase implements GRPCHandler { - private final MeterProcessContext processContext; + private final IMeterProcessService processService; - public MeterServiceHandler(MeterProcessContext processContext) { - this.processContext = processContext; + public MeterServiceHandler(IMeterProcessService processService) { + this.processService = processService; } @Override public StreamObserver collect(StreamObserver responseObserver) { - final MeterProcessor processor = processContext.createProcessor(); + final MeterProcessor processor = processService.createProcessor(); return new StreamObserver() { @Override public void onNext(MeterData meterData) { diff --git a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/config/MeterConfigsTest.java b/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/config/MeterConfigsTest.java index 08b4a3c13..b4cfb0c47 100644 --- a/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/config/MeterConfigsTest.java +++ b/oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/config/MeterConfigsTest.java @@ -18,9 +18,10 @@ package org.apache.skywalking.oap.server.receiver.meter.provider.config; +import org.apache.skywalking.oap.server.analyzer.provider.meter.config.MeterConfig; +import org.apache.skywalking.oap.server.analyzer.provider.meter.config.MeterConfigs; import org.apache.skywalking.oap.server.core.analysis.meter.ScopeType; import org.apache.skywalking.oap.server.library.module.ModuleStartException; -import org.apache.skywalking.oap.server.receiver.meter.provider.MeterReceiverConfig; import org.junit.Assert; import org.junit.Test; @@ -28,10 +29,11 @@ import java.util.Arrays; import java.util.List; public class MeterConfigsTest { + private static final String CONFIG_PATH = "meter-receive-config"; @Test public void testLoadConfig() throws ModuleStartException { - final List meterConfigs = MeterConfigs.loadConfig(new MeterReceiverConfig().getConfigPath()); + final List meterConfigs = MeterConfigs.loadConfig(CONFIG_PATH); Assert.assertEquals(3, meterConfigs.size()); diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/pom.xml b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/pom.xml index 17952b1b8..15538b5d4 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/pom.xml +++ b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/pom.xml @@ -34,5 +34,10 @@ skywalking-sharing-server-plugin ${project.version} + + org.apache.skywalking + agent-analyzer + ${project.version} + \ No newline at end of file diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/module/TraceModule.java b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/module/TraceModule.java index c03652b01..21df09fe3 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/module/TraceModule.java +++ b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/module/TraceModule.java @@ -19,7 +19,6 @@ package org.apache.skywalking.oap.server.receiver.trace.module; import org.apache.skywalking.oap.server.library.module.ModuleDefine; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.ISegmentParserService; public class TraceModule extends ModuleDefine { public static final String NAME = "receiver-trace"; @@ -30,6 +29,6 @@ public class TraceModule extends ModuleDefine { @Override public Class[] services() { - return new Class[] {ISegmentParserService.class}; + return new Class[] {}; } } diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/TraceModuleProvider.java b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/TraceModuleProvider.java index 071f0a5ae..98b07e1da 100755 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/TraceModuleProvider.java +++ b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/TraceModuleProvider.java @@ -18,43 +18,23 @@ package org.apache.skywalking.oap.server.receiver.trace.provider; -import lombok.Getter; +import org.apache.skywalking.oap.server.analyzer.module.AnalyzerModule; import org.apache.skywalking.oap.server.configuration.api.ConfigurationModule; -import org.apache.skywalking.oap.server.configuration.api.DynamicConfigurationService; import org.apache.skywalking.oap.server.core.CoreModule; -import org.apache.skywalking.oap.server.core.oal.rt.CoreOALDefine; -import org.apache.skywalking.oap.server.core.oal.rt.OALEngineLoaderService; import org.apache.skywalking.oap.server.core.server.GRPCHandlerRegister; import org.apache.skywalking.oap.server.core.server.JettyHandlerRegister; import org.apache.skywalking.oap.server.library.module.ModuleConfig; import org.apache.skywalking.oap.server.library.module.ModuleDefine; import org.apache.skywalking.oap.server.library.module.ModuleProvider; -import org.apache.skywalking.oap.server.library.module.ModuleStartException; import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; import org.apache.skywalking.oap.server.receiver.sharing.server.SharingServerModule; import org.apache.skywalking.oap.server.receiver.trace.module.TraceModule; import org.apache.skywalking.oap.server.receiver.trace.provider.handler.v8.grpc.TraceSegmentReportServiceHandler; import org.apache.skywalking.oap.server.receiver.trace.provider.handler.v8.rest.TraceSegmentReportListServletHandler; import org.apache.skywalking.oap.server.receiver.trace.provider.handler.v8.rest.TraceSegmentReportSingleServletHandler; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.ISegmentParserService; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.SegmentParserListenerManager; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.SegmentParserServiceImpl; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener.MultiScopesAnalysisListener; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener.NetworkAddressAliasMappingListener; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener.SegmentAnalysisListener; import org.apache.skywalking.oap.server.telemetry.TelemetryModule; public class TraceModuleProvider extends ModuleProvider { - @Getter - private final TraceServiceModuleConfig moduleConfig; - private DBLatencyThresholdsAndWatcher thresholds; - private UninstrumentedGatewaysConfig uninstrumentedGatewaysConfig; - private SegmentParserServiceImpl segmentParserService; - private TraceSampleRateWatcher traceSampleRateWatcher; - - public TraceModuleProvider() { - this.moduleConfig = new TraceServiceModuleConfig(); - } @Override public String name() { @@ -68,55 +48,27 @@ public class TraceModuleProvider extends ModuleProvider { @Override public ModuleConfig createConfigBeanIfAbsent() { - return moduleConfig; + return null; } @Override public void prepare() throws ServiceNotProvidedException { - thresholds = new DBLatencyThresholdsAndWatcher(moduleConfig.getSlowDBAccessThreshold(), this); - uninstrumentedGatewaysConfig = new UninstrumentedGatewaysConfig(this); - - traceSampleRateWatcher = new TraceSampleRateWatcher(this); - - moduleConfig.setDbLatencyThresholdsAndWatcher(thresholds); - moduleConfig.setUninstrumentedGatewaysConfig(uninstrumentedGatewaysConfig); - moduleConfig.setTraceSampleRateWatcher(traceSampleRateWatcher); - - segmentParserService = new SegmentParserServiceImpl(getManager(), moduleConfig); - this.registerServiceImplementation(ISegmentParserService.class, segmentParserService); } @Override - public void start() throws ModuleStartException { - // load official analysis - getManager().find(CoreModule.NAME) - .provider() - .getService(OALEngineLoaderService.class) - .load(CoreOALDefine.INSTANCE); - - DynamicConfigurationService dynamicConfigurationService = getManager().find(ConfigurationModule.NAME) - .provider() - .getService( - DynamicConfigurationService.class); + public void start() { GRPCHandlerRegister grpcHandlerRegister = getManager().find(SharingServerModule.NAME) .provider() .getService(GRPCHandlerRegister.class); JettyHandlerRegister jettyHandlerRegister = getManager().find(SharingServerModule.NAME) .provider() .getService(JettyHandlerRegister.class); - dynamicConfigurationService.registerConfigChangeWatcher(thresholds); - dynamicConfigurationService.registerConfigChangeWatcher(uninstrumentedGatewaysConfig); - dynamicConfigurationService.registerConfigChangeWatcher(traceSampleRateWatcher); - segmentParserService.setListenerManager(listenerManager()); - grpcHandlerRegister.addHandler( - new TraceSegmentReportServiceHandler(getManager(), listenerManager(), moduleConfig)); + grpcHandlerRegister.addHandler(new TraceSegmentReportServiceHandler(getManager())); - jettyHandlerRegister.addHandler( - new TraceSegmentReportListServletHandler(getManager(), listenerManager(), moduleConfig)); - jettyHandlerRegister.addHandler( - new TraceSegmentReportSingleServletHandler(getManager(), listenerManager(), moduleConfig)); + jettyHandlerRegister.addHandler(new TraceSegmentReportListServletHandler(getManager())); + jettyHandlerRegister.addHandler(new TraceSegmentReportSingleServletHandler(getManager())); } @Override @@ -129,19 +81,10 @@ public class TraceModuleProvider extends ModuleProvider { return new String[] { TelemetryModule.NAME, CoreModule.NAME, + AnalyzerModule.NAME, SharingServerModule.NAME, ConfigurationModule.NAME }; } - private SegmentParserListenerManager listenerManager() { - SegmentParserListenerManager listenerManager = new SegmentParserListenerManager(); - if (moduleConfig.isTraceAnalysis()) { - listenerManager.add(new MultiScopesAnalysisListener.Factory(getManager())); - listenerManager.add(new NetworkAddressAliasMappingListener.Factory(getManager())); - } - listenerManager.add(new SegmentAnalysisListener.Factory(getManager(), moduleConfig)); - - return listenerManager; - } } diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/grpc/TraceSegmentReportServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/grpc/TraceSegmentReportServiceHandler.java index 8e5df3f73..b9bbc4ef6 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/grpc/TraceSegmentReportServiceHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/grpc/TraceSegmentReportServiceHandler.java @@ -23,11 +23,10 @@ import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.apm.network.common.v3.Commands; import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; import org.apache.skywalking.apm.network.language.agent.v3.TraceSegmentReportServiceGrpc; +import org.apache.skywalking.oap.server.analyzer.module.AnalyzerModule; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.ISegmentParserService; import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.library.server.grpc.GRPCHandler; -import org.apache.skywalking.oap.server.receiver.trace.provider.TraceServiceModuleConfig; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.SegmentParserListenerManager; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.TraceAnalyzer; import org.apache.skywalking.oap.server.telemetry.TelemetryModule; import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics; import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics; @@ -37,17 +36,17 @@ import org.apache.skywalking.oap.server.telemetry.api.MetricsTag; @Slf4j public class TraceSegmentReportServiceHandler extends TraceSegmentReportServiceGrpc.TraceSegmentReportServiceImplBase implements GRPCHandler { private final ModuleManager moduleManager; - private final SegmentParserListenerManager listenerManager; - private final TraceServiceModuleConfig config; private HistogramMetrics histogram; private CounterMetrics errorCounter; - public TraceSegmentReportServiceHandler(ModuleManager moduleManager, - SegmentParserListenerManager listenerManager, - TraceServiceModuleConfig config) { + private ISegmentParserService segmentParserService; + + public TraceSegmentReportServiceHandler(ModuleManager moduleManager) { this.moduleManager = moduleManager; - this.listenerManager = listenerManager; - this.config = config; + this.segmentParserService = moduleManager.find(AnalyzerModule.NAME) + .provider() + .getService(ISegmentParserService.class); + MetricsCreator metricsCreator = moduleManager.find(TelemetryModule.NAME) .provider() .getService(MetricsCreator.class); @@ -71,9 +70,7 @@ public class TraceSegmentReportServiceHandler extends TraceSegmentReportServiceG HistogramMetrics.Timer timer = histogram.createTimer(); try { - final TraceAnalyzer traceAnalyzer = new TraceAnalyzer( - moduleManager, listenerManager, config); - traceAnalyzer.doAnalysis(segment); + segmentParserService.send(segment); } catch (Exception e) { errorCounter.inc(); } finally { diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/rest/TraceSegmentReportBaseServletHandler.java b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/rest/TraceSegmentReportBaseServletHandler.java index 63cfb4138..c822a7ba8 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/rest/TraceSegmentReportBaseServletHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/rest/TraceSegmentReportBaseServletHandler.java @@ -24,11 +24,10 @@ import java.util.List; import javax.servlet.http.HttpServletRequest; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; +import org.apache.skywalking.oap.server.analyzer.module.AnalyzerModule; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.ISegmentParserService; import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.library.server.jetty.JettyJsonHandler; -import org.apache.skywalking.oap.server.receiver.trace.provider.TraceServiceModuleConfig; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.SegmentParserListenerManager; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.TraceAnalyzer; import org.apache.skywalking.oap.server.telemetry.TelemetryModule; import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics; import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics; @@ -39,17 +38,15 @@ import org.apache.skywalking.oap.server.telemetry.api.MetricsTag; public abstract class TraceSegmentReportBaseServletHandler extends JettyJsonHandler { private final ModuleManager moduleManager; - private final SegmentParserListenerManager listenerManager; - private final TraceServiceModuleConfig config; + private final ISegmentParserService segmentParserService; private HistogramMetrics histogram; private CounterMetrics errorCounter; - public TraceSegmentReportBaseServletHandler(ModuleManager moduleManager, - SegmentParserListenerManager listenerManager, - TraceServiceModuleConfig config) { + public TraceSegmentReportBaseServletHandler(ModuleManager moduleManager) { this.moduleManager = moduleManager; - this.listenerManager = listenerManager; - this.config = config; + this.segmentParserService = moduleManager.find(AnalyzerModule.NAME) + .provider() + .getService(ISegmentParserService.class); MetricsCreator metricsCreator = moduleManager.find(TelemetryModule.NAME) .provider() .getService(MetricsCreator.class); @@ -78,8 +75,7 @@ public abstract class TraceSegmentReportBaseServletHandler extends JettyJsonHand final List segments = parseSegments(req); for (SegmentObject segment : segments) { - final TraceAnalyzer traceAnalyzer = new TraceAnalyzer(moduleManager, listenerManager, config); - traceAnalyzer.doAnalysis(segment); + segmentParserService.send(segment); } } catch (Exception e) { errorCounter.inc(); diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/rest/TraceSegmentReportListServletHandler.java b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/rest/TraceSegmentReportListServletHandler.java index 64374d599..b07df59dd 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/rest/TraceSegmentReportListServletHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/rest/TraceSegmentReportListServletHandler.java @@ -21,18 +21,15 @@ package org.apache.skywalking.oap.server.receiver.trace.provider.handler.v8.rest import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonElement; -import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; -import org.apache.skywalking.oap.server.library.module.ModuleManager; -import org.apache.skywalking.oap.server.library.util.ProtoBufJsonUtils; -import org.apache.skywalking.oap.server.receiver.trace.provider.TraceServiceModuleConfig; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.SegmentParserListenerManager; - -import javax.servlet.http.HttpServletRequest; import java.io.BufferedReader; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import javax.servlet.http.HttpServletRequest; +import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.util.ProtoBufJsonUtils; /** * Parsing segments from list json @@ -41,8 +38,8 @@ public class TraceSegmentReportListServletHandler extends TraceSegmentReportBase private final Gson gson = new Gson(); - public TraceSegmentReportListServletHandler(ModuleManager moduleManager, SegmentParserListenerManager listenerManager, TraceServiceModuleConfig config) { - super(moduleManager, listenerManager, config); + public TraceSegmentReportListServletHandler(ModuleManager moduleManager) { + super(moduleManager); } @Override diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/rest/TraceSegmentReportSingleServletHandler.java b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/rest/TraceSegmentReportSingleServletHandler.java index 20d171ddf..b7bab98a6 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/rest/TraceSegmentReportSingleServletHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/rest/TraceSegmentReportSingleServletHandler.java @@ -21,8 +21,6 @@ package org.apache.skywalking.oap.server.receiver.trace.provider.handler.v8.rest import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.library.util.ProtoBufJsonUtils; -import org.apache.skywalking.oap.server.receiver.trace.provider.TraceServiceModuleConfig; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.SegmentParserListenerManager; import javax.servlet.http.HttpServletRequest; import java.io.BufferedReader; @@ -35,8 +33,8 @@ import java.util.List; */ public class TraceSegmentReportSingleServletHandler extends TraceSegmentReportBaseServletHandler { - public TraceSegmentReportSingleServletHandler(ModuleManager moduleManager, SegmentParserListenerManager listenerManager, TraceServiceModuleConfig config) { - super(moduleManager, listenerManager, config); + public TraceSegmentReportSingleServletHandler(ModuleManager moduleManager) { + super(moduleManager); } @Override diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/rest/TraceSegmentReportServletHandlerTest.java b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/rest/TraceSegmentReportServletHandlerTest.java index 15498415c..b47efc2cc 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/rest/TraceSegmentReportServletHandlerTest.java +++ b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/rest/TraceSegmentReportServletHandlerTest.java @@ -18,7 +18,17 @@ package org.apache.skywalking.oap.server.receiver.trace.provider.handler.v8.rest; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; +import java.util.List; +import javax.servlet.http.HttpServletRequest; import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; +import org.apache.skywalking.oap.server.analyzer.module.AnalyzerModule; +import org.apache.skywalking.oap.server.analyzer.provider.AnalyzerModuleConfig; +import org.apache.skywalking.oap.server.analyzer.provider.AnalyzerModuleProvider; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.ISegmentParserService; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.SegmentParserServiceImpl; import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.telemetry.TelemetryModule; import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator; @@ -34,12 +44,6 @@ import org.mockito.internal.util.reflection.Whitebox; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.modules.junit4.PowerMockRunner; -import javax.servlet.http.HttpServletRequest; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.StringReader; -import java.util.List; - import static org.mockito.Mockito.when; @RunWith(PowerMockRunner.class) @@ -53,14 +57,24 @@ public class TraceSegmentReportServletHandlerTest { @Mock private NoneTelemetryProvider telemetryProvider; + @Mock + private AnalyzerModuleProvider analyzerModuleProvider; + @Before public void init() throws IOException { TelemetryModule telemetryModule = Mockito.spy(TelemetryModule.class); Whitebox.setInternalState(telemetryModule, "loadedProvider", telemetryProvider); Mockito.when(moduleManager.find(TelemetryModule.NAME)).thenReturn(telemetryModule); + AnalyzerModule analyzerModule = Mockito.spy(AnalyzerModule.class); + Whitebox.setInternalState(analyzerModule, "loadedProvider", analyzerModuleProvider); + Mockito.when(moduleManager.find(AnalyzerModule.NAME)).thenReturn(analyzerModule); + Mockito.when(telemetryProvider.getService(MetricsCreator.class)) .thenReturn(new MetricsCreatorNoop()); + + when(analyzerModuleProvider.getService(ISegmentParserService.class)) + .thenReturn(new SegmentParserServiceImpl(moduleManager, new AnalyzerModuleConfig())); } @Test @@ -73,7 +87,7 @@ public class TraceSegmentReportServletHandlerTest { "}"; final TraceSegmentReportSingleServletHandler singleServletHandler = - new TraceSegmentReportSingleServletHandler(moduleManager, null, null); + new TraceSegmentReportSingleServletHandler(moduleManager); when(request.getReader()).thenReturn(new BufferedReader(new StringReader(singleJson))); final List segmentObjects = singleServletHandler.parseSegments(request); @@ -95,7 +109,7 @@ public class TraceSegmentReportServletHandlerTest { "}]"; final TraceSegmentReportListServletHandler singleServletHandler = - new TraceSegmentReportListServletHandler(moduleManager, null, null); + new TraceSegmentReportListServletHandler(moduleManager); when(request.getReader()).thenReturn(new BufferedReader(new StringReader(listJson))); final List segmentObjects = singleServletHandler.parseSegments(request); diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/MultiScopesAnalysisListenerTest.java b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/MultiScopesAnalysisListenerTest.java index b715e7c55..7cdd71d3d 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/MultiScopesAnalysisListenerTest.java +++ b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/MultiScopesAnalysisListenerTest.java @@ -27,6 +27,11 @@ import org.apache.skywalking.apm.network.language.agent.v3.SegmentReference; import org.apache.skywalking.apm.network.language.agent.v3.SpanLayer; import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; import org.apache.skywalking.apm.network.language.agent.v3.SpanType; +import org.apache.skywalking.oap.server.analyzer.provider.AnalyzerModuleConfig; +import org.apache.skywalking.oap.server.analyzer.provider.trace.UninstrumentedGatewaysConfig; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.SpanTags; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.AnalysisListener; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.MultiScopesAnalysisListener; import org.apache.skywalking.oap.server.core.Const; import org.apache.skywalking.oap.server.core.analysis.IDManager; import org.apache.skywalking.oap.server.core.analysis.NodeType; @@ -44,9 +49,6 @@ import org.apache.skywalking.oap.server.core.source.ServiceInstanceRelation; import org.apache.skywalking.oap.server.core.source.ServiceMeta; import org.apache.skywalking.oap.server.core.source.ServiceRelation; import org.apache.skywalking.oap.server.core.source.Source; -import org.apache.skywalking.oap.server.receiver.trace.provider.TraceServiceModuleConfig; -import org.apache.skywalking.oap.server.receiver.trace.provider.UninstrumentedGatewaysConfig; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.SpanTags; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -54,7 +56,7 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; -import static org.apache.skywalking.oap.server.receiver.trace.provider.parser.SpanTags.LOGIC_ENDPOINT; +import static org.apache.skywalking.oap.server.analyzer.provider.trace.parser.SpanTags.LOGIC_ENDPOINT; import static org.mockito.Matchers.any; import static org.mockito.Mockito.when; @@ -66,7 +68,7 @@ import static org.mockito.Mockito.when; */ public class MultiScopesAnalysisListenerTest { @Mock - private static TraceServiceModuleConfig CONFIG; + private static AnalyzerModuleConfig CONFIG; @Mock private static NetworkAddressAliasCache CACHE; @Mock diff --git a/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/ZipkinReceiverProvider.java b/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/ZipkinReceiverProvider.java index bfdafe68a..9e8269b33 100644 --- a/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/ZipkinReceiverProvider.java +++ b/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/ZipkinReceiverProvider.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.receiver.zipkin; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.ISegmentParserService; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.library.module.ModuleConfig; import org.apache.skywalking.oap.server.library.module.ModuleDefine; @@ -28,7 +29,6 @@ import org.apache.skywalking.oap.server.library.server.ServerException; import org.apache.skywalking.oap.server.library.server.jetty.JettyServer; import org.apache.skywalking.oap.server.library.server.jetty.JettyServerConfig; import org.apache.skywalking.oap.server.receiver.trace.module.TraceModule; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.ISegmentParserService; import org.apache.skywalking.oap.server.receiver.zipkin.analysis.Receiver2AnalysisBridge; import org.apache.skywalking.oap.server.receiver.zipkin.analysis.transform.Zipkin2SkyWalkingTransfer; import org.apache.skywalking.oap.server.receiver.zipkin.handler.SpanV1JettyHandler; diff --git a/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/analysis/Receiver2AnalysisBridge.java b/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/analysis/Receiver2AnalysisBridge.java index 41b43ba55..e9300bab4 100644 --- a/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/analysis/Receiver2AnalysisBridge.java +++ b/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/analysis/Receiver2AnalysisBridge.java @@ -18,7 +18,7 @@ package org.apache.skywalking.oap.server.receiver.zipkin.analysis; -import org.apache.skywalking.oap.server.receiver.trace.provider.parser.ISegmentParserService; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.ISegmentParserService; import org.apache.skywalking.oap.server.receiver.zipkin.analysis.data.SkyWalkingTrace; import org.apache.skywalking.oap.server.receiver.zipkin.analysis.transform.SegmentListener; diff --git a/test/e2e/e2e-test/docker/kafka/Dockerfile.provider b/test/e2e/e2e-test/docker/kafka/Dockerfile.provider new file mode 100644 index 000000000..bdd89a02d --- /dev/null +++ b/test/e2e/e2e-test/docker/kafka/Dockerfile.provider @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG SW_AGENT_JDK_VERSION=8 +ARG AGENT_JDK_BASE=adoptopenjdk/openjdk${SW_AGENT_JDK_VERSION}:alpine + +FROM ${AGENT_JDK_BASE} + +WORKDIR /skywalking + +ARG DIST_PACKAGE=dist/apache-skywalking-apm-bin.tar.gz +ADD ${DIST_PACKAGE} . +RUN tar -zxf apache-skywalking-apm*.gz --strip=1 || true +RUN mv apache-skywalking-apm-bin/* . || true + +# activate kafka-reporter plugin +RUN mv agent/optional-reporter-plugins/kafka-reporter-plugin-*.jar agent/plugins/ || true + +VOLUME /services + +ADD e2e-service-provider/target/e2e-service-provider-1.0.0.jar /services/ + +ENV JAVA_OPTS="" + +CMD ["sh", "-c", "java $JAVA_OPTS -jar /services/e2e-service-provider-1.0.0.jar"] diff --git a/test/e2e/e2e-test/docker/kafka/docker-compose.base.yml b/test/e2e/e2e-test/docker/kafka/docker-compose.base.yml new file mode 100644 index 000000000..2b8652b13 --- /dev/null +++ b/test/e2e/e2e-test/docker/kafka/docker-compose.base.yml @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + oap: + extends: + file: ../base-compose.yml + service: oap + environment: + SW_PROMETHEUS_FETCHER_ACTIVE: "true" + SW_TELEMETRY: prometheus + SW_KAFKA_FETCHER: default + SW_KAFKA_FETCHER_SERVERS: broker-a:9092,broker-b:9092 + SW_KAFKA_FETCHER_PARTITIONS: 2 + SW_KAFKA_FETCHER_PARTITIONS_FACTOR: 1 + depends_on: + broker-a: + condition: service_healthy + broker-b: + condition: service_healthy + + + provider_kafka: + build: + context: ../../../ + dockerfile: e2e-test/docker/kafka/Dockerfile.provider + args: + - SW_AGENT_JDK_VERSION=${SW_AGENT_JDK_VERSION} + networks: + - e2e + expose: + - 9090 + volumes: + - ../../../../jacoco:/jacoco + environment: + SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11800 + SW_KAFKA_BOOTSTRAP_SERVERS: broker-a:9092,broker-b:9092 + JAVA_OPTS: >- + -javaagent:/jacoco/jacocoagent.jar=classdumpdir=/jacoco/classes/provider,destfile=/jacoco/provider.exec,includes=org.apache.skywalking.*,excludes=org.apache.skywalking.apm.dependencies.* + -javaagent:/skywalking/agent/skywalking-agent.jar=logging.output=CONSOLE + healthcheck: + test: ["CMD", "sh", "-c", "nc -nz 127.0.0.1 9090"] + interval: 5s + timeout: 60s + retries: 120 + depends_on: + oap: + condition: service_healthy + broker-a: + condition: service_healthy + broker-b: + condition: service_healthy + +networks: + e2e: diff --git a/test/e2e/e2e-test/docker/kafka/docker-compose.meter.yml b/test/e2e/e2e-test/docker/kafka/docker-compose.meter.yml new file mode 100644 index 000000000..da4577fbe --- /dev/null +++ b/test/e2e/e2e-test/docker/kafka/docker-compose.meter.yml @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + oap: + extends: + file: ../base-compose.yml + service: oap + environment: + SW_RECEIVER_METER: default + SW_KAFKA_FETCHER: default + SW_KAFKA_FETCHER_SERVERS: broker-a:9092,broker-b:9092 + SW_KAFKA_FETCHER_PARTITIONS: 2 + SW_KAFKA_FETCHER_PARTITIONS_FACTOR: 1 + SW_KAFKA_FETCHER_ENABLE_METER_SYSTEM: "true" + depends_on: + broker-a: + condition: service_healthy + broker-b: + condition: service_healthy + + provider_kafka: + build: + context: ../../../ + dockerfile: e2e-test/docker/kafka/Dockerfile.provider + args: + - SW_AGENT_JDK_VERSION=${SW_AGENT_JDK_VERSION} + networks: + - e2e + expose: + - 9090 + volumes: + - ../../../../jacoco:/jacoco + environment: + SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11800 + SW_KAFKA_BOOTSTRAP_SERVERS: broker-a:9092,broker-b:9092 + JAVA_OPTS: >- + -javaagent:/jacoco/jacocoagent.jar=classdumpdir=/jacoco/classes/provider,destfile=/jacoco/provider.exec,includes=org.apache.skywalking.*,excludes=org.apache.skywalking.apm.dependencies.* + -javaagent:/skywalking/agent/skywalking-agent.jar=logging.output=CONSOLE + healthcheck: + test: ["CMD", "sh", "-c", "nc -nz 127.0.0.1 9090"] + interval: 5s + timeout: 60s + retries: 120 + depends_on: + oap: + condition: service_healthy + broker-a: + condition: service_healthy + broker-b: + condition: service_healthy + +networks: + e2e: diff --git a/test/e2e/e2e-test/docker/kafka/docker-compose.profiling.yml b/test/e2e/e2e-test/docker/kafka/docker-compose.profiling.yml new file mode 100644 index 000000000..4b232edc0 --- /dev/null +++ b/test/e2e/e2e-test/docker/kafka/docker-compose.profiling.yml @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + influxdb: + image: influxdb:1.7.9 + expose: + - 8086 + networks: + - e2e + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/8086"] + interval: 5s + timeout: 60s + retries: 120 + + oap: + extends: + file: ../base-compose.yml + service: oap + environment: + SW_STORAGE: ${SW_STORAGE} + SW_KAFKA_FETCHER: default + SW_KAFKA_FETCHER_SERVERS: broker-a:9092,broker-b:9092 + SW_KAFKA_FETCHER_PARTITIONS: 2 + SW_KAFKA_FETCHER_PARTITIONS_FACTOR: 1 + depends_on: + influxdb: + condition: service_healthy + broker-a: + condition: service_healthy + broker-b: + condition: service_healthy + + provider_kafka: + build: + context: ../../../ + dockerfile: e2e-test/docker/kafka/Dockerfile.provider + args: + - SW_AGENT_JDK_VERSION=${SW_AGENT_JDK_VERSION} + networks: + - e2e + expose: + - 9090 + volumes: + - ../../../../jacoco:/jacoco + environment: + SW_AGENT_NAME: e2e-profile-service + SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11800 + SW_KAFKA_BOOTSTRAP_SERVERS: broker-a:9092,broker-b:9092 + JAVA_OPTS: >- + -javaagent:/jacoco/jacocoagent.jar=classdumpdir=/jacoco/classes/provider,destfile=/jacoco/provider.exec,includes=org.apache.skywalking.*,excludes=org.apache.skywalking.apm.dependencies.* + -javaagent:/skywalking/agent/skywalking-agent.jar=logging.output=CONSOLE + healthcheck: + test: ["CMD", "sh", "-c", "nc -nz 127.0.0.1 9090"] + interval: 5s + timeout: 60s + retries: 120 + depends_on: + oap: + condition: service_healthy + broker-a: + condition: service_healthy + broker-b: + condition: service_healthy + +networks: + e2e: diff --git a/test/e2e/e2e-test/docker/kafka/docker-compose.yml b/test/e2e/e2e-test/docker/kafka/docker-compose.yml new file mode 100644 index 000000000..ec5d89eae --- /dev/null +++ b/test/e2e/e2e-test/docker/kafka/docker-compose.yml @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + ui: + extends: + file: ../base-compose.yml + service: ui + depends_on: + oap: + condition: service_healthy + + zookeeper: + image: zookeeper:3.4 + hostname: zookeeper + expose: + - 2181 + networks: + - e2e + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + healthcheck: + test: ["CMD", "sh", "-c", "nc -nz 127.0.0.1 2181"] + interval: 5s + timeout: 60s + retries: 120 + + broker-a: + image: bitnami/kafka:2.4.1 + hostname: broker-a + expose: + - 9092 + networks: + - e2e + environment: + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_BROKER_ID=10 + - ALLOW_PLAINTEXT_LISTENER=yes + depends_on: + zookeeper: + condition: service_healthy + healthcheck: + test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"] + interval: 5s + timeout: 60s + retries: 120 + + broker-b: + image: bitnami/kafka:2.4.1 + hostname: broker-b + expose: + - 9092 + networks: + - e2e + environment: + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_BROKER_ID=24 + - ALLOW_PLAINTEXT_LISTENER=yes + depends_on: + zookeeper: + condition: service_healthy + healthcheck: + test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"] + interval: 5s + timeout: 60s + retries: 120 + +networks: + e2e: diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaE2E.java new file mode 100644 index 000000000..7a0283a14 --- /dev/null +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaE2E.java @@ -0,0 +1,355 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.e2e.kafka; + +import java.util.List; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.e2e.annotation.ContainerHostAndPort; +import org.apache.skywalking.e2e.annotation.DockerCompose; +import org.apache.skywalking.e2e.base.SkyWalkingE2E; +import org.apache.skywalking.e2e.base.SkyWalkingTestAdapter; +import org.apache.skywalking.e2e.common.HostAndPort; +import org.apache.skywalking.e2e.metrics.AtLeastOneOfMetricsMatcher; +import org.apache.skywalking.e2e.metrics.Metrics; +import org.apache.skywalking.e2e.metrics.MetricsQuery; +import org.apache.skywalking.e2e.metrics.MetricsValueMatcher; +import org.apache.skywalking.e2e.metrics.ReadMetrics; +import org.apache.skywalking.e2e.metrics.ReadMetricsQuery; +import org.apache.skywalking.e2e.retryable.RetryableTest; +import org.apache.skywalking.e2e.service.Service; +import org.apache.skywalking.e2e.service.ServicesMatcher; +import org.apache.skywalking.e2e.service.ServicesQuery; +import org.apache.skywalking.e2e.service.endpoint.Endpoint; +import org.apache.skywalking.e2e.service.endpoint.EndpointQuery; +import org.apache.skywalking.e2e.service.endpoint.Endpoints; +import org.apache.skywalking.e2e.service.endpoint.EndpointsMatcher; +import org.apache.skywalking.e2e.service.instance.Instance; +import org.apache.skywalking.e2e.service.instance.Instances; +import org.apache.skywalking.e2e.service.instance.InstancesMatcher; +import org.apache.skywalking.e2e.service.instance.InstancesQuery; +import org.apache.skywalking.e2e.topo.Call; +import org.apache.skywalking.e2e.topo.ServiceInstanceTopology; +import org.apache.skywalking.e2e.topo.ServiceInstanceTopologyMatcher; +import org.apache.skywalking.e2e.topo.ServiceInstanceTopologyQuery; +import org.apache.skywalking.e2e.topo.TopoMatcher; +import org.apache.skywalking.e2e.topo.TopoQuery; +import org.apache.skywalking.e2e.topo.Topology; +import org.apache.skywalking.e2e.trace.Trace; +import org.apache.skywalking.e2e.trace.TracesMatcher; +import org.apache.skywalking.e2e.trace.TracesQuery; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.containers.DockerComposeContainer; + +import static org.apache.skywalking.e2e.metrics.MetricsMatcher.verifyMetrics; +import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_ENDPOINT_METRICS; +import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_INSTANCE_JVM_METRICS; +import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_INSTANCE_METRICS; +import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_INSTANCE_RELATION_CLIENT_METRICS; +import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_INSTANCE_RELATION_SERVER_METRICS; +import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_METRICS; +import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_RELATION_CLIENT_METRICS; +import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_RELATION_SERVER_METRICS; +import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SO11Y_LABELED_METRICS; +import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SO11Y_LINER_METRICS; +import static org.apache.skywalking.e2e.utils.Times.now; +import static org.apache.skywalking.e2e.utils.Yamls.load; + +@Slf4j +@SkyWalkingE2E +public class KafkaE2E extends SkyWalkingTestAdapter { + + @SuppressWarnings("unused") + @DockerCompose({ + "docker/kafka/docker-compose.yml", + "docker/kafka/docker-compose.base.yml" + }) + protected DockerComposeContainer compose; + + @SuppressWarnings("unused") + @ContainerHostAndPort(name = "provider_kafka", port = 9090) + protected HostAndPort serviceHostPort; + + @SuppressWarnings("unused") + @ContainerHostAndPort(name = "ui", port = 8080) + protected HostAndPort swWebappHostPort; + + @BeforeAll + void setUp() throws Exception { + queryClient(swWebappHostPort); + + trafficController(serviceHostPort, "/users"); + } + + @AfterAll + public void tearDown() { + trafficController.stop(); + } + + @RetryableTest + void services() throws Exception { + List services = graphql.services(new ServicesQuery().start(startTime).end(now())); + + services = services.stream().filter(s -> !s.getLabel().equals("oap-server")).collect(Collectors.toList()); + LOGGER.info("services: {}", services); + + load("expected/simple/services.yml").as(ServicesMatcher.class).verify(services); + + for (final Service service : services) { + LOGGER.info("verifying service instances: {}", service); + + verifyServiceMetrics(service); + + final Instances instances = verifyServiceInstances(service); + + verifyInstancesMetrics(instances); + + verifyInstancesJVMMetrics(instances); + + final Endpoints endpoints = verifyServiceEndpoints(service); + + verifyEndpointsMetrics(endpoints); + } + } + + @RetryableTest + void traces() throws Exception { + final List traces = graphql.traces(new TracesQuery().start(startTime).end(now()).orderByDuration()); + + LOGGER.info("traces: {}", traces); + + load("expected/simple/traces.yml").as(TracesMatcher.class).verifyLoosely(traces); + } + + @RetryableTest + void topology() throws Exception { + final Topology topology = graphql.topo(new TopoQuery().stepByMinute().start(startTime.minusDays(1)).end(now())); + + LOGGER.info("topology: {}", topology); + + load("expected/simple/topo.yml").as(TopoMatcher.class).verify(topology); + + verifyServiceRelationMetrics(topology.getCalls()); + } + + @RetryableTest + void serviceInstances() throws Exception { + final ServiceInstanceTopology topology = graphql.serviceInstanceTopo( + new ServiceInstanceTopologyQuery().stepByMinute() + .start(startTime.minusDays(1)) + .end(now()) + .clientServiceId("VXNlcg==.0") + .serverServiceId("WW91cl9BcHBsaWNhdGlvbk5hbWU=.1")); + + LOGGER.info("instance topology: {}", topology); + + load("expected/simple/serviceInstanceTopo.yml").as(ServiceInstanceTopologyMatcher.class).verify(topology); + + verifyServiceInstanceRelationMetrics(topology.getCalls()); + } + + @RetryableTest + void so11y() throws Exception { + List services = graphql.services(new ServicesQuery().start(startTime).end(now())); + + services = services.stream().filter(s -> s.getLabel().equals("oap-server")).collect(Collectors.toList()); + LOGGER.info("services: {}", services); + load("expected/simple/so11y-services.yml").as(ServicesMatcher.class).verify(services); + for (final Service service : services) { + final Instances instances = graphql.instances( + new InstancesQuery().serviceId(service.getKey()).start(startTime).end(now()) + ); + + LOGGER.info("instances: {}", instances); + + load("expected/simple/so11y-instances.yml").as(InstancesMatcher.class).verify(instances); + for (Instance instance : instances.getInstances()) { + for (String metricsName : ALL_SO11Y_LINER_METRICS) { + LOGGER.info("verifying service instance response time: {}", instance); + final ReadMetrics instanceMetrics = graphql.readMetrics( + new ReadMetricsQuery().stepByMinute().metricsName(metricsName) + .serviceName(service.getLabel()).instanceName(instance.getLabel()) + ); + + LOGGER.info("{}: {}", metricsName, instanceMetrics); + final AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher(); + final MetricsValueMatcher greaterThanZero = new MetricsValueMatcher(); + greaterThanZero.setValue("gt 0"); + instanceRespTimeMatcher.setValue(greaterThanZero); + instanceRespTimeMatcher.verify(instanceMetrics.getValues()); + } + for (String metricsName : ALL_SO11Y_LABELED_METRICS) { + LOGGER.info("verifying service instance response time: {}", instance); + final List instanceMetrics = graphql.readLabeledMetrics( + new ReadMetricsQuery().stepByMinute().metricsName(metricsName) + .serviceName(service.getLabel()).instanceName(instance.getLabel()) + ); + + LOGGER.info("{}: {}", metricsName, instanceMetrics); + Metrics allValues = new Metrics(); + for (ReadMetrics readMetrics : instanceMetrics) { + allValues.getValues().addAll(readMetrics.getValues().getValues()); + } + final AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher(); + final MetricsValueMatcher greaterThanZero = new MetricsValueMatcher(); + greaterThanZero.setValue("gt 0"); + instanceRespTimeMatcher.setValue(greaterThanZero); + instanceRespTimeMatcher.verify(allValues); + } + } + } + } + + private Instances verifyServiceInstances(final Service service) throws Exception { + final Instances instances = graphql.instances( + new InstancesQuery().serviceId(service.getKey()).start(startTime).end(now()) + ); + + LOGGER.info("instances: {}", instances); + + load("expected/simple/instances.yml").as(InstancesMatcher.class).verify(instances); + + return instances; + } + + private Endpoints verifyServiceEndpoints(final Service service) throws Exception { + final Endpoints endpoints = graphql.endpoints(new EndpointQuery().serviceId(service.getKey())); + + LOGGER.info("endpoints: {}", endpoints); + + load("expected/simple/endpoints.yml").as(EndpointsMatcher.class).verify(endpoints); + + return endpoints; + } + + private void verifyInstancesMetrics(final Instances instances) throws Exception { + for (Instance instance : instances.getInstances()) { + for (String metricsName : ALL_INSTANCE_METRICS) { + LOGGER.info("verifying service instance response time: {}", instance); + final Metrics instanceMetrics = graphql.metrics( + new MetricsQuery().stepByMinute().metricsName(metricsName).id(instance.getKey()) + ); + + LOGGER.info("instance metrics: {}", instanceMetrics); + + final AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher(); + final MetricsValueMatcher greaterThanZero = new MetricsValueMatcher(); + greaterThanZero.setValue("gt 0"); + instanceRespTimeMatcher.setValue(greaterThanZero); + instanceRespTimeMatcher.verify(instanceMetrics); + LOGGER.info("{}: {}", metricsName, instanceMetrics); + } + } + } + + private void verifyInstancesJVMMetrics(final Instances instances) throws Exception { + for (Instance instance : instances.getInstances()) { + for (String metricsName : ALL_INSTANCE_JVM_METRICS) { + LOGGER.info("verifying service instance response time: {}", instance); + final Metrics instanceJVMMetrics = graphql.metrics( + new MetricsQuery().stepByMinute().metricsName(metricsName).id(instance.getKey()) + ); + + LOGGER.info("instance jvm metrics: {}", instanceJVMMetrics); + + final AtLeastOneOfMetricsMatcher instanceThreadMatcher = new AtLeastOneOfMetricsMatcher(); + final MetricsValueMatcher greaterThanZero = new MetricsValueMatcher(); + greaterThanZero.setValue("gt 0"); + instanceThreadMatcher.setValue(greaterThanZero); + instanceThreadMatcher.verify(instanceJVMMetrics); + LOGGER.info("{}: {}", metricsName, instanceJVMMetrics); + } + } + } + + private void verifyEndpointsMetrics(final Endpoints endpoints) throws Exception { + for (Endpoint endpoint : endpoints.getEndpoints()) { + if (!endpoint.getLabel().equals("/users")) { + continue; + } + for (final String metricName : ALL_ENDPOINT_METRICS) { + LOGGER.info("verifying endpoint {}: {}", endpoint, metricName); + + final Metrics metrics = graphql.metrics( + new MetricsQuery().stepByMinute().metricsName(metricName).id(endpoint.getKey()) + ); + + LOGGER.info("metrics: {}", metrics); + + final AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher(); + final MetricsValueMatcher greaterThanZero = new MetricsValueMatcher(); + greaterThanZero.setValue("gt 0"); + instanceRespTimeMatcher.setValue(greaterThanZero); + instanceRespTimeMatcher.verify(metrics); + + LOGGER.info("{}: {}", metricName, metrics); + } + } + } + + private void verifyServiceMetrics(final Service service) throws Exception { + for (String metricName : ALL_SERVICE_METRICS) { + LOGGER.info("verifying service {}, metrics: {}", service, metricName); + final Metrics serviceMetrics = graphql.metrics( + new MetricsQuery().stepByMinute().metricsName(metricName).id(service.getKey()) + ); + LOGGER.info("serviceMetrics: {}", serviceMetrics); + final AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher(); + final MetricsValueMatcher greaterThanZero = new MetricsValueMatcher(); + greaterThanZero.setValue("gt 0"); + instanceRespTimeMatcher.setValue(greaterThanZero); + instanceRespTimeMatcher.verify(serviceMetrics); + LOGGER.info("{}: {}", metricName, serviceMetrics); + } + } + + private void verifyServiceInstanceRelationMetrics(final List calls) throws Exception { + verifyRelationMetrics( + calls, ALL_SERVICE_INSTANCE_RELATION_CLIENT_METRICS, + ALL_SERVICE_INSTANCE_RELATION_SERVER_METRICS + ); + } + + private void verifyServiceRelationMetrics(final List calls) throws Exception { + verifyRelationMetrics(calls, ALL_SERVICE_RELATION_CLIENT_METRICS, ALL_SERVICE_RELATION_SERVER_METRICS); + } + + private void verifyRelationMetrics(final List calls, + final String[] relationClientMetrics, + final String[] relationServerMetrics) throws Exception { + for (Call call : calls) { + for (String detectPoint : call.getDetectPoints()) { + switch (detectPoint) { + case "CLIENT": { + for (String metricName : relationClientMetrics) { + verifyMetrics(graphql, metricName, call.getId(), startTime); + } + break; + } + case "SERVER": { + for (String metricName : relationServerMetrics) { + verifyMetrics(graphql, metricName, call.getId(), startTime); + } + break; + } + } + } + } + } +} diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaMeterE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaMeterE2E.java new file mode 100644 index 000000000..e6474f2d8 --- /dev/null +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaMeterE2E.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e.kafka; + +import java.util.List; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.e2e.annotation.ContainerHostAndPort; +import org.apache.skywalking.e2e.annotation.DockerCompose; +import org.apache.skywalking.e2e.base.SkyWalkingE2E; +import org.apache.skywalking.e2e.base.SkyWalkingTestAdapter; +import org.apache.skywalking.e2e.common.HostAndPort; +import org.apache.skywalking.e2e.metrics.AtLeastOneOfMetricsMatcher; +import org.apache.skywalking.e2e.metrics.MetricsValueMatcher; +import org.apache.skywalking.e2e.metrics.ReadMetrics; +import org.apache.skywalking.e2e.metrics.ReadMetricsQuery; +import org.apache.skywalking.e2e.retryable.RetryableTest; +import org.apache.skywalking.e2e.service.Service; +import org.apache.skywalking.e2e.service.ServicesMatcher; +import org.apache.skywalking.e2e.service.ServicesQuery; +import org.apache.skywalking.e2e.service.instance.Instances; +import org.apache.skywalking.e2e.service.instance.InstancesMatcher; +import org.apache.skywalking.e2e.service.instance.InstancesQuery; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.containers.DockerComposeContainer; + +import static org.apache.skywalking.e2e.metrics.MetricsQuery.SIMPLE_MICROMETER_METERS; +import static org.apache.skywalking.e2e.utils.Times.now; +import static org.apache.skywalking.e2e.utils.Yamls.load; + +@Slf4j +@SkyWalkingE2E +public class KafkaMeterE2E extends SkyWalkingTestAdapter { + + @SuppressWarnings("unused") + @DockerCompose({ + "docker/kafka/docker-compose.yml", + "docker/kafka/docker-compose.meter.yml" + }) + protected DockerComposeContainer justForSideEffects; + + @SuppressWarnings("unused") + @ContainerHostAndPort(name = "ui", port = 8080) + protected HostAndPort swWebappHostPort; + + @SuppressWarnings("unused") + @ContainerHostAndPort(name = "provider_kafka", port = 9090) + protected HostAndPort serviceHostPort; + + @BeforeAll + public void setUp() throws Exception { + queryClient(swWebappHostPort); + + trafficController(serviceHostPort, "/users"); + } + + @AfterAll + public void tearDown() { + trafficController.stop(); + } + + @RetryableTest + public void meters() throws Exception { + List services = graphql.services(new ServicesQuery().start(startTime).end(now())); + + services = services.stream().filter(s -> !s.getLabel().equals("oap-server")).collect(Collectors.toList()); + LOGGER.info("services: {}", services); + + load("expected/meter/services.yml").as(ServicesMatcher.class).verify(services); + + for (final Service service : services) { + final Instances instances = verifyServiceInstances(service); + for (String meterName : SIMPLE_MICROMETER_METERS) { + LOGGER.info("verifying service {}, metrics: {}", service, meterName); + final ReadMetrics instanceMetrics = graphql.readMetrics( + new ReadMetricsQuery().stepByMinute().metricsName(meterName).serviceName(service.getLabel()) + .instanceName(instances.getInstances().get(0).getLabel()) + ); + LOGGER.info("instanceMetrics: {}", instanceMetrics); + final AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher(); + final MetricsValueMatcher greaterThanZero = new MetricsValueMatcher(); + greaterThanZero.setValue("gt 0"); + instanceRespTimeMatcher.setValue(greaterThanZero); + instanceRespTimeMatcher.verify(instanceMetrics.getValues()); + LOGGER.info("{}: {}", meterName, instanceMetrics); + } + } + } + + private Instances verifyServiceInstances(final Service service) throws Exception { + final Instances instances = graphql.instances( + new InstancesQuery().serviceId(service.getKey()).start(startTime).end(now()) + ); + + LOGGER.info("instances: {}", instances); + + load("expected/simple/instances.yml").as(InstancesMatcher.class).verify(instances); + + return instances; + } +} diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaProfileE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaProfileE2E.java new file mode 100644 index 000000000..f6b33a3d7 --- /dev/null +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaProfileE2E.java @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e.kafka; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.e2e.ProfileClient; +import org.apache.skywalking.e2e.annotation.ContainerHostAndPort; +import org.apache.skywalking.e2e.annotation.DockerCompose; +import org.apache.skywalking.e2e.annotation.DockerContainer; +import org.apache.skywalking.e2e.base.SkyWalkingE2E; +import org.apache.skywalking.e2e.base.SkyWalkingTestAdapter; +import org.apache.skywalking.e2e.common.HostAndPort; +import org.apache.skywalking.e2e.profile.creation.ProfileTaskCreationRequest; +import org.apache.skywalking.e2e.profile.creation.ProfileTaskCreationResult; +import org.apache.skywalking.e2e.profile.creation.ProfileTaskCreationResultMatcher; +import org.apache.skywalking.e2e.profile.query.ProfileAnalyzation; +import org.apache.skywalking.e2e.profile.query.ProfileStackTreeMatcher; +import org.apache.skywalking.e2e.profile.query.ProfileTaskQuery; +import org.apache.skywalking.e2e.profile.query.ProfileTasks; +import org.apache.skywalking.e2e.profile.query.ProfiledSegment; +import org.apache.skywalking.e2e.profile.query.ProfiledSegmentMatcher; +import org.apache.skywalking.e2e.profile.query.ProfilesTasksMatcher; +import org.apache.skywalking.e2e.retryable.RetryableTest; +import org.apache.skywalking.e2e.service.Service; +import org.apache.skywalking.e2e.service.ServicesMatcher; +import org.apache.skywalking.e2e.service.ServicesQuery; +import org.apache.skywalking.e2e.service.endpoint.EndpointQuery; +import org.apache.skywalking.e2e.service.endpoint.Endpoints; +import org.apache.skywalking.e2e.service.endpoint.EndpointsMatcher; +import org.apache.skywalking.e2e.service.instance.Instances; +import org.apache.skywalking.e2e.service.instance.InstancesMatcher; +import org.apache.skywalking.e2e.service.instance.InstancesQuery; +import org.apache.skywalking.e2e.trace.Trace; +import org.apache.skywalking.e2e.trace.TracesMatcher; +import org.apache.skywalking.e2e.trace.TracesQuery; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerState; +import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; + +import static org.apache.skywalking.e2e.utils.Times.now; +import static org.apache.skywalking.e2e.utils.Yamls.load; +import static org.assertj.core.api.Assertions.assertThat; + +@Slf4j +@SkyWalkingE2E +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class KafkaProfileE2E extends SkyWalkingTestAdapter { + + @SuppressWarnings("unused") + @DockerCompose({ + "docker/kafka/docker-compose.yml", + "docker/kafka/docker-compose.profiling.yml" + }) + protected DockerComposeContainer compose; + + @SuppressWarnings("unused") + @ContainerHostAndPort(name = "provider_kafka", port = 9090) + protected HostAndPort serviceHostPort; + + @SuppressWarnings("unused") + @ContainerHostAndPort(name = "ui", port = 8080) + protected HostAndPort swWebappHostPort; + + @SuppressWarnings("unused") + @DockerContainer("oap") + private ContainerState oapContainer; + + private ProfileClient graphql; + private String instrumentedServiceUrl; + + @BeforeAll + public void setUp() { + graphql = new ProfileClient(swWebappHostPort.host(), swWebappHostPort.port()); + instrumentedServiceUrl = "http://" + serviceHostPort.host() + ":" + serviceHostPort.port(); + } + + @Order(1) + @RetryableTest + void traces() throws Exception { + final ResponseEntity response = sendRequest(false); + + LOGGER.info("response: {}", response); + + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK); + + final List traces = graphql.traces( + new TracesQuery().start(startTime).end(now()).orderByDuration() + ); + + LOGGER.info("traces: {}", traces); + + assertThat(traces).isNotEmpty(); + } + + @Order(2) + @RetryableTest + void services() throws Exception { + final List services = graphql.services(new ServicesQuery().start(startTime).end(now())); + + LOGGER.info("services: {}", services); + + load("expected/profile/services.yml").as(ServicesMatcher.class).verify(services); + + for (Service service : services) { + LOGGER.info("verifying service instances: {}", service); + + verifyServiceInstances(service); + + verifyServiceEndpoints(service); + } + } + + @Test + @Order(3) + void createProfileTask() throws Exception { + final ProfileTaskCreationRequest creationRequest = ProfileTaskCreationRequest.builder() + .serviceId( + "ZTJlLXByb2ZpbGUtc2VydmljZQ==.1") + .endpointName("/profile/{name}") + .duration(1) + .startTime(-1) + .minDurationThreshold(1500) + .dumpPeriod(500) + .maxSamplingCount(5) + .build(); + + final ProfileTaskCreationResult creationResult = graphql.createProfileTask(creationRequest); + + LOGGER.info("create profile task result: {}", creationResult); + + final ProfileTaskCreationResultMatcher creationResultMatcher = new ProfileTaskCreationResultMatcher(); + creationResultMatcher.verify(creationResult); + + verifyProfileTask(creationRequest.getServiceId(), "expected/profile/notified.yml"); + + sendRequest(true); + + verifyProfileTask(creationRequest.getServiceId(), "expected/profile/finished.yml"); + + verifyProfiledSegment(creationResult.getId()); + } + + private ResponseEntity sendRequest(boolean needProfiling) { + final Map user = ImmutableMap.of( + "name", "SkyWalking", "enableProfiling", String.valueOf(needProfiling) + ); + return restTemplate.postForEntity(instrumentedServiceUrl + "/profile/users?e2e=true", user, String.class); + } + + private void verifyProfiledSegment(String taskId) throws Exception { + Trace foundedTrace = null; + for (int i = 0; i < 10; i++) { + try { + final List traces = graphql.getProfiledTraces(taskId); + + LOGGER.info("get profiled segemnt list: {}", traces); + + load("expected/profile/profileSegments.yml").as(TracesMatcher.class).verifyLoosely(traces); + + foundedTrace = traces.get(0); + break; + } catch (Exception e) { + if (i == 10 - 1) { + throw new IllegalStateException("match profiled segment list fail!", e); + } + TimeUnit.SECONDS.sleep(30); + } + } + assert foundedTrace != null; + final String segmentId = foundedTrace.getKey(); + final ProfiledSegment.ProfiledSegmentData segmentData = graphql.getProfiledSegment(foundedTrace.getKey()); + + LOGGER.info("get profiled segment : {}", segmentData); + + load("expected/profile/profileSegment.yml").as(ProfiledSegmentMatcher.class).verify(segmentData); + + final long start = Long.parseLong(foundedTrace.getStart()); + final long end = start + foundedTrace.getDuration(); + + final ProfileAnalyzation analyzation = graphql.getProfileAnalyzation(segmentId, start, end); + + LOGGER.info("get profile analyzation : {}", analyzation); + + load("expected/profile/profileAnayzation.yml").as(ProfileStackTreeMatcher.class) + .verify(analyzation.getData().getTrees().get(0)); + + validateExporter(taskId, foundedTrace.getTraceIds().get(0)); + } + + private void verifyProfileTask(String serviceId, String verifyResources) throws Exception { + for (int i = 0; i < 10; i++) { + try { + final ProfileTasks tasks = graphql.getProfileTaskList( + new ProfileTaskQuery().serviceId(serviceId).endpointName("") + ); + + LOGGER.info("get profile task list: {}", tasks); + + load(verifyResources).as(ProfilesTasksMatcher.class).verify(tasks); + break; + } catch (Throwable e) { + if (i == 10 - 1) { + throw new IllegalStateException("match profile task list fail!", e); + } + TimeUnit.SECONDS.sleep(30); + } + } + } + + private void verifyServiceInstances(final Service service) throws Exception { + final Instances instances = graphql.instances( + new InstancesQuery().serviceId(service.getKey()).start(startTime).end(now()) + ); + + LOGGER.info("instances: {}", instances); + + load("expected/profile/instances.yml").as(InstancesMatcher.class).verify(instances); + + } + + private void verifyServiceEndpoints(final Service service) throws Exception { + final Endpoints endpoints = graphql.endpoints(new EndpointQuery().serviceId(service.getKey())); + + LOGGER.info("endpoints: {}", endpoints); + + load("expected/profile/endpoints.yml").as(EndpointsMatcher.class).verify(endpoints); + + } + + private void validateExporter(final String taskId, final String traceId) throws Exception { + final String exportShell = String.format( + "/skywalking/tools/profile-exporter/profile_exporter.sh --taskid=%s --traceid=%s /tmp", + taskId, traceId + ); + final Container.ExecResult exportResult = oapContainer.execInContainer("/bin/sh", "-c", exportShell); + + LOGGER.info("exported result: {}", exportResult); + + assertThat(exportResult.getExitCode()).isEqualTo(0); + + final String lsExportedFileShell = String.format("ls /tmp/%s.tar.gz", traceId); + final Container.ExecResult checkExportedFileResult = oapContainer.execInContainer( + "/bin/sh", "-c", lsExportedFileShell); + + LOGGER.info("check exported file result: {}", checkExportedFileResult); + + assertThat(checkExportedFileResult.getExitCode()).isEqualTo(0); + } +} diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/meter/MeterE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/meter/MeterE2E.java index 533541e2d..828bb52bf 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/meter/MeterE2E.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/meter/MeterE2E.java @@ -55,11 +55,11 @@ public class MeterE2E extends SkyWalkingTestAdapter { @SuppressWarnings("unused") @ContainerHostAndPort(name = "ui", port = 8080) - private HostAndPort swWebappHostPort; + protected HostAndPort swWebappHostPort; @SuppressWarnings("unused") @ContainerHostAndPort(name = "provider", port = 9090) - private HostAndPort serviceHostPort; + protected HostAndPort serviceHostPort; @BeforeAll void setUp() throws Exception { diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/profile/ProfileE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/profile/ProfileE2E.java index 77256bf83..b89b8d208 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/profile/ProfileE2E.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/profile/ProfileE2E.java @@ -81,11 +81,11 @@ public class ProfileE2E extends SkyWalkingTestAdapter { @SuppressWarnings("unused") @ContainerHostAndPort(name = "ui", port = 8080) - private HostAndPort swWebappHostPort; + protected HostAndPort swWebappHostPort; @SuppressWarnings("unused") @ContainerHostAndPort(name = "provider", port = 9090) - private HostAndPort serviceHostPort; + protected HostAndPort serviceHostPort; @SuppressWarnings("unused") @DockerContainer("oap") diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java index 9a56df2da..c7e4f7c0f 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java @@ -116,11 +116,11 @@ public class SimpleE2E extends SkyWalkingTestAdapter { @SuppressWarnings("unused") @ContainerHostAndPort(name = "ui", port = 8080) - private HostAndPort swWebappHostPort; + protected HostAndPort swWebappHostPort; @SuppressWarnings("unused") @ContainerHostAndPort(name = "provider", port = 9090) - private HostAndPort serviceHostPort; + protected HostAndPort serviceHostPort; @BeforeAll void setUp() throws Exception { diff --git a/tools/dependencies/known-oap-backend-dependencies-es7.txt b/tools/dependencies/known-oap-backend-dependencies-es7.txt index 3940be02d..4c7384ac2 100755 --- a/tools/dependencies/known-oap-backend-dependencies-es7.txt +++ b/tools/dependencies/known-oap-backend-dependencies-es7.txt @@ -167,3 +167,7 @@ t-digest-3.2.jar vavr-0.10.3.jar vavr-match-0.10.3.jar zookeeper-3.4.10.jar +kafka-clients-2.4.1.jar +lz4-java-1.6.0.jar +snappy-java-1.1.7.3.jar +zstd-jni-1.4.3-1.jar \ No newline at end of file diff --git a/tools/dependencies/known-oap-backend-dependencies.txt b/tools/dependencies/known-oap-backend-dependencies.txt index 3020c8c9d..9b5e0d016 100755 --- a/tools/dependencies/known-oap-backend-dependencies.txt +++ b/tools/dependencies/known-oap-backend-dependencies.txt @@ -166,3 +166,7 @@ vavr-0.10.3.jar vavr-match-0.10.3.jar zipkin-2.9.1.jar zookeeper-3.4.10.jar +kafka-clients-2.4.1.jar +lz4-java-1.6.0.jar +snappy-java-1.1.7.3.jar +zstd-jni-1.4.3-1.jar