From 7a3355f956f6fdcbb8f01ad9cd3ea5de1d107c14 Mon Sep 17 00:00:00 2001 From: liqiangz Date: Fri, 11 Jun 2021 22:21:10 +0800 Subject: [PATCH] Use MAL to calculate JVM metrics, remove OAL dependency. (#7091) --- CHANGES.md | 1 + apm-dist-es7/src/main/assembly/binary-es7.xml | 1 + apm-dist/src/main/assembly/binary.xml | 1 + .../concepts-and-designs/scope-definitions.md | 56 ---- .../provider/jvm/JVMSourceDispatcher.java | 285 +++++++++--------- .../skywalking/oal/rt/grammar/OALLexer.g4 | 5 - .../skywalking/oal/rt/grammar/OALParser.g4 | 1 - oap-server/server-bootstrap/pom.xml | 1 + .../main/resources/jvm-metrics-rules/jvm.yaml | 61 ++++ .../src/main/resources/oal/java-agent.oal | 31 -- .../core/source/ServiceInstanceJVMCPU.java | 58 ---- .../core/source/ServiceInstanceJVMGC.java | 64 ---- .../core/source/ServiceInstanceJVMMemory.java | 70 ----- .../source/ServiceInstanceJVMMemoryPool.java | 70 ----- .../core/source/ServiceInstanceJVMThread.java | 64 ---- .../kafka/provider/KafkaFetcherProvider.java | 10 +- .../provider/handler/JVMMetricsHandler.java | 7 +- .../handler/JVMMetricsHandlerTest.java | 29 +- .../jvm/provider/JVMModuleProvider.java | 14 +- .../receiver/jvm/provider/JVMOALDefine.java | 35 --- .../JVMMetricReportServiceHandler.java | 11 +- .../skywalking/e2e/metrics/MetricsQuery.java | 10 +- .../skywalking/e2e/simple/SimpleE2E.java | 6 +- 23 files changed, 241 insertions(+), 650 deletions(-) create mode 100644 oap-server/server-bootstrap/src/main/resources/jvm-metrics-rules/jvm.yaml delete mode 100644 oap-server/server-bootstrap/src/main/resources/oal/java-agent.oal delete mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMCPU.java delete mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMGC.java delete mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMMemory.java delete mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMMemoryPool.java delete mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMThread.java delete mode 100644 oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/JVMOALDefine.java diff --git a/CHANGES.md b/CHANGES.md index 56409f0ea..9b28d50f0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,6 +12,7 @@ Release Notes. #### OAP-Backend * Disable Spring sleuth meter analyzer by default. +* Use MAL to calculate JVM metrics, remove OAL dependency. #### UI diff --git a/apm-dist-es7/src/main/assembly/binary-es7.xml b/apm-dist-es7/src/main/assembly/binary-es7.xml index 337217b65..4e3e3f307 100644 --- a/apm-dist-es7/src/main/assembly/binary-es7.xml +++ b/apm-dist-es7/src/main/assembly/binary-es7.xml @@ -67,6 +67,7 @@ ui-initialized-templates/* lal/* log-mal-rules/* + jvm-metrics-rules/*.yaml config diff --git a/apm-dist/src/main/assembly/binary.xml b/apm-dist/src/main/assembly/binary.xml index 562fe8b11..b78466a0f 100644 --- a/apm-dist/src/main/assembly/binary.xml +++ b/apm-dist/src/main/assembly/binary.xml @@ -67,6 +67,7 @@ ui-initialized-templates/* lal/* log-mal-rules/* + jvm-metrics-rules/*.yaml config diff --git a/docs/en/concepts-and-designs/scope-definitions.md b/docs/en/concepts-and-designs/scope-definitions.md index cc7518ff8..0bdb725c4 100644 --- a/docs/en/concepts-and-designs/scope-definitions.md +++ b/docs/en/concepts-and-designs/scope-definitions.md @@ -52,62 +52,6 @@ This calculates the metrics data from each request of the service instance. | tcpInfo.receivedBytes | The received bytes of the TCP traffic, if this request is a TCP call. | | long | | tcpInfo.sentBytes | The sent bytes of the TCP traffic, if this request is a TCP call. | | long | -#### Secondary scopes of `ServiceInstance` - -This calculates the metrics data if the service instance is a JVM and collects through javaagent. - -1. SCOPE `ServiceInstanceJVMCPU` - -| Name | Remarks | Group Key | Type | -|---|---|---|---| -| name | The name of the service instance, such as `ip:port@Service Name`. **Note**: Currently, the native agent uses `uuid@ipv4` as the instance name, which does not assist in setting up a filter in aggregation. | | string| -| serviceName | The name of the service. | | string | -| usePercent | The percentage of CPU time spent.| | double| - -2. SCOPE `ServiceInstanceJVMMemory` - -| Name | Remarks | Group Key | Type | -|---|---|---|---| -| name | The name of the service instance, such as `ip:port@Service Name`. **Note**: Currently, the native agent uses `uuid@ipv4` as the instance name, which does not assist in setting up a filter in aggregation. | | string| -| serviceName | The name of the service. | | string | -| heapStatus | Indicates whether the metric has a heap property or not. | | bool | -| init | See the JVM documentation. | | long | -| max | See the JVM documentation. | | long | -| used | See the JVM documentation. | | long | -| committed | See the JVM documentation. | | long | - -3. SCOPE `ServiceInstanceJVMMemoryPool` - -| Name | Remarks | Group Key | Type | -|---|---|---|---| -| name | The name of the service instance, such as `ip:port@Service Name`. **Note**: Currently, the native agent uses `uuid@ipv4` as the instance name, which does not assist in setting up a filter in aggregation. | | string| -| serviceName | The name of the service. | | string | -| poolType | The type may be CODE_CACHE_USAGE, NEWGEN_USAGE, OLDGEN_USAGE, SURVIVOR_USAGE, PERMGEN_USAGE, or METASPACE_USAGE based on different versions of JVM. | | enum | -| init | See the JVM documentation. | | long | -| max | See the JVM documentation. | | long | -| used | See the JVM documentation. | | long | -| committed | See the JVM documentation. | | long | - -4. SCOPE `ServiceInstanceJVMGC` - -| Name | Remarks | Group Key | Type | -|---|---|---|---| -| name | The name of the service instance, such as `ip:port@Service Name`. **Note**: Currently, the native agent uses `uuid@ipv4` as the instance name, which does not assist in setting up a filter in aggregation. | | string| -| serviceName | The name of the service. | | string | -| phrase | Includes both NEW and OLD. | | Enum | -| time | The time spent in GC. | | long | -| count | The count in GC operations. | | long | - -5. SCOPE `ServiceInstanceJVMThread` - -| Name | Remarks | Group Key | Type | -|---|---|---|---| -| name | The name of the service instance, such as `ip:port@Service Name`. **Note**: Currently, the native agent uses `uuid@ipv4` as the instance name, which does not assist in setting up a filter in aggregation. | | string| -| serviceName | The name of the service. | | string | -| liveCount | The current number of live threads. | | int | -| daemonCount | The current number of daemon threads. | | int | -| peakCount | The current number of peak threads. | | int | - ### SCOPE `Endpoint` This calculates the metrics data from each request of the endpoint in the service. diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/jvm/JVMSourceDispatcher.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/jvm/JVMSourceDispatcher.java index f1ab7b955..ca9586a7b 100644 --- a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/jvm/JVMSourceDispatcher.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/jvm/JVMSourceDispatcher.java @@ -18,182 +18,167 @@ package org.apache.skywalking.oap.server.analyzer.provider.jvm; +import java.util.Arrays; +import java.util.Collection; import java.util.List; +import java.util.stream.Collectors; +import java.util.Collections; + +import com.google.common.collect.ImmutableMap; import lombok.extern.slf4j.Slf4j; -import org.apache.skywalking.apm.network.common.v3.CPU; import org.apache.skywalking.apm.network.language.agent.v3.GC; import org.apache.skywalking.apm.network.language.agent.v3.JVMMetric; import org.apache.skywalking.apm.network.language.agent.v3.Memory; import org.apache.skywalking.apm.network.language.agent.v3.MemoryPool; import org.apache.skywalking.apm.network.language.agent.v3.Thread; +import org.apache.skywalking.oap.meter.analyzer.MetricConvert; +import org.apache.skywalking.oap.meter.analyzer.dsl.Sample; +import org.apache.skywalking.oap.meter.analyzer.dsl.SampleFamily; +import org.apache.skywalking.oap.meter.analyzer.dsl.SampleFamilyBuilder; +import org.apache.skywalking.oap.meter.analyzer.prometheus.rule.Rule; +import org.apache.skywalking.oap.server.analyzer.provider.meter.process.SampleBuilder; import org.apache.skywalking.oap.server.core.CoreModule; -import org.apache.skywalking.oap.server.core.analysis.IDManager; -import org.apache.skywalking.oap.server.core.analysis.NodeType; -import org.apache.skywalking.oap.server.core.analysis.TimeBucket; -import org.apache.skywalking.oap.server.core.source.GCPhrase; -import org.apache.skywalking.oap.server.core.source.MemoryPoolType; -import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMCPU; -import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMGC; -import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMMemory; -import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMMemoryPool; -import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMThread; -import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.core.analysis.meter.MeterSystem; import org.apache.skywalking.oap.server.library.module.ModuleManager; @Slf4j public class JVMSourceDispatcher { - private final SourceReceiver sourceReceiver; - public JVMSourceDispatcher(ModuleManager moduleManager) { - this.sourceReceiver = moduleManager.find(CoreModule.NAME).provider().getService(SourceReceiver.class); + private final List metricConverts; + + public JVMSourceDispatcher(ModuleManager moduleManager, List rules) { + this.metricConverts = rules.stream() + .map(it -> new MetricConvert(it, moduleManager.find(CoreModule.NAME).provider().getService(MeterSystem.class))) + .collect(Collectors.toList()); } - public void sendMetric(String service, String serviceInstance, JVMMetric metrics) { - long minuteTimeBucket = TimeBucket.getMinuteTimeBucket(metrics.getTime()); + public void sendMetric(String service, String serviceInstance, JVMMetric jvmMetric) { + List cpuSamples = Collections.singletonList(parseCpuData(service, serviceInstance, jvmMetric)); + List memorySamples = parseMemoryData(service, serviceInstance, jvmMetric); + List memoryPoolSamples = parseMemoryPollData(service, serviceInstance, jvmMetric); + List gcCountSamples = parseGcCountData(service, serviceInstance, jvmMetric); + List gcTimeSamples = parseGcTimeData(service, serviceInstance, jvmMetric); + List threadSamples = parseThreadData(service, serviceInstance, jvmMetric); - final String serviceId = IDManager.ServiceID.buildId(service, NodeType.Normal); - final String serviceInstanceId = IDManager.ServiceInstanceID.buildId(serviceId, serviceInstance); + ImmutableMap sampleFamilies = ImmutableMap.builder() + .put("sw_jvm_gc_time", SampleFamilyBuilder.newBuilder(gcTimeSamples.toArray(new Sample[0])).build()) + .put("sw_jvm_gc_count", SampleFamilyBuilder.newBuilder(gcCountSamples.toArray(new Sample[0])).build()) + .put("sw_jvm_cpu", SampleFamilyBuilder.newBuilder(cpuSamples.toArray(new Sample[0])).build()) + .put("sw_jvm_thread", SampleFamilyBuilder.newBuilder(threadSamples.toArray(new Sample[0])).build()) + .put("sw_jvm_memory", SampleFamilyBuilder.newBuilder(memorySamples.toArray(new Sample[0])).build()) + .put("sw_jvm_memory_poll", SampleFamilyBuilder.newBuilder(memoryPoolSamples.toArray(new Sample[0])).build()) + .build(); - this.sendToCpuMetricProcess( - service, serviceId, serviceInstance, serviceInstanceId, minuteTimeBucket, metrics.getCpu()); - this.sendToMemoryMetricProcess( - service, serviceId, serviceInstance, serviceInstanceId, minuteTimeBucket, metrics.getMemoryList()); - this.sendToMemoryPoolMetricProcess( - service, serviceId, serviceInstance, serviceInstanceId, minuteTimeBucket, metrics.getMemoryPoolList()); - this.sendToGCMetricProcess( - service, serviceId, serviceInstance, serviceInstanceId, minuteTimeBucket, metrics.getGcList()); - this.sendToThreadMetricProcess( - service, serviceId, serviceInstance, serviceInstanceId, minuteTimeBucket, metrics.getThread()); + metricConverts.forEach(metricConvert -> metricConvert.toMeter(sampleFamilies)); } - private void sendToCpuMetricProcess(String service, - String serviceId, - String serviceInstance, - String serviceInstanceId, - long timeBucket, - CPU cpu) { - ServiceInstanceJVMCPU serviceInstanceJVMCPU = new ServiceInstanceJVMCPU(); - serviceInstanceJVMCPU.setId(serviceInstanceId); - serviceInstanceJVMCPU.setName(serviceInstance); - serviceInstanceJVMCPU.setServiceId(serviceId); - serviceInstanceJVMCPU.setServiceName(service); - // If the cpu usage percent is less than 1, will set to 1 - double adjustedCpuUsagePercent = Math.max(cpu.getUsagePercent(), 1.0); - serviceInstanceJVMCPU.setUsePercent(adjustedCpuUsagePercent); - serviceInstanceJVMCPU.setTimeBucket(timeBucket); - sourceReceiver.receive(serviceInstanceJVMCPU); + private List parseThreadData(String service, String serviceInstance, JVMMetric jvmMetric) { + Thread thread = jvmMetric.getThread(); + return Arrays.asList( + buildThreadSample(thread.getDaemonCount(), "daemon", service, serviceInstance, jvmMetric.getTime()), + buildThreadSample(thread.getLiveCount(), "live", service, serviceInstance, jvmMetric.getTime()), + buildThreadSample(thread.getPeakCount(), "peak", service, serviceInstance, jvmMetric.getTime()) + ); } - private void sendToGCMetricProcess(String service, - String serviceId, - String serviceInstance, - String serviceInstanceId, - long timeBucket, - List gcs) { - gcs.forEach(gc -> { - ServiceInstanceJVMGC serviceInstanceJVMGC = new ServiceInstanceJVMGC(); - serviceInstanceJVMGC.setId(serviceInstanceId); - serviceInstanceJVMGC.setName(serviceInstance); - serviceInstanceJVMGC.setServiceId(serviceId); - serviceInstanceJVMGC.setServiceName(service); - - switch (gc.getPhrase()) { - case NEW: - serviceInstanceJVMGC.setPhrase(GCPhrase.NEW); - break; - case OLD: - serviceInstanceJVMGC.setPhrase(GCPhrase.OLD); - break; - } - - serviceInstanceJVMGC.setTime(gc.getTime()); - serviceInstanceJVMGC.setCount(gc.getCount()); - serviceInstanceJVMGC.setTimeBucket(timeBucket); - sourceReceiver.receive(serviceInstanceJVMGC); - }); + private List parseGcCountData(String service, String serviceInstance, JVMMetric jvmMetric) { + return jvmMetric.getGcList().stream().map(gc -> + buildGcSample(gc, gc.getCount(), "sw_jvm_gc_count", service, serviceInstance, jvmMetric.getTime()) + ).collect(Collectors.toList()); } - private void sendToMemoryMetricProcess(String service, - String serviceId, - String serviceInstance, - String serviceInstanceId, - long timeBucket, - List memories) { - memories.forEach(memory -> { - ServiceInstanceJVMMemory serviceInstanceJVMMemory = new ServiceInstanceJVMMemory(); - serviceInstanceJVMMemory.setId(serviceInstanceId); - serviceInstanceJVMMemory.setName(serviceInstance); - serviceInstanceJVMMemory.setServiceId(serviceId); - serviceInstanceJVMMemory.setServiceName(service); - serviceInstanceJVMMemory.setHeapStatus(memory.getIsHeap()); - serviceInstanceJVMMemory.setInit(memory.getInit()); - serviceInstanceJVMMemory.setMax(memory.getMax()); - serviceInstanceJVMMemory.setUsed(memory.getUsed()); - serviceInstanceJVMMemory.setCommitted(memory.getCommitted()); - serviceInstanceJVMMemory.setTimeBucket(timeBucket); - sourceReceiver.receive(serviceInstanceJVMMemory); - }); + private List parseGcTimeData(String service, String serviceInstance, JVMMetric jvmMetric) { + return jvmMetric.getGcList().stream().map(gc -> + buildGcSample(gc, gc.getTime(), "sw_jvm_gc_time", service, serviceInstance, jvmMetric.getTime()) + ).collect(Collectors.toList()); } - private void sendToMemoryPoolMetricProcess(String service, - String serviceId, - String serviceInstance, - String serviceInstanceId, - long timeBucket, - List memoryPools) { - - memoryPools.forEach(memoryPool -> { - ServiceInstanceJVMMemoryPool serviceInstanceJVMMemoryPool = new ServiceInstanceJVMMemoryPool(); - serviceInstanceJVMMemoryPool.setId(serviceInstanceId); - serviceInstanceJVMMemoryPool.setName(serviceInstance); - serviceInstanceJVMMemoryPool.setServiceId(serviceId); - serviceInstanceJVMMemoryPool.setServiceName(service); - - switch (memoryPool.getType()) { - case NEWGEN_USAGE: - serviceInstanceJVMMemoryPool.setPoolType(MemoryPoolType.NEWGEN_USAGE); - break; - case OLDGEN_USAGE: - serviceInstanceJVMMemoryPool.setPoolType(MemoryPoolType.OLDGEN_USAGE); - break; - case PERMGEN_USAGE: - serviceInstanceJVMMemoryPool.setPoolType(MemoryPoolType.PERMGEN_USAGE); - break; - case SURVIVOR_USAGE: - serviceInstanceJVMMemoryPool.setPoolType(MemoryPoolType.SURVIVOR_USAGE); - break; - case METASPACE_USAGE: - serviceInstanceJVMMemoryPool.setPoolType(MemoryPoolType.METASPACE_USAGE); - break; - case CODE_CACHE_USAGE: - serviceInstanceJVMMemoryPool.setPoolType(MemoryPoolType.CODE_CACHE_USAGE); - break; - } - - serviceInstanceJVMMemoryPool.setInit(memoryPool.getInit()); - serviceInstanceJVMMemoryPool.setMax(memoryPool.getMax()); - serviceInstanceJVMMemoryPool.setUsed(memoryPool.getUsed()); - serviceInstanceJVMMemoryPool.setCommitted(memoryPool.getCommitted()); - serviceInstanceJVMMemoryPool.setTimeBucket(timeBucket); - sourceReceiver.receive(serviceInstanceJVMMemoryPool); - }); + private Sample parseCpuData(String service, String serviceInstance, JVMMetric jvmMetric) { + SampleBuilder.SampleBuilderBuilder sampleBuilderBuilder = SampleBuilder.builder(); + double adjustedCpuUsagePercent = Math.max(jvmMetric.getCpu().getUsagePercent(), 1.0); + sampleBuilderBuilder.name("sw_jvm_cpu"); + sampleBuilderBuilder.value(adjustedCpuUsagePercent); + sampleBuilderBuilder.labels(ImmutableMap.builder().build()); + return sampleBuilderBuilder.build().build(service, serviceInstance, jvmMetric.getTime()); } - private void sendToThreadMetricProcess(String service, - String serviceId, - String serviceInstance, - String serviceInstanceId, - long timeBucket, - Thread thread) { - ServiceInstanceJVMThread serviceInstanceJVMThread = new ServiceInstanceJVMThread(); - serviceInstanceJVMThread.setId(serviceInstanceId); - serviceInstanceJVMThread.setName(serviceInstance); - serviceInstanceJVMThread.setServiceId(serviceId); - serviceInstanceJVMThread.setServiceName(service); - serviceInstanceJVMThread.setLiveCount(thread.getLiveCount()); - serviceInstanceJVMThread.setDaemonCount(thread.getDaemonCount()); - serviceInstanceJVMThread.setPeakCount(thread.getPeakCount()); - serviceInstanceJVMThread.setTimeBucket(timeBucket); - sourceReceiver.receive(serviceInstanceJVMThread); + private List parseMemoryData(String service, String serviceInstance, JVMMetric jvmMetric) { + return jvmMetric.getMemoryList().stream().map(memory -> Arrays.asList( + buildMemorySample(memory, memory.getInit(), "init", service, serviceInstance, jvmMetric.getTime()), + buildMemorySample(memory, memory.getMax(), "max", service, serviceInstance, jvmMetric.getTime()), + buildMemorySample(memory, memory.getCommitted(), "committed", service, serviceInstance, jvmMetric.getTime()), + buildMemorySample(memory, memory.getUsed(), "used", service, serviceInstance, jvmMetric.getTime()) + )).flatMap(Collection::stream).collect(Collectors.toList()); + } + + private List parseMemoryPollData(String service, String serviceInstance, JVMMetric jvmMetric) { + return jvmMetric.getMemoryPoolList().stream().map(memoryPool -> Arrays.asList( + buildMemoryPoolSample(memoryPool, memoryPool.getInit(), "init", service, serviceInstance, jvmMetric.getTime()), + buildMemoryPoolSample(memoryPool, memoryPool.getMax(), "max", service, serviceInstance, jvmMetric.getTime()), + buildMemoryPoolSample(memoryPool, memoryPool.getCommitted(), "committed", service, serviceInstance, jvmMetric.getTime()), + buildMemoryPoolSample(memoryPool, memoryPool.getUsed(), "used", service, serviceInstance, jvmMetric.getTime()) + )).flatMap(Collection::stream).collect(Collectors.toList()); + } + + private Sample buildGcSample(GC gc, long value, String name, String service, String serviceInstance, long time) { + SampleBuilder.SampleBuilderBuilder sampleBuilderBuilder = SampleBuilder.builder(); + sampleBuilderBuilder.name(name); + sampleBuilderBuilder.value(value); + switch (gc.getPhrase()) { + case NEW: + sampleBuilderBuilder.labels(ImmutableMap.of("gc_phrase", "new")); + break; + case OLD: + sampleBuilderBuilder.labels(ImmutableMap.of("gc_phrase", "old")); + break; + default: + } + return sampleBuilderBuilder.build().build(service, serviceInstance, time); + } + + private Sample buildThreadSample(long value, String threadType, String service, String serviceInstance, long time) { + SampleBuilder.SampleBuilderBuilder sampleBuilderBuilder = SampleBuilder.builder(); + sampleBuilderBuilder.name("sw_jvm_thread"); + sampleBuilderBuilder.value(value); + sampleBuilderBuilder.labels(ImmutableMap.of("thread_type", threadType)); + return sampleBuilderBuilder.build().build(service, serviceInstance, time); + } + + private Sample buildMemorySample(Memory memory, long value, String memoryType, String service, String serviceInstance, long time) { + SampleBuilder.SampleBuilderBuilder sampleBuilderBuilder = SampleBuilder.builder(); + sampleBuilderBuilder.name("sw_jvm_memory"); + sampleBuilderBuilder.labels(ImmutableMap.of("heap_status", String.valueOf(memory.getIsHeap()), "memory_type", memoryType)); + sampleBuilderBuilder.value(value); + return sampleBuilderBuilder.build().build(service, serviceInstance, time); + } + + private Sample buildMemoryPoolSample(MemoryPool memoryPool, long value, String memoryType, String service, String serviceInstance, long time) { + SampleBuilder.SampleBuilderBuilder sampleBuilderBuilder = SampleBuilder.builder(); + sampleBuilderBuilder.name("sw_jvm_memory_poll"); + sampleBuilderBuilder.value(value); + String pollType = "poll_type"; + String memoryTypeKey = "memory_type"; + switch (memoryPool.getType()) { + case NEWGEN_USAGE: + sampleBuilderBuilder.labels(ImmutableMap.of(pollType, "memoryTypeKey", memoryTypeKey, memoryType)); + break; + case OLDGEN_USAGE: + sampleBuilderBuilder.labels(ImmutableMap.of(pollType, "oldgenUsage", memoryTypeKey, memoryType)); + break; + case PERMGEN_USAGE: + sampleBuilderBuilder.labels(ImmutableMap.of(pollType, "permgenUsage", memoryTypeKey, memoryType)); + break; + case SURVIVOR_USAGE: + sampleBuilderBuilder.labels(ImmutableMap.of(pollType, "survivorUsage", memoryTypeKey, memoryType)); + break; + case METASPACE_USAGE: + sampleBuilderBuilder.labels(ImmutableMap.of(pollType, "metaspaceUsage", memoryTypeKey, memoryType)); + break; + case CODE_CACHE_USAGE: + sampleBuilderBuilder.labels(ImmutableMap.of(pollType, "codeCacheUsage", memoryTypeKey, memoryType)); + break; + default: + } + return sampleBuilderBuilder.build().build(service, serviceInstance, time); } } diff --git a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 index 8c3acdd8a..86830f20f 100644 --- a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 +++ b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 @@ -34,11 +34,6 @@ SRC_ENDPOINT: 'Endpoint'; SRC_SERVICE_RELATION: 'ServiceRelation'; SRC_SERVICE_INSTANCE_RELATION: 'ServiceInstanceRelation'; SRC_ENDPOINT_RELATION: 'EndpointRelation'; -SRC_SERVICE_INSTANCE_JVM_CPU: 'ServiceInstanceJVMCPU'; -SRC_SERVICE_INSTANCE_JVM_MEMORY: 'ServiceInstanceJVMMemory'; -SRC_SERVICE_INSTANCE_JVM_MEMORY_POOL: 'ServiceInstanceJVMMemoryPool'; -SRC_SERVICE_INSTANCE_JVM_GC: 'ServiceInstanceJVMGC'; -SRC_SERVICE_INSTANCE_JVM_THREAD: 'ServiceInstanceJVMThread'; SRC_DATABASE_ACCESS: 'DatabaseAccess'; SRC_SERVICE_INSTANCE_CLR_CPU: 'ServiceInstanceCLRCPU'; SRC_SERVICE_INSTANCE_CLR_GC: 'ServiceInstanceCLRGC'; diff --git a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 index 71b030a9d..08f40087c 100644 --- a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 +++ b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 @@ -52,7 +52,6 @@ filterExpression source : SRC_ALL | SRC_SERVICE | SRC_DATABASE_ACCESS | SRC_SERVICE_INSTANCE | SRC_ENDPOINT | SRC_SERVICE_RELATION | SRC_SERVICE_INSTANCE_RELATION | SRC_ENDPOINT_RELATION | - SRC_SERVICE_INSTANCE_JVM_CPU | SRC_SERVICE_INSTANCE_JVM_MEMORY | SRC_SERVICE_INSTANCE_JVM_MEMORY_POOL | SRC_SERVICE_INSTANCE_JVM_GC | SRC_SERVICE_INSTANCE_JVM_THREAD |// JVM source of service instance SRC_SERVICE_INSTANCE_CLR_CPU | SRC_SERVICE_INSTANCE_CLR_GC | SRC_SERVICE_INSTANCE_CLR_THREAD | SRC_ENVOY_INSTANCE_METRIC | SRC_BROWSER_APP_PERF | SRC_BROWSER_APP_PAGE_PERF | SRC_BROWSER_APP_SINGLE_VERSION_PERF | diff --git a/oap-server/server-bootstrap/pom.xml b/oap-server/server-bootstrap/pom.xml index e9dbccada..dd0c38233 100644 --- a/oap-server/server-bootstrap/pom.xml +++ b/oap-server/server-bootstrap/pom.xml @@ -285,6 +285,7 @@ zabbix-rules/ lal/ log-mal-rules/ + jvm-metrics-rules/ diff --git a/oap-server/server-bootstrap/src/main/resources/jvm-metrics-rules/jvm.yaml b/oap-server/server-bootstrap/src/main/resources/jvm-metrics-rules/jvm.yaml new file mode 100644 index 000000000..1aeb7bb3d --- /dev/null +++ b/oap-server/server-bootstrap/src/main/resources/jvm-metrics-rules/jvm.yaml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This will parse a textual representation of a duration. The formats +# accepted are based on the ISO-8601 duration format {@code PnDTnHnMn.nS} +# with days considered to be exactly 24 hours. +#

+# Examples: +# 

+#    "PT20.345S" -- parses as "20.345 seconds"
+#    "PT15M"     -- parses as "15 minutes" (where a minute is 60 seconds)
+#    "PT10H"     -- parses as "10 hours" (where an hour is 3600 seconds)
+#    "P2D"       -- parses as "2 days" (where a day is 24 hours or 86400 seconds)
+#    "P2DT3H4M"  -- parses as "2 days, 3 hours and 4 minutes"
+#    "P-6H3M"    -- parses as "-6 hours and +3 minutes"
+#    "-P6H3M"    -- parses as "-6 hours and -3 minutes"
+#    "-P-6H+3M"  -- parses as "+6 hours and -3 minutes"
+#
+expSuffix: instance(['service'], ['instance']) +metricPrefix: instance_jvm +metricsRules: + - name: cpu + exp: sw_jvm_cpu + + - name: memory_heap + exp: sw_jvm_memory.tagEqual('memory_type', 'used', 'heap_status', 'true').avg(['service', 'instance']) + - name: memory_noheap + exp: sw_jvm_memory.tagEqual('memory_type', 'used', 'heap_status', 'false').avg(['service', 'instance']) + - name: memory_heap_max + exp: sw_jvm_memory.tagEqual('memory_type', 'max', 'heap_status', 'true').avg(['service', 'instance']) + - name: memory_noheap_max + exp: sw_jvm_memory.tagEqual('memory_type', 'max', 'heap_status', 'false').avg(['service', 'instance']) + + - name: young_gc_time + exp: sw_jvm_gc_time.tagEqual('gc_phrase', 'new').sum(['service', 'instance']) + - name: old_gc_time + exp: sw_jvm_gc_time.tagEqual('gc_phrase', 'old').sum(['service', 'instance']) + + - name: young_gc_count + exp: sw_jvm_gc_count.tagEqual('gc_phrase', 'new').sum(['service', 'instance']) + - name: old_gc_count + exp: sw_jvm_gc_count.tagEqual('gc_phrase', 'old').sum(['service', 'instance']) + + - name: thread_live_count + exp: sw_jvm_thread.tagEqual('thread_type', 'live').avg(['service', 'instance']) + - name: thread_daemon_count + exp: sw_jvm_thread.tagEqual('thread_type', 'daemon').avg(['service', 'instance']) + - name: thread_peak_count + exp: sw_jvm_thread.tagEqual('thread_type', 'peak').avg(['service', 'instance']) diff --git a/oap-server/server-bootstrap/src/main/resources/oal/java-agent.oal b/oap-server/server-bootstrap/src/main/resources/oal/java-agent.oal deleted file mode 100644 index ca344605f..000000000 --- a/oap-server/server-bootstrap/src/main/resources/oal/java-agent.oal +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// JVM instance metrics -instance_jvm_cpu = from(ServiceInstanceJVMCPU.usePercent).doubleAvg(); -instance_jvm_memory_heap = from(ServiceInstanceJVMMemory.used).filter(heapStatus == true).longAvg(); -instance_jvm_memory_noheap = from(ServiceInstanceJVMMemory.used).filter(heapStatus == false).longAvg(); -instance_jvm_memory_heap_max = from(ServiceInstanceJVMMemory.max).filter(heapStatus == true).longAvg(); -instance_jvm_memory_noheap_max = from(ServiceInstanceJVMMemory.max).filter(heapStatus == false).longAvg(); -instance_jvm_young_gc_time = from(ServiceInstanceJVMGC.time).filter(phrase == GCPhrase.NEW).sum(); -instance_jvm_old_gc_time = from(ServiceInstanceJVMGC.time).filter(phrase == GCPhrase.OLD).sum(); -instance_jvm_young_gc_count = from(ServiceInstanceJVMGC.count).filter(phrase == GCPhrase.NEW).sum(); -instance_jvm_old_gc_count = from(ServiceInstanceJVMGC.count).filter(phrase == GCPhrase.OLD).sum(); -instance_jvm_thread_live_count = from(ServiceInstanceJVMThread.liveCount).longAvg(); -instance_jvm_thread_daemon_count = from(ServiceInstanceJVMThread.daemonCount).longAvg(); -instance_jvm_thread_peak_count = from(ServiceInstanceJVMThread.peakCount).longAvg(); \ No newline at end of file diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMCPU.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMCPU.java deleted file mode 100644 index 58c99255d..000000000 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMCPU.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.server.core.source; - -import lombok.Getter; -import lombok.Setter; - -import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_INSTANCE_CATALOG_NAME; -import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_INSTANCE_JVM_CPU; - -@ScopeDeclaration(id = SERVICE_INSTANCE_JVM_CPU, name = "ServiceInstanceJVMCPU", catalog = SERVICE_INSTANCE_CATALOG_NAME) -@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class) -public class ServiceInstanceJVMCPU extends Source { - @Override - public int scope() { - return DefaultScopeDefine.SERVICE_INSTANCE_JVM_CPU; - } - - @Override - public String getEntityId() { - return String.valueOf(id); - } - - @Getter - @Setter - private String id; - @Getter - @Setter - @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) - private String name; - @Getter - @Setter - @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) - private String serviceName; - @Getter - @Setter - @ScopeDefaultColumn.DefinedByField(columnName = "service_id") - private String serviceId; - @Getter - @Setter - private double usePercent; -} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMGC.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMGC.java deleted file mode 100644 index 64a4dd8bc..000000000 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMGC.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.server.core.source; - -import lombok.Getter; -import lombok.Setter; - -import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_INSTANCE_CATALOG_NAME; -import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_INSTANCE_JVM_GC; - -@ScopeDeclaration(id = SERVICE_INSTANCE_JVM_GC, name = "ServiceInstanceJVMGC", catalog = SERVICE_INSTANCE_CATALOG_NAME) -@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class) -public class ServiceInstanceJVMGC extends Source { - @Override - public int scope() { - return DefaultScopeDefine.SERVICE_INSTANCE_JVM_GC; - } - - @Override - public String getEntityId() { - return String.valueOf(id); - } - - @Getter - @Setter - private String id; - @Getter - @Setter - @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) - private String name; - @Getter - @Setter - @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) - private String serviceName; - @Getter - @Setter - @ScopeDefaultColumn.DefinedByField(columnName = "service_id") - private String serviceId; - @Getter - @Setter - private GCPhrase phrase; - @Getter - @Setter - private long time; - @Getter - @Setter - private long count; -} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMMemory.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMMemory.java deleted file mode 100644 index ea715012f..000000000 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMMemory.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.server.core.source; - -import lombok.Getter; -import lombok.Setter; - -import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_INSTANCE_CATALOG_NAME; -import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_INSTANCE_JVM_MEMORY; - -@ScopeDeclaration(id = SERVICE_INSTANCE_JVM_MEMORY, name = "ServiceInstanceJVMMemory", catalog = SERVICE_INSTANCE_CATALOG_NAME) -@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class) -public class ServiceInstanceJVMMemory extends Source { - @Override - public int scope() { - return DefaultScopeDefine.SERVICE_INSTANCE_JVM_MEMORY; - } - - @Override - public String getEntityId() { - return String.valueOf(id); - } - - @Getter - @Setter - private String id; - @Getter - @Setter - @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) - private String name; - @Getter - @Setter - @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) - private String serviceName; - @Getter - @Setter - @ScopeDefaultColumn.DefinedByField(columnName = "service_id") - private String serviceId; - @Getter - @Setter - private boolean heapStatus; - @Getter - @Setter - private long init; - @Getter - @Setter - private long max; - @Getter - @Setter - private long used; - @Getter - @Setter - private long committed; -} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMMemoryPool.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMMemoryPool.java deleted file mode 100644 index 193fbc1a8..000000000 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMMemoryPool.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.server.core.source; - -import lombok.Getter; -import lombok.Setter; - -import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_INSTANCE_CATALOG_NAME; -import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_INSTANCE_JVM_MEMORY_POOL; - -@ScopeDeclaration(id = SERVICE_INSTANCE_JVM_MEMORY_POOL, name = "ServiceInstanceJVMMemoryPool", catalog = SERVICE_INSTANCE_CATALOG_NAME) -@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class) -public class ServiceInstanceJVMMemoryPool extends Source { - @Override - public int scope() { - return DefaultScopeDefine.SERVICE_INSTANCE_JVM_MEMORY_POOL; - } - - @Override - public String getEntityId() { - return String.valueOf(id); - } - - @Getter - @Setter - private String id; - @Getter - @Setter - @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) - private String name; - @Getter - @Setter - @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) - private String serviceName; - @Getter - @Setter - @ScopeDefaultColumn.DefinedByField(columnName = "service_id") - private String serviceId; - @Getter - @Setter - private MemoryPoolType poolType; - @Getter - @Setter - private long init; - @Getter - @Setter - private long max; - @Getter - @Setter - private long used; - @Getter - @Setter - private long committed; -} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMThread.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMThread.java deleted file mode 100644 index 42ee6a2e2..000000000 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMThread.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.server.core.source; - -import lombok.Getter; -import lombok.Setter; - -import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_INSTANCE_CATALOG_NAME; -import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_INSTANCE_JVM_THREAD; - -@ScopeDeclaration(id = SERVICE_INSTANCE_JVM_THREAD, name = "ServiceInstanceJVMThread", catalog = SERVICE_INSTANCE_CATALOG_NAME) -@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class) -public class ServiceInstanceJVMThread extends Source { - @Override - public int scope() { - return SERVICE_INSTANCE_JVM_THREAD; - } - - @Override - public String getEntityId() { - return String.valueOf(id); - } - - @Getter - @Setter - private String id; - @Getter - @Setter - @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) - private String name; - @Getter - @Setter - @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) - private String serviceName; - @Getter - @Setter - @ScopeDefaultColumn.DefinedByField(columnName = "service_id") - private String serviceId; - @Getter - @Setter - private long liveCount; - @Getter - @Setter - private long daemonCount; - @Getter - @Setter - private long peakCount; -} diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/KafkaFetcherProvider.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/KafkaFetcherProvider.java index 94be3913f..d187bdafe 100644 --- a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/KafkaFetcherProvider.java +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/KafkaFetcherProvider.java @@ -20,6 +20,8 @@ package org.apache.skywalking.oap.server.analyzer.agent.kafka.provider; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.oap.log.analyzer.module.LogAnalyzerModule; +import org.apache.skywalking.oap.meter.analyzer.prometheus.rule.Rule; +import org.apache.skywalking.oap.meter.analyzer.prometheus.rule.Rules; import org.apache.skywalking.oap.server.analyzer.agent.kafka.KafkaFetcherHandlerRegister; import org.apache.skywalking.oap.server.analyzer.agent.kafka.module.KafkaFetcherConfig; import org.apache.skywalking.oap.server.analyzer.agent.kafka.module.KafkaFetcherModule; @@ -39,6 +41,9 @@ import org.apache.skywalking.oap.server.library.module.ModuleStartException; import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; import org.apache.skywalking.oap.server.telemetry.TelemetryModule; +import java.util.Collections; +import java.util.List; + @Slf4j public class KafkaFetcherProvider extends ModuleProvider { private KafkaFetcherHandlerRegister handlerRegister; @@ -69,8 +74,9 @@ public class KafkaFetcherProvider extends ModuleProvider { } @Override - public void start() throws ServiceNotProvidedException { - handlerRegister.register(new JVMMetricsHandler(getManager(), config)); + public void start() throws ServiceNotProvidedException, ModuleStartException { + List rules = Rules.loadRules("jvm-metrics-rules", Collections.singletonList("jvm")); + handlerRegister.register(new JVMMetricsHandler(getManager(), config, rules)); handlerRegister.register(new ServiceManagementHandler(getManager(), config)); handlerRegister.register(new TraceSegmentHandler(getManager(), config)); handlerRegister.register(new ProfileTaskHandler(getManager(), config)); diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/JVMMetricsHandler.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/JVMMetricsHandler.java index 0a5365f5d..55e3780c8 100644 --- a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/JVMMetricsHandler.java +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/JVMMetricsHandler.java @@ -22,6 +22,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.utils.Bytes; import org.apache.skywalking.apm.network.language.agent.v3.JVMMetricCollection; +import org.apache.skywalking.oap.meter.analyzer.prometheus.rule.Rule; import org.apache.skywalking.oap.server.analyzer.agent.kafka.module.KafkaFetcherConfig; import org.apache.skywalking.oap.server.analyzer.provider.jvm.JVMSourceDispatcher; import org.apache.skywalking.oap.server.core.CoreModule; @@ -34,6 +35,8 @@ import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics.Timer; import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator; import org.apache.skywalking.oap.server.telemetry.api.MetricsTag; +import java.util.List; + /** * A handler deserializes the message of JVM Metrics and pushes it to downstream. */ @@ -47,9 +50,9 @@ public class JVMMetricsHandler extends AbstractKafkaHandler { private final HistogramMetrics histogramBatch; private final CounterMetrics errorCounter; - public JVMMetricsHandler(ModuleManager manager, KafkaFetcherConfig config) { + public JVMMetricsHandler(ModuleManager manager, KafkaFetcherConfig config, List rules) { super(manager, config); - this.jvmSourceDispatcher = new JVMSourceDispatcher(manager); + this.jvmSourceDispatcher = new JVMSourceDispatcher(manager, rules); this.namingLengthControl = manager.find(CoreModule.NAME) .provider() .getService(NamingControl.class); diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/JVMMetricsHandlerTest.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/JVMMetricsHandlerTest.java index 8ac4824f2..4d56a56c6 100644 --- a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/JVMMetricsHandlerTest.java +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/provider/handler/JVMMetricsHandlerTest.java @@ -19,7 +19,8 @@ package org.apache.skywalking.oap.server.analyzer.agent.kafka.provider.handler; import com.google.common.collect.Lists; -import java.util.List; + +import java.util.ArrayList; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.utils.Bytes; import org.apache.skywalking.apm.network.common.v3.CPU; @@ -31,12 +32,6 @@ import org.apache.skywalking.apm.network.language.agent.v3.MemoryPool; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.config.NamingControl; import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping; -import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMCPU; -import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMGC; -import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMMemory; -import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMMemoryPool; -import org.apache.skywalking.oap.server.core.source.Source; -import org.apache.skywalking.oap.server.core.source.SourceReceiver; import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.analyzer.agent.kafka.module.KafkaFetcherConfig; import org.apache.skywalking.oap.server.analyzer.agent.kafka.mock.MockModuleManager; @@ -46,11 +41,8 @@ import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator; import org.apache.skywalking.oap.server.telemetry.none.MetricsCreatorNoop; import org.junit.Assert; import org.junit.Before; -import org.junit.ClassRule; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; - public class JVMMetricsHandlerTest { private static final String TOPIC_NAME = "skywalking-metrics"; private JVMMetricsHandler handler = null; @@ -58,20 +50,6 @@ public class JVMMetricsHandlerTest { private ModuleManager manager; - @ClassRule - public static SourceReceiverRule SOURCE_RECEIVER = new SourceReceiverRule() { - - @Override - protected void verify(final List sourceList) throws Throwable { - Assert.assertTrue(sourceList.get(0) instanceof ServiceInstanceJVMCPU); - ServiceInstanceJVMCPU serviceInstanceJVMCPU = (ServiceInstanceJVMCPU) sourceList.get(0); - Assert.assertThat(serviceInstanceJVMCPU.getUsePercent(), is(1.0)); - Assert.assertTrue(sourceList.get(1) instanceof ServiceInstanceJVMMemory); - Assert.assertTrue(sourceList.get(2) instanceof ServiceInstanceJVMMemoryPool); - Assert.assertTrue(sourceList.get(3) instanceof ServiceInstanceJVMGC); - } - }; - @Before public void setup() { manager = new MockModuleManager() { @@ -82,7 +60,6 @@ public class JVMMetricsHandlerTest { protected void register() { registerServiceImplementation(NamingControl.class, new NamingControl( 512, 512, 512, new EndpointNameGrouping())); - registerServiceImplementation(SourceReceiver.class, SOURCE_RECEIVER); } }); register(TelemetryModule.NAME, () -> new MockModuleProvider() { @@ -93,7 +70,7 @@ public class JVMMetricsHandlerTest { }); } }; - handler = new JVMMetricsHandler(manager, config); + handler = new JVMMetricsHandler(manager, config, new ArrayList<>()); } @Test diff --git a/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/JVMModuleProvider.java b/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/JVMModuleProvider.java index 202f9e8be..a55159b75 100644 --- a/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/JVMModuleProvider.java +++ b/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/JVMModuleProvider.java @@ -18,8 +18,9 @@ package org.apache.skywalking.oap.server.receiver.jvm.provider; +import org.apache.skywalking.oap.meter.analyzer.prometheus.rule.Rule; +import org.apache.skywalking.oap.meter.analyzer.prometheus.rule.Rules; import org.apache.skywalking.oap.server.core.CoreModule; -import org.apache.skywalking.oap.server.core.oal.rt.OALEngineLoaderService; import org.apache.skywalking.oap.server.core.server.GRPCHandlerRegister; import org.apache.skywalking.oap.server.library.module.ModuleConfig; import org.apache.skywalking.oap.server.library.module.ModuleDefine; @@ -30,6 +31,9 @@ import org.apache.skywalking.oap.server.receiver.jvm.provider.handler.JVMMetricR import org.apache.skywalking.oap.server.receiver.jvm.provider.handler.JVMMetricReportServiceHandlerCompat; import org.apache.skywalking.oap.server.receiver.sharing.server.SharingServerModule; +import java.util.Collections; +import java.util.List; + public class JVMModuleProvider extends ModuleProvider { @Override @@ -53,16 +57,12 @@ public class JVMModuleProvider extends ModuleProvider { @Override public void start() throws ModuleStartException { - // load official analysis - getManager().find(CoreModule.NAME) - .provider() - .getService(OALEngineLoaderService.class) - .load(JVMOALDefine.INSTANCE); + List rules = Rules.loadRules("jvm-metrics-rules", Collections.singletonList("jvm")); + JVMMetricReportServiceHandler jvmMetricReportServiceHandler = new JVMMetricReportServiceHandler(getManager(), rules); GRPCHandlerRegister grpcHandlerRegister = getManager().find(SharingServerModule.NAME) .provider() .getService(GRPCHandlerRegister.class); - JVMMetricReportServiceHandler jvmMetricReportServiceHandler = new JVMMetricReportServiceHandler(getManager()); grpcHandlerRegister.addHandler(jvmMetricReportServiceHandler); grpcHandlerRegister.addHandler(new JVMMetricReportServiceHandlerCompat(jvmMetricReportServiceHandler)); } diff --git a/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/JVMOALDefine.java b/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/JVMOALDefine.java deleted file mode 100644 index 0ff62ae3b..000000000 --- a/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/JVMOALDefine.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.server.receiver.jvm.provider; - -import org.apache.skywalking.oap.server.core.oal.rt.OALDefine; - -/** - * JVM OAl script includes the metrics related to JVM only. - */ -public class JVMOALDefine extends OALDefine { - public static final JVMOALDefine INSTANCE = new JVMOALDefine(); - - private JVMOALDefine() { - super( - "oal/java-agent.oal", - "org.apache.skywalking.oap.server.core.source" - ); - } -} diff --git a/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/handler/JVMMetricReportServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/handler/JVMMetricReportServiceHandler.java index f39d0057b..8ba14bb5d 100644 --- a/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/handler/JVMMetricReportServiceHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/handler/JVMMetricReportServiceHandler.java @@ -23,19 +23,23 @@ import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.apm.network.common.v3.Commands; import org.apache.skywalking.apm.network.language.agent.v3.JVMMetricCollection; import org.apache.skywalking.apm.network.language.agent.v3.JVMMetricReportServiceGrpc; +import org.apache.skywalking.oap.meter.analyzer.prometheus.rule.Rule; import org.apache.skywalking.oap.server.analyzer.provider.jvm.JVMSourceDispatcher; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.config.NamingControl; import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.library.server.grpc.GRPCHandler; +import java.util.List; + @Slf4j public class JVMMetricReportServiceHandler extends JVMMetricReportServiceGrpc.JVMMetricReportServiceImplBase implements GRPCHandler { - private final JVMSourceDispatcher jvmSourceDispatcher; private final NamingControl namingControl; - public JVMMetricReportServiceHandler(ModuleManager moduleManager) { - this.jvmSourceDispatcher = new JVMSourceDispatcher(moduleManager); + private final JVMSourceDispatcher jvmSourceDispatcher; + + public JVMMetricReportServiceHandler(ModuleManager moduleManager, List rules) { + this.jvmSourceDispatcher = new JVMSourceDispatcher(moduleManager, rules); this.namingControl = moduleManager.find(CoreModule.NAME) .provider() .getService(NamingControl.class); @@ -61,5 +65,4 @@ public class JVMMetricReportServiceHandler extends JVMMetricReportServiceGrpc.JV responseObserver.onNext(Commands.newBuilder().build()); responseObserver.onCompleted(); } - } diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java index e50d2f65f..7117760df 100644 --- a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java @@ -69,13 +69,19 @@ public class MetricsQuery extends AbstractQuery { SERVICE_INSTANCE_SLA }; + public static String INSTANCE_JVM_MEMORY_HEAP = "instance_jvm_memory_heap"; + public static String INSTANCE_JVM_MEMORY_HEAP_MAX = "instance_jvm_memory_heap_max"; + public static String INSTANCE_JVM_MEMORY_NOHEAP = "instance_jvm_memory_noheap"; public static String INSTANCE_JVM_THREAD_LIVE_COUNT = "instance_jvm_thread_live_count"; public static String INSTANCE_JVM_THREAD_DAEMON_COUNT = "instance_jvm_thread_daemon_count"; public static String INSTANCE_JVM_THREAD_PEAK_COUNT = "instance_jvm_thread_peak_count"; public static String [] ALL_INSTANCE_JVM_METRICS = { + INSTANCE_JVM_THREAD_LIVE_COUNT, INSTANCE_JVM_THREAD_DAEMON_COUNT, - INSTANCE_JVM_THREAD_DAEMON_COUNT, - INSTANCE_JVM_THREAD_PEAK_COUNT + INSTANCE_JVM_THREAD_PEAK_COUNT, + INSTANCE_JVM_MEMORY_NOHEAP, + INSTANCE_JVM_MEMORY_HEAP_MAX, + INSTANCE_JVM_MEMORY_HEAP, }; public static String SERVICE_RELATION_CLIENT_CPM = "service_relation_client_cpm"; diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java index 09ab8461a..0a8065507 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java @@ -246,11 +246,11 @@ public class SimpleE2E extends SkyWalkingTestAdapter { LOGGER.info("instance jvm metrics: {}", instanceJVMMetrics); - final AtLeastOneOfMetricsMatcher instanceThreadMatcher = new AtLeastOneOfMetricsMatcher(); + final AtLeastOneOfMetricsMatcher instanceMatcher = new AtLeastOneOfMetricsMatcher(); final MetricsValueMatcher greaterThanZero = new MetricsValueMatcher(); greaterThanZero.setValue("gt 0"); - instanceThreadMatcher.setValue(greaterThanZero); - instanceThreadMatcher.verify(instanceJVMMetrics); + instanceMatcher.setValue(greaterThanZero); + instanceMatcher.verify(instanceJVMMetrics); LOGGER.info("{}: {}", metricsName, instanceJVMMetrics); } }