From 1877d0fcf41a320dc236c173c18b44f260e3c6f5 Mon Sep 17 00:00:00 2001 From: Stalary <452024236@qq.com> Date: Wed, 22 Jul 2020 09:05:32 +0800 Subject: [PATCH] Add jvm_thread metrics (#5129) --- apm-protocol/apm-network/src/main/proto | 2 +- .../apm/agent/core/jvm/JVMService.java | 2 + .../agent/core/jvm/thread/ThreadProvider.java | 42 ++++++++++++ .../concepts-and-designs/scope-definitions.md | 11 ++++ .../skywalking/oal/rt/grammar/OALLexer.g4 | 1 + .../skywalking/oal/rt/grammar/OALParser.g4 | 2 +- .../oal-rt/src/test/resources/oal_test.oal | 3 + .../src/main/resources/oal/java-agent.oal | 5 +- .../resources/ui-initialized-templates.yml | 11 ++++ .../core/source/DefaultScopeDefine.java | 1 + .../core/source/ServiceInstanceJVMThread.java | 64 +++++++++++++++++++ .../provider/handler/JVMSourceDispatcher.java | 22 +++++++ .../skywalking/e2e/metrics/MetricsQuery.java | 9 +++ .../skywalking/e2e/simple/SimpleE2E.java | 23 +++++++ 14 files changed, 195 insertions(+), 3 deletions(-) create mode 100644 apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/jvm/thread/ThreadProvider.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMThread.java diff --git a/apm-protocol/apm-network/src/main/proto b/apm-protocol/apm-network/src/main/proto index 24788cf41..61550492d 160000 --- a/apm-protocol/apm-network/src/main/proto +++ b/apm-protocol/apm-network/src/main/proto @@ -1 +1 @@ -Subproject commit 24788cf41807048dcbe8dba0958688aaaf630d73 +Subproject commit 61550492d82de2bfd71c6f43cb6b7d3251fa9d76 diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/jvm/JVMService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/jvm/JVMService.java index 2a1e81461..02b116c49 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/jvm/JVMService.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/jvm/JVMService.java @@ -34,6 +34,7 @@ import org.apache.skywalking.apm.agent.core.jvm.cpu.CPUProvider; import org.apache.skywalking.apm.agent.core.jvm.gc.GCProvider; import org.apache.skywalking.apm.agent.core.jvm.memory.MemoryProvider; import org.apache.skywalking.apm.agent.core.jvm.memorypool.MemoryPoolProvider; +import org.apache.skywalking.apm.agent.core.jvm.thread.ThreadProvider; import org.apache.skywalking.apm.agent.core.logging.api.ILog; import org.apache.skywalking.apm.agent.core.logging.api.LogManager; import org.apache.skywalking.apm.agent.core.remote.GRPCChannelListener; @@ -121,6 +122,7 @@ public class JVMService implements BootService, Runnable { jvmBuilder.addAllMemory(MemoryProvider.INSTANCE.getMemoryMetricList()); jvmBuilder.addAllMemoryPool(MemoryPoolProvider.INSTANCE.getMemoryPoolMetricsList()); jvmBuilder.addAllGc(GCProvider.INSTANCE.getGCList()); + jvmBuilder.setThread(ThreadProvider.INSTANCE.getThreadMetrics()); JVMMetric jvmMetric = jvmBuilder.build(); if (!queue.offer(jvmMetric)) { diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/jvm/thread/ThreadProvider.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/jvm/thread/ThreadProvider.java new file mode 100644 index 000000000..25243f27d --- /dev/null +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/jvm/thread/ThreadProvider.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.agent.core.jvm.thread; + +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadMXBean; +import org.apache.skywalking.apm.network.language.agent.v3.Thread; + +public enum ThreadProvider { + INSTANCE; + private final ThreadMXBean threadMXBean; + + ThreadProvider() { + this.threadMXBean = ManagementFactory.getThreadMXBean(); + } + + public Thread getThreadMetrics() { + int threadCount = threadMXBean.getThreadCount(); + int daemonThreadCount = threadMXBean.getDaemonThreadCount(); + int peakThreadCount = threadMXBean.getPeakThreadCount(); + return Thread.newBuilder().setLiveCount(threadCount) + .setDaemonCount(daemonThreadCount) + .setPeakCount(peakThreadCount).build(); + } + +} diff --git a/docs/en/concepts-and-designs/scope-definitions.md b/docs/en/concepts-and-designs/scope-definitions.md index db8925860..0a368aa73 100644 --- a/docs/en/concepts-and-designs/scope-definitions.md +++ b/docs/en/concepts-and-designs/scope-definitions.md @@ -93,6 +93,17 @@ Calculate the metrics data if the service instance is a JVM and collected by jav | time | GC time cost | | long | | count | Count of GC op | | long | +5. SCOPE `ServiceInstanceJVMThread` + +| Name | Remarks | Group Key | Type | +|---|---|---|---| +| id | Represent the unique id of the service instance, usually a number. | yes | int | +| name | Represent the name of the service instance. Such as `ip:port@Service Name`. **Notice**: current native agent uses `processId@Service name` as instance name, which is useless when you want to setup a filter in aggregation. | | string| +| serviceName | Represent the name of the service. | | string | +| liveCount | Represent Current number of live threads | | int | +| daemonCount | Represent Current number of daemon threads | | int | +| peakCount | Represent Current number of peak threads | | int | + ### SCOPE `Endpoint` Calculate the metrics data from each request of the endpoint in the service. diff --git a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 index 03ff22a4c..8ed3398bd 100644 --- a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 +++ b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 @@ -38,6 +38,7 @@ SRC_SERVICE_INSTANCE_JVM_CPU: 'ServiceInstanceJVMCPU'; SRC_SERVICE_INSTANCE_JVM_MEMORY: 'ServiceInstanceJVMMemory'; SRC_SERVICE_INSTANCE_JVM_MEMORY_POOL: 'ServiceInstanceJVMMemoryPool'; SRC_SERVICE_INSTANCE_JVM_GC: 'ServiceInstanceJVMGC'; +SRC_SERVICE_INSTANCE_JVM_THREAD: 'ServiceInstanceJVMThread'; SRC_DATABASE_ACCESS: 'DatabaseAccess'; SRC_SERVICE_INSTANCE_CLR_CPU: 'ServiceInstanceCLRCPU'; SRC_SERVICE_INSTANCE_CLR_GC: 'ServiceInstanceCLRGC'; diff --git a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 index 75a2a2de7..46818ee7f 100644 --- a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 +++ b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 @@ -52,7 +52,7 @@ filterExpression source : SRC_ALL | SRC_SERVICE | SRC_DATABASE_ACCESS | SRC_SERVICE_INSTANCE | SRC_ENDPOINT | SRC_SERVICE_RELATION | SRC_SERVICE_INSTANCE_RELATION | SRC_ENDPOINT_RELATION | - SRC_SERVICE_INSTANCE_JVM_CPU | SRC_SERVICE_INSTANCE_JVM_MEMORY | SRC_SERVICE_INSTANCE_JVM_MEMORY_POOL | SRC_SERVICE_INSTANCE_JVM_GC |// JVM source of service instance + SRC_SERVICE_INSTANCE_JVM_CPU | SRC_SERVICE_INSTANCE_JVM_MEMORY | SRC_SERVICE_INSTANCE_JVM_MEMORY_POOL | SRC_SERVICE_INSTANCE_JVM_GC | SRC_SERVICE_INSTANCE_JVM_THREAD |// JVM source of service instance SRC_SERVICE_INSTANCE_CLR_CPU | SRC_SERVICE_INSTANCE_CLR_GC | SRC_SERVICE_INSTANCE_CLR_THREAD | SRC_ENVOY_INSTANCE_METRIC ; diff --git a/oap-server/oal-rt/src/test/resources/oal_test.oal b/oap-server/oal-rt/src/test/resources/oal_test.oal index a0613c91d..a7f7deeef 100644 --- a/oap-server/oal-rt/src/test/resources/oal_test.oal +++ b/oap-server/oal-rt/src/test/resources/oal_test.oal @@ -71,5 +71,8 @@ instance_jvm_young_gc_time = from(ServiceInstanceJVMGC.time).filter(phrase == GC instance_jvm_old_gc_time = from(ServiceInstanceJVMGC.time).filter(phrase == GCPhrase.OLD).longAvg(); instance_jvm_young_gc_count = from(ServiceInstanceJVMGC.count).filter(phrase == GCPhrase.NEW).sum(); instance_jvm_old_gc_count = from(ServiceInstanceJVMGC.count).filter(phrase == GCPhrase.OLD).sum(); +instance_jvm_thread_live_count = from(ServiceInstanceJVMThread.liveCount).longAvg(); +instance_jvm_thread_daemon_count = from(ServiceInstanceJVMThread.daemonCount).longAvg(); +instance_jvm_thread_peak_count = from(ServiceInstanceJVMThread.peakCount).longAvg(); // endpoint_Avg_for_prod_serv = from(Endpoint.latency).filter(name == "/product/service").longAvg(); \ No newline at end of file diff --git a/oap-server/server-bootstrap/src/main/resources/oal/java-agent.oal b/oap-server/server-bootstrap/src/main/resources/oal/java-agent.oal index f2e7aedca..ca344605f 100644 --- a/oap-server/server-bootstrap/src/main/resources/oal/java-agent.oal +++ b/oap-server/server-bootstrap/src/main/resources/oal/java-agent.oal @@ -25,4 +25,7 @@ instance_jvm_memory_noheap_max = from(ServiceInstanceJVMMemory.max).filter(heapS instance_jvm_young_gc_time = from(ServiceInstanceJVMGC.time).filter(phrase == GCPhrase.NEW).sum(); instance_jvm_old_gc_time = from(ServiceInstanceJVMGC.time).filter(phrase == GCPhrase.OLD).sum(); instance_jvm_young_gc_count = from(ServiceInstanceJVMGC.count).filter(phrase == GCPhrase.NEW).sum(); -instance_jvm_old_gc_count = from(ServiceInstanceJVMGC.count).filter(phrase == GCPhrase.OLD).sum(); \ No newline at end of file +instance_jvm_old_gc_count = from(ServiceInstanceJVMGC.count).filter(phrase == GCPhrase.OLD).sum(); +instance_jvm_thread_live_count = from(ServiceInstanceJVMThread.liveCount).longAvg(); +instance_jvm_thread_daemon_count = from(ServiceInstanceJVMThread.daemonCount).longAvg(); +instance_jvm_thread_peak_count = from(ServiceInstanceJVMThread.peakCount).longAvg(); \ No newline at end of file diff --git a/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates.yml b/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates.yml index 0a2dcfd83..328c5ced5 100644 --- a/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates.yml +++ b/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates.yml @@ -359,6 +359,17 @@ templates: "chartType": "ChartBar", "metricName": "instance_jvm_young_gc_count, instance_jvm_old_gc_count" }, + { + "width": 3, + "title": "JVM Thread Count (Java Service)", + "height": "250", + "entityType": "ServiceInstance", + "independentSelector": false, + "metricType": "REGULAR_VALUE", + "queryMetricType": "readMetricsValues", + "chartType": "ChartLine", + "metricName": "instance_jvm_thread_live_count, instance_jvm_thread_daemon_count, instance_jvm_thread_peak_count" + }, { "width": 3, "title": "CLR CPU (.NET Service)", diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java index 4883446c7..e753c629d 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java @@ -67,6 +67,7 @@ public class DefaultScopeDefine { public static final int SERVICE_INSTANCE_UPDATE = 30; public static final int NETWORK_ADDRESS_ALIAS = 31; public static final int UI_TEMPLATE = 32; + public static final int SERVICE_INSTANCE_JVM_THREAD = 33; /** * Catalog of scope, the metrics processor could use this to group all generated metrics by oal rt. diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMThread.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMThread.java new file mode 100644 index 000000000..42ee6a2e2 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMThread.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.source; + +import lombok.Getter; +import lombok.Setter; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_INSTANCE_CATALOG_NAME; +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_INSTANCE_JVM_THREAD; + +@ScopeDeclaration(id = SERVICE_INSTANCE_JVM_THREAD, name = "ServiceInstanceJVMThread", catalog = SERVICE_INSTANCE_CATALOG_NAME) +@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class) +public class ServiceInstanceJVMThread extends Source { + @Override + public int scope() { + return SERVICE_INSTANCE_JVM_THREAD; + } + + @Override + public String getEntityId() { + return String.valueOf(id); + } + + @Getter + @Setter + private String id; + @Getter + @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) + private String name; + @Getter + @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) + private String serviceName; + @Getter + @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "service_id") + private String serviceId; + @Getter + @Setter + private long liveCount; + @Getter + @Setter + private long daemonCount; + @Getter + @Setter + private long peakCount; +} diff --git a/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/handler/JVMSourceDispatcher.java b/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/handler/JVMSourceDispatcher.java index f22e11301..e847a14cc 100644 --- a/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/handler/JVMSourceDispatcher.java +++ b/oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/handler/JVMSourceDispatcher.java @@ -24,6 +24,7 @@ import org.apache.skywalking.apm.network.language.agent.v3.GC; import org.apache.skywalking.apm.network.language.agent.v3.JVMMetric; import org.apache.skywalking.apm.network.language.agent.v3.Memory; import org.apache.skywalking.apm.network.language.agent.v3.MemoryPool; +import org.apache.skywalking.apm.network.language.agent.v3.Thread; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.analysis.IDManager; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; @@ -34,6 +35,7 @@ import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMCPU; import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMGC; import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMMemory; import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMMemoryPool; +import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMThread; import org.apache.skywalking.oap.server.core.source.SourceReceiver; import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.slf4j.Logger; @@ -61,6 +63,8 @@ public class JVMSourceDispatcher { service, serviceId, serviceInstance, serviceInstanceId, minuteTimeBucket, metrics.getMemoryPoolList()); this.sendToGCMetricProcess( service, serviceId, serviceInstance, serviceInstanceId, minuteTimeBucket, metrics.getGcList()); + this.sendToThreadMetricProcess( + service, serviceId, serviceInstance, serviceInstanceId, minuteTimeBucket, metrics.getThread()); } private void sendToCpuMetricProcess(String service, @@ -173,4 +177,22 @@ public class JVMSourceDispatcher { sourceReceiver.receive(serviceInstanceJVMMemoryPool); }); } + + private void sendToThreadMetricProcess(String service, + String serviceId, + String serviceInstance, + String serviceInstanceId, + long timeBucket, + Thread thread) { + ServiceInstanceJVMThread serviceInstanceJVMThread = new ServiceInstanceJVMThread(); + serviceInstanceJVMThread.setId(serviceInstanceId); + serviceInstanceJVMThread.setName(service); + serviceInstanceJVMThread.setServiceId(serviceId); + serviceInstanceJVMThread.setServiceName(serviceInstance); + serviceInstanceJVMThread.setLiveCount(thread.getLiveCount()); + serviceInstanceJVMThread.setDaemonCount(thread.getDaemonCount()); + serviceInstanceJVMThread.setPeakCount(thread.getPeakCount()); + serviceInstanceJVMThread.setTimeBucket(timeBucket); + sourceReceiver.receive(serviceInstanceJVMThread); + } } diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java index 97f8e275b..4b86ac02b 100644 --- a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java @@ -64,6 +64,15 @@ public class MetricsQuery extends AbstractQuery { SERVICE_INSTANCE_SLA }; + public static String INSTANCE_JVM_THREAD_LIVE_COUNT = "instance_jvm_thread_live_count"; + public static String INSTANCE_JVM_THREAD_DAEMON_COUNT = "instance_jvm_thread_daemon_count"; + public static String INSTANCE_JVM_THREAD_PEAK_COUNT = "instance_jvm_thread_peak_count"; + public static String [] ALL_INSTANCE_JVM_METRICS = { + INSTANCE_JVM_THREAD_DAEMON_COUNT, + INSTANCE_JVM_THREAD_DAEMON_COUNT, + INSTANCE_JVM_THREAD_PEAK_COUNT + }; + public static String SERVICE_RELATION_CLIENT_CPM = "service_relation_client_cpm"; public static String SERVICE_RELATION_SERVER_CPM = "service_relation_server_cpm"; public static String SERVICE_RELATION_CLIENT_CALL_SLA = "service_relation_client_call_sla"; diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java index 67af38853..9a56df2da 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java @@ -62,6 +62,7 @@ import org.testcontainers.containers.DockerComposeContainer; import static org.apache.skywalking.e2e.metrics.MetricsMatcher.verifyMetrics; import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_ENDPOINT_METRICS; import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_INSTANCE_METRICS; +import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_INSTANCE_JVM_METRICS; import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_INSTANCE_RELATION_CLIENT_METRICS; import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_INSTANCE_RELATION_SERVER_METRICS; import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_METRICS; @@ -151,6 +152,8 @@ public class SimpleE2E extends SkyWalkingTestAdapter { verifyInstancesMetrics(instances); + verifyInstancesJVMMetrics(instances); + final Endpoints endpoints = verifyServiceEndpoints(service); verifyEndpointsMetrics(endpoints); @@ -287,6 +290,26 @@ public class SimpleE2E extends SkyWalkingTestAdapter { } } + private void verifyInstancesJVMMetrics(final Instances instances) throws Exception { + for (Instance instance : instances.getInstances()) { + for (String metricsName : ALL_INSTANCE_JVM_METRICS) { + LOGGER.info("verifying service instance response time: {}", instance); + final Metrics instanceJVMMetrics = graphql.metrics( + new MetricsQuery().stepByMinute().metricsName(metricsName).id(instance.getKey()) + ); + + LOGGER.info("instance jvm metrics: {}", instanceJVMMetrics); + + final AtLeastOneOfMetricsMatcher instanceThreadMatcher = new AtLeastOneOfMetricsMatcher(); + final MetricsValueMatcher greaterThanZero = new MetricsValueMatcher(); + greaterThanZero.setValue("gt 0"); + instanceThreadMatcher.setValue(greaterThanZero); + instanceThreadMatcher.verify(instanceJVMMetrics); + LOGGER.info("{}: {}", metricsName, instanceJVMMetrics); + } + } + } + private void verifyEndpointsMetrics(final Endpoints endpoints) throws Exception { for (Endpoint endpoint : endpoints.getEndpoints()) { if (!endpoint.getLabel().equals("/users")) {