kazusa
/
skywalking-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add jvm_thread metrics (#5129)

This commit is contained in:
Stalary 2020-07-22 09:05:32 +08:00 committed by GitHub
parent b363b0ea4f
commit 1877d0fcf4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 195 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apm-protocol/apm-network/src/main/proto

@ -1 +1 @@
Subproject commit 24788cf41807048dcbe8dba0958688aaaf630d73
Subproject commit 61550492d82de2bfd71c6f43cb6b7d3251fa9d76

2
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/jvm/JVMService.java
View File

@ -34,6 +34,7 @@ import org.apache.skywalking.apm.agent.core.jvm.cpu.CPUProvider;
import org.apache.skywalking.apm.agent.core.jvm.gc.GCProvider;
import org.apache.skywalking.apm.agent.core.jvm.memory.MemoryProvider;
import org.apache.skywalking.apm.agent.core.jvm.memorypool.MemoryPoolProvider;
import org.apache.skywalking.apm.agent.core.jvm.thread.ThreadProvider;
import org.apache.skywalking.apm.agent.core.logging.api.ILog;
import org.apache.skywalking.apm.agent.core.logging.api.LogManager;
import org.apache.skywalking.apm.agent.core.remote.GRPCChannelListener;
@ -121,6 +122,7 @@ public class JVMService implements BootService, Runnable {
jvmBuilder.addAllMemory(MemoryProvider.INSTANCE.getMemoryMetricList());
jvmBuilder.addAllMemoryPool(MemoryPoolProvider.INSTANCE.getMemoryPoolMetricsList());
jvmBuilder.addAllGc(GCProvider.INSTANCE.getGCList());
jvmBuilder.setThread(ThreadProvider.INSTANCE.getThreadMetrics());
JVMMetric jvmMetric = jvmBuilder.build();
if (!queue.offer(jvmMetric)) {

42
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/jvm/thread/ThreadProvider.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.apm.agent.core.jvm.thread;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadMXBean;
import org.apache.skywalking.apm.network.language.agent.v3.Thread;
public enum ThreadProvider {
INSTANCE;
private final ThreadMXBean threadMXBean;
ThreadProvider() {
this.threadMXBean = ManagementFactory.getThreadMXBean();
}
public Thread getThreadMetrics() {
int threadCount = threadMXBean.getThreadCount();
int daemonThreadCount = threadMXBean.getDaemonThreadCount();
int peakThreadCount = threadMXBean.getPeakThreadCount();
return Thread.newBuilder().setLiveCount(threadCount)
.setDaemonCount(daemonThreadCount)
.setPeakCount(peakThreadCount).build();
}
}

11
docs/en/concepts-and-designs/scope-definitions.md
View File

@ -93,6 +93,17 @@ Calculate the metrics data if the service instance is a JVM and collected by jav
| time | GC time cost | | long |
| count | Count of GC op | | long |
5. SCOPE `ServiceInstanceJVMThread`
| Name | Remarks | Group Key | Type |
|---|---|---|---|
| id | Represent the unique id of the service instance, usually a number. | yes | int |
| name | Represent the name of the service instance. Such as `ip:port@Service Name`. **Notice**: current native agent uses `processId@Service name` as instance name, which is useless when you want to setup a filter in aggregation. | | string|
| serviceName | Represent the name of the service. | | string |
| liveCount | Represent Current number of live threads | | int |
| daemonCount | Represent Current number of daemon threads | | int |
| peakCount | Represent Current number of peak threads | | int |
### SCOPE `Endpoint`
Calculate the metrics data from each request of the endpoint in the service.

1
oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4
View File

@ -38,6 +38,7 @@ SRC_SERVICE_INSTANCE_JVM_CPU: 'ServiceInstanceJVMCPU';
SRC_SERVICE_INSTANCE_JVM_MEMORY: 'ServiceInstanceJVMMemory';
SRC_SERVICE_INSTANCE_JVM_MEMORY_POOL: 'ServiceInstanceJVMMemoryPool';
SRC_SERVICE_INSTANCE_JVM_GC: 'ServiceInstanceJVMGC';
SRC_SERVICE_INSTANCE_JVM_THREAD: 'ServiceInstanceJVMThread';
SRC_DATABASE_ACCESS: 'DatabaseAccess';
SRC_SERVICE_INSTANCE_CLR_CPU: 'ServiceInstanceCLRCPU';
SRC_SERVICE_INSTANCE_CLR_GC: 'ServiceInstanceCLRGC';

2
oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4
View File

@ -52,7 +52,7 @@ filterExpression
source
: SRC_ALL | SRC_SERVICE | SRC_DATABASE_ACCESS | SRC_SERVICE_INSTANCE | SRC_ENDPOINT |
SRC_SERVICE_RELATION | SRC_SERVICE_INSTANCE_RELATION | SRC_ENDPOINT_RELATION |
SRC_SERVICE_INSTANCE_JVM_CPU | SRC_SERVICE_INSTANCE_JVM_MEMORY | SRC_SERVICE_INSTANCE_JVM_MEMORY_POOL | SRC_SERVICE_INSTANCE_JVM_GC |// JVM source of service instance
SRC_SERVICE_INSTANCE_JVM_CPU | SRC_SERVICE_INSTANCE_JVM_MEMORY | SRC_SERVICE_INSTANCE_JVM_MEMORY_POOL | SRC_SERVICE_INSTANCE_JVM_GC | SRC_SERVICE_INSTANCE_JVM_THREAD |// JVM source of service instance
SRC_SERVICE_INSTANCE_CLR_CPU | SRC_SERVICE_INSTANCE_CLR_GC | SRC_SERVICE_INSTANCE_CLR_THREAD |
SRC_ENVOY_INSTANCE_METRIC
;

3
oap-server/oal-rt/src/test/resources/oal_test.oal
View File

@ -71,5 +71,8 @@ instance_jvm_young_gc_time = from(ServiceInstanceJVMGC.time).filter(phrase == GC
instance_jvm_old_gc_time = from(ServiceInstanceJVMGC.time).filter(phrase == GCPhrase.OLD).longAvg();
instance_jvm_young_gc_count = from(ServiceInstanceJVMGC.count).filter(phrase == GCPhrase.NEW).sum();
instance_jvm_old_gc_count = from(ServiceInstanceJVMGC.count).filter(phrase == GCPhrase.OLD).sum();
instance_jvm_thread_live_count = from(ServiceInstanceJVMThread.liveCount).longAvg();
instance_jvm_thread_daemon_count = from(ServiceInstanceJVMThread.daemonCount).longAvg();
instance_jvm_thread_peak_count = from(ServiceInstanceJVMThread.peakCount).longAvg();
// endpoint_Avg_for_prod_serv = from(Endpoint.latency).filter(name == "/product/service").longAvg();

5
oap-server/server-bootstrap/src/main/resources/oal/java-agent.oal
View File

@ -25,4 +25,7 @@ instance_jvm_memory_noheap_max = from(ServiceInstanceJVMMemory.max).filter(heapS
instance_jvm_young_gc_time = from(ServiceInstanceJVMGC.time).filter(phrase == GCPhrase.NEW).sum();
instance_jvm_old_gc_time = from(ServiceInstanceJVMGC.time).filter(phrase == GCPhrase.OLD).sum();
instance_jvm_young_gc_count = from(ServiceInstanceJVMGC.count).filter(phrase == GCPhrase.NEW).sum();
instance_jvm_old_gc_count = from(ServiceInstanceJVMGC.count).filter(phrase == GCPhrase.OLD).sum();
instance_jvm_old_gc_count = from(ServiceInstanceJVMGC.count).filter(phrase == GCPhrase.OLD).sum();
instance_jvm_thread_live_count = from(ServiceInstanceJVMThread.liveCount).longAvg();
instance_jvm_thread_daemon_count = from(ServiceInstanceJVMThread.daemonCount).longAvg();
instance_jvm_thread_peak_count = from(ServiceInstanceJVMThread.peakCount).longAvg();

11
oap-server/server-bootstrap/src/main/resources/ui-initialized-templates.yml
View File

@ -359,6 +359,17 @@ templates:
"chartType": "ChartBar",
"metricName": "instance_jvm_young_gc_count, instance_jvm_old_gc_count"
},
{
"width": 3,
"title": "JVM Thread Count (Java Service)",
"height": "250",
"entityType": "ServiceInstance",
"independentSelector": false,
"metricType": "REGULAR_VALUE",
"queryMetricType": "readMetricsValues",
"chartType": "ChartLine",
"metricName": "instance_jvm_thread_live_count, instance_jvm_thread_daemon_count, instance_jvm_thread_peak_count"
},
{
"width": 3,
"title": "CLR CPU (.NET Service)",

1
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java
View File

@ -67,6 +67,7 @@ public class DefaultScopeDefine {
public static final int SERVICE_INSTANCE_UPDATE = 30;
public static final int NETWORK_ADDRESS_ALIAS = 31;
public static final int UI_TEMPLATE = 32;
public static final int SERVICE_INSTANCE_JVM_THREAD = 33;
/**
* Catalog of scope, the metrics processor could use this to group all generated metrics by oal rt.

64
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMThread.java Normal file
View File

@ -0,0 +1,64 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.source;
import lombok.Getter;
import lombok.Setter;
import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_INSTANCE_CATALOG_NAME;
import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_INSTANCE_JVM_THREAD;
@ScopeDeclaration(id = SERVICE_INSTANCE_JVM_THREAD, name = "ServiceInstanceJVMThread", catalog = SERVICE_INSTANCE_CATALOG_NAME)
@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class)
public class ServiceInstanceJVMThread extends Source {
@Override
public int scope() {
return SERVICE_INSTANCE_JVM_THREAD;
}
@Override
public String getEntityId() {
return String.valueOf(id);
}
@Getter
@Setter
private String id;
@Getter
@Setter
@ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
private String name;
@Getter
@Setter
@ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true)
private String serviceName;
@Getter
@Setter
@ScopeDefaultColumn.DefinedByField(columnName = "service_id")
private String serviceId;
@Getter
@Setter
private long liveCount;
@Getter
@Setter
private long daemonCount;
@Getter
@Setter
private long peakCount;
}

22
oap-server/server-receiver-plugin/skywalking-jvm-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/jvm/provider/handler/JVMSourceDispatcher.java
View File

@ -24,6 +24,7 @@ import org.apache.skywalking.apm.network.language.agent.v3.GC;
import org.apache.skywalking.apm.network.language.agent.v3.JVMMetric;
import org.apache.skywalking.apm.network.language.agent.v3.Memory;
import org.apache.skywalking.apm.network.language.agent.v3.MemoryPool;
import org.apache.skywalking.apm.network.language.agent.v3.Thread;
import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.analysis.IDManager;
import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
@ -34,6 +35,7 @@ import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMCPU;
import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMGC;
import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMMemory;
import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMMemoryPool;
import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMThread;
import org.apache.skywalking.oap.server.core.source.SourceReceiver;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
import org.slf4j.Logger;
@ -61,6 +63,8 @@ public class JVMSourceDispatcher {
service, serviceId, serviceInstance, serviceInstanceId, minuteTimeBucket, metrics.getMemoryPoolList());
this.sendToGCMetricProcess(
service, serviceId, serviceInstance, serviceInstanceId, minuteTimeBucket, metrics.getGcList());
this.sendToThreadMetricProcess(
service, serviceId, serviceInstance, serviceInstanceId, minuteTimeBucket, metrics.getThread());
}
private void sendToCpuMetricProcess(String service,
@ -173,4 +177,22 @@ public class JVMSourceDispatcher {
sourceReceiver.receive(serviceInstanceJVMMemoryPool);
});
}
private void sendToThreadMetricProcess(String service,
String serviceId,
String serviceInstance,
String serviceInstanceId,
long timeBucket,
Thread thread) {
ServiceInstanceJVMThread serviceInstanceJVMThread = new ServiceInstanceJVMThread();
serviceInstanceJVMThread.setId(serviceInstanceId);
serviceInstanceJVMThread.setName(service);
serviceInstanceJVMThread.setServiceId(serviceId);
serviceInstanceJVMThread.setServiceName(serviceInstance);
serviceInstanceJVMThread.setLiveCount(thread.getLiveCount());
serviceInstanceJVMThread.setDaemonCount(thread.getDaemonCount());
serviceInstanceJVMThread.setPeakCount(thread.getPeakCount());
serviceInstanceJVMThread.setTimeBucket(timeBucket);
sourceReceiver.receive(serviceInstanceJVMThread);
}
}

9
test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java
View File

@ -64,6 +64,15 @@ public class MetricsQuery extends AbstractQuery<MetricsQuery> {
SERVICE_INSTANCE_SLA
};
public static String INSTANCE_JVM_THREAD_LIVE_COUNT = "instance_jvm_thread_live_count";
public static String INSTANCE_JVM_THREAD_DAEMON_COUNT = "instance_jvm_thread_daemon_count";
public static String INSTANCE_JVM_THREAD_PEAK_COUNT = "instance_jvm_thread_peak_count";
public static String [] ALL_INSTANCE_JVM_METRICS = {
INSTANCE_JVM_THREAD_DAEMON_COUNT,
INSTANCE_JVM_THREAD_DAEMON_COUNT,
INSTANCE_JVM_THREAD_PEAK_COUNT
};
public static String SERVICE_RELATION_CLIENT_CPM = "service_relation_client_cpm";
public static String SERVICE_RELATION_SERVER_CPM = "service_relation_server_cpm";
public static String SERVICE_RELATION_CLIENT_CALL_SLA = "service_relation_client_call_sla";

23
test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java
View File

@ -62,6 +62,7 @@ import org.testcontainers.containers.DockerComposeContainer;
import static org.apache.skywalking.e2e.metrics.MetricsMatcher.verifyMetrics;
import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_ENDPOINT_METRICS;
import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_INSTANCE_METRICS;
import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_INSTANCE_JVM_METRICS;
import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_INSTANCE_RELATION_CLIENT_METRICS;
import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_INSTANCE_RELATION_SERVER_METRICS;
import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_METRICS;
@ -151,6 +152,8 @@ public class SimpleE2E extends SkyWalkingTestAdapter {
verifyInstancesMetrics(instances);
verifyInstancesJVMMetrics(instances);
final Endpoints endpoints = verifyServiceEndpoints(service);
verifyEndpointsMetrics(endpoints);
@ -287,6 +290,26 @@ public class SimpleE2E extends SkyWalkingTestAdapter {
}
}
private void verifyInstancesJVMMetrics(final Instances instances) throws Exception {
for (Instance instance : instances.getInstances()) {
for (String metricsName : ALL_INSTANCE_JVM_METRICS) {
LOGGER.info("verifying service instance response time: {}", instance);
final Metrics instanceJVMMetrics = graphql.metrics(
new MetricsQuery().stepByMinute().metricsName(metricsName).id(instance.getKey())
);
LOGGER.info("instance jvm metrics: {}", instanceJVMMetrics);
final AtLeastOneOfMetricsMatcher instanceThreadMatcher = new AtLeastOneOfMetricsMatcher();
final MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
greaterThanZero.setValue("gt 0");
instanceThreadMatcher.setValue(greaterThanZero);
instanceThreadMatcher.verify(instanceJVMMetrics);
LOGGER.info("{}: {}", metricsName, instanceJVMMetrics);
}
}
}
private void verifyEndpointsMetrics(final Endpoints endpoints) throws Exception {
for (Endpoint endpoint : endpoints.getEndpoints()) {
if (!endpoint.getLabel().equals("/users")) {