kazusa
/
skywalking-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Support keep trace profiling when cross-thread (#479)

This commit is contained in:
mrproliu 2023-03-14 17:55:28 +08:00 committed by GitHub
parent f04866cf24
commit d357bdef7a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 196 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.md
View File

@ -20,6 +20,7 @@ Release Notes.
* Adapt Armeria's plugins to the latest version 1.22.x
* Fix tomcat-10x-plugin and add test case to support tomcat7.x-8.x-9.x.
* Fix thrift plugin generate duplicate traceid when `sendBase` error occurs
* Support keep trace profiling when cross-thread.
#### Documentation
* Update docs of Tracing APIs, reorganize the API docs into six parts.

7
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java
View File

@ -241,10 +241,15 @@ public class Config {
public static boolean ACTIVE = true;
/**
* Parallel monitor segment count
* Parallel monitor endpoint thread count
*/
public static int MAX_PARALLEL = 5;
/**
* Max monitoring sub-tasks count of one single endpoint access
*/
public static int MAX_ACCEPT_SUB_PARALLEL = 5;
/**
* Max monitor segment time(minutes), if current segment monitor time out of limit, then stop it.
*/

6
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/context/ContextSnapshot.java
View File

@ -20,6 +20,7 @@ package org.apache.skywalking.apm.agent.core.context;
import lombok.Getter;
import org.apache.skywalking.apm.agent.core.context.ids.DistributedTraceId;
import org.apache.skywalking.apm.agent.core.profile.ProfileStatusContext;
/**
* The <code>ContextSnapshot</code> is a snapshot for current context. The snapshot carries the info for building
@ -34,19 +35,22 @@ public class ContextSnapshot {
private CorrelationContext correlationContext;
private ExtensionContext extensionContext;
private ProfileStatusContext profileStatusContext;
ContextSnapshot(String traceSegmentId,
int spanId,
DistributedTraceId primaryTraceId,
String parentEndpoint,
CorrelationContext correlationContext,
ExtensionContext extensionContext) {
ExtensionContext extensionContext,
ProfileStatusContext profileStatusContext) {
this.traceSegmentId = traceSegmentId;
this.spanId = spanId;
this.traceId = primaryTraceId;
this.parentEndpoint = parentEndpoint;
this.correlationContext = correlationContext.clone();
this.extensionContext = extensionContext.clone();
this.profileStatusContext = profileStatusContext.clone();
}
public boolean isFromCurrent() {

5
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/context/IgnoredTracerContext.java
View File

@ -22,6 +22,7 @@ import java.util.LinkedList;
import java.util.List;
import org.apache.skywalking.apm.agent.core.context.trace.AbstractSpan;
import org.apache.skywalking.apm.agent.core.context.trace.NoopSpan;
import org.apache.skywalking.apm.agent.core.profile.ProfileStatusContext;
/**
* The <code>IgnoredTracerContext</code> represent a context should be ignored. So it just maintains the stack with an
@ -35,6 +36,7 @@ public class IgnoredTracerContext implements AbstractTracerContext {
private final CorrelationContext correlationContext;
private final ExtensionContext extensionContext;
private final ProfileStatusContext profileStatusContext;
private int stackDepth;
@ -42,6 +44,7 @@ public class IgnoredTracerContext implements AbstractTracerContext {
this.stackDepth = 0;
this.correlationContext = new CorrelationContext();
this.extensionContext = new ExtensionContext();
this.profileStatusContext = ProfileStatusContext.createWithNone();
}
@Override
@ -56,7 +59,7 @@ public class IgnoredTracerContext implements AbstractTracerContext {
@Override
public ContextSnapshot capture() {
return new ContextSnapshot(null, -1, null, null, correlationContext, extensionContext);
return new ContextSnapshot(null, -1, null, null, correlationContext, extensionContext, profileStatusContext);
}
@Override

12
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/context/TracingContext.java
View File

@ -41,7 +41,7 @@ import org.apache.skywalking.apm.agent.core.context.trace.TraceSegment;
import org.apache.skywalking.apm.agent.core.context.trace.TraceSegmentRef;
import org.apache.skywalking.apm.agent.core.logging.api.ILog;
import org.apache.skywalking.apm.agent.core.logging.api.LogManager;
import org.apache.skywalking.apm.agent.core.profile.ProfileStatusReference;
import org.apache.skywalking.apm.agent.core.profile.ProfileStatusContext;
import org.apache.skywalking.apm.agent.core.profile.ProfileTaskExecutionService;
import org.apache.skywalking.apm.util.StringUtil;
@ -106,7 +106,7 @@ public class TracingContext implements AbstractTracerContext {
/**
* profile status
*/
private final ProfileStatusReference profileStatus;
private final ProfileStatusContext profileStatus;
@Getter(AccessLevel.PACKAGE)
private final CorrelationContext correlationContext;
@Getter(AccessLevel.PACKAGE)
@ -213,7 +213,8 @@ public class TracingContext implements AbstractTracerContext {
getPrimaryTraceId(),
primaryEndpoint.getName(),
this.correlationContext,
this.extensionContext
this.extensionContext,
this.profileStatus
);
return snapshot;
@ -234,6 +235,9 @@ public class TracingContext implements AbstractTracerContext {
this.correlationContext.continued(snapshot);
this.extensionContext.continued(snapshot);
this.extensionContext.handle(this.activeSpan());
if (this.profileStatus.continued(snapshot)) {
PROFILE_TASK_EXECUTION_SERVICE.continueProfiling(this, this.segment.getTraceSegmentId());
}
}
}
@ -571,7 +575,7 @@ public class TracingContext implements AbstractTracerContext {
return this.createTime;
}
public ProfileStatusReference profileStatus() {
public ProfileStatusContext profileStatus() {
return this.profileStatus;
}

110
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ProfileStatusContext.java Normal file
View File

@ -0,0 +1,110 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.apm.agent.core.profile;
import org.apache.skywalking.apm.agent.core.conf.Config;
import org.apache.skywalking.apm.agent.core.context.ContextSnapshot;
import org.apache.skywalking.apm.agent.core.context.TracingContext;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Wrapper {@link ProfileStatus}, make sure {@link org.apache.skywalking.apm.agent.core.context.TracingContext} with {@link ThreadProfiler} have same reference with {@link ProfileStatus},
* And only the profile module could change the status
*/
public class ProfileStatusContext {
private volatile ProfileStatus status;
private volatile long firstSegmentCreateTime;
private volatile AtomicInteger subThreadProfilingCount;
private ProfileStatusContext(ProfileStatus status, long firstSegmentCreateTime, AtomicInteger subThreadProfilingCount) {
this.status = status;
this.firstSegmentCreateTime = firstSegmentCreateTime;
this.subThreadProfilingCount = subThreadProfilingCount;
}
/**
* Create with not watching
*/
public static ProfileStatusContext createWithNone() {
return new ProfileStatusContext(ProfileStatus.NONE, 0, null);
}
/**
* Create with pending to profile
*/
public static ProfileStatusContext createWithPending(long firstSegmentCreateTime) {
return new ProfileStatusContext(ProfileStatus.PENDING, firstSegmentCreateTime, new AtomicInteger(0));
}
public ProfileStatus get() {
return this.status;
}
public long firstSegmentCreateTime() {
return this.firstSegmentCreateTime;
}
/**
* The profile monitoring is watching, wait for some profile conditions.
*/
public boolean isBeingWatched() {
return this.status != ProfileStatus.NONE;
}
public boolean isProfiling() {
return this.status == ProfileStatus.PROFILING;
}
public ProfileStatusContext clone() {
return new ProfileStatusContext(this.status, this.firstSegmentCreateTime, this.subThreadProfilingCount);
}
/**
* Continued profile status context
* @return is needs to keep profile
*/
public boolean continued(ContextSnapshot snapshot) {
this.status = snapshot.getProfileStatusContext().get();
this.firstSegmentCreateTime = snapshot.getProfileStatusContext().firstSegmentCreateTime();
this.subThreadProfilingCount = snapshot.getProfileStatusContext().subThreadProfilingCount;
return this.isBeingWatched() &&
// validate is reach the count of sub-thread
this.subThreadProfilingCount != null &&
this.subThreadProfilingCount.incrementAndGet() <= Config.Profile.MAX_ACCEPT_SUB_PARALLEL;
}
/**
* Update status, only access with profile module
*/
void updateStatus(ProfileStatus status, TracingContext tracingContext) {
this.status = status;
if (this.firstSegmentCreateTime == 0 && tracingContext != null) {
this.firstSegmentCreateTime = tracingContext.createTime();
}
}
void updateStatus(ProfileStatusContext statusContext) {
this.status = statusContext.get();
this.firstSegmentCreateTime = statusContext.firstSegmentCreateTime();
this.subThreadProfilingCount = statusContext.subThreadProfilingCount;
}
}

69
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ProfileStatusReference.java
View File

@ -1,69 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.apm.agent.core.profile;
/**
* Wrapper {@link ProfileStatus}, make sure {@link org.apache.skywalking.apm.agent.core.context.TracingContext} with {@link ThreadProfiler} have same reference with {@link ProfileStatus},
* And only the profile module could change the status
*/
public class ProfileStatusReference {
private volatile ProfileStatus status;
private ProfileStatusReference(ProfileStatus status) {
this.status = status;
}
/**
* Create with not watching
*/
public static ProfileStatusReference createWithNone() {
return new ProfileStatusReference(ProfileStatus.NONE);
}
/**
* Create with pending to profile
*/
public static ProfileStatusReference createWithPending() {
return new ProfileStatusReference(ProfileStatus.PENDING);
}
public ProfileStatus get() {
return this.status;
}
/**
* The profile monitoring is watching, wait for some profile conditions.
*/
public boolean isBeingWatched() {
return this.status != ProfileStatus.NONE;
}
public boolean isProfiling() {
return this.status == ProfileStatus.PROFILING;
}
/**
* Update status, only access with profile module
*/
void updateStatus(ProfileStatus status) {
this.status = status;
}
}

49
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ProfileTaskExecutionContext.java
View File

@ -34,8 +34,8 @@ public class ProfileTaskExecutionContext {
// task data
private final ProfileTask task;
// record current profiling count, use this to check has available profile slot
private final AtomicInteger currentProfilingCount = new AtomicInteger(0);
// record current first endpoint profiling count, use this to check has available profile slot
private final AtomicInteger currentEndpointProfilingCount = new AtomicInteger(0);
// profiling segment slot
private volatile AtomicReferenceArray<ThreadProfiler> profilingSegmentSlots;
@ -48,7 +48,7 @@ public class ProfileTaskExecutionContext {
public ProfileTaskExecutionContext(ProfileTask task) {
this.task = task;
profilingSegmentSlots = new AtomicReferenceArray<>(Config.Profile.MAX_PARALLEL);
profilingSegmentSlots = new AtomicReferenceArray<>(Config.Profile.MAX_PARALLEL * (Config.Profile.MAX_ACCEPT_SUB_PARALLEL + 1));
}
/**
@ -72,39 +72,52 @@ public class ProfileTaskExecutionContext {
*
* @return is add profile success
*/
public ProfileStatusReference attemptProfiling(TracingContext tracingContext,
String traceSegmentId,
String firstSpanOPName) {
// check has available slot
final int usingSlotCount = currentProfilingCount.get();
if (usingSlotCount >= Config.Profile.MAX_PARALLEL) {
return ProfileStatusReference.createWithNone();
public ProfileStatusContext attemptProfiling(TracingContext tracingContext,
String traceSegmentId,
String firstSpanOPName) {
// check has limited the max parallel profiling count
final int profilingEndpointCount = currentEndpointProfilingCount.get();
if (profilingEndpointCount >= Config.Profile.MAX_PARALLEL) {
return ProfileStatusContext.createWithNone();
}
// check first operation name matches
if (!Objects.equals(task.getFirstSpanOPName(), firstSpanOPName)) {
return ProfileStatusReference.createWithNone();
return ProfileStatusContext.createWithNone();
}
// if out limit started profiling count then stop add profiling
if (totalStartedProfilingCount.get() > task.getMaxSamplingCount()) {
return ProfileStatusReference.createWithNone();
return ProfileStatusContext.createWithNone();
}
// try to occupy slot
if (!currentProfilingCount.compareAndSet(usingSlotCount, usingSlotCount + 1)) {
return ProfileStatusReference.createWithNone();
if (!currentEndpointProfilingCount.compareAndSet(profilingEndpointCount, profilingEndpointCount + 1)) {
return ProfileStatusContext.createWithNone();
}
ThreadProfiler profiler;
if ((profiler = addProfilingThread(tracingContext, traceSegmentId)) != null) {
return profiler.profilingStatus();
}
return ProfileStatusContext.createWithNone();
}
public boolean continueProfiling(TracingContext tracingContext, String traceSegmentId) {
return addProfilingThread(tracingContext, traceSegmentId) != null;
}
private ThreadProfiler addProfilingThread(TracingContext tracingContext, String traceSegmentId) {
final ThreadProfiler threadProfiler = new ThreadProfiler(
tracingContext, traceSegmentId, Thread.currentThread(), this);
int slotLength = profilingSegmentSlots.length();
for (int slot = 0; slot < slotLength; slot++) {
if (profilingSegmentSlots.compareAndSet(slot, null, threadProfiler)) {
return threadProfiler.profilingStatus();
return threadProfiler;
}
}
return ProfileStatusReference.createWithNone();
// add profiling thread failure, so ignore it
return null;
}
/**
@ -118,7 +131,7 @@ public class ProfileTaskExecutionContext {
// update profiling status
tracingContext.profileStatus()
.updateStatus(attemptProfiling(tracingContext, traceSegmentId, firstSpanOPName).get());
.updateStatus(attemptProfiling(tracingContext, traceSegmentId, firstSpanOPName));
}
/**
@ -134,7 +147,7 @@ public class ProfileTaskExecutionContext {
// setting stop running
currentProfiler.stopProfiling();
currentProfilingCount.addAndGet(-1);
currentEndpointProfilingCount.addAndGet(-1);
break;
}
}

20
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ProfileTaskExecutionService.java
View File

@ -90,18 +90,30 @@ public class ProfileTaskExecutionService implements BootService, TracingThreadLi
/**
* check and add {@link TracingContext} profiling
*/
public ProfileStatusReference addProfiling(TracingContext tracingContext,
String traceSegmentId,
String firstSpanOPName) {
public ProfileStatusContext addProfiling(TracingContext tracingContext,
String traceSegmentId,
String firstSpanOPName) {
// get current profiling task, check need profiling
final ProfileTaskExecutionContext executionContext = taskExecutionContext.get();
if (executionContext == null) {
return ProfileStatusReference.createWithNone();
return ProfileStatusContext.createWithNone();
}
return executionContext.attemptProfiling(tracingContext, traceSegmentId, firstSpanOPName);
}
/**
* continue profiling task when cross-thread
*/
public void continueProfiling(TracingContext tracingContext, String traceSegmentId) {
final ProfileTaskExecutionContext executionContext = taskExecutionContext.get();
if (executionContext == null) {
return;
}
executionContext.continueProfiling(tracingContext, traceSegmentId);
}
/**
* Re-check current trace need profiling, in case that third-party plugins change the operation name.
*/

16
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ThreadProfiler.java
View File

@ -40,7 +40,7 @@ public class ThreadProfiler {
private long profilingMaxTimeMills;
// after min duration threshold check, it will start dump
private final ProfileStatusReference profilingStatus;
private final ProfileStatusContext profilingStatus;
// thread dump sequence
private int dumpSequence = 0;
@ -51,10 +51,10 @@ public class ThreadProfiler {
this.profilingThread = profilingThread;
this.executionContext = executionContext;
if (tracingContext.profileStatus() == null) {
this.profilingStatus = ProfileStatusReference.createWithPending();
this.profilingStatus = ProfileStatusContext.createWithPending(tracingContext().createTime());
} else {
this.profilingStatus = tracingContext.profileStatus();
this.profilingStatus.updateStatus(ProfileStatus.PENDING);
this.profilingStatus.updateStatus(ProfileStatus.PENDING, tracingContext);
}
this.profilingMaxTimeMills = TimeUnit.MINUTES.toMillis(Config.Profile.MAX_DURATION);
}
@ -63,10 +63,10 @@ public class ThreadProfiler {
* If tracing start time greater than {@link ProfileTask#getMinDurationThreshold()}, then start to profiling trace
*/
public void startProfilingIfNeed() {
if (System.currentTimeMillis() - tracingContext.createTime() > executionContext.getTask()
if (System.currentTimeMillis() - profilingStatus.firstSegmentCreateTime() > executionContext.getTask()
.getMinDurationThreshold()) {
this.profilingStartTime = System.currentTimeMillis();
this.tracingContext.profileStatus().updateStatus(ProfileStatus.PROFILING);
this.profilingStatus.updateStatus(ProfileStatus.PROFILING, tracingContext);
}
}
@ -74,7 +74,7 @@ public class ThreadProfiler {
* Stop profiling status
*/
public void stopProfiling() {
this.tracingContext.profileStatus().updateStatus(ProfileStatus.STOPPED);
this.profilingStatus.updateStatus(ProfileStatus.STOPPED, tracingContext);
}
/**
@ -133,7 +133,7 @@ public class ThreadProfiler {
*/
public boolean matches(TracingContext context) {
// match trace id
return Objects.equal(context.getReadablePrimaryTraceId(), tracingContext.getReadablePrimaryTraceId());
return Objects.equal(context.getSegmentId(), tracingContext.getSegmentId());
}
/**
@ -149,7 +149,7 @@ public class ThreadProfiler {
return tracingContext;
}
public ProfileStatusReference profilingStatus() {
public ProfileStatusContext profilingStatus() {
return profilingStatus;
}

4
apm-sniffer/apm-test-tools/src/main/java/org/apache/skywalking/apm/agent/core/context/MockContextSnapshot.java
View File

@ -19,6 +19,7 @@
package org.apache.skywalking.apm.agent.core.context;
import org.apache.skywalking.apm.agent.core.context.ids.NewDistributedTraceId;
import org.apache.skywalking.apm.agent.core.profile.ProfileStatusContext;
public enum MockContextSnapshot {
INSTANCE;
@ -32,7 +33,8 @@ public enum MockContextSnapshot {
new NewDistributedTraceId(),
"/for-test-entryOperationName",
new CorrelationContext(),
new ExtensionContext()
new ExtensionContext(),
ProfileStatusContext.createWithNone()
);
}

4
apm-sniffer/config/agent.config
View File

@ -161,8 +161,10 @@ buffer.channel_size=${SW_BUFFER_CHANNEL_SIZE:5}
buffer.buffer_size=${SW_BUFFER_BUFFER_SIZE:300}
# If true, skywalking agent will enable profile when user create a new profile task. Otherwise disable profile.
profile.active=${SW_AGENT_PROFILE_ACTIVE:true}
# Parallel monitor segment count
# Parallel monitor endpoint thread count
profile.max_parallel=${SW_AGENT_PROFILE_MAX_PARALLEL:5}
# Max monitoring sub-tasks count of one single endpoint access
profile.max_accept_sub_parallel=${SW_AGENT_PROFILE_MAX_ACCEPT_SUB_PARALLEL:5}
# Max monitor segment time(minutes), if current segment monitor time out of limit, then stop it.
profile.duration=${SW_AGENT_PROFILE_DURATION:10}
# Max dump thread stack depth

1
docs/en/setup/service-agent/java-agent/configurations.md
View File

@ -54,6 +54,7 @@ This is the properties list supported in `agent/config/agent.config`.
| `buffer.buffer_size` | The buffer size. | SW_BUFFER_BUFFER_SIZE | `300` |
| `profile.active` | If true, skywalking agent will enable profile when user create a new profile task. Otherwise disable profile. | SW_AGENT_PROFILE_ACTIVE | `true` |
| `profile.max_parallel` | Parallel monitor segment count | SW_AGENT_PROFILE_MAX_PARALLEL | `5` |
| `profile.max_accept_sub_parallel` | Max monitoring sub-tasks count of one single endpoint access | SW_AGENT_PROFILE_MAX_ACCEPT_SUB_PARALLEL | `5` |
| `profile.duration` | Max monitor segment time(minutes), if current segment monitor time out of limit, then stop it. | SW_AGENT_PROFILE_DURATION | `10` |
| `profile.dump_max_stack_depth` | Max dump thread stack depth | SW_AGENT_PROFILE_DUMP_MAX_STACK_DEPTH | `500` |
| `profile.snapshot_transport_buffer_size` | Snapshot transport to backend buffer size | SW_AGENT_PROFILE_SNAPSHOT_TRANSPORT_BUFFER_SIZE | `4500` |