kazusa
/
skywalking-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Support async profiler feature (#720)

This commit is contained in:
zhengziyi0117 2024-10-30 17:22:33 +08:00 committed by GitHub
parent 576550a8db
commit 2027a98b1e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 866 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

133
apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java Normal file
View File

@ -0,0 +1,133 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.apm.network.trace.component.command;
import org.apache.skywalking.apm.network.common.v3.Command;
import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair;
import java.util.List;
import java.util.Objects;
public class AsyncProfilerTaskCommand extends BaseCommand implements Serializable, Deserializable<AsyncProfilerTaskCommand> {
public static final Deserializable<AsyncProfilerTaskCommand> DESERIALIZER = new AsyncProfilerTaskCommand("", "", 0, null, "", 0);
public static final String NAME = "AsyncProfilerTaskQuery";
/**
* async-profiler taskId
*/
private final String taskId;
/**
* run profiling for duration (second)
*/
private final int duration;
/**
* async profiler extended parameters. Here is a table of optional parameters.
*
* <p>lock[=DURATION] - profile contended locks overflowing the DURATION ns bucket (default: 10us)</p>
* <p>alloc[=BYTES] - profile allocations with BYTES interval</p>
* <p>interval=N - sampling interval in ns (default: 10'000'000, i.e. 10 ms)</p>
* <p>jstackdepth=N - maximum Java stack depth (default: 2048)</p>
* <p>chunksize=N - approximate size of JFR chunk in bytes (default: 100 MB) </p>
* <p>chunktime=N - duration of JFR chunk in seconds (default: 1 hour) </p>
* details @see <a href="https://github.com/async-profiler/async-profiler/blob/master/src/arguments.cpp#L44">async-profiler argument</a>
*/
private final String execArgs;
/**
* task create time
*/
private final long createTime;
public AsyncProfilerTaskCommand(String serialNumber, String taskId, int duration,
List<String> events, String execArgs, long createTime) {
super(NAME, serialNumber);
this.taskId = taskId;
this.duration = duration;
this.createTime = createTime;
String comma = ",";
StringBuilder sb = new StringBuilder();
if (Objects.nonNull(events) && !events.isEmpty()) {
sb.append("event=")
.append(String.join(comma, events))
.append(comma);
}
if (execArgs != null && !execArgs.isEmpty()) {
sb.append(execArgs);
}
this.execArgs = sb.toString();
}
public AsyncProfilerTaskCommand(String serialNumber, String taskId, int duration,
String execArgs, long createTime) {
super(NAME, serialNumber);
this.taskId = taskId;
this.duration = duration;
this.execArgs = execArgs;
this.createTime = createTime;
}
@Override
public AsyncProfilerTaskCommand deserialize(Command command) {
final List<KeyStringValuePair> argsList = command.getArgsList();
String taskId = null;
int duration = 0;
String execArgs = null;
long createTime = 0;
String serialNumber = null;
for (final KeyStringValuePair pair : argsList) {
if ("SerialNumber".equals(pair.getKey())) {
serialNumber = pair.getValue();
} else if ("TaskId".equals(pair.getKey())) {
taskId = pair.getValue();
} else if ("Duration".equals(pair.getKey())) {
duration = Integer.parseInt(pair.getValue());
} else if ("ExecArgs".equals(pair.getKey())) {
execArgs = pair.getValue();
} else if ("CreateTime".equals(pair.getKey())) {
createTime = Long.parseLong(pair.getValue());
}
}
return new AsyncProfilerTaskCommand(serialNumber, taskId, duration, execArgs, createTime);
}
@Override
public Command.Builder serialize() {
final Command.Builder builder = commandBuilder();
builder.addArgs(KeyStringValuePair.newBuilder().setKey("TaskId").setValue(taskId))
.addArgs(KeyStringValuePair.newBuilder().setKey("Duration").setValue(String.valueOf(duration)))
.addArgs(KeyStringValuePair.newBuilder().setKey("ExecArgs").setValue(execArgs))
.addArgs(KeyStringValuePair.newBuilder().setKey("CreateTime").setValue(String.valueOf(createTime)));
return builder;
}
public String getTaskId() {
return taskId;
}
public int getDuration() {
return duration;
}
public String getExecArgs() {
return execArgs;
}
public long getCreateTime() {
return createTime;
}
}

3
apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/CommandDeserializer.java
View File

@ -27,7 +27,10 @@ public class CommandDeserializer {
return ProfileTaskCommand.DESERIALIZER.deserialize(command);
} else if (ConfigurationDiscoveryCommand.NAME.equals(commandName)) {
return ConfigurationDiscoveryCommand.DESERIALIZER.deserialize(command);
} else if (AsyncProfilerTaskCommand.NAME.equals(commandName)) {
return AsyncProfilerTaskCommand.DESERIALIZER.deserialize(command);
}
throw new UnsupportedCommandException(command);
}

2
apm-protocol/apm-network/src/main/proto

@ -1 +1 @@
Subproject commit d4da5699915ee52288f8ff1c954decf6363485bc
Subproject commit bd1f91f7e1cb4de9d9b5ccb71f36ce6b1c7c97f5

4
apm-sniffer/apm-agent-core/pom.xml
View File

@ -143,6 +143,10 @@
<artifactId>jmh-generator-annprocess</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>tools.profiler</groupId>
<artifactId>async-profiler</artifactId>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>

194
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java Normal file
View File

@ -0,0 +1,194 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.apm.agent.core.asyncprofiler;
import com.google.protobuf.ByteString;
import io.grpc.Channel;
import io.grpc.stub.ClientCallStreamObserver;
import io.grpc.stub.ClientResponseObserver;
import io.grpc.stub.StreamObserver;
import org.apache.skywalking.apm.agent.core.boot.BootService;
import org.apache.skywalking.apm.agent.core.boot.DefaultImplementor;
import org.apache.skywalking.apm.agent.core.boot.ServiceManager;
import org.apache.skywalking.apm.agent.core.conf.Config;
import org.apache.skywalking.apm.agent.core.logging.api.ILog;
import org.apache.skywalking.apm.agent.core.logging.api.LogManager;
import org.apache.skywalking.apm.agent.core.remote.GRPCChannelListener;
import org.apache.skywalking.apm.agent.core.remote.GRPCChannelManager;
import org.apache.skywalking.apm.agent.core.remote.GRPCChannelStatus;
import org.apache.skywalking.apm.agent.core.remote.GRPCStreamServiceStatus;
import org.apache.skywalking.apm.network.language.asyncprofiler.v10.AsyncProfilerCollectionResponse;
import org.apache.skywalking.apm.network.language.asyncprofiler.v10.AsyncProfilerData;
import org.apache.skywalking.apm.network.language.asyncprofiler.v10.AsyncProfilerMetaData;
import org.apache.skywalking.apm.network.language.asyncprofiler.v10.AsyncProfilerTaskGrpc;
import org.apache.skywalking.apm.network.language.asyncprofiler.v10.AsyncProfilingStatus;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import static org.apache.skywalking.apm.agent.core.conf.Config.AsyncProfiler.DATA_CHUNK_SIZE;
import static org.apache.skywalking.apm.agent.core.conf.Config.Collector.GRPC_UPSTREAM_TIMEOUT;
@DefaultImplementor
public class AsyncProfilerDataSender implements BootService, GRPCChannelListener {
private static final ILog LOGGER = LogManager.getLogger(AsyncProfilerDataSender.class);
private volatile GRPCChannelStatus status = GRPCChannelStatus.DISCONNECT;
private volatile AsyncProfilerTaskGrpc.AsyncProfilerTaskStub asyncProfilerTaskStub;
@Override
public void prepare() throws Throwable {
ServiceManager.INSTANCE.findService(GRPCChannelManager.class).addChannelListener(this);
}
@Override
public void boot() throws Throwable {
}
@Override
public void onComplete() throws Throwable {
}
@Override
public void shutdown() throws Throwable {
}
@Override
public void statusChanged(GRPCChannelStatus status) {
if (GRPCChannelStatus.CONNECTED.equals(status)) {
Channel channel = ServiceManager.INSTANCE.findService(GRPCChannelManager.class).getChannel();
asyncProfilerTaskStub = AsyncProfilerTaskGrpc.newStub(channel);
} else {
asyncProfilerTaskStub = null;
}
this.status = status;
}
public void sendData(AsyncProfilerTask task, FileChannel channel) throws IOException, InterruptedException {
if (status != GRPCChannelStatus.CONNECTED || Objects.isNull(channel) || !channel.isOpen()) {
return;
}
int size = Math.toIntExact(channel.size());
final GRPCStreamServiceStatus status = new GRPCStreamServiceStatus(false);
StreamObserver<AsyncProfilerData> dataStreamObserver = asyncProfilerTaskStub.withDeadlineAfter(
GRPC_UPSTREAM_TIMEOUT, TimeUnit.SECONDS
).collect(new ClientResponseObserver<AsyncProfilerData, AsyncProfilerCollectionResponse>() {
ClientCallStreamObserver<AsyncProfilerData> requestStream;
@Override
public void beforeStart(ClientCallStreamObserver<AsyncProfilerData> requestStream) {
this.requestStream = requestStream;
}
@Override
public void onNext(AsyncProfilerCollectionResponse value) {
if (AsyncProfilingStatus.TERMINATED_BY_OVERSIZE.equals(value.getType())) {
LOGGER.warn("JFR is too large to be received by the oap server");
} else {
ByteBuffer buf = ByteBuffer.allocateDirect(DATA_CHUNK_SIZE);
try {
while (channel.read(buf) > 0) {
buf.flip();
AsyncProfilerData asyncProfilerData = AsyncProfilerData.newBuilder()
.setContent(ByteString.copyFrom(buf))
.build();
requestStream.onNext(asyncProfilerData);
buf.clear();
}
} catch (IOException e) {
LOGGER.error("Failed to read JFR file and failed to upload to oap", e);
}
}
requestStream.onCompleted();
}
@Override
public void onError(Throwable t) {
status.finished();
LOGGER.error(t, "Send async profiler task data to collector fail with a grpc internal exception.");
ServiceManager.INSTANCE.findService(GRPCChannelManager.class).reportError(t);
}
@Override
public void onCompleted() {
status.finished();
}
});
AsyncProfilerMetaData metaData = AsyncProfilerMetaData.newBuilder()
.setService(Config.Agent.SERVICE_NAME)
.setServiceInstance(Config.Agent.INSTANCE_NAME)
.setType(AsyncProfilingStatus.PROFILING_SUCCESS)
.setContentSize(size)
.setTaskId(task.getTaskId())
.build();
AsyncProfilerData asyncProfilerData = AsyncProfilerData.newBuilder().setMetaData(metaData).build();
dataStreamObserver.onNext(asyncProfilerData);
status.wait4Finish();
}
public void sendError(AsyncProfilerTask task, String errorMessage) {
if (status != GRPCChannelStatus.CONNECTED) {
return;
}
final GRPCStreamServiceStatus status = new GRPCStreamServiceStatus(false);
StreamObserver<AsyncProfilerData> dataStreamObserver = asyncProfilerTaskStub.withDeadlineAfter(
GRPC_UPSTREAM_TIMEOUT, TimeUnit.SECONDS
).collect(new StreamObserver<AsyncProfilerCollectionResponse>() {
@Override
public void onNext(AsyncProfilerCollectionResponse value) {
}
@Override
public void onError(Throwable t) {
status.finished();
LOGGER.error(t, "Send async profiler task execute error fail with a grpc internal exception.");
ServiceManager.INSTANCE.findService(GRPCChannelManager.class).reportError(t);
}
@Override
public void onCompleted() {
status.finished();
}
});
AsyncProfilerMetaData metaData = AsyncProfilerMetaData.newBuilder()
.setService(Config.Agent.SERVICE_NAME)
.setServiceInstance(Config.Agent.INSTANCE_NAME)
.setTaskId(task.getTaskId())
.setType(AsyncProfilingStatus.EXECUTION_TASK_ERROR)
.setContentSize(-1)
.build();
AsyncProfilerData asyncProfilerData = AsyncProfilerData.newBuilder()
.setMetaData(metaData)
.setErrorMessage(errorMessage)
.build();
dataStreamObserver.onNext(asyncProfilerData);
dataStreamObserver.onCompleted();
status.wait4Finish();
}
}

146
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java Normal file
View File

@ -0,0 +1,146 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.apm.agent.core.asyncprofiler;
import one.profiler.AsyncProfiler;
import org.apache.skywalking.apm.agent.core.conf.Config;
import org.apache.skywalking.apm.agent.core.logging.api.ILog;
import org.apache.skywalking.apm.agent.core.logging.api.LogManager;
import org.apache.skywalking.apm.util.StringUtil;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
public class AsyncProfilerTask {
private static final ILog LOGGER = LogManager.getLogger(AsyncProfilerTask.class);
private static final String COMMA = ",";
/**
* task id
*/
private String taskId;
/**
* async profiler optional extended parameters
*
* @see org.apache.skywalking.apm.network.trace.component.command.AsyncProfilerTaskCommand
*/
private String execArgs;
/**
* run profiling for duration (second)
*/
private int duration;
/**
* The time when oap server created this task
*/
private long createTime;
/**
* tempFile generated by async-profiler execution
*/
private Path tempFile;
private static String execute(AsyncProfiler asyncProfiler, String args)
throws IllegalArgumentException, IOException {
LOGGER.info("async profiler execute args:{}", args);
return asyncProfiler.execute(args);
}
/**
* start async profiler
*/
public String start(AsyncProfiler asyncProfiler) throws IOException {
tempFile = getProfilerFilePath();
StringBuilder startArgs = new StringBuilder();
startArgs.append("start").append(COMMA);
if (StringUtil.isNotEmpty(execArgs)) {
startArgs.append(execArgs).append(COMMA);
}
startArgs.append("file=").append(tempFile.toString());
return execute(asyncProfiler, startArgs.toString());
}
/**
* stop async profiler and get file
*/
public File stop(AsyncProfiler asyncProfiler) throws IOException {
LOGGER.info("async profiler process stop and dump file");
String stopArgs = "stop" + COMMA + "file=" + tempFile.toAbsolutePath();
execute(asyncProfiler, stopArgs);
return tempFile.toFile();
}
/**
* if outputPath is configured, the JFR file will be generated at outputPath,
* otherwise createTemp will be used to create the file
*/
public Path getProfilerFilePath() throws IOException {
if (StringUtil.isNotEmpty(Config.AsyncProfiler.OUTPUT_PATH)) {
Path tempFilePath = Paths.get(Config.AsyncProfiler.OUTPUT_PATH, taskId + getFileExtension());
return Files.createFile(tempFilePath.toAbsolutePath());
} else {
return Files.createTempFile(taskId, getFileExtension());
}
}
private String getFileExtension() {
return ".jfr";
}
public void setExecArgs(String execArgs) {
this.execArgs = execArgs;
}
public void setDuration(int duration) {
this.duration = duration;
}
public void setTempFile(Path tempFile) {
this.tempFile = tempFile;
}
public void setTaskId(String taskId) {
this.taskId = taskId;
}
public void setCreateTime(long createTime) {
this.createTime = createTime;
}
public String getExecArgs() {
return execArgs;
}
public int getDuration() {
return duration;
}
public Path getTempFile() {
return tempFile;
}
public String getTaskId() {
return taskId;
}
public long getCreateTime() {
return createTime;
}
}

131
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java Normal file
View File

@ -0,0 +1,131 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.apm.agent.core.asyncprofiler;
import io.grpc.Channel;
import io.grpc.Status;
import io.grpc.StatusRuntimeException;
import org.apache.skywalking.apm.agent.core.boot.BootService;
import org.apache.skywalking.apm.agent.core.boot.DefaultImplementor;
import org.apache.skywalking.apm.agent.core.boot.DefaultNamedThreadFactory;
import org.apache.skywalking.apm.agent.core.boot.ServiceManager;
import org.apache.skywalking.apm.agent.core.commands.CommandService;
import org.apache.skywalking.apm.agent.core.conf.Config;
import org.apache.skywalking.apm.agent.core.logging.api.ILog;
import org.apache.skywalking.apm.agent.core.logging.api.LogManager;
import org.apache.skywalking.apm.agent.core.remote.GRPCChannelListener;
import org.apache.skywalking.apm.agent.core.remote.GRPCChannelManager;
import org.apache.skywalking.apm.agent.core.remote.GRPCChannelStatus;
import org.apache.skywalking.apm.network.common.v3.Commands;
import org.apache.skywalking.apm.network.language.asyncprofiler.v10.AsyncProfilerTaskCommandQuery;
import org.apache.skywalking.apm.network.language.asyncprofiler.v10.AsyncProfilerTaskGrpc;
import org.apache.skywalking.apm.util.RunnableWithExceptionProtection;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import static org.apache.skywalking.apm.agent.core.conf.Config.Collector.GRPC_UPSTREAM_TIMEOUT;
@DefaultImplementor
public class AsyncProfilerTaskChannelService implements BootService, Runnable, GRPCChannelListener {
private static final ILog LOGGER = LogManager.getLogger(AsyncProfilerTaskChannelService.class);
// channel status
private volatile GRPCChannelStatus status = GRPCChannelStatus.DISCONNECT;
private volatile AsyncProfilerTaskGrpc.AsyncProfilerTaskBlockingStub asyncProfilerTaskBlockingStub;
// query task schedule
private volatile ScheduledFuture<?> getTaskFuture;
@Override
public void run() {
if (status == GRPCChannelStatus.CONNECTED) {
try {
// test start command and 10s after put stop command
long lastCommandCreateTime = ServiceManager.INSTANCE
.findService(AsyncProfilerTaskExecutionService.class).getLastCommandCreateTime();
AsyncProfilerTaskCommandQuery query = AsyncProfilerTaskCommandQuery.newBuilder()
.setServiceInstance(Config.Agent.INSTANCE_NAME)
.setService(Config.Agent.SERVICE_NAME)
.setLastCommandTime(lastCommandCreateTime)
.build();
Commands commands = asyncProfilerTaskBlockingStub.withDeadlineAfter(GRPC_UPSTREAM_TIMEOUT, TimeUnit.SECONDS)
.getAsyncProfilerTaskCommands(query);
ServiceManager.INSTANCE.findService(CommandService.class).receiveCommand(commands);
} catch (Throwable t) {
if (!(t instanceof StatusRuntimeException)) {
LOGGER.error(t, "fail to query async-profiler task from backend");
return;
}
final StatusRuntimeException statusRuntimeException = (StatusRuntimeException) t;
if (Status.Code.UNIMPLEMENTED.equals(statusRuntimeException.getStatus().getCode())) {
LOGGER.warn("Backend doesn't support async-profiler, async-profiler will be disabled");
if (getTaskFuture != null) {
getTaskFuture.cancel(true);
}
}
}
}
}
@Override
public void statusChanged(GRPCChannelStatus status) {
if (GRPCChannelStatus.CONNECTED.equals(status)) {
Channel channel = ServiceManager.INSTANCE.findService(GRPCChannelManager.class).getChannel();
asyncProfilerTaskBlockingStub = AsyncProfilerTaskGrpc.newBlockingStub(channel);
} else {
asyncProfilerTaskBlockingStub = null;
}
this.status = status;
}
@Override
public void prepare() throws Throwable {
ServiceManager.INSTANCE.findService(GRPCChannelManager.class).addChannelListener(this);
}
@Override
public void boot() throws Throwable {
if (Config.AsyncProfiler.ACTIVE) {
getTaskFuture = Executors.newSingleThreadScheduledExecutor(
new DefaultNamedThreadFactory("AsyncProfilerGetTaskService")
).scheduleWithFixedDelay(
new RunnableWithExceptionProtection(
this,
t -> LOGGER.error("Query async profiler task list failure.", t)
), 0, Config.Collector.GET_PROFILE_TASK_INTERVAL, TimeUnit.SECONDS
);
}
}
@Override
public void onComplete() throws Throwable {
}
@Override
public void shutdown() throws Throwable {
if (getTaskFuture != null) {
getTaskFuture.cancel(true);
}
}
}

160
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java Normal file
View File

@ -0,0 +1,160 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.apm.agent.core.asyncprofiler;
import one.profiler.AsyncProfiler;
import org.apache.skywalking.apm.agent.core.boot.BootService;
import org.apache.skywalking.apm.agent.core.boot.DefaultImplementor;
import org.apache.skywalking.apm.agent.core.boot.DefaultNamedThreadFactory;
import org.apache.skywalking.apm.agent.core.boot.ServiceManager;
import org.apache.skywalking.apm.agent.core.logging.api.ILog;
import org.apache.skywalking.apm.agent.core.logging.api.LogManager;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.util.Objects;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
@DefaultImplementor
public class AsyncProfilerTaskExecutionService implements BootService {
private static final ILog LOGGER = LogManager.getLogger(AsyncProfilerTaskChannelService.class);
private AsyncProfiler asyncProfilerInstance;
private static final String SUCCESS_RESULT = "Profiling started\n";
// profile executor thread pool, only running one thread
private volatile ScheduledExecutorService asyncProfilerExecutor;
// last command create time, use to next query task list
private volatile long lastCommandCreateTime = -1;
// task schedule future
private volatile ScheduledFuture<?> scheduledFuture;
public void processAsyncProfilerTask(AsyncProfilerTask task) {
if (task.getCreateTime() <= lastCommandCreateTime) {
LOGGER.warn("get repeat task because createTime is less than lastCommandCreateTime");
return;
}
lastCommandCreateTime = task.getCreateTime();
LOGGER.info("add async profiler task: {}", task.getTaskId());
// add task to list
getAsyncProfilerExecutor().execute(() -> {
try {
if (Objects.nonNull(scheduledFuture) && !scheduledFuture.isDone()) {
LOGGER.info("AsyncProfilerTask already running");
return;
}
String result = task.start(getAsyncProfiler());
if (!SUCCESS_RESULT.equals(result)) {
stopWhenError(task, result);
return;
}
scheduledFuture = getAsyncProfilerExecutor().schedule(
() -> stopWhenSuccess(task), task.getDuration(), TimeUnit.SECONDS
);
} catch (IOException e) {
LOGGER.error("AsyncProfilerTask executor error:" + e.getMessage(), e);
}
});
}
private void stopWhenError(AsyncProfilerTask task, String errorMessage) {
LOGGER.error("AsyncProfilerTask fails to start: " + errorMessage);
AsyncProfilerDataSender dataSender = ServiceManager.INSTANCE.findService(AsyncProfilerDataSender.class);
dataSender.sendError(task, errorMessage);
}
private void stopWhenSuccess(AsyncProfilerTask task) {
try {
File dumpFile = task.stop(getAsyncProfiler());
// stop task
try (FileInputStream fileInputStream = new FileInputStream(dumpFile)) {
// upload file
FileChannel channel = fileInputStream.getChannel();
AsyncProfilerDataSender dataSender = ServiceManager.INSTANCE.findService(AsyncProfilerDataSender.class);
dataSender.sendData(task, channel);
}
if (!dumpFile.delete()) {
LOGGER.warn("Fail to delete the dump file of async profiler.");
}
} catch (Exception e) {
LOGGER.error("stop async profiler task error", e);
return;
}
}
public long getLastCommandCreateTime() {
return lastCommandCreateTime;
}
@Override
public void prepare() throws Throwable {
}
@Override
public void boot() throws Throwable {
}
@Override
public void onComplete() throws Throwable {
}
@Override
public void shutdown() throws Throwable {
getAsyncProfilerExecutor().shutdown();
if (Objects.nonNull(scheduledFuture)) {
scheduledFuture.cancel(true);
scheduledFuture = null;
}
}
private AsyncProfiler getAsyncProfiler() {
if (asyncProfilerInstance == null) {
asyncProfilerInstance = AsyncProfiler.getInstance();
}
return asyncProfilerInstance;
}
private ScheduledExecutorService getAsyncProfilerExecutor() {
if (asyncProfilerExecutor == null) {
synchronized (this) {
if (asyncProfilerExecutor == null) {
asyncProfilerExecutor = Executors.newSingleThreadScheduledExecutor(
new DefaultNamedThreadFactory("ASYNC-PROFILING-TASK"));
}
}
}
return asyncProfilerExecutor;
}
}

5
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/CommandExecutorService.java
View File

@ -21,9 +21,11 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.skywalking.apm.agent.core.boot.BootService;
import org.apache.skywalking.apm.agent.core.boot.DefaultImplementor;
import org.apache.skywalking.apm.agent.core.commands.executor.AsyncProfilerCommandExecutor;
import org.apache.skywalking.apm.agent.core.commands.executor.ConfigurationDiscoveryCommandExecutor;
import org.apache.skywalking.apm.agent.core.commands.executor.NoopCommandExecutor;
import org.apache.skywalking.apm.agent.core.commands.executor.ProfileTaskCommandExecutor;
import org.apache.skywalking.apm.network.trace.component.command.AsyncProfilerTaskCommand;
import org.apache.skywalking.apm.network.trace.component.command.BaseCommand;
import org.apache.skywalking.apm.network.trace.component.command.ConfigurationDiscoveryCommand;
import org.apache.skywalking.apm.network.trace.component.command.ProfileTaskCommand;
@ -48,6 +50,9 @@ public class CommandExecutorService implements BootService, CommandExecutor {
//Get ConfigurationDiscoveryCommand executor.
commandExecutorMap.put(ConfigurationDiscoveryCommand.NAME, new ConfigurationDiscoveryCommandExecutor());
// AsyncProfiler task executor
commandExecutorMap.put(AsyncProfilerTaskCommand.NAME, new AsyncProfilerCommandExecutor());
}
@Override

44
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java Normal file
View File

@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.apm.agent.core.commands.executor;
import org.apache.skywalking.apm.agent.core.asyncprofiler.AsyncProfilerTask;
import org.apache.skywalking.apm.agent.core.asyncprofiler.AsyncProfilerTaskExecutionService;
import org.apache.skywalking.apm.agent.core.boot.ServiceManager;
import org.apache.skywalking.apm.agent.core.commands.CommandExecutionException;
import org.apache.skywalking.apm.agent.core.commands.CommandExecutor;
import org.apache.skywalking.apm.agent.core.conf.Config;
import org.apache.skywalking.apm.network.trace.component.command.AsyncProfilerTaskCommand;
import org.apache.skywalking.apm.network.trace.component.command.BaseCommand;
public class AsyncProfilerCommandExecutor implements CommandExecutor {
@Override
public void execute(BaseCommand command) throws CommandExecutionException {
AsyncProfilerTaskCommand asyncProfilerTaskCommand = (AsyncProfilerTaskCommand) command;
AsyncProfilerTask asyncProfilerTask = new AsyncProfilerTask();
asyncProfilerTask.setTaskId(asyncProfilerTaskCommand.getTaskId());
int duration = Math.min(Config.AsyncProfiler.MAX_DURATION, asyncProfilerTaskCommand.getDuration());
asyncProfilerTask.setDuration(duration);
asyncProfilerTask.setExecArgs(asyncProfilerTaskCommand.getExecArgs());
asyncProfilerTask.setCreateTime(asyncProfilerTaskCommand.getCreateTime());
ServiceManager.INSTANCE.findService(AsyncProfilerTaskExecutionService.class)
.processAsyncProfilerTask(asyncProfilerTask);
}
}

27
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java
View File

@ -252,6 +252,33 @@ public class Config {
public static int SNAPSHOT_TRANSPORT_BUFFER_SIZE = 500;
}
public static class AsyncProfiler {
/**
* If true, async profiler will be enabled when user creates a new async profiler task.
* If false, it will be disabled.
* The default value is true.
*/
public static boolean ACTIVE = true;
/**
* Max execution time(second) for the Async Profiler. The task will be stopped even if a longer time is specified.
* default 10min.
*/
public static int MAX_DURATION = 600;
/**
* Path for the JFR outputs from the Async Profiler.
* If the parameter is not empty, the file will be created in the specified directory,
* otherwise the Files.createTemp method will be used to create the file.
*/
public static String OUTPUT_PATH = "";
/**
* The size of the chunk when uploading jfr
*/
public static final int DATA_CHUNK_SIZE = 1024 * 1024;
}
public static class Meter {
/**
* If true, skywalking agent will enable sending meters. Otherwise disable meter report.

3
apm-sniffer/apm-agent-core/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService
View File

@ -36,3 +36,6 @@ org.apache.skywalking.apm.agent.core.remote.LogReportServiceClient
org.apache.skywalking.apm.agent.core.conf.dynamic.ConfigurationDiscoveryService
org.apache.skywalking.apm.agent.core.remote.EventReportServiceClient
org.apache.skywalking.apm.agent.core.ServiceInstanceGenerator
org.apache.skywalking.apm.agent.core.asyncprofiler.AsyncProfilerTaskExecutionService
org.apache.skywalking.apm.agent.core.asyncprofiler.AsyncProfilerTaskChannelService
org.apache.skywalking.apm.agent.core.asyncprofiler.AsyncProfilerDataSender

4
apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/boot/ServiceManagerTest.java
View File

@ -59,7 +59,7 @@ public class ServiceManagerTest {
public void testServiceDependencies() throws Exception {
HashMap<Class, BootService> registryService = getFieldValue(ServiceManager.INSTANCE, "bootedServices");
assertThat(registryService.size(), is(20));
assertThat(registryService.size(), is(23));
assertTraceSegmentServiceClient(ServiceManager.INSTANCE.findService(TraceSegmentServiceClient.class));
assertContextManager(ServiceManager.INSTANCE.findService(ContextManager.class));
@ -109,7 +109,7 @@ public class ServiceManagerTest {
assertNotNull(service);
List<GRPCChannelListener> listeners = getFieldValue(service, "listeners");
assertEquals(listeners.size(), 10);
assertEquals(listeners.size(), 12);
}
private void assertSamplingService(SamplingService service) {

6
apm-sniffer/config/agent.config
View File

@ -164,6 +164,12 @@ profile.duration=${SW_AGENT_PROFILE_DURATION:10}
profile.dump_max_stack_depth=${SW_AGENT_PROFILE_DUMP_MAX_STACK_DEPTH:500}
# Snapshot transport to backend buffer size
profile.snapshot_transport_buffer_size=${SW_AGENT_PROFILE_SNAPSHOT_TRANSPORT_BUFFER_SIZE:4500}
# If true, async profiler will be enabled when user creates a new async profiler task. If false, it will be disabled. The default value is true.
asyncprofiler.active=${SW_AGENT_ASYNC_PROFILER_ACTIVE:true}
# Max execution time(second) for the Async Profiler. The task will be stopped even if a longer time is specified. default 10min.
asyncprofiler.max_duration=${SW_AGENT_ASYNC_PROFILER_MAX_DURATION:600}
# Path for the JFR outputs from the Async Profiler. If the parameter is not empty, the file will be created in the specified directory, otherwise the Files.createTemp method will be used to create the file.
asyncprofiler.output_path=${SW_AGENT_ASYNC_PROFILER_OUTPUT_PATH:}
# If true, the agent collects and reports metrics to the backend.
meter.active=${SW_METER_ACTIVE:true}
# Report meters interval. The unit is second

1
dist-material/LICENSE
View File

@ -222,6 +222,7 @@ The text of each license is the standard Apache 2.0 license.
Google: jsr305 3.0.2: http://central.maven.org/maven2/com/google/code/findbugs/jsr305/3.0.0/jsr305-3.0.0.pom , Apache 2.0
Google: guava 32.0.1: https://github.com/google/guava , Apache 2.0
netty 4.1.100: https://github.com/netty/netty/blob/4.1/LICENSE.txt, Apache 2.0
async-profiler 3.0: https://github.com/async-profiler/async-profiler/blob/v3.0/LICENSE, Apache 2.0
========================================================================
BSD licenses

6
pom.xml
View File

@ -97,6 +97,7 @@
<netty-tcnative-boringssl-static.version>2.0.48.Final</netty-tcnative-boringssl-static.version>
<javax.annotation-api.version>1.3.2</javax.annotation-api.version>
<objenesis.version>3.1</objenesis.version>
<async-profiler.version>3.0</async-profiler.version>
<!-- necessary for Java 9+ -->
<org.apache.tomcat.annotations-api.version>6.0.53</org.apache.tomcat.annotations-api.version>
@ -260,6 +261,11 @@
<version>${jmh.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>tools.profiler</groupId>
<artifactId>async-profiler</artifactId>
<version>${async-profiler.version}</version>
</dependency>
</dependencies>
</dependencyManagement>