diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 000000000..9ca28e951 --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: E2E + +on: + pull_request: + push: + branches: + - main + +jobs: + E2E: + name: Basic function + gRPC transport(Java8) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Build SkyWalking agent docker image + run: make build docker.java8 + - name: Build consumer & provider images + shell: bash + run: | + echo "::group::Install SNAPSHOT apm-application-toolkit" + ./mvnw -DskipTests -N install + ./mvnw -f apm-application-toolkit -DskipTests -am install + echo "::endgroup::" + echo "::group::" + SW_VERSION=$(./mvnw -q -DforceStdout -N org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate -Dexpression=project.version) + ./mvnw -B package -f ./test/e2e/base/pom.xml -Dsw.version=${SW_VERSION} + echo "::endgroup::" + - name: Run E2E Tests + uses: apache/skywalking-infra-e2e@7d55a252c35bf69f65dc64675a80699dc3c21bde + with: + e2e-file: test/e2e/case/base/e2e.yaml diff --git a/CHANGES.md b/CHANGES.md index be31fc26f..84872b93e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -21,8 +21,10 @@ Release Notes. * Add benchmark result for `exception-ignore` plugin and polish plugin guide. * Provide Alibaba Druid database connection pool plugin. * Provide HikariCP database connection pool plugin. -* fix the bug that springMVC plugin will not stop span when exception throws. - +* Fix a tracing context leak of SpringMVC plugin, when an internal exception throws due to response can't be found. +* Make GRPC log reporter sharing GRPC channel with other reporters of agent. Remove config items of `agent.conf`, `plugin.toolkit.log.grpc.reporter.server_host`, `plugin.toolkit.log.grpc.reporter.server_port`, and `plugin.toolkit.log.grpc.reporter.upstream_timeout`. + rename `plugin.toolkit.log.grpc.reporter.max_message_size` to `log.max_message_size`. +* Implement Kafka Log Reporter. Add config item of `agnt.conf`, `plugin.kafka.topic_logging`. #### Documentation diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java index b6464160b..4195d1943 100755 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java @@ -246,6 +246,13 @@ public class Config { public static int BUFFER_SIZE = 60 * 10; } + public static class Log { + /** + * The max size of message to send to server.Default is 10 MB. + */ + public static int MAX_MESSAGE_SIZE = 10 * 1024 * 1024; + } + public static class Buffer { public static int CHANNEL_SIZE = 5; diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/remote/LogReportServiceClient.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/remote/LogReportServiceClient.java index ec9ef863e..8db7fbd3c 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/remote/LogReportServiceClient.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/remote/LogReportServiceClient.java @@ -18,25 +18,51 @@ package org.apache.skywalking.apm.agent.core.remote; +import io.grpc.Channel; +import io.grpc.stub.StreamObserver; import java.util.List; +import java.util.Objects; import java.util.Properties; +import java.util.concurrent.TimeUnit; import org.apache.skywalking.apm.agent.core.boot.BootService; import org.apache.skywalking.apm.agent.core.boot.DefaultImplementor; +import org.apache.skywalking.apm.agent.core.boot.ServiceManager; +import org.apache.skywalking.apm.agent.core.conf.Config; +import org.apache.skywalking.apm.agent.core.conf.Config.Collector; +import org.apache.skywalking.apm.agent.core.conf.Config.Log; +import org.apache.skywalking.apm.agent.core.logging.api.ILog; +import org.apache.skywalking.apm.agent.core.logging.api.LogManager; +import org.apache.skywalking.apm.agent.core.util.CollectionUtil; +import org.apache.skywalking.apm.commons.datacarrier.DataCarrier; +import org.apache.skywalking.apm.commons.datacarrier.buffer.BufferStrategy; import org.apache.skywalking.apm.commons.datacarrier.consumer.IConsumer; +import org.apache.skywalking.apm.network.common.v3.Commands; import org.apache.skywalking.apm.network.logging.v3.LogData; +import org.apache.skywalking.apm.network.logging.v3.LogReportServiceGrpc; @DefaultImplementor -public class LogReportServiceClient implements BootService, IConsumer { +public class LogReportServiceClient implements BootService, GRPCChannelListener, IConsumer { + private static final ILog LOGGER = LogManager.getLogger(LogReportServiceClient.class); + + private volatile DataCarrier carrier; + private volatile GRPCChannelStatus status; + + private volatile LogReportServiceGrpc.LogReportServiceStub logReportServiceStub; @Override public void prepare() throws Throwable { - + ServiceManager.INSTANCE.findService(GRPCChannelManager.class).addChannelListener(this); } @Override public void boot() throws Throwable { - + carrier = new DataCarrier<>("gRPC-log", "gRPC-log", + Config.Buffer.CHANNEL_SIZE, + Config.Buffer.BUFFER_SIZE, + BufferStrategy.IF_POSSIBLE + ); + carrier.consume(this, 1); } @Override @@ -44,9 +70,12 @@ public class LogReportServiceClient implements BootService, IConsumer { } - @Override - public void shutdown() throws Throwable { - + public void produce(LogData logData) { + if (Objects.nonNull(logData) && !carrier.produce(logData)) { + if (LOGGER.isDebugEnable()) { + LOGGER.debug("One log has been abandoned, cause by buffer is full."); + } + } } @Override @@ -54,22 +83,71 @@ public class LogReportServiceClient implements BootService, IConsumer { } - public void produce(LogData logData) { + @Override + public void consume(final List dataList) { + if (CollectionUtil.isEmpty(dataList)) { + return; + } + if (GRPCChannelStatus.CONNECTED.equals(status)) { + GRPCStreamServiceStatus status = new GRPCStreamServiceStatus(false); + + StreamObserver logDataStreamObserver = logReportServiceStub + .withDeadlineAfter(Collector.GRPC_UPSTREAM_TIMEOUT, TimeUnit.SECONDS) + .collect( + new StreamObserver() { + @Override + public void onNext(final Commands commands) { + + } + + @Override + public void onError(final Throwable throwable) { + status.finished(); + LOGGER.error(throwable, "Try to send {} log data to collector, with unexpected exception.", + dataList.size() + ); + ServiceManager.INSTANCE + .findService(GRPCChannelManager.class) + .reportError(throwable); + } + + @Override + public void onCompleted() { + status.finished(); + } + }); + + for (final LogData logData : dataList) { + logDataStreamObserver.onNext(logData); + } + logDataStreamObserver.onCompleted(); + status.wait4Finish(); + } } @Override - public void consume(List data) { - - } - - @Override - public void onError(List data, Throwable t) { - + public void onError(final List data, final Throwable t) { + LOGGER.error(t, "Try to consume {} log data to sender, with unexpected exception.", data.size()); } @Override public void onExit() { } + + @Override + public void statusChanged(GRPCChannelStatus status) { + if (GRPCChannelStatus.CONNECTED.equals(status)) { + Channel channel = ServiceManager.INSTANCE.findService(GRPCChannelManager.class).getChannel(); + logReportServiceStub = LogReportServiceGrpc.newStub(channel) + .withMaxOutboundMessageSize(Log.MAX_MESSAGE_SIZE); + } + this.status = status; + } + + @Override + public void shutdown() { + carrier.shutdownConsumers(); + } } diff --git a/apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/boot/ServiceManagerTest.java b/apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/boot/ServiceManagerTest.java index b51ce2d12..48af242a4 100644 --- a/apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/boot/ServiceManagerTest.java +++ b/apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/boot/ServiceManagerTest.java @@ -32,6 +32,7 @@ import org.apache.skywalking.apm.agent.core.profile.ProfileTaskChannelService; import org.apache.skywalking.apm.agent.core.profile.ProfileTaskExecutionService; import org.apache.skywalking.apm.agent.core.remote.GRPCChannelListener; import org.apache.skywalking.apm.agent.core.remote.GRPCChannelManager; +import org.apache.skywalking.apm.agent.core.remote.LogReportServiceClient; import org.apache.skywalking.apm.agent.core.remote.TraceSegmentServiceClient; import org.apache.skywalking.apm.agent.core.sampling.SamplingService; import org.apache.skywalking.apm.agent.core.test.tools.AgentServiceRule; @@ -68,6 +69,7 @@ public class ServiceManagerTest { assertProfileTaskQueryService(ServiceManager.INSTANCE.findService(ProfileTaskChannelService.class)); assertProfileTaskExecuteService(ServiceManager.INSTANCE.findService(ProfileTaskExecutionService.class)); assertMeterRegisterService(ServiceManager.INSTANCE.findService(MeterService.class)); + assertLogReporterServiceClient(ServiceManager.INSTANCE.findService(LogReportServiceClient.class)); assertTracingContextListener(); assertIgnoreTracingContextListener(); @@ -107,7 +109,7 @@ public class ServiceManagerTest { assertNotNull(service); List listeners = getFieldValue(service, "listeners"); - assertEquals(listeners.size(), 9); + assertEquals(listeners.size(), 10); } private void assertSamplingService(SamplingService service) { @@ -126,6 +128,10 @@ public class ServiceManagerTest { assertNotNull(service); } + private void assertLogReporterServiceClient(LogReportServiceClient service) { + assertNotNull(service); + } + private T getFieldValue(Object instance, String fieldName) throws Exception { Field field = instance.getClass().getDeclaredField(fieldName); field.setAccessible(true); diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/GRPCLogReportServiceClient.java b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/GRPCLogReportServiceClient.java deleted file mode 100644 index 94259852d..000000000 --- a/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/GRPCLogReportServiceClient.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.apm.toolkit.logging.common.log; - -import io.grpc.CallOptions; -import io.grpc.Channel; -import io.grpc.ClientCall; -import io.grpc.ClientInterceptor; -import io.grpc.ClientInterceptors; -import io.grpc.ForwardingClientCall; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.Metadata; -import io.grpc.MethodDescriptor; -import io.grpc.StatusRuntimeException; -import io.grpc.stub.StreamObserver; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.skywalking.apm.agent.core.boot.OverrideImplementor; -import org.apache.skywalking.apm.agent.core.conf.Config; -import org.apache.skywalking.apm.agent.core.logging.api.ILog; -import org.apache.skywalking.apm.agent.core.logging.api.LogManager; -import org.apache.skywalking.apm.agent.core.remote.GRPCStreamServiceStatus; -import org.apache.skywalking.apm.agent.core.remote.LogReportServiceClient; -import org.apache.skywalking.apm.agent.core.util.CollectionUtil; -import org.apache.skywalking.apm.commons.datacarrier.DataCarrier; -import org.apache.skywalking.apm.commons.datacarrier.buffer.BufferStrategy; -import org.apache.skywalking.apm.network.common.v3.Commands; -import org.apache.skywalking.apm.network.logging.v3.LogData; -import org.apache.skywalking.apm.network.logging.v3.LogReportServiceGrpc; -import org.apache.skywalking.apm.util.StringUtil; - -/** - * Report log to server by grpc - */ -@OverrideImplementor(LogReportServiceClient.class) -public class GRPCLogReportServiceClient extends LogReportServiceClient { - - private static final ILog LOGGER = LogManager.getLogger(GRPCLogReportServiceClient.class); - - private volatile DataCarrier carrier; - - private LogReportServiceGrpc.LogReportServiceStub asyncStub; - - private ManagedChannel channel; - - private AtomicBoolean disconnected = new AtomicBoolean(false); - - private static final Metadata.Key AUTH_HEAD_HEADER_NAME = Metadata.Key.of("Authentication", Metadata.ASCII_STRING_MARSHALLER); - - @Override - public void boot() throws Throwable { - carrier = new DataCarrier<>("gRPC-log", "gRPC-log", - Config.Buffer.CHANNEL_SIZE, - Config.Buffer.BUFFER_SIZE, - BufferStrategy.IF_POSSIBLE - ); - carrier.consume(this, 1); - channel = ManagedChannelBuilder - .forAddress( - ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.SERVER_HOST, - ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.SERVER_PORT - ) - .usePlaintext() - .build(); - - Channel decoratedChannel = decorateLogChannelWithAuthentication(channel); - asyncStub = LogReportServiceGrpc.newStub(decoratedChannel) - .withMaxOutboundMessageSize( - ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.MAX_MESSAGE_SIZE); - } - - @Override - public void shutdown() { - try { - carrier.shutdownConsumers(); - if (channel != null) { - channel.shutdownNow(); - } - } catch (Throwable t) { - LOGGER.error(t.getMessage(), t); - } - } - - @Override - public void produce(LogData logData) { - if (Objects.nonNull(logData) && !carrier.produce(logData)) { - if (LOGGER.isDebugEnable()) { - LOGGER.debug("One log has been abandoned, cause by buffer is full."); - } - } - } - - @Override - public void consume(final List dataList) { - if (CollectionUtil.isEmpty(dataList)) { - return; - } - StreamObserver reportStreamObserver = null; - final GRPCStreamServiceStatus waitStatus = new GRPCStreamServiceStatus(false); - try { - reportStreamObserver = asyncStub.withDeadlineAfter( - ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.UPSTREAM_TIMEOUT, TimeUnit.SECONDS - ).collect(new StreamObserver() { - @Override - public void onNext(Commands commands) { - } - - @Override - public void onError(Throwable t) { - waitStatus.finished(); - if (disconnected.compareAndSet(false, true)) { - LOGGER.error("Send log to gRPC server fail with an internal exception.", t); - } - - LOGGER.error(t, "Try to send {} log data to collector, with unexpected exception.", - dataList.size() - ); - } - - @Override - public void onCompleted() { - disconnected.compareAndSet(true, false); - waitStatus.finished(); - } - }); - - for (final LogData logData : dataList) { - reportStreamObserver.onNext(logData); - } - } catch (Throwable e) { - if (!(e instanceof StatusRuntimeException)) { - LOGGER.error(e, "Report log failure with the gRPC client."); - } - } finally { - if (reportStreamObserver != null) { - reportStreamObserver.onCompleted(); - } - waitStatus.wait4Finish(); - } - } - - private Channel decorateLogChannelWithAuthentication(Channel channel) { - if (StringUtil.isEmpty(Config.Agent.AUTHENTICATION)) { - return channel; - } - - return ClientInterceptors.intercept(channel, new ClientInterceptor() { - @Override - public ClientCall interceptCall(MethodDescriptor method, - CallOptions options, Channel channel) { - return new ForwardingClientCall.SimpleForwardingClientCall(channel.newCall(method, options)) { - @Override - public void start(Listener responseListener, Metadata headers) { - headers.put(AUTH_HEAD_HEADER_NAME, Config.Agent.AUTHENTICATION); - super.start(responseListener, headers); - } - }; - } - }); - } - -} diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/ToolkitConfig.java b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/ToolkitConfig.java index ee6d001bc..1337e171f 100644 --- a/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/ToolkitConfig.java +++ b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/ToolkitConfig.java @@ -30,31 +30,6 @@ public class ToolkitConfig { * Whether or not to transmit logged data as formatted or un-formatted. */ public static boolean TRANSMIT_FORMATTED = true; - - public static class GRPC { - @PluginConfig(root = ToolkitConfig.class) - public static class Reporter { - /** - * The host of gRPC log server. - */ - public static String SERVER_HOST = "127.0.0.1"; - - /** - * The port of gRPC log server. - */ - public static int SERVER_PORT = 11800; - - /** - * The max size of message to send to server.Default is 10 MB. - */ - public static int MAX_MESSAGE_SIZE = 10 * 1024 * 1024; - - /** - * How long grpc client will timeout in sending data to upstream. The unit is second. - */ - public static int UPSTREAM_TIMEOUT = 30; - } - } } } } diff --git a/apm-sniffer/config/agent.config b/apm-sniffer/config/agent.config index 89ae6e2a3..c8d10d760 100755 --- a/apm-sniffer/config/agent.config +++ b/apm-sniffer/config/agent.config @@ -160,8 +160,10 @@ profile.snapshot_transport_buffer_size=${SW_AGENT_PROFILE_SNAPSHOT_TRANSPORT_BUF meter.active=${SW_METER_ACTIVE:true} # Report meters interval. The unit is second meter.report_interval=${SW_METER_REPORT_INTERVAL:20} -# Max size of the meter pool +# Max size of the meter pool meter.max_meter_size=${SW_METER_MAX_METER_SIZE:500} +# The max size of message to send to server.Default is 10 MB +log.max_message_size=${SW_GRPC_LOG_MAX_MESSAGE_SIZE:10485760} # Mount the specific folders of the plugins. Plugins in mounted folders would work. plugin.mount=${SW_MOUNT_FOLDERS:plugins,activations} @@ -221,7 +223,7 @@ plugin.dubbo.collect_consumer_arguments=${SW_PLUGIN_DUBBO_COLLECT_CONSUMER_ARGUM plugin.dubbo.consumer_arguments_length_threshold=${SW_PLUGIN_DUBBO_CONSUMER_ARGUMENTS_LENGTH_THRESHOLD:256} # Apache Dubbo provider collect `arguments` in RPC call, use `Object#toString` to collect `arguments`. plugin.dubbo.collect_provider_arguments=${SW_PLUGIN_DUBBO_COLLECT_PROVIDER_ARGUMENTS:false} -# When `plugin.dubbo.collect_provider_arguments` is `true`, Arguments of length from the front will to the OAP backend +# When `plugin.dubbo.collect_provider_arguments` is `true`, Arguments of length from the front will to the OAP backend plugin.dubbo.provider_arguments_length_threshold=${SW_PLUGIN_DUBBO_PROVIDER_ARGUMENTS_LENGTH_THRESHOLD:256} # A list of host/port pairs to use for establishing the initial connection to the Kafka cluster. plugin.kafka.bootstrap_servers=${SW_KAFKA_BOOTSTRAP_SERVERS:localhost:9092} @@ -242,20 +244,14 @@ plugin.kafka.topic_segment=${SW_PLUGIN_KAFKA_TOPIC_SEGMENT:skywalking-segments} plugin.kafka.topic_profiling=${SW_PLUGIN_KAFKA_TOPIC_PROFILINGS:skywalking-profilings} # Specify which Kafka topic name for the register or heartbeat data of Service Instance to report to. plugin.kafka.topic_management=${SW_PLUGIN_KAFKA_TOPIC_MANAGEMENT:skywalking-managements} -# isolate multi OAP server when using same Kafka cluster (final topic name will append namespace before Kafka topics with `-` ). +# Specify which Kafka topic name for the logging data to report to. +plugin.kafka.topic_logging=${SW_PLUGIN_KAFKA_TOPIC_LOGGING:skywalking-logging} +# isolate multi OAP server when using same Kafka cluster (final topic name will append namespace before Kafka topics with `-` ). plugin.kafka.namespace=${SW_KAFKA_NAMESPACE:} # Match spring beans with regular expression for the class name. Multiple expressions could be separated by a comma. This only works when `Spring annotation plugin` has been activated. plugin.springannotation.classname_match_regex=${SW_SPRINGANNOTATION_CLASSNAME_MATCH_REGEX:} # Whether or not to transmit logged data as formatted or un-formatted. plugin.toolkit.log.transmit_formatted=${SW_PLUGIN_TOOLKIT_LOG_TRANSMIT_FORMATTED:true} -# Specify which grpc server's host for log data to report to. -plugin.toolkit.log.grpc.reporter.server_host=${SW_GRPC_LOG_SERVER_HOST:127.0.0.1} -# Specify which grpc server's port for log data to report to. -plugin.toolkit.log.grpc.reporter.server_port=${SW_GRPC_LOG_SERVER_PORT:11800} -# Specify the maximum size of log data for grpc client to report to. -plugin.toolkit.log.grpc.reporter.max_message_size=${SW_PLUGIN_TOOLKIT_LOG_GRPC_REPORTER_MAX_MESSAGE_SIZE:10485760} -# How long grpc client will timeout in sending data to upstream. Unit is second. -plugin.toolkit.log.grpc.reporter.upstream_timeout=${SW_PLUGIN_TOOLKIT_LOG_GRPC_REPORTER_UPSTREAM_TIMEOUT:30} # If set to true, the parameters of Redis commands would be collected by Lettuce agent. plugin.lettuce.trace_redis_parameters=${SW_PLUGIN_LETTUCE_TRACE_REDIS_PARAMETERS:false} # If set to positive number and `plugin.lettuce.trace_redis_parameters` is set to `true`, Redis command parameters would be collected and truncated to this length. diff --git a/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaLogReporterServiceClient.java b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaLogReporterServiceClient.java new file mode 100644 index 000000000..5c65145f3 --- /dev/null +++ b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaLogReporterServiceClient.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.apm.agent.core.kafka; + +import java.util.List; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.utils.Bytes; +import org.apache.skywalking.apm.agent.core.boot.OverrideImplementor; +import org.apache.skywalking.apm.agent.core.boot.ServiceManager; +import org.apache.skywalking.apm.agent.core.remote.LogReportServiceClient; +import org.apache.skywalking.apm.agent.core.util.CollectionUtil; +import org.apache.skywalking.apm.network.logging.v3.LogData; + +@OverrideImplementor(LogReportServiceClient.class) +public class KafkaLogReporterServiceClient extends LogReportServiceClient implements KafkaConnectionStatusListener { + + private String topic; + private KafkaProducer producer; + + @Override + public void prepare() { + KafkaProducerManager producerManager = ServiceManager.INSTANCE.findService(KafkaProducerManager.class); + producerManager.addListener(this); + topic = producerManager.formatTopicNameThenRegister(KafkaReporterPluginConfig.Plugin.Kafka.TOPIC_LOGGING); + } + + @Override + public void produce(final LogData logData) { + super.produce(logData); + } + + @Override + public void consume(final List dataList) { + if (producer == null || CollectionUtil.isEmpty(dataList)) { + return; + } + + for (LogData data : dataList) { + producer.send(new ProducerRecord<>(topic, data.getService(), Bytes.wrap(data.toByteArray()))); + } + } + + @Override + public void onStatusChanged(final org.apache.skywalking.apm.agent.core.kafka.KafkaConnectionStatus status) { + if (status == KafkaConnectionStatus.CONNECTED) { + producer = ServiceManager.INSTANCE.findService(KafkaProducerManager.class).getProducer(); + } + } +} diff --git a/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaReporterPluginConfig.java b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaReporterPluginConfig.java index 1b4d582f0..e3e43d63e 100644 --- a/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaReporterPluginConfig.java +++ b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaReporterPluginConfig.java @@ -42,6 +42,8 @@ public class KafkaReporterPluginConfig { public static String TOPIC_METER = "skywalking-meters"; + public static String TOPIC_LOGGING = "skywalking-logging"; + public static String NAMESPACE = ""; /** diff --git a/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService index 8e5fafef0..df7049893 100644 --- a/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService +++ b/apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService @@ -20,4 +20,5 @@ org.apache.skywalking.apm.agent.core.kafka.KafkaServiceManagementServiceClient org.apache.skywalking.apm.agent.core.kafka.KafkaJVMMetricsSender org.apache.skywalking.apm.agent.core.kafka.KafkaTraceSegmentServiceClient org.apache.skywalking.apm.agent.core.kafka.KafkaProfileSnapshotSender -org.apache.skywalking.apm.agent.core.kafka.KafkaMeterSender \ No newline at end of file +org.apache.skywalking.apm.agent.core.kafka.KafkaMeterSender +org.apache.skywalking.apm.agent.core.kafka.KafkaLogReporterServiceClient diff --git a/docs/en/setup/service-agent/java-agent/Application-toolkit-log4j-1.x.md b/docs/en/setup/service-agent/java-agent/Application-toolkit-log4j-1.x.md index a14c23885..b180d164a 100644 --- a/docs/en/setup/service-agent/java-agent/Application-toolkit-log4j-1.x.md +++ b/docs/en/setup/service-agent/java-agent/Application-toolkit-log4j-1.x.md @@ -43,8 +43,5 @@ log4j.appender.CustomAppender.layout.ConversionPattern=[%t] %-5p %c %x - %m%n * Add config of the plugin or use default ```properties -plugin.toolkit.log.grpc.reporter.server_host=${SW_GRPC_LOG_SERVER_HOST:127.0.0.1} -plugin.toolkit.log.grpc.reporter.server_port=${SW_GRPC_LOG_SERVER_PORT:11800} -plugin.toolkit.log.grpc.reporter.max_message_size=${SW_GRPC_LOG_MAX_MESSAGE_SIZE:10485760} -plugin.toolkit.log.grpc.reporter.upstream_timeout=${SW_GRPC_LOG_GRPC_UPSTREAM_TIMEOUT:30} +log.max_message_size=${SW_GRPC_LOG_MAX_MESSAGE_SIZE:10485760} ``` diff --git a/docs/en/setup/service-agent/java-agent/Application-toolkit-log4j-2.x.md b/docs/en/setup/service-agent/java-agent/Application-toolkit-log4j-2.x.md index d688ce84c..18267da32 100644 --- a/docs/en/setup/service-agent/java-agent/Application-toolkit-log4j-2.x.md +++ b/docs/en/setup/service-agent/java-agent/Application-toolkit-log4j-2.x.md @@ -125,10 +125,7 @@ The gRPC report could forward the collected logs to SkyWalking OAP server, or [S * Add config of the plugin or use default ```properties -plugin.toolkit.log.grpc.reporter.server_host=${SW_GRPC_LOG_SERVER_HOST:127.0.0.1} -plugin.toolkit.log.grpc.reporter.server_port=${SW_GRPC_LOG_SERVER_PORT:11800} -plugin.toolkit.log.grpc.reporter.max_message_size=${SW_GRPC_LOG_MAX_MESSAGE_SIZE:10485760} -plugin.toolkit.log.grpc.reporter.upstream_timeout=${SW_GRPC_LOG_GRPC_UPSTREAM_TIMEOUT:30} +log.max_message_size=${SW_GRPC_LOG_MAX_MESSAGE_SIZE:10485760} ``` diff --git a/docs/en/setup/service-agent/java-agent/Application-toolkit-logback-1.x.md b/docs/en/setup/service-agent/java-agent/Application-toolkit-logback-1.x.md index 133f1cf7a..8023dfed5 100644 --- a/docs/en/setup/service-agent/java-agent/Application-toolkit-logback-1.x.md +++ b/docs/en/setup/service-agent/java-agent/Application-toolkit-logback-1.x.md @@ -143,10 +143,7 @@ The gRPC reporter could forward the collected logs to SkyWalking OAP server, or * Add config of the plugin or use default ```properties -plugin.toolkit.log.grpc.reporter.server_host=${SW_GRPC_LOG_SERVER_HOST:127.0.0.1} -plugin.toolkit.log.grpc.reporter.server_port=${SW_GRPC_LOG_SERVER_PORT:11800} -plugin.toolkit.log.grpc.reporter.max_message_size=${SW_GRPC_LOG_MAX_MESSAGE_SIZE:10485760} -plugin.toolkit.log.grpc.reporter.upstream_timeout=${SW_GRPC_LOG_GRPC_UPSTREAM_TIMEOUT:30} +log.max_message_size=${SW_GRPC_LOG_MAX_MESSAGE_SIZE:10485760} ``` ## Transmitting un-formatted messages diff --git a/docs/en/setup/service-agent/java-agent/configurations.md b/docs/en/setup/service-agent/java-agent/configurations.md index 324dbd960..1dfe1708e 100644 --- a/docs/en/setup/service-agent/java-agent/configurations.md +++ b/docs/en/setup/service-agent/java-agent/configurations.md @@ -56,6 +56,7 @@ property key | Description | Default | `meter.active`|If true, the agent collects and reports metrics to the backend.|`true`| `meter.report_interval`|Report meters interval. The unit is second|`20`| `meter.max_meter_size`| Max size of the meter pool |`500`| +`log.max_message_size`| The max size of message to send to server.Default is 10 MB. |`10485760`| `plugin.mount` | Mount the specific folders of the plugins. Plugins in mounted folders would work. | `plugins,activations` | `plugin.peer_max_length `|Peer maximum description limit.|`200`| `plugin.exclude_plugins `|Exclude some plugins define in plugins dir.Plugin names is defined in [Agent plugin list](Plugin-list.md)|`""`| @@ -95,13 +96,10 @@ property key | Description | Default | `plugin.kafka.topic_segment` | Specify which Kafka topic name for traces data to report to. | `skywalking-segments` | `plugin.kafka.topic_profiling` | Specify which Kafka topic name for Thread Profiling snapshot to report to. | `skywalking-profilings` | `plugin.kafka.topic_management` | Specify which Kafka topic name for the register or heartbeat data of Service Instance to report to. | `skywalking-managements` | +`plugin.kafka.topic_logging` | Specify which Kafka topic name for the logging data to report to. | `skywalking-logging` | `plugin.kafka.namespace` | isolate multi OAP server when using same Kafka cluster (final topic name will append namespace before Kafka topics with `-` ). | `` | `plugin.springannotation.classname_match_regex` | Match spring beans with regular expression for the class name. Multiple expressions could be separated by a comma. This only works when `Spring annotation plugin` has been activated. | `All the spring beans tagged with @Bean,@Service,@Dao, or @Repository.` | `plugin.toolkit.log.transmit_formatted` | Whether or not to transmit logged data as formatted or un-formatted. | `true` | -`plugin.toolkit.log.grpc.reporter.server_host` | Specify which grpc server's host for log data to report to. | `127.0.0.1` | -`plugin.toolkit.log.grpc.reporter.server_port` | Specify which grpc server's port for log data to report to. | `11800` | -`plugin.toolkit.log.grpc.reporter.max_message_size` | Specify the maximum size of log data for grpc client to report to. | `10485760` | -`plugin.toolkit.log.grpc.reporter.upstream_timeout` | How long grpc client will timeout in sending data to upstream. Unit is second.|`30` seconds| `plugin.lettuce.trace_redis_parameters` | If set to true, the parameters of Redis commands would be collected by Lettuce agent.| `false` | `plugin.lettuce.redis_parameter_max_length` | If set to positive number and `plugin.lettuce.trace_redis_parameters` is set to `true`, Redis command parameters would be collected and truncated to this length.| `128` | `plugin.neo4j.trace_cypher_parameters`|If set to true, the parameters of the cypher would be collected.|`false`| diff --git a/test/e2e/base/base-compose.yml b/test/e2e/base/base-compose.yml new file mode 100644 index 000000000..c492b9de9 --- /dev/null +++ b/test/e2e/base/base-compose.yml @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + + oap: + image: ghcr.io/apache/skywalking/oap:1730f2c84bbd4da999ec2c74d1c26db31d5a0d24 + expose: + - 11800 + - 12800 + networks: + - e2e + restart: on-failure + healthcheck: + test: ["CMD", "sh", "-c", "nc -zn 127.0.0.1 11800"] + interval: 5s + timeout: 60s + retries: 120 + + provider: + build: + context: provider + dockerfile: Dockerfile + expose: + - 9090 + networks: + - e2e + environment: + SW_AGENT_NAME: e2e-service-provider + SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11800 + SW_INSTANCE_NAME: provider1 + SW_LOGGING_OUTPUT: CONSOLE + healthcheck: + test: [ "CMD", "sh", "-c", "nc -zn 127.0.0.1 9090"] + interval: 5s + timeout: 60s + retries: 120 + + consumer: + build: + context: consumer + dockerfile: Dockerfile + expose: + - 9090 + networks: + - e2e + environment: + PROVIDER_URL: http://provider:9090 + SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11800 + SW_AGENT_NAME: e2e-service-consumer + SW_INSTANCE_NAME: consumer1 + SW_LOGGING_OUTPUT: CONSOLE + healthcheck: + test: [ "CMD", "sh", "-c", "nc -zn 127.0.0.1 9090"] + interval: 5s + timeout: 60s + retries: 120 + +networks: + e2e: diff --git a/test/e2e/base/consumer/Dockerfile b/test/e2e/base/consumer/Dockerfile new file mode 100644 index 000000000..0a6b88bfd --- /dev/null +++ b/test/e2e/base/consumer/Dockerfile @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM skywalking/skywalking-java:latest-java8 + +VOLUME /services +ADD target/e2e-service-consumer.jar /services/ + +ENV JAVA_OPTS="" + +CMD ["sh", "-c", "java $JAVA_OPTS -jar /services/e2e-service-consumer.jar"] diff --git a/test/e2e/base/consumer/pom.xml b/test/e2e/base/consumer/pom.xml new file mode 100644 index 000000000..f5d051055 --- /dev/null +++ b/test/e2e/base/consumer/pom.xml @@ -0,0 +1,92 @@ + + + + + + + org.apache.skywalking + apache-skywalking-e2e + 1.0.0 + + 4.0.0 + + jar + + e2e-service-consumer + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-test + + + org.springframework.boot + spring-boot-configuration-processor + true + + + + com.google.guava + guava + ${guava.version} + + + + org.projectlombok + lombok + ${lombok.version} + provided + + + org.apache.skywalking + apm-toolkit-trace + ${sw.version} + + + + + e2e-service-consumer + + + org.springframework.boot + spring-boot-maven-plugin + ${spring.boot.version} + + true + true + true + + + + + repackage + + + + + + + + diff --git a/test/e2e/base/consumer/src/main/java/org/apache/skywalking/e2e/E2EConfiguration.java b/test/e2e/base/consumer/src/main/java/org/apache/skywalking/e2e/E2EConfiguration.java new file mode 100644 index 000000000..2e7bda2f7 --- /dev/null +++ b/test/e2e/base/consumer/src/main/java/org/apache/skywalking/e2e/E2EConfiguration.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Data +@Configuration +@ConfigurationProperties("e2e") +public class E2EConfiguration { + private String providerBaseUrl; +} diff --git a/test/e2e/base/consumer/src/main/java/org/apache/skywalking/e2e/Service1Application.java b/test/e2e/base/consumer/src/main/java/org/apache/skywalking/e2e/Service1Application.java new file mode 100644 index 000000000..138f9f88e --- /dev/null +++ b/test/e2e/base/consumer/src/main/java/org/apache/skywalking/e2e/Service1Application.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class Service1Application { + public static void main(String[] args) { + SpringApplication.run(Service1Application.class, args); + } +} diff --git a/test/e2e/base/consumer/src/main/java/org/apache/skywalking/e2e/User.java b/test/e2e/base/consumer/src/main/java/org/apache/skywalking/e2e/User.java new file mode 100644 index 000000000..66d7c0d6b --- /dev/null +++ b/test/e2e/base/consumer/src/main/java/org/apache/skywalking/e2e/User.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e; + +import lombok.Data; + +@Data +public class User { + private Long id; + private String name; +} diff --git a/test/e2e/base/consumer/src/main/java/org/apache/skywalking/e2e/controller/HealthController.java b/test/e2e/base/consumer/src/main/java/org/apache/skywalking/e2e/controller/HealthController.java new file mode 100644 index 000000000..999a0155d --- /dev/null +++ b/test/e2e/base/consumer/src/main/java/org/apache/skywalking/e2e/controller/HealthController.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e.controller; + +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequiredArgsConstructor +public class HealthController { + @GetMapping("/health") + @SuppressWarnings("EmptyMethod") + @ResponseStatus(code = HttpStatus.OK) + public void hello() { + } + +} diff --git a/test/e2e/base/consumer/src/main/java/org/apache/skywalking/e2e/controller/UserController.java b/test/e2e/base/consumer/src/main/java/org/apache/skywalking/e2e/controller/UserController.java new file mode 100644 index 000000000..2db2c303e --- /dev/null +++ b/test/e2e/base/consumer/src/main/java/org/apache/skywalking/e2e/controller/UserController.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e.controller; + +import com.google.common.base.Strings; +import lombok.RequiredArgsConstructor; +import org.apache.skywalking.apm.toolkit.trace.TraceContext; +import org.apache.skywalking.e2e.E2EConfiguration; +import org.apache.skywalking.e2e.User; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.client.RestTemplate; + +import java.util.Optional; +import java.util.Random; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@RestController +@RequiredArgsConstructor +public class UserController { + private final RestTemplate restTemplate = new RestTemplate(); + + private final E2EConfiguration configuration; + + private final int sleepMin = 500; + + private final int sleepMax = 1000; + + @PostMapping("/info") + public String info() throws InterruptedException { + Thread.sleep(randomSleepLong(sleepMin, sleepMax)); + + Optional> optionalResponseEntity = Stream.of( + Strings.nullToEmpty(configuration.getProviderBaseUrl()).split(",")) + .map(baseUrl -> restTemplate.postForEntity( + baseUrl + "/info", null, String.class)) + .findFirst(); + if (optionalResponseEntity.isPresent() && optionalResponseEntity.get().getStatusCodeValue() == 200) { + return optionalResponseEntity.get().getBody(); + } + throw new RuntimeException(); + } + + @PostMapping("/users") + public Object createAuthor() throws InterruptedException { + Thread.sleep(randomSleepLong(sleepMin, sleepMax)); + + return Stream.of(Strings.nullToEmpty(configuration.getProviderBaseUrl()).split(",")) + .map(baseUrl -> restTemplate.postForEntity(baseUrl + "/users", null, User.class)) + .collect(Collectors.toList()); + } + + @PostMapping("/correlation") + public String correlation() throws InterruptedException { + Thread.sleep(randomSleepLong(sleepMin, sleepMax)); + TraceContext.putCorrelation("CONSUMER_KEY", "consumer"); + + String baseUrl = configuration.getProviderBaseUrl(); + ResponseEntity resp = restTemplate.postForEntity(baseUrl + "/correlation", null, String.class); + return resp.getBody(); + } + + private long randomSleepLong(int min, int max) { + Random rand = new Random(); + int randomNumber = rand.nextInt((max - min) + 1) + min; + return randomNumber; + } +} diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService b/test/e2e/base/consumer/src/main/resources/application.yml similarity index 86% rename from apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService rename to test/e2e/base/consumer/src/main/resources/application.yml index 69de3fb6b..13022a93a 100644 --- a/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService +++ b/test/e2e/base/consumer/src/main/resources/application.yml @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -13,7 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -org.apache.skywalking.apm.toolkit.logging.common.log.GRPCLogReportServiceClient \ No newline at end of file +server: + port: 9090 + +spring: + main: + banner-mode: 'off' + +e2e: + provider-base-url: ${PROVIDER_URL:http://127.0.0.1:9090} diff --git a/test/e2e/base/pom.xml b/test/e2e/base/pom.xml new file mode 100644 index 000000000..9581d8aed --- /dev/null +++ b/test/e2e/base/pom.xml @@ -0,0 +1,59 @@ + + + + + + org.springframework.boot + spring-boot-starter-parent + 2.2.5.RELEASE + + + 4.0.0 + SkyWalking End to End Tests + org.apache.skywalking + apache-skywalking-e2e + 1.0.0 + + pom + + + provider + consumer + + + + + + + 1.8 + ${java.version} + ${java.version} + UTF-8 + 2.2.5.RELEASE + 28.2-jre + 1.18.20 + + + \ No newline at end of file diff --git a/test/e2e/base/provider/Dockerfile b/test/e2e/base/provider/Dockerfile new file mode 100644 index 000000000..a7424cc2a --- /dev/null +++ b/test/e2e/base/provider/Dockerfile @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM skywalking/skywalking-java:latest-java8 + +VOLUME /services +ADD target/e2e-service-provider.jar /services/ + +ENV JAVA_OPTS="" + +CMD ["sh", "-c", "java $JAVA_OPTS -jar /services/e2e-service-provider.jar"] diff --git a/test/e2e/base/provider/pom.xml b/test/e2e/base/provider/pom.xml new file mode 100644 index 000000000..841c045b3 --- /dev/null +++ b/test/e2e/base/provider/pom.xml @@ -0,0 +1,154 @@ + + + + + org.apache.skywalking + apache-skywalking-e2e + 1.0.0 + + 4.0.0 + + jar + + e2e-service-provider + + + 1.2.17 + 2.7 + 1.2.3 + + 2.1.2.RELEASE + 1.4.199 + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-test + + + org.springframework.boot + spring-boot-configuration-processor + true + + + + com.google.guava + guava + ${guava.version} + + + + org.projectlombok + lombok + ${lombok.version} + provided + + + org.springframework.boot + spring-boot-starter-data-jpa + ${spring.boot.version} + + + com.h2database + h2 + ${h2.version} + + + org.springframework.boot + spring-boot-starter-actuator + + + log4j-to-slf4j + org.apache.logging.log4j + + + + + org.apache.skywalking + apm-toolkit-micrometer-registry + ${sw.version} + + + log4j + log4j + ${log4j.version} + + + org.apache.logging.log4j + log4j-core + ${log4j2.version} + provided + + + ch.qos.logback + logback-classic + ${logback.version} + + + org.apache.skywalking + apm-toolkit-logback-1.x + ${sw.version} + + + org.apache.skywalking + apm-toolkit-log4j-1.x + ${sw.version} + + + org.apache.skywalking + apm-toolkit-log4j-2.x + ${sw.version} + + + org.apache.skywalking + apm-toolkit-trace + ${sw.version} + + + + + e2e-service-provider + + + org.springframework.boot + spring-boot-maven-plugin + ${spring.boot.version} + + true + true + true + + + + + repackage + + + + + + + diff --git a/test/e2e/base/provider/src/main/java/org/apache/skywalking/e2e/KeyValue.java b/test/e2e/base/provider/src/main/java/org/apache/skywalking/e2e/KeyValue.java new file mode 100644 index 000000000..64bb447d1 --- /dev/null +++ b/test/e2e/base/provider/src/main/java/org/apache/skywalking/e2e/KeyValue.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.e2e; + +import lombok.Data; +import lombok.experimental.Accessors; + +@Data +@Accessors(chain = true) +public class KeyValue { + + private String key; + private String value; +} diff --git a/test/e2e/base/provider/src/main/java/org/apache/skywalking/e2e/Service0Application.java b/test/e2e/base/provider/src/main/java/org/apache/skywalking/e2e/Service0Application.java new file mode 100644 index 000000000..c5f87977e --- /dev/null +++ b/test/e2e/base/provider/src/main/java/org/apache/skywalking/e2e/Service0Application.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e; + +import org.apache.skywalking.apm.meter.micrometer.SkywalkingMeterRegistry; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; + +@EnableJpaRepositories +@SpringBootApplication +@ComponentScan({"org.apache.skywalking", "test.apache.skywalking"}) +public class Service0Application { + public static void main(String[] args) { + SpringApplication.run(Service0Application.class, args); + } + + @Bean + SkywalkingMeterRegistry skywalkingMeterRegistry() { + return new SkywalkingMeterRegistry(); + } + +} diff --git a/test/e2e/base/provider/src/main/java/org/apache/skywalking/e2e/User.java b/test/e2e/base/provider/src/main/java/org/apache/skywalking/e2e/User.java new file mode 100644 index 000000000..2acc78a79 --- /dev/null +++ b/test/e2e/base/provider/src/main/java/org/apache/skywalking/e2e/User.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.RequiredArgsConstructor; + +@Data +@Entity +@Builder +@AllArgsConstructor +@RequiredArgsConstructor +public class User { + @Id + @GeneratedValue + private Long id; + + @Column + private String name; +} diff --git a/test/e2e/base/provider/src/main/java/org/apache/skywalking/e2e/UserRepo.java b/test/e2e/base/provider/src/main/java/org/apache/skywalking/e2e/UserRepo.java new file mode 100644 index 000000000..bafea9b30 --- /dev/null +++ b/test/e2e/base/provider/src/main/java/org/apache/skywalking/e2e/UserRepo.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e; + +import org.springframework.data.jpa.repository.JpaRepository; + +public interface UserRepo extends JpaRepository { +} diff --git a/test/e2e/base/provider/src/main/java/org/apache/skywalking/e2e/controller/HealthController.java b/test/e2e/base/provider/src/main/java/org/apache/skywalking/e2e/controller/HealthController.java new file mode 100644 index 000000000..987177343 --- /dev/null +++ b/test/e2e/base/provider/src/main/java/org/apache/skywalking/e2e/controller/HealthController.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e.controller; + +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@SuppressWarnings("SameReturnValue") +public class HealthController { + @GetMapping("/health") + public String hello() { + return "healthy"; + } +} diff --git a/test/e2e/base/provider/src/main/java/org/apache/skywalking/e2e/controller/UserController.java b/test/e2e/base/provider/src/main/java/org/apache/skywalking/e2e/controller/UserController.java new file mode 100644 index 000000000..14f40de95 --- /dev/null +++ b/test/e2e/base/provider/src/main/java/org/apache/skywalking/e2e/controller/UserController.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e.controller; + +import lombok.RequiredArgsConstructor; +import org.apache.skywalking.apm.toolkit.trace.TraceContext; +import org.apache.skywalking.e2e.User; +import org.apache.skywalking.e2e.UserRepo; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; +import org.slf4j.LoggerFactory; + +import java.util.Random; + +@RestController +@RequiredArgsConstructor +@SuppressWarnings("SameReturnValue") +public class UserController { + private static final org.slf4j.Logger LOGBACK_LOGGER = LoggerFactory.getLogger(UserController.class); + + private final UserRepo userRepo; + private final int sleepMin = 500; + private final int sleepMax = 1000; + + @PostMapping("/info") + public String info() throws InterruptedException { + Thread.sleep(randomSleepLong(sleepMin, sleepMax)); + LOGBACK_LOGGER.info("logback message==> now: {}", System.currentTimeMillis()); + return "whatever"; + } + + @PostMapping("/users") + public User createAuthor() throws InterruptedException { + Thread.sleep(randomSleepLong(sleepMin, sleepMax)); + return userRepo.save(User.builder().name("").build()); + } + + @PostMapping("/correlation") + public String correlation() throws InterruptedException { + Thread.sleep(randomSleepLong(sleepMin, sleepMax)); + TraceContext.putCorrelation("PROVIDER_KEY", "provider"); + return TraceContext.getCorrelation("CONSUMER_KEY").orElse("") + "_" + + TraceContext.getCorrelation("MIDDLE_KEY").orElse("") + "_" + + TraceContext.getCorrelation("PROVIDER_KEY").orElse(""); + } + + private long randomSleepLong(int min, int max) { + Random rand = new Random(); + int randomNumber = rand.nextInt((max - min) + 1) + min; + return randomNumber; + } +} diff --git a/test/e2e/base/provider/src/main/java/test/apache/skywalking/e2e/profile/CreateUser.java b/test/e2e/base/provider/src/main/java/test/apache/skywalking/e2e/profile/CreateUser.java new file mode 100644 index 000000000..30cb74ae2 --- /dev/null +++ b/test/e2e/base/provider/src/main/java/test/apache/skywalking/e2e/profile/CreateUser.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test.apache.skywalking.e2e.profile; + +import lombok.Data; +import org.apache.skywalking.e2e.User; + +@Data +public class CreateUser { + private String name; + private boolean enableProfiling; + + public User toUser() { + return User.builder().name(name).build(); + } +} diff --git a/test/e2e/base/provider/src/main/java/test/apache/skywalking/e2e/profile/ProfileController.java b/test/e2e/base/provider/src/main/java/test/apache/skywalking/e2e/profile/ProfileController.java new file mode 100644 index 000000000..b939a8426 --- /dev/null +++ b/test/e2e/base/provider/src/main/java/test/apache/skywalking/e2e/profile/ProfileController.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test.apache.skywalking.e2e.profile; + +import java.util.concurrent.TimeUnit; +import lombok.RequiredArgsConstructor; +import org.apache.skywalking.e2e.User; +import org.apache.skywalking.e2e.UserRepo; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequiredArgsConstructor +public class ProfileController { + private final UserRepo userRepo; + + @PostMapping("/profile/{name}") + public User createAuthor(@RequestBody final CreateUser createUser) throws InterruptedException { + final User user = userRepo.save(createUser.toUser()); + if (createUser.isEnableProfiling()) { + TimeUnit.MILLISECONDS.sleep(6200); + } + return user; + } +} diff --git a/test/e2e/base/provider/src/main/resources/application.yml b/test/e2e/base/provider/src/main/resources/application.yml new file mode 100644 index 000000000..360c10c4b --- /dev/null +++ b/test/e2e/base/provider/src/main/resources/application.yml @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +server: + port: 9090 + +spring: + main: + banner-mode: 'off' + datasource: + url: jdbc:h2:mem:testdb + driver-class-name: org.h2.Driver + data-username: sa + password: sa + platform: org.hibernate.dialect.H2Dialect + jpa: + generate-ddl: true + hibernate: + ddl-auto: create-drop + properties: + hibernate.format_sql: true + show-sql: true + +management: + metrics: + use-global-registry: true + web: + server: + request: + autotime: + enabled: true diff --git a/test/e2e/base/provider/src/main/resources/log4j.properties b/test/e2e/base/provider/src/main/resources/log4j.properties new file mode 100644 index 000000000..56a721e0a --- /dev/null +++ b/test/e2e/base/provider/src/main/resources/log4j.properties @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +log4j.rootLogger=info,CustomAppender +log4j.appender.CustomAppender=org.apache.skywalking.apm.toolkit.log.log4j.v1.x.log.GRPCLogClientAppender +log4j.appender.CustomAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.CustomAppender.layout.ConversionPattern=[%t] %-5p %c %x - %m%n + +log4j.logger.fileLogger=info,FileAppender +log4j.appender.FileAppender=org.apache.log4j.FileAppender +log4j.appender.FileAppender.ImmediateFlush=true +log4j.appender.FileAppender.Append=true +log4j.appender.FileAppender.File=/tmp/skywalking-logs/log4j1/e2e-service-provider.log +log4j.appender.FileAppender.layout=org.apache.skywalking.apm.toolkit.log.log4j.v1.x.TraceIdPatternLayout +log4j.appender.FileAppender.layout.ConversionPattern=[%T{SW_CTX}] [%p] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c:%L - %m%n \ No newline at end of file diff --git a/test/e2e/base/provider/src/main/resources/log4j2.xml b/test/e2e/base/provider/src/main/resources/log4j2.xml new file mode 100644 index 000000000..65b0bbd43 --- /dev/null +++ b/test/e2e/base/provider/src/main/resources/log4j2.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + [%sw_ctx] [%p] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c:%L - %m%n + + + + + + + + + + + + + + + + diff --git a/test/e2e/base/provider/src/main/resources/logback.xml b/test/e2e/base/provider/src/main/resources/logback.xml new file mode 100644 index 000000000..e091cdac1 --- /dev/null +++ b/test/e2e/base/provider/src/main/resources/logback.xml @@ -0,0 +1,52 @@ + + + + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%X{tid}] [%thread] %-5level %logger{36} -%msg%n + + + + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%X{tid}] [%thread] %-5level %logger{36} -%msg%n + + + + + + /tmp/skywalking-logs/logback/e2e-service-provider.log + + + [%sw_ctx] [%level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %logger:%line - %msg%n + + + + + + + + + + + + diff --git a/test/e2e/case/base/docker-compose.yml b/test/e2e/case/base/docker-compose.yml new file mode 100644 index 000000000..2e655e86c --- /dev/null +++ b/test/e2e/case/base/docker-compose.yml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + oap: + extends: + file: ../../base/base-compose.yml + service: oap + ports: + - 12800 + + provider: + extends: + file: ../../base/base-compose.yml + service: provider + environment: + SW_AGENT_INSTANCE_NAME: provider1 + SW_LOGGING_OUTPUT: CONSOLE + ports: + - 9090 + depends_on: + oap: + condition: service_healthy + + consumer: + extends: + file: ../../base/base-compose.yml + service: consumer + environment: + SW_AGENT_INSTANCE_NAME: consumer1 + SW_LOGGING_OUTPUT: CONSOLE + ports: + - 9090 + depends_on: + provider: + condition: service_healthy + +networks: + e2e: diff --git a/test/e2e/case/base/e2e.yaml b/test/e2e/case/base/e2e.yaml new file mode 100644 index 000000000..bdf113a09 --- /dev/null +++ b/test/e2e/case/base/e2e.yaml @@ -0,0 +1,100 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 1200 + steps: + - name: install yq + command: | + if ! command -v yq &> /dev/null; then + mkdir -p /tmp/skywalking-infra-e2e/bin && cd /tmp/skywalking-infra-e2e + mkdir -p yq && cd yq + curl -kLo yq.tar.gz https://github.com/mikefarah/yq/archive/v4.11.1.tar.gz + tar -zxf yq.tar.gz --strip=1 + go install && go build -ldflags -s && mv yq ../bin/yq + export PATH="$PATH:/tmp/skywalking-infra-e2e/bin" + echo "success to install yq" + fi + - name: install swctl + command: | + if ! command -v swctl &> /dev/null; then + mkdir -p /tmp/skywalking-infra-e2e/bin && cd /tmp/skywalking-infra-e2e + mkdir -p swctl && cd swctl + curl -kLo skywalking-cli.tar.gz https://github.com/apache/skywalking-cli/archive/4d1cb83e24ff58988f4aba0daa50259593b11670.tar.gz + tar -zxf skywalking-cli.tar.gz --strip=1 + utype=$(uname | awk '{print tolower($0)}') + make $utype && mv bin/swctl-*-$utype-amd64 ../bin/swctl + export PATH="$PATH:/tmp/skywalking-infra-e2e/bin" + echo "success to install swctl" + fi + +cleanup: + # always never success failure + on: always + +trigger: + action: http + interval: 3s + times: 5 + url: http://${consumer_host}:${consumer_9090}/info + method: POST + +verify: + # verify with retry strategy + retry: + # max retry count + count: 20 + # the interval between two retries, in millisecond. + interval: 10000 + cases: + # basic check: service list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls + expected: expected/service.yml + # basic check: service metrics + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name service_sla --service=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + # basic check: service endpoint + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=info --service-id=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls|grep -B 1 'provider'|yq e '.[0].id' -) + expected: expected/service-endpoint.yml + # basic check: service endpoint metrics + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name endpoint_cpm --endpoint=/info --service=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + + # native management: service instance list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider + expected: expected/service-instance.yml + + # native jvm: service instance jvm metrics + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name instance_jvm_thread_live_count --instance=provider1 --service=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + + # native tracing: trace segment list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls + expected: expected/traces-list.yml + # native tracing: trace detail + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace $(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls|grep -A 5 '/info'|tail -n1|awk -F ' ' '{print $2}') + expected: expected/trace-info-detail.yml + + # native event: event list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql event list --service=e2e-service-provider --instance=provider1 + expected: expected/event-list.yml + + # native log: logs list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql logs list --service-id=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls|grep -B 1 'provider'|yq e '.[0].id' -) --trace-id=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls|grep -A 5 '/info'|tail -n1|awk -F ' ' '{print $2}') + expected: expected/logs-list.yml diff --git a/test/e2e/case/base/expected/event-list.yml b/test/e2e/case/base/expected/event-list.yml new file mode 100644 index 000000000..8c41d3dc8 --- /dev/null +++ b/test/e2e/case/base/expected/event-list.yml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +events: +{{- range .events }} +- uuid: {{ notEmpty .uuid }} + source: + service: e2e-service-provider + serviceinstance: provider1 + endpoint: "" + name: Start + type: Normal + message: Start Java Application + parameters: + {{- range .parameters }} + {{- if eq .key "OPTS"}} + - key: OPTS + value: {{ notEmpty .value }} + {{- end }} + {{- end }} + starttime: {{ gt .starttime 0 }} + endtime: {{ gt .endtime 0 }} +{{- end }} +total: {{ gt .total 0 }} \ No newline at end of file diff --git a/test/e2e/case/base/expected/logs-list.yml b/test/e2e/case/base/expected/logs-list.yml new file mode 100644 index 000000000..329491058 --- /dev/null +++ b/test/e2e/case/base/expected/logs-list.yml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +logs: + {{- range .logs }} + - servicename: e2e-service-provider + serviceid: {{ b64enc "e2e-service-provider" }}.1 + serviceinstancename: provider1 + serviceinstanceid: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} + endpointname: null + endpointid: null + traceid: {{ notEmpty .traceid }} + timestamp: {{ gt .timestamp 0 }} + contenttype: TEXT + content: | + {{ notEmpty .content }} + tags: + {{- range .tags }} + {{- if eq .key "level"}} + - key: level + value: INFO + {{- end }} + {{- if eq .key "logger" }} + - key: logger + value: {{ notEmpty .value }} + {{- end }} + {{- if eq .key "thread" }} + - key: thread + value: {{ notEmpty .value }} + {{- end }} + {{- end }} + {{- end }} +total: {{ gt .total 0 }} \ No newline at end of file diff --git a/test/e2e/case/base/expected/metrics-has-value.yml b/test/e2e/case/base/expected/metrics-has-value.yml new file mode 100644 index 000000000..27ae47c8f --- /dev/null +++ b/test/e2e/case/base/expected/metrics-has-value.yml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- contains . }} +- key: {{ notEmpty .key }} + value: {{ ge .value 0 }} +- key: {{ notEmpty .key }} + value: {{ ge .value 1 }} +{{- end }} \ No newline at end of file diff --git a/test/e2e/case/base/expected/service-endpoint.yml b/test/e2e/case/base/expected/service-endpoint.yml new file mode 100644 index 000000000..76bf68888 --- /dev/null +++ b/test/e2e/case/base/expected/service-endpoint.yml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- range .}} +{{- if eq .name "/info" }} +- id: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "/info" }} + name: /info +{{- end}} +{{- end}} \ No newline at end of file diff --git a/test/e2e/case/base/expected/service-instance.yml b/test/e2e/case/base/expected/service-instance.yml new file mode 100644 index 000000000..ef74087af --- /dev/null +++ b/test/e2e/case/base/expected/service-instance.yml @@ -0,0 +1,54 @@ +# Licensed to Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Apache Software Foundation (ASF) licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{{- range .}} +- id: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} + name: {{ notEmpty .name }} + attributes: + {{- range .attributes }} + {{- if eq .name "OS Name" }} + - name: OS Name + value: Linux + {{- end }} + {{- if eq .name "hostname" }} + - name: hostname + value: {{ notEmpty .value }} + {{- end }} + {{- if eq .name "Process No." }} + - name: Process No. + value: "1" + {{- end }} + {{- if eq .name "Start Time" }} + - name: Start Time + value: {{ notEmpty .value }} + {{- end }} + {{- if eq .name "JVM Arguments" }} + - name: JVM Arguments + value: '{{ notEmpty .value }}' + {{- end }} + {{- if eq .name "Jar Dependencies" }} + - name: Jar Dependencies + value: '{{ notEmpty .value }}' + {{- end }} + {{- if eq .name "ipv4s" }} + - name: ipv4s + value: {{ notEmpty .value }} + {{- end }} + {{- end}} + language: JAVA + instanceuuid: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} +{{- end}} \ No newline at end of file diff --git a/test/e2e/case/base/expected/service.yml b/test/e2e/case/base/expected/service.yml new file mode 100644 index 000000000..129f9d3e8 --- /dev/null +++ b/test/e2e/case/base/expected/service.yml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- id: {{ b64enc "e2e-service-provider" }}.1 + name: e2e-service-provider + group: "" +- id: {{ b64enc "e2e-service-consumer" }}.1 + name: e2e-service-consumer + group: "" \ No newline at end of file diff --git a/test/e2e/case/base/expected/trace-info-detail.yml b/test/e2e/case/base/expected/trace-info-detail.yml new file mode 100644 index 000000000..2598ffb4b --- /dev/null +++ b/test/e2e/case/base/expected/trace-info-detail.yml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +spans: + {{- range .spans}} + - traceid: {{ notEmpty .traceid }} + segmentid: {{ notEmpty .segmentid }} + spanid: {{ .spanid }} + parentspanid: {{ .parentspanid }} + refs: + {{- if eq .servicecode "e2e-service-provider" }} + {{- range .refs }} + - traceid: {{ notEmpty .traceid }} + parentsegmentid: {{ notEmpty .parentsegmentid }} + parentspanid: 1 + type: CROSS_PROCESS + {{- end }} + {{- end }} + {{- if eq .servicecode "e2e-service-consumer" }} + [] + {{- end }} + servicecode: {{ notEmpty .servicecode }} + serviceinstancename: {{ notEmpty .serviceinstancename }} + starttime: {{ gt .starttime 0 }} + endtime: {{ gt .endtime 0 }} + endpointname: /info + type: {{ notEmpty .type }} + peer: + {{- if eq .type "Exit" }} + provider:9090 + {{ else }} + "" + {{- end }} + component: + {{- if eq .type "Exit" }} + SpringRestTemplate + {{- end }} + {{- if eq .type "Entry" }} + Tomcat + {{- end }} + iserror: false + layer: Http + tags: + {{- range .tags }} + {{- if eq .key "http.method" }} + - key: http.method + value: POST + {{- end }} + {{- if eq .key "url" }} + - key: url + value: {{ notEmpty .value }} + {{- end }} + {{- end }} + logs: [] + {{- end }} \ No newline at end of file diff --git a/test/e2e/case/base/expected/trace-users-detail.yml b/test/e2e/case/base/expected/trace-users-detail.yml new file mode 100644 index 000000000..5129243ba --- /dev/null +++ b/test/e2e/case/base/expected/trace-users-detail.yml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +spans: + {{- range .spans}} + - traceid: {{ notEmpty .traceid }} + segmentid: {{ notEmpty .segmentid }} + spanid: 0 + parentspanid: -1 + refs: [] + servicecode: {{ notEmpty .servicecode }} + serviceinstancename: {{ notEmpty .serviceinstancename }} + starttime: {{ gt .starttime 0 }} + endtime: {{ gt .endtime 0 }} + endpointname: /users + type: Entry + peer: "" + component: Tomcat + iserror: false + layer: Http + tags: + {{- range .tags }} + {{- if eq .key "http.method" }} + - key: http.method + value: POST + {{- end }} + {{- if eq .key "url" }} + - key: url + value: {{ notEmpty .value }} + {{- end }} + {{- end }} + logs: [] + {{- end }} \ No newline at end of file diff --git a/test/e2e/case/base/expected/traces-list.yml b/test/e2e/case/base/expected/traces-list.yml new file mode 100644 index 000000000..b5f79add0 --- /dev/null +++ b/test/e2e/case/base/expected/traces-list.yml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +traces: +{{- range .traces }} +- segmentid: {{ notEmpty .segmentid }} + endpointnames: + {{- if eq (index .endpointnames 0) "HikariCP/Connection/getConnection" }} + - HikariCP/Connection/getConnection + {{- end }} + {{- if eq (index .endpointnames 0) "HikariCP/Connection/close" }} + - HikariCP/Connection/close + {{- end }} + {{- if eq (index .endpointnames 0) "H2/JDBI/Statement/execute" }} + - H2/JDBI/Statement/execute + {{- end}} + {{- if eq (index .endpointnames 0) "H2/JDBI/Statement/executeQuery" }} + - H2/JDBI/Statement/executeQuery + {{- end}} + {{- if eq (index .endpointnames 0) "H2/JDBI/PreparedStatement/executeQuery" }} + - H2/JDBI/PreparedStatement/executeQuery + {{- end }} + {{- if eq (index .endpointnames 0) "/info" }} + - /info + {{- end }} + {{- if eq (index .endpointnames 0) "/users" }} + - /users + {{- end }} + duration: {{ ge .duration 0 }} + start: "{{ notEmpty .start}}" + iserror: false + traceids: + - {{ (index .traceids 0) }} +{{- end }} +total: {{ gt .total 0 }}