From 01d1eff6c796510b5698ad5d84d0800d56e06c6b Mon Sep 17 00:00:00 2001 From: zhang-wei Date: Fri, 29 Jan 2021 00:07:16 +0800 Subject: [PATCH] Enhancement log-recerive e2e test (#6248) --- .github/workflows/e2e.log.yaml | 13 +- .../log/GRPCLogReportServiceClient.java | 33 +++-- .../logging/common/log/ToolkitConfig.java | 46 +++--- .../setup/backend/configuration-vocabulary.md | 2 +- .../src/main/resources/application.yml | 2 +- .../server/core/query/LogQueryService.java | 18 ++- .../plugin/influxdb/query/LogQuery.java | 2 +- .../skywalking/e2e/SimpleQueryClient.java | 4 - .../apache/skywalking/e2e/log/LogsQuery.java | 44 +----- test/e2e/e2e-data/src/main/resources/logs.gql | 6 - test/e2e/e2e-log-producer/pom.xml | 66 --------- .../e2e/LogProducerApplication.java | 28 ---- .../e2e/controller/LogController.java | 134 ------------------ test/e2e/e2e-service-provider/pom.xml | 14 ++ .../apm/toolkit/trace/TraceContext.java | 57 -------- .../e2e/controller/LogController.java} | 22 ++- .../src/main/resources/logback.xml | 34 +++++ test/e2e/e2e-test/docker/Dockerfile.provider | 4 + ...file.log.producer => Dockerfile.satellite} | 23 ++- test/e2e/e2e-test/docker/base-compose.yml | 4 +- .../e2e-test/docker/kafka/Dockerfile.provider | 4 + .../docker/kafka/docker-compose.log.yml | 30 ++-- .../e2e-test/docker/kafka/docker-compose.yml | 8 -- .../docker/kafka/satellite_config.yaml | 61 ++++++++ .../docker/log/docker-compose.es6.yml | 9 +- .../docker/log/docker-compose.es7.yml | 7 + .../e2e-test/docker/log/docker-compose.h2.yml | 8 ++ .../docker/log/docker-compose.influxdb.yml | 8 ++ .../docker/log/docker-compose.mysql.yml | 8 ++ .../apache/skywalking/e2e/kafka/KafkaE2E.java | 2 +- .../skywalking/e2e/kafka/KafkaLogE2E.java | 24 +--- .../skywalking/e2e/kafka/KafkaMeterE2E.java | 2 +- .../skywalking/e2e/kafka/KafkaProfileE2E.java | 2 +- .../org/apache/skywalking/e2e/log/LogE2E.java | 116 ++------------- .../test/resources/expected/log/endpoints.yml | 2 +- .../test/resources/expected/log/instances.yml | 2 +- .../src/test/resources/expected/log/logs.yml | 12 +- .../test/resources/expected/log/services.yml | 2 +- test/e2e/pom.xml | 16 ++- 39 files changed, 313 insertions(+), 566 deletions(-) delete mode 100644 test/e2e/e2e-log-producer/pom.xml delete mode 100644 test/e2e/e2e-log-producer/src/main/java/org/apache/skywalking/e2e/LogProducerApplication.java delete mode 100644 test/e2e/e2e-log-producer/src/main/java/org/apache/skywalking/e2e/controller/LogController.java delete mode 100644 test/e2e/e2e-service-provider/src/main/java/org/apache/skywalking/apm/toolkit/trace/TraceContext.java rename test/e2e/{e2e-log-producer/src/main/java/org/apache/skywalking/e2e/controller/HealthController.java => e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/LogController.java} (70%) create mode 100644 test/e2e/e2e-service-provider/src/main/resources/logback.xml rename test/e2e/e2e-test/docker/{kafka/Dockerfile.log.producer => Dockerfile.satellite} (59%) create mode 100644 test/e2e/e2e-test/docker/kafka/satellite_config.yaml diff --git a/.github/workflows/e2e.log.yaml b/.github/workflows/e2e.log.yaml index 313a06626..8cc53b6b3 100644 --- a/.github/workflows/e2e.log.yaml +++ b/.github/workflows/e2e.log.yaml @@ -23,9 +23,8 @@ on: - '!**.md' schedule: - cron: '0 18 * * *' - env: - SKIP_TEST: true + SW_AGENT_JDK_VERSION: 8 jobs: Log: @@ -47,12 +46,10 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- - - name: Compile and Build - run: make docker && ES_VERSION=es7 TAG=latest-es7 make docker - - name: Copy dist package - run: cp -R dist test/e2e/ - - name: Log Receiver ${{ matrix.storage }} - run: ./mvnw --batch-mode -f test/e2e/pom.xml -am -DfailIfNoTests=false verify -Dit.test=org.apache.skywalking.e2e.log.LogE2E + - name: Run E2E Test + uses: ./.github/actions/e2e-test + with: + test_class: org.apache.skywalking.e2e.log.LogE2E - uses: actions/upload-artifact@v1 if: failure() with: diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/GRPCLogReportServiceClient.java b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/GRPCLogReportServiceClient.java index 568c50817..8f374063a 100644 --- a/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/GRPCLogReportServiceClient.java +++ b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/GRPCLogReportServiceClient.java @@ -18,11 +18,14 @@ package org.apache.skywalking.apm.toolkit.logging.common.log; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.StreamObserver; import java.util.List; import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; - import org.apache.skywalking.apm.agent.core.boot.OverrideImplementor; import org.apache.skywalking.apm.agent.core.conf.Config; import org.apache.skywalking.apm.agent.core.logging.api.ILog; @@ -36,11 +39,6 @@ import org.apache.skywalking.apm.network.common.v3.Commands; import org.apache.skywalking.apm.network.logging.v3.LogData; import org.apache.skywalking.apm.network.logging.v3.LogReportServiceGrpc; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.StatusRuntimeException; -import io.grpc.stub.StreamObserver; - /** * Report log to server by grpc */ @@ -59,14 +57,22 @@ public class GRPCLogReportServiceClient extends LogReportServiceClient { @Override public void boot() throws Throwable { - carrier = new DataCarrier<>("gRPC-log", "gRPC-log", Config.Buffer.CHANNEL_SIZE, Config.Buffer.BUFFER_SIZE, - BufferStrategy.IF_POSSIBLE); + carrier = new DataCarrier<>("gRPC-log", "gRPC-log", + Config.Buffer.CHANNEL_SIZE, + Config.Buffer.BUFFER_SIZE, + BufferStrategy.IF_POSSIBLE + ); carrier.consume(this, 1); channel = ManagedChannelBuilder - .forAddress(ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.SERVER_HOST, - ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.SERVER_PORT).usePlaintext().build(); + .forAddress( + ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.SERVER_HOST, + ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.SERVER_PORT + ) + .usePlaintext() + .build(); asyncStub = LogReportServiceGrpc.newStub(channel) - .withMaxOutboundMessageSize(ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.MAX_MESSAGE_SIZE); + .withMaxOutboundMessageSize( + ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.MAX_MESSAGE_SIZE); } @Override @@ -99,7 +105,7 @@ public class GRPCLogReportServiceClient extends LogReportServiceClient { final GRPCStreamServiceStatus waitStatus = new GRPCStreamServiceStatus(false); try { reportStreamObserver = asyncStub.withDeadlineAfter( - ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.UPSTREAM_TIMEOUT, TimeUnit.SECONDS + ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.UPSTREAM_TIMEOUT, TimeUnit.SECONDS ).collect(new StreamObserver() { @Override public void onNext(Commands commands) { @@ -113,7 +119,8 @@ public class GRPCLogReportServiceClient extends LogReportServiceClient { } LOGGER.error(t, "Try to send {} log data to collector, with unexpected exception.", - dataList.size()); + dataList.size() + ); } @Override diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/ToolkitConfig.java b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/ToolkitConfig.java index c0bc233b9..6cc51184a 100644 --- a/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/ToolkitConfig.java +++ b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/ToolkitConfig.java @@ -23,33 +23,33 @@ import org.apache.skywalking.apm.agent.core.boot.PluginConfig; public class ToolkitConfig { public static class Plugin { - @PluginConfig(root = ToolkitConfig.class) public static class Toolkit { - public static class Log { - public static class GRPC { - public static class Reporter { - /** - * The host of gRPC log server. - */ - public static String SERVER_HOST = "127.0.0.1"; + public static class Log { + public static class GRPC { + @PluginConfig(root = ToolkitConfig.class) + public static class Reporter { + /** + * The host of gRPC log server. + */ + public static String SERVER_HOST = "127.0.0.1"; - /** - * The port of gRPC log server. - */ - public static int SERVER_PORT = 11800; + /** + * The port of gRPC log server. + */ + public static int SERVER_PORT = 11800; - /** - * The max size of message to send to server.Default is 10 MB. - */ - public static int MAX_MESSAGE_SIZE = 10 * 1024 * 1024; + /** + * The max size of message to send to server.Default is 10 MB. + */ + public static int MAX_MESSAGE_SIZE = 10 * 1024 * 1024; - /** - * How long grpc client will timeout in sending data to upstream. The unit is second. - */ - public static int UPSTREAM_TIMEOUT = 30; - } - } - } + /** + * How long grpc client will timeout in sending data to upstream. The unit is second. + */ + public static int UPSTREAM_TIMEOUT = 30; + } + } + } } } } \ No newline at end of file diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md index 669697b11..441726f00 100644 --- a/docs/en/setup/backend/configuration-vocabulary.md +++ b/docs/en/setup/backend/configuration-vocabulary.md @@ -30,7 +30,7 @@ core|default|role|Option values, `Mixed/Receiver/Aggregator`. **Receiver** mode | - | - | instanceNameMaxLength| Max length limitation of service instance name. The max length of service + instance names should be less than 200.|SW_INSTANCE_NAME_MAX_LENGTH|70| | - | - | endpointNameMaxLength| Max length limitation of endpoint name. The max length of service + endpoint names should be less than 240.|SW_ENDPOINT_NAME_MAX_LENGTH|150| | - | - | searchableTracesTags | Define the set of span tag keys, which should be searchable through the GraphQL. Multiple values should be separated through the comma. | SW_SEARCHABLE_TAG_KEYS | http.method,status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker| -| - | - | searchableLogsTags | Define the set of log tag keys, which should be searchable through the GraphQL. Multiple values should be separated through the comma. | SW_SEARCHABLE_LOGS_TAG_KEYS | - | +| - | - | searchableLogsTags | Define the set of log tag keys, which should be searchable through the GraphQL. Multiple values should be separated through the comma. | SW_SEARCHABLE_LOGS_TAG_KEYS | level,logger,thread | | - | - | gRPCThreadPoolSize|Pool size of gRPC server| SW_CORE_GRPC_THREAD_POOL_SIZE | CPU core * 4| | - | - | gRPCThreadPoolQueueSize| The queue size of gRPC server| SW_CORE_GRPC_POOL_QUEUE_SIZE | 10000| | - | - | maxConcurrentCallsPerConnection | The maximum number of concurrent calls permitted for each incoming connection. Defaults to no limit. | SW_CORE_GRPC_MAX_CONCURRENT_CALL | - | diff --git a/oap-server/server-bootstrap/src/main/resources/application.yml b/oap-server/server-bootstrap/src/main/resources/application.yml index de3fef97d..595a7e350 100755 --- a/oap-server/server-bootstrap/src/main/resources/application.yml +++ b/oap-server/server-bootstrap/src/main/resources/application.yml @@ -101,7 +101,7 @@ core: # Define the set of span tag keys, which should be searchable through the GraphQL. searchableTracesTags: ${SW_SEARCHABLE_TAG_KEYS:http.method,status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker} # Define the set of log tag keys, which should be searchable through the GraphQL. - searchableLogsTags: ${SW_SEARCHABLE_LOGS_TAG_KEYS:""} + searchableLogsTags: ${SW_SEARCHABLE_LOGS_TAG_KEYS:level,logger,thread} storage: selector: ${SW_STORAGE:h2} elasticsearch: diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/LogQueryService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/LogQueryService.java index 4bc5fc4d3..49cb0070d 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/LogQueryService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/LogQueryService.java @@ -20,6 +20,7 @@ package org.apache.skywalking.oap.server.core.query; import java.io.IOException; import java.util.List; +import java.util.stream.Collectors; import org.apache.skywalking.apm.util.StringUtil; import org.apache.skywalking.oap.server.core.analysis.IDManager; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; @@ -33,6 +34,8 @@ import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO; import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.library.module.Service; +import static java.util.Objects.nonNull; + public class LogQueryService implements Service { private final ModuleManager moduleManager; @@ -64,10 +67,21 @@ public class LogQueryService implements Service { final long startTB, final long endTB, final List tags, - final List keywordsOfContent, - final List excludingKeywordsOfContent) throws IOException { + List keywordsOfContent, + List excludingKeywordsOfContent) throws IOException { PaginationUtils.Page page = PaginationUtils.INSTANCE.exchange(paging); + if (nonNull(keywordsOfContent)) { + keywordsOfContent = keywordsOfContent.stream() + .filter(StringUtil::isNotEmpty) + .collect(Collectors.toList()); + } + if (nonNull(excludingKeywordsOfContent)) { + excludingKeywordsOfContent = excludingKeywordsOfContent.stream() + .filter(StringUtil::isNotEmpty) + .collect(Collectors.toList()); + } + Logs logs = getLogQueryDAO().queryLogs(serviceId, serviceInstanceId, endpointId, diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/LogQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/LogQuery.java index 6b130ea02..5a21f31e8 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/LogQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/LogQuery.java @@ -143,7 +143,7 @@ public class LogQuery implements ILogQueryDAO { nested.close(); } - SelectQueryImpl countQuery = select().count(ENDPOINT_ID).from(client.getDatabase(), LogRecord.INDEX_NAME); + SelectQueryImpl countQuery = select().count(SERVICE_ID).from(client.getDatabase(), LogRecord.INDEX_NAME); for (ConjunctionClause clause : recallQuery.getClauses()) { countQuery.where(clause); } diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/SimpleQueryClient.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/SimpleQueryClient.java index 90d20adcc..72bb57823 100644 --- a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/SimpleQueryClient.java +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/SimpleQueryClient.java @@ -398,12 +398,8 @@ public class SimpleQueryClient { .collect(Collectors.joining()) .replace("{state}", query.state()) .replace("{serviceId}", query.serviceId()) - .replace("{serviceInstanceId}", query.serviceInstanceId()) .replace("{endpointId}", query.endpointId()) .replace("{endpointName}", query.endpointName()) - .replace("{traceId}", query.traceId()) - .replace("{segmentId}", query.segmentId()) - .replace("{spanId}", query.spanId()) .replace("{start}", query.start()) .replace("{end}", query.end()) .replace("{step}", query.step()) diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/log/LogsQuery.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/log/LogsQuery.java index 4fd1fca74..3dc7fdc63 100644 --- a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/log/LogsQuery.java +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/log/LogsQuery.java @@ -23,12 +23,8 @@ public class LogsQuery extends AbstractQuery { private String state = "ALL"; private String serviceId; - private String serviceInstanceId; - private String endpointId; - private String endpointName; - private String traceId; - private String segmentId; - private String spanId; + private String endpointId = ""; + private String endpointName = ""; private String tagKey; private String tagValue; private String pageNum = "1"; @@ -55,15 +51,6 @@ public class LogsQuery extends AbstractQuery { return this; } - public String serviceInstanceId() { - return serviceInstanceId; - } - - public LogsQuery serviceInstanceId(String serviceInstanceId) { - this.serviceInstanceId = serviceInstanceId; - return this; - } - public String endpointId() { return endpointId; } @@ -82,33 +69,6 @@ public class LogsQuery extends AbstractQuery { return this; } - public String traceId() { - return traceId; - } - - public LogsQuery traceId(String traceId) { - this.traceId = traceId; - return this; - } - - public String segmentId() { - return segmentId; - } - - public LogsQuery segmentId(String segmentId) { - this.segmentId = segmentId; - return this; - } - - public String spanId() { - return spanId; - } - - public LogsQuery spanId(String spanId) { - this.spanId = spanId; - return this; - } - public String tagKey() { return tagKey; } diff --git a/test/e2e/e2e-data/src/main/resources/logs.gql b/test/e2e/e2e-data/src/main/resources/logs.gql index b757c21ed..07283ce63 100644 --- a/test/e2e/e2e-data/src/main/resources/logs.gql +++ b/test/e2e/e2e-data/src/main/resources/logs.gql @@ -38,14 +38,8 @@ "condition": { "state": "{state}", "serviceId": "{serviceId}", - "serviceInstanceId": "{serviceInstanceId}", "endpointId": "{endpointId}", "endpointName": "{endpointName}", - "relatedTrace": { - "traceId": "{traceId}", - "segmentId": "{segmentId}", - "spanId": {spanId} - }, "queryDuration": { "start": "{start}", "end": "{end}", diff --git a/test/e2e/e2e-log-producer/pom.xml b/test/e2e/e2e-log-producer/pom.xml deleted file mode 100644 index 8c30f3ab3..000000000 --- a/test/e2e/e2e-log-producer/pom.xml +++ /dev/null @@ -1,66 +0,0 @@ - - - - - - apache-skywalking-e2e - org.apache.skywalking - 1.0.0 - - 4.0.0 - - e2e-log-producer - - - - org.apache.skywalking - e2e-protocol - ${project.version} - - - org.apache.kafka - kafka-clients - ${kafka-clients.version} - - - - - - - org.springframework.boot - spring-boot-maven-plugin - ${spring.boot.version} - - true - true - true - - - - - repackage - - - - - - - - \ No newline at end of file diff --git a/test/e2e/e2e-log-producer/src/main/java/org/apache/skywalking/e2e/LogProducerApplication.java b/test/e2e/e2e-log-producer/src/main/java/org/apache/skywalking/e2e/LogProducerApplication.java deleted file mode 100644 index baf0f1a1d..000000000 --- a/test/e2e/e2e-log-producer/src/main/java/org/apache/skywalking/e2e/LogProducerApplication.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.skywalking.e2e; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; - -@SpringBootApplication -public class LogProducerApplication { - public static void main(String[] args) { - SpringApplication.run(LogProducerApplication.class, args); - } -} diff --git a/test/e2e/e2e-log-producer/src/main/java/org/apache/skywalking/e2e/controller/LogController.java b/test/e2e/e2e-log-producer/src/main/java/org/apache/skywalking/e2e/controller/LogController.java deleted file mode 100644 index a189ab526..000000000 --- a/test/e2e/e2e-log-producer/src/main/java/org/apache/skywalking/e2e/controller/LogController.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.skywalking.e2e.controller; - -import java.util.Collections; -import java.util.Objects; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.AdminClient; -import org.apache.kafka.clients.admin.DescribeTopicsResult; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.serialization.BytesSerializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.kafka.common.utils.Bytes; -import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; -import org.apache.skywalking.apm.network.logging.v3.LogData; -import org.apache.skywalking.apm.network.logging.v3.LogDataBody; -import org.apache.skywalking.apm.network.logging.v3.LogTags; -import org.apache.skywalking.apm.network.logging.v3.TextLog; -import org.apache.skywalking.apm.network.logging.v3.TraceContext; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.HttpStatus; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.ResponseStatus; -import org.springframework.web.bind.annotation.RestController; - -import static java.util.Objects.nonNull; - -@Slf4j -@RestController -public class LogController { - - private static final String TOPIC = "skywalking-logs"; - - private KafkaProducer producer; - - @Value("#{systemProperties['bootstrap.service'] ?: 'localhost:9092'}") - private String bootstrapService; - - @PostConstruct - public void up() { - Properties properties = new Properties(); - properties.setProperty( - ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapService); - producer = new KafkaProducer<>(properties, new StringSerializer(), new BytesSerializer()); - AdminClient adminClient = AdminClient.create(properties); - DescribeTopicsResult topicsResult = adminClient.describeTopics(Collections.singletonList(TOPIC)); - Set topics = topicsResult.values().entrySet().stream() - .map(entry -> { - try { - entry.getValue().get(10, TimeUnit.SECONDS); - return null; - } catch (InterruptedException | ExecutionException | TimeoutException e) { - LOGGER.error("Get KAFKA topic:" + entry.getKey() + " error", e); - } - return entry.getKey(); - }).filter(Objects::nonNull).collect(Collectors.toSet()); - if (!topics.isEmpty()) { - throw new RuntimeException("These topics" + topics + " don't exist."); - } - } - - @PreDestroy - public void down() { - if (nonNull(producer)) { - producer.close(); - } - } - - @PostMapping("/sendLog") - @SuppressWarnings("EmptyMethod") - @ResponseStatus(code = HttpStatus.OK) - public void sendLog() { - try { - LogData logData = LogData.newBuilder() - .setService("e2e") - .setServiceInstance("e2e-instance") - .setEndpoint("/traffic") - .setBody( - LogDataBody.newBuilder() - .setText( - TextLog.newBuilder().setText("[main] INFO log message").build()) - .build()) - .setTags(LogTags.newBuilder() - .addData(KeyStringValuePair.newBuilder() - .setKey("status_code") - .setValue("200") - .build()) - .build()) - .setTraceContext(TraceContext.newBuilder() - .setTraceId("ac81b308-0d66-4c69-a7af-a023a536bd3e") - .setTraceSegmentId( - "6024a2b1fcff48e4a641d69d388bac53.41.16088574455279608") - .setSpanId(0) - .build()) - .build(); - - producer.send( - new ProducerRecord<>(TOPIC, logData.getService(), Bytes.wrap(logData.toByteArray())), - (m, e) -> { - if (nonNull(e)) { - LOGGER.error("Failed to report logs.", e); - } - } - ); - } catch (Exception e) { - LOGGER.error(e.getMessage(), e); - } - } -} diff --git a/test/e2e/e2e-service-provider/pom.xml b/test/e2e/e2e-service-provider/pom.xml index 19c31c460..94ff0b141 100644 --- a/test/e2e/e2e-service-provider/pom.xml +++ b/test/e2e/e2e-service-provider/pom.xml @@ -33,6 +33,10 @@ e2e-service-provider + + 8.4.0-SNAPSHOT + + org.springframework.boot @@ -53,6 +57,16 @@ apm-toolkit-micrometer-registry 8.2.0 + + org.apache.skywalking + apm-toolkit-logback-1.x + ${sw.snapshot.version} + + + org.apache.skywalking + apm-toolkit-trace + ${sw.snapshot.version} + diff --git a/test/e2e/e2e-service-provider/src/main/java/org/apache/skywalking/apm/toolkit/trace/TraceContext.java b/test/e2e/e2e-service-provider/src/main/java/org/apache/skywalking/apm/toolkit/trace/TraceContext.java deleted file mode 100644 index c846f44e0..000000000 --- a/test/e2e/e2e-service-provider/src/main/java/org/apache/skywalking/apm/toolkit/trace/TraceContext.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.apm.toolkit.trace; - -import java.util.Optional; - -/** - * Try to access the sky-walking tracer context. The context is not existed, always. only the middleware, component, or - * rpc-framework are supported in the current invoke stack, in the same thread, the context will be available. - *

- */ -public class TraceContext { - - /** - * Try to get the traceId of current trace context. - * - * @return traceId, if it exists, or empty {@link String}. - */ - public static String traceId() { - return ""; - } - - /** - * Try to get the custom value from trace context. - * - * @return custom data value. - */ - public static Optional getCorrelation(String key) { - return Optional.empty(); - } - - /** - * Put the custom key/value into trace context. - * - * @return previous value if it exists. - */ - public static Optional putCorrelation(String key, String value) { - return Optional.empty(); - } - -} diff --git a/test/e2e/e2e-log-producer/src/main/java/org/apache/skywalking/e2e/controller/HealthController.java b/test/e2e/e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/LogController.java similarity index 70% rename from test/e2e/e2e-log-producer/src/main/java/org/apache/skywalking/e2e/controller/HealthController.java rename to test/e2e/e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/LogController.java index b3cd2a300..baa1b2317 100644 --- a/test/e2e/e2e-log-producer/src/main/java/org/apache/skywalking/e2e/controller/HealthController.java +++ b/test/e2e/e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/LogController.java @@ -17,20 +17,18 @@ package org.apache.skywalking.e2e.controller; -import lombok.RequiredArgsConstructor; -import org.springframework.http.HttpStatus; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.ResponseStatus; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.apm.toolkit.trace.TraceContext; +import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +@Slf4j @RestController -@RequiredArgsConstructor -public class HealthController { - @GetMapping("/health") - @SuppressWarnings("EmptyMethod") - @ResponseStatus(code = HttpStatus.OK) - public void hello() { +public class LogController { + + @RequestMapping(value = "/logs/trigger") + public String trigger() { + LOGGER.info("now: {}", System.currentTimeMillis()); + return TraceContext.traceId(); } - } - diff --git a/test/e2e/e2e-service-provider/src/main/resources/logback.xml b/test/e2e/e2e-service-provider/src/main/resources/logback.xml new file mode 100644 index 000000000..f47372cb2 --- /dev/null +++ b/test/e2e/e2e-service-provider/src/main/resources/logback.xml @@ -0,0 +1,34 @@ + + + + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%X{tid}] [%thread] %-5level %logger{36} -%msg%n + + + + + + + + + + + \ No newline at end of file diff --git a/test/e2e/e2e-test/docker/Dockerfile.provider b/test/e2e/e2e-test/docker/Dockerfile.provider index 89d2def39..68dfb017f 100644 --- a/test/e2e/e2e-test/docker/Dockerfile.provider +++ b/test/e2e/e2e-test/docker/Dockerfile.provider @@ -25,6 +25,10 @@ ADD ${DIST_PACKAGE} . RUN tar -zxf apache-skywalking-apm*.gz --strip=1 || true RUN mv apache-skywalking-apm-bin/* . || true +# add log reporter configuration +RUN echo 'plugin.toolkit.log.grpc.reporter.server_host=${SW_GRPC_LOG_SERVER_HOST:oap}' >> agent/config/agent.config +RUN echo 'plugin.toolkit.log.grpc.reporter.server_port=${SW_GRPC_LOG_SERVER_PORT:11800}' >> agent/config/agent.config + VOLUME /services ADD e2e-service-provider/target/e2e-service-provider-1.0.0.jar /services/ diff --git a/test/e2e/e2e-test/docker/kafka/Dockerfile.log.producer b/test/e2e/e2e-test/docker/Dockerfile.satellite similarity index 59% rename from test/e2e/e2e-test/docker/kafka/Dockerfile.log.producer rename to test/e2e/e2e-test/docker/Dockerfile.satellite index c5a8a3d4a..2e0a7a2f3 100644 --- a/test/e2e/e2e-test/docker/kafka/Dockerfile.log.producer +++ b/test/e2e/e2e-test/docker/Dockerfile.satellite @@ -13,15 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG SW_AGENT_JDK_VERSION=8 -ARG AGENT_JDK_BASE=adoptopenjdk/openjdk${SW_AGENT_JDK_VERSION}:alpine +FROM golang:1.14 AS builder -FROM ${AGENT_JDK_BASE} +ARG COMMIT_HASH=827f3a45e44cb2a64f874ff17d9eb454a680d08f +ARG SATELLITE_CODE=${COMMIT_HASH}.tar.gz +ARG SATELLITE_CODE_URL=https://github.com/apache/skywalking-satellite/archive/${SATELLITE_CODE} -VOLUME /services +ENV CGO_ENABLED=0 -ADD e2e-log-producer/target/e2e-log-producer-1.0.0.jar /services/ +WORKDIR /skywalking-satellite -ENV JAVA_OPTS="" +ADD ${SATELLITE_CODE_URL} . +RUN tar -xf ${SATELLITE_CODE} --strip 1 +RUN rm ${SATELLITE_CODE} -CMD ["sh", "-c", "java $JAVA_OPTS -jar /services/e2e-log-producer-1.0.0.jar"] \ No newline at end of file +RUN make linux + +FROM alpine:3.10 + +COPY --from=builder /skywalking-satellite/bin/skywalking-satellite-latest-linux-amd64 /sw-satellite + +ENTRYPOINT ["/sw-satellite"] \ No newline at end of file diff --git a/test/e2e/e2e-test/docker/base-compose.yml b/test/e2e/e2e-test/docker/base-compose.yml index 19e76ae1a..9bc0f2f88 100644 --- a/test/e2e/e2e-test/docker/base-compose.yml +++ b/test/e2e/e2e-test/docker/base-compose.yml @@ -33,7 +33,6 @@ services: SW_STORAGE_ES_CLUSTER_NODES: es:9200 SW_JDBC_URL: jdbc:mysql://mysql:3306/swtest SW_STORAGE_INFLUXDB_URL: http://influxdb:8086 - SW_SEARCHABLE_LOGS_TAG_KEYS: http.method,status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker JAVA_OPTS: >- -javaagent:/jacoco/jacocoagent.jar=classdumpdir=/jacoco/classes/oap,destfile=/jacoco/oap.exec,includes=org.apache.skywalking.*,excludes=org.apache.skywalking.oap.query.*:org.apache.skywalking.oap.server.core.query.* -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 @@ -60,7 +59,6 @@ services: SW_STORAGE_ES_CLUSTER_NODES: es:9200 SW_JDBC_URL: jdbc:mysql://mysql:3306/swtest SW_STORAGE_INFLUXDB_URL: http://influxdb:8086 - SW_SEARCHABLE_LOGS_TAG_KEYS: http.method,status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker JAVA_OPTS: >- -javaagent:/jacoco/jacocoagent.jar=classdumpdir=/jacoco/classes/oap,destfile=/jacoco/oap.exec,includes=org.apache.skywalking.*,excludes=org.apache.skywalking.oap.query.*:org.apache.skywalking.oap.server.core.query.* -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 @@ -94,6 +92,8 @@ services: - ../../../jacoco:/jacoco environment: SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11800 + SW_GRPC_LOG_SERVER_HOST: oap + SW_GRPC_LOG_SERVER_PORT: 11800 JAVA_OPTS: >- -javaagent:/jacoco/jacocoagent.jar=classdumpdir=/jacoco/classes/provider,destfile=/jacoco/provider.exec,includes=org.apache.skywalking.*,excludes=org.apache.skywalking.apm.dependencies.* -javaagent:/skywalking/agent/skywalking-agent.jar=logging.output=CONSOLE diff --git a/test/e2e/e2e-test/docker/kafka/Dockerfile.provider b/test/e2e/e2e-test/docker/kafka/Dockerfile.provider index 621bf8fde..3e61dc06e 100644 --- a/test/e2e/e2e-test/docker/kafka/Dockerfile.provider +++ b/test/e2e/e2e-test/docker/kafka/Dockerfile.provider @@ -28,6 +28,10 @@ RUN mv apache-skywalking-apm-bin/* . || true RUN sed -i 's/# plugin.kafka.bootstrap_servers/plugin.kafka.bootstrap_servers/g' agent/config/agent.config RUN echo 'plugin.kafka.get_topic_timeout=${SW_GET_TOPIC_TIMEOUT:60}' >> agent/config/agent.config +# add log reporter configuration +RUN echo 'plugin.toolkit.log.grpc.reporter.server_host=${SW_GRPC_LOG_SERVER_HOST:oap}' >> agent/config/agent.config +RUN echo 'plugin.toolkit.log.grpc.reporter.server_port=${SW_GRPC_LOG_SERVER_PORT:11800}' >> agent/config/agent.config + # activate kafka-reporter plugin RUN mv agent/optional-reporter-plugins/kafka-reporter-plugin-*.jar agent/plugins/ || true diff --git a/test/e2e/e2e-test/docker/kafka/docker-compose.log.yml b/test/e2e/e2e-test/docker/kafka/docker-compose.log.yml index 2367d2017..8df544e1f 100644 --- a/test/e2e/e2e-test/docker/kafka/docker-compose.log.yml +++ b/test/e2e/e2e-test/docker/kafka/docker-compose.log.yml @@ -31,24 +31,20 @@ services: condition: service_healthy broker-b: condition: service_healthy - provider_kafka: + + sw-satellite: build: context: ../../../ - dockerfile: e2e-test/docker/kafka/Dockerfile.log.producer - args: - - SW_AGENT_JDK_VERSION=${SW_AGENT_JDK_VERSION} + dockerfile: e2e-test/docker/Dockerfile.satellite networks: - e2e expose: - - 8080 + - 11800 volumes: - - ../../../../jacoco:/jacoco - environment: - JAVA_OPTS: >- - -javaagent:/jacoco/jacocoagent.jar=classdumpdir=/jacoco/classes/provider,destfile=/jacoco/provider.exec,includes=org.apache.skywalking.*,excludes=org.apache.skywalking.apm.dependencies.* - -Dbootstrap.service=broker-a:9092,broker-b:9092 + - ./satellite_config.yaml:/configs/satellite_config.yaml + command: ["start"] healthcheck: - test: ["CMD", "sh", "-c", "nc -nz 127.0.0.1 8080"] + test: [ "CMD", "sh", "-c", "nc -z 127.0.0.1 11800" ] interval: 5s timeout: 60s retries: 120 @@ -59,5 +55,17 @@ services: condition: service_healthy broker-b: condition: service_healthy + + provider: + extends: + file: ../base-compose.yml + service: provider + environment: + SW_GRPC_LOG_SERVER_HOST: sw-satellite + depends_on: + oap: + condition: service_healthy + sw-satellite: + condition: service_healthy networks: e2e: diff --git a/test/e2e/e2e-test/docker/kafka/docker-compose.yml b/test/e2e/e2e-test/docker/kafka/docker-compose.yml index ec5d89eae..b0841889b 100644 --- a/test/e2e/e2e-test/docker/kafka/docker-compose.yml +++ b/test/e2e/e2e-test/docker/kafka/docker-compose.yml @@ -16,14 +16,6 @@ version: '2.1' services: - ui: - extends: - file: ../base-compose.yml - service: ui - depends_on: - oap: - condition: service_healthy - zookeeper: image: zookeeper:3.4 hostname: zookeeper diff --git a/test/e2e/e2e-test/docker/kafka/satellite_config.yaml b/test/e2e/e2e-test/docker/kafka/satellite_config.yaml new file mode 100644 index 000000000..01aca88f4 --- /dev/null +++ b/test/e2e/e2e-test/docker/kafka/satellite_config.yaml @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +logger: + log_pattern: "%time [%level][%field] - %msg" + time_pattern: "2006-01-02 15:04:05.000" + level: "info" + +telemetry: + cluster: cluster1 + service: service1 + instance: instance1 + +sharing: + common_config: + pipe_name: sharing + clients: + - plugin_name: "kafka-client" + brokers: broker-a:9092,broker-b:9092 + version: 2.4.1 + servers: + - plugin_name: "grpc-server" + address: ":11800" +pipes: + - common_config: + pipe_name: pipe1 + gatherer: + server_name: "grpc-server" + receiver: + plugin_name: "grpc-nativelog-receiver" + queue: + plugin_name: "mmap-queue" + segment_size: 524288 + max_in_mem_segments: 6 + queue_dir: "pipe1-log-grpc-receiver-queue" + processor: + filters: + sender: + fallbacker: + plugin_name: none-fallbacker + flush_time: 1000 + max_buffer_size: 200 + min_flush_events: 5 + client_name: kafka-client + forwarders: + - plugin_name: nativelog-kafka-forwarder + topic: skywalking-logs \ No newline at end of file diff --git a/test/e2e/e2e-test/docker/log/docker-compose.es6.yml b/test/e2e/e2e-test/docker/log/docker-compose.es6.yml index beefbff91..595f97663 100644 --- a/test/e2e/e2e-test/docker/log/docker-compose.es6.yml +++ b/test/e2e/e2e-test/docker/log/docker-compose.es6.yml @@ -25,7 +25,7 @@ services: environment: - discovery.type=single-node healthcheck: - test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"] + test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200" ] interval: 5s timeout: 60s retries: 120 @@ -40,5 +40,12 @@ services: es: condition: service_healthy + provider: + extends: + file: ../base-compose.yml + service: provider + depends_on: + oap: + condition: service_healthy networks: e2e: \ No newline at end of file diff --git a/test/e2e/e2e-test/docker/log/docker-compose.es7.yml b/test/e2e/e2e-test/docker/log/docker-compose.es7.yml index 28c79a7ec..37adf3f83 100644 --- a/test/e2e/e2e-test/docker/log/docker-compose.es7.yml +++ b/test/e2e/e2e-test/docker/log/docker-compose.es7.yml @@ -40,5 +40,12 @@ services: es: condition: service_healthy + provider: + extends: + file: ../base-compose.yml + service: provider + depends_on: + oap: + condition: service_healthy networks: e2e: \ No newline at end of file diff --git a/test/e2e/e2e-test/docker/log/docker-compose.h2.yml b/test/e2e/e2e-test/docker/log/docker-compose.h2.yml index 7d21486d0..128621b52 100644 --- a/test/e2e/e2e-test/docker/log/docker-compose.h2.yml +++ b/test/e2e/e2e-test/docker/log/docker-compose.h2.yml @@ -41,5 +41,13 @@ services: h2db: condition: service_healthy + provider: + extends: + file: ../base-compose.yml + service: provider + depends_on: + oap: + condition: service_healthy + networks: e2e: \ No newline at end of file diff --git a/test/e2e/e2e-test/docker/log/docker-compose.influxdb.yml b/test/e2e/e2e-test/docker/log/docker-compose.influxdb.yml index 6313f352f..2cb6abb6e 100644 --- a/test/e2e/e2e-test/docker/log/docker-compose.influxdb.yml +++ b/test/e2e/e2e-test/docker/log/docker-compose.influxdb.yml @@ -38,5 +38,13 @@ services: influxdb: condition: service_healthy + provider: + extends: + file: ../base-compose.yml + service: provider + depends_on: + oap: + condition: service_healthy + networks: e2e: \ No newline at end of file diff --git a/test/e2e/e2e-test/docker/log/docker-compose.mysql.yml b/test/e2e/e2e-test/docker/log/docker-compose.mysql.yml index d577079f1..4661e5e3f 100644 --- a/test/e2e/e2e-test/docker/log/docker-compose.mysql.yml +++ b/test/e2e/e2e-test/docker/log/docker-compose.mysql.yml @@ -43,5 +43,13 @@ services: condition: service_healthy entrypoint: ['sh', '-c', '/download-mysql.sh && /skywalking/docker-entrypoint.sh'] + provider: + extends: + file: ../base-compose.yml + service: provider + depends_on: + oap: + condition: service_healthy + networks: e2e: \ No newline at end of file diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaE2E.java index c1272a294..c63cbb31b 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaE2E.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaE2E.java @@ -90,7 +90,7 @@ public class KafkaE2E extends SkyWalkingTestAdapter { protected HostAndPort serviceHostPort; @SuppressWarnings("unused") - @ContainerHostAndPort(name = "ui", port = 8080) + @ContainerHostAndPort(name = "oap", port = 12800) protected HostAndPort swWebappHostPort; @BeforeAll diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaLogE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaLogE2E.java index 85b41b6ba..9c34a6dfe 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaLogE2E.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaLogE2E.java @@ -61,17 +61,13 @@ public class KafkaLogE2E extends SkyWalkingTestAdapter { private HostAndPort swWebappHostPort; @SuppressWarnings("unused") - @ContainerHostAndPort(name = "oap", port = 11800) - private HostAndPort oapHostPost; - - @SuppressWarnings("unused") - @ContainerHostAndPort(name = "provider_kafka", port = 8080) + @ContainerHostAndPort(name = "provider", port = 9090) private HostAndPort serviceHostPort; @BeforeAll public void setUp() throws Exception { queryClient(swWebappHostPort); - trafficController(serviceHostPort, "/sendLog"); + trafficController(serviceHostPort, "/logs/trigger"); } @AfterAll @@ -94,27 +90,17 @@ public class KafkaLogE2E extends SkyWalkingTestAdapter { LOGGER.info("verifying service instance: {}", service); // instance verifyServiceInstances(service); - // endpoint - verifyServiceEndpoints(service); } } @RetryableTest public void verifyLog() throws Exception { - LogsQuery logsQuery = new LogsQuery().serviceId("ZTJl.1") - .serviceInstanceId("ZTJl.1_ZTJlLWluc3RhbmNl") - .endpointId("ZTJl.1_L3RyYWZmaWM=") - .endpointName("/traffic") - .traceId("ac81b308-0d66-4c69-a7af-a023a536bd3e") - .segmentId( - "6024a2b1fcff48e4a641d69d388bac53.41.16088574455279608") - .spanId("0") - .tag("status_code", "200") + LogsQuery logsQuery = new LogsQuery().serviceId("WW91cl9BcHBsaWNhdGlvbk5hbWU=.1") + .tag("logger", "org.apache.skywalking.e2e.controller.LogController") .start(startTime) .end(Times.now()); if (graphql.supportQueryLogsByKeywords()) { - logsQuery.keywordsOfContent("main", "INFO") - .excludingKeywordsOfContent("ERROR"); + logsQuery.keywordsOfContent("now"); } final List logs = graphql.logs(logsQuery); LOGGER.info("logs: {}", logs); diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaMeterE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaMeterE2E.java index e6474f2d8..2738f6d72 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaMeterE2E.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaMeterE2E.java @@ -57,7 +57,7 @@ public class KafkaMeterE2E extends SkyWalkingTestAdapter { protected DockerComposeContainer justForSideEffects; @SuppressWarnings("unused") - @ContainerHostAndPort(name = "ui", port = 8080) + @ContainerHostAndPort(name = "oap", port = 12800) protected HostAndPort swWebappHostPort; @SuppressWarnings("unused") diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaProfileE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaProfileE2E.java index cf53fc05f..aacefc1f6 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaProfileE2E.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaProfileE2E.java @@ -85,7 +85,7 @@ public class KafkaProfileE2E extends SkyWalkingTestAdapter { protected HostAndPort serviceHostPort; @SuppressWarnings("unused") - @ContainerHostAndPort(name = "ui", port = 8080) + @ContainerHostAndPort(name = "oap", port = 12800) protected HostAndPort swWebappHostPort; @SuppressWarnings("unused") diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/LogE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/LogE2E.java index c24416221..de07f3a41 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/LogE2E.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/LogE2E.java @@ -17,27 +17,13 @@ package org.apache.skywalking.e2e.log; -import io.grpc.ManagedChannel; -import io.grpc.internal.DnsNameResolverProvider; -import io.grpc.netty.NettyChannelBuilder; -import io.grpc.stub.StreamObserver; import java.util.List; -import java.util.concurrent.CountDownLatch; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; -import org.apache.skywalking.apm.network.common.v3.Commands; -import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; -import org.apache.skywalking.apm.network.logging.v3.LogData; -import org.apache.skywalking.apm.network.logging.v3.LogDataBody; -import org.apache.skywalking.apm.network.logging.v3.LogReportServiceGrpc; -import org.apache.skywalking.apm.network.logging.v3.LogTags; -import org.apache.skywalking.apm.network.logging.v3.TextLog; -import org.apache.skywalking.apm.network.logging.v3.TraceContext; import org.apache.skywalking.e2e.annotation.ContainerHostAndPort; import org.apache.skywalking.e2e.annotation.DockerCompose; import org.apache.skywalking.e2e.base.SkyWalkingE2E; import org.apache.skywalking.e2e.base.SkyWalkingTestAdapter; -import org.apache.skywalking.e2e.base.TrafficController; import org.apache.skywalking.e2e.common.HostAndPort; import org.apache.skywalking.e2e.retryable.RetryableTest; import org.apache.skywalking.e2e.service.Service; @@ -59,9 +45,6 @@ import static org.apache.skywalking.e2e.utils.Yamls.load; @Slf4j @SkyWalkingE2E public class LogE2E extends SkyWalkingTestAdapter { - - private final static int MAX_INBOUND_MESSAGE_SIZE = 1024 * 1024 * 50; - @SuppressWarnings("unused") @DockerCompose({ "docker/log/docker-compose.${SW_STORAGE}.yml" @@ -73,25 +56,20 @@ public class LogE2E extends SkyWalkingTestAdapter { private HostAndPort swWebappHostPort; @SuppressWarnings("unused") - @ContainerHostAndPort(name = "oap", port = 11800) - private HostAndPort oapHostPost; - - private LogReportServiceGrpc.LogReportServiceStub logReportServiceStub; + @ContainerHostAndPort(name = "provider", port = 9090) + protected HostAndPort providerHostPort; @BeforeAll - public void setUp() { + public void setUp() throws Exception { queryClient(swWebappHostPort); - final ManagedChannel channel = NettyChannelBuilder.forAddress(oapHostPost.host(), oapHostPost.port()) - .nameResolverFactory(new DnsNameResolverProvider()) - .maxInboundMessageSize(MAX_INBOUND_MESSAGE_SIZE) - .usePlaintext().build(); - logReportServiceStub = LogReportServiceGrpc.newStub(channel); - generateTraffic(); + trafficController(providerHostPort, "/logs/trigger"); } @AfterAll public void tearDown() { - trafficController.stop(); + if (trafficController != null) { + trafficController.stop(); + } } @RetryableTest @@ -107,27 +85,17 @@ public class LogE2E extends SkyWalkingTestAdapter { LOGGER.info("verifying service instance: {}", service); // instance verifyServiceInstances(service); - // endpoint - verifyServiceEndpoints(service); } } @RetryableTest public void verifyLog() throws Exception { - LogsQuery logsQuery = new LogsQuery().serviceId("ZTJl.1") - .serviceInstanceId("ZTJl.1_ZTJlLWluc3RhbmNl") - .endpointId("ZTJl.1_L3RyYWZmaWM=") - .endpointName("/traffic") - .traceId("ac81b308-0d66-4c69-a7af-a023a536bd3e") - .segmentId( - "6024a2b1fcff48e4a641d69d388bac53.41.16088574455279608") - .spanId("0") - .tag("status_code", "200") + LogsQuery logsQuery = new LogsQuery().serviceId("WW91cl9BcHBsaWNhdGlvbk5hbWU=.1") + .tag("logger", "org.apache.skywalking.e2e.controller.LogController") .start(startTime) .end(Times.now()); if (graphql.supportQueryLogsByKeywords()) { - logsQuery.keywordsOfContent("main", "INFO") - .excludingKeywordsOfContent("ERROR"); + logsQuery.keywordsOfContent("now"); } final List logs = graphql.logs(logsQuery); LOGGER.info("logs: {}", logs); @@ -149,68 +117,4 @@ public class LogE2E extends SkyWalkingTestAdapter { load("expected/log/endpoints.yml").as(EndpointsMatcher.class).verify(endpoints); } - - private void generateTraffic() { - trafficController = TrafficController.builder() - .sender(this::sendLog) - .build(); - trafficController.start(); - } - - private boolean sendLog() { - try { - LogData logData = LogData.newBuilder() - .setService("e2e") - .setServiceInstance("e2e-instance") - .setEndpoint("/traffic") - .setBody( - LogDataBody.newBuilder() - .setText(TextLog.newBuilder() - .setText( - "[main] INFO log message") - .build()) - .build()) - .setTags(LogTags.newBuilder() - .addData(KeyStringValuePair.newBuilder() - .setKey("status_code") - .setValue("200") - .build()) - .build()) - .setTraceContext(TraceContext.newBuilder() - .setTraceId("ac81b308-0d66-4c69-a7af-a023a536bd3e") - .setTraceSegmentId( - "6024a2b1fcff48e4a641d69d388bac53.41.16088574455279608") - .setSpanId(0) - .build()) - .build(); - - CountDownLatch downLatch = new CountDownLatch(1); - - StreamObserver streamObserver = logReportServiceStub.collect(new StreamObserver() { - @Override - public void onNext(final Commands commands) { - - } - - @Override - public void onError(final Throwable throwable) { - LOGGER.error(throwable.getMessage(), throwable); - downLatch.countDown(); - } - - @Override - public void onCompleted() { - downLatch.countDown(); - } - }); - - streamObserver.onNext(logData); - streamObserver.onCompleted(); - downLatch.await(); - return true; - } catch (Exception e) { - LOGGER.warn(e.getMessage(), e); - return false; - } - } } diff --git a/test/e2e/e2e-test/src/test/resources/expected/log/endpoints.yml b/test/e2e/e2e-test/src/test/resources/expected/log/endpoints.yml index 0e83a1d4f..8d95b80c0 100644 --- a/test/e2e/e2e-test/src/test/resources/expected/log/endpoints.yml +++ b/test/e2e/e2e-test/src/test/resources/expected/log/endpoints.yml @@ -15,4 +15,4 @@ endpoints: - key: not null - label: /traffic + label: /logs/traffic diff --git a/test/e2e/e2e-test/src/test/resources/expected/log/instances.yml b/test/e2e/e2e-test/src/test/resources/expected/log/instances.yml index 3ccf0f6a9..7de572a77 100644 --- a/test/e2e/e2e-test/src/test/resources/expected/log/instances.yml +++ b/test/e2e/e2e-test/src/test/resources/expected/log/instances.yml @@ -15,4 +15,4 @@ instances: - key: not null - label: e2e-instance + label: not null diff --git a/test/e2e/e2e-test/src/test/resources/expected/log/logs.yml b/test/e2e/e2e-test/src/test/resources/expected/log/logs.yml index b8702b98a..13abc991d 100644 --- a/test/e2e/e2e-test/src/test/resources/expected/log/logs.yml +++ b/test/e2e/e2e-test/src/test/resources/expected/log/logs.yml @@ -14,17 +14,15 @@ # limitations under the License. logs: - - serviceName: e2e + - serviceName: not null serviceId: not null - serviceInstanceName: "e2e-instance" + serviceInstanceName: not null serviceInstanceId: not null - endpointName: /traffic - endpointId: not null - traceId: "ac81b308-0d66-4c69-a7af-a023a536bd3e" + traceId: not null timestamp: not null isError: false contentType: TEXT content: not null tags: - - key: status_code - value: 200 \ No newline at end of file + - key: level + value: INFO \ No newline at end of file diff --git a/test/e2e/e2e-test/src/test/resources/expected/log/services.yml b/test/e2e/e2e-test/src/test/resources/expected/log/services.yml index d87fe90a3..3583691d7 100644 --- a/test/e2e/e2e-test/src/test/resources/expected/log/services.yml +++ b/test/e2e/e2e-test/src/test/resources/expected/log/services.yml @@ -15,4 +15,4 @@ services: - key: not null - label: "e2e" + label: not null diff --git a/test/e2e/pom.xml b/test/e2e/pom.xml index be7b7ba14..1e5288593 100644 --- a/test/e2e/pom.xml +++ b/test/e2e/pom.xml @@ -40,7 +40,6 @@ e2e-protocol e2e-service-provider e2e-service-consumer - e2e-log-producer e2e-test @@ -68,6 +67,21 @@ 1.15.0 + + + apache.snapshots + Apache Development Snapshot Repository + https://repository.apache.org/content/groups/snapshots/ + + false + + + true + + + + +