Enhancement log-recerive e2e test (#6248)

This commit is contained in:
zhang-wei 2021-01-29 00:07:16 +08:00 committed by GitHub
parent fd02bcd6f6
commit 01d1eff6c7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
39 changed files with 313 additions and 566 deletions

View File

@ -23,9 +23,8 @@ on:
- '!**.md'
schedule:
- cron: '0 18 * * *'
env:
SKIP_TEST: true
SW_AGENT_JDK_VERSION: 8
jobs:
Log:
@ -47,12 +46,10 @@ jobs:
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Compile and Build
run: make docker && ES_VERSION=es7 TAG=latest-es7 make docker
- name: Copy dist package
run: cp -R dist test/e2e/
- name: Log Receiver ${{ matrix.storage }}
run: ./mvnw --batch-mode -f test/e2e/pom.xml -am -DfailIfNoTests=false verify -Dit.test=org.apache.skywalking.e2e.log.LogE2E
- name: Run E2E Test
uses: ./.github/actions/e2e-test
with:
test_class: org.apache.skywalking.e2e.log.LogE2E
- uses: actions/upload-artifact@v1
if: failure()
with:

View File

@ -18,11 +18,14 @@
package org.apache.skywalking.apm.toolkit.logging.common.log;
import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
import io.grpc.StatusRuntimeException;
import io.grpc.stub.StreamObserver;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.skywalking.apm.agent.core.boot.OverrideImplementor;
import org.apache.skywalking.apm.agent.core.conf.Config;
import org.apache.skywalking.apm.agent.core.logging.api.ILog;
@ -36,11 +39,6 @@ import org.apache.skywalking.apm.network.common.v3.Commands;
import org.apache.skywalking.apm.network.logging.v3.LogData;
import org.apache.skywalking.apm.network.logging.v3.LogReportServiceGrpc;
import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
import io.grpc.StatusRuntimeException;
import io.grpc.stub.StreamObserver;
/**
* Report log to server by grpc
*/
@ -59,14 +57,22 @@ public class GRPCLogReportServiceClient extends LogReportServiceClient {
@Override
public void boot() throws Throwable {
carrier = new DataCarrier<>("gRPC-log", "gRPC-log", Config.Buffer.CHANNEL_SIZE, Config.Buffer.BUFFER_SIZE,
BufferStrategy.IF_POSSIBLE);
carrier = new DataCarrier<>("gRPC-log", "gRPC-log",
Config.Buffer.CHANNEL_SIZE,
Config.Buffer.BUFFER_SIZE,
BufferStrategy.IF_POSSIBLE
);
carrier.consume(this, 1);
channel = ManagedChannelBuilder
.forAddress(ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.SERVER_HOST,
ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.SERVER_PORT).usePlaintext().build();
.forAddress(
ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.SERVER_HOST,
ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.SERVER_PORT
)
.usePlaintext()
.build();
asyncStub = LogReportServiceGrpc.newStub(channel)
.withMaxOutboundMessageSize(ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.MAX_MESSAGE_SIZE);
.withMaxOutboundMessageSize(
ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.MAX_MESSAGE_SIZE);
}
@Override
@ -99,7 +105,7 @@ public class GRPCLogReportServiceClient extends LogReportServiceClient {
final GRPCStreamServiceStatus waitStatus = new GRPCStreamServiceStatus(false);
try {
reportStreamObserver = asyncStub.withDeadlineAfter(
ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.UPSTREAM_TIMEOUT, TimeUnit.SECONDS
ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.UPSTREAM_TIMEOUT, TimeUnit.SECONDS
).collect(new StreamObserver<Commands>() {
@Override
public void onNext(Commands commands) {
@ -113,7 +119,8 @@ public class GRPCLogReportServiceClient extends LogReportServiceClient {
}
LOGGER.error(t, "Try to send {} log data to collector, with unexpected exception.",
dataList.size());
dataList.size()
);
}
@Override

View File

@ -23,33 +23,33 @@ import org.apache.skywalking.apm.agent.core.boot.PluginConfig;
public class ToolkitConfig {
public static class Plugin {
@PluginConfig(root = ToolkitConfig.class)
public static class Toolkit {
public static class Log {
public static class GRPC {
public static class Reporter {
/**
* The host of gRPC log server.
*/
public static String SERVER_HOST = "127.0.0.1";
public static class Log {
public static class GRPC {
@PluginConfig(root = ToolkitConfig.class)
public static class Reporter {
/**
* The host of gRPC log server.
*/
public static String SERVER_HOST = "127.0.0.1";
/**
* The port of gRPC log server.
*/
public static int SERVER_PORT = 11800;
/**
* The port of gRPC log server.
*/
public static int SERVER_PORT = 11800;
/**
* The max size of message to send to server.Default is 10 MB.
*/
public static int MAX_MESSAGE_SIZE = 10 * 1024 * 1024;
/**
* The max size of message to send to server.Default is 10 MB.
*/
public static int MAX_MESSAGE_SIZE = 10 * 1024 * 1024;
/**
* How long grpc client will timeout in sending data to upstream. The unit is second.
*/
public static int UPSTREAM_TIMEOUT = 30;
}
}
}
/**
* How long grpc client will timeout in sending data to upstream. The unit is second.
*/
public static int UPSTREAM_TIMEOUT = 30;
}
}
}
}
}
}

View File

@ -30,7 +30,7 @@ core|default|role|Option values, `Mixed/Receiver/Aggregator`. **Receiver** mode
| - | - | instanceNameMaxLength| Max length limitation of service instance name. The max length of service + instance names should be less than 200.|SW_INSTANCE_NAME_MAX_LENGTH|70|
| - | - | endpointNameMaxLength| Max length limitation of endpoint name. The max length of service + endpoint names should be less than 240.|SW_ENDPOINT_NAME_MAX_LENGTH|150|
| - | - | searchableTracesTags | Define the set of span tag keys, which should be searchable through the GraphQL. Multiple values should be separated through the comma. | SW_SEARCHABLE_TAG_KEYS | http.method,status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker|
| - | - | searchableLogsTags | Define the set of log tag keys, which should be searchable through the GraphQL. Multiple values should be separated through the comma. | SW_SEARCHABLE_LOGS_TAG_KEYS | - |
| - | - | searchableLogsTags | Define the set of log tag keys, which should be searchable through the GraphQL. Multiple values should be separated through the comma. | SW_SEARCHABLE_LOGS_TAG_KEYS | level,logger,thread |
| - | - | gRPCThreadPoolSize|Pool size of gRPC server| SW_CORE_GRPC_THREAD_POOL_SIZE | CPU core * 4|
| - | - | gRPCThreadPoolQueueSize| The queue size of gRPC server| SW_CORE_GRPC_POOL_QUEUE_SIZE | 10000|
| - | - | maxConcurrentCallsPerConnection | The maximum number of concurrent calls permitted for each incoming connection. Defaults to no limit. | SW_CORE_GRPC_MAX_CONCURRENT_CALL | - |

View File

@ -101,7 +101,7 @@ core:
# Define the set of span tag keys, which should be searchable through the GraphQL.
searchableTracesTags: ${SW_SEARCHABLE_TAG_KEYS:http.method,status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker}
# Define the set of log tag keys, which should be searchable through the GraphQL.
searchableLogsTags: ${SW_SEARCHABLE_LOGS_TAG_KEYS:""}
searchableLogsTags: ${SW_SEARCHABLE_LOGS_TAG_KEYS:level,logger,thread}
storage:
selector: ${SW_STORAGE:h2}
elasticsearch:

View File

@ -20,6 +20,7 @@ package org.apache.skywalking.oap.server.core.query;
import java.io.IOException;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.skywalking.apm.util.StringUtil;
import org.apache.skywalking.oap.server.core.analysis.IDManager;
import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag;
@ -33,6 +34,8 @@ import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
import org.apache.skywalking.oap.server.library.module.Service;
import static java.util.Objects.nonNull;
public class LogQueryService implements Service {
private final ModuleManager moduleManager;
@ -64,10 +67,21 @@ public class LogQueryService implements Service {
final long startTB,
final long endTB,
final List<Tag> tags,
final List<String> keywordsOfContent,
final List<String> excludingKeywordsOfContent) throws IOException {
List<String> keywordsOfContent,
List<String> excludingKeywordsOfContent) throws IOException {
PaginationUtils.Page page = PaginationUtils.INSTANCE.exchange(paging);
if (nonNull(keywordsOfContent)) {
keywordsOfContent = keywordsOfContent.stream()
.filter(StringUtil::isNotEmpty)
.collect(Collectors.toList());
}
if (nonNull(excludingKeywordsOfContent)) {
excludingKeywordsOfContent = excludingKeywordsOfContent.stream()
.filter(StringUtil::isNotEmpty)
.collect(Collectors.toList());
}
Logs logs = getLogQueryDAO().queryLogs(serviceId,
serviceInstanceId,
endpointId,

View File

@ -143,7 +143,7 @@ public class LogQuery implements ILogQueryDAO {
nested.close();
}
SelectQueryImpl countQuery = select().count(ENDPOINT_ID).from(client.getDatabase(), LogRecord.INDEX_NAME);
SelectQueryImpl countQuery = select().count(SERVICE_ID).from(client.getDatabase(), LogRecord.INDEX_NAME);
for (ConjunctionClause clause : recallQuery.getClauses()) {
countQuery.where(clause);
}

View File

@ -398,12 +398,8 @@ public class SimpleQueryClient {
.collect(Collectors.joining())
.replace("{state}", query.state())
.replace("{serviceId}", query.serviceId())
.replace("{serviceInstanceId}", query.serviceInstanceId())
.replace("{endpointId}", query.endpointId())
.replace("{endpointName}", query.endpointName())
.replace("{traceId}", query.traceId())
.replace("{segmentId}", query.segmentId())
.replace("{spanId}", query.spanId())
.replace("{start}", query.start())
.replace("{end}", query.end())
.replace("{step}", query.step())

View File

@ -23,12 +23,8 @@ public class LogsQuery extends AbstractQuery<LogsQuery> {
private String state = "ALL";
private String serviceId;
private String serviceInstanceId;
private String endpointId;
private String endpointName;
private String traceId;
private String segmentId;
private String spanId;
private String endpointId = "";
private String endpointName = "";
private String tagKey;
private String tagValue;
private String pageNum = "1";
@ -55,15 +51,6 @@ public class LogsQuery extends AbstractQuery<LogsQuery> {
return this;
}
public String serviceInstanceId() {
return serviceInstanceId;
}
public LogsQuery serviceInstanceId(String serviceInstanceId) {
this.serviceInstanceId = serviceInstanceId;
return this;
}
public String endpointId() {
return endpointId;
}
@ -82,33 +69,6 @@ public class LogsQuery extends AbstractQuery<LogsQuery> {
return this;
}
public String traceId() {
return traceId;
}
public LogsQuery traceId(String traceId) {
this.traceId = traceId;
return this;
}
public String segmentId() {
return segmentId;
}
public LogsQuery segmentId(String segmentId) {
this.segmentId = segmentId;
return this;
}
public String spanId() {
return spanId;
}
public LogsQuery spanId(String spanId) {
this.spanId = spanId;
return this;
}
public String tagKey() {
return tagKey;
}

View File

@ -38,14 +38,8 @@
"condition": {
"state": "{state}",
"serviceId": "{serviceId}",
"serviceInstanceId": "{serviceInstanceId}",
"endpointId": "{endpointId}",
"endpointName": "{endpointName}",
"relatedTrace": {
"traceId": "{traceId}",
"segmentId": "{segmentId}",
"spanId": {spanId}
},
"queryDuration": {
"start": "{start}",
"end": "{end}",

View File

@ -1,66 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>apache-skywalking-e2e</artifactId>
<groupId>org.apache.skywalking</groupId>
<version>1.0.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>e2e-log-producer</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.skywalking</groupId>
<artifactId>e2e-protocol</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>${kafka-clients.version}</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${spring.boot.version}</version>
<configuration>
<executable>true</executable>
<addResources>true</addResources>
<excludeDevtools>true</excludeDevtools>
</configuration>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

View File

@ -1,28 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.skywalking.e2e;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class LogProducerApplication {
public static void main(String[] args) {
SpringApplication.run(LogProducerApplication.class, args);
}
}

View File

@ -1,134 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.skywalking.e2e.controller;
import java.util.Collections;
import java.util.Objects;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.admin.AdminClient;
import org.apache.kafka.clients.admin.DescribeTopicsResult;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.BytesSerializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.apache.kafka.common.utils.Bytes;
import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair;
import org.apache.skywalking.apm.network.logging.v3.LogData;
import org.apache.skywalking.apm.network.logging.v3.LogDataBody;
import org.apache.skywalking.apm.network.logging.v3.LogTags;
import org.apache.skywalking.apm.network.logging.v3.TextLog;
import org.apache.skywalking.apm.network.logging.v3.TraceContext;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestController;
import static java.util.Objects.nonNull;
@Slf4j
@RestController
public class LogController {
private static final String TOPIC = "skywalking-logs";
private KafkaProducer<String, Bytes> producer;
@Value("#{systemProperties['bootstrap.service'] ?: 'localhost:9092'}")
private String bootstrapService;
@PostConstruct
public void up() {
Properties properties = new Properties();
properties.setProperty(
ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapService);
producer = new KafkaProducer<>(properties, new StringSerializer(), new BytesSerializer());
AdminClient adminClient = AdminClient.create(properties);
DescribeTopicsResult topicsResult = adminClient.describeTopics(Collections.singletonList(TOPIC));
Set<String> topics = topicsResult.values().entrySet().stream()
.map(entry -> {
try {
entry.getValue().get(10, TimeUnit.SECONDS);
return null;
} catch (InterruptedException | ExecutionException | TimeoutException e) {
LOGGER.error("Get KAFKA topic:" + entry.getKey() + " error", e);
}
return entry.getKey();
}).filter(Objects::nonNull).collect(Collectors.toSet());
if (!topics.isEmpty()) {
throw new RuntimeException("These topics" + topics + " don't exist.");
}
}
@PreDestroy
public void down() {
if (nonNull(producer)) {
producer.close();
}
}
@PostMapping("/sendLog")
@SuppressWarnings("EmptyMethod")
@ResponseStatus(code = HttpStatus.OK)
public void sendLog() {
try {
LogData logData = LogData.newBuilder()
.setService("e2e")
.setServiceInstance("e2e-instance")
.setEndpoint("/traffic")
.setBody(
LogDataBody.newBuilder()
.setText(
TextLog.newBuilder().setText("[main] INFO log message").build())
.build())
.setTags(LogTags.newBuilder()
.addData(KeyStringValuePair.newBuilder()
.setKey("status_code")
.setValue("200")
.build())
.build())
.setTraceContext(TraceContext.newBuilder()
.setTraceId("ac81b308-0d66-4c69-a7af-a023a536bd3e")
.setTraceSegmentId(
"6024a2b1fcff48e4a641d69d388bac53.41.16088574455279608")
.setSpanId(0)
.build())
.build();
producer.send(
new ProducerRecord<>(TOPIC, logData.getService(), Bytes.wrap(logData.toByteArray())),
(m, e) -> {
if (nonNull(e)) {
LOGGER.error("Failed to report logs.", e);
}
}
);
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
}
}
}

View File

@ -33,6 +33,10 @@
<artifactId>e2e-service-provider</artifactId>
<properties>
<sw.snapshot.version>8.4.0-SNAPSHOT</sw.snapshot.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
@ -53,6 +57,16 @@
<artifactId>apm-toolkit-micrometer-registry</artifactId>
<version>8.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.skywalking</groupId>
<artifactId>apm-toolkit-logback-1.x</artifactId>
<version>${sw.snapshot.version}</version>
</dependency>
<dependency>
<groupId>org.apache.skywalking</groupId>
<artifactId>apm-toolkit-trace</artifactId>
<version>${sw.snapshot.version}</version>
</dependency>
</dependencies>
<build>

View File

@ -1,57 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.apm.toolkit.trace;
import java.util.Optional;
/**
* Try to access the sky-walking tracer context. The context is not existed, always. only the middleware, component, or
* rpc-framework are supported in the current invoke stack, in the same thread, the context will be available.
* <p>
*/
public class TraceContext {
/**
* Try to get the traceId of current trace context.
*
* @return traceId, if it exists, or empty {@link String}.
*/
public static String traceId() {
return "";
}
/**
* Try to get the custom value from trace context.
*
* @return custom data value.
*/
public static Optional<String> getCorrelation(String key) {
return Optional.empty();
}
/**
* Put the custom key/value into trace context.
*
* @return previous value if it exists.
*/
public static Optional<String> putCorrelation(String key, String value) {
return Optional.empty();
}
}

View File

@ -17,20 +17,18 @@
package org.apache.skywalking.e2e.controller;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.ResponseStatus;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.apm.toolkit.trace.TraceContext;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@Slf4j
@RestController
@RequiredArgsConstructor
public class HealthController {
@GetMapping("/health")
@SuppressWarnings("EmptyMethod")
@ResponseStatus(code = HttpStatus.OK)
public void hello() {
public class LogController {
@RequestMapping(value = "/logs/trigger")
public String trigger() {
LOGGER.info("now: {}", System.currentTimeMillis());
return TraceContext.traceId();
}
}

View File

@ -0,0 +1,34 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration scan="true" scanPeriod=" 5 seconds">
<appender name="stdout" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.core.encoder.LayoutWrappingEncoder">
<layout class="org.apache.skywalking.apm.toolkit.log.logback.v1.x.mdc.TraceIdMDCPatternLogbackLayout">
<Pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%X{tid}] [%thread] %-5level %logger{36} -%msg%n</Pattern>
</layout>
</encoder>
</appender>
<appender name="grpc-log" class="org.apache.skywalking.apm.toolkit.log.logback.v1.x.log.GRPCLogClientAppender"/>
<root level="INFO">
<appender-ref ref="grpc-log"/>
<appender-ref ref="stdout"/>
</root>
</configuration>

View File

@ -25,6 +25,10 @@ ADD ${DIST_PACKAGE} .
RUN tar -zxf apache-skywalking-apm*.gz --strip=1 || true
RUN mv apache-skywalking-apm-bin/* . || true
# add log reporter configuration
RUN echo 'plugin.toolkit.log.grpc.reporter.server_host=${SW_GRPC_LOG_SERVER_HOST:oap}' >> agent/config/agent.config
RUN echo 'plugin.toolkit.log.grpc.reporter.server_port=${SW_GRPC_LOG_SERVER_PORT:11800}' >> agent/config/agent.config
VOLUME /services
ADD e2e-service-provider/target/e2e-service-provider-1.0.0.jar /services/

View File

@ -13,15 +13,24 @@
# See the License for the specific language governing permissions and
# limitations under the License.
ARG SW_AGENT_JDK_VERSION=8
ARG AGENT_JDK_BASE=adoptopenjdk/openjdk${SW_AGENT_JDK_VERSION}:alpine
FROM golang:1.14 AS builder
FROM ${AGENT_JDK_BASE}
ARG COMMIT_HASH=827f3a45e44cb2a64f874ff17d9eb454a680d08f
ARG SATELLITE_CODE=${COMMIT_HASH}.tar.gz
ARG SATELLITE_CODE_URL=https://github.com/apache/skywalking-satellite/archive/${SATELLITE_CODE}
VOLUME /services
ENV CGO_ENABLED=0
ADD e2e-log-producer/target/e2e-log-producer-1.0.0.jar /services/
WORKDIR /skywalking-satellite
ENV JAVA_OPTS=""
ADD ${SATELLITE_CODE_URL} .
RUN tar -xf ${SATELLITE_CODE} --strip 1
RUN rm ${SATELLITE_CODE}
CMD ["sh", "-c", "java $JAVA_OPTS -jar /services/e2e-log-producer-1.0.0.jar"]
RUN make linux
FROM alpine:3.10
COPY --from=builder /skywalking-satellite/bin/skywalking-satellite-latest-linux-amd64 /sw-satellite
ENTRYPOINT ["/sw-satellite"]

View File

@ -33,7 +33,6 @@ services:
SW_STORAGE_ES_CLUSTER_NODES: es:9200
SW_JDBC_URL: jdbc:mysql://mysql:3306/swtest
SW_STORAGE_INFLUXDB_URL: http://influxdb:8086
SW_SEARCHABLE_LOGS_TAG_KEYS: http.method,status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker
JAVA_OPTS: >-
-javaagent:/jacoco/jacocoagent.jar=classdumpdir=/jacoco/classes/oap,destfile=/jacoco/oap.exec,includes=org.apache.skywalking.*,excludes=org.apache.skywalking.oap.query.*:org.apache.skywalking.oap.server.core.query.*
-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005
@ -60,7 +59,6 @@ services:
SW_STORAGE_ES_CLUSTER_NODES: es:9200
SW_JDBC_URL: jdbc:mysql://mysql:3306/swtest
SW_STORAGE_INFLUXDB_URL: http://influxdb:8086
SW_SEARCHABLE_LOGS_TAG_KEYS: http.method,status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker
JAVA_OPTS: >-
-javaagent:/jacoco/jacocoagent.jar=classdumpdir=/jacoco/classes/oap,destfile=/jacoco/oap.exec,includes=org.apache.skywalking.*,excludes=org.apache.skywalking.oap.query.*:org.apache.skywalking.oap.server.core.query.*
-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005
@ -94,6 +92,8 @@ services:
- ../../../jacoco:/jacoco
environment:
SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11800
SW_GRPC_LOG_SERVER_HOST: oap
SW_GRPC_LOG_SERVER_PORT: 11800
JAVA_OPTS: >-
-javaagent:/jacoco/jacocoagent.jar=classdumpdir=/jacoco/classes/provider,destfile=/jacoco/provider.exec,includes=org.apache.skywalking.*,excludes=org.apache.skywalking.apm.dependencies.*
-javaagent:/skywalking/agent/skywalking-agent.jar=logging.output=CONSOLE

View File

@ -28,6 +28,10 @@ RUN mv apache-skywalking-apm-bin/* . || true
RUN sed -i 's/# plugin.kafka.bootstrap_servers/plugin.kafka.bootstrap_servers/g' agent/config/agent.config
RUN echo 'plugin.kafka.get_topic_timeout=${SW_GET_TOPIC_TIMEOUT:60}' >> agent/config/agent.config
# add log reporter configuration
RUN echo 'plugin.toolkit.log.grpc.reporter.server_host=${SW_GRPC_LOG_SERVER_HOST:oap}' >> agent/config/agent.config
RUN echo 'plugin.toolkit.log.grpc.reporter.server_port=${SW_GRPC_LOG_SERVER_PORT:11800}' >> agent/config/agent.config
# activate kafka-reporter plugin
RUN mv agent/optional-reporter-plugins/kafka-reporter-plugin-*.jar agent/plugins/ || true

View File

@ -31,24 +31,20 @@ services:
condition: service_healthy
broker-b:
condition: service_healthy
provider_kafka:
sw-satellite:
build:
context: ../../../
dockerfile: e2e-test/docker/kafka/Dockerfile.log.producer
args:
- SW_AGENT_JDK_VERSION=${SW_AGENT_JDK_VERSION}
dockerfile: e2e-test/docker/Dockerfile.satellite
networks:
- e2e
expose:
- 8080
- 11800
volumes:
- ../../../../jacoco:/jacoco
environment:
JAVA_OPTS: >-
-javaagent:/jacoco/jacocoagent.jar=classdumpdir=/jacoco/classes/provider,destfile=/jacoco/provider.exec,includes=org.apache.skywalking.*,excludes=org.apache.skywalking.apm.dependencies.*
-Dbootstrap.service=broker-a:9092,broker-b:9092
- ./satellite_config.yaml:/configs/satellite_config.yaml
command: ["start"]
healthcheck:
test: ["CMD", "sh", "-c", "nc -nz 127.0.0.1 8080"]
test: [ "CMD", "sh", "-c", "nc -z 127.0.0.1 11800" ]
interval: 5s
timeout: 60s
retries: 120
@ -59,5 +55,17 @@ services:
condition: service_healthy
broker-b:
condition: service_healthy
provider:
extends:
file: ../base-compose.yml
service: provider
environment:
SW_GRPC_LOG_SERVER_HOST: sw-satellite
depends_on:
oap:
condition: service_healthy
sw-satellite:
condition: service_healthy
networks:
e2e:

View File

@ -16,14 +16,6 @@
version: '2.1'
services:
ui:
extends:
file: ../base-compose.yml
service: ui
depends_on:
oap:
condition: service_healthy
zookeeper:
image: zookeeper:3.4
hostname: zookeeper

View File

@ -0,0 +1,61 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
logger:
log_pattern: "%time [%level][%field] - %msg"
time_pattern: "2006-01-02 15:04:05.000"
level: "info"
telemetry:
cluster: cluster1
service: service1
instance: instance1
sharing:
common_config:
pipe_name: sharing
clients:
- plugin_name: "kafka-client"
brokers: broker-a:9092,broker-b:9092
version: 2.4.1
servers:
- plugin_name: "grpc-server"
address: ":11800"
pipes:
- common_config:
pipe_name: pipe1
gatherer:
server_name: "grpc-server"
receiver:
plugin_name: "grpc-nativelog-receiver"
queue:
plugin_name: "mmap-queue"
segment_size: 524288
max_in_mem_segments: 6
queue_dir: "pipe1-log-grpc-receiver-queue"
processor:
filters:
sender:
fallbacker:
plugin_name: none-fallbacker
flush_time: 1000
max_buffer_size: 200
min_flush_events: 5
client_name: kafka-client
forwarders:
- plugin_name: nativelog-kafka-forwarder
topic: skywalking-logs

View File

@ -25,7 +25,7 @@ services:
environment:
- discovery.type=single-node
healthcheck:
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"]
test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200" ]
interval: 5s
timeout: 60s
retries: 120
@ -40,5 +40,12 @@ services:
es:
condition: service_healthy
provider:
extends:
file: ../base-compose.yml
service: provider
depends_on:
oap:
condition: service_healthy
networks:
e2e:

View File

@ -40,5 +40,12 @@ services:
es:
condition: service_healthy
provider:
extends:
file: ../base-compose.yml
service: provider
depends_on:
oap:
condition: service_healthy
networks:
e2e:

View File

@ -41,5 +41,13 @@ services:
h2db:
condition: service_healthy
provider:
extends:
file: ../base-compose.yml
service: provider
depends_on:
oap:
condition: service_healthy
networks:
e2e:

View File

@ -38,5 +38,13 @@ services:
influxdb:
condition: service_healthy
provider:
extends:
file: ../base-compose.yml
service: provider
depends_on:
oap:
condition: service_healthy
networks:
e2e:

View File

@ -43,5 +43,13 @@ services:
condition: service_healthy
entrypoint: ['sh', '-c', '/download-mysql.sh && /skywalking/docker-entrypoint.sh']
provider:
extends:
file: ../base-compose.yml
service: provider
depends_on:
oap:
condition: service_healthy
networks:
e2e:

View File

@ -90,7 +90,7 @@ public class KafkaE2E extends SkyWalkingTestAdapter {
protected HostAndPort serviceHostPort;
@SuppressWarnings("unused")
@ContainerHostAndPort(name = "ui", port = 8080)
@ContainerHostAndPort(name = "oap", port = 12800)
protected HostAndPort swWebappHostPort;
@BeforeAll

View File

@ -61,17 +61,13 @@ public class KafkaLogE2E extends SkyWalkingTestAdapter {
private HostAndPort swWebappHostPort;
@SuppressWarnings("unused")
@ContainerHostAndPort(name = "oap", port = 11800)
private HostAndPort oapHostPost;
@SuppressWarnings("unused")
@ContainerHostAndPort(name = "provider_kafka", port = 8080)
@ContainerHostAndPort(name = "provider", port = 9090)
private HostAndPort serviceHostPort;
@BeforeAll
public void setUp() throws Exception {
queryClient(swWebappHostPort);
trafficController(serviceHostPort, "/sendLog");
trafficController(serviceHostPort, "/logs/trigger");
}
@AfterAll
@ -94,27 +90,17 @@ public class KafkaLogE2E extends SkyWalkingTestAdapter {
LOGGER.info("verifying service instance: {}", service);
// instance
verifyServiceInstances(service);
// endpoint
verifyServiceEndpoints(service);
}
}
@RetryableTest
public void verifyLog() throws Exception {
LogsQuery logsQuery = new LogsQuery().serviceId("ZTJl.1")
.serviceInstanceId("ZTJl.1_ZTJlLWluc3RhbmNl")
.endpointId("ZTJl.1_L3RyYWZmaWM=")
.endpointName("/traffic")
.traceId("ac81b308-0d66-4c69-a7af-a023a536bd3e")
.segmentId(
"6024a2b1fcff48e4a641d69d388bac53.41.16088574455279608")
.spanId("0")
.tag("status_code", "200")
LogsQuery logsQuery = new LogsQuery().serviceId("WW91cl9BcHBsaWNhdGlvbk5hbWU=.1")
.tag("logger", "org.apache.skywalking.e2e.controller.LogController")
.start(startTime)
.end(Times.now());
if (graphql.supportQueryLogsByKeywords()) {
logsQuery.keywordsOfContent("main", "INFO")
.excludingKeywordsOfContent("ERROR");
logsQuery.keywordsOfContent("now");
}
final List<Log> logs = graphql.logs(logsQuery);
LOGGER.info("logs: {}", logs);

View File

@ -57,7 +57,7 @@ public class KafkaMeterE2E extends SkyWalkingTestAdapter {
protected DockerComposeContainer<?> justForSideEffects;
@SuppressWarnings("unused")
@ContainerHostAndPort(name = "ui", port = 8080)
@ContainerHostAndPort(name = "oap", port = 12800)
protected HostAndPort swWebappHostPort;
@SuppressWarnings("unused")

View File

@ -85,7 +85,7 @@ public class KafkaProfileE2E extends SkyWalkingTestAdapter {
protected HostAndPort serviceHostPort;
@SuppressWarnings("unused")
@ContainerHostAndPort(name = "ui", port = 8080)
@ContainerHostAndPort(name = "oap", port = 12800)
protected HostAndPort swWebappHostPort;
@SuppressWarnings("unused")

View File

@ -17,27 +17,13 @@
package org.apache.skywalking.e2e.log;
import io.grpc.ManagedChannel;
import io.grpc.internal.DnsNameResolverProvider;
import io.grpc.netty.NettyChannelBuilder;
import io.grpc.stub.StreamObserver;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.apm.network.common.v3.Commands;
import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair;
import org.apache.skywalking.apm.network.logging.v3.LogData;
import org.apache.skywalking.apm.network.logging.v3.LogDataBody;
import org.apache.skywalking.apm.network.logging.v3.LogReportServiceGrpc;
import org.apache.skywalking.apm.network.logging.v3.LogTags;
import org.apache.skywalking.apm.network.logging.v3.TextLog;
import org.apache.skywalking.apm.network.logging.v3.TraceContext;
import org.apache.skywalking.e2e.annotation.ContainerHostAndPort;
import org.apache.skywalking.e2e.annotation.DockerCompose;
import org.apache.skywalking.e2e.base.SkyWalkingE2E;
import org.apache.skywalking.e2e.base.SkyWalkingTestAdapter;
import org.apache.skywalking.e2e.base.TrafficController;
import org.apache.skywalking.e2e.common.HostAndPort;
import org.apache.skywalking.e2e.retryable.RetryableTest;
import org.apache.skywalking.e2e.service.Service;
@ -59,9 +45,6 @@ import static org.apache.skywalking.e2e.utils.Yamls.load;
@Slf4j
@SkyWalkingE2E
public class LogE2E extends SkyWalkingTestAdapter {
private final static int MAX_INBOUND_MESSAGE_SIZE = 1024 * 1024 * 50;
@SuppressWarnings("unused")
@DockerCompose({
"docker/log/docker-compose.${SW_STORAGE}.yml"
@ -73,25 +56,20 @@ public class LogE2E extends SkyWalkingTestAdapter {
private HostAndPort swWebappHostPort;
@SuppressWarnings("unused")
@ContainerHostAndPort(name = "oap", port = 11800)
private HostAndPort oapHostPost;
private LogReportServiceGrpc.LogReportServiceStub logReportServiceStub;
@ContainerHostAndPort(name = "provider", port = 9090)
protected HostAndPort providerHostPort;
@BeforeAll
public void setUp() {
public void setUp() throws Exception {
queryClient(swWebappHostPort);
final ManagedChannel channel = NettyChannelBuilder.forAddress(oapHostPost.host(), oapHostPost.port())
.nameResolverFactory(new DnsNameResolverProvider())
.maxInboundMessageSize(MAX_INBOUND_MESSAGE_SIZE)
.usePlaintext().build();
logReportServiceStub = LogReportServiceGrpc.newStub(channel);
generateTraffic();
trafficController(providerHostPort, "/logs/trigger");
}
@AfterAll
public void tearDown() {
trafficController.stop();
if (trafficController != null) {
trafficController.stop();
}
}
@RetryableTest
@ -107,27 +85,17 @@ public class LogE2E extends SkyWalkingTestAdapter {
LOGGER.info("verifying service instance: {}", service);
// instance
verifyServiceInstances(service);
// endpoint
verifyServiceEndpoints(service);
}
}
@RetryableTest
public void verifyLog() throws Exception {
LogsQuery logsQuery = new LogsQuery().serviceId("ZTJl.1")
.serviceInstanceId("ZTJl.1_ZTJlLWluc3RhbmNl")
.endpointId("ZTJl.1_L3RyYWZmaWM=")
.endpointName("/traffic")
.traceId("ac81b308-0d66-4c69-a7af-a023a536bd3e")
.segmentId(
"6024a2b1fcff48e4a641d69d388bac53.41.16088574455279608")
.spanId("0")
.tag("status_code", "200")
LogsQuery logsQuery = new LogsQuery().serviceId("WW91cl9BcHBsaWNhdGlvbk5hbWU=.1")
.tag("logger", "org.apache.skywalking.e2e.controller.LogController")
.start(startTime)
.end(Times.now());
if (graphql.supportQueryLogsByKeywords()) {
logsQuery.keywordsOfContent("main", "INFO")
.excludingKeywordsOfContent("ERROR");
logsQuery.keywordsOfContent("now");
}
final List<Log> logs = graphql.logs(logsQuery);
LOGGER.info("logs: {}", logs);
@ -149,68 +117,4 @@ public class LogE2E extends SkyWalkingTestAdapter {
load("expected/log/endpoints.yml").as(EndpointsMatcher.class).verify(endpoints);
}
private void generateTraffic() {
trafficController = TrafficController.builder()
.sender(this::sendLog)
.build();
trafficController.start();
}
private boolean sendLog() {
try {
LogData logData = LogData.newBuilder()
.setService("e2e")
.setServiceInstance("e2e-instance")
.setEndpoint("/traffic")
.setBody(
LogDataBody.newBuilder()
.setText(TextLog.newBuilder()
.setText(
"[main] INFO log message")
.build())
.build())
.setTags(LogTags.newBuilder()
.addData(KeyStringValuePair.newBuilder()
.setKey("status_code")
.setValue("200")
.build())
.build())
.setTraceContext(TraceContext.newBuilder()
.setTraceId("ac81b308-0d66-4c69-a7af-a023a536bd3e")
.setTraceSegmentId(
"6024a2b1fcff48e4a641d69d388bac53.41.16088574455279608")
.setSpanId(0)
.build())
.build();
CountDownLatch downLatch = new CountDownLatch(1);
StreamObserver<LogData> streamObserver = logReportServiceStub.collect(new StreamObserver<Commands>() {
@Override
public void onNext(final Commands commands) {
}
@Override
public void onError(final Throwable throwable) {
LOGGER.error(throwable.getMessage(), throwable);
downLatch.countDown();
}
@Override
public void onCompleted() {
downLatch.countDown();
}
});
streamObserver.onNext(logData);
streamObserver.onCompleted();
downLatch.await();
return true;
} catch (Exception e) {
LOGGER.warn(e.getMessage(), e);
return false;
}
}
}

View File

@ -15,4 +15,4 @@
endpoints:
- key: not null
label: /traffic
label: /logs/traffic

View File

@ -15,4 +15,4 @@
instances:
- key: not null
label: e2e-instance
label: not null

View File

@ -14,17 +14,15 @@
# limitations under the License.
logs:
- serviceName: e2e
- serviceName: not null
serviceId: not null
serviceInstanceName: "e2e-instance"
serviceInstanceName: not null
serviceInstanceId: not null
endpointName: /traffic
endpointId: not null
traceId: "ac81b308-0d66-4c69-a7af-a023a536bd3e"
traceId: not null
timestamp: not null
isError: false
contentType: TEXT
content: not null
tags:
- key: status_code
value: 200
- key: level
value: INFO

View File

@ -15,4 +15,4 @@
services:
- key: not null
label: "e2e"
label: not null

View File

@ -40,7 +40,6 @@
<module>e2e-protocol</module>
<module>e2e-service-provider</module>
<module>e2e-service-consumer</module>
<module>e2e-log-producer</module>
<module>e2e-test</module>
</modules>
@ -68,6 +67,21 @@
<testcontainers.version>1.15.0</testcontainers.version>
</properties>
<repositories>
<repository>
<id>apache.snapshots</id>
<name>Apache Development Snapshot Repository</name>
<url>https://repository.apache.org/content/groups/snapshots/</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>
<dependencyManagement>
<dependencies>
<dependency>