Enhancement log-recerive e2e test (#6248)
This commit is contained in:
parent
fd02bcd6f6
commit
01d1eff6c7
|
|
@ -23,9 +23,8 @@ on:
|
|||
- '!**.md'
|
||||
schedule:
|
||||
- cron: '0 18 * * *'
|
||||
|
||||
env:
|
||||
SKIP_TEST: true
|
||||
SW_AGENT_JDK_VERSION: 8
|
||||
|
||||
jobs:
|
||||
Log:
|
||||
|
|
@ -47,12 +46,10 @@ jobs:
|
|||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
- name: Compile and Build
|
||||
run: make docker && ES_VERSION=es7 TAG=latest-es7 make docker
|
||||
- name: Copy dist package
|
||||
run: cp -R dist test/e2e/
|
||||
- name: Log Receiver ${{ matrix.storage }}
|
||||
run: ./mvnw --batch-mode -f test/e2e/pom.xml -am -DfailIfNoTests=false verify -Dit.test=org.apache.skywalking.e2e.log.LogE2E
|
||||
- name: Run E2E Test
|
||||
uses: ./.github/actions/e2e-test
|
||||
with:
|
||||
test_class: org.apache.skywalking.e2e.log.LogE2E
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: failure()
|
||||
with:
|
||||
|
|
|
|||
|
|
@ -18,11 +18,14 @@
|
|||
|
||||
package org.apache.skywalking.apm.toolkit.logging.common.log;
|
||||
|
||||
import io.grpc.ManagedChannel;
|
||||
import io.grpc.ManagedChannelBuilder;
|
||||
import io.grpc.StatusRuntimeException;
|
||||
import io.grpc.stub.StreamObserver;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import org.apache.skywalking.apm.agent.core.boot.OverrideImplementor;
|
||||
import org.apache.skywalking.apm.agent.core.conf.Config;
|
||||
import org.apache.skywalking.apm.agent.core.logging.api.ILog;
|
||||
|
|
@ -36,11 +39,6 @@ import org.apache.skywalking.apm.network.common.v3.Commands;
|
|||
import org.apache.skywalking.apm.network.logging.v3.LogData;
|
||||
import org.apache.skywalking.apm.network.logging.v3.LogReportServiceGrpc;
|
||||
|
||||
import io.grpc.ManagedChannel;
|
||||
import io.grpc.ManagedChannelBuilder;
|
||||
import io.grpc.StatusRuntimeException;
|
||||
import io.grpc.stub.StreamObserver;
|
||||
|
||||
/**
|
||||
* Report log to server by grpc
|
||||
*/
|
||||
|
|
@ -59,14 +57,22 @@ public class GRPCLogReportServiceClient extends LogReportServiceClient {
|
|||
|
||||
@Override
|
||||
public void boot() throws Throwable {
|
||||
carrier = new DataCarrier<>("gRPC-log", "gRPC-log", Config.Buffer.CHANNEL_SIZE, Config.Buffer.BUFFER_SIZE,
|
||||
BufferStrategy.IF_POSSIBLE);
|
||||
carrier = new DataCarrier<>("gRPC-log", "gRPC-log",
|
||||
Config.Buffer.CHANNEL_SIZE,
|
||||
Config.Buffer.BUFFER_SIZE,
|
||||
BufferStrategy.IF_POSSIBLE
|
||||
);
|
||||
carrier.consume(this, 1);
|
||||
channel = ManagedChannelBuilder
|
||||
.forAddress(ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.SERVER_HOST,
|
||||
ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.SERVER_PORT).usePlaintext().build();
|
||||
.forAddress(
|
||||
ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.SERVER_HOST,
|
||||
ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.SERVER_PORT
|
||||
)
|
||||
.usePlaintext()
|
||||
.build();
|
||||
asyncStub = LogReportServiceGrpc.newStub(channel)
|
||||
.withMaxOutboundMessageSize(ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.MAX_MESSAGE_SIZE);
|
||||
.withMaxOutboundMessageSize(
|
||||
ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.MAX_MESSAGE_SIZE);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -99,7 +105,7 @@ public class GRPCLogReportServiceClient extends LogReportServiceClient {
|
|||
final GRPCStreamServiceStatus waitStatus = new GRPCStreamServiceStatus(false);
|
||||
try {
|
||||
reportStreamObserver = asyncStub.withDeadlineAfter(
|
||||
ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.UPSTREAM_TIMEOUT, TimeUnit.SECONDS
|
||||
ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.UPSTREAM_TIMEOUT, TimeUnit.SECONDS
|
||||
).collect(new StreamObserver<Commands>() {
|
||||
@Override
|
||||
public void onNext(Commands commands) {
|
||||
|
|
@ -113,7 +119,8 @@ public class GRPCLogReportServiceClient extends LogReportServiceClient {
|
|||
}
|
||||
|
||||
LOGGER.error(t, "Try to send {} log data to collector, with unexpected exception.",
|
||||
dataList.size());
|
||||
dataList.size()
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -23,33 +23,33 @@ import org.apache.skywalking.apm.agent.core.boot.PluginConfig;
|
|||
public class ToolkitConfig {
|
||||
|
||||
public static class Plugin {
|
||||
@PluginConfig(root = ToolkitConfig.class)
|
||||
public static class Toolkit {
|
||||
public static class Log {
|
||||
public static class GRPC {
|
||||
public static class Reporter {
|
||||
/**
|
||||
* The host of gRPC log server.
|
||||
*/
|
||||
public static String SERVER_HOST = "127.0.0.1";
|
||||
public static class Log {
|
||||
public static class GRPC {
|
||||
@PluginConfig(root = ToolkitConfig.class)
|
||||
public static class Reporter {
|
||||
/**
|
||||
* The host of gRPC log server.
|
||||
*/
|
||||
public static String SERVER_HOST = "127.0.0.1";
|
||||
|
||||
/**
|
||||
* The port of gRPC log server.
|
||||
*/
|
||||
public static int SERVER_PORT = 11800;
|
||||
/**
|
||||
* The port of gRPC log server.
|
||||
*/
|
||||
public static int SERVER_PORT = 11800;
|
||||
|
||||
/**
|
||||
* The max size of message to send to server.Default is 10 MB.
|
||||
*/
|
||||
public static int MAX_MESSAGE_SIZE = 10 * 1024 * 1024;
|
||||
/**
|
||||
* The max size of message to send to server.Default is 10 MB.
|
||||
*/
|
||||
public static int MAX_MESSAGE_SIZE = 10 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* How long grpc client will timeout in sending data to upstream. The unit is second.
|
||||
*/
|
||||
public static int UPSTREAM_TIMEOUT = 30;
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* How long grpc client will timeout in sending data to upstream. The unit is second.
|
||||
*/
|
||||
public static int UPSTREAM_TIMEOUT = 30;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -30,7 +30,7 @@ core|default|role|Option values, `Mixed/Receiver/Aggregator`. **Receiver** mode
|
|||
| - | - | instanceNameMaxLength| Max length limitation of service instance name. The max length of service + instance names should be less than 200.|SW_INSTANCE_NAME_MAX_LENGTH|70|
|
||||
| - | - | endpointNameMaxLength| Max length limitation of endpoint name. The max length of service + endpoint names should be less than 240.|SW_ENDPOINT_NAME_MAX_LENGTH|150|
|
||||
| - | - | searchableTracesTags | Define the set of span tag keys, which should be searchable through the GraphQL. Multiple values should be separated through the comma. | SW_SEARCHABLE_TAG_KEYS | http.method,status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker|
|
||||
| - | - | searchableLogsTags | Define the set of log tag keys, which should be searchable through the GraphQL. Multiple values should be separated through the comma. | SW_SEARCHABLE_LOGS_TAG_KEYS | - |
|
||||
| - | - | searchableLogsTags | Define the set of log tag keys, which should be searchable through the GraphQL. Multiple values should be separated through the comma. | SW_SEARCHABLE_LOGS_TAG_KEYS | level,logger,thread |
|
||||
| - | - | gRPCThreadPoolSize|Pool size of gRPC server| SW_CORE_GRPC_THREAD_POOL_SIZE | CPU core * 4|
|
||||
| - | - | gRPCThreadPoolQueueSize| The queue size of gRPC server| SW_CORE_GRPC_POOL_QUEUE_SIZE | 10000|
|
||||
| - | - | maxConcurrentCallsPerConnection | The maximum number of concurrent calls permitted for each incoming connection. Defaults to no limit. | SW_CORE_GRPC_MAX_CONCURRENT_CALL | - |
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ core:
|
|||
# Define the set of span tag keys, which should be searchable through the GraphQL.
|
||||
searchableTracesTags: ${SW_SEARCHABLE_TAG_KEYS:http.method,status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker}
|
||||
# Define the set of log tag keys, which should be searchable through the GraphQL.
|
||||
searchableLogsTags: ${SW_SEARCHABLE_LOGS_TAG_KEYS:""}
|
||||
searchableLogsTags: ${SW_SEARCHABLE_LOGS_TAG_KEYS:level,logger,thread}
|
||||
storage:
|
||||
selector: ${SW_STORAGE:h2}
|
||||
elasticsearch:
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ package org.apache.skywalking.oap.server.core.query;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
import org.apache.skywalking.apm.util.StringUtil;
|
||||
import org.apache.skywalking.oap.server.core.analysis.IDManager;
|
||||
import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag;
|
||||
|
|
@ -33,6 +34,8 @@ import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO;
|
|||
import org.apache.skywalking.oap.server.library.module.ModuleManager;
|
||||
import org.apache.skywalking.oap.server.library.module.Service;
|
||||
|
||||
import static java.util.Objects.nonNull;
|
||||
|
||||
public class LogQueryService implements Service {
|
||||
|
||||
private final ModuleManager moduleManager;
|
||||
|
|
@ -64,10 +67,21 @@ public class LogQueryService implements Service {
|
|||
final long startTB,
|
||||
final long endTB,
|
||||
final List<Tag> tags,
|
||||
final List<String> keywordsOfContent,
|
||||
final List<String> excludingKeywordsOfContent) throws IOException {
|
||||
List<String> keywordsOfContent,
|
||||
List<String> excludingKeywordsOfContent) throws IOException {
|
||||
PaginationUtils.Page page = PaginationUtils.INSTANCE.exchange(paging);
|
||||
|
||||
if (nonNull(keywordsOfContent)) {
|
||||
keywordsOfContent = keywordsOfContent.stream()
|
||||
.filter(StringUtil::isNotEmpty)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
if (nonNull(excludingKeywordsOfContent)) {
|
||||
excludingKeywordsOfContent = excludingKeywordsOfContent.stream()
|
||||
.filter(StringUtil::isNotEmpty)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
Logs logs = getLogQueryDAO().queryLogs(serviceId,
|
||||
serviceInstanceId,
|
||||
endpointId,
|
||||
|
|
|
|||
|
|
@ -143,7 +143,7 @@ public class LogQuery implements ILogQueryDAO {
|
|||
nested.close();
|
||||
}
|
||||
|
||||
SelectQueryImpl countQuery = select().count(ENDPOINT_ID).from(client.getDatabase(), LogRecord.INDEX_NAME);
|
||||
SelectQueryImpl countQuery = select().count(SERVICE_ID).from(client.getDatabase(), LogRecord.INDEX_NAME);
|
||||
for (ConjunctionClause clause : recallQuery.getClauses()) {
|
||||
countQuery.where(clause);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -398,12 +398,8 @@ public class SimpleQueryClient {
|
|||
.collect(Collectors.joining())
|
||||
.replace("{state}", query.state())
|
||||
.replace("{serviceId}", query.serviceId())
|
||||
.replace("{serviceInstanceId}", query.serviceInstanceId())
|
||||
.replace("{endpointId}", query.endpointId())
|
||||
.replace("{endpointName}", query.endpointName())
|
||||
.replace("{traceId}", query.traceId())
|
||||
.replace("{segmentId}", query.segmentId())
|
||||
.replace("{spanId}", query.spanId())
|
||||
.replace("{start}", query.start())
|
||||
.replace("{end}", query.end())
|
||||
.replace("{step}", query.step())
|
||||
|
|
|
|||
|
|
@ -23,12 +23,8 @@ public class LogsQuery extends AbstractQuery<LogsQuery> {
|
|||
|
||||
private String state = "ALL";
|
||||
private String serviceId;
|
||||
private String serviceInstanceId;
|
||||
private String endpointId;
|
||||
private String endpointName;
|
||||
private String traceId;
|
||||
private String segmentId;
|
||||
private String spanId;
|
||||
private String endpointId = "";
|
||||
private String endpointName = "";
|
||||
private String tagKey;
|
||||
private String tagValue;
|
||||
private String pageNum = "1";
|
||||
|
|
@ -55,15 +51,6 @@ public class LogsQuery extends AbstractQuery<LogsQuery> {
|
|||
return this;
|
||||
}
|
||||
|
||||
public String serviceInstanceId() {
|
||||
return serviceInstanceId;
|
||||
}
|
||||
|
||||
public LogsQuery serviceInstanceId(String serviceInstanceId) {
|
||||
this.serviceInstanceId = serviceInstanceId;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String endpointId() {
|
||||
return endpointId;
|
||||
}
|
||||
|
|
@ -82,33 +69,6 @@ public class LogsQuery extends AbstractQuery<LogsQuery> {
|
|||
return this;
|
||||
}
|
||||
|
||||
public String traceId() {
|
||||
return traceId;
|
||||
}
|
||||
|
||||
public LogsQuery traceId(String traceId) {
|
||||
this.traceId = traceId;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String segmentId() {
|
||||
return segmentId;
|
||||
}
|
||||
|
||||
public LogsQuery segmentId(String segmentId) {
|
||||
this.segmentId = segmentId;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String spanId() {
|
||||
return spanId;
|
||||
}
|
||||
|
||||
public LogsQuery spanId(String spanId) {
|
||||
this.spanId = spanId;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String tagKey() {
|
||||
return tagKey;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,14 +38,8 @@
|
|||
"condition": {
|
||||
"state": "{state}",
|
||||
"serviceId": "{serviceId}",
|
||||
"serviceInstanceId": "{serviceInstanceId}",
|
||||
"endpointId": "{endpointId}",
|
||||
"endpointName": "{endpointName}",
|
||||
"relatedTrace": {
|
||||
"traceId": "{traceId}",
|
||||
"segmentId": "{segmentId}",
|
||||
"spanId": {spanId}
|
||||
},
|
||||
"queryDuration": {
|
||||
"start": "{start}",
|
||||
"end": "{end}",
|
||||
|
|
|
|||
|
|
@ -1,66 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
~ contributor license agreements. See the NOTICE file distributed with
|
||||
~ this work for additional information regarding copyright ownership.
|
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
~ (the "License"); you may not use this file except in compliance with
|
||||
~ the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>apache-skywalking-e2e</artifactId>
|
||||
<groupId>org.apache.skywalking</groupId>
|
||||
<version>1.0.0</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>e2e-log-producer</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.skywalking</groupId>
|
||||
<artifactId>e2e-protocol</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka-clients</artifactId>
|
||||
<version>${kafka-clients.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<version>${spring.boot.version}</version>
|
||||
<configuration>
|
||||
<executable>true</executable>
|
||||
<addResources>true</addResources>
|
||||
<excludeDevtools>true</excludeDevtools>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>repackage</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.skywalking.e2e;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
@SpringBootApplication
|
||||
public class LogProducerApplication {
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(LogProducerApplication.class, args);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,134 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.skywalking.e2e.controller;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.Objects;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.stream.Collectors;
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.annotation.PreDestroy;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.kafka.clients.admin.AdminClient;
|
||||
import org.apache.kafka.clients.admin.DescribeTopicsResult;
|
||||
import org.apache.kafka.clients.producer.KafkaProducer;
|
||||
import org.apache.kafka.clients.producer.ProducerConfig;
|
||||
import org.apache.kafka.clients.producer.ProducerRecord;
|
||||
import org.apache.kafka.common.serialization.BytesSerializer;
|
||||
import org.apache.kafka.common.serialization.StringSerializer;
|
||||
import org.apache.kafka.common.utils.Bytes;
|
||||
import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair;
|
||||
import org.apache.skywalking.apm.network.logging.v3.LogData;
|
||||
import org.apache.skywalking.apm.network.logging.v3.LogDataBody;
|
||||
import org.apache.skywalking.apm.network.logging.v3.LogTags;
|
||||
import org.apache.skywalking.apm.network.logging.v3.TextLog;
|
||||
import org.apache.skywalking.apm.network.logging.v3.TraceContext;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.ResponseStatus;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import static java.util.Objects.nonNull;
|
||||
|
||||
@Slf4j
|
||||
@RestController
|
||||
public class LogController {
|
||||
|
||||
private static final String TOPIC = "skywalking-logs";
|
||||
|
||||
private KafkaProducer<String, Bytes> producer;
|
||||
|
||||
@Value("#{systemProperties['bootstrap.service'] ?: 'localhost:9092'}")
|
||||
private String bootstrapService;
|
||||
|
||||
@PostConstruct
|
||||
public void up() {
|
||||
Properties properties = new Properties();
|
||||
properties.setProperty(
|
||||
ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapService);
|
||||
producer = new KafkaProducer<>(properties, new StringSerializer(), new BytesSerializer());
|
||||
AdminClient adminClient = AdminClient.create(properties);
|
||||
DescribeTopicsResult topicsResult = adminClient.describeTopics(Collections.singletonList(TOPIC));
|
||||
Set<String> topics = topicsResult.values().entrySet().stream()
|
||||
.map(entry -> {
|
||||
try {
|
||||
entry.getValue().get(10, TimeUnit.SECONDS);
|
||||
return null;
|
||||
} catch (InterruptedException | ExecutionException | TimeoutException e) {
|
||||
LOGGER.error("Get KAFKA topic:" + entry.getKey() + " error", e);
|
||||
}
|
||||
return entry.getKey();
|
||||
}).filter(Objects::nonNull).collect(Collectors.toSet());
|
||||
if (!topics.isEmpty()) {
|
||||
throw new RuntimeException("These topics" + topics + " don't exist.");
|
||||
}
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void down() {
|
||||
if (nonNull(producer)) {
|
||||
producer.close();
|
||||
}
|
||||
}
|
||||
|
||||
@PostMapping("/sendLog")
|
||||
@SuppressWarnings("EmptyMethod")
|
||||
@ResponseStatus(code = HttpStatus.OK)
|
||||
public void sendLog() {
|
||||
try {
|
||||
LogData logData = LogData.newBuilder()
|
||||
.setService("e2e")
|
||||
.setServiceInstance("e2e-instance")
|
||||
.setEndpoint("/traffic")
|
||||
.setBody(
|
||||
LogDataBody.newBuilder()
|
||||
.setText(
|
||||
TextLog.newBuilder().setText("[main] INFO log message").build())
|
||||
.build())
|
||||
.setTags(LogTags.newBuilder()
|
||||
.addData(KeyStringValuePair.newBuilder()
|
||||
.setKey("status_code")
|
||||
.setValue("200")
|
||||
.build())
|
||||
.build())
|
||||
.setTraceContext(TraceContext.newBuilder()
|
||||
.setTraceId("ac81b308-0d66-4c69-a7af-a023a536bd3e")
|
||||
.setTraceSegmentId(
|
||||
"6024a2b1fcff48e4a641d69d388bac53.41.16088574455279608")
|
||||
.setSpanId(0)
|
||||
.build())
|
||||
.build();
|
||||
|
||||
producer.send(
|
||||
new ProducerRecord<>(TOPIC, logData.getService(), Bytes.wrap(logData.toByteArray())),
|
||||
(m, e) -> {
|
||||
if (nonNull(e)) {
|
||||
LOGGER.error("Failed to report logs.", e);
|
||||
}
|
||||
}
|
||||
);
|
||||
} catch (Exception e) {
|
||||
LOGGER.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -33,6 +33,10 @@
|
|||
|
||||
<artifactId>e2e-service-provider</artifactId>
|
||||
|
||||
<properties>
|
||||
<sw.snapshot.version>8.4.0-SNAPSHOT</sw.snapshot.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
|
|
@ -53,6 +57,16 @@
|
|||
<artifactId>apm-toolkit-micrometer-registry</artifactId>
|
||||
<version>8.2.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.skywalking</groupId>
|
||||
<artifactId>apm-toolkit-logback-1.x</artifactId>
|
||||
<version>${sw.snapshot.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.skywalking</groupId>
|
||||
<artifactId>apm-toolkit-trace</artifactId>
|
||||
<version>${sw.snapshot.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
|
|
|||
|
|
@ -1,57 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.apache.skywalking.apm.toolkit.trace;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Try to access the sky-walking tracer context. The context is not existed, always. only the middleware, component, or
|
||||
* rpc-framework are supported in the current invoke stack, in the same thread, the context will be available.
|
||||
* <p>
|
||||
*/
|
||||
public class TraceContext {
|
||||
|
||||
/**
|
||||
* Try to get the traceId of current trace context.
|
||||
*
|
||||
* @return traceId, if it exists, or empty {@link String}.
|
||||
*/
|
||||
public static String traceId() {
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to get the custom value from trace context.
|
||||
*
|
||||
* @return custom data value.
|
||||
*/
|
||||
public static Optional<String> getCorrelation(String key) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
/**
|
||||
* Put the custom key/value into trace context.
|
||||
*
|
||||
* @return previous value if it exists.
|
||||
*/
|
||||
public static Optional<String> putCorrelation(String key, String value) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -17,20 +17,18 @@
|
|||
|
||||
package org.apache.skywalking.e2e.controller;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.ResponseStatus;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.skywalking.apm.toolkit.trace.TraceContext;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
public class HealthController {
|
||||
@GetMapping("/health")
|
||||
@SuppressWarnings("EmptyMethod")
|
||||
@ResponseStatus(code = HttpStatus.OK)
|
||||
public void hello() {
|
||||
public class LogController {
|
||||
|
||||
@RequestMapping(value = "/logs/trigger")
|
||||
public String trigger() {
|
||||
LOGGER.info("now: {}", System.currentTimeMillis());
|
||||
return TraceContext.traceId();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
~ contributor license agreements. See the NOTICE file distributed with
|
||||
~ this work for additional information regarding copyright ownership.
|
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
~ (the "License"); you may not use this file except in compliance with
|
||||
~ the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<configuration scan="true" scanPeriod=" 5 seconds">
|
||||
|
||||
<appender name="stdout" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder class="ch.qos.logback.core.encoder.LayoutWrappingEncoder">
|
||||
<layout class="org.apache.skywalking.apm.toolkit.log.logback.v1.x.mdc.TraceIdMDCPatternLogbackLayout">
|
||||
<Pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%X{tid}] [%thread] %-5level %logger{36} -%msg%n</Pattern>
|
||||
</layout>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<appender name="grpc-log" class="org.apache.skywalking.apm.toolkit.log.logback.v1.x.log.GRPCLogClientAppender"/>
|
||||
|
||||
<root level="INFO">
|
||||
<appender-ref ref="grpc-log"/>
|
||||
<appender-ref ref="stdout"/>
|
||||
</root>
|
||||
</configuration>
|
||||
|
|
@ -25,6 +25,10 @@ ADD ${DIST_PACKAGE} .
|
|||
RUN tar -zxf apache-skywalking-apm*.gz --strip=1 || true
|
||||
RUN mv apache-skywalking-apm-bin/* . || true
|
||||
|
||||
# add log reporter configuration
|
||||
RUN echo 'plugin.toolkit.log.grpc.reporter.server_host=${SW_GRPC_LOG_SERVER_HOST:oap}' >> agent/config/agent.config
|
||||
RUN echo 'plugin.toolkit.log.grpc.reporter.server_port=${SW_GRPC_LOG_SERVER_PORT:11800}' >> agent/config/agent.config
|
||||
|
||||
VOLUME /services
|
||||
|
||||
ADD e2e-service-provider/target/e2e-service-provider-1.0.0.jar /services/
|
||||
|
|
|
|||
|
|
@ -13,15 +13,24 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
ARG SW_AGENT_JDK_VERSION=8
|
||||
ARG AGENT_JDK_BASE=adoptopenjdk/openjdk${SW_AGENT_JDK_VERSION}:alpine
|
||||
FROM golang:1.14 AS builder
|
||||
|
||||
FROM ${AGENT_JDK_BASE}
|
||||
ARG COMMIT_HASH=827f3a45e44cb2a64f874ff17d9eb454a680d08f
|
||||
ARG SATELLITE_CODE=${COMMIT_HASH}.tar.gz
|
||||
ARG SATELLITE_CODE_URL=https://github.com/apache/skywalking-satellite/archive/${SATELLITE_CODE}
|
||||
|
||||
VOLUME /services
|
||||
ENV CGO_ENABLED=0
|
||||
|
||||
ADD e2e-log-producer/target/e2e-log-producer-1.0.0.jar /services/
|
||||
WORKDIR /skywalking-satellite
|
||||
|
||||
ENV JAVA_OPTS=""
|
||||
ADD ${SATELLITE_CODE_URL} .
|
||||
RUN tar -xf ${SATELLITE_CODE} --strip 1
|
||||
RUN rm ${SATELLITE_CODE}
|
||||
|
||||
CMD ["sh", "-c", "java $JAVA_OPTS -jar /services/e2e-log-producer-1.0.0.jar"]
|
||||
RUN make linux
|
||||
|
||||
FROM alpine:3.10
|
||||
|
||||
COPY --from=builder /skywalking-satellite/bin/skywalking-satellite-latest-linux-amd64 /sw-satellite
|
||||
|
||||
ENTRYPOINT ["/sw-satellite"]
|
||||
|
|
@ -33,7 +33,6 @@ services:
|
|||
SW_STORAGE_ES_CLUSTER_NODES: es:9200
|
||||
SW_JDBC_URL: jdbc:mysql://mysql:3306/swtest
|
||||
SW_STORAGE_INFLUXDB_URL: http://influxdb:8086
|
||||
SW_SEARCHABLE_LOGS_TAG_KEYS: http.method,status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker
|
||||
JAVA_OPTS: >-
|
||||
-javaagent:/jacoco/jacocoagent.jar=classdumpdir=/jacoco/classes/oap,destfile=/jacoco/oap.exec,includes=org.apache.skywalking.*,excludes=org.apache.skywalking.oap.query.*:org.apache.skywalking.oap.server.core.query.*
|
||||
-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005
|
||||
|
|
@ -60,7 +59,6 @@ services:
|
|||
SW_STORAGE_ES_CLUSTER_NODES: es:9200
|
||||
SW_JDBC_URL: jdbc:mysql://mysql:3306/swtest
|
||||
SW_STORAGE_INFLUXDB_URL: http://influxdb:8086
|
||||
SW_SEARCHABLE_LOGS_TAG_KEYS: http.method,status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker
|
||||
JAVA_OPTS: >-
|
||||
-javaagent:/jacoco/jacocoagent.jar=classdumpdir=/jacoco/classes/oap,destfile=/jacoco/oap.exec,includes=org.apache.skywalking.*,excludes=org.apache.skywalking.oap.query.*:org.apache.skywalking.oap.server.core.query.*
|
||||
-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005
|
||||
|
|
@ -94,6 +92,8 @@ services:
|
|||
- ../../../jacoco:/jacoco
|
||||
environment:
|
||||
SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11800
|
||||
SW_GRPC_LOG_SERVER_HOST: oap
|
||||
SW_GRPC_LOG_SERVER_PORT: 11800
|
||||
JAVA_OPTS: >-
|
||||
-javaagent:/jacoco/jacocoagent.jar=classdumpdir=/jacoco/classes/provider,destfile=/jacoco/provider.exec,includes=org.apache.skywalking.*,excludes=org.apache.skywalking.apm.dependencies.*
|
||||
-javaagent:/skywalking/agent/skywalking-agent.jar=logging.output=CONSOLE
|
||||
|
|
|
|||
|
|
@ -28,6 +28,10 @@ RUN mv apache-skywalking-apm-bin/* . || true
|
|||
RUN sed -i 's/# plugin.kafka.bootstrap_servers/plugin.kafka.bootstrap_servers/g' agent/config/agent.config
|
||||
RUN echo 'plugin.kafka.get_topic_timeout=${SW_GET_TOPIC_TIMEOUT:60}' >> agent/config/agent.config
|
||||
|
||||
# add log reporter configuration
|
||||
RUN echo 'plugin.toolkit.log.grpc.reporter.server_host=${SW_GRPC_LOG_SERVER_HOST:oap}' >> agent/config/agent.config
|
||||
RUN echo 'plugin.toolkit.log.grpc.reporter.server_port=${SW_GRPC_LOG_SERVER_PORT:11800}' >> agent/config/agent.config
|
||||
|
||||
# activate kafka-reporter plugin
|
||||
RUN mv agent/optional-reporter-plugins/kafka-reporter-plugin-*.jar agent/plugins/ || true
|
||||
|
||||
|
|
|
|||
|
|
@ -31,24 +31,20 @@ services:
|
|||
condition: service_healthy
|
||||
broker-b:
|
||||
condition: service_healthy
|
||||
provider_kafka:
|
||||
|
||||
sw-satellite:
|
||||
build:
|
||||
context: ../../../
|
||||
dockerfile: e2e-test/docker/kafka/Dockerfile.log.producer
|
||||
args:
|
||||
- SW_AGENT_JDK_VERSION=${SW_AGENT_JDK_VERSION}
|
||||
dockerfile: e2e-test/docker/Dockerfile.satellite
|
||||
networks:
|
||||
- e2e
|
||||
expose:
|
||||
- 8080
|
||||
- 11800
|
||||
volumes:
|
||||
- ../../../../jacoco:/jacoco
|
||||
environment:
|
||||
JAVA_OPTS: >-
|
||||
-javaagent:/jacoco/jacocoagent.jar=classdumpdir=/jacoco/classes/provider,destfile=/jacoco/provider.exec,includes=org.apache.skywalking.*,excludes=org.apache.skywalking.apm.dependencies.*
|
||||
-Dbootstrap.service=broker-a:9092,broker-b:9092
|
||||
- ./satellite_config.yaml:/configs/satellite_config.yaml
|
||||
command: ["start"]
|
||||
healthcheck:
|
||||
test: ["CMD", "sh", "-c", "nc -nz 127.0.0.1 8080"]
|
||||
test: [ "CMD", "sh", "-c", "nc -z 127.0.0.1 11800" ]
|
||||
interval: 5s
|
||||
timeout: 60s
|
||||
retries: 120
|
||||
|
|
@ -59,5 +55,17 @@ services:
|
|||
condition: service_healthy
|
||||
broker-b:
|
||||
condition: service_healthy
|
||||
|
||||
provider:
|
||||
extends:
|
||||
file: ../base-compose.yml
|
||||
service: provider
|
||||
environment:
|
||||
SW_GRPC_LOG_SERVER_HOST: sw-satellite
|
||||
depends_on:
|
||||
oap:
|
||||
condition: service_healthy
|
||||
sw-satellite:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
e2e:
|
||||
|
|
|
|||
|
|
@ -16,14 +16,6 @@
|
|||
version: '2.1'
|
||||
|
||||
services:
|
||||
ui:
|
||||
extends:
|
||||
file: ../base-compose.yml
|
||||
service: ui
|
||||
depends_on:
|
||||
oap:
|
||||
condition: service_healthy
|
||||
|
||||
zookeeper:
|
||||
image: zookeeper:3.4
|
||||
hostname: zookeeper
|
||||
|
|
|
|||
|
|
@ -0,0 +1,61 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
logger:
|
||||
log_pattern: "%time [%level][%field] - %msg"
|
||||
time_pattern: "2006-01-02 15:04:05.000"
|
||||
level: "info"
|
||||
|
||||
telemetry:
|
||||
cluster: cluster1
|
||||
service: service1
|
||||
instance: instance1
|
||||
|
||||
sharing:
|
||||
common_config:
|
||||
pipe_name: sharing
|
||||
clients:
|
||||
- plugin_name: "kafka-client"
|
||||
brokers: broker-a:9092,broker-b:9092
|
||||
version: 2.4.1
|
||||
servers:
|
||||
- plugin_name: "grpc-server"
|
||||
address: ":11800"
|
||||
pipes:
|
||||
- common_config:
|
||||
pipe_name: pipe1
|
||||
gatherer:
|
||||
server_name: "grpc-server"
|
||||
receiver:
|
||||
plugin_name: "grpc-nativelog-receiver"
|
||||
queue:
|
||||
plugin_name: "mmap-queue"
|
||||
segment_size: 524288
|
||||
max_in_mem_segments: 6
|
||||
queue_dir: "pipe1-log-grpc-receiver-queue"
|
||||
processor:
|
||||
filters:
|
||||
sender:
|
||||
fallbacker:
|
||||
plugin_name: none-fallbacker
|
||||
flush_time: 1000
|
||||
max_buffer_size: 200
|
||||
min_flush_events: 5
|
||||
client_name: kafka-client
|
||||
forwarders:
|
||||
- plugin_name: nativelog-kafka-forwarder
|
||||
topic: skywalking-logs
|
||||
|
|
@ -25,7 +25,7 @@ services:
|
|||
environment:
|
||||
- discovery.type=single-node
|
||||
healthcheck:
|
||||
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"]
|
||||
test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200" ]
|
||||
interval: 5s
|
||||
timeout: 60s
|
||||
retries: 120
|
||||
|
|
@ -40,5 +40,12 @@ services:
|
|||
es:
|
||||
condition: service_healthy
|
||||
|
||||
provider:
|
||||
extends:
|
||||
file: ../base-compose.yml
|
||||
service: provider
|
||||
depends_on:
|
||||
oap:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
e2e:
|
||||
|
|
@ -40,5 +40,12 @@ services:
|
|||
es:
|
||||
condition: service_healthy
|
||||
|
||||
provider:
|
||||
extends:
|
||||
file: ../base-compose.yml
|
||||
service: provider
|
||||
depends_on:
|
||||
oap:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
e2e:
|
||||
|
|
@ -41,5 +41,13 @@ services:
|
|||
h2db:
|
||||
condition: service_healthy
|
||||
|
||||
provider:
|
||||
extends:
|
||||
file: ../base-compose.yml
|
||||
service: provider
|
||||
depends_on:
|
||||
oap:
|
||||
condition: service_healthy
|
||||
|
||||
networks:
|
||||
e2e:
|
||||
|
|
@ -38,5 +38,13 @@ services:
|
|||
influxdb:
|
||||
condition: service_healthy
|
||||
|
||||
provider:
|
||||
extends:
|
||||
file: ../base-compose.yml
|
||||
service: provider
|
||||
depends_on:
|
||||
oap:
|
||||
condition: service_healthy
|
||||
|
||||
networks:
|
||||
e2e:
|
||||
|
|
@ -43,5 +43,13 @@ services:
|
|||
condition: service_healthy
|
||||
entrypoint: ['sh', '-c', '/download-mysql.sh && /skywalking/docker-entrypoint.sh']
|
||||
|
||||
provider:
|
||||
extends:
|
||||
file: ../base-compose.yml
|
||||
service: provider
|
||||
depends_on:
|
||||
oap:
|
||||
condition: service_healthy
|
||||
|
||||
networks:
|
||||
e2e:
|
||||
|
|
@ -90,7 +90,7 @@ public class KafkaE2E extends SkyWalkingTestAdapter {
|
|||
protected HostAndPort serviceHostPort;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@ContainerHostAndPort(name = "ui", port = 8080)
|
||||
@ContainerHostAndPort(name = "oap", port = 12800)
|
||||
protected HostAndPort swWebappHostPort;
|
||||
|
||||
@BeforeAll
|
||||
|
|
|
|||
|
|
@ -61,17 +61,13 @@ public class KafkaLogE2E extends SkyWalkingTestAdapter {
|
|||
private HostAndPort swWebappHostPort;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@ContainerHostAndPort(name = "oap", port = 11800)
|
||||
private HostAndPort oapHostPost;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@ContainerHostAndPort(name = "provider_kafka", port = 8080)
|
||||
@ContainerHostAndPort(name = "provider", port = 9090)
|
||||
private HostAndPort serviceHostPort;
|
||||
|
||||
@BeforeAll
|
||||
public void setUp() throws Exception {
|
||||
queryClient(swWebappHostPort);
|
||||
trafficController(serviceHostPort, "/sendLog");
|
||||
trafficController(serviceHostPort, "/logs/trigger");
|
||||
}
|
||||
|
||||
@AfterAll
|
||||
|
|
@ -94,27 +90,17 @@ public class KafkaLogE2E extends SkyWalkingTestAdapter {
|
|||
LOGGER.info("verifying service instance: {}", service);
|
||||
// instance
|
||||
verifyServiceInstances(service);
|
||||
// endpoint
|
||||
verifyServiceEndpoints(service);
|
||||
}
|
||||
}
|
||||
|
||||
@RetryableTest
|
||||
public void verifyLog() throws Exception {
|
||||
LogsQuery logsQuery = new LogsQuery().serviceId("ZTJl.1")
|
||||
.serviceInstanceId("ZTJl.1_ZTJlLWluc3RhbmNl")
|
||||
.endpointId("ZTJl.1_L3RyYWZmaWM=")
|
||||
.endpointName("/traffic")
|
||||
.traceId("ac81b308-0d66-4c69-a7af-a023a536bd3e")
|
||||
.segmentId(
|
||||
"6024a2b1fcff48e4a641d69d388bac53.41.16088574455279608")
|
||||
.spanId("0")
|
||||
.tag("status_code", "200")
|
||||
LogsQuery logsQuery = new LogsQuery().serviceId("WW91cl9BcHBsaWNhdGlvbk5hbWU=.1")
|
||||
.tag("logger", "org.apache.skywalking.e2e.controller.LogController")
|
||||
.start(startTime)
|
||||
.end(Times.now());
|
||||
if (graphql.supportQueryLogsByKeywords()) {
|
||||
logsQuery.keywordsOfContent("main", "INFO")
|
||||
.excludingKeywordsOfContent("ERROR");
|
||||
logsQuery.keywordsOfContent("now");
|
||||
}
|
||||
final List<Log> logs = graphql.logs(logsQuery);
|
||||
LOGGER.info("logs: {}", logs);
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ public class KafkaMeterE2E extends SkyWalkingTestAdapter {
|
|||
protected DockerComposeContainer<?> justForSideEffects;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@ContainerHostAndPort(name = "ui", port = 8080)
|
||||
@ContainerHostAndPort(name = "oap", port = 12800)
|
||||
protected HostAndPort swWebappHostPort;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ public class KafkaProfileE2E extends SkyWalkingTestAdapter {
|
|||
protected HostAndPort serviceHostPort;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@ContainerHostAndPort(name = "ui", port = 8080)
|
||||
@ContainerHostAndPort(name = "oap", port = 12800)
|
||||
protected HostAndPort swWebappHostPort;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
|
|
|
|||
|
|
@ -17,27 +17,13 @@
|
|||
|
||||
package org.apache.skywalking.e2e.log;
|
||||
|
||||
import io.grpc.ManagedChannel;
|
||||
import io.grpc.internal.DnsNameResolverProvider;
|
||||
import io.grpc.netty.NettyChannelBuilder;
|
||||
import io.grpc.stub.StreamObserver;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.skywalking.apm.network.common.v3.Commands;
|
||||
import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair;
|
||||
import org.apache.skywalking.apm.network.logging.v3.LogData;
|
||||
import org.apache.skywalking.apm.network.logging.v3.LogDataBody;
|
||||
import org.apache.skywalking.apm.network.logging.v3.LogReportServiceGrpc;
|
||||
import org.apache.skywalking.apm.network.logging.v3.LogTags;
|
||||
import org.apache.skywalking.apm.network.logging.v3.TextLog;
|
||||
import org.apache.skywalking.apm.network.logging.v3.TraceContext;
|
||||
import org.apache.skywalking.e2e.annotation.ContainerHostAndPort;
|
||||
import org.apache.skywalking.e2e.annotation.DockerCompose;
|
||||
import org.apache.skywalking.e2e.base.SkyWalkingE2E;
|
||||
import org.apache.skywalking.e2e.base.SkyWalkingTestAdapter;
|
||||
import org.apache.skywalking.e2e.base.TrafficController;
|
||||
import org.apache.skywalking.e2e.common.HostAndPort;
|
||||
import org.apache.skywalking.e2e.retryable.RetryableTest;
|
||||
import org.apache.skywalking.e2e.service.Service;
|
||||
|
|
@ -59,9 +45,6 @@ import static org.apache.skywalking.e2e.utils.Yamls.load;
|
|||
@Slf4j
|
||||
@SkyWalkingE2E
|
||||
public class LogE2E extends SkyWalkingTestAdapter {
|
||||
|
||||
private final static int MAX_INBOUND_MESSAGE_SIZE = 1024 * 1024 * 50;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@DockerCompose({
|
||||
"docker/log/docker-compose.${SW_STORAGE}.yml"
|
||||
|
|
@ -73,25 +56,20 @@ public class LogE2E extends SkyWalkingTestAdapter {
|
|||
private HostAndPort swWebappHostPort;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@ContainerHostAndPort(name = "oap", port = 11800)
|
||||
private HostAndPort oapHostPost;
|
||||
|
||||
private LogReportServiceGrpc.LogReportServiceStub logReportServiceStub;
|
||||
@ContainerHostAndPort(name = "provider", port = 9090)
|
||||
protected HostAndPort providerHostPort;
|
||||
|
||||
@BeforeAll
|
||||
public void setUp() {
|
||||
public void setUp() throws Exception {
|
||||
queryClient(swWebappHostPort);
|
||||
final ManagedChannel channel = NettyChannelBuilder.forAddress(oapHostPost.host(), oapHostPost.port())
|
||||
.nameResolverFactory(new DnsNameResolverProvider())
|
||||
.maxInboundMessageSize(MAX_INBOUND_MESSAGE_SIZE)
|
||||
.usePlaintext().build();
|
||||
logReportServiceStub = LogReportServiceGrpc.newStub(channel);
|
||||
generateTraffic();
|
||||
trafficController(providerHostPort, "/logs/trigger");
|
||||
}
|
||||
|
||||
@AfterAll
|
||||
public void tearDown() {
|
||||
trafficController.stop();
|
||||
if (trafficController != null) {
|
||||
trafficController.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@RetryableTest
|
||||
|
|
@ -107,27 +85,17 @@ public class LogE2E extends SkyWalkingTestAdapter {
|
|||
LOGGER.info("verifying service instance: {}", service);
|
||||
// instance
|
||||
verifyServiceInstances(service);
|
||||
// endpoint
|
||||
verifyServiceEndpoints(service);
|
||||
}
|
||||
}
|
||||
|
||||
@RetryableTest
|
||||
public void verifyLog() throws Exception {
|
||||
LogsQuery logsQuery = new LogsQuery().serviceId("ZTJl.1")
|
||||
.serviceInstanceId("ZTJl.1_ZTJlLWluc3RhbmNl")
|
||||
.endpointId("ZTJl.1_L3RyYWZmaWM=")
|
||||
.endpointName("/traffic")
|
||||
.traceId("ac81b308-0d66-4c69-a7af-a023a536bd3e")
|
||||
.segmentId(
|
||||
"6024a2b1fcff48e4a641d69d388bac53.41.16088574455279608")
|
||||
.spanId("0")
|
||||
.tag("status_code", "200")
|
||||
LogsQuery logsQuery = new LogsQuery().serviceId("WW91cl9BcHBsaWNhdGlvbk5hbWU=.1")
|
||||
.tag("logger", "org.apache.skywalking.e2e.controller.LogController")
|
||||
.start(startTime)
|
||||
.end(Times.now());
|
||||
if (graphql.supportQueryLogsByKeywords()) {
|
||||
logsQuery.keywordsOfContent("main", "INFO")
|
||||
.excludingKeywordsOfContent("ERROR");
|
||||
logsQuery.keywordsOfContent("now");
|
||||
}
|
||||
final List<Log> logs = graphql.logs(logsQuery);
|
||||
LOGGER.info("logs: {}", logs);
|
||||
|
|
@ -149,68 +117,4 @@ public class LogE2E extends SkyWalkingTestAdapter {
|
|||
|
||||
load("expected/log/endpoints.yml").as(EndpointsMatcher.class).verify(endpoints);
|
||||
}
|
||||
|
||||
private void generateTraffic() {
|
||||
trafficController = TrafficController.builder()
|
||||
.sender(this::sendLog)
|
||||
.build();
|
||||
trafficController.start();
|
||||
}
|
||||
|
||||
private boolean sendLog() {
|
||||
try {
|
||||
LogData logData = LogData.newBuilder()
|
||||
.setService("e2e")
|
||||
.setServiceInstance("e2e-instance")
|
||||
.setEndpoint("/traffic")
|
||||
.setBody(
|
||||
LogDataBody.newBuilder()
|
||||
.setText(TextLog.newBuilder()
|
||||
.setText(
|
||||
"[main] INFO log message")
|
||||
.build())
|
||||
.build())
|
||||
.setTags(LogTags.newBuilder()
|
||||
.addData(KeyStringValuePair.newBuilder()
|
||||
.setKey("status_code")
|
||||
.setValue("200")
|
||||
.build())
|
||||
.build())
|
||||
.setTraceContext(TraceContext.newBuilder()
|
||||
.setTraceId("ac81b308-0d66-4c69-a7af-a023a536bd3e")
|
||||
.setTraceSegmentId(
|
||||
"6024a2b1fcff48e4a641d69d388bac53.41.16088574455279608")
|
||||
.setSpanId(0)
|
||||
.build())
|
||||
.build();
|
||||
|
||||
CountDownLatch downLatch = new CountDownLatch(1);
|
||||
|
||||
StreamObserver<LogData> streamObserver = logReportServiceStub.collect(new StreamObserver<Commands>() {
|
||||
@Override
|
||||
public void onNext(final Commands commands) {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onError(final Throwable throwable) {
|
||||
LOGGER.error(throwable.getMessage(), throwable);
|
||||
downLatch.countDown();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onCompleted() {
|
||||
downLatch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
streamObserver.onNext(logData);
|
||||
streamObserver.onCompleted();
|
||||
downLatch.await();
|
||||
return true;
|
||||
} catch (Exception e) {
|
||||
LOGGER.warn(e.getMessage(), e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,4 +15,4 @@
|
|||
|
||||
endpoints:
|
||||
- key: not null
|
||||
label: /traffic
|
||||
label: /logs/traffic
|
||||
|
|
|
|||
|
|
@ -15,4 +15,4 @@
|
|||
|
||||
instances:
|
||||
- key: not null
|
||||
label: e2e-instance
|
||||
label: not null
|
||||
|
|
|
|||
|
|
@ -14,17 +14,15 @@
|
|||
# limitations under the License.
|
||||
|
||||
logs:
|
||||
- serviceName: e2e
|
||||
- serviceName: not null
|
||||
serviceId: not null
|
||||
serviceInstanceName: "e2e-instance"
|
||||
serviceInstanceName: not null
|
||||
serviceInstanceId: not null
|
||||
endpointName: /traffic
|
||||
endpointId: not null
|
||||
traceId: "ac81b308-0d66-4c69-a7af-a023a536bd3e"
|
||||
traceId: not null
|
||||
timestamp: not null
|
||||
isError: false
|
||||
contentType: TEXT
|
||||
content: not null
|
||||
tags:
|
||||
- key: status_code
|
||||
value: 200
|
||||
- key: level
|
||||
value: INFO
|
||||
|
|
@ -15,4 +15,4 @@
|
|||
|
||||
services:
|
||||
- key: not null
|
||||
label: "e2e"
|
||||
label: not null
|
||||
|
|
|
|||
|
|
@ -40,7 +40,6 @@
|
|||
<module>e2e-protocol</module>
|
||||
<module>e2e-service-provider</module>
|
||||
<module>e2e-service-consumer</module>
|
||||
<module>e2e-log-producer</module>
|
||||
<module>e2e-test</module>
|
||||
</modules>
|
||||
|
||||
|
|
@ -68,6 +67,21 @@
|
|||
<testcontainers.version>1.15.0</testcontainers.version>
|
||||
</properties>
|
||||
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>apache.snapshots</id>
|
||||
<name>Apache Development Snapshot Repository</name>
|
||||
<url>https://repository.apache.org/content/groups/snapshots/</url>
|
||||
<releases>
|
||||
<enabled>false</enabled>
|
||||
</releases>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
</repository>
|
||||
</repositories>
|
||||
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
|
|
|
|||
Loading…
Reference in New Issue