kazusa
/
skywalking
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add FluentBit integration E2E test and doc (#7138)

This commit is contained in:
kezhenxu94 2021-06-20 21:56:45 +08:00 committed by GitHub
parent 3536b3c1c4
commit 6caf29733f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 270 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/actions/e2e-test/action.yml vendored
View File

@ -79,5 +79,5 @@ runs:
CODECOV_TOKEN: d2065307-8f01-4637-9715-2781ef096db7
run: |
echo "::group::Uploading Code Coverage"
bash -x tools/coverage/report.sh
bash -x tools/coverage/report.sh || true
echo "::endgroup::"

24
.github/workflows/e2e.log.yaml vendored
View File

@ -60,3 +60,27 @@ jobs:
uses: ./.github/actions/e2e-test
with:
test_class: org.apache.skywalking.e2e.log.LogE2E
LogFluentBit:
if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule')
name: Log (FluentBit)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Set Skip Env Var
uses: ./.github/actions/skip
- name: Cache local Maven repository
if: env.SKIP_CI != 'true'
uses: actions/cache@v2
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Run E2E Test
if: env.SKIP_CI != 'true'
uses: ./.github/actions/e2e-test
with:
test_class: org.apache.skywalking.e2e.log.LogFluentBitE2E

10
docs/en/setup/backend/log-analyzer.md
View File

@ -5,14 +5,16 @@ There are various ways to collect logs from application.
### Log files collector
You can use [Filebeat](https://www.elastic.co/cn/beats/filebeat) 、[Fluentd](https://fluentd.org) to
collect file logs including to use Kafka MQ to transport [native-json](../../protocols/Log-Data-Protocol.md#Native-Json-Protocol)
format logs. When use this, need to open [kafka-fetcher](backend-fetcher.md#kafka-fetcher)
and enable configs `enableNativeJsonLog`.
You can use [Filebeat](https://www.elastic.co/cn/beats/filebeat), [Fluentd](https://fluentd.org)
and [FluentBit](http://fluentbit.io) to collect logs, and then transport the logs to SkyWalking OAP through Kafka or
HTTP protocol, with the formats [Kafka JSON](../../protocols/Log-Data-Protocol.md#native-kafka-protocol)
or [HTTP JSON array](../../protocols/Log-Data-Protocol.md#http-api). When using Kafka protocol, you need to
open [kafka-fetcher](backend-fetcher.md#kafka-fetcher) and enable configs `enableNativeJsonLog`.
Collector config examples:
- [filebeat.yml](../../../../test/e2e/e2e-test/docker/kafka/filebeat.yml)
- [fluentd.conf](../../../../test/e2e/e2e-test/docker/kafka/fluentd.conf)
- [fluent-bit.conf](../../../../test/e2e/e2e-test/docker/log/fluent-bit)
### Java agent's toolkits
Java agent provides toolkit for

71
test/e2e/e2e-test/docker/log/docker-compose.fluentbit.yml Normal file
View File

@ -0,0 +1,71 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
version: '2.1'
services:
es:
image: elastic/elasticsearch:6.3.2
expose:
- 9200
networks:
- e2e
environment:
- discovery.type=single-node
healthcheck:
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"]
interval: 5s
timeout: 60s
retries: 120
oap:
extends:
file: ../base-compose.yml
service: oap
environment:
SW_STORAGE: elasticsearch
depends_on:
es:
condition: service_healthy
networks:
- e2e
provider:
extends:
file: ../base-compose.yml
service: provider
volumes:
- /tmp/skywalking-logs/:/tmp/skywalking-logs/
networks:
- e2e
depends_on:
oap:
condition: service_healthy
fluentbit:
image: fluent/fluent-bit:1.7
volumes:
- /tmp/skywalking-logs:/tmp/skywalking-logs
- ./fluent-bit:/fluent-bit/etc
networks:
- e2e
depends_on:
oap:
condition: service_healthy
provider:
condition: service_healthy
networks:
e2e:

21
test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit-parser.conf Normal file
View File

@ -0,0 +1,21 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[PARSER]
Name my-log-format
Format regex
Regex ^\[SW_CTX: ?\[(?<service>[^,]+),(?<serviceInstance>[^,]+),(?<traceId>[^,]+),(?<traceSegmentId>[^,]+),(?<spanId>[^\]]+)\]\] \[(?<level>.+?)\] (?<logtime>[^\]]*) \[(?<thread>[^ ]*)\] (?<logger>[^\]]*):(?<line>[^\]]*) - (?<body>[^\]]*)$
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z

23
test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit-script.lua Normal file
View File

@ -0,0 +1,23 @@
--
-- Licensed to the Apache Software Foundation (ASF) under one or more
-- contributor license agreements. See the NOTICE file distributed with
-- this work for additional information regarding copyright ownership.
-- The ASF licenses this file to You under the Apache License, Version 2.0
-- (the "License"); you may not use this file except in compliance with
-- the License. You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
function rewrite_body(tag, timestamp, record)
record["body"] = {text={text="fluentbit " .. record["body"]}}
record["tags"] = {data={{key="level", value="INFO"}}}
record["traceContext"] = {traceId=record["traceId"],traceSegmentId=record["traceSegmentId"],spanId=record["spanId"]}
return 1, timestamp, record
end

40
test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit.conf Normal file
View File

@ -0,0 +1,40 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[SERVICE]
Flush 5
Daemon Off
Log_Level warn
Parsers_File fluent-bit-parser.conf
[INPUT]
Name tail
Path /tmp/skywalking-logs/*/e2e-service-provider.log
Parser my-log-format
[FILTER]
Name lua
Match *
Script fluent-bit-script.lua
Call rewrite_body
[OUTPUT]
Name stdout
Match *
Format json
[OUTPUT]
Name http
Match *
Host oap
Port 12800
URI /v3/logs
Format json

75
test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/FluentBitE2E.java Normal file
View File

@ -0,0 +1,75 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.skywalking.e2e.log;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.e2e.annotation.ContainerHostAndPort;
import org.apache.skywalking.e2e.annotation.DockerCompose;
import org.apache.skywalking.e2e.base.SkyWalkingE2E;
import org.apache.skywalking.e2e.common.HostAndPort;
import org.apache.skywalking.e2e.retryable.RetryableTest;
import org.junit.jupiter.api.BeforeAll;
import org.testcontainers.containers.DockerComposeContainer;
@Slf4j
@SkyWalkingE2E
public class FluentBitE2E extends LogE2E {
@SuppressWarnings("unused")
@DockerCompose({
"docker/log/docker-compose.fluentbit.yml"
})
protected DockerComposeContainer<?> justForSideEffects;
@SuppressWarnings("unused")
@ContainerHostAndPort(name = "oap", port = 12800)
protected HostAndPort swWebappHostPort;
@SuppressWarnings("unused")
@ContainerHostAndPort(name = "provider", port = 9090)
protected HostAndPort providerHostPort;
@Override
@BeforeAll
public void setUp() throws Exception {
queryClient(swWebappHostPort);
trafficController(providerHostPort, "/file/logs/trigger");
}
@Override
@RetryableTest
public void verifyService() throws Exception {
}
@Override
@RetryableTest
public void verifyLog4jLog() throws Exception {
verify("fluentbit log4j fileLogger");
}
@Override
@RetryableTest
public void verifyLog4j2Log() throws Exception {
verify("fluentbit log4j2 fileLogger");
}
@Override
@RetryableTest
public void verifyLogbackLog() throws Exception {
verify("fluentbit logback fileLogger");
}
}

34
test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/LogE2E.java
View File

@ -51,11 +51,11 @@ public class LogE2E extends SkyWalkingTestAdapter {
@DockerCompose({
"docker/log/docker-compose.${SW_STORAGE}.yml"
})
private DockerComposeContainer<?> justForSideEffects;
protected DockerComposeContainer<?> justForSideEffects;
@SuppressWarnings("unused")
@ContainerHostAndPort(name = "oap", port = 12800)
private HostAndPort swWebappHostPort;
protected HostAndPort swWebappHostPort;
@SuppressWarnings("unused")
@ContainerHostAndPort(name = "provider", port = 9090)
@ -92,42 +92,26 @@ public class LogE2E extends SkyWalkingTestAdapter {
@RetryableTest
public void verifyLog4jLog() throws Exception {
LogsQuery logsQuery = new LogsQuery().serviceId("WW91cl9BcHBsaWNhdGlvbk5hbWU=.1")
.start(startTime)
.end(Times.now())
.addTag("level", "INFO");
if (graphql.supportQueryLogsByKeywords()) {
logsQuery.keywordsOfContent("log4j message");
}
final List<Log> logs = graphql.logs(logsQuery);
LOGGER.info("logs: {}", logs);
load("expected/log/logs.yml").as(LogsMatcher.class).verifyLoosely(logs);
verify("log4j message");
}
@RetryableTest
public void verifyLog4j2Log() throws Exception {
LogsQuery logsQuery = new LogsQuery().serviceId("WW91cl9BcHBsaWNhdGlvbk5hbWU=.1")
.start(startTime)
.end(Times.now())
.addTag("level", "INFO");
if (graphql.supportQueryLogsByKeywords()) {
logsQuery.keywordsOfContent("log4j2 message");
}
final List<Log> logs = graphql.logs(logsQuery);
LOGGER.info("logs: {}", logs);
load("expected/log/logs.yml").as(LogsMatcher.class).verifyLoosely(logs);
verify("log4j2 message");
}
@RetryableTest
public void verifyLogbackLog() throws Exception {
verify("logback message");
}
protected void verify(String keyword) throws Exception {
LogsQuery logsQuery = new LogsQuery().serviceId("WW91cl9BcHBsaWNhdGlvbk5hbWU=.1")
.start(startTime)
.end(Times.now())
.addTag("level", "INFO");
if (graphql.supportQueryLogsByKeywords()) {
logsQuery.keywordsOfContent("logback message");
logsQuery.keywordsOfContent(keyword);
}
final List<Log> logs = graphql.logs(logsQuery);
LOGGER.info("logs: {}", logs);