diff --git a/.github/actions/e2e-test/action.yml b/.github/actions/e2e-test/action.yml index 67f3dbb9b..6cd1ce35b 100644 --- a/.github/actions/e2e-test/action.yml +++ b/.github/actions/e2e-test/action.yml @@ -79,5 +79,5 @@ runs: CODECOV_TOKEN: d2065307-8f01-4637-9715-2781ef096db7 run: | echo "::group::Uploading Code Coverage" - bash -x tools/coverage/report.sh + bash -x tools/coverage/report.sh || true echo "::endgroup::" diff --git a/.github/workflows/e2e.log.yaml b/.github/workflows/e2e.log.yaml index 88c82122b..116b15e1f 100644 --- a/.github/workflows/e2e.log.yaml +++ b/.github/workflows/e2e.log.yaml @@ -60,3 +60,27 @@ jobs: uses: ./.github/actions/e2e-test with: test_class: org.apache.skywalking.e2e.log.LogE2E + + LogFluentBit: + if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') + name: Log (FluentBit) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Set Skip Env Var + uses: ./.github/actions/skip + - name: Cache local Maven repository + if: env.SKIP_CI != 'true' + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Run E2E Test + if: env.SKIP_CI != 'true' + uses: ./.github/actions/e2e-test + with: + test_class: org.apache.skywalking.e2e.log.LogFluentBitE2E diff --git a/docs/en/setup/backend/log-analyzer.md b/docs/en/setup/backend/log-analyzer.md index 68f8a2887..b323136d8 100644 --- a/docs/en/setup/backend/log-analyzer.md +++ b/docs/en/setup/backend/log-analyzer.md @@ -5,14 +5,16 @@ There are various ways to collect logs from application. ### Log files collector -You can use [Filebeat](https://www.elastic.co/cn/beats/filebeat) 、[Fluentd](https://fluentd.org) to -collect file logs including to use Kafka MQ to transport [native-json](../../protocols/Log-Data-Protocol.md#Native-Json-Protocol) -format logs. When use this, need to open [kafka-fetcher](backend-fetcher.md#kafka-fetcher) -and enable configs `enableNativeJsonLog`. +You can use [Filebeat](https://www.elastic.co/cn/beats/filebeat), [Fluentd](https://fluentd.org) +and [FluentBit](http://fluentbit.io) to collect logs, and then transport the logs to SkyWalking OAP through Kafka or +HTTP protocol, with the formats [Kafka JSON](../../protocols/Log-Data-Protocol.md#native-kafka-protocol) +or [HTTP JSON array](../../protocols/Log-Data-Protocol.md#http-api). When using Kafka protocol, you need to +open [kafka-fetcher](backend-fetcher.md#kafka-fetcher) and enable configs `enableNativeJsonLog`. Collector config examples: - [filebeat.yml](../../../../test/e2e/e2e-test/docker/kafka/filebeat.yml) - [fluentd.conf](../../../../test/e2e/e2e-test/docker/kafka/fluentd.conf) +- [fluent-bit.conf](../../../../test/e2e/e2e-test/docker/log/fluent-bit) ### Java agent's toolkits Java agent provides toolkit for diff --git a/test/e2e/e2e-test/docker/log/docker-compose.fluentbit.yml b/test/e2e/e2e-test/docker/log/docker-compose.fluentbit.yml new file mode 100644 index 000000000..2fa84b3d5 --- /dev/null +++ b/test/e2e/e2e-test/docker/log/docker-compose.fluentbit.yml @@ -0,0 +1,71 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + es: + image: elastic/elasticsearch:6.3.2 + expose: + - 9200 + networks: + - e2e + environment: + - discovery.type=single-node + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"] + interval: 5s + timeout: 60s + retries: 120 + + oap: + extends: + file: ../base-compose.yml + service: oap + environment: + SW_STORAGE: elasticsearch + depends_on: + es: + condition: service_healthy + networks: + - e2e + + provider: + extends: + file: ../base-compose.yml + service: provider + volumes: + - /tmp/skywalking-logs/:/tmp/skywalking-logs/ + networks: + - e2e + depends_on: + oap: + condition: service_healthy + + fluentbit: + image: fluent/fluent-bit:1.7 + volumes: + - /tmp/skywalking-logs:/tmp/skywalking-logs + - ./fluent-bit:/fluent-bit/etc + networks: + - e2e + depends_on: + oap: + condition: service_healthy + provider: + condition: service_healthy + +networks: + e2e: diff --git a/test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit-parser.conf b/test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit-parser.conf new file mode 100644 index 000000000..be298d631 --- /dev/null +++ b/test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit-parser.conf @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[PARSER] + Name my-log-format + Format regex + Regex ^\[SW_CTX: ?\[(?[^,]+),(?[^,]+),(?[^,]+),(?[^,]+),(?[^\]]+)\]\] \[(?.+?)\] (?[^\]]*) \[(?[^ ]*)\] (?[^\]]*):(?[^\]]*) - (?[^\]]*)$ + Time_Key time + Time_Format %d/%b/%Y:%H:%M:%S %z diff --git a/test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit-script.lua b/test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit-script.lua new file mode 100644 index 000000000..274d62e46 --- /dev/null +++ b/test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit-script.lua @@ -0,0 +1,23 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +function rewrite_body(tag, timestamp, record) + record["body"] = {text={text="fluentbit " .. record["body"]}} + record["tags"] = {data={{key="level", value="INFO"}}} + record["traceContext"] = {traceId=record["traceId"],traceSegmentId=record["traceSegmentId"],spanId=record["spanId"]} + return 1, timestamp, record +end diff --git a/test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit.conf b/test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit.conf new file mode 100644 index 000000000..28d9554ee --- /dev/null +++ b/test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit.conf @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[SERVICE] + Flush 5 + Daemon Off + Log_Level warn + Parsers_File fluent-bit-parser.conf +[INPUT] + Name tail + Path /tmp/skywalking-logs/*/e2e-service-provider.log + Parser my-log-format +[FILTER] + Name lua + Match * + Script fluent-bit-script.lua + Call rewrite_body +[OUTPUT] + Name stdout + Match * + Format json +[OUTPUT] + Name http + Match * + Host oap + Port 12800 + URI /v3/logs + Format json diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/FluentBitE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/FluentBitE2E.java new file mode 100644 index 000000000..75071661d --- /dev/null +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/FluentBitE2E.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.e2e.log; + +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.e2e.annotation.ContainerHostAndPort; +import org.apache.skywalking.e2e.annotation.DockerCompose; +import org.apache.skywalking.e2e.base.SkyWalkingE2E; +import org.apache.skywalking.e2e.common.HostAndPort; +import org.apache.skywalking.e2e.retryable.RetryableTest; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.containers.DockerComposeContainer; + +@Slf4j +@SkyWalkingE2E +public class FluentBitE2E extends LogE2E { + @SuppressWarnings("unused") + @DockerCompose({ + "docker/log/docker-compose.fluentbit.yml" + }) + protected DockerComposeContainer justForSideEffects; + + @SuppressWarnings("unused") + @ContainerHostAndPort(name = "oap", port = 12800) + protected HostAndPort swWebappHostPort; + + @SuppressWarnings("unused") + @ContainerHostAndPort(name = "provider", port = 9090) + protected HostAndPort providerHostPort; + + @Override + @BeforeAll + public void setUp() throws Exception { + queryClient(swWebappHostPort); + trafficController(providerHostPort, "/file/logs/trigger"); + } + + @Override + @RetryableTest + public void verifyService() throws Exception { + } + + @Override + @RetryableTest + public void verifyLog4jLog() throws Exception { + verify("fluentbit log4j fileLogger"); + } + + @Override + @RetryableTest + public void verifyLog4j2Log() throws Exception { + verify("fluentbit log4j2 fileLogger"); + } + + @Override + @RetryableTest + public void verifyLogbackLog() throws Exception { + verify("fluentbit logback fileLogger"); + } +} diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/LogE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/LogE2E.java index 0ed77ab6f..ad1f10c06 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/LogE2E.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/LogE2E.java @@ -51,11 +51,11 @@ public class LogE2E extends SkyWalkingTestAdapter { @DockerCompose({ "docker/log/docker-compose.${SW_STORAGE}.yml" }) - private DockerComposeContainer justForSideEffects; + protected DockerComposeContainer justForSideEffects; @SuppressWarnings("unused") @ContainerHostAndPort(name = "oap", port = 12800) - private HostAndPort swWebappHostPort; + protected HostAndPort swWebappHostPort; @SuppressWarnings("unused") @ContainerHostAndPort(name = "provider", port = 9090) @@ -92,42 +92,26 @@ public class LogE2E extends SkyWalkingTestAdapter { @RetryableTest public void verifyLog4jLog() throws Exception { - LogsQuery logsQuery = new LogsQuery().serviceId("WW91cl9BcHBsaWNhdGlvbk5hbWU=.1") - .start(startTime) - .end(Times.now()) - .addTag("level", "INFO"); - if (graphql.supportQueryLogsByKeywords()) { - logsQuery.keywordsOfContent("log4j message"); - } - final List logs = graphql.logs(logsQuery); - LOGGER.info("logs: {}", logs); - - load("expected/log/logs.yml").as(LogsMatcher.class).verifyLoosely(logs); + verify("log4j message"); } @RetryableTest public void verifyLog4j2Log() throws Exception { - LogsQuery logsQuery = new LogsQuery().serviceId("WW91cl9BcHBsaWNhdGlvbk5hbWU=.1") - .start(startTime) - .end(Times.now()) - .addTag("level", "INFO"); - if (graphql.supportQueryLogsByKeywords()) { - logsQuery.keywordsOfContent("log4j2 message"); - } - final List logs = graphql.logs(logsQuery); - LOGGER.info("logs: {}", logs); - - load("expected/log/logs.yml").as(LogsMatcher.class).verifyLoosely(logs); + verify("log4j2 message"); } @RetryableTest public void verifyLogbackLog() throws Exception { + verify("logback message"); + } + + protected void verify(String keyword) throws Exception { LogsQuery logsQuery = new LogsQuery().serviceId("WW91cl9BcHBsaWNhdGlvbk5hbWU=.1") .start(startTime) .end(Times.now()) .addTag("level", "INFO"); if (graphql.supportQueryLogsByKeywords()) { - logsQuery.keywordsOfContent("logback message"); + logsQuery.keywordsOfContent(keyword); } final List logs = graphql.logs(logsQuery); LOGGER.info("logs: {}", logs);