From 85e22f5124f96cafd3a7eb7bd22ea548d1e9d620 Mon Sep 17 00:00:00 2001 From: kezhenxu94 Date: Tue, 10 Aug 2021 15:36:53 +0800 Subject: [PATCH] Fix issues in ES7.14 and add log cases to ES7.14 (#7429) --- .github/workflows/e2e.log.yaml | 2 +- CHANGES.md | 1 + .../manual/log/AbstractLogRecord.java | 7 ++- .../errorlog/BrowserErrorLogRecord.java | 6 +- .../docker/log/docker-compose.es7.14.yml | 56 +++++++++++++++++++ 5 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 test/e2e/e2e-test/docker/log/docker-compose.es7.14.yml diff --git a/.github/workflows/e2e.log.yaml b/.github/workflows/e2e.log.yaml index c8a4d9a76..3eea86d5f 100644 --- a/.github/workflows/e2e.log.yaml +++ b/.github/workflows/e2e.log.yaml @@ -39,7 +39,7 @@ jobs: timeout-minutes: 90 strategy: matrix: - storage: ['h2', 'mysql', 'es6', 'es7', 'influxdb'] + storage: ['h2', 'mysql', 'es6', 'es7', 'es7.14', 'influxdb'] env: SW_STORAGE: ${{ matrix.storage }} steps: diff --git a/CHANGES.md b/CHANGES.md index 3e4bc9ccb..7998e759c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -23,6 +23,7 @@ Release Notes. * Trace segment wouldn't be recognized as a TopN sample service. Add through #4694 experimentally, but it caused performance impact. * Remove `version` and `endTime` in the segment entity. Reduce indexing payload. +* Fix `mapper_parsing_exception` in ElasticSearch 7.14. #### UI diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java index 8988d0033..66557d75d 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java @@ -33,6 +33,8 @@ import org.apache.skywalking.oap.server.core.storage.StorageHashMapBuilder; import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import static java.util.Objects.nonNull; + public abstract class AbstractLogRecord extends Record { public static final String SERVICE_ID = "service_id"; @@ -44,7 +46,6 @@ public abstract class AbstractLogRecord extends Record { public static final String SPAN_ID = "span_id"; public static final String CONTENT_TYPE = "content_type"; public static final String CONTENT = "content"; - public static final String CONTENT_TYPE_CLASS = "content_type_class"; public static final String TAGS_RAW_DATA = "tags_raw_data"; public static final String TIMESTAMP = "timestamp"; public static final String TAGS = "tags"; @@ -121,7 +122,9 @@ public abstract class AbstractLogRecord extends Record { map.put(SERVICE_ID, record.getServiceId()); map.put(SERVICE_INSTANCE_ID, record.getServiceInstanceId()); map.put(ENDPOINT_ID, record.getEndpointId()); - map.put(ENDPOINT_NAME, record.getEndpointName()); + if (nonNull(record.getEndpointName())) { + map.put(ENDPOINT_NAME, record.getEndpointName()); + } map.put(TRACE_ID, record.getTraceId()); map.put(TRACE_SEGMENT_ID, record.getTraceSegmentId()); map.put(SPAN_ID, record.getSpanId()); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java index 7cc404bb9..7a856c53c 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java @@ -33,6 +33,8 @@ import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset; import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import static java.util.Objects.nonNull; + @SuperDataset @Stream(name = BrowserErrorLogRecord.INDEX_NAME, scopeId = DefaultScopeDefine.BROWSER_ERROR_LOG, builder = BrowserErrorLogRecord.Builder.class, processor = RecordStreamProcessor.class) public class BrowserErrorLogRecord extends Record { @@ -119,7 +121,9 @@ public class BrowserErrorLogRecord extends Record { map.put(SERVICE_ID, storageData.getServiceId()); map.put(SERVICE_VERSION_ID, storageData.getServiceVersionId()); map.put(PAGE_PATH_ID, storageData.getPagePathId()); - map.put(PAGE_PATH, storageData.getPagePath()); + if (nonNull(storageData.getPagePath())) { + map.put(PAGE_PATH, storageData.getPagePath()); + } map.put(TIMESTAMP, storageData.getTimestamp()); map.put(TIME_BUCKET, storageData.getTimeBucket()); map.put(ERROR_CATEGORY, storageData.getErrorCategory()); diff --git a/test/e2e/e2e-test/docker/log/docker-compose.es7.14.yml b/test/e2e/e2e-test/docker/log/docker-compose.es7.14.yml new file mode 100644 index 000000000..2a7cadcd6 --- /dev/null +++ b/test/e2e/e2e-test/docker/log/docker-compose.es7.14.yml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + es: + image: elastic/elasticsearch:7.14.0 + expose: + - 9200 + networks: + - e2e + environment: + - discovery.type=single-node + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"] + interval: 5s + timeout: 60s + retries: 120 + + oap: + extends: + file: ../base-compose.yml + service: oap-es7 + environment: + SW_STORAGE: elasticsearch7 + SW_LOG_LAL_FILES: test + SW_LOG_MAL_FILES: test + volumes: + - ./lal.yaml:/skywalking/config/lal/test.yaml + - ./log-mal.yaml:/skywalking/config/log-mal-rules/test.yaml + depends_on: + es: + condition: service_healthy + + provider: + extends: + file: ../base-compose.yml + service: provider + depends_on: + oap: + condition: service_healthy +networks: + e2e: