From 73c20eedbc1bf4a0411fa18098891968af8abc8b Mon Sep 17 00:00:00 2001 From: Wan Kai Date: Fri, 27 Sep 2024 10:02:21 +0800 Subject: [PATCH] BanyanDB: use `TimestampRange` to improve "events" query for BanyanDB. (#12651) --- .github/workflows/skywalking.yaml | 2 + docs/en/changes/changes.md | 1 + .../measure/BanyanDBEventQueryDAO.java | 35 ++++++++----- .../cases/alarm/banyandb/docker-compose.yml | 52 +++++++++++++++++++ test/e2e-v2/cases/alarm/banyandb/e2e.yaml | 47 +++++++++++++++++ 5 files changed, 125 insertions(+), 12 deletions(-) create mode 100644 test/e2e-v2/cases/alarm/banyandb/docker-compose.yml create mode 100644 test/e2e-v2/cases/alarm/banyandb/e2e.yaml diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index f55e8888d6..144bc6bb3b 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -405,6 +405,8 @@ jobs: config: test/e2e-v2/cases/alarm/mysql/e2e.yaml - name: Alarm PostgreSQL config: test/e2e-v2/cases/alarm/postgres/e2e.yaml + - name: Alarm BanyanDB + config: test/e2e-v2/cases/alarm/banyandb/e2e.yaml - name: TTL ES 7.16.3 config: test/e2e-v2/cases/ttl/es/e2e.yaml diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index e94c6cf8da..0317c25684 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -79,6 +79,7 @@ * PromQL service: query API support RFC3399 time format. * Improve the performance of OTEL metrics handler. * PromQL service: fix operators result missing `rangeExpression` flag. +* BanyanDB: use `TimestampRange` to improve "events" query for BanyanDB. #### UI diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBEventQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBEventQueryDAO.java index 7b56d35026..09174d399e 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBEventQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBEventQueryDAO.java @@ -30,8 +30,10 @@ import org.apache.skywalking.banyandb.v1.client.DataPoint; import org.apache.skywalking.banyandb.v1.client.MeasureQuery; import org.apache.skywalking.banyandb.v1.client.MeasureQueryResponse; import org.apache.skywalking.banyandb.v1.client.PairQueryCondition; +import org.apache.skywalking.banyandb.v1.client.TimestampRange; import org.apache.skywalking.oap.server.core.analysis.DownSampling; import org.apache.skywalking.oap.server.core.analysis.Layer; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.query.PaginationUtils; import org.apache.skywalking.oap.server.core.query.enumeration.Order; import org.apache.skywalking.oap.server.core.query.input.Duration; @@ -60,8 +62,17 @@ public class BanyanDBEventQueryDAO extends AbstractBanyanDBDAO implements IEvent @Override public Events queryEvents(EventQueryCondition condition) throws Exception { MetadataRegistry.Schema schema = MetadataRegistry.INSTANCE.findMetadata(Event.INDEX_NAME, DownSampling.Minute); + final Duration time = condition.getTime(); + TimestampRange tsRange = null; + if (time != null) { + long startTB = time.getStartTimeBucketInSec(); + long endTB = time.getEndTimeBucketInSec(); + if (startTB > 0 && endTB > 0) { + tsRange = new TimestampRange(TimeBucket.getTimestamp(startTB), TimeBucket.getTimestamp(endTB)); + } + } MeasureQueryResponse resp = query(schema, TAGS, - Collections.emptySet(), buildQuery(Collections.singletonList(condition))); + Collections.emptySet(), tsRange, buildQuery(Collections.singletonList(condition))); Events events = new Events(); if (resp.size() == 0) { return events; @@ -75,8 +86,18 @@ public class BanyanDBEventQueryDAO extends AbstractBanyanDBDAO implements IEvent @Override public Events queryEvents(List conditionList) throws Exception { MetadataRegistry.Schema schema = MetadataRegistry.INSTANCE.findMetadata(Event.INDEX_NAME, DownSampling.Minute); + // Duration should be same for all conditions + final Duration time = conditionList.get(0).getTime(); + TimestampRange tsRange = null; + if (time != null) { + long startTB = time.getStartTimeBucketInSec(); + long endTB = time.getEndTimeBucketInSec(); + if (startTB > 0 && endTB > 0) { + tsRange = new TimestampRange(TimeBucket.getTimestamp(startTB), TimeBucket.getTimestamp(endTB)); + } + } MeasureQueryResponse resp = query(schema, TAGS, - Collections.emptySet(), buildQuery(conditionList)); + Collections.emptySet(), tsRange, buildQuery(conditionList)); Events events = new Events(); if (resp.size() == 0) { return events; @@ -129,16 +150,6 @@ public class BanyanDBEventQueryDAO extends AbstractBanyanDBDAO implements IEvent queryConditions.add(eq(Event.TYPE, condition.getType().name())); } - final Duration startTime = condition.getTime(); - if (startTime != null) { - if (startTime.getStartTimestamp() > 0) { - queryConditions.add(gte(Event.START_TIME, startTime.getStartTimestamp())); - } - if (startTime.getEndTimestamp() > 0) { - queryConditions.add(lte(Event.END_TIME, startTime.getEndTimestamp())); - } - } - if (!isNullOrEmpty(condition.getLayer())) { queryConditions.add(eq(Event.LAYER, Layer.valueOf(condition.getLayer()).value())); } diff --git a/test/e2e-v2/cases/alarm/banyandb/docker-compose.yml b/test/e2e-v2/cases/alarm/banyandb/docker-compose.yml new file mode 100644 index 0000000000..a08b02cad0 --- /dev/null +++ b/test/e2e-v2/cases/alarm/banyandb/docker-compose.yml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + banyandb: + extends: + file: ../../../script/docker-compose/base-compose.yml + service: banyandb + networks: + - e2e + + oap: + extends: + file: ../../../script/docker-compose/base-compose.yml + service: oap + environment: + SW_STORAGE: banyandb + SW_SEARCHABLE_ALARM_TAG_KEYS: level,receivers + ports: + - 12800 + depends_on: + banyandb: + condition: service_healthy + volumes: + - ../alarm-settings.yml:/skywalking/config/alarm-settings.yml + + provider: + extends: + file: ../../../script/docker-compose/base-compose.yml + service: provider + ports: + - 9090 + depends_on: + oap: + condition: service_healthy + +networks: + e2e: diff --git a/test/e2e-v2/cases/alarm/banyandb/e2e.yaml b/test/e2e-v2/cases/alarm/banyandb/e2e.yaml new file mode 100644 index 0000000000..4dcb90bf0f --- /dev/null +++ b/test/e2e-v2/cases/alarm/banyandb/e2e.yaml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 20m + init-system-environment: ../../../script/env + steps: + - name: set PATH + command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + +trigger: + action: http + interval: 3s + times: 30 + url: http://${provider_host}:${provider_9090}/users + method: POST + body: '{"id":"123","name":"skywalking"}' + headers: + "Content-Type": "application/json" + +verify: + retry: + count: 20 + interval: 3s + cases: + - includes: + - ../alarm-cases.yaml