From 492b6f1535d0ff24fc5cf2ea3bd8bab14c1f04ed Mon Sep 17 00:00:00 2001 From: Wan Kai Date: Mon, 14 Jul 2025 13:07:31 +0800 Subject: [PATCH] BanyanDB: support new Index rule type `SKIPPING/TREE`, and update the record `log`'s `trace_id` indexType to `SKIPPING` (#13366) --- docs/en/changes/changes.md | 2 + oap-server-bom/pom.xml | 2 +- .../manual/log/AbstractLogRecord.java | 1 + .../core/storage/annotation/BanyanDB.java | 11 +++++ .../core/storage/model/BanyanDBExtension.java | 3 +- .../plugin/banyandb/MetadataRegistry.java | 43 +++++++++++++++---- .../server/telemetry/api/MetricsCreator.java | 2 +- test/e2e-v2/script/env | 2 +- 8 files changed, 52 insertions(+), 14 deletions(-) diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 2f8b4a3c46..375103ce33 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -39,6 +39,8 @@ * OAP gRPC-Client support `Health Check`. * [Break Change] `health_check_xx` metrics make response 1 represents healthy, 0 represents unhealthy. * Bump up grpc to 1.70.0. +* BanyanDB: support new Index rule type `SKIPPING/TREE`, and update the record `log`'s `trace_id` indexType to `SKIPPING` +* BanyanDB: remove `index-only` from tag setting. #### UI diff --git a/oap-server-bom/pom.xml b/oap-server-bom/pom.xml index 050d3ad115..73d3d71377 100644 --- a/oap-server-bom/pom.xml +++ b/oap-server-bom/pom.xml @@ -72,7 +72,7 @@ 4.4.16 4.1.5 1.21 - 0.9.0-rc2 + 0.9.0-rc3 3.4.0 2.4.6.RELEASE 1.5.3 diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java index de11f1f20f..15726e3d7c 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java @@ -68,6 +68,7 @@ public abstract class AbstractLogRecord extends Record { @Setter @Getter @Column(name = TRACE_ID, length = 150) + @BanyanDB.IndexRule(indexType = BanyanDB.IndexRule.IndexType.SKIPPING) private String traceId; @Setter @Getter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDB.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDB.java index 8c6afc4771..9908f24d8c 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDB.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDB.java @@ -24,6 +24,7 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import lombok.Getter; +import org.apache.skywalking.oap.server.core.analysis.manual.log.LogRecord; import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.analysis.record.Record; import org.apache.skywalking.oap.server.core.storage.StorageID; @@ -132,6 +133,16 @@ public @interface BanyanDB { * It's suitable for most tag indexing due to a better memory usage ratio and query performance. */ INVERTED, + /** + * The `SKIPPING` index is optimized for the majority of stream tags, which prioritizes efficient space utilization. + * Such as the `trace_id` in the {@link LogRecord}. + */ + SKIPPING, + /** + * The `TREE` index is designed for storing hierarchical data. + * Such as Trace Span. + */ + TREE } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java index 9053a3c184..d22b52ab2b 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java @@ -54,9 +54,8 @@ public class BanyanDBExtension { * indexType is the type of index built for a {@link ModelColumn} in BanyanDB. * * @since 9.3.0 - * @deprecated since 10.2. Only support {@link BanyanDB.IndexRule.IndexType#INVERTED} now. There was IndexType#TREE, - * but removed. */ + @Getter private final BanyanDB.IndexRule.IndexType indexType; /** diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java index ef9d47d549..a30c0de90b 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java @@ -81,6 +81,10 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import static org.apache.skywalking.banyandb.database.v1.BanyandbDatabase.IndexRule.Type.TYPE_INVERTED; +import static org.apache.skywalking.banyandb.database.v1.BanyandbDatabase.IndexRule.Type.TYPE_SKIPPING; +import static org.apache.skywalking.banyandb.database.v1.BanyandbDatabase.IndexRule.Type.TYPE_TREE; + @Slf4j public enum MetadataRegistry { INSTANCE; @@ -160,7 +164,7 @@ public enum MetadataRegistry { .collect(Collectors.toList()); if (model.getBanyanDBModelExtension().isStoreIDTag()) { - indexRules.add(indexRule(schemaMetadata.group, BanyanDBConverter.ID, false, null)); + indexRules.add(indexRule(schemaMetadata.group, BanyanDBConverter.ID, false, null, null)); } final Measure.Builder builder = Measure.newBuilder(); @@ -368,12 +372,35 @@ public enum MetadataRegistry { } } - IndexRule indexRule(String group, String tagName, boolean enableSort, BanyanDB.MatchQuery.AnalyzerType analyzer) { + IndexRule indexRule(String group, + String tagName, + boolean enableSort, + BanyanDB.MatchQuery.AnalyzerType analyzer, + BanyanDB.IndexRule.IndexType type) { IndexRule.Builder builder = IndexRule.newBuilder() .setMetadata(Metadata.newBuilder().setName(tagName).setGroup(group)) - .setType(IndexRule.Type.TYPE_INVERTED).addTags(tagName); + .addTags(tagName); // *Notice*: here is a reverse logic, if enableSort is true, then setNoSort is false builder.setNoSort(!enableSort); + + if (type != null) { + switch (type) { + case INVERTED: + builder.setType(TYPE_INVERTED); + break; + case TREE: + builder.setType(TYPE_TREE); + break; + case SKIPPING: + builder.setType(TYPE_SKIPPING); + break; + default: + throw new UnsupportedOperationException("unsupported index type: " + type); + } + } else { + builder.setType(TYPE_INVERTED); + } + if (analyzer != null) { switch (analyzer) { case KEYWORD: @@ -449,7 +476,8 @@ public enum MetadataRegistry { tagMetadataList.add(new TagMetadata( indexRule( group, tagSpec.getName(), col.getBanyanDBExtension().isEnableSort(), - col.getBanyanDBExtension().getAnalyzer() + col.getBanyanDBExtension().getAnalyzer(), + col.getBanyanDBExtension().getIndexType() ), tagSpec)); } else { tagMetadataList.add(new TagMetadata(null, tagSpec)); @@ -499,7 +527,8 @@ public enum MetadataRegistry { result.tag(new TagMetadata( indexRule( group, tagSpec.getName(), col.getBanyanDBExtension().isEnableSort(), - col.getBanyanDBExtension().getAnalyzer() + col.getBanyanDBExtension().getAnalyzer(), + col.getBanyanDBExtension().getIndexType() ), tagSpec)); } else { result.tag(new TagMetadata(null, tagSpec)); @@ -544,10 +573,6 @@ public enum MetadataRegistry { } else { throw new IllegalStateException("type " + modelColumn.getType().toString() + " is not supported"); } - - if (modelColumn.isIndexOnly()) { - tagSpec.setIndexedOnly(true); - } return tagSpec.build(); } diff --git a/oap-server/server-telemetry/telemetry-api/src/main/java/org/apache/skywalking/oap/server/telemetry/api/MetricsCreator.java b/oap-server/server-telemetry/telemetry-api/src/main/java/org/apache/skywalking/oap/server/telemetry/api/MetricsCreator.java index 476bf3f0de..f1183bf3aa 100644 --- a/oap-server/server-telemetry/telemetry-api/src/main/java/org/apache/skywalking/oap/server/telemetry/api/MetricsCreator.java +++ b/oap-server/server-telemetry/telemetry-api/src/main/java/org/apache/skywalking/oap/server/telemetry/api/MetricsCreator.java @@ -53,7 +53,7 @@ public interface MetricsCreator extends Service { default HealthCheckMetrics createHealthCheckerGauge(String name, MetricsTag.Keys tagKeys, MetricsTag.Values tagValues) { Preconditions.checkArgument(!Strings.isNullOrEmpty(name), "Require non-null or empty metric name"); return new HealthCheckMetrics(createGauge(Strings.lenientFormat("%s%s", HEALTH_METRIC_PREFIX, name), - Strings.lenientFormat("%s health check. 1 health, 0 not health, -1 unknown", name), + Strings.lenientFormat("%s health check. 1 healthy, 0 not healthy, -1 unknown", name), tagKeys, tagValues)); } diff --git a/test/e2e-v2/script/env b/test/e2e-v2/script/env index 020252abbe..ad47277695 100644 --- a/test/e2e-v2/script/env +++ b/test/e2e-v2/script/env @@ -23,7 +23,7 @@ SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449 SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016 SW_KUBERNETES_COMMIT_SHA=6fe5e6f0d3b7686c6be0457733e825ee68cb9b35 SW_ROVER_COMMIT=79292fe07f17f98f486e0c4471213e1961fb2d1d -SW_BANYANDB_COMMIT=b90a0ff31a4d8560422c04d152e7e7188d72ea89 +SW_BANYANDB_COMMIT=0f3b90d9b4f628d2de02cb39ef678d636358df44 SW_AGENT_PHP_COMMIT=d1114e7be5d89881eec76e5b56e69ff844691e35 SW_PREDICTOR_COMMIT=54a0197654a3781a6f73ce35146c712af297c994