kazusa
/
skywalking
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

BanyanDB: support new Index rule type `SKIPPING/TREE`, and update the record `log`'s `trace_id` indexType to `SKIPPING` (#13366)

This commit is contained in:
Wan Kai 2025-07-14 13:07:31 +08:00 committed by GitHub
parent 5d80ac2e98
commit 492b6f1535
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 52 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/en/changes/changes.md
View File

@ -39,6 +39,8 @@
* OAP gRPC-Client support `Health Check`.
* [Break Change] `health_check_xx` metrics make response 1 represents healthy, 0 represents unhealthy.
* Bump up grpc to 1.70.0.
* BanyanDB: support new Index rule type `SKIPPING/TREE`, and update the record `log`'s `trace_id` indexType to `SKIPPING`
* BanyanDB: remove `index-only` from tag setting.
#### UI

2
oap-server-bom/pom.xml
View File

@ -72,7 +72,7 @@
<httpcore.version>4.4.16</httpcore.version>
<httpasyncclient.version>4.1.5</httpasyncclient.version>
<commons-compress.version>1.21</commons-compress.version>
<banyandb-java-client.version>0.9.0-rc2</banyandb-java-client.version>
<banyandb-java-client.version>0.9.0-rc3</banyandb-java-client.version>
<kafka-clients.version>3.4.0</kafka-clients.version>
<spring-kafka-test.version>2.4.6.RELEASE</spring-kafka-test.version>
<consul.client.version>1.5.3</consul.client.version>

1
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java
View File

@ -68,6 +68,7 @@ public abstract class AbstractLogRecord extends Record {
@Setter
@Getter
@Column(name = TRACE_ID, length = 150)
@BanyanDB.IndexRule(indexType = BanyanDB.IndexRule.IndexType.SKIPPING)
private String traceId;
@Setter
@Getter

11
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDB.java
View File

@ -24,6 +24,7 @@ import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import lombok.Getter;
import org.apache.skywalking.oap.server.core.analysis.manual.log.LogRecord;
import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.storage.StorageID;
@ -132,6 +133,16 @@ public @interface BanyanDB {
* It's suitable for most tag indexing due to a better memory usage ratio and query performance.
*/
INVERTED,
/**
* The `SKIPPING` index is optimized for the majority of stream tags, which prioritizes efficient space utilization.
* Such as the `trace_id` in the {@link LogRecord}.
*/
SKIPPING,
/**
* The `TREE` index is designed for storing hierarchical data.
* Such as Trace Span.
*/
TREE
}
}

3
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java
View File

@ -54,9 +54,8 @@ public class BanyanDBExtension {
* indexType is the type of index built for a {@link ModelColumn} in BanyanDB.
*
* @since 9.3.0
* @deprecated since 10.2. Only support {@link BanyanDB.IndexRule.IndexType#INVERTED} now. There was IndexType#TREE,
* but removed.
*/
@Getter
private final BanyanDB.IndexRule.IndexType indexType;
/**

43
oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java
View File

@ -81,6 +81,10 @@ import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import static org.apache.skywalking.banyandb.database.v1.BanyandbDatabase.IndexRule.Type.TYPE_INVERTED;
import static org.apache.skywalking.banyandb.database.v1.BanyandbDatabase.IndexRule.Type.TYPE_SKIPPING;
import static org.apache.skywalking.banyandb.database.v1.BanyandbDatabase.IndexRule.Type.TYPE_TREE;
@Slf4j
public enum MetadataRegistry {
INSTANCE;
@ -160,7 +164,7 @@ public enum MetadataRegistry {
.collect(Collectors.toList());
if (model.getBanyanDBModelExtension().isStoreIDTag()) {
indexRules.add(indexRule(schemaMetadata.group, BanyanDBConverter.ID, false, null));
indexRules.add(indexRule(schemaMetadata.group, BanyanDBConverter.ID, false, null, null));
}
final Measure.Builder builder = Measure.newBuilder();
@ -368,12 +372,35 @@ public enum MetadataRegistry {
}
}
IndexRule indexRule(String group, String tagName, boolean enableSort, BanyanDB.MatchQuery.AnalyzerType analyzer) {
IndexRule indexRule(String group,
String tagName,
boolean enableSort,
BanyanDB.MatchQuery.AnalyzerType analyzer,
BanyanDB.IndexRule.IndexType type) {
IndexRule.Builder builder = IndexRule.newBuilder()
.setMetadata(Metadata.newBuilder().setName(tagName).setGroup(group))
.setType(IndexRule.Type.TYPE_INVERTED).addTags(tagName);
.addTags(tagName);
// *Notice*: here is a reverse logic, if enableSort is true, then setNoSort is false
builder.setNoSort(!enableSort);
if (type != null) {
switch (type) {
case INVERTED:
builder.setType(TYPE_INVERTED);
break;
case TREE:
builder.setType(TYPE_TREE);
break;
case SKIPPING:
builder.setType(TYPE_SKIPPING);
break;
default:
throw new UnsupportedOperationException("unsupported index type: " + type);
}
} else {
builder.setType(TYPE_INVERTED);
}
if (analyzer != null) {
switch (analyzer) {
case KEYWORD:
@ -449,7 +476,8 @@ public enum MetadataRegistry {
tagMetadataList.add(new TagMetadata(
indexRule(
group, tagSpec.getName(), col.getBanyanDBExtension().isEnableSort(),
col.getBanyanDBExtension().getAnalyzer()
col.getBanyanDBExtension().getAnalyzer(),
col.getBanyanDBExtension().getIndexType()
), tagSpec));
} else {
tagMetadataList.add(new TagMetadata(null, tagSpec));
@ -499,7 +527,8 @@ public enum MetadataRegistry {
result.tag(new TagMetadata(
indexRule(
group, tagSpec.getName(), col.getBanyanDBExtension().isEnableSort(),
col.getBanyanDBExtension().getAnalyzer()
col.getBanyanDBExtension().getAnalyzer(),
col.getBanyanDBExtension().getIndexType()
), tagSpec));
} else {
result.tag(new TagMetadata(null, tagSpec));
@ -544,10 +573,6 @@ public enum MetadataRegistry {
} else {
throw new IllegalStateException("type " + modelColumn.getType().toString() + " is not supported");
}
if (modelColumn.isIndexOnly()) {
tagSpec.setIndexedOnly(true);
}
return tagSpec.build();
}

2
oap-server/server-telemetry/telemetry-api/src/main/java/org/apache/skywalking/oap/server/telemetry/api/MetricsCreator.java
View File

@ -53,7 +53,7 @@ public interface MetricsCreator extends Service {
default HealthCheckMetrics createHealthCheckerGauge(String name, MetricsTag.Keys tagKeys, MetricsTag.Values tagValues) {
Preconditions.checkArgument(!Strings.isNullOrEmpty(name), "Require non-null or empty metric name");
return new HealthCheckMetrics(createGauge(Strings.lenientFormat("%s%s", HEALTH_METRIC_PREFIX, name),
Strings.lenientFormat("%s health check. 1 health, 0 not health, -1 unknown", name),
Strings.lenientFormat("%s health check. 1 healthy, 0 not healthy, -1 unknown", name),
tagKeys, tagValues));
}

2
test/e2e-v2/script/env
View File

@ -23,7 +23,7 @@ SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449
SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016
SW_KUBERNETES_COMMIT_SHA=6fe5e6f0d3b7686c6be0457733e825ee68cb9b35
SW_ROVER_COMMIT=79292fe07f17f98f486e0c4471213e1961fb2d1d
SW_BANYANDB_COMMIT=b90a0ff31a4d8560422c04d152e7e7188d72ea89
SW_BANYANDB_COMMIT=0f3b90d9b4f628d2de02cb39ef678d636358df44
SW_AGENT_PHP_COMMIT=d1114e7be5d89881eec76e5b56e69ff844691e35
SW_PREDICTOR_COMMIT=54a0197654a3781a6f73ce35146c712af297c994