kazusa
/
skywalking
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

BanyanDB: Support `@EnableSort` on the column to enable sorting for `IndexRule` and set the default to false. (#12823)

This commit is contained in:
Wan Kai 2024-12-03 17:46:33 +08:00 committed by GitHub
parent 88cd1eb025
commit 1cd5733aac
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 141 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/en/changes/changes.md
View File

@ -32,6 +32,7 @@
* BanyanDB: Introduce index mode and refactor banyandb group settings.
* BanyanDB: Support update the Schema when OAP starting.
* BanyanDB: Speed up OAP booting while initializing BanyanDB.
* BanyanDB: Support `@EnableSort` on the column to enable sorting for `IndexRule` and set the default to false.
#### UI

1
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java
View File

@ -91,6 +91,7 @@ public class SegmentRecord extends Record {
@Setter
@Getter
@ElasticSearch.EnableDocValues
@BanyanDB.EnableSort
@Column(name = LATENCY)
private int latency;
@Setter

1
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/trace/SampledSlowTraceRecord.java
View File

@ -62,6 +62,7 @@ public class SampledSlowTraceRecord extends Record {
@Column(name = URI, storageOnly = true)
private String uri;
@ElasticSearch.EnableDocValues
@BanyanDB.EnableSort
@Column(name = LATENCY, dataType = Column.ValueDataType.SAMPLED_RECORD)
private long latency;
@Setter

1
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/trace/SampledStatus4xxTraceRecord.java
View File

@ -63,6 +63,7 @@ public class SampledStatus4xxTraceRecord extends Record {
@Column(name = URI, storageOnly = true)
private String uri;
@ElasticSearch.EnableDocValues
@BanyanDB.EnableSort
@Column(name = LATENCY, dataType = Column.ValueDataType.SAMPLED_RECORD)
private long latency;
@Setter

1
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/trace/SampledStatus5xxTraceRecord.java
View File

@ -63,6 +63,7 @@ public class SampledStatus5xxTraceRecord extends Record {
@Column(name = URI, storageOnly = true)
private String uri;
@ElasticSearch.EnableDocValues
@BanyanDB.EnableSort
@Column(name = LATENCY, dataType = Column.ValueDataType.SAMPLED_RECORD)
private long latency;
@Setter

1
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/Event.java
View File

@ -109,6 +109,7 @@ public class Event extends Metrics {
private String parameters;
@ElasticSearch.EnableDocValues
@BanyanDB.EnableSort
@Column(name = START_TIME)
private long startTime;

1
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/topn/TopN.java
View File

@ -39,6 +39,7 @@ public abstract class TopN extends Record implements ComparableStorageData {
@Getter
@Setter
@ElasticSearch.EnableDocValues
@BanyanDB.EnableSort
@Column(name = LATENCY, dataType = Column.ValueDataType.SAMPLED_RECORD)
private long latency;
@Getter

3
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppPagePerf.java
View File

@ -40,7 +40,8 @@ public class BrowserAppPagePerf extends BrowserAppPerfSource {
}
@Getter
@ScopeDefaultColumn.DefinedByField(columnName = "service_id", groupByCondInTopN = true)
@ScopeDefaultColumn.DefinedByField(columnName = "service_id")
@ScopeDefaultColumn.BanyanDB(groupByCondInTopN = true)
private String serviceId;
@Getter
@Setter

1
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingScheduleRecord.java
View File

@ -67,6 +67,7 @@ public class EBPFProfilingScheduleRecord extends Metrics {
@Column(name = PROCESS_ID, length = 600)
private String processId;
@ElasticSearch.EnableDocValues
@BanyanDB.EnableSort
@Column(name = START_TIME)
private long startTime;
@Column(name = END_TIME)

8
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java
View File

@ -234,12 +234,18 @@ public class DefaultScopeDefine {
for (Field field : scopeClassField) {
ScopeDefaultColumn.DefinedByField definedByField = field.getAnnotation(
ScopeDefaultColumn.DefinedByField.class);
ScopeDefaultColumn.BanyanDB banyanDB = field.getAnnotation(
ScopeDefaultColumn.BanyanDB.class);
boolean groupByCondInTopN = false;
if (banyanDB != null) {
groupByCondInTopN = banyanDB.groupByCondInTopN();
}
if (definedByField != null) {
if (!definedByField.requireDynamicActive() || ACTIVE_EXTRA_MODEL_COLUMNS) {
scopeDefaultColumns.add(
new ScopeDefaultColumn(
field.getName(), definedByField.columnName(), field.getType(), false,
definedByField.length(), definedByField.groupByCondInTopN(), definedByField.isAttribute()
definedByField.length(), groupByCondInTopN, definedByField.isAttribute()
));
}
}

3
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java
View File

@ -56,7 +56,8 @@ public class Endpoint extends Source {
@ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
private String name;
@Getter
@ScopeDefaultColumn.DefinedByField(columnName = "service_id", groupByCondInTopN = true)
@ScopeDefaultColumn.DefinedByField(columnName = "service_id")
@ScopeDefaultColumn.BanyanDB(groupByCondInTopN = true)
private String serviceId;
@Getter
@Setter

3
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/K8SEndpoint.java
View File

@ -31,7 +31,8 @@ import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.K8
public class K8SEndpoint extends K8SMetrics.ProtocolMetrics {
private volatile String entityId;
@ScopeDefaultColumn.DefinedByField(columnName = "service_id", groupByCondInTopN = true)
@ScopeDefaultColumn.DefinedByField(columnName = "service_id")
@ScopeDefaultColumn.BanyanDB(groupByCondInTopN = true)
private String serviceId;
@ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true)
private String serviceName;

19
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ScopeDefaultColumn.java
View File

@ -67,13 +67,6 @@ public class ScopeDefaultColumn {
*/
int length() default 256;
/**
* Indicate whether this column is a condition for groupBy in the TopN Aggregation.
*
* @since 9.5.0
*/
boolean groupByCondInTopN() default false;
/**
* Indicate whether this column is an attribute.
* Attributes are optional fields, which are set by the source decorator and can be used for query conditions.
@ -83,6 +76,18 @@ public class ScopeDefaultColumn {
boolean isAttribute() default false;
}
@Target({ElementType.FIELD})
@Retention(RetentionPolicy.RUNTIME)
public @interface BanyanDB {
/**
* Indicate whether this column is a condition for groupBy in the TopN Aggregation.
*
* @since 9.5.0
* @since 10.2.0 moved out from {@link DefinedByField} to {@link BanyanDB}
*/
boolean groupByCondInTopN() default false;
}
@Target({ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
public @interface VirtualColumnDefinition {

9
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDB.java
View File

@ -207,6 +207,15 @@ public @interface BanyanDB {
}
}
/**
* EnableSort is used to indicate the IndexRule supports sorting.
* @since 10.2.0
*/
@Target({ElementType.FIELD})
@Retention(RetentionPolicy.RUNTIME)
@interface EnableSort {
}
/**
* IndexMode is used to indicate the index mode of the metric.
* All columns in the metric will be stored in the index exclusively.

17
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java
View File

@ -30,14 +30,14 @@ import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB;
@RequiredArgsConstructor
public class BanyanDBExtension {
/**
* Sharding key is used to group time series data per metric of one entity. See {@link
* SeriesID is used to group time series data per metric of one entity. See {@link
* BanyanDB.SeriesID#index()}.
*
* @since 9.1.0 moved into BanyanDBExtension
* @since 9.0.0 added into {@link ModelColumn}
*/
@Getter
private final int shardingKeyIdx;
private final int seriesIDIdx;
/**
* {@link BanyanDB.NoIndexing} exists to override {@link ModelColumn#shouldIndex()}, or be the same as {@link
@ -64,15 +64,22 @@ public class BanyanDBExtension {
/**
* The analyzer policy appointed to fuzzy query, especially for BanyanDB.
* See {@link BanyanDB.MatchQuery}.
*/
@Getter
private final BanyanDB.MatchQuery.AnalyzerType analyzer;
/**
* @return true if this column is a part of sharding key
* Enable sort for this column. See {@link BanyanDB.EnableSort}.
*/
public boolean isShardingKey() {
return this.shardingKeyIdx > -1;
@Getter
private final boolean enableSort;
/**
* @return true if this column is a part of SeriesID
*/
public boolean isSeriesID() {
return this.seriesIDIdx > -1;
}
/**

2
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBModelExtension.java
View File

@ -42,7 +42,7 @@ public class BanyanDBModelExtension {
/**
* storeIDTag indicates whether a metric stores its ID as a tag.
* The installer will create a virtual string ID tag with a tree index rule.
* The installer will create a virtual string ID tag without timestamp.
*/
@Getter
@Setter

5
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java
View File

@ -104,6 +104,11 @@ public class ModelColumn {
throw new IllegalArgumentException(
"The column " + columnName + " should be indexed if require MatchQuery.");
}
if (!this.banyanDBExtension.shouldIndex() && this.banyanDBExtension.isEnableSort()) {
throw new IllegalArgumentException(
"The column " + columnName + " should be indexed if require EnableSort.");
}
}
/**

25
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
View File

@ -58,7 +58,7 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
DefaultScopeDefine.nameOf(scopeId);
List<ModelColumn> modelColumns = new ArrayList<>();
ShardingKeyChecker checker = new ShardingKeyChecker();
SeriesIDChecker checker = new SeriesIDChecker();
SQLDatabaseModelExtension sqlDBModelExtension = new SQLDatabaseModelExtension();
BanyanDBModelExtension banyanDBModelExtension = new BanyanDBModelExtension();
ElasticSearchModelExtension elasticSearchModelExtension = new ElasticSearchModelExtension();
@ -153,7 +153,7 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
final String modelName,
final List<ModelColumn> modelColumns,
final int scopeId,
ShardingKeyChecker checker,
SeriesIDChecker checker,
final SQLDatabaseModelExtension sqlDBModelExtension,
final BanyanDBModelExtension banyanDBModelExtension) {
if (log.isDebugEnabled()) {
@ -222,13 +222,16 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
BanyanDB.TopNAggregation.class);
final BanyanDB.MatchQuery analyzer = field.getAnnotation(
BanyanDB.MatchQuery.class);
final BanyanDB.EnableSort enableSort = field.getAnnotation(
BanyanDB.EnableSort.class);
final boolean shouldIndex = (banyanDBNoIndex == null) && !column.storageOnly();
BanyanDBExtension banyanDBExtension = new BanyanDBExtension(
banyanDBSeriesID == null ? -1 : banyanDBSeriesID.index(),
shouldIndex,
banyanDBIndexRule == null ? BanyanDB.IndexRule.IndexType.INVERTED : banyanDBIndexRule.indexType(),
banyanDBMeasureField != null,
analyzer == null ? null : analyzer.analyzer()
analyzer == null ? null : analyzer.analyzer(),
enableSort != null
);
if (topNAggregation != null) {
@ -251,7 +254,7 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
elasticSearchExtension,
banyanDBExtension
);
if (banyanDBExtension.isShardingKey()) {
if (banyanDBExtension.isSeriesID()) {
checker.accept(modelName, modelColumn);
}
@ -321,14 +324,14 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
return models;
}
private static class ShardingKeyChecker {
private static class SeriesIDChecker {
private final ArrayList<ModelColumn> keys = new ArrayList<>();
/**
* @throws IllegalStateException if sharding key indices are conflicting.
* @throws IllegalStateException if seriesID indices are conflicting.
*/
private void accept(String modelName, ModelColumn modelColumn) throws IllegalStateException {
final int idx = modelColumn.getBanyanDBExtension().getShardingKeyIdx();
final int idx = modelColumn.getBanyanDBExtension().getSeriesIDIdx();
while (idx + 1 > keys.size()) {
keys.add(null);
}
@ -337,21 +340,21 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
throw new IllegalStateException(
modelName + "'s "
+ "Column [" + exist.getColumnName() + "] and column [" + modelColumn.getColumnName()
+ " are conflicting with sharding key index=" + modelColumn.getBanyanDBExtension()
.getShardingKeyIdx());
+ " are conflicting with seriesID index=" + modelColumn.getBanyanDBExtension()
.getSeriesIDIdx());
}
keys.set(idx, modelColumn);
}
/**
* @param modelName model name of the entity
* @throws IllegalStateException if sharding key indices are not continuous
* @throws IllegalStateException if seriesIDs indices are not continuous
*/
private void check(String modelName) throws IllegalStateException {
for (int i = 0; i < keys.size(); i++) {
final ModelColumn modelColumn = keys.get(i);
if (modelColumn == null) {
throw new IllegalStateException("Sharding key index=" + i + " is missing in " + modelName);
throw new IllegalStateException("seriesID index=" + i + " is missing in " + modelName);
}
}
}

39
oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java
View File

@ -49,7 +49,10 @@ public class ModelColumnTest {
new SQLDatabaseExtension(),
new ElasticSearchExtension(
ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, null, false, false, true),
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE)
new BanyanDBExtension(
-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false,
BanyanDB.MatchQuery.AnalyzerType.SIMPLE, true
)
);
Assertions.assertTrue(column.isStorageOnly());
Assertions.assertEquals("abc", column.getColumnName().getName());
@ -58,7 +61,10 @@ public class ModelColumnTest {
false, false, true, 200,
new SQLDatabaseExtension(),
new ElasticSearchExtension(ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, null, false, false, true),
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB. MatchQuery.AnalyzerType.SIMPLE)
new BanyanDBExtension(
-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false,
BanyanDB.MatchQuery.AnalyzerType.SIMPLE, true
)
);
Assertions.assertTrue(column.isStorageOnly());
Assertions.assertEquals("abc", column.getColumnName().getName());
@ -68,7 +74,7 @@ public class ModelColumnTest {
false, false, true, 200,
new SQLDatabaseExtension(),
new ElasticSearchExtension(ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, null, false, false, true),
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE)
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE, true)
);
Assertions.assertFalse(column.isStorageOnly());
Assertions.assertEquals("abc", column.getColumnName().getName());
@ -82,7 +88,10 @@ public class ModelColumnTest {
new SQLDatabaseExtension(),
new ElasticSearchExtension(
ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, "abc", false, false, true),
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE)
new BanyanDBExtension(
-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false,
BanyanDB.MatchQuery.AnalyzerType.SIMPLE, true
)
);
});
}
@ -95,7 +104,9 @@ public class ModelColumnTest {
new SQLDatabaseExtension(),
new ElasticSearchExtension(
ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, "abc", false, false, true),
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE)
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false,
BanyanDB.MatchQuery.AnalyzerType.SIMPLE, true
)
);
});
}
@ -110,7 +121,23 @@ public class ModelColumnTest {
ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, "abc", false, false, true),
new BanyanDBExtension(
-1, false, BanyanDB.IndexRule.IndexType.INVERTED, false,
BanyanDB.MatchQuery.AnalyzerType.SIMPLE
BanyanDB.MatchQuery.AnalyzerType.SIMPLE, true
)
);
});
}
@Test
public void testConflictDefinitionEnableSort() {
assertThrows(IllegalArgumentException.class, () -> {
new ModelColumn(new ColumnName(c), String.class, String.class,
true, false, false, 200,
new SQLDatabaseExtension(),
new ElasticSearchExtension(
ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, "abc", false, false, true),
new BanyanDBExtension(
-1, false, BanyanDB.IndexRule.IndexType.INVERTED, true,
null, true
)
);
});

58
oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java
View File

@ -84,16 +84,16 @@ public enum MetadataRegistry {
Schema.SchemaBuilder schemaBuilder = Schema.builder().metadata(schemaMetadata);
Map<String, ModelColumn> modelColumnMap = model.getColumns().stream()
.collect(Collectors.toMap(modelColumn -> modelColumn.getColumnName().getStorageName(), Function.identity()));
// parse and set sharding keys
List<String> shardingColumns = parseEntityNames(modelColumnMap);
if (shardingColumns.isEmpty()) {
throw new IllegalStateException("sharding keys of model[stream." + model.getName() + "] must not be empty");
// parse and set seriesIDs
List<String> seriesIDColumns = parseEntityNames(modelColumnMap);
if (seriesIDColumns.isEmpty()) {
throw new IllegalStateException("seriesID of model[stream." + model.getName() + "] must not be empty");
}
// parse tag metadata
// this can be used to build both
// 1) a list of TagFamilySpec,
// 2) a list of IndexRule,
List<TagMetadata> tags = parseTagMetadata(model, schemaBuilder, shardingColumns, schemaMetadata.group);
List<TagMetadata> tags = parseTagMetadata(model, schemaBuilder, seriesIDColumns, schemaMetadata.group);
List<TagFamilySpec> tagFamilySpecs = schemaMetadata.extractTagFamilySpec(tags, false);
// iterate over tagFamilySpecs to save tag names
for (final TagFamilySpec tagFamilySpec : tagFamilySpecs) {
@ -115,7 +115,7 @@ public enum MetadataRegistry {
final Stream.Builder builder = Stream.newBuilder();
builder.setMetadata(BanyandbCommon.Metadata.newBuilder().setGroup(schemaMetadata.getGroup())
.setName(schemaMetadata.name()));
builder.setEntity(BanyandbDatabase.Entity.newBuilder().addAllTagNames(shardingColumns));
builder.setEntity(BanyandbDatabase.Entity.newBuilder().addAllTagNames(seriesIDColumns));
builder.addAllTagFamilies(tagFamilySpecs);
registry.put(schemaMetadata.name(), schemaBuilder.build());
@ -127,16 +127,16 @@ public enum MetadataRegistry {
Schema.SchemaBuilder schemaBuilder = Schema.builder().metadata(schemaMetadata);
Map<String, ModelColumn> modelColumnMap = model.getColumns().stream()
.collect(Collectors.toMap(modelColumn -> modelColumn.getColumnName().getStorageName(), Function.identity()));
// parse and set sharding keys
List<String> shardingColumns = parseEntityNames(modelColumnMap);
if (shardingColumns.isEmpty()) {
// parse and set seriesIDs
List<String> seriesIDColumns = parseEntityNames(modelColumnMap);
if (seriesIDColumns.isEmpty()) {
throw new StorageException("model " + model.getName() + " doesn't contain series id");
}
// parse tag metadata
// this can be used to build both
// 1) a list of TagFamilySpec,
// 2) a list of IndexRule,
MeasureMetadata tagsAndFields = parseTagAndFieldMetadata(model, schemaBuilder, shardingColumns, schemaMetadata.group);
MeasureMetadata tagsAndFields = parseTagAndFieldMetadata(model, schemaBuilder, seriesIDColumns, schemaMetadata.group);
List<TagFamilySpec> tagFamilySpecs = schemaMetadata.extractTagFamilySpec(tagsAndFields.tags, model.getBanyanDBModelExtension().isStoreIDTag());
// iterate over tagFamilySpecs to save tag names
for (final TagFamilySpec tagFamilySpec : tagFamilySpecs) {
@ -150,14 +150,14 @@ public enum MetadataRegistry {
.collect(Collectors.toList());
if (model.getBanyanDBModelExtension().isStoreIDTag()) {
indexRules.add(indexRule(schemaMetadata.group, BanyanDBConverter.ID, null));
indexRules.add(indexRule(schemaMetadata.group, BanyanDBConverter.ID, false, null));
}
final Measure.Builder builder = Measure.newBuilder();
builder.setMetadata(BanyandbCommon.Metadata.newBuilder().setGroup(schemaMetadata.getGroup())
.setName(schemaMetadata.name()));
builder.setInterval(downSamplingDuration(model.getDownsampling()).format());
builder.setEntity(BanyandbDatabase.Entity.newBuilder().addAllTagNames(shardingColumns));
builder.setEntity(BanyandbDatabase.Entity.newBuilder().addAllTagNames(seriesIDColumns));
builder.addAllTagFamilies(tagFamilySpecs);
if (model.getBanyanDBModelExtension().isIndexMode()) {
builder.setIndexMode(true);
@ -290,10 +290,12 @@ public enum MetadataRegistry {
}
}
IndexRule indexRule(String group, String tagName, BanyanDB.MatchQuery.AnalyzerType analyzer) {
IndexRule indexRule(String group, String tagName, boolean enableSort, BanyanDB.MatchQuery.AnalyzerType analyzer) {
IndexRule.Builder builder = IndexRule.newBuilder()
.setMetadata(Metadata.newBuilder().setName(tagName).setGroup(group))
.setType(IndexRule.Type.TYPE_INVERTED).addTags(tagName);
// *Notice*: here is a reverse logic, if enableSort is true, then setNoSort is false
builder.setNoSort(!enableSort);
if (analyzer != null) {
switch (analyzer) {
case KEYWORD:
@ -316,20 +318,20 @@ public enum MetadataRegistry {
}
/**
* Parse sharding keys from the {@link Model}
* Parse SeriesID from the {@link Model}
*
* @param modelColumnMap the mapping between column storageName and {@link ModelColumn}
* @return a list of column names in strict order
*/
List<String> parseEntityNames(Map<String, ModelColumn> modelColumnMap) {
List<ModelColumn> shardingColumns = new ArrayList<>();
List<ModelColumn> seriesIDColumns = new ArrayList<>();
for (final ModelColumn col : modelColumnMap.values()) {
if (col.getBanyanDBExtension().isShardingKey()) {
shardingColumns.add(col);
if (col.getBanyanDBExtension().isSeriesID()) {
seriesIDColumns.add(col);
}
}
return shardingColumns.stream()
.sorted(Comparator.comparingInt(col -> col.getBanyanDBExtension().getShardingKeyIdx()))
return seriesIDColumns.stream()
.sorted(Comparator.comparingInt(col -> col.getBanyanDBExtension().getSeriesIDIdx()))
.map(col -> col.getColumnName().getName())
.collect(Collectors.toList());
}
@ -341,7 +343,7 @@ public enum MetadataRegistry {
*
* @since 9.4.0 Skip {@link Record#TIME_BUCKET}
*/
List<TagMetadata> parseTagMetadata(Model model, Schema.SchemaBuilder builder, List<String> shardingColumns, String group) {
List<TagMetadata> parseTagMetadata(Model model, Schema.SchemaBuilder builder, List<String> seriesIDColumns, String group) {
List<TagMetadata> tagMetadataList = new ArrayList<>();
for (final ModelColumn col : model.getColumns()) {
final String columnStorageName = col.getColumnName().getStorageName();
@ -352,9 +354,12 @@ public enum MetadataRegistry {
builder.spec(columnStorageName, new ColumnSpec(ColumnType.TAG, col.getType()));
String colName = col.getColumnName().getStorageName();
if (col.getBanyanDBExtension().shouldIndex()) {
if (!shardingColumns.contains(colName) || null != col.getBanyanDBExtension().getAnalyzer()) {
if (!seriesIDColumns.contains(colName) || null != col.getBanyanDBExtension().getAnalyzer()) {
tagMetadataList.add(new TagMetadata(
indexRule(group, tagSpec.getName(), col.getBanyanDBExtension().getAnalyzer()), tagSpec));
indexRule(
group, tagSpec.getName(), col.getBanyanDBExtension().isEnableSort(),
col.getBanyanDBExtension().getAnalyzer()
), tagSpec));
} else {
tagMetadataList.add(new TagMetadata(null, tagSpec));
}
@ -381,7 +386,7 @@ public enum MetadataRegistry {
*
* @since 9.4.0 Skip {@link Metrics#TIME_BUCKET}
*/
MeasureMetadata parseTagAndFieldMetadata(Model model, Schema.SchemaBuilder builder, List<String> shardingColumns, String group) {
MeasureMetadata parseTagAndFieldMetadata(Model model, Schema.SchemaBuilder builder, List<String> seriesIDColumns, String group) {
// skip metric
MeasureMetadata.MeasureMetadataBuilder result = MeasureMetadata.builder();
for (final ModelColumn col : model.getColumns()) {
@ -399,9 +404,12 @@ public enum MetadataRegistry {
String colName = col.getColumnName().getStorageName();
if (col.getBanyanDBExtension().shouldIndex()) {
if (!shardingColumns.contains(colName) || null != col.getBanyanDBExtension().getAnalyzer()) {
if (!seriesIDColumns.contains(colName) || null != col.getBanyanDBExtension().getAnalyzer()) {
result.tag(new TagMetadata(
indexRule(group, tagSpec.getName(), col.getBanyanDBExtension().getAnalyzer()), tagSpec));
indexRule(
group, tagSpec.getName(), col.getBanyanDBExtension().isEnableSort(),
col.getBanyanDBExtension().getAnalyzer()
), tagSpec));
} else {
result.tag(new TagMetadata(null, tagSpec));
}

2
oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBMetricsDAO.java
View File

@ -77,7 +77,7 @@ public class BanyanDBMetricsDAO extends AbstractBanyanDBDAO implements IMetricsD
if (ext == null) {
return;
}
if (ext.isShardingKey()) {
if (ext.isSeriesID()) {
seriesIDColumns.put(c.getColumnName().getName(), new ArrayList<>());
}
});