diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 4dda6f327c..96a95ddc4c 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -32,6 +32,7 @@ * BanyanDB: Introduce index mode and refactor banyandb group settings. * BanyanDB: Support update the Schema when OAP starting. * BanyanDB: Speed up OAP booting while initializing BanyanDB. +* BanyanDB: Support `@EnableSort` on the column to enable sorting for `IndexRule` and set the default to false. #### UI diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java index c72dc4a24b..41595a9063 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java @@ -91,6 +91,7 @@ public class SegmentRecord extends Record { @Setter @Getter @ElasticSearch.EnableDocValues + @BanyanDB.EnableSort @Column(name = LATENCY) private int latency; @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/trace/SampledSlowTraceRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/trace/SampledSlowTraceRecord.java index 6815d6d879..fda70e6778 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/trace/SampledSlowTraceRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/trace/SampledSlowTraceRecord.java @@ -62,6 +62,7 @@ public class SampledSlowTraceRecord extends Record { @Column(name = URI, storageOnly = true) private String uri; @ElasticSearch.EnableDocValues + @BanyanDB.EnableSort @Column(name = LATENCY, dataType = Column.ValueDataType.SAMPLED_RECORD) private long latency; @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/trace/SampledStatus4xxTraceRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/trace/SampledStatus4xxTraceRecord.java index d74674205c..54b11f6470 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/trace/SampledStatus4xxTraceRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/trace/SampledStatus4xxTraceRecord.java @@ -63,6 +63,7 @@ public class SampledStatus4xxTraceRecord extends Record { @Column(name = URI, storageOnly = true) private String uri; @ElasticSearch.EnableDocValues + @BanyanDB.EnableSort @Column(name = LATENCY, dataType = Column.ValueDataType.SAMPLED_RECORD) private long latency; @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/trace/SampledStatus5xxTraceRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/trace/SampledStatus5xxTraceRecord.java index 403ee05c8b..34fe3f9299 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/trace/SampledStatus5xxTraceRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/trace/SampledStatus5xxTraceRecord.java @@ -63,6 +63,7 @@ public class SampledStatus5xxTraceRecord extends Record { @Column(name = URI, storageOnly = true) private String uri; @ElasticSearch.EnableDocValues + @BanyanDB.EnableSort @Column(name = LATENCY, dataType = Column.ValueDataType.SAMPLED_RECORD) private long latency; @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/Event.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/Event.java index 09d343a3bd..98d2fbe220 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/Event.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/Event.java @@ -109,6 +109,7 @@ public class Event extends Metrics { private String parameters; @ElasticSearch.EnableDocValues + @BanyanDB.EnableSort @Column(name = START_TIME) private long startTime; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/topn/TopN.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/topn/TopN.java index b86f2e39a6..3920d83a20 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/topn/TopN.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/topn/TopN.java @@ -39,6 +39,7 @@ public abstract class TopN extends Record implements ComparableStorageData { @Getter @Setter @ElasticSearch.EnableDocValues + @BanyanDB.EnableSort @Column(name = LATENCY, dataType = Column.ValueDataType.SAMPLED_RECORD) private long latency; @Getter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppPagePerf.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppPagePerf.java index bede297ef0..adab7546cc 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppPagePerf.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppPagePerf.java @@ -40,7 +40,8 @@ public class BrowserAppPagePerf extends BrowserAppPerfSource { } @Getter - @ScopeDefaultColumn.DefinedByField(columnName = "service_id", groupByCondInTopN = true) + @ScopeDefaultColumn.DefinedByField(columnName = "service_id") + @ScopeDefaultColumn.BanyanDB(groupByCondInTopN = true) private String serviceId; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingScheduleRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingScheduleRecord.java index b4bcdac390..4ec6953194 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingScheduleRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingScheduleRecord.java @@ -67,6 +67,7 @@ public class EBPFProfilingScheduleRecord extends Metrics { @Column(name = PROCESS_ID, length = 600) private String processId; @ElasticSearch.EnableDocValues + @BanyanDB.EnableSort @Column(name = START_TIME) private long startTime; @Column(name = END_TIME) diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java index 4b7cce9832..88595ad457 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java @@ -234,12 +234,18 @@ public class DefaultScopeDefine { for (Field field : scopeClassField) { ScopeDefaultColumn.DefinedByField definedByField = field.getAnnotation( ScopeDefaultColumn.DefinedByField.class); + ScopeDefaultColumn.BanyanDB banyanDB = field.getAnnotation( + ScopeDefaultColumn.BanyanDB.class); + boolean groupByCondInTopN = false; + if (banyanDB != null) { + groupByCondInTopN = banyanDB.groupByCondInTopN(); + } if (definedByField != null) { if (!definedByField.requireDynamicActive() || ACTIVE_EXTRA_MODEL_COLUMNS) { scopeDefaultColumns.add( new ScopeDefaultColumn( field.getName(), definedByField.columnName(), field.getType(), false, - definedByField.length(), definedByField.groupByCondInTopN(), definedByField.isAttribute() + definedByField.length(), groupByCondInTopN, definedByField.isAttribute() )); } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java index 7ac0fc6f43..acc5955f58 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java @@ -56,7 +56,8 @@ public class Endpoint extends Source { @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) private String name; @Getter - @ScopeDefaultColumn.DefinedByField(columnName = "service_id", groupByCondInTopN = true) + @ScopeDefaultColumn.DefinedByField(columnName = "service_id") + @ScopeDefaultColumn.BanyanDB(groupByCondInTopN = true) private String serviceId; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/K8SEndpoint.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/K8SEndpoint.java index 9f4715ca29..0f16aaa02d 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/K8SEndpoint.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/K8SEndpoint.java @@ -31,7 +31,8 @@ import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.K8 public class K8SEndpoint extends K8SMetrics.ProtocolMetrics { private volatile String entityId; - @ScopeDefaultColumn.DefinedByField(columnName = "service_id", groupByCondInTopN = true) + @ScopeDefaultColumn.DefinedByField(columnName = "service_id") + @ScopeDefaultColumn.BanyanDB(groupByCondInTopN = true) private String serviceId; @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) private String serviceName; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ScopeDefaultColumn.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ScopeDefaultColumn.java index 71ca602d0e..7688668087 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ScopeDefaultColumn.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ScopeDefaultColumn.java @@ -67,13 +67,6 @@ public class ScopeDefaultColumn { */ int length() default 256; - /** - * Indicate whether this column is a condition for groupBy in the TopN Aggregation. - * - * @since 9.5.0 - */ - boolean groupByCondInTopN() default false; - /** * Indicate whether this column is an attribute. * Attributes are optional fields, which are set by the source decorator and can be used for query conditions. @@ -83,6 +76,18 @@ public class ScopeDefaultColumn { boolean isAttribute() default false; } + @Target({ElementType.FIELD}) + @Retention(RetentionPolicy.RUNTIME) + public @interface BanyanDB { + /** + * Indicate whether this column is a condition for groupBy in the TopN Aggregation. + * + * @since 9.5.0 + * @since 10.2.0 moved out from {@link DefinedByField} to {@link BanyanDB} + */ + boolean groupByCondInTopN() default false; + } + @Target({ElementType.TYPE}) @Retention(RetentionPolicy.RUNTIME) public @interface VirtualColumnDefinition { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDB.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDB.java index 3e95ad11ef..af31fab4b1 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDB.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDB.java @@ -207,6 +207,15 @@ public @interface BanyanDB { } } + /** + * EnableSort is used to indicate the IndexRule supports sorting. + * @since 10.2.0 + */ + @Target({ElementType.FIELD}) + @Retention(RetentionPolicy.RUNTIME) + @interface EnableSort { + } + /** * IndexMode is used to indicate the index mode of the metric. * All columns in the metric will be stored in the index exclusively. diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java index d680176574..7cd55b4be4 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java @@ -30,14 +30,14 @@ import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB; @RequiredArgsConstructor public class BanyanDBExtension { /** - * Sharding key is used to group time series data per metric of one entity. See {@link + * SeriesID is used to group time series data per metric of one entity. See {@link * BanyanDB.SeriesID#index()}. * * @since 9.1.0 moved into BanyanDBExtension * @since 9.0.0 added into {@link ModelColumn} */ @Getter - private final int shardingKeyIdx; + private final int seriesIDIdx; /** * {@link BanyanDB.NoIndexing} exists to override {@link ModelColumn#shouldIndex()}, or be the same as {@link @@ -64,15 +64,22 @@ public class BanyanDBExtension { /** * The analyzer policy appointed to fuzzy query, especially for BanyanDB. + * See {@link BanyanDB.MatchQuery}. */ @Getter private final BanyanDB.MatchQuery.AnalyzerType analyzer; /** - * @return true if this column is a part of sharding key + * Enable sort for this column. See {@link BanyanDB.EnableSort}. */ - public boolean isShardingKey() { - return this.shardingKeyIdx > -1; + @Getter + private final boolean enableSort; + + /** + * @return true if this column is a part of SeriesID + */ + public boolean isSeriesID() { + return this.seriesIDIdx > -1; } /** diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBModelExtension.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBModelExtension.java index 5e2601cb11..198091d842 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBModelExtension.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBModelExtension.java @@ -42,7 +42,7 @@ public class BanyanDBModelExtension { /** * storeIDTag indicates whether a metric stores its ID as a tag. - * The installer will create a virtual string ID tag with a tree index rule. + * The installer will create a virtual string ID tag without timestamp. */ @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java index 9712a8fca1..876fbe82a7 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java @@ -104,6 +104,11 @@ public class ModelColumn { throw new IllegalArgumentException( "The column " + columnName + " should be indexed if require MatchQuery."); } + + if (!this.banyanDBExtension.shouldIndex() && this.banyanDBExtension.isEnableSort()) { + throw new IllegalArgumentException( + "The column " + columnName + " should be indexed if require EnableSort."); + } } /** diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java index 729e55a69f..fe45c226e4 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java @@ -58,7 +58,7 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula DefaultScopeDefine.nameOf(scopeId); List modelColumns = new ArrayList<>(); - ShardingKeyChecker checker = new ShardingKeyChecker(); + SeriesIDChecker checker = new SeriesIDChecker(); SQLDatabaseModelExtension sqlDBModelExtension = new SQLDatabaseModelExtension(); BanyanDBModelExtension banyanDBModelExtension = new BanyanDBModelExtension(); ElasticSearchModelExtension elasticSearchModelExtension = new ElasticSearchModelExtension(); @@ -153,7 +153,7 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula final String modelName, final List modelColumns, final int scopeId, - ShardingKeyChecker checker, + SeriesIDChecker checker, final SQLDatabaseModelExtension sqlDBModelExtension, final BanyanDBModelExtension banyanDBModelExtension) { if (log.isDebugEnabled()) { @@ -222,13 +222,16 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula BanyanDB.TopNAggregation.class); final BanyanDB.MatchQuery analyzer = field.getAnnotation( BanyanDB.MatchQuery.class); + final BanyanDB.EnableSort enableSort = field.getAnnotation( + BanyanDB.EnableSort.class); final boolean shouldIndex = (banyanDBNoIndex == null) && !column.storageOnly(); BanyanDBExtension banyanDBExtension = new BanyanDBExtension( banyanDBSeriesID == null ? -1 : banyanDBSeriesID.index(), shouldIndex, banyanDBIndexRule == null ? BanyanDB.IndexRule.IndexType.INVERTED : banyanDBIndexRule.indexType(), banyanDBMeasureField != null, - analyzer == null ? null : analyzer.analyzer() + analyzer == null ? null : analyzer.analyzer(), + enableSort != null ); if (topNAggregation != null) { @@ -251,7 +254,7 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula elasticSearchExtension, banyanDBExtension ); - if (banyanDBExtension.isShardingKey()) { + if (banyanDBExtension.isSeriesID()) { checker.accept(modelName, modelColumn); } @@ -321,14 +324,14 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula return models; } - private static class ShardingKeyChecker { + private static class SeriesIDChecker { private final ArrayList keys = new ArrayList<>(); /** - * @throws IllegalStateException if sharding key indices are conflicting. + * @throws IllegalStateException if seriesID indices are conflicting. */ private void accept(String modelName, ModelColumn modelColumn) throws IllegalStateException { - final int idx = modelColumn.getBanyanDBExtension().getShardingKeyIdx(); + final int idx = modelColumn.getBanyanDBExtension().getSeriesIDIdx(); while (idx + 1 > keys.size()) { keys.add(null); } @@ -337,21 +340,21 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula throw new IllegalStateException( modelName + "'s " + "Column [" + exist.getColumnName() + "] and column [" + modelColumn.getColumnName() - + " are conflicting with sharding key index=" + modelColumn.getBanyanDBExtension() - .getShardingKeyIdx()); + + " are conflicting with seriesID index=" + modelColumn.getBanyanDBExtension() + .getSeriesIDIdx()); } keys.set(idx, modelColumn); } /** * @param modelName model name of the entity - * @throws IllegalStateException if sharding key indices are not continuous + * @throws IllegalStateException if seriesIDs indices are not continuous */ private void check(String modelName) throws IllegalStateException { for (int i = 0; i < keys.size(); i++) { final ModelColumn modelColumn = keys.get(i); if (modelColumn == null) { - throw new IllegalStateException("Sharding key index=" + i + " is missing in " + modelName); + throw new IllegalStateException("seriesID index=" + i + " is missing in " + modelName); } } } diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java index e57b968196..32fa3edab5 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java @@ -49,7 +49,10 @@ public class ModelColumnTest { new SQLDatabaseExtension(), new ElasticSearchExtension( ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, null, false, false, true), - new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE) + new BanyanDBExtension( + -1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, + BanyanDB.MatchQuery.AnalyzerType.SIMPLE, true + ) ); Assertions.assertTrue(column.isStorageOnly()); Assertions.assertEquals("abc", column.getColumnName().getName()); @@ -58,7 +61,10 @@ public class ModelColumnTest { false, false, true, 200, new SQLDatabaseExtension(), new ElasticSearchExtension(ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, null, false, false, true), - new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB. MatchQuery.AnalyzerType.SIMPLE) + new BanyanDBExtension( + -1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, + BanyanDB.MatchQuery.AnalyzerType.SIMPLE, true + ) ); Assertions.assertTrue(column.isStorageOnly()); Assertions.assertEquals("abc", column.getColumnName().getName()); @@ -68,7 +74,7 @@ public class ModelColumnTest { false, false, true, 200, new SQLDatabaseExtension(), new ElasticSearchExtension(ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, null, false, false, true), - new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE) + new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE, true) ); Assertions.assertFalse(column.isStorageOnly()); Assertions.assertEquals("abc", column.getColumnName().getName()); @@ -82,7 +88,10 @@ public class ModelColumnTest { new SQLDatabaseExtension(), new ElasticSearchExtension( ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, "abc", false, false, true), - new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE) + new BanyanDBExtension( + -1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, + BanyanDB.MatchQuery.AnalyzerType.SIMPLE, true + ) ); }); } @@ -95,7 +104,9 @@ public class ModelColumnTest { new SQLDatabaseExtension(), new ElasticSearchExtension( ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, "abc", false, false, true), - new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE) + new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, + BanyanDB.MatchQuery.AnalyzerType.SIMPLE, true + ) ); }); } @@ -110,7 +121,23 @@ public class ModelColumnTest { ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, "abc", false, false, true), new BanyanDBExtension( -1, false, BanyanDB.IndexRule.IndexType.INVERTED, false, - BanyanDB.MatchQuery.AnalyzerType.SIMPLE + BanyanDB.MatchQuery.AnalyzerType.SIMPLE, true + ) + ); + }); + } + + @Test + public void testConflictDefinitionEnableSort() { + assertThrows(IllegalArgumentException.class, () -> { + new ModelColumn(new ColumnName(c), String.class, String.class, + true, false, false, 200, + new SQLDatabaseExtension(), + new ElasticSearchExtension( + ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, "abc", false, false, true), + new BanyanDBExtension( + -1, false, BanyanDB.IndexRule.IndexType.INVERTED, true, + null, true ) ); }); diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java index c5214b986c..67ff29b655 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java @@ -84,16 +84,16 @@ public enum MetadataRegistry { Schema.SchemaBuilder schemaBuilder = Schema.builder().metadata(schemaMetadata); Map modelColumnMap = model.getColumns().stream() .collect(Collectors.toMap(modelColumn -> modelColumn.getColumnName().getStorageName(), Function.identity())); - // parse and set sharding keys - List shardingColumns = parseEntityNames(modelColumnMap); - if (shardingColumns.isEmpty()) { - throw new IllegalStateException("sharding keys of model[stream." + model.getName() + "] must not be empty"); + // parse and set seriesIDs + List seriesIDColumns = parseEntityNames(modelColumnMap); + if (seriesIDColumns.isEmpty()) { + throw new IllegalStateException("seriesID of model[stream." + model.getName() + "] must not be empty"); } // parse tag metadata // this can be used to build both // 1) a list of TagFamilySpec, // 2) a list of IndexRule, - List tags = parseTagMetadata(model, schemaBuilder, shardingColumns, schemaMetadata.group); + List tags = parseTagMetadata(model, schemaBuilder, seriesIDColumns, schemaMetadata.group); List tagFamilySpecs = schemaMetadata.extractTagFamilySpec(tags, false); // iterate over tagFamilySpecs to save tag names for (final TagFamilySpec tagFamilySpec : tagFamilySpecs) { @@ -115,7 +115,7 @@ public enum MetadataRegistry { final Stream.Builder builder = Stream.newBuilder(); builder.setMetadata(BanyandbCommon.Metadata.newBuilder().setGroup(schemaMetadata.getGroup()) .setName(schemaMetadata.name())); - builder.setEntity(BanyandbDatabase.Entity.newBuilder().addAllTagNames(shardingColumns)); + builder.setEntity(BanyandbDatabase.Entity.newBuilder().addAllTagNames(seriesIDColumns)); builder.addAllTagFamilies(tagFamilySpecs); registry.put(schemaMetadata.name(), schemaBuilder.build()); @@ -127,16 +127,16 @@ public enum MetadataRegistry { Schema.SchemaBuilder schemaBuilder = Schema.builder().metadata(schemaMetadata); Map modelColumnMap = model.getColumns().stream() .collect(Collectors.toMap(modelColumn -> modelColumn.getColumnName().getStorageName(), Function.identity())); - // parse and set sharding keys - List shardingColumns = parseEntityNames(modelColumnMap); - if (shardingColumns.isEmpty()) { + // parse and set seriesIDs + List seriesIDColumns = parseEntityNames(modelColumnMap); + if (seriesIDColumns.isEmpty()) { throw new StorageException("model " + model.getName() + " doesn't contain series id"); } // parse tag metadata // this can be used to build both // 1) a list of TagFamilySpec, // 2) a list of IndexRule, - MeasureMetadata tagsAndFields = parseTagAndFieldMetadata(model, schemaBuilder, shardingColumns, schemaMetadata.group); + MeasureMetadata tagsAndFields = parseTagAndFieldMetadata(model, schemaBuilder, seriesIDColumns, schemaMetadata.group); List tagFamilySpecs = schemaMetadata.extractTagFamilySpec(tagsAndFields.tags, model.getBanyanDBModelExtension().isStoreIDTag()); // iterate over tagFamilySpecs to save tag names for (final TagFamilySpec tagFamilySpec : tagFamilySpecs) { @@ -150,14 +150,14 @@ public enum MetadataRegistry { .collect(Collectors.toList()); if (model.getBanyanDBModelExtension().isStoreIDTag()) { - indexRules.add(indexRule(schemaMetadata.group, BanyanDBConverter.ID, null)); + indexRules.add(indexRule(schemaMetadata.group, BanyanDBConverter.ID, false, null)); } final Measure.Builder builder = Measure.newBuilder(); builder.setMetadata(BanyandbCommon.Metadata.newBuilder().setGroup(schemaMetadata.getGroup()) .setName(schemaMetadata.name())); builder.setInterval(downSamplingDuration(model.getDownsampling()).format()); - builder.setEntity(BanyandbDatabase.Entity.newBuilder().addAllTagNames(shardingColumns)); + builder.setEntity(BanyandbDatabase.Entity.newBuilder().addAllTagNames(seriesIDColumns)); builder.addAllTagFamilies(tagFamilySpecs); if (model.getBanyanDBModelExtension().isIndexMode()) { builder.setIndexMode(true); @@ -290,10 +290,12 @@ public enum MetadataRegistry { } } - IndexRule indexRule(String group, String tagName, BanyanDB.MatchQuery.AnalyzerType analyzer) { + IndexRule indexRule(String group, String tagName, boolean enableSort, BanyanDB.MatchQuery.AnalyzerType analyzer) { IndexRule.Builder builder = IndexRule.newBuilder() .setMetadata(Metadata.newBuilder().setName(tagName).setGroup(group)) .setType(IndexRule.Type.TYPE_INVERTED).addTags(tagName); + // *Notice*: here is a reverse logic, if enableSort is true, then setNoSort is false + builder.setNoSort(!enableSort); if (analyzer != null) { switch (analyzer) { case KEYWORD: @@ -316,20 +318,20 @@ public enum MetadataRegistry { } /** - * Parse sharding keys from the {@link Model} + * Parse SeriesID from the {@link Model} * * @param modelColumnMap the mapping between column storageName and {@link ModelColumn} * @return a list of column names in strict order */ List parseEntityNames(Map modelColumnMap) { - List shardingColumns = new ArrayList<>(); + List seriesIDColumns = new ArrayList<>(); for (final ModelColumn col : modelColumnMap.values()) { - if (col.getBanyanDBExtension().isShardingKey()) { - shardingColumns.add(col); + if (col.getBanyanDBExtension().isSeriesID()) { + seriesIDColumns.add(col); } } - return shardingColumns.stream() - .sorted(Comparator.comparingInt(col -> col.getBanyanDBExtension().getShardingKeyIdx())) + return seriesIDColumns.stream() + .sorted(Comparator.comparingInt(col -> col.getBanyanDBExtension().getSeriesIDIdx())) .map(col -> col.getColumnName().getName()) .collect(Collectors.toList()); } @@ -341,7 +343,7 @@ public enum MetadataRegistry { * * @since 9.4.0 Skip {@link Record#TIME_BUCKET} */ - List parseTagMetadata(Model model, Schema.SchemaBuilder builder, List shardingColumns, String group) { + List parseTagMetadata(Model model, Schema.SchemaBuilder builder, List seriesIDColumns, String group) { List tagMetadataList = new ArrayList<>(); for (final ModelColumn col : model.getColumns()) { final String columnStorageName = col.getColumnName().getStorageName(); @@ -352,9 +354,12 @@ public enum MetadataRegistry { builder.spec(columnStorageName, new ColumnSpec(ColumnType.TAG, col.getType())); String colName = col.getColumnName().getStorageName(); if (col.getBanyanDBExtension().shouldIndex()) { - if (!shardingColumns.contains(colName) || null != col.getBanyanDBExtension().getAnalyzer()) { + if (!seriesIDColumns.contains(colName) || null != col.getBanyanDBExtension().getAnalyzer()) { tagMetadataList.add(new TagMetadata( - indexRule(group, tagSpec.getName(), col.getBanyanDBExtension().getAnalyzer()), tagSpec)); + indexRule( + group, tagSpec.getName(), col.getBanyanDBExtension().isEnableSort(), + col.getBanyanDBExtension().getAnalyzer() + ), tagSpec)); } else { tagMetadataList.add(new TagMetadata(null, tagSpec)); } @@ -381,7 +386,7 @@ public enum MetadataRegistry { * * @since 9.4.0 Skip {@link Metrics#TIME_BUCKET} */ - MeasureMetadata parseTagAndFieldMetadata(Model model, Schema.SchemaBuilder builder, List shardingColumns, String group) { + MeasureMetadata parseTagAndFieldMetadata(Model model, Schema.SchemaBuilder builder, List seriesIDColumns, String group) { // skip metric MeasureMetadata.MeasureMetadataBuilder result = MeasureMetadata.builder(); for (final ModelColumn col : model.getColumns()) { @@ -399,9 +404,12 @@ public enum MetadataRegistry { String colName = col.getColumnName().getStorageName(); if (col.getBanyanDBExtension().shouldIndex()) { - if (!shardingColumns.contains(colName) || null != col.getBanyanDBExtension().getAnalyzer()) { + if (!seriesIDColumns.contains(colName) || null != col.getBanyanDBExtension().getAnalyzer()) { result.tag(new TagMetadata( - indexRule(group, tagSpec.getName(), col.getBanyanDBExtension().getAnalyzer()), tagSpec)); + indexRule( + group, tagSpec.getName(), col.getBanyanDBExtension().isEnableSort(), + col.getBanyanDBExtension().getAnalyzer() + ), tagSpec)); } else { result.tag(new TagMetadata(null, tagSpec)); } diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBMetricsDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBMetricsDAO.java index f736fc5ece..e2abb01b17 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBMetricsDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBMetricsDAO.java @@ -77,7 +77,7 @@ public class BanyanDBMetricsDAO extends AbstractBanyanDBDAO implements IMetricsD if (ext == null) { return; } - if (ext.isShardingKey()) { + if (ext.isSeriesID()) { seriesIDColumns.put(c.getColumnName().getName(), new ArrayList<>()); } });