From 6d9decddceaffd2f10a26cc3d818c6bac4e94ba2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E6=99=9F=20Wu=20Sheng?= Date: Tue, 12 Apr 2022 08:21:44 +0800 Subject: [PATCH] Support BanyanDB global index for entities (#8852) * Move ElasticSearch and BanyanDB specific configurations out of column. * Support BanyanDB global index for entities. Log and Segment record entities declare this new feature. * Remove unnecessary analyzer settings in columns of templates. Many were added due to analyzer's default value. * Move SQLDatabase(H2/MySQL/PostgreSQL) specific configurations out of column. --- docs/en/changes/changes.md | 10 +- .../apache/skywalking/oal/rt/OALRuntime.java | 4 +- .../oap/server/core/alarm/AlarmRecord.java | 4 +- .../manual/endpoint/EndpointTraffic.java | 4 +- .../manual/log/AbstractLogRecord.java | 14 ++- .../networkalias/NetworkAddressAlias.java | 4 +- .../EndpointRelationServerSideMetrics.java | 4 +- ...viceInstanceRelationServerSideMetrics.java | 4 +- .../ServiceRelationServerSideMetrics.java | 4 +- .../manual/segment/SegmentRecord.java | 12 ++- .../manual/service/ServiceTraffic.java | 4 +- .../meter/function/HistogramFunction.java | 4 +- .../meter/function/PercentileFunction.java | 4 +- .../meter/function/avg/AvgFunction.java | 4 +- .../function/avg/AvgHistogramFunction.java | 4 +- .../avg/AvgHistogramPercentileFunction.java | 4 +- .../function/avg/AvgLabeledFunction.java | 4 +- .../meter/function/latest/LatestFunction.java | 4 +- .../meter/function/sum/SumFunction.java | 4 +- .../errorlog/BrowserErrorLogRecord.java | 4 +- .../ebpf/storage/EBPFProfilingDataRecord.java | 4 +- .../annotation/BanyanDBGlobalIndex.java | 46 +++++++++ .../annotation/BanyanDBShardingKey.java | 59 +++++++++++ .../core/storage/annotation/Column.java | 56 ----------- .../annotation/ElasticSearchMatchQuery.java | 64 ++++++++++++ .../core/storage/model/BanyanDBExtension.java | 67 +++++++++++++ .../storage/model/ElasticSearchExtension.java | 42 ++++++++ .../core/storage/model/ExtraQueryIndex.java | 56 ----------- .../oap/server/core/storage/model/Model.java | 3 - .../core/storage/model/ModelColumn.java | 37 ++++--- .../storage/model/SQLDatabaseExtension.java | 70 +++++++++++++ .../core/storage/model/StorageModels.java | 97 +++++++++++++------ .../core/storage/model/ModelColumnTest.java | 35 ++++--- ...est.java => SQLDatabaseExtensionTest.java} | 8 +- .../core/storage/model/StorageModelsTest.java | 4 +- .../elasticsearch/base/AnalyzerSetting.java | 12 +-- .../base/StorageEsInstaller.java | 12 ++- .../base/TimeSeriesUtilsTest.java | 6 +- .../jdbc/mysql/MySQLTableInstaller.java | 33 ++++--- .../plugin/zipkin/ZipkinSpanRecord.java | 12 ++- 40 files changed, 592 insertions(+), 235 deletions(-) create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBGlobalIndex.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBShardingKey.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/ElasticSearchMatchQuery.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ElasticSearchExtension.java delete mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ExtraQueryIndex.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/SQLDatabaseExtension.java rename oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/{ExtraQueryIndexTest.java => SQLDatabaseExtensionTest.java} (78%) diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 05812ca036..1e3679b17d 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -1,18 +1,24 @@ ## 9.1.0 #### Project + * Upgrade zipkin to 2.23.16. #### OAP Server + * Add component definition(ID=127) for `Apache ShenYu (incubating)`. -* Fix Zipkin receiver: Decode spans error, missing `Layer` for V9 and wrong time bucket for generate Service and Endpoint. +* Fix Zipkin receiver: Decode spans error, missing `Layer` for V9 and wrong time bucket for generate Service and + Endpoint. +* [Refactor] Move SQLDatabase(H2/MySQL/PostgreSQL), ElasticSearch and BanyanDB specific configurations out of column. +* Support BanyanDB global index for entities. Log and Segment record entities declare this new feature. +* Remove unnecessary analyzer settings in columns of templates. Many were added due to analyzer's default value. #### UI + * General service instance: move `Thread Pool` from JVM to Overview, fix `JVM GC Count` calculation. * Add Apache ShenYu (incubating) component LOGO. #### Documentation - All issues and pull requests are [here](https://github.com/apache/skywalking/milestone/128?closed=1) diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java index 4ccf35f460..98b67d02d6 100644 --- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java @@ -71,6 +71,7 @@ import org.apache.skywalking.oap.server.core.source.oal.rt.metrics.MetricClassPa import org.apache.skywalking.oap.server.core.source.oal.rt.metrics.builder.MetricBuilderClassPackageHolder; import org.apache.skywalking.oap.server.core.storage.StorageBuilderFactory; import org.apache.skywalking.oap.server.core.storage.StorageException; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey; import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.library.module.ModuleStartException; import org.apache.skywalking.oap.server.library.util.ResourceUtils; @@ -269,7 +270,8 @@ public class OALRuntime implements OALEngine { } if (field.isID()) { // Add shardingKeyIdx = 0 to column annotation. - columnAnnotation.addMemberValue("shardingKeyIdx", new IntegerMemberValue(constPool, 0)); + Annotation banyanShardingKeyAnnotation = new Annotation(BanyanDBShardingKey.class.getName(), constPool); + banyanShardingKeyAnnotation.addMemberValue("index", new IntegerMemberValue(constPool, 0)); } annotationsAttribute.addAnnotation(columnAnnotation); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java index 8c292dc759..b55585b7ad 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java @@ -29,6 +29,7 @@ import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcess import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; import org.apache.skywalking.oap.server.core.source.ScopeDeclaration; import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery; import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; import org.apache.skywalking.oap.server.core.storage.type.HashMapConverter; @@ -68,7 +69,8 @@ public class AlarmRecord extends Record { private String id1; @Column(columnName = START_TIME) private long startTime; - @Column(columnName = ALARM_MESSAGE, matchQuery = true) + @Column(columnName = ALARM_MESSAGE) + @ElasticSearchMatchQuery private String alarmMessage; @Column(columnName = RULE_NAME) private String ruleName; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java index 432c7503b5..62da3fcfbb 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java @@ -31,6 +31,7 @@ import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProces import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery; import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; @@ -52,7 +53,8 @@ public class EndpointTraffic extends Metrics { private String serviceId; @Setter @Getter - @Column(columnName = NAME, matchQuery = true) + @Column(columnName = NAME) + @ElasticSearchMatchQuery private String name = Const.EMPTY_STRING; @Override diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java index fa405a0906..0ea0ee01ee 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java @@ -25,7 +25,10 @@ import org.apache.skywalking.oap.server.core.UnexpectedException; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; import org.apache.skywalking.oap.server.core.analysis.record.Record; import org.apache.skywalking.oap.server.core.query.type.ContentType; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBGlobalIndex; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey; import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery; import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; import org.apache.skywalking.oap.server.core.storage.type.HashMapConverter; @@ -47,11 +50,13 @@ public abstract class AbstractLogRecord extends Record { @Setter @Getter - @Column(columnName = SERVICE_ID, shardingKeyIdx = 0) + @Column(columnName = SERVICE_ID) + @BanyanDBShardingKey(index = 0) private String serviceId; @Setter @Getter - @Column(columnName = SERVICE_INSTANCE_ID, shardingKeyIdx = 1) + @Column(columnName = SERVICE_INSTANCE_ID) + @BanyanDBShardingKey(index = 1) private String serviceInstanceId; @Setter @Getter @@ -60,10 +65,12 @@ public abstract class AbstractLogRecord extends Record { @Setter @Getter @Column(columnName = TRACE_ID, length = 150) + @BanyanDBGlobalIndex(extraFields = {}) private String traceId; @Setter @Getter @Column(columnName = TRACE_SEGMENT_ID, length = 150) + @BanyanDBGlobalIndex(extraFields = {SPAN_ID}) private String traceSegmentId; @Setter @Getter @@ -75,7 +82,8 @@ public abstract class AbstractLogRecord extends Record { private int contentType = ContentType.NONE.value(); @Setter @Getter - @Column(columnName = CONTENT, length = 1_000_000, matchQuery = true, analyzer = Column.AnalyzerType.OAP_LOG_ANALYZER) + @Column(columnName = CONTENT, length = 1_000_000) + @ElasticSearchMatchQuery(analyzer = ElasticSearchMatchQuery.AnalyzerType.OAP_LOG_ANALYZER) private String content; @Setter @Getter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/networkalias/NetworkAddressAlias.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/networkalias/NetworkAddressAlias.java index c07c3c33f8..bddf8bda14 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/networkalias/NetworkAddressAlias.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/networkalias/NetworkAddressAlias.java @@ -28,6 +28,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor; import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData; import org.apache.skywalking.oap.server.core.source.ScopeDeclaration; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey; import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; @@ -51,7 +52,8 @@ public class NetworkAddressAlias extends Metrics { @Setter @Getter - @Column(columnName = ADDRESS, shardingKeyIdx = 0) + @Column(columnName = ADDRESS) + @BanyanDBShardingKey(index = 0) private String address; @Setter @Getter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/endpoint/EndpointRelationServerSideMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/endpoint/EndpointRelationServerSideMetrics.java index f07aada393..08863f11b0 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/endpoint/EndpointRelationServerSideMetrics.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/endpoint/EndpointRelationServerSideMetrics.java @@ -27,6 +27,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor; import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey; import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; @@ -58,7 +59,8 @@ public class EndpointRelationServerSideMetrics extends Metrics { private int componentId; @Setter @Getter - @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0) + @Column(columnName = ENTITY_ID, length = 512) + @BanyanDBShardingKey(index = 0) private String entityId; @Override diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java index 484f5de214..dd674cef84 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java @@ -27,6 +27,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor; import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey; import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; @@ -68,7 +69,8 @@ public class ServiceInstanceRelationServerSideMetrics extends Metrics { private int componentId; @Setter @Getter - @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0) + @Column(columnName = ENTITY_ID, length = 512) + @BanyanDBShardingKey(index = 0) private String entityId; @Override diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java index b68ddddb69..435ab6997a 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java @@ -27,6 +27,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor; import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey; import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; @@ -58,7 +59,8 @@ public class ServiceRelationServerSideMetrics extends Metrics { private int componentId; @Setter @Getter - @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0) + @Column(columnName = ENTITY_ID, length = 512) + @BanyanDBShardingKey(index = 0) private String entityId; @Override diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java index 90d13792c8..d25091779c 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java @@ -26,6 +26,8 @@ import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; import org.apache.skywalking.oap.server.core.analysis.record.Record; import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBGlobalIndex; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey; import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset; import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; @@ -56,14 +58,17 @@ public class SegmentRecord extends Record { @Setter @Getter @Column(columnName = TRACE_ID, length = 150) + @BanyanDBGlobalIndex(extraFields = {}) private String traceId; @Setter @Getter - @Column(columnName = SERVICE_ID, shardingKeyIdx = 0) + @Column(columnName = SERVICE_ID) + @BanyanDBShardingKey(index = 0) private String serviceId; @Setter @Getter - @Column(columnName = SERVICE_INSTANCE_ID, shardingKeyIdx = 1) + @Column(columnName = SERVICE_INSTANCE_ID) + @BanyanDBShardingKey(index = 1) private String serviceInstanceId; @Setter @Getter @@ -79,7 +84,8 @@ public class SegmentRecord extends Record { private int latency; @Setter @Getter - @Column(columnName = IS_ERROR, shardingKeyIdx = 2) + @Column(columnName = IS_ERROR) + @BanyanDBShardingKey(index = 2) private int isError; @Setter @Getter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java index 0d56840c18..6ad15a8f4b 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java @@ -31,6 +31,7 @@ import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProces import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery; import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; @@ -60,7 +61,8 @@ public class ServiceTraffic extends Metrics { @Setter @Getter - @Column(columnName = NAME, matchQuery = true) + @Column(columnName = NAME) + @ElasticSearchMatchQuery private String name = Const.EMPTY_STRING; @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/HistogramFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/HistogramFunction.java index e0ceaa962e..0974829ec7 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/HistogramFunction.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/HistogramFunction.java @@ -31,6 +31,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable; import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.query.type.Bucket; import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey; import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; @@ -48,7 +49,8 @@ public abstract class HistogramFunction extends Meter implements AcceptableValue @Setter @Getter - @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0) + @Column(columnName = ENTITY_ID, length = 512) + @BanyanDBShardingKey(index = 0) private String entityId; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java index 70ec29ca14..8d5760e4ce 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java @@ -37,6 +37,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.MultiIntValuesHold import org.apache.skywalking.oap.server.core.analysis.metrics.PercentileMetrics; import org.apache.skywalking.oap.server.core.query.type.Bucket; import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey; import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; @@ -55,7 +56,8 @@ public abstract class PercentileFunction extends Meter implements AcceptableValu @Setter @Getter - @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0) + @Column(columnName = ENTITY_ID, length = 512) + @BanyanDBShardingKey(index = 0) private String entityId; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgFunction.java index 6476746865..7b1bb9d97a 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgFunction.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgFunction.java @@ -36,6 +36,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.Entranc import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.SourceFrom; import org.apache.skywalking.oap.server.core.query.sql.Function; import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey; import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; @@ -50,7 +51,8 @@ public abstract class AvgFunction extends Meter implements AcceptableValue @Setter @Getter - @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0) + @Column(columnName = ENTITY_ID, length = 512) + @BanyanDBShardingKey(index = 0) private String entityId; /** diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java index 0222c971e2..8a88a6e17e 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java @@ -34,6 +34,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable; import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.query.type.Bucket; import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey; import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; @@ -59,7 +60,8 @@ public abstract class AvgHistogramFunction extends Meter implements AcceptableVa @Setter @Getter - @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0) + @Column(columnName = ENTITY_ID, length = 512) + @BanyanDBShardingKey(index = 0) private String entityId; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java index f785c39a81..794f8668d7 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java @@ -43,6 +43,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.analysis.metrics.MultiIntValuesHolder; import org.apache.skywalking.oap.server.core.query.type.Bucket; import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey; import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; @@ -74,7 +75,8 @@ public abstract class AvgHistogramPercentileFunction extends Meter implements Ac @Setter @Getter - @Column(columnName = ENTITY_ID, shardingKeyIdx = 0) + @Column(columnName = ENTITY_ID) + @BanyanDBShardingKey(index = 0) private String entityId; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java index 8721a77cda..7a44ee569d 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java @@ -34,6 +34,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable; import org.apache.skywalking.oap.server.core.analysis.metrics.LabeledValueHolder; import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey; import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; @@ -48,7 +49,8 @@ public abstract class AvgLabeledFunction extends Meter implements AcceptableValu @Setter @Getter - @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0) + @Column(columnName = ENTITY_ID, length = 512) + @BanyanDBShardingKey(index = 0) private String entityId; /** diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/latest/LatestFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/latest/LatestFunction.java index 354988e646..c791422ab5 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/latest/LatestFunction.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/latest/LatestFunction.java @@ -35,6 +35,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.Entranc import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.SourceFrom; import org.apache.skywalking.oap.server.core.query.sql.Function; import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey; import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; @@ -47,7 +48,8 @@ public abstract class LatestFunction extends Meter implements AcceptableValue @Setter @Getter - @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0) + @Column(columnName = ENTITY_ID, length = 512) + @BanyanDBShardingKey(index = 0) private String entityId; @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java index df4e49b435..df12484b0b 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java @@ -23,6 +23,7 @@ import org.apache.skywalking.oap.server.core.analysis.Stream; import org.apache.skywalking.oap.server.core.analysis.record.Record; import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey; import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset; import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; @@ -54,7 +55,8 @@ public class BrowserErrorLogRecord extends Record { @Setter @Getter - @Column(columnName = SERVICE_ID, shardingKeyIdx = 0) + @Column(columnName = SERVICE_ID) + @BanyanDBShardingKey(index = 0) private String serviceId; @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingDataRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingDataRecord.java index dc2433bd7f..98b1519cef 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingDataRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingDataRecord.java @@ -24,6 +24,7 @@ import lombok.Data; import org.apache.skywalking.oap.server.core.analysis.Stream; import org.apache.skywalking.oap.server.core.analysis.record.Record; import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey; import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; @@ -48,7 +49,8 @@ public class EBPFProfilingDataRecord extends Record { public static final String STACKS_BINARY = "dump_binary"; public static final String UPLOAD_TIME = "upload_time"; - @Column(columnName = TASK_ID, length = 600, shardingKeyIdx = 0) + @Column(columnName = TASK_ID, length = 600) + @BanyanDBShardingKey(index = 0) private String taskId; @Column(columnName = SCHEDULE_ID, length = 600) private String scheduleId; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBGlobalIndex.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBGlobalIndex.java new file mode 100644 index 0000000000..a8dc1b761f --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBGlobalIndex.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.storage.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * BanyanDBGlobalIndex declares advanced global index, which are only available in BanyanDB. + * + * Global index should only be considered if a column value has a huge value candidates, but we will need a direct equal + * query without timestamp. + * The typical global index is designed for huge candidate of indexed values, + * such as `trace ID` or `segment ID + span ID` + * + * Only work with {@link Column} + */ +@Target({ElementType.FIELD}) +@Retention(RetentionPolicy.RUNTIME) +public @interface BanyanDBGlobalIndex { + /** + * The current column should be indexed through global index. + * + * @return empty array if only the current column should be indexed in global index. Or list of column names if this + * global index includes multiple columns. + */ + String[] extraFields(); +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBShardingKey.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBShardingKey.java new file mode 100644 index 0000000000..8c8e21a172 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBShardingKey.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.storage.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Sharding key is used to group time series data per metric of one entity in one place (same sharding and/or same + * row for column-oriented database). + * For example, + * ServiceA's traffic gauge, service call per minute, includes following timestamp values, then it should be sharded + * by service ID + * [ServiceA(encoded ID): 01-28 18:30 values-1, 01-28 18:31 values-2, 01-28 18:32 values-3, 01-28 18:32 values-4] + * + * BanyanDB is the 1st storage implementation supporting this. It would make continuous time series metrics stored + * closely and compressed better. + * + * 1. One entity could have multiple sharding keys + * 2. If no column is appointed for this, {@link org.apache.skywalking.oap.server.core.storage.StorageData#id} + * would be used by the storage implementation accordingly. + * + * NOTICE, this sharding concept is NOT just for splitting data into different database instances or physical + * files. + * + * Only work with {@link Column} + * + * @return non-negative if this column be used for sharding. -1 means not as a sharding key + * @since 9.1.0 created as a new annotation. + * @since 9.0.0 added in {@link Column} + */ +@Target({ElementType.FIELD}) +@Retention(RetentionPolicy.RUNTIME) +public @interface BanyanDBShardingKey { + /** + * Relative entity tag + * + * @return index, from zero. + */ + int index() default -1; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java index 077fdfe6dd..44d2305024 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java @@ -48,11 +48,6 @@ public @interface Column { */ int defaultValue() default 0; - /** - * Match query means using analyzer(if storage have) to do key word match query. - */ - boolean matchQuery() default false; - /** * The column is just saved, never used in query. */ @@ -94,57 +89,6 @@ public @interface Column { */ ValueDataType dataType() default ValueDataType.NOT_VALUE; - /** - * The storage analyzer mode. - * - * @since 8.4.0 - */ - AnalyzerType analyzer() default AnalyzerType.OAP_ANALYZER; - - /** - * Sharding key is used to group time series data per metric of one entity in one place (same sharding and/or same - * row for column-oriented database). - * For example, - * ServiceA's traffic gauge, service call per minute, includes following timestamp values, then it should be sharded - * by service ID - * [ServiceA(encoded ID): 01-28 18:30 values-1, 01-28 18:31 values-2, 01-28 18:32 values-3, 01-28 18:32 values-4] - * - * BanyanDB is the 1st storage implementation supporting this. It would make continuous time series metrics stored - * closely and compressed better. - * - * 1. One entity could have multiple sharding keys - * 2. If no column is appointed for this, {@link org.apache.skywalking.oap.server.core.storage.StorageData#id} - * would be used by the storage implementation accordingly. - * - * NOTICE, this sharding concept is NOT just for splitting data into different database instances or physical - * files. - * - * @return non-negative if this column be used for sharding. -1 means not as a sharding key - * @since 9.0.0 - */ - int shardingKeyIdx() default -1; - - /** - * The analyzer declares the text analysis mode. - */ - enum AnalyzerType { - /** - * The default analyzer. - */ - OAP_ANALYZER("oap_analyzer"), - /** - * The log analyzer. - */ - OAP_LOG_ANALYZER("oap_log_analyzer"); - - @Getter - private final String name; - - AnalyzerType(final String name) { - this.name = name; - } - } - /** * ValueDataType represents the data structure of value column. The persistent way of the value column determine the * available ways to query the data. diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/ElasticSearchMatchQuery.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/ElasticSearchMatchQuery.java new file mode 100644 index 0000000000..19084754de --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/ElasticSearchMatchQuery.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.storage.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import lombok.Getter; + +/** + * Match query is designed for ElasticSearch match query with specific analyzer. It is a fuzzy query implementation + * powered by analyzer. + * + * @since 9.1.0 This used to be {@link Column}'s matchQuery and analyzer attributes. + */ +@Target({ElementType.FIELD}) +@Retention(RetentionPolicy.RUNTIME) +public @interface ElasticSearchMatchQuery { + /** + * The storage analyzer mode. + * + * @since 9.1.0 created as a new annotation. + * @since 8.4.0 added in {@link Column} + */ + AnalyzerType analyzer() default AnalyzerType.OAP_ANALYZER; + + /** + * The analyzer declares the text analysis mode. + */ + enum AnalyzerType { + /** + * The default analyzer. + */ + OAP_ANALYZER("oap_analyzer"), + /** + * The log analyzer. + */ + OAP_LOG_ANALYZER("oap_log_analyzer"); + + @Getter + private final String name; + + AnalyzerType(final String name) { + this.name = name; + } + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java new file mode 100644 index 0000000000..2a8c82ee26 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.storage.model; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * BanyanDBExtension represents extra metadata for columns, but specific for BanyanDB usages. + * + * @since 9.1.0 + */ +@Getter +@RequiredArgsConstructor +public class BanyanDBExtension { + /** + * Sharding key is used to group time series data per metric of one entity. See {@link + * org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey#index()}. + * + * @since 9.1.0 moved into BanyanDBExtension + * @since 9.0.0 added into {@link ModelColumn} + */ + private final int shardingKeyIdx; + + /** + * Global index column values has 3 conditions + * 1. NULL, this column should be as a global index. + * 2. Empty array(declared by @BanyanDBGlobalIndex(extraFields = {}) in codes) represents this single column should + * be a global index. + * 3. Not empty array(declared by @BanyanDBGlobalIndex(extraFields = {"col1", "col2"}) in codes) represents this + * column and other declared columns should be as a global index. The values of these columns should be joint by + * underline(_) + * + * @since 9.1.0 + */ + private final String[] globalIndexColumns; + + /** + * @return true if this column is a part of sharding key + */ + public boolean isShardingKey() { + return this.shardingKeyIdx > -1; + } + + /** + * @return null or array, see {@link #globalIndexColumns} for more details. + */ + public boolean isGlobalIndexing() { + return globalIndexColumns != null; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ElasticSearchExtension.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ElasticSearchExtension.java new file mode 100644 index 0000000000..4ec91d990b --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ElasticSearchExtension.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.storage.model; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery; + +/** + * ElasticSearchExtension represents extra metadata for columns, but specific for ElasticSearch usages. + * + * @since 9.1.0 + */ +@Getter +@RequiredArgsConstructor +public class ElasticSearchExtension { + /** + * The analyzer policy appointed to fuzzy query, especially for ElasticSearch. + * When it is null, it means no need to build match query, no `copy_to` column, and no analyzer assigned. + */ + private final ElasticSearchMatchQuery.AnalyzerType analyzer; + + public boolean needMatchQuery() { + return analyzer != null; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ExtraQueryIndex.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ExtraQueryIndex.java deleted file mode 100644 index d5f8fa8b9d..0000000000 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ExtraQueryIndex.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.server.core.storage.model; - -import lombok.Getter; -import org.apache.skywalking.oap.server.library.util.CollectionUtils; - -/** - * The extra query index if the storage could support this mode. Many NO-SQL support one column index only, in that - * case, this could be ignored in the implementation level. - */ -@Getter -public class ExtraQueryIndex { - private String[] columns; - - public ExtraQueryIndex(String mainColumn, final String[] withColumns) { - if (CollectionUtils.isNotEmpty(withColumns)) { - columns = new String[withColumns.length + 1]; - columns[0] = mainColumn; - System.arraycopy(withColumns, 0, columns, 1, withColumns.length); - } else { - throw new IllegalArgumentException("ExtraQueryIndex required withColumns as a not empty list."); - } - - } - - /** - * Keep the same name replacement as {@link ColumnName#overrideName(String, String)} - * - * @param oldName to be replaced. - * @param newName to use in the storage level. - */ - public void overrideName(String oldName, String newName) { - for (int i = 0; i < columns.length; i++) { - if (columns[i].equals(oldName)) { - columns[i] = newName; - } - } - } -} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/Model.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/Model.java index b37375688f..1550e40ccf 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/Model.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/Model.java @@ -31,7 +31,6 @@ import org.apache.skywalking.oap.server.core.analysis.DownSampling; public class Model { private final String name; private final List columns; - private final List extraQueryIndices; private final int scopeId; private final DownSampling downsampling; private final boolean record; @@ -42,7 +41,6 @@ public class Model { public Model(final String name, final List columns, - final List extraQueryIndices, final int scopeId, final DownSampling downsampling, final boolean record, @@ -51,7 +49,6 @@ public class Model { boolean timeRelativeID) { this.name = name; this.columns = columns; - this.extraQueryIndices = extraQueryIndices; this.scopeId = scopeId; this.downsampling = downsampling; this.isTimeSeries = !DownSampling.None.equals(downsampling); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java index 9d381f19ae..b5dbfbe739 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java @@ -22,7 +22,6 @@ import java.lang.reflect.Type; import lombok.Getter; import lombok.ToString; import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable; -import org.apache.skywalking.oap.server.core.storage.annotation.Column; @Getter @ToString @@ -30,7 +29,6 @@ public class ModelColumn { private final ColumnName columnName; private final Class type; private final Type genericType; - private final boolean matchQuery; /** * Storage this column for query result, but can't be as a condition . Conflict with {@link #indexOnly} */ @@ -45,33 +43,42 @@ public class ModelColumn { * The max length of column value for length sensitive database. */ private final int length; + /** - * The analyzer policy appointed to fuzzy query, especially for ElasticSearch - */ - private final Column.AnalyzerType analyzer; - /** - * Sharding key is used to group time series data per metric of one entity. See {@link Column#shardingKeyIdx()}. + * Hold configurations especially for SQL Database, such as MySQL, H2, PostgreSQL * - * @since 9.0.0 + * @since 9.1.0 */ - private int shardingKeyIdx; + private SQLDatabaseExtension sqlDatabaseExtension; + /** + * Hold configurations especially for ElasticSearch + * + * @since 9.1.0 + */ + private ElasticSearchExtension elasticSearchExtension; + /** + * Hold configurations especially for BanyanDB relevant + * + * @since 9.1.0 + */ + private BanyanDBExtension banyanDBExtension; public ModelColumn(ColumnName columnName, Class type, Type genericType, - boolean matchQuery, boolean storageOnly, boolean indexOnly, boolean isValue, int length, - Column.AnalyzerType analyzer, - int shardingKeyIdx) { + SQLDatabaseExtension sqlDatabaseExtension, + ElasticSearchExtension elasticSearchExtension, + BanyanDBExtension banyanDBExtension) { this.columnName = columnName; this.type = type; this.genericType = genericType; - this.matchQuery = matchQuery; this.length = length; - this.analyzer = analyzer; + this.sqlDatabaseExtension = sqlDatabaseExtension; + this.elasticSearchExtension = elasticSearchExtension; /* * byte[] and {@link IntKeyLongValueHashMap} could never be query. */ @@ -90,6 +97,6 @@ public class ModelColumn { "The column " + columnName + " can't be defined as both indexOnly and storageOnly."); } this.indexOnly = indexOnly; - this.shardingKeyIdx = shardingKeyIdx; + this.banyanDBExtension = banyanDBExtension; } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/SQLDatabaseExtension.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/SQLDatabaseExtension.java new file mode 100644 index 0000000000..c7df66d4d5 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/SQLDatabaseExtension.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.storage.model; + +import java.util.ArrayList; +import java.util.List; +import lombok.Getter; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; + +/** + * @since 9.0.0 + */ +@Getter +public class SQLDatabaseExtension { + private final List indices = new ArrayList<>(5); + + public void appendIndex(MultiColumnsIndex index) { + indices.add(index); + } + + /** + * The multiple columns index if the storage could support this mode. Many NO-SQL support one column index only, in that + * case, this could be ignored in the implementation level. + */ + @Getter + public static class MultiColumnsIndex { + private String[] columns; + + public MultiColumnsIndex(String mainColumn, final String[] withColumns) { + if (CollectionUtils.isNotEmpty(withColumns)) { + columns = new String[withColumns.length + 1]; + columns[0] = mainColumn; + System.arraycopy(withColumns, 0, columns, 1, withColumns.length); + } else { + throw new IllegalArgumentException("ExtraQueryIndex required withColumns as a not empty list."); + } + + } + + /** + * Keep the same name replacement as {@link ColumnName#overrideName(String, String)} + * + * @param oldName to be replaced. + * @param newName to use in the storage level. + */ + public void overrideName(String oldName, String newName) { + for (int i = 0; i < columns.length; i++) { + if (columns[i].equals(oldName)) { + columns[i] = newName; + } + } + } + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java index 8aebfb1dc4..e71aef4408 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java @@ -27,7 +27,10 @@ import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.oap.server.core.analysis.FunctionCategory; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; import org.apache.skywalking.oap.server.core.storage.StorageException; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBGlobalIndex; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey; import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery; import org.apache.skywalking.oap.server.core.storage.annotation.MultipleQueryUnifiedIndex; import org.apache.skywalking.oap.server.core.storage.annotation.QueryUnifiedIndex; import org.apache.skywalking.oap.server.core.storage.annotation.Storage; @@ -56,14 +59,16 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula DefaultScopeDefine.nameOf(scopeId); List modelColumns = new ArrayList<>(); - List extraQueryIndices = new ArrayList<>(); ShardingKeyChecker checker = new ShardingKeyChecker(); - retrieval(aClass, storage.getModelName(), modelColumns, extraQueryIndices, scopeId, checker); + retrieval(aClass, storage.getModelName(), modelColumns, scopeId, checker); checker.check(storage.getModelName()); Model model = new Model( - storage.getModelName(), modelColumns, extraQueryIndices, scopeId, - storage.getDownsampling(), record, + storage.getModelName(), + modelColumns, + scopeId, + storage.getDownsampling(), + record, isSuperDatasetModel(aClass), FunctionCategory.uniqueFunctionName(aClass), storage.isTimeRelativeID() @@ -100,7 +105,6 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula private void retrieval(final Class clazz, final String modelName, final List modelColumns, - final List extraQueryIndices, final int scopeId, ShardingKeyChecker checker) { if (log.isDebugEnabled()) { @@ -129,13 +133,55 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula } } - final ModelColumn modelColumn = new ModelColumn( - new ColumnName(modelName, column.columnName()), field.getType(), field.getGenericType(), - column.matchQuery(), column.storageOnly(), column.indexOnly(), column.dataType().isValue(), - columnLength, - column.analyzer(), column.shardingKeyIdx() + // SQL Database extension + SQLDatabaseExtension sqlDatabaseExtension = new SQLDatabaseExtension(); + List indexDefinitions = new ArrayList<>(); + if (field.isAnnotationPresent(QueryUnifiedIndex.class)) { + indexDefinitions.add(field.getAnnotation(QueryUnifiedIndex.class)); + } + + if (field.isAnnotationPresent(MultipleQueryUnifiedIndex.class)) { + Collections.addAll(indexDefinitions, field.getAnnotation(MultipleQueryUnifiedIndex.class).value()); + } + + indexDefinitions.forEach(indexDefinition -> { + sqlDatabaseExtension.appendIndex(new SQLDatabaseExtension.MultiColumnsIndex( + column.columnName(), + indexDefinition.withColumns() + )); + }); + + // ElasticSearch extension + final ElasticSearchMatchQuery elasticSearchAnalyzer = field.getAnnotation( + ElasticSearchMatchQuery.class); + ElasticSearchExtension elasticSearchExtension = new ElasticSearchExtension( + elasticSearchAnalyzer == null ? null : elasticSearchAnalyzer.analyzer() ); - if (column.shardingKeyIdx() > -1) { + + // BanyanDB extension + final BanyanDBShardingKey banyanDBShardingKey = field.getAnnotation(BanyanDBShardingKey.class); + final BanyanDBGlobalIndex banyanDBGlobalIndex = field.getAnnotation(BanyanDBGlobalIndex.class); + BanyanDBExtension banyanDBExtension = new BanyanDBExtension( + banyanDBShardingKey == null ? -1 : banyanDBShardingKey.index(), + banyanDBGlobalIndex == null ? null : banyanDBGlobalIndex.extraFields() + ); + + final ModelColumn modelColumn = new ModelColumn( + new ColumnName( + modelName, + column.columnName() + ), + field.getType(), + field.getGenericType(), + column.storageOnly(), + column.indexOnly(), + column.dataType().isValue(), + columnLength, + sqlDatabaseExtension, + elasticSearchExtension, + banyanDBExtension + ); + if (banyanDBExtension.isShardingKey()) { checker.accept(modelName, modelColumn); } @@ -149,25 +195,11 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula column.defaultValue(), scopeId ); } - - List indexDefinitions = new ArrayList<>(); - if (field.isAnnotationPresent(QueryUnifiedIndex.class)) { - indexDefinitions.add(field.getAnnotation(QueryUnifiedIndex.class)); - } - - if (field.isAnnotationPresent(MultipleQueryUnifiedIndex.class)) { - Collections.addAll(indexDefinitions, field.getAnnotation(MultipleQueryUnifiedIndex.class).value()); - } - - indexDefinitions.forEach(indexDefinition -> extraQueryIndices.add(new ExtraQueryIndex( - column.columnName(), - indexDefinition.withColumns() - ))); } } if (Objects.nonNull(clazz.getSuperclass())) { - retrieval(clazz.getSuperclass(), modelName, modelColumns, extraQueryIndices, scopeId, checker); + retrieval(clazz.getSuperclass(), modelName, modelColumns, scopeId, checker); } } @@ -180,8 +212,12 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula private void followColumnNameRules(Model model) { columnNameOverrideRule.forEach((oldName, newName) -> { - model.getColumns().forEach(column -> column.getColumnName().overrideName(oldName, newName)); - model.getExtraQueryIndices().forEach(extraQueryIndex -> extraQueryIndex.overrideName(oldName, newName)); + model.getColumns().forEach(column -> { + column.getColumnName().overrideName(oldName, newName); + column.getSqlDatabaseExtension() + .getIndices() + .forEach(extraQueryIndex -> extraQueryIndex.overrideName(oldName, newName)); + }); }); } @@ -197,7 +233,7 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula * @throws IllegalStateException if sharding key indices are conflicting. */ private void accept(String modelName, ModelColumn modelColumn) throws IllegalStateException { - final int idx = modelColumn.getShardingKeyIdx(); + final int idx = modelColumn.getBanyanDBExtension().getShardingKeyIdx(); while (idx + 1 > keys.size()) { keys.add(null); } @@ -206,7 +242,8 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula throw new IllegalStateException( modelName + "'s " + "Column [" + exist.getColumnName() + "] and column [" + modelColumn.getColumnName() - + " are conflicting with sharding key index=" + modelColumn.getShardingKeyIdx()); + + " are conflicting with sharding key index=" + modelColumn.getBanyanDBExtension() + .getShardingKeyIdx()); } keys.set(idx, modelColumn); } diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java index 90591834b3..be61edb872 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java @@ -19,31 +19,38 @@ package org.apache.skywalking.oap.server.core.storage.model; import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable; -import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery; import org.junit.Assert; import org.junit.Test; public class ModelColumnTest { @Test public void testColumnDefine() { - ModelColumn column = new ModelColumn(new ColumnName("", "abc"), byte[].class, byte[].class, true, + ModelColumn column = new ModelColumn(new ColumnName("", "abc"), byte[].class, byte[].class, false, false, true, 0, - Column.AnalyzerType.OAP_ANALYZER, 0 + new SQLDatabaseExtension(), + new ElasticSearchExtension( + ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER), + new BanyanDBExtension(-1, null) ); Assert.assertEquals(true, column.isStorageOnly()); Assert.assertEquals("abc", column.getColumnName().getName()); - column = new ModelColumn(new ColumnName("", "abc"), DataTable.class, DataTable.class, true, + column = new ModelColumn(new ColumnName("", "abc"), DataTable.class, DataTable.class, false, false, true, 200, - Column.AnalyzerType.OAP_ANALYZER, 0 + new SQLDatabaseExtension(), + new ElasticSearchExtension(ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER), + new BanyanDBExtension(-1, null) ); Assert.assertEquals(true, column.isStorageOnly()); Assert.assertEquals("abc", column.getColumnName().getName()); Assert.assertEquals(200, column.getLength()); - column = new ModelColumn(new ColumnName("", "abc"), String.class, String.class, true, + column = new ModelColumn(new ColumnName("", "abc"), String.class, String.class, false, false, true, 200, - Column.AnalyzerType.OAP_ANALYZER, 0 + new SQLDatabaseExtension(), + new ElasticSearchExtension(ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER), + new BanyanDBExtension(-1, null) ); Assert.assertEquals(false, column.isStorageOnly()); Assert.assertEquals("abc", column.getColumnName().getName()); @@ -52,16 +59,22 @@ public class ModelColumnTest { @Test(expected = IllegalArgumentException.class) public void testConflictDefinition() { ModelColumn column = new ModelColumn(new ColumnName("", "abc"), String.class, String.class, - true, true, false, true, 200, - Column.AnalyzerType.OAP_ANALYZER, 0 + true, false, true, 200, + new SQLDatabaseExtension(), + new ElasticSearchExtension( + ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER), + new BanyanDBExtension(-1, null) ); } @Test(expected = IllegalArgumentException.class) public void testConflictDefinitionIndexOnly() { ModelColumn column = new ModelColumn(new ColumnName("", "abc"), String.class, String.class, - true, true, true, false, 200, - Column.AnalyzerType.OAP_ANALYZER, 0 + true, true, false, 200, + new SQLDatabaseExtension(), + new ElasticSearchExtension( + ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER), + new BanyanDBExtension(-1, null) ); } } diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ExtraQueryIndexTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/SQLDatabaseExtensionTest.java similarity index 78% rename from oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ExtraQueryIndexTest.java rename to oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/SQLDatabaseExtensionTest.java index 7331ac0ae4..1f789a4894 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ExtraQueryIndexTest.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/SQLDatabaseExtensionTest.java @@ -21,10 +21,11 @@ package org.apache.skywalking.oap.server.core.storage.model; import org.junit.Assert; import org.junit.Test; -public class ExtraQueryIndexTest { +public class SQLDatabaseExtensionTest { @Test public void testIndexColumns() { - final ExtraQueryIndex extraQueryIndex = new ExtraQueryIndex("a1", new String[] {"a2"}); + final SQLDatabaseExtension.MultiColumnsIndex extraQueryIndex = new SQLDatabaseExtension.MultiColumnsIndex( + "a1", new String[] {"a2"}); Assert.assertArrayEquals(new String[] { "a1", "a2" @@ -33,6 +34,7 @@ public class ExtraQueryIndexTest { @Test(expected = IllegalArgumentException.class) public void testIllegalIndexColumns() { - ExtraQueryIndex extraQueryIndex = new ExtraQueryIndex("a1", new String[0]); + SQLDatabaseExtension.MultiColumnsIndex extraQueryIndex = new SQLDatabaseExtension.MultiColumnsIndex( + "a1", new String[0]); } } diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/StorageModelsTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/StorageModelsTest.java index 961c015f23..65d734ef47 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/StorageModelsTest.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/StorageModelsTest.java @@ -74,12 +74,10 @@ public class StorageModelsTest { Assert.assertEquals(false, model.getColumns().get(2).isStorageOnly()); Assert.assertEquals(true, model.getColumns().get(3).isStorageOnly()); - final List extraQueryIndices = model.getExtraQueryIndices(); - Assert.assertEquals(3, extraQueryIndices.size()); Assert.assertArrayEquals(new String[] { "column2", "column" - }, extraQueryIndices.get(2).getColumns()); + }, model.getColumns().get(2).getSqlDatabaseExtension().getIndices().get(1).getColumns()); } @Stream(name = "StorageModelsTest", scopeId = -1, builder = TestModel.Builder.class, processor = MetricsStreamProcessor.class) diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/AnalyzerSetting.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/AnalyzerSetting.java index 1de991027b..57e79ad0a4 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/AnalyzerSetting.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/AnalyzerSetting.java @@ -27,7 +27,7 @@ import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.oap.server.core.storage.StorageException; -import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.StorageModuleElasticsearchConfig; @Getter @@ -79,18 +79,18 @@ public class AnalyzerSetting { public enum Generator { OAP_ANALYZER_SETTING_GENERATOR( - Column.AnalyzerType.OAP_ANALYZER, + ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER, config -> new Gson().fromJson(config.getOapAnalyzer(), AnalyzerSetting.class) ), OAP_LOG_ANALYZER_SETTING_GENERATOR( - Column.AnalyzerType.OAP_LOG_ANALYZER, + ElasticSearchMatchQuery.AnalyzerType.OAP_LOG_ANALYZER, config -> new Gson().fromJson(config.getOapLogAnalyzer(), AnalyzerSetting.class) ); - private final Column.AnalyzerType type; + private final ElasticSearchMatchQuery.AnalyzerType type; private final GenerateAnalyzerSettingFunc func; - Generator(final Column.AnalyzerType type, + Generator(final ElasticSearchMatchQuery.AnalyzerType type, final GenerateAnalyzerSettingFunc func) { this.type = type; this.func = func; @@ -100,7 +100,7 @@ public class AnalyzerSetting { return this.func; } - public static Generator getGenerator(Column.AnalyzerType type) throws StorageException { + public static Generator getGenerator(ElasticSearchMatchQuery.AnalyzerType type) throws StorageException { for (final Generator value : Generator.values()) { if (value.type == type) { return value; diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java index e77c666adb..3e59f249ec 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import lombok.extern.slf4j.Slf4j; -import org.apache.skywalking.oap.server.library.util.StringUtil; import org.apache.skywalking.library.elasticsearch.response.Index; import org.apache.skywalking.library.elasticsearch.response.IndexTemplate; import org.apache.skywalking.library.elasticsearch.response.Mappings; @@ -36,6 +35,7 @@ import org.apache.skywalking.oap.server.core.storage.model.ModelInstaller; import org.apache.skywalking.oap.server.library.client.Client; import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.util.StringUtil; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.StorageModuleElasticsearchConfig; @Slf4j @@ -214,7 +214,11 @@ public class StorageEsInstaller extends ModelInstaller { private Map getAnalyzerSetting(List analyzerTypes) throws StorageException { AnalyzerSetting analyzerSetting = new AnalyzerSetting(); for (final ModelColumn column : analyzerTypes) { - AnalyzerSetting setting = AnalyzerSetting.Generator.getGenerator(column.getAnalyzer()) + if (!column.getElasticSearchExtension().needMatchQuery()) { + continue; + } + AnalyzerSetting setting = AnalyzerSetting.Generator.getGenerator( + column.getElasticSearchExtension().getAnalyzer()) .getGenerateFunc() .generate(config); analyzerSetting.combine(setting); @@ -227,7 +231,7 @@ public class StorageEsInstaller extends ModelInstaller { Mappings.Source source = new Mappings.Source(); for (ModelColumn columnDefine : model.getColumns()) { final String type = columnTypeEsMapping.transform(columnDefine.getType(), columnDefine.getGenericType()); - if (columnDefine.isMatchQuery()) { + if (columnDefine.getElasticSearchExtension().needMatchQuery()) { String matchCName = MatchCNameBuilder.INSTANCE.build(columnDefine.getColumnName().getName()); Map originalColumn = new HashMap<>(); @@ -237,7 +241,7 @@ public class StorageEsInstaller extends ModelInstaller { Map matchColumn = new HashMap<>(); matchColumn.put("type", "text"); - matchColumn.put("analyzer", columnDefine.getAnalyzer().getName()); + matchColumn.put("analyzer", columnDefine.getElasticSearchExtension().getAnalyzer().getName()); properties.put(matchCName, matchColumn); } else { Map column = new HashMap<>(); diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/TimeSeriesUtilsTest.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/TimeSeriesUtilsTest.java index 9d6a3d3869..369275d74c 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/TimeSeriesUtilsTest.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/TimeSeriesUtilsTest.java @@ -36,13 +36,13 @@ public class TimeSeriesUtilsTest { @Before public void prepare() { - superDatasetModel = new Model("superDatasetModel", Lists.newArrayList(), Lists.newArrayList(), + superDatasetModel = new Model("superDatasetModel", Lists.newArrayList(), 0, DownSampling.Minute, true, true, "", true ); - normalRecordModel = new Model("normalRecordModel", Lists.newArrayList(), Lists.newArrayList(), + normalRecordModel = new Model("normalRecordModel", Lists.newArrayList(), 0, DownSampling.Minute, true, false, "", true ); - normalMetricsModel = new Model("normalMetricsModel", Lists.newArrayList(), Lists.newArrayList(), + normalMetricsModel = new Model("normalMetricsModel", Lists.newArrayList(), 0, DownSampling.Minute, false, false, "", true ); TimeSeriesUtils.setSUPER_DATASET_DAY_STEP(1); diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java index 3efd6890b3..cd4b4da0d7 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java @@ -25,9 +25,9 @@ import java.sql.SQLException; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.oap.server.core.storage.StorageException; -import org.apache.skywalking.oap.server.core.storage.model.ExtraQueryIndex; import org.apache.skywalking.oap.server.core.storage.model.Model; import org.apache.skywalking.oap.server.core.storage.model.ModelColumn; +import org.apache.skywalking.oap.server.core.storage.model.SQLDatabaseExtension; import org.apache.skywalking.oap.server.core.storage.type.StorageDataComplexObject; import org.apache.skywalking.oap.server.library.client.Client; import org.apache.skywalking.oap.server.library.client.jdbc.JDBCClientException; @@ -104,22 +104,25 @@ public class MySQLTableInstaller extends H2TableInstaller { } } - for (final ExtraQueryIndex extraQueryIndex : model.getExtraQueryIndices()) { - SQLBuilder tableIndexSQL = new SQLBuilder("CREATE INDEX "); - tableIndexSQL.append(model.getName().toUpperCase()) - .append("_") - .append(String.valueOf(indexSeq++)) - .append("_IDX "); - tableIndexSQL.append(" ON ").append(model.getName()).append("("); - final String[] columns = extraQueryIndex.getColumns(); - for (int i = 0; i < columns.length; i++) { - tableIndexSQL.append(columns[i]); - if (i < columns.length - 1) { - tableIndexSQL.append(","); + for (final ModelColumn modelColumn : model.getColumns()) { + for (final SQLDatabaseExtension.MultiColumnsIndex index : modelColumn.getSqlDatabaseExtension() + .getIndices()) { + SQLBuilder tableIndexSQL = new SQLBuilder("CREATE INDEX "); + tableIndexSQL.append(model.getName().toUpperCase()) + .append("_") + .append(String.valueOf(indexSeq++)) + .append("_IDX "); + tableIndexSQL.append(" ON ").append(model.getName()).append("("); + final String[] columns = index.getColumns(); + for (int i = 0; i < columns.length; i++) { + tableIndexSQL.append(columns[i]); + if (i < columns.length - 1) { + tableIndexSQL.append(","); + } } + tableIndexSQL.append(")"); + createIndex(client, connection, model, tableIndexSQL); } - tableIndexSQL.append(")"); - createIndex(client, connection, model, tableIndexSQL); } } diff --git a/oap-server/server-storage-plugin/storage-zipkin-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java b/oap-server/server-storage-plugin/storage-zipkin-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java index ea4bbb13e4..16376804f8 100644 --- a/oap-server/server-storage-plugin/storage-zipkin-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java +++ b/oap-server/server-storage-plugin/storage-zipkin-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java @@ -25,6 +25,7 @@ import org.apache.skywalking.oap.server.core.analysis.Stream; import org.apache.skywalking.oap.server.core.analysis.record.Record; import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey; import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset; import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; @@ -60,15 +61,17 @@ public class ZipkinSpanRecord extends Record { private String spanId; @Setter @Getter - @Column(columnName = SERVICE_ID, shardingKeyIdx = 0) + @Column(columnName = SERVICE_ID) + @BanyanDBShardingKey(index = 0) private String serviceId; @Setter @Getter - @Column(columnName = SERVICE_INSTANCE_ID, shardingKeyIdx = 1) + @Column(columnName = SERVICE_INSTANCE_ID) + @BanyanDBShardingKey(index = 1) private String serviceInstanceId; @Setter @Getter - @Column(columnName = ENDPOINT_NAME, matchQuery = true) + @Column(columnName = ENDPOINT_NAME) private String endpointName; @Setter @Getter @@ -88,7 +91,8 @@ public class ZipkinSpanRecord extends Record { private int latency; @Setter @Getter - @Column(columnName = IS_ERROR, shardingKeyIdx = 2) + @Column(columnName = IS_ERROR) + @BanyanDBShardingKey(index = 2) private int isError; @Setter @Getter