kazusa
/
skywalking
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Support BanyanDB global index for entities (#8852) 

* Move ElasticSearch and BanyanDB specific configurations out of column.
* Support BanyanDB global index for entities. Log and Segment record entities declare this new feature.
* Remove unnecessary analyzer settings in columns of templates. Many were added due to analyzer's default value.
* Move SQLDatabase(H2/MySQL/PostgreSQL) specific configurations out of column.
This commit is contained in:
吴晟 Wu Sheng 2022-04-12 08:21:44 +08:00 committed by GitHub
parent f3478b187d
commit 6d9decddce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 592 additions and 235 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/en/changes/changes.md
View File

@ -1,18 +1,24 @@
## 9.1.0
#### Project
* Upgrade zipkin to 2.23.16.
#### OAP Server
* Add component definition(ID=127) for `Apache ShenYu (incubating)`.
* Fix Zipkin receiver: Decode spans error, missing `Layer` for V9 and wrong time bucket for generate Service and Endpoint.
* Fix Zipkin receiver: Decode spans error, missing `Layer` for V9 and wrong time bucket for generate Service and
Endpoint.
* [Refactor] Move SQLDatabase(H2/MySQL/PostgreSQL), ElasticSearch and BanyanDB specific configurations out of column.
* Support BanyanDB global index for entities. Log and Segment record entities declare this new feature.
* Remove unnecessary analyzer settings in columns of templates. Many were added due to analyzer's default value.
#### UI
* General service instance: move `Thread Pool` from JVM to Overview, fix `JVM GC Count` calculation.
* Add Apache ShenYu (incubating) component LOGO.
#### Documentation
All issues and pull requests are [here](https://github.com/apache/skywalking/milestone/128?closed=1)

4
oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java
View File

@ -71,6 +71,7 @@ import org.apache.skywalking.oap.server.core.source.oal.rt.metrics.MetricClassPa
import org.apache.skywalking.oap.server.core.source.oal.rt.metrics.builder.MetricBuilderClassPackageHolder;
import org.apache.skywalking.oap.server.core.storage.StorageBuilderFactory;
import org.apache.skywalking.oap.server.core.storage.StorageException;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
import org.apache.skywalking.oap.server.library.util.ResourceUtils;
@ -269,7 +270,8 @@ public class OALRuntime implements OALEngine {
}
if (field.isID()) {
// Add shardingKeyIdx = 0 to column annotation.
columnAnnotation.addMemberValue("shardingKeyIdx", new IntegerMemberValue(constPool, 0));
Annotation banyanShardingKeyAnnotation = new Annotation(BanyanDBShardingKey.class.getName(), constPool);
banyanShardingKeyAnnotation.addMemberValue("index", new IntegerMemberValue(constPool, 0));
}
annotationsAttribute.addAnnotation(columnAnnotation);

4
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java
View File

@ -29,6 +29,7 @@ import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcess
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.source.ScopeDeclaration;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
import org.apache.skywalking.oap.server.core.storage.type.HashMapConverter;
@ -68,7 +69,8 @@ public class AlarmRecord extends Record {
private String id1;
@Column(columnName = START_TIME)
private long startTime;
@Column(columnName = ALARM_MESSAGE, matchQuery = true)
@Column(columnName = ALARM_MESSAGE)
@ElasticSearchMatchQuery
private String alarmMessage;
@Column(columnName = RULE_NAME)
private String ruleName;

4
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java
View File

@ -31,6 +31,7 @@ import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProces
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder;
@ -52,7 +53,8 @@ public class EndpointTraffic extends Metrics {
private String serviceId;
@Setter
@Getter
@Column(columnName = NAME, matchQuery = true)
@Column(columnName = NAME)
@ElasticSearchMatchQuery
private String name = Const.EMPTY_STRING;
@Override

14
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java
View File

@ -25,7 +25,10 @@ import org.apache.skywalking.oap.server.core.UnexpectedException;
import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag;
import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.query.type.ContentType;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBGlobalIndex;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
import org.apache.skywalking.oap.server.core.storage.type.HashMapConverter;
@ -47,11 +50,13 @@ public abstract class AbstractLogRecord extends Record {
@Setter
@Getter
@Column(columnName = SERVICE_ID, shardingKeyIdx = 0)
@Column(columnName = SERVICE_ID)
@BanyanDBShardingKey(index = 0)
private String serviceId;
@Setter
@Getter
@Column(columnName = SERVICE_INSTANCE_ID, shardingKeyIdx = 1)
@Column(columnName = SERVICE_INSTANCE_ID)
@BanyanDBShardingKey(index = 1)
private String serviceInstanceId;
@Setter
@Getter
@ -60,10 +65,12 @@ public abstract class AbstractLogRecord extends Record {
@Setter
@Getter
@Column(columnName = TRACE_ID, length = 150)
@BanyanDBGlobalIndex(extraFields = {})
private String traceId;
@Setter
@Getter
@Column(columnName = TRACE_SEGMENT_ID, length = 150)
@BanyanDBGlobalIndex(extraFields = {SPAN_ID})
private String traceSegmentId;
@Setter
@Getter
@ -75,7 +82,8 @@ public abstract class AbstractLogRecord extends Record {
private int contentType = ContentType.NONE.value();
@Setter
@Getter
@Column(columnName = CONTENT, length = 1_000_000, matchQuery = true, analyzer = Column.AnalyzerType.OAP_LOG_ANALYZER)
@Column(columnName = CONTENT, length = 1_000_000)
@ElasticSearchMatchQuery(analyzer = ElasticSearchMatchQuery.AnalyzerType.OAP_LOG_ANALYZER)
private String content;
@Setter
@Getter

4
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/networkalias/NetworkAddressAlias.java
View File

@ -28,6 +28,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.source.ScopeDeclaration;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@ -51,7 +52,8 @@ public class NetworkAddressAlias extends Metrics {
@Setter
@Getter
@Column(columnName = ADDRESS, shardingKeyIdx = 0)
@Column(columnName = ADDRESS)
@BanyanDBShardingKey(index = 0)
private String address;
@Setter
@Getter

4
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/endpoint/EndpointRelationServerSideMetrics.java
View File

@ -27,6 +27,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@ -58,7 +59,8 @@ public class EndpointRelationServerSideMetrics extends Metrics {
private int componentId;
@Setter
@Getter
@Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
@Column(columnName = ENTITY_ID, length = 512)
@BanyanDBShardingKey(index = 0)
private String entityId;
@Override

4
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java
View File

@ -27,6 +27,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@ -68,7 +69,8 @@ public class ServiceInstanceRelationServerSideMetrics extends Metrics {
private int componentId;
@Setter
@Getter
@Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
@Column(columnName = ENTITY_ID, length = 512)
@BanyanDBShardingKey(index = 0)
private String entityId;
@Override

4
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java
View File

@ -27,6 +27,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@ -58,7 +59,8 @@ public class ServiceRelationServerSideMetrics extends Metrics {
private int componentId;
@Setter
@Getter
@Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
@Column(columnName = ENTITY_ID, length = 512)
@BanyanDBShardingKey(index = 0)
private String entityId;
@Override

12
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java
View File

@ -26,6 +26,8 @@ import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag;
import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBGlobalIndex;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
@ -56,14 +58,17 @@ public class SegmentRecord extends Record {
@Setter
@Getter
@Column(columnName = TRACE_ID, length = 150)
@BanyanDBGlobalIndex(extraFields = {})
private String traceId;
@Setter
@Getter
@Column(columnName = SERVICE_ID, shardingKeyIdx = 0)
@Column(columnName = SERVICE_ID)
@BanyanDBShardingKey(index = 0)
private String serviceId;
@Setter
@Getter
@Column(columnName = SERVICE_INSTANCE_ID, shardingKeyIdx = 1)
@Column(columnName = SERVICE_INSTANCE_ID)
@BanyanDBShardingKey(index = 1)
private String serviceInstanceId;
@Setter
@Getter
@ -79,7 +84,8 @@ public class SegmentRecord extends Record {
private int latency;
@Setter
@Getter
@Column(columnName = IS_ERROR, shardingKeyIdx = 2)
@Column(columnName = IS_ERROR)
@BanyanDBShardingKey(index = 2)
private int isError;
@Setter
@Getter

4
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java
View File

@ -31,6 +31,7 @@ import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProces
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder;
@ -60,7 +61,8 @@ public class ServiceTraffic extends Metrics {
@Setter
@Getter
@Column(columnName = NAME, matchQuery = true)
@Column(columnName = NAME)
@ElasticSearchMatchQuery
private String name = Const.EMPTY_STRING;
@Setter

4
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/HistogramFunction.java
View File

@ -31,6 +31,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.query.type.Bucket;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@ -48,7 +49,8 @@ public abstract class HistogramFunction extends Meter implements AcceptableValue
@Setter
@Getter
@Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
@Column(columnName = ENTITY_ID, length = 512)
@BanyanDBShardingKey(index = 0)
private String entityId;
@Getter
@Setter

4
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java
View File

@ -37,6 +37,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.MultiIntValuesHold
import org.apache.skywalking.oap.server.core.analysis.metrics.PercentileMetrics;
import org.apache.skywalking.oap.server.core.query.type.Bucket;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@ -55,7 +56,8 @@ public abstract class PercentileFunction extends Meter implements AcceptableValu
@Setter
@Getter
@Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
@Column(columnName = ENTITY_ID, length = 512)
@BanyanDBShardingKey(index = 0)
private String entityId;
@Getter
@Setter

4
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgFunction.java
View File

@ -36,6 +36,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.Entranc
import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.SourceFrom;
import org.apache.skywalking.oap.server.core.query.sql.Function;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@ -50,7 +51,8 @@ public abstract class AvgFunction extends Meter implements AcceptableValue<Long>
@Setter
@Getter
@Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
@Column(columnName = ENTITY_ID, length = 512)
@BanyanDBShardingKey(index = 0)
private String entityId;
/**

4
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java
View File

@ -34,6 +34,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.query.type.Bucket;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@ -59,7 +60,8 @@ public abstract class AvgHistogramFunction extends Meter implements AcceptableVa
@Setter
@Getter
@Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
@Column(columnName = ENTITY_ID, length = 512)
@BanyanDBShardingKey(index = 0)
private String entityId;
@Getter
@Setter

4
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java
View File

@ -43,6 +43,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.analysis.metrics.MultiIntValuesHolder;
import org.apache.skywalking.oap.server.core.query.type.Bucket;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@ -74,7 +75,8 @@ public abstract class AvgHistogramPercentileFunction extends Meter implements Ac
@Setter
@Getter
@Column(columnName = ENTITY_ID, shardingKeyIdx = 0)
@Column(columnName = ENTITY_ID)
@BanyanDBShardingKey(index = 0)
private String entityId;
@Getter
@Setter

4
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java
View File

@ -34,6 +34,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
import org.apache.skywalking.oap.server.core.analysis.metrics.LabeledValueHolder;
import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@ -48,7 +49,8 @@ public abstract class AvgLabeledFunction extends Meter implements AcceptableValu
@Setter
@Getter
@Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
@Column(columnName = ENTITY_ID, length = 512)
@BanyanDBShardingKey(index = 0)
private String entityId;
/**

4
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/latest/LatestFunction.java
View File

@ -35,6 +35,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.Entranc
import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.SourceFrom;
import org.apache.skywalking.oap.server.core.query.sql.Function;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@ -47,7 +48,8 @@ public abstract class LatestFunction extends Meter implements AcceptableValue<Lo
@Setter
@Getter
@Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
@Column(columnName = ENTITY_ID, length = 512)
@BanyanDBShardingKey(index = 0)
private String entityId;
/**

4
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/sum/SumFunction.java
View File

@ -35,6 +35,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.Entranc
import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.SourceFrom;
import org.apache.skywalking.oap.server.core.query.sql.Function;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@ -47,7 +48,8 @@ public abstract class SumFunction extends Meter implements AcceptableValue<Long>
@Setter
@Getter
@Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
@Column(columnName = ENTITY_ID, length = 512)
@BanyanDBShardingKey(index = 0)
private String entityId;
@Setter

4
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java
View File

@ -23,6 +23,7 @@ import org.apache.skywalking.oap.server.core.analysis.Stream;
import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
@ -54,7 +55,8 @@ public class BrowserErrorLogRecord extends Record {
@Setter
@Getter
@Column(columnName = SERVICE_ID, shardingKeyIdx = 0)
@Column(columnName = SERVICE_ID)
@BanyanDBShardingKey(index = 0)
private String serviceId;
@Setter

4
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingDataRecord.java
View File

@ -24,6 +24,7 @@ import lombok.Data;
import org.apache.skywalking.oap.server.core.analysis.Stream;
import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@ -48,7 +49,8 @@ public class EBPFProfilingDataRecord extends Record {
public static final String STACKS_BINARY = "dump_binary";
public static final String UPLOAD_TIME = "upload_time";
@Column(columnName = TASK_ID, length = 600, shardingKeyIdx = 0)
@Column(columnName = TASK_ID, length = 600)
@BanyanDBShardingKey(index = 0)
private String taskId;
@Column(columnName = SCHEDULE_ID, length = 600)
private String scheduleId;

46
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBGlobalIndex.java Normal file
View File

@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.storage.annotation;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* BanyanDBGlobalIndex declares advanced global index, which are only available in BanyanDB.
*
* Global index should only be considered if a column value has a huge value candidates, but we will need a direct equal
* query without timestamp.
* The typical global index is designed for huge candidate of indexed values,
* such as `trace ID` or `segment ID + span ID`
*
* Only work with {@link Column}
*/
@Target({ElementType.FIELD})
@Retention(RetentionPolicy.RUNTIME)
public @interface BanyanDBGlobalIndex {
/**
* The current column should be indexed through global index.
*
* @return empty array if only the current column should be indexed in global index. Or list of column names if this
* global index includes multiple columns.
*/
String[] extraFields();
}

59
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBShardingKey.java Normal file
View File

@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.storage.annotation;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Sharding key is used to group time series data per metric of one entity in one place (same sharding and/or same
* row for column-oriented database).
* For example,
* ServiceA's traffic gauge, service call per minute, includes following timestamp values, then it should be sharded
* by service ID
* [ServiceA(encoded ID): 01-28 18:30 values-1, 01-28 18:31 values-2, 01-28 18:32 values-3, 01-28 18:32 values-4]
*
* BanyanDB is the 1st storage implementation supporting this. It would make continuous time series metrics stored
* closely and compressed better.
*
* 1. One entity could have multiple sharding keys
* 2. If no column is appointed for this, {@link org.apache.skywalking.oap.server.core.storage.StorageData#id}
* would be used by the storage implementation accordingly.
*
* NOTICE, this sharding concept is NOT just for splitting data into different database instances or physical
* files.
*
* Only work with {@link Column}
*
* @return non-negative if this column be used for sharding. -1 means not as a sharding key
* @since 9.1.0 created as a new annotation.
* @since 9.0.0 added in {@link Column}
*/
@Target({ElementType.FIELD})
@Retention(RetentionPolicy.RUNTIME)
public @interface BanyanDBShardingKey {
/**
* Relative entity tag
*
* @return index, from zero.
*/
int index() default -1;
}

56
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java
View File

@ -48,11 +48,6 @@ public @interface Column {
*/
int defaultValue() default 0;
/**
* Match query means using analyzer(if storage have) to do key word match query.
*/
boolean matchQuery() default false;
/**
* The column is just saved, never used in query.
*/
@ -94,57 +89,6 @@ public @interface Column {
*/
ValueDataType dataType() default ValueDataType.NOT_VALUE;
/**
* The storage analyzer mode.
*
* @since 8.4.0
*/
AnalyzerType analyzer() default AnalyzerType.OAP_ANALYZER;
/**
* Sharding key is used to group time series data per metric of one entity in one place (same sharding and/or same
* row for column-oriented database).
* For example,
* ServiceA's traffic gauge, service call per minute, includes following timestamp values, then it should be sharded
* by service ID
* [ServiceA(encoded ID): 01-28 18:30 values-1, 01-28 18:31 values-2, 01-28 18:32 values-3, 01-28 18:32 values-4]
*
* BanyanDB is the 1st storage implementation supporting this. It would make continuous time series metrics stored
* closely and compressed better.
*
* 1. One entity could have multiple sharding keys
* 2. If no column is appointed for this, {@link org.apache.skywalking.oap.server.core.storage.StorageData#id}
* would be used by the storage implementation accordingly.
*
* NOTICE, this sharding concept is NOT just for splitting data into different database instances or physical
* files.
*
* @return non-negative if this column be used for sharding. -1 means not as a sharding key
* @since 9.0.0
*/
int shardingKeyIdx() default -1;
/**
* The analyzer declares the text analysis mode.
*/
enum AnalyzerType {
/**
* The default analyzer.
*/
OAP_ANALYZER("oap_analyzer"),
/**
* The log analyzer.
*/
OAP_LOG_ANALYZER("oap_log_analyzer");
@Getter
private final String name;
AnalyzerType(final String name) {
this.name = name;
}
}
/**
* ValueDataType represents the data structure of value column. The persistent way of the value column determine the
* available ways to query the data.

64
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/ElasticSearchMatchQuery.java Normal file
View File

@ -0,0 +1,64 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.storage.annotation;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import lombok.Getter;
/**
* Match query is designed for ElasticSearch match query with specific analyzer. It is a fuzzy query implementation
* powered by analyzer.
*
* @since 9.1.0 This used to be {@link Column}'s matchQuery and analyzer attributes.
*/
@Target({ElementType.FIELD})
@Retention(RetentionPolicy.RUNTIME)
public @interface ElasticSearchMatchQuery {
/**
* The storage analyzer mode.
*
* @since 9.1.0 created as a new annotation.
* @since 8.4.0 added in {@link Column}
*/
AnalyzerType analyzer() default AnalyzerType.OAP_ANALYZER;
/**
* The analyzer declares the text analysis mode.
*/
enum AnalyzerType {
/**
* The default analyzer.
*/
OAP_ANALYZER("oap_analyzer"),
/**
* The log analyzer.
*/
OAP_LOG_ANALYZER("oap_log_analyzer");
@Getter
private final String name;
AnalyzerType(final String name) {
this.name = name;
}
}
}

67
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java Normal file
View File

@ -0,0 +1,67 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.storage.model;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
/**
* BanyanDBExtension represents extra metadata for columns, but specific for BanyanDB usages.
*
* @since 9.1.0
*/
@Getter
@RequiredArgsConstructor
public class BanyanDBExtension {
/**
* Sharding key is used to group time series data per metric of one entity. See {@link
* org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey#index()}.
*
* @since 9.1.0 moved into BanyanDBExtension
* @since 9.0.0 added into {@link ModelColumn}
*/
private final int shardingKeyIdx;
/**
* Global index column values has 3 conditions
* 1. NULL, this column should be as a global index.
* 2. Empty array(declared by @BanyanDBGlobalIndex(extraFields = {}) in codes) represents this single column should
* be a global index.
* 3. Not empty array(declared by @BanyanDBGlobalIndex(extraFields = {"col1", "col2"}) in codes) represents this
* column and other declared columns should be as a global index. The values of these columns should be joint by
* underline(_)
*
* @since 9.1.0
*/
private final String[] globalIndexColumns;
/**
* @return true if this column is a part of sharding key
*/
public boolean isShardingKey() {
return this.shardingKeyIdx > -1;
}
/**
* @return null or array, see {@link #globalIndexColumns} for more details.
*/
public boolean isGlobalIndexing() {
return globalIndexColumns != null;
}
}

42
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ElasticSearchExtension.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.storage.model;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
/**
* ElasticSearchExtension represents extra metadata for columns, but specific for ElasticSearch usages.
*
* @since 9.1.0
*/
@Getter
@RequiredArgsConstructor
public class ElasticSearchExtension {
/**
* The analyzer policy appointed to fuzzy query, especially for ElasticSearch.
* When it is null, it means no need to build match query, no `copy_to` column, and no analyzer assigned.
*/
private final ElasticSearchMatchQuery.AnalyzerType analyzer;
public boolean needMatchQuery() {
return analyzer != null;
}
}

56
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ExtraQueryIndex.java
View File

@ -1,56 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.storage.model;
import lombok.Getter;
import org.apache.skywalking.oap.server.library.util.CollectionUtils;
/**
* The extra query index if the storage could support this mode. Many NO-SQL support one column index only, in that
* case, this could be ignored in the implementation level.
*/
@Getter
public class ExtraQueryIndex {
private String[] columns;
public ExtraQueryIndex(String mainColumn, final String[] withColumns) {
if (CollectionUtils.isNotEmpty(withColumns)) {
columns = new String[withColumns.length + 1];
columns[0] = mainColumn;
System.arraycopy(withColumns, 0, columns, 1, withColumns.length);
} else {
throw new IllegalArgumentException("ExtraQueryIndex required withColumns as a not empty list.");
}
}
/**
* Keep the same name replacement as {@link ColumnName#overrideName(String, String)}
*
* @param oldName to be replaced.
* @param newName to use in the storage level.
*/
public void overrideName(String oldName, String newName) {
for (int i = 0; i < columns.length; i++) {
if (columns[i].equals(oldName)) {
columns[i] = newName;
}
}
}
}

3
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/Model.java
View File

@ -31,7 +31,6 @@ import org.apache.skywalking.oap.server.core.analysis.DownSampling;
public class Model {
private final String name;
private final List<ModelColumn> columns;
private final List<ExtraQueryIndex> extraQueryIndices;
private final int scopeId;
private final DownSampling downsampling;
private final boolean record;
@ -42,7 +41,6 @@ public class Model {
public Model(final String name,
final List<ModelColumn> columns,
final List<ExtraQueryIndex> extraQueryIndices,
final int scopeId,
final DownSampling downsampling,
final boolean record,
@ -51,7 +49,6 @@ public class Model {
boolean timeRelativeID) {
this.name = name;
this.columns = columns;
this.extraQueryIndices = extraQueryIndices;
this.scopeId = scopeId;
this.downsampling = downsampling;
this.isTimeSeries = !DownSampling.None.equals(downsampling);

37
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java
View File

@ -22,7 +22,6 @@ import java.lang.reflect.Type;
import lombok.Getter;
import lombok.ToString;
import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
@Getter
@ToString
@ -30,7 +29,6 @@ public class ModelColumn {
private final ColumnName columnName;
private final Class<?> type;
private final Type genericType;
private final boolean matchQuery;
/**
* Storage this column for query result, but can't be as a condition . Conflict with {@link #indexOnly}
*/
@ -45,33 +43,42 @@ public class ModelColumn {
* The max length of column value for length sensitive database.
*/
private final int length;
/**
* The analyzer policy appointed to fuzzy query, especially for ElasticSearch
*/
private final Column.AnalyzerType analyzer;
/**
* Sharding key is used to group time series data per metric of one entity. See {@link Column#shardingKeyIdx()}.
* Hold configurations especially for SQL Database, such as MySQL, H2, PostgreSQL
*
* @since 9.0.0
* @since 9.1.0
*/
private int shardingKeyIdx;
private SQLDatabaseExtension sqlDatabaseExtension;
/**
* Hold configurations especially for ElasticSearch
*
* @since 9.1.0
*/
private ElasticSearchExtension elasticSearchExtension;
/**
* Hold configurations especially for BanyanDB relevant
*
* @since 9.1.0
*/
private BanyanDBExtension banyanDBExtension;
public ModelColumn(ColumnName columnName,
Class<?> type,
Type genericType,
boolean matchQuery,
boolean storageOnly,
boolean indexOnly,
boolean isValue,
int length,
Column.AnalyzerType analyzer,
int shardingKeyIdx) {
SQLDatabaseExtension sqlDatabaseExtension,
ElasticSearchExtension elasticSearchExtension,
BanyanDBExtension banyanDBExtension) {
this.columnName = columnName;
this.type = type;
this.genericType = genericType;
this.matchQuery = matchQuery;
this.length = length;
this.analyzer = analyzer;
this.sqlDatabaseExtension = sqlDatabaseExtension;
this.elasticSearchExtension = elasticSearchExtension;
/*
* byte[] and {@link IntKeyLongValueHashMap} could never be query.
*/
@ -90,6 +97,6 @@ public class ModelColumn {
"The column " + columnName + " can't be defined as both indexOnly and storageOnly.");
}
this.indexOnly = indexOnly;
this.shardingKeyIdx = shardingKeyIdx;
this.banyanDBExtension = banyanDBExtension;
}
}

70
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/SQLDatabaseExtension.java Normal file
View File

@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.storage.model;
import java.util.ArrayList;
import java.util.List;
import lombok.Getter;
import org.apache.skywalking.oap.server.library.util.CollectionUtils;
/**
* @since 9.0.0
*/
@Getter
public class SQLDatabaseExtension {
private final List<MultiColumnsIndex> indices = new ArrayList<>(5);
public void appendIndex(MultiColumnsIndex index) {
indices.add(index);
}
/**
* The multiple columns index if the storage could support this mode. Many NO-SQL support one column index only, in that
* case, this could be ignored in the implementation level.
*/
@Getter
public static class MultiColumnsIndex {
private String[] columns;
public MultiColumnsIndex(String mainColumn, final String[] withColumns) {
if (CollectionUtils.isNotEmpty(withColumns)) {
columns = new String[withColumns.length + 1];
columns[0] = mainColumn;
System.arraycopy(withColumns, 0, columns, 1, withColumns.length);
} else {
throw new IllegalArgumentException("ExtraQueryIndex required withColumns as a not empty list.");
}
}
/**
* Keep the same name replacement as {@link ColumnName#overrideName(String, String)}
*
* @param oldName to be replaced.
* @param newName to use in the storage level.
*/
public void overrideName(String oldName, String newName) {
for (int i = 0; i < columns.length; i++) {
if (columns[i].equals(oldName)) {
columns[i] = newName;
}
}
}
}
}

97
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
View File

@ -27,7 +27,10 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.oap.server.core.analysis.FunctionCategory;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.storage.StorageException;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBGlobalIndex;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.apache.skywalking.oap.server.core.storage.annotation.MultipleQueryUnifiedIndex;
import org.apache.skywalking.oap.server.core.storage.annotation.QueryUnifiedIndex;
import org.apache.skywalking.oap.server.core.storage.annotation.Storage;
@ -56,14 +59,16 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
DefaultScopeDefine.nameOf(scopeId);
List<ModelColumn> modelColumns = new ArrayList<>();
List<ExtraQueryIndex> extraQueryIndices = new ArrayList<>();
ShardingKeyChecker checker = new ShardingKeyChecker();
retrieval(aClass, storage.getModelName(), modelColumns, extraQueryIndices, scopeId, checker);
retrieval(aClass, storage.getModelName(), modelColumns, scopeId, checker);
checker.check(storage.getModelName());
Model model = new Model(
storage.getModelName(), modelColumns, extraQueryIndices, scopeId,
storage.getDownsampling(), record,
storage.getModelName(),
modelColumns,
scopeId,
storage.getDownsampling(),
record,
isSuperDatasetModel(aClass),
FunctionCategory.uniqueFunctionName(aClass),
storage.isTimeRelativeID()
@ -100,7 +105,6 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
private void retrieval(final Class<?> clazz,
final String modelName,
final List<ModelColumn> modelColumns,
final List<ExtraQueryIndex> extraQueryIndices,
final int scopeId,
ShardingKeyChecker checker) {
if (log.isDebugEnabled()) {
@ -129,13 +133,55 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
}
}
final ModelColumn modelColumn = new ModelColumn(
new ColumnName(modelName, column.columnName()), field.getType(), field.getGenericType(),
column.matchQuery(), column.storageOnly(), column.indexOnly(), column.dataType().isValue(),
columnLength,
column.analyzer(), column.shardingKeyIdx()
// SQL Database extension
SQLDatabaseExtension sqlDatabaseExtension = new SQLDatabaseExtension();
List<QueryUnifiedIndex> indexDefinitions = new ArrayList<>();
if (field.isAnnotationPresent(QueryUnifiedIndex.class)) {
indexDefinitions.add(field.getAnnotation(QueryUnifiedIndex.class));
}
if (field.isAnnotationPresent(MultipleQueryUnifiedIndex.class)) {
Collections.addAll(indexDefinitions, field.getAnnotation(MultipleQueryUnifiedIndex.class).value());
}
indexDefinitions.forEach(indexDefinition -> {
sqlDatabaseExtension.appendIndex(new SQLDatabaseExtension.MultiColumnsIndex(
column.columnName(),
indexDefinition.withColumns()
));
});
// ElasticSearch extension
final ElasticSearchMatchQuery elasticSearchAnalyzer = field.getAnnotation(
ElasticSearchMatchQuery.class);
ElasticSearchExtension elasticSearchExtension = new ElasticSearchExtension(
elasticSearchAnalyzer == null ? null : elasticSearchAnalyzer.analyzer()
);
if (column.shardingKeyIdx() > -1) {
// BanyanDB extension
final BanyanDBShardingKey banyanDBShardingKey = field.getAnnotation(BanyanDBShardingKey.class);
final BanyanDBGlobalIndex banyanDBGlobalIndex = field.getAnnotation(BanyanDBGlobalIndex.class);
BanyanDBExtension banyanDBExtension = new BanyanDBExtension(
banyanDBShardingKey == null ? -1 : banyanDBShardingKey.index(),
banyanDBGlobalIndex == null ? null : banyanDBGlobalIndex.extraFields()
);
final ModelColumn modelColumn = new ModelColumn(
new ColumnName(
modelName,
column.columnName()
),
field.getType(),
field.getGenericType(),
column.storageOnly(),
column.indexOnly(),
column.dataType().isValue(),
columnLength,
sqlDatabaseExtension,
elasticSearchExtension,
banyanDBExtension
);
if (banyanDBExtension.isShardingKey()) {
checker.accept(modelName, modelColumn);
}
@ -149,25 +195,11 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
column.defaultValue(), scopeId
);
}
List<QueryUnifiedIndex> indexDefinitions = new ArrayList<>();
if (field.isAnnotationPresent(QueryUnifiedIndex.class)) {
indexDefinitions.add(field.getAnnotation(QueryUnifiedIndex.class));
}
if (field.isAnnotationPresent(MultipleQueryUnifiedIndex.class)) {
Collections.addAll(indexDefinitions, field.getAnnotation(MultipleQueryUnifiedIndex.class).value());
}
indexDefinitions.forEach(indexDefinition -> extraQueryIndices.add(new ExtraQueryIndex(
column.columnName(),
indexDefinition.withColumns()
)));
}
}
if (Objects.nonNull(clazz.getSuperclass())) {
retrieval(clazz.getSuperclass(), modelName, modelColumns, extraQueryIndices, scopeId, checker);
retrieval(clazz.getSuperclass(), modelName, modelColumns, scopeId, checker);
}
}
@ -180,8 +212,12 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
private void followColumnNameRules(Model model) {
columnNameOverrideRule.forEach((oldName, newName) -> {
model.getColumns().forEach(column -> column.getColumnName().overrideName(oldName, newName));
model.getExtraQueryIndices().forEach(extraQueryIndex -> extraQueryIndex.overrideName(oldName, newName));
model.getColumns().forEach(column -> {
column.getColumnName().overrideName(oldName, newName);
column.getSqlDatabaseExtension()
.getIndices()
.forEach(extraQueryIndex -> extraQueryIndex.overrideName(oldName, newName));
});
});
}
@ -197,7 +233,7 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
* @throws IllegalStateException if sharding key indices are conflicting.
*/
private void accept(String modelName, ModelColumn modelColumn) throws IllegalStateException {
final int idx = modelColumn.getShardingKeyIdx();
final int idx = modelColumn.getBanyanDBExtension().getShardingKeyIdx();
while (idx + 1 > keys.size()) {
keys.add(null);
}
@ -206,7 +242,8 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
throw new IllegalStateException(
modelName + "'s "
+ "Column [" + exist.getColumnName() + "] and column [" + modelColumn.getColumnName()
+ " are conflicting with sharding key index=" + modelColumn.getShardingKeyIdx());
+ " are conflicting with sharding key index=" + modelColumn.getBanyanDBExtension()
.getShardingKeyIdx());
}
keys.set(idx, modelColumn);
}

35
oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java
View File

@ -19,31 +19,38 @@
package org.apache.skywalking.oap.server.core.storage.model;
import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.junit.Assert;
import org.junit.Test;
public class ModelColumnTest {
@Test
public void testColumnDefine() {
ModelColumn column = new ModelColumn(new ColumnName("", "abc"), byte[].class, byte[].class, true,
ModelColumn column = new ModelColumn(new ColumnName("", "abc"), byte[].class, byte[].class,
false, false, true, 0,
Column.AnalyzerType.OAP_ANALYZER, 0
new SQLDatabaseExtension(),
new ElasticSearchExtension(
ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER),
new BanyanDBExtension(-1, null)
);
Assert.assertEquals(true, column.isStorageOnly());
Assert.assertEquals("abc", column.getColumnName().getName());
column = new ModelColumn(new ColumnName("", "abc"), DataTable.class, DataTable.class, true,
column = new ModelColumn(new ColumnName("", "abc"), DataTable.class, DataTable.class,
false, false, true, 200,
Column.AnalyzerType.OAP_ANALYZER, 0
new SQLDatabaseExtension(),
new ElasticSearchExtension(ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER),
new BanyanDBExtension(-1, null)
);
Assert.assertEquals(true, column.isStorageOnly());
Assert.assertEquals("abc", column.getColumnName().getName());
Assert.assertEquals(200, column.getLength());
column = new ModelColumn(new ColumnName("", "abc"), String.class, String.class, true,
column = new ModelColumn(new ColumnName("", "abc"), String.class, String.class,
false, false, true, 200,
Column.AnalyzerType.OAP_ANALYZER, 0
new SQLDatabaseExtension(),
new ElasticSearchExtension(ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER),
new BanyanDBExtension(-1, null)
);
Assert.assertEquals(false, column.isStorageOnly());
Assert.assertEquals("abc", column.getColumnName().getName());
@ -52,16 +59,22 @@ public class ModelColumnTest {
@Test(expected = IllegalArgumentException.class)
public void testConflictDefinition() {
ModelColumn column = new ModelColumn(new ColumnName("", "abc"), String.class, String.class,
true, true, false, true, 200,
Column.AnalyzerType.OAP_ANALYZER, 0
true, false, true, 200,
new SQLDatabaseExtension(),
new ElasticSearchExtension(
ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER),
new BanyanDBExtension(-1, null)
);
}
@Test(expected = IllegalArgumentException.class)
public void testConflictDefinitionIndexOnly() {
ModelColumn column = new ModelColumn(new ColumnName("", "abc"), String.class, String.class,
true, true, true, false, 200,
Column.AnalyzerType.OAP_ANALYZER, 0
true, true, false, 200,
new SQLDatabaseExtension(),
new ElasticSearchExtension(
ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER),
new BanyanDBExtension(-1, null)
);
}
}

8
oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ExtraQueryIndexTest.java → oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/SQLDatabaseExtensionTest.java
View File

@ -21,10 +21,11 @@ package org.apache.skywalking.oap.server.core.storage.model;
import org.junit.Assert;
import org.junit.Test;
public class ExtraQueryIndexTest {
public class SQLDatabaseExtensionTest {
@Test
public void testIndexColumns() {
final ExtraQueryIndex extraQueryIndex = new ExtraQueryIndex("a1", new String[] {"a2"});
final SQLDatabaseExtension.MultiColumnsIndex extraQueryIndex = new SQLDatabaseExtension.MultiColumnsIndex(
"a1", new String[] {"a2"});
Assert.assertArrayEquals(new String[] {
"a1",
"a2"
@ -33,6 +34,7 @@ public class ExtraQueryIndexTest {
@Test(expected = IllegalArgumentException.class)
public void testIllegalIndexColumns() {
ExtraQueryIndex extraQueryIndex = new ExtraQueryIndex("a1", new String[0]);
SQLDatabaseExtension.MultiColumnsIndex extraQueryIndex = new SQLDatabaseExtension.MultiColumnsIndex(
"a1", new String[0]);
}
}

4
oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/StorageModelsTest.java
View File

@ -74,12 +74,10 @@ public class StorageModelsTest {
Assert.assertEquals(false, model.getColumns().get(2).isStorageOnly());
Assert.assertEquals(true, model.getColumns().get(3).isStorageOnly());
final List<ExtraQueryIndex> extraQueryIndices = model.getExtraQueryIndices();
Assert.assertEquals(3, extraQueryIndices.size());
Assert.assertArrayEquals(new String[] {
"column2",
"column"
}, extraQueryIndices.get(2).getColumns());
}, model.getColumns().get(2).getSqlDatabaseExtension().getIndices().get(1).getColumns());
}
@Stream(name = "StorageModelsTest", scopeId = -1, builder = TestModel.Builder.class, processor = MetricsStreamProcessor.class)

12
oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/AnalyzerSetting.java
View File

@ -27,7 +27,7 @@ import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.oap.server.core.storage.StorageException;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.StorageModuleElasticsearchConfig;
@Getter
@ -79,18 +79,18 @@ public class AnalyzerSetting {
public enum Generator {
OAP_ANALYZER_SETTING_GENERATOR(
Column.AnalyzerType.OAP_ANALYZER,
ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER,
config -> new Gson().fromJson(config.getOapAnalyzer(), AnalyzerSetting.class)
),
OAP_LOG_ANALYZER_SETTING_GENERATOR(
Column.AnalyzerType.OAP_LOG_ANALYZER,
ElasticSearchMatchQuery.AnalyzerType.OAP_LOG_ANALYZER,
config -> new Gson().fromJson(config.getOapLogAnalyzer(), AnalyzerSetting.class)
);
private final Column.AnalyzerType type;
private final ElasticSearchMatchQuery.AnalyzerType type;
private final GenerateAnalyzerSettingFunc func;
Generator(final Column.AnalyzerType type,
Generator(final ElasticSearchMatchQuery.AnalyzerType type,
final GenerateAnalyzerSettingFunc func) {
this.type = type;
this.func = func;
@ -100,7 +100,7 @@ public class AnalyzerSetting {
return this.func;
}
public static Generator getGenerator(Column.AnalyzerType type) throws StorageException {
public static Generator getGenerator(ElasticSearchMatchQuery.AnalyzerType type) throws StorageException {
for (final Generator value : Generator.values()) {
if (value.type == type) {
return value;

12
oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
View File

@ -25,7 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.oap.server.library.util.StringUtil;
import org.apache.skywalking.library.elasticsearch.response.Index;
import org.apache.skywalking.library.elasticsearch.response.IndexTemplate;
import org.apache.skywalking.library.elasticsearch.response.Mappings;
@ -36,6 +35,7 @@ import org.apache.skywalking.oap.server.core.storage.model.ModelInstaller;
import org.apache.skywalking.oap.server.library.client.Client;
import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
import org.apache.skywalking.oap.server.library.util.StringUtil;
import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.StorageModuleElasticsearchConfig;
@Slf4j
@ -214,7 +214,11 @@ public class StorageEsInstaller extends ModelInstaller {
private Map getAnalyzerSetting(List<ModelColumn> analyzerTypes) throws StorageException {
AnalyzerSetting analyzerSetting = new AnalyzerSetting();
for (final ModelColumn column : analyzerTypes) {
AnalyzerSetting setting = AnalyzerSetting.Generator.getGenerator(column.getAnalyzer())
if (!column.getElasticSearchExtension().needMatchQuery()) {
continue;
}
AnalyzerSetting setting = AnalyzerSetting.Generator.getGenerator(
column.getElasticSearchExtension().getAnalyzer())
.getGenerateFunc()
.generate(config);
analyzerSetting.combine(setting);
@ -227,7 +231,7 @@ public class StorageEsInstaller extends ModelInstaller {
Mappings.Source source = new Mappings.Source();
for (ModelColumn columnDefine : model.getColumns()) {
final String type = columnTypeEsMapping.transform(columnDefine.getType(), columnDefine.getGenericType());
if (columnDefine.isMatchQuery()) {
if (columnDefine.getElasticSearchExtension().needMatchQuery()) {
String matchCName = MatchCNameBuilder.INSTANCE.build(columnDefine.getColumnName().getName());
Map<String, Object> originalColumn = new HashMap<>();
@ -237,7 +241,7 @@ public class StorageEsInstaller extends ModelInstaller {
Map<String, Object> matchColumn = new HashMap<>();
matchColumn.put("type", "text");
matchColumn.put("analyzer", columnDefine.getAnalyzer().getName());
matchColumn.put("analyzer", columnDefine.getElasticSearchExtension().getAnalyzer().getName());
properties.put(matchCName, matchColumn);
} else {
Map<String, Object> column = new HashMap<>();

6
oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/TimeSeriesUtilsTest.java
View File

@ -36,13 +36,13 @@ public class TimeSeriesUtilsTest {
@Before
public void prepare() {
superDatasetModel = new Model("superDatasetModel", Lists.newArrayList(), Lists.newArrayList(),
superDatasetModel = new Model("superDatasetModel", Lists.newArrayList(),
0, DownSampling.Minute, true, true, "", true
);
normalRecordModel = new Model("normalRecordModel", Lists.newArrayList(), Lists.newArrayList(),
normalRecordModel = new Model("normalRecordModel", Lists.newArrayList(),
0, DownSampling.Minute, true, false, "", true
);
normalMetricsModel = new Model("normalMetricsModel", Lists.newArrayList(), Lists.newArrayList(),
normalMetricsModel = new Model("normalMetricsModel", Lists.newArrayList(),
0, DownSampling.Minute, false, false, "", true
);
TimeSeriesUtils.setSUPER_DATASET_DAY_STEP(1);

33
oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java
View File

@ -25,9 +25,9 @@ import java.sql.SQLException;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.oap.server.core.storage.StorageException;
import org.apache.skywalking.oap.server.core.storage.model.ExtraQueryIndex;
import org.apache.skywalking.oap.server.core.storage.model.Model;
import org.apache.skywalking.oap.server.core.storage.model.ModelColumn;
import org.apache.skywalking.oap.server.core.storage.model.SQLDatabaseExtension;
import org.apache.skywalking.oap.server.core.storage.type.StorageDataComplexObject;
import org.apache.skywalking.oap.server.library.client.Client;
import org.apache.skywalking.oap.server.library.client.jdbc.JDBCClientException;
@ -104,22 +104,25 @@ public class MySQLTableInstaller extends H2TableInstaller {
}
}
for (final ExtraQueryIndex extraQueryIndex : model.getExtraQueryIndices()) {
SQLBuilder tableIndexSQL = new SQLBuilder("CREATE INDEX ");
tableIndexSQL.append(model.getName().toUpperCase())
.append("_")
.append(String.valueOf(indexSeq++))
.append("_IDX ");
tableIndexSQL.append(" ON ").append(model.getName()).append("(");
final String[] columns = extraQueryIndex.getColumns();
for (int i = 0; i < columns.length; i++) {
tableIndexSQL.append(columns[i]);
if (i < columns.length - 1) {
tableIndexSQL.append(",");
for (final ModelColumn modelColumn : model.getColumns()) {
for (final SQLDatabaseExtension.MultiColumnsIndex index : modelColumn.getSqlDatabaseExtension()
.getIndices()) {
SQLBuilder tableIndexSQL = new SQLBuilder("CREATE INDEX ");
tableIndexSQL.append(model.getName().toUpperCase())
.append("_")
.append(String.valueOf(indexSeq++))
.append("_IDX ");
tableIndexSQL.append(" ON ").append(model.getName()).append("(");
final String[] columns = index.getColumns();
for (int i = 0; i < columns.length; i++) {
tableIndexSQL.append(columns[i]);
if (i < columns.length - 1) {
tableIndexSQL.append(",");
}
}
tableIndexSQL.append(")");
createIndex(client, connection, model, tableIndexSQL);
}
tableIndexSQL.append(")");
createIndex(client, connection, model, tableIndexSQL);
}
}

12
oap-server/server-storage-plugin/storage-zipkin-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java
View File

@ -25,6 +25,7 @@ import org.apache.skywalking.oap.server.core.analysis.Stream;
import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
@ -60,15 +61,17 @@ public class ZipkinSpanRecord extends Record {
private String spanId;
@Setter
@Getter
@Column(columnName = SERVICE_ID, shardingKeyIdx = 0)
@Column(columnName = SERVICE_ID)
@BanyanDBShardingKey(index = 0)
private String serviceId;
@Setter
@Getter
@Column(columnName = SERVICE_INSTANCE_ID, shardingKeyIdx = 1)
@Column(columnName = SERVICE_INSTANCE_ID)
@BanyanDBShardingKey(index = 1)
private String serviceInstanceId;
@Setter
@Getter
@Column(columnName = ENDPOINT_NAME, matchQuery = true)
@Column(columnName = ENDPOINT_NAME)
private String endpointName;
@Setter
@Getter
@ -88,7 +91,8 @@ public class ZipkinSpanRecord extends Record {
private int latency;
@Setter
@Getter
@Column(columnName = IS_ERROR, shardingKeyIdx = 2)
@Column(columnName = IS_ERROR)
@BanyanDBShardingKey(index = 2)
private int isError;
@Setter
@Getter