Bump BanyanDB Java Client to 0.6.0 (#12195)

This commit is contained in:
Gao Hongtao 2024-05-09 17:31:52 +08:00 committed by GitHub
parent b3044ee714
commit 28bf55219f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
30 changed files with 235 additions and 140 deletions

View File

@ -236,7 +236,7 @@ The text of each license is the standard Apache 2.0 license.
https://mvnrepository.com/artifact/com.google.guava/guava/32.0.1-jre Apache-2.0
https://mvnrepository.com/artifact/com.google.guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava Apache-2.0
https://mvnrepository.com/artifact/com.google.inject/guice/4.1.0 Apache-2.0
https://mvnrepository.com/artifact/com.google.j2objc/j2objc-annotations/1.3 Apache-2.0
https://mvnrepository.com/artifact/com.google.j2objc/j2objc-annotations/2.8 Apache-2.0
https://mvnrepository.com/artifact/com.graphql-java/java-dataloader/3.2.0 Apache-2.0
https://mvnrepository.com/artifact/com.linecorp.armeria/armeria/1.27.3 Apache-2.0
https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-graphql/1.27.3 Apache-2.0
@ -289,15 +289,15 @@ The text of each license is the standard Apache 2.0 license.
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-scheduling/6.7.1 Apache-2.0
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-storageclass/6.7.1 Apache-2.0
https://mvnrepository.com/artifact/io.fabric8/zjsonpatch/0.3.0 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-api/1.62.2 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-context/1.62.2 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-core/1.62.2 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-grpclb/1.62.2 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-netty/1.62.2 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-protobuf/1.62.2 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-protobuf-lite/1.62.2 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-stub/1.62.2 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-util/1.62.2 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-api/1.63.0 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-context/1.63.0 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-core/1.63.0 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-grpclb/1.63.0 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-netty/1.63.0 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-protobuf/1.63.0 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-protobuf-lite/1.63.0 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-stub/1.63.0 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-util/1.63.0 Apache-2.0
https://mvnrepository.com/artifact/io.micrometer/micrometer-commons/1.12.2 Apache-2.0
https://mvnrepository.com/artifact/io.micrometer/micrometer-core/1.12.2 Apache-2.0
https://mvnrepository.com/artifact/io.micrometer/micrometer-observation/1.12.2 Apache-2.0
@ -389,8 +389,8 @@ BSD-3-Clause licenses
The following components are provided under the BSD-3-Clause License. See project link for details.
The text of each license is also included in licenses/LICENSE-[project].txt.
https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java/3.21.8 BSD-3-Clause
https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java-util/3.21.8 BSD-3-Clause
https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java/3.25.1 BSD-3-Clause
https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java-util/3.25.1 BSD-3-Clause
https://npmjs.com/package/d3-collection/v/1.0.7 1.0.7 BSD-3-Clause
https://npmjs.com/package/d3-ease/v/3.0.1 3.0.1 BSD-3-Clause
https://npmjs.com/package/d3-tip/node_modules/d3-selection/v/1.4.2 1.4.2 BSD-3-Clause

View File

@ -43,7 +43,7 @@ services:
banyandb:
profiles:
- "banyandb"
image: ${BANYANDB_IMAGE:-ghcr.io/apache/skywalking-banyandb:84f32b3969cdcc676aaee428383b34b3b67dbdf5}
image: ${BANYANDB_IMAGE:-ghcr.io/apache/skywalking-banyandb:a091ac0c3efa7305288ae9fb8853bffb2186583a}
container_name: banyandb
networks:
- demo

View File

@ -10,6 +10,7 @@
* Remove CLI_VERSION variable from Makefile build.
* Add BanyanDB to docker-compose quickstart.
* Bump up Armeria, jackson, netty, jetcd and grpc to fix CVEs.
* Bump up BanyanDB Java Client to 0.6.0.
#### OAP Server

View File

@ -3,24 +3,55 @@
[BanyanDB](https://github.com/apache/skywalking-banyandb) is a dedicated storage implementation developed by the SkyWalking Team and the community.
Activate BanyanDB as the storage, and set storage provider to **banyandb**.
The OAP requires BanyanDB 0.5 server. As BanyanDB is still in the beta phase, we don't provide any compatibility besides the required version.
The OAP requires BanyanDB 0.6 server. From this version, BanyanDB provides general compatibility.
```yaml
storage:
banyandb:
# Targets is the list of BanyanDB servers, separated by commas.
# Each target is a BanyanDB server in the format of `host:port`
# The host is the IP address or domain name of the BanyanDB server, and the port is the port number of the BanyanDB server.
targets: ${SW_STORAGE_BANYANDB_TARGETS:127.0.0.1:17912}
# The max number of records in a bulk write request.
# Bigger value can improve the write performance, but also increase the OAP and BanyanDB Server memory usage.
maxBulkSize: ${SW_STORAGE_BANYANDB_MAX_BULK_SIZE:5000}
# The minimum seconds between two bulk flushes.
# If the data in a bulk is less than maxBulkSize, the data will be flushed after this period.
# If the data in a bulk is more than maxBulkSize, the data will be flushed immediately.
# Bigger value can reduce the write pressure on BanyanDB Server, but also increase the latency of the data.
flushInterval: ${SW_STORAGE_BANYANDB_FLUSH_INTERVAL:15}
# The shard number of `measure` groups that store the metrics data.
metricsShardsNumber: ${SW_STORAGE_BANYANDB_METRICS_SHARDS_NUMBER:1}
# The shard number of `stream` groups that store the trace, log and profile data.
recordShardsNumber: ${SW_STORAGE_BANYANDB_RECORD_SHARDS_NUMBER:1}
# The multiplier of the number of shards of the super dataset.
# Super dataset is a special dataset that stores the trace or log data that is too large to be stored in the normal dataset.
# If the normal dataset has `n` shards, the super dataset will have `n * superDatasetShardsFactor` shards.
# For example, supposing `recordShardsNumber` is 3, and `superDatasetShardsFactor` is 2,
# `segment-default` is a normal dataset that has 3 shards, and `segment-minute` is a super dataset that has 6 shards.
superDatasetShardsFactor: ${SW_STORAGE_BANYANDB_SUPERDATASET_SHARDS_FACTOR:2}
# The number of threads that write data to BanyanDB concurrently.
# Bigger value can improve the write performance, but also increase the OAP and BanyanDB Server CPU usage.
concurrentWriteThreads: ${SW_STORAGE_BANYANDB_CONCURRENT_WRITE_THREADS:15}
profileTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_PROFILE_TASK_QUERY_MAX_SIZE:200} # the max number of fetch task in a request
streamBlockInterval: ${SW_STORAGE_BANYANDB_STREAM_BLOCK_INTERVAL:4} # Unit is hour
streamSegmentInterval: ${SW_STORAGE_BANYANDB_STREAM_SEGMENT_INTERVAL:24} # Unit is hour
measureBlockInterval: ${SW_STORAGE_BANYANDB_MEASURE_BLOCK_INTERVAL:4} # Unit is hour
measureSegmentInterval: ${SW_STORAGE_BANYANDB_MEASURE_SEGMENT_INTERVAL:24} # Unit is hour
# The max number of profile task query in a request.
profileTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_PROFILE_TASK_QUERY_MAX_SIZE:200}
# Data is stored in BanyanDB in segments. A segment is a time range of data.
# The segment interval is the time range of a segment.
# The value should be less or equal to data TTL relevant settings.
segmentIntervalDays: ${SW_STORAGE_BANYANDB_SEGMENT_INTERVAL_DAYS:1}
# The super dataset segment interval is the time range of a segment in the super dataset.
superDatasetSegmentIntervalDays: ${SW_STORAGE_BANYANDB_SUPER_DATASET_SEGMENT_INTERVAL_DAYS:1}
# Specific groups settings.
# For example, {"group1": {"blockIntervalHours": 4, "segmentIntervalDays": 1}}
# Please refer to https://github.com/apache/skywalking-banyandb/blob/${BANYANDB_RELEASE}/docs/crud/group.md#create-operation
# for group setting details.
specificGroupSettings: ${SW_STORAGE_BANYANDB_SPECIFIC_GROUP_SETTINGS:""}
```
For more details, please refer to the documents of [BanyanDB](https://skywalking.apache.org/docs/skywalking-banyandb/next/readme/)
BanyanDB Server supports two installation modes: standalone and cluster. The standalone mode is suitable for small-scale deployments, while the cluster mode is suitable for large-scale deployments.
* Standalone mode: `targets` is the IP address/host name and port of the BanyanDB server.
* Cluster mode: `targets` is the IP address/host name and port of the `liaison` nodes, separated by commas. `liaison` nodes are the entry points of the BanyanDB cluster.
For more details, please refer to the documents of [BanyanDB](https://skywalking.apache.org/docs/skywalking-banyandb/latest/readme/)
and [BanyanDB Java Client](https://github.com/apache/skywalking-banyandb-java-client) subprojects.

View File

@ -41,8 +41,8 @@
<zookeeper.version>3.5.7</zookeeper.version>
<guava.version>32.0.1-jre</guava.version>
<snakeyaml.version>2.0</snakeyaml.version>
<protobuf-java.version>3.21.8</protobuf-java.version>
<protobuf-java-util.version>3.21.8</protobuf-java-util.version>
<protobuf-java.version>3.25.1</protobuf-java.version>
<protobuf-java-util.version>3.25.1</protobuf-java-util.version>
<commons-codec.version>1.11</commons-codec.version>
<commons-lang3.version>3.12.0</commons-lang3.version>
<commons-io.version>2.7</commons-io.version>
@ -72,7 +72,7 @@
<awaitility.version>3.0.0</awaitility.version>
<httpcore.version>4.4.13</httpcore.version>
<commons-compress.version>1.21</commons-compress.version>
<banyandb-java-client.version>0.5.0</banyandb-java-client.version>
<banyandb-java-client.version>0.6.0</banyandb-java-client.version>
<kafka-clients.version>3.4.0</kafka-clients.version>
<spring-kafka-test.version>2.4.6.RELEASE</spring-kafka-test.version>
<consul.client.version>1.5.3</consul.client.version>

View File

@ -68,12 +68,10 @@ public abstract class AbstractLogRecord extends Record {
@Setter
@Getter
@Column(name = TRACE_ID, length = 150)
@BanyanDB.GlobalIndex
private String traceId;
@Setter
@Getter
@Column(name = TRACE_SEGMENT_ID, length = 150)
@BanyanDB.GlobalIndex
private String traceSegmentId;
@Setter
@Getter

View File

@ -65,7 +65,6 @@ public class SegmentRecord extends Record {
@Setter
@Getter
@Column(name = TRACE_ID, length = 150)
@BanyanDB.GlobalIndex
@ElasticSearch.Routing
private String traceId;
@Setter

View File

@ -67,7 +67,6 @@ public class SpanAttachedEventRecord extends Record {
@Column(name = TRACE_REF_TYPE)
private int traceRefType;
@Column(name = RELATED_TRACE_ID)
@BanyanDB.GlobalIndex
private String relatedTraceId;
@Column(name = TRACE_SEGMENT_ID)
private String traceSegmentId;

View File

@ -32,21 +32,6 @@ import org.apache.skywalking.oap.server.core.analysis.record.Record;
* @since 9.1.0
*/
public @interface BanyanDB {
/**
* GlobalIndex declares advanced global index, which are only available in BanyanDB.
* <p>
* Global index should only be considered if a column value has a huge value candidates, but we will need a direct
* equal
* query without timestamp.
* The typical global index is designed for huge candidate of indexed values, such as `trace ID` or `segment ID`.
* <p>
* Only work with {@link Column}
*/
@Target({ElementType.FIELD})
@Retention(RetentionPolicy.RUNTIME)
@interface GlobalIndex {
}
/**
* Series key is used to group time series data per metric of one entity in one place.
* <p>
@ -92,8 +77,7 @@ public @interface BanyanDB {
/**
* Force disabling indexing declare through {@link Column}.
* In BanyanDB, {@link GlobalIndex} provides the high performance capability to filter trace/log from super large
* dataset, some additional conditions could be done in server memory, no indexing required in this case.
* In BanyanDB, some additional conditions could be done in server memory, no indexing required in this case.
*
* @since 9.1.0
*/

View File

@ -39,14 +39,6 @@ public class BanyanDBExtension {
@Getter
private final int shardingKeyIdx;
/**
* Global index column.
*
* @since 9.1.0
*/
@Getter
private final boolean isGlobalIndexing;
/**
* {@link BanyanDB.NoIndexing} exists to override {@link ModelColumn#shouldIndex()}, or be the same as {@link
* ModelColumn#shouldIndex()}.

View File

@ -207,8 +207,6 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
// BanyanDB extension
final BanyanDB.SeriesID banyanDBSeriesID = field.getAnnotation(
BanyanDB.SeriesID.class);
final BanyanDB.GlobalIndex banyanDBGlobalIndex = field.getAnnotation(
BanyanDB.GlobalIndex.class);
final BanyanDB.NoIndexing banyanDBNoIndex = field.getAnnotation(
BanyanDB.NoIndexing.class);
final BanyanDB.IndexRule banyanDBIndexRule = field.getAnnotation(
@ -219,7 +217,6 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
BanyanDB.TopNAggregation.class);
BanyanDBExtension banyanDBExtension = new BanyanDBExtension(
banyanDBSeriesID == null ? -1 : banyanDBSeriesID.index(),
banyanDBGlobalIndex != null,
banyanDBNoIndex == null && !column.storageOnly(),
banyanDBIndexRule == null ? BanyanDB.IndexRule.IndexType.INVERTED : banyanDBIndexRule.indexType(),
banyanDBMeasureField != null

View File

@ -49,7 +49,7 @@ public class ModelColumnTest {
new SQLDatabaseExtension(),
new ElasticSearchExtension(
ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, null, false, false),
new BanyanDBExtension(-1, false, true, BanyanDB.IndexRule.IndexType.INVERTED, false)
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false)
);
Assertions.assertTrue(column.isStorageOnly());
Assertions.assertEquals("abc", column.getColumnName().getName());
@ -58,7 +58,7 @@ public class ModelColumnTest {
false, false, true, 200,
new SQLDatabaseExtension(),
new ElasticSearchExtension(ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, null, false, false),
new BanyanDBExtension(-1, false, true, BanyanDB.IndexRule.IndexType.INVERTED, false)
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false)
);
Assertions.assertTrue(column.isStorageOnly());
Assertions.assertEquals("abc", column.getColumnName().getName());
@ -68,7 +68,7 @@ public class ModelColumnTest {
false, false, true, 200,
new SQLDatabaseExtension(),
new ElasticSearchExtension(ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, null, false, false),
new BanyanDBExtension(-1, false, true, BanyanDB.IndexRule.IndexType.INVERTED, false)
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false)
);
Assertions.assertFalse(column.isStorageOnly());
Assertions.assertEquals("abc", column.getColumnName().getName());
@ -82,7 +82,7 @@ public class ModelColumnTest {
new SQLDatabaseExtension(),
new ElasticSearchExtension(
ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, "abc", false, false),
new BanyanDBExtension(-1, false, true, BanyanDB.IndexRule.IndexType.INVERTED, false)
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false)
);
});
}
@ -95,7 +95,7 @@ public class ModelColumnTest {
new SQLDatabaseExtension(),
new ElasticSearchExtension(
ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, "abc", false, false),
new BanyanDBExtension(-1, false, true, BanyanDB.IndexRule.IndexType.INVERTED, false)
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false)
);
});
}

View File

@ -216,19 +216,44 @@ storage:
maxSizeOfBatchSql: ${SW_STORAGE_MAX_SIZE_OF_BATCH_SQL:2000}
asyncBatchPersistentPoolSize: ${SW_STORAGE_ASYNC_BATCH_PERSISTENT_POOL_SIZE:4}
banyandb:
# Targets is the list of BanyanDB servers, separated by commas.
# Each target is a BanyanDB server in the format of `host:port`
# The host is the IP address or domain name of the BanyanDB server, and the port is the port number of the BanyanDB server.
targets: ${SW_STORAGE_BANYANDB_TARGETS:127.0.0.1:17912}
# The max number of records in a bulk write request.
# Bigger value can improve the write performance, but also increase the OAP and BanyanDB Server memory usage.
maxBulkSize: ${SW_STORAGE_BANYANDB_MAX_BULK_SIZE:5000}
# The minimum seconds between two bulk flushes.
# If the data in a bulk is less than maxBulkSize, the data will be flushed after this period.
# If the data in a bulk is more than maxBulkSize, the data will be flushed immediately.
# Bigger value can reduce the write pressure on BanyanDB Server, but also increase the latency of the data.
flushInterval: ${SW_STORAGE_BANYANDB_FLUSH_INTERVAL:15}
# The shard number of `measure` groups that store the metrics data.
metricsShardsNumber: ${SW_STORAGE_BANYANDB_METRICS_SHARDS_NUMBER:1}
# The shard number of `stream` groups that store the trace, log and profile data.
recordShardsNumber: ${SW_STORAGE_BANYANDB_RECORD_SHARDS_NUMBER:1}
# The multiplier of the number of shards of the super dataset.
# Super dataset is a special dataset that stores the trace or log data that is too large to be stored in the normal dataset.
# If the normal dataset has `n` shards, the super dataset will have `n * superDatasetShardsFactor` shards.
# For example, supposing `recordShardsNumber` is 3, and `superDatasetShardsFactor` is 2,
# `segment-default` is a normal dataset that has 3 shards, and `segment-minute` is a super dataset that has 6 shards.
superDatasetShardsFactor: ${SW_STORAGE_BANYANDB_SUPERDATASET_SHARDS_FACTOR:2}
# The number of threads that write data to BanyanDB concurrently.
# Bigger value can improve the write performance, but also increase the OAP and BanyanDB Server CPU usage.
concurrentWriteThreads: ${SW_STORAGE_BANYANDB_CONCURRENT_WRITE_THREADS:15}
profileTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_PROFILE_TASK_QUERY_MAX_SIZE:200} # the max number of fetch task in a request
blockIntervalHours: ${SW_STORAGE_BANYANDB_BLOCK_INTERVAL_HOURS:24} # Unit is hour
segmentIntervalDays: ${SW_STORAGE_BANYANDB_SEGMENT_INTERVAL_DAYS:1} # Unit is day
superDatasetBlockIntervalHours: ${SW_STORAGE_BANYANDB_SUPER_DATASET_BLOCK_INTERVAL_HOURS:4} # Unit is hour
superDatasetSegmentIntervalDays: ${SW_STORAGE_BANYANDB_SUPER_DATASET_SEGMENT_INTERVAL_DAYS:1} # Unit is day
specificGroupSettings: ${SW_STORAGE_BANYANDB_SPECIFIC_GROUP_SETTINGS:""} # For example, {"group1": {"blockIntervalHours": 4, "segmentIntervalDays": 1}}
# The max number of profile task query in a request.
profileTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_PROFILE_TASK_QUERY_MAX_SIZE:200}
# Data is stored in BanyanDB in segments. A segment is a time range of data.
# The segment interval is the time range of a segment.
# The value should be less or equal to data TTL relevant settings.
segmentIntervalDays: ${SW_STORAGE_BANYANDB_SEGMENT_INTERVAL_DAYS:1}
# The super dataset segment interval is the time range of a segment in the super dataset.
superDatasetSegmentIntervalDays: ${SW_STORAGE_BANYANDB_SUPER_DATASET_SEGMENT_INTERVAL_DAYS:1}
# Specific groups settings.
# For example, {"group1": {"blockIntervalHours": 4, "segmentIntervalDays": 1}}
# Please refer to https://github.com/apache/skywalking-banyandb/blob/${BANYANDB_RELEASE}/docs/crud/group.md#create-operation
# for group setting details.
specificGroupSettings: ${SW_STORAGE_BANYANDB_SPECIFIC_GROUP_SETTINGS:""}
agent-analyzer:
selector: ${SW_AGENT_ANALYZER:default}

View File

@ -62,13 +62,6 @@ public class BanyanDBStorageConfig extends ModuleConfig {
* Shards Factor for a super dataset
*/
private int superDatasetShardsFactor;
/**
* Default global block interval for non-super-dataset models.
* Unit is hour.
*
* @since 9.4.0
*/
private int blockIntervalHours;
/**
* Default global segment interval for non-super-dataset models.
* Unit is day.
@ -76,13 +69,6 @@ public class BanyanDBStorageConfig extends ModuleConfig {
* @since 9.4.0
*/
private int segmentIntervalDays;
/**
* Default global block interval for super-dataset models.
* Unit is hour.
*
* @since 9.4.0
*/
private int superDatasetBlockIntervalHours;
/**
* Default global segment interval for super-dataset models.
* Unit is day.

View File

@ -19,7 +19,6 @@
package org.apache.skywalking.oap.server.storage.plugin.banyandb;
import org.apache.skywalking.banyandb.v1.client.metadata.Group;
import org.apache.skywalking.banyandb.v1.client.metadata.IntervalRule;
import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.storage.IBatchDAO;
import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO;
@ -89,8 +88,6 @@ public class BanyanDBStorageProvider extends ModuleProvider {
private BanyanDBStorageClient client;
private ModelInstaller modelInstaller;
private IntervalRule bypass = IntervalRule.create(IntervalRule.Unit.UNSPECIFIED, 0);
@Override
public String name() {
return "banyandb";

View File

@ -69,7 +69,6 @@ import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.config.ConfigService;
import org.apache.skywalking.oap.server.core.query.enumeration.Step;
import org.apache.skywalking.oap.server.core.storage.StorageException;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
import org.apache.skywalking.oap.server.core.storage.model.Model;
@ -156,7 +155,7 @@ public enum MetadataRegistry {
.collect(Collectors.toList());
if (model.getBanyanDBModelExtension().isStoreIDTag()) {
indexRules.add(IndexRule.create(BanyanDBConverter.ID, IndexRule.IndexType.TREE, IndexRule.IndexLocation.SERIES));
indexRules.add(IndexRule.create(BanyanDBConverter.ID, IndexRule.IndexType.INVERTED));
}
final Measure.Builder builder = Measure.create(schemaMetadata.getGroup(), schemaMetadata.name(),
@ -285,25 +284,8 @@ public enum MetadataRegistry {
}
}
IndexRule parseIndexRule(String tagName, ModelColumn modelColumn) {
// TODO: we need to add support index type in the OAP core
// Currently, we only register INVERTED type
// if it is null, it must be a user-defined tag
if (modelColumn == null) {
return IndexRule.create(tagName, IndexRule.IndexType.INVERTED, IndexRule.IndexLocation.SERIES);
}
if (modelColumn.getBanyanDBExtension().isGlobalIndexing()) {
return IndexRule.create(tagName, parseIndexType(modelColumn.getBanyanDBExtension().getIndexType()), IndexRule.IndexLocation.GLOBAL);
} else {
return IndexRule.create(tagName, parseIndexType(modelColumn.getBanyanDBExtension().getIndexType()), IndexRule.IndexLocation.SERIES);
}
}
private static IndexRule.IndexType parseIndexType(BanyanDB.IndexRule.IndexType indexType) {
if (indexType == BanyanDB.IndexRule.IndexType.INVERTED) {
return IndexRule.IndexType.INVERTED;
}
return IndexRule.IndexType.TREE;
IndexRule indexRule(String tagName) {
return IndexRule.create(tagName, IndexRule.IndexType.INVERTED);
}
/**
@ -343,9 +325,7 @@ public enum MetadataRegistry {
builder.spec(columnStorageName, new ColumnSpec(ColumnType.TAG, col.getType()));
String colName = col.getColumnName().getStorageName();
if (!shardingColumns.contains(colName) && col.getBanyanDBExtension().shouldIndex()) {
// build indexRule
IndexRule indexRule = parseIndexRule(tagSpec.getTagName(), col);
tagMetadataList.add(new TagMetadata(indexRule, tagSpec));
tagMetadataList.add(new TagMetadata(indexRule(tagSpec.getTagName()), tagSpec));
} else {
tagMetadataList.add(new TagMetadata(null, tagSpec));
}
@ -385,7 +365,7 @@ public enum MetadataRegistry {
final TagFamilySpec.TagSpec tagSpec = parseTagSpec(col);
builder.spec(columnStorageName, new ColumnSpec(ColumnType.TAG, col.getType()));
String colName = col.getColumnName().getStorageName();
result.tag(new TagMetadata(!shardingColumns.contains(colName) && col.getBanyanDBExtension().shouldIndex() ? parseIndexRule(tagSpec.getTagName(), col) : null, tagSpec));
result.tag(new TagMetadata(!shardingColumns.contains(colName) && col.getBanyanDBExtension().shouldIndex() ? indexRule(tagSpec.getTagName()) : null, tagSpec));
}
return result.build();
@ -443,10 +423,8 @@ public enum MetadataRegistry {
}
public SchemaMetadata parseMetadata(Model model, BanyanDBStorageConfig config, ConfigService configService) {
int blockIntervalHrs = config.getBlockIntervalHours();
int segmentIntervalDays = config.getSegmentIntervalDays();
if (model.isSuperDataset()) {
blockIntervalHrs = config.getSuperDatasetBlockIntervalHours();
segmentIntervalDays = config.getSuperDatasetSegmentIntervalDays();
}
String group;
@ -460,7 +438,6 @@ public enum MetadataRegistry {
} else if (model.getDownsampling() == DownSampling.Minute && model.isTimeRelativeID()) { // measure
group = "measure-minute";
// apply super dataset's settings to measure-minute
blockIntervalHrs = config.getSuperDatasetBlockIntervalHours();
segmentIntervalDays = config.getSuperDatasetSegmentIntervalDays();
metricShardNum = metricShardNum * config.getSuperDatasetShardsFactor();
} else {
@ -471,14 +448,12 @@ public enum MetadataRegistry {
return new SchemaMetadata("measure-default", model.getName(), Kind.MEASURE,
model.getDownsampling(),
config.getMetricsShardsNumber(),
intervalDays * 24,
intervalDays, // use 10-day/240-hour strategy
configService.getMetricsDataTTL());
}
GroupSetting groupSetting = this.specificGroupSettings.get(group);
if (groupSetting != null) {
blockIntervalHrs = groupSetting.getBlockIntervalHours();
segmentIntervalDays = groupSetting.getSegmentIntervalDays();
}
if (model.isRecord()) {
@ -488,7 +463,6 @@ public enum MetadataRegistry {
model.getDownsampling(),
config.getRecordShardsNumber() *
(model.isSuperDataset() ? config.getSuperDatasetShardsFactor() : 1),
blockIntervalHrs,
segmentIntervalDays,
configService.getRecordDataTTL()
);
@ -497,7 +471,6 @@ public enum MetadataRegistry {
return new SchemaMetadata(group, model.getName(), Kind.MEASURE,
model.getDownsampling(),
metricShardNum,
blockIntervalHrs,
segmentIntervalDays,
configService.getMetricsDataTTL());
}
@ -517,7 +490,6 @@ public enum MetadataRegistry {
*/
private final DownSampling downSampling;
private final int shard;
private final int blockIntervalHrs;
private final int segmentIntervalDays;
private final int ttlDays;
@ -580,7 +552,6 @@ public enum MetadataRegistry {
resourceExist = client.existStream(this.group, this.name());
if (!resourceExist.hasGroup()) {
Group g = client.define(Group.create(this.group, Catalog.STREAM, this.shard,
IntervalRule.create(IntervalRule.Unit.HOUR, this.blockIntervalHrs),
IntervalRule.create(IntervalRule.Unit.DAY, this.segmentIntervalDays),
IntervalRule.create(IntervalRule.Unit.DAY, this.ttlDays)));
if (g != null) {
@ -592,7 +563,6 @@ public enum MetadataRegistry {
resourceExist = client.existMeasure(this.group, this.name());
if (!resourceExist.hasGroup()) {
Group g = client.define(Group.create(this.group, Catalog.MEASURE, this.shard,
IntervalRule.create(IntervalRule.Unit.HOUR, this.blockIntervalHrs),
IntervalRule.create(IntervalRule.Unit.DAY, this.segmentIntervalDays),
IntervalRule.create(IntervalRule.Unit.DAY, this.ttlDays)));
if (g != null) {
@ -726,7 +696,6 @@ public enum MetadataRegistry {
@Setter
@NoArgsConstructor
public static class GroupSetting {
private int blockIntervalHours;
private int segmentIntervalDays;
}
}

View File

@ -155,9 +155,9 @@ public class BanyanDBEventQueryDAO extends AbstractBanyanDBDAO implements IEvent
event.setUuid(dataPoint.getTagValue(Event.UUID));
String service = dataPoint.getTagValue(Event.SERVICE);
String serviceInstance = dataPoint.getTagValue(Event.SERVICE_INSTANCE);
String endpoint = dataPoint.getTagValue(Event.ENDPOINT);
String service = getValueOrDefault(dataPoint, Event.SERVICE, "");
String serviceInstance = getValueOrDefault(dataPoint, Event.SERVICE_INSTANCE, "");
String endpoint = getValueOrDefault(dataPoint, Event.ENDPOINT, "");
event.setSource(new Source(service, serviceInstance, endpoint));
event.setName(dataPoint.getTagValue(Event.NAME));
@ -171,4 +171,9 @@ public class BanyanDBEventQueryDAO extends AbstractBanyanDBDAO implements IEvent
return event;
}
private <T> T getValueOrDefault(DataPoint dataPoint, String tagName, T defaultValue) {
T v = dataPoint.getTagValue(tagName);
return v == null ? defaultValue : v;
}
}

View File

@ -65,9 +65,9 @@ public abstract class AbstractBanyanDBDAO extends AbstractDAO<BanyanDBStorageCli
}
final StreamQuery query;
if (timestampRange == null) {
query = new StreamQuery(schema.getMetadata().getGroup(), schema.getMetadata().name(), LARGEST_TIME_RANGE, tags);
query = new StreamQuery(List.of(schema.getMetadata().getGroup()), schema.getMetadata().name(), LARGEST_TIME_RANGE, tags);
} else {
query = new StreamQuery(schema.getMetadata().getGroup(), schema.getMetadata().name(), timestampRange, tags);
query = new StreamQuery(List.of(schema.getMetadata().getGroup()), schema.getMetadata().name(), timestampRange, tags);
}
builder.apply(query);
@ -92,7 +92,7 @@ public abstract class AbstractBanyanDBDAO extends AbstractDAO<BanyanDBStorageCli
private TopNQueryResponse topNQuery(MetadataRegistry.Schema schema, TimestampRange timestampRange, int number,
AbstractQuery.Sort sort, List<KeyValue> additionalConditions) throws IOException {
final TopNQuery q = new TopNQuery(schema.getMetadata().getGroup(), schema.getTopNSpec().getName(),
final TopNQuery q = new TopNQuery(List.of(schema.getMetadata().getGroup()), schema.getTopNSpec().getName(),
timestampRange,
number, sort);
q.setAggregationType(MeasureQuery.Aggregation.Type.MEAN);
@ -119,9 +119,9 @@ public abstract class AbstractBanyanDBDAO extends AbstractDAO<BanyanDBStorageCli
TimestampRange timestampRange, QueryBuilder<MeasureQuery> builder) throws IOException {
final MeasureQuery query;
if (timestampRange == null) {
query = new MeasureQuery(schema.getMetadata().getGroup(), schema.getMetadata().name(), LARGEST_TIME_RANGE, tags, fields);
query = new MeasureQuery(List.of(schema.getMetadata().getGroup()), schema.getMetadata().name(), LARGEST_TIME_RANGE, tags, fields);
} else {
query = new MeasureQuery(schema.getMetadata().getGroup(), schema.getMetadata().name(), timestampRange, tags, fields);
query = new MeasureQuery(List.of(schema.getMetadata().getGroup()), schema.getMetadata().name(), timestampRange, tags, fields);
}
builder.apply(query);

View File

@ -35,6 +35,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
@Type(value = FixedStringGenerator.class, name = "fixedString"),
@Type(value = SequenceGenerator.class, name = "sequence"),
@Type(value = UUIDGenerator.class, name = "uuid"),
@Type(value = TimeGenerator.class, name = "time"),
})
public interface Generator<I, T> {
public T next(I input);

View File

@ -0,0 +1,73 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.skywalking.generator;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import lombok.Data;
import java.util.concurrent.atomic.AtomicLong;
@JsonDeserialize(builder = TimeGenerator.Builder.class)
public class TimeGenerator implements Generator<Object, Long> {
private final Long stepMillisecond;
private final Long waitMillisecond;
private AtomicLong last;
private AtomicLong lastSync;
public TimeGenerator(Builder builder) {
this.waitMillisecond = builder.waitMillisecond;
this.stepMillisecond = builder.stepMillisecond;
reset();
}
@Override
public Long next(Object ignored) {
long current = System.currentTimeMillis();
if (current - lastSync.longValue() < waitMillisecond) {
// in case of the time is not sync
return last.incrementAndGet();
}
lastSync.set(current);
return last.addAndGet(stepMillisecond);
}
@Override
public void reset() {
last = new AtomicLong(System.currentTimeMillis());
lastSync = new AtomicLong(last.longValue());
}
@Data
public static class Builder {
private Long stepMillisecond;
private Long waitMillisecond;
public TimeGenerator build() {
if (stepMillisecond == null) {
stepMillisecond = 1000L;
}
if (waitMillisecond == null) {
waitMillisecond = 0L;
}
return new TimeGenerator(this);
}
}
}

View File

@ -24,6 +24,7 @@ import com.google.common.base.Strings;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject;
import org.apache.skywalking.apm.network.language.agent.v3.SegmentReference;
import org.apache.skywalking.apm.network.language.agent.v3.SpanObject;
@ -42,6 +43,7 @@ import java.util.stream.IntStream;
@Getter
@Setter
@JsonTypeInfo(use = JsonTypeInfo.Id.NONE)
@Slf4j
public class SegmentGenerator implements Generator<SegmentGenerator.SegmentContext, SegmentGenerator.SegmentResult> {
private Generator<String, String> segmentId;
@ -49,10 +51,11 @@ public class SegmentGenerator implements Generator<SegmentGenerator.SegmentConte
private Generator<Object, Long> error;
private Generator<Object, List<TagGenerator>> tags;
private Generator<Object, List<SpanGenerator>> spans;
private Generator<Object, Long> now;
@Override
public SegmentResult next(SegmentContext ctx) {
long now = System.currentTimeMillis();
long n = now.next(null);
final String serviceName = ctx.serviceName;
final String serviceInstanceName = ctx.serviceInstanceName;
final String endpointName = getEndpointName().next(null);
@ -89,7 +92,7 @@ public class SegmentGenerator implements Generator<SegmentGenerator.SegmentConte
IntStream.range(0, size)
.mapToObj(i -> {
SpanGenerator sg = spanGenerators.get(i);
return sg.next(new SpanGenerator.SpanGeneratorContext(i, size, sr, ctx.peer));
return sg.next(new SpanGenerator.SpanGeneratorContext(i, size, sr, ctx.peer, n));
})
.collect(Collectors.<SpanObject>toList()))
.setService(serviceName)
@ -114,7 +117,7 @@ public class SegmentGenerator implements Generator<SegmentGenerator.SegmentConte
IDManager.EndpointID.buildId(
segment.getServiceId(),
endpointName));
segment.setStartTime(now - latency);
segment.setStartTime(n - latency);
segment.setLatency(latency.intValue());
segment.setIsError(getError().next(null).intValue());
segment.setTimeBucket(TimeBucket.getRecordTimeBucket(segment.getStartTime()));

View File

@ -47,7 +47,7 @@ public final class SpanGenerator implements Generator<SpanGenerator.SpanGenerato
public SpanObject next(SpanGenerator.SpanGeneratorContext ctx) {
long now;
if (endTime == null) {
now = System.currentTimeMillis();
now = ctx.now;
} else {
now = endTime.next(null);
}
@ -101,5 +101,6 @@ public final class SpanGenerator implements Generator<SpanGenerator.SpanGenerato
final int length;
final SegmentReference ref;
final String peer;
final long now;
}
}

View File

@ -164,19 +164,44 @@ storage:
maxSizeOfBatchSql: ${SW_STORAGE_MAX_SIZE_OF_BATCH_SQL:2000}
asyncBatchPersistentPoolSize: ${SW_STORAGE_ASYNC_BATCH_PERSISTENT_POOL_SIZE:4}
banyandb:
# Targets is the list of BanyanDB servers, separated by commas.
# Each target is a BanyanDB server in the format of `host:port`
# The host is the IP address or domain name of the BanyanDB server, and the port is the port number of the BanyanDB server.
targets: ${SW_STORAGE_BANYANDB_TARGETS:127.0.0.1:17912}
# The max number of records in a bulk write request.
# Bigger value can improve the write performance, but also increase the OAP and BanyanDB Server memory usage.
maxBulkSize: ${SW_STORAGE_BANYANDB_MAX_BULK_SIZE:5000}
# The minimum seconds between two bulk flushes.
# If the data in a bulk is less than maxBulkSize, the data will be flushed after this period.
# If the data in a bulk is more than maxBulkSize, the data will be flushed immediately.
# Bigger value can reduce the write pressure on BanyanDB Server, but also increase the latency of the data.
flushInterval: ${SW_STORAGE_BANYANDB_FLUSH_INTERVAL:15}
# The shard number of `measure` groups that store the metrics data.
metricsShardsNumber: ${SW_STORAGE_BANYANDB_METRICS_SHARDS_NUMBER:1}
# The shard number of `stream` groups that store the trace, log and profile data.
recordShardsNumber: ${SW_STORAGE_BANYANDB_RECORD_SHARDS_NUMBER:1}
# The multiplier of the number of shards of the super dataset.
# Super dataset is a special dataset that stores the trace or log data that is too large to be stored in the normal dataset.
# If the normal dataset has `n` shards, the super dataset will have `n * superDatasetShardsFactor` shards.
# For example, supposing `recordShardsNumber` is 3, and `superDatasetShardsFactor` is 2,
# `segment-default` is a normal dataset that has 3 shards, and `segment-minute` is a super dataset that has 6 shards.
superDatasetShardsFactor: ${SW_STORAGE_BANYANDB_SUPERDATASET_SHARDS_FACTOR:2}
# The number of threads that write data to BanyanDB concurrently.
# Bigger value can improve the write performance, but also increase the OAP and BanyanDB Server CPU usage.
concurrentWriteThreads: ${SW_STORAGE_BANYANDB_CONCURRENT_WRITE_THREADS:15}
profileTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_PROFILE_TASK_QUERY_MAX_SIZE:200} # the max number of fetch task in a request
blockIntervalHours: ${SW_STORAGE_BANYANDB_BLOCK_INTERVAL_HOURS:24} # Unit is hour
segmentIntervalDays: ${SW_STORAGE_BANYANDB_SEGMENT_INTERVAL_DAYS:1} # Unit is day
superDatasetBlockIntervalHours: ${SW_STORAGE_BANYANDB_SUPER_DATASET_BLOCK_INTERVAL_HOURS:4} # Unit is hour
superDatasetSegmentIntervalDays: ${SW_STORAGE_BANYANDB_SUPER_DATASET_SEGMENT_INTERVAL_DAYS:1} # Unit is day
specificGroupSettings: ${SW_STORAGE_BANYANDB_SPECIFIC_GROUP_SETTINGS:""} # For example, {"group1": {"blockIntervalHours": 4, "segmentIntervalDays": 1}}
# The max number of profile task query in a request.
profileTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_PROFILE_TASK_QUERY_MAX_SIZE:200}
# Data is stored in BanyanDB in segments. A segment is a time range of data.
# The segment interval is the time range of a segment.
# The value should be less or equal to data TTL relevant settings.
segmentIntervalDays: ${SW_STORAGE_BANYANDB_SEGMENT_INTERVAL_DAYS:1}
# The super dataset segment interval is the time range of a segment in the super dataset.
superDatasetSegmentIntervalDays: ${SW_STORAGE_BANYANDB_SUPER_DATASET_SEGMENT_INTERVAL_DAYS:1}
# Specific groups settings.
# For example, {"group1": {"blockIntervalHours": 4, "segmentIntervalDays": 1}}
# Please refer to https://github.com/apache/skywalking-banyandb/blob/${BANYANDB_RELEASE}/docs/crud/group.md#create-operation
# for group setting details.
specificGroupSettings: ${SW_STORAGE_BANYANDB_SPECIFIC_GROUP_SETTINGS:""}
agent-analyzer:
selector: ${SW_AGENT_ANALYZER:default}

View File

@ -31,6 +31,11 @@
"min": 1,
"max": 1
},
"now": {
"type": "time",
"stepMillisecond": 10000,
"waitMillisecond": 1000
},
"tags": {
"type": "randomList",
"size": 5,

View File

@ -163,7 +163,7 @@
<byte-buddy.version>1.14.9</byte-buddy.version>
<!-- core lib dependency -->
<grpc.version>1.62.2</grpc.version>
<grpc.version>1.63.0</grpc.version>
<netty.version>4.1.108.Final</netty.version>
<netty-tcnative-boringssl-static.version>2.0.52.Final</netty-tcnative-boringssl-static.version>
<gson.version>2.9.0</gson.version>

View File

@ -50,6 +50,7 @@ setup:
--set fullnameOverride=skywalking \
--set elasticsearch.enabled=false \
--set banyandb.enabled=true \
--set banyandb.standalone.image.tag=${SW_BANYANDB_COMMIT} \
--set oap.replicas=1 \
--set ui.image.repository=skywalking/ui \
--set ui.image.tag=latest \

View File

@ -53,6 +53,7 @@ setup:
--set fullnameOverride=skywalking \
--set elasticsearch.enabled=false \
--set banyandb.enabled=true \
--set banyandb.standalone.image.tag=${SW_BANYANDB_COMMIT} \
--set oap.replicas=1 \
--set ui.image.repository=skywalking/ui \
--set ui.image.tag=latest \

View File

@ -50,6 +50,7 @@ setup:
--set fullnameOverride=skywalking \
--set elasticsearch.enabled=false \
--set banyandb.enabled=true \
--set banyandb.standalone.image.tag=${SW_BANYANDB_COMMIT} \
--set oap.replicas=1 \
--set ui.image.repository=skywalking/ui \
--set ui.image.tag=latest \

View File

@ -50,6 +50,7 @@ setup:
--set fullnameOverride=skywalking \
--set elasticsearch.enabled=false \
--set banyandb.enabled=true \
--set banyandb.standalone.image.tag=${SW_BANYANDB_COMMIT} \
--set oap.replicas=1 \
--set ui.image.repository=skywalking/ui \
--set ui.image.tag=latest \

View File

@ -21,9 +21,9 @@ SW_AGENT_GO_COMMIT=774a6d56baba1187eb03bf1861af542c923b3dff
SW_AGENT_PYTHON_COMMIT=c76a6ec51a478ac91abb20ec8f22a99b8d4d6a58
SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449
SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016
SW_KUBERNETES_COMMIT_SHA=3eb9fc8235fd3a773650d1f08adfccfd5d0745fc
SW_KUBERNETES_COMMIT_SHA=1741f5a0959b85faaf0a2fc6bbd7b550ec330280
SW_ROVER_COMMIT=6bbd39aa701984482330d9dfb4dbaaff0527d55c
SW_BANYANDB_COMMIT=84f32b3969cdcc676aaee428383b34b3b67dbdf5
SW_BANYANDB_COMMIT=a091ac0c3efa7305288ae9fb8853bffb2186583a
SW_AGENT_PHP_COMMIT=3192c553002707d344bd6774cfab5bc61f67a1d3
SW_CTL_COMMIT=d5f3597733aa5217373986d776a3ee5ee8b3c468