From 8b678bebe6662665e2fa9d3c3d2e1aa8d5a732c1 Mon Sep 17 00:00:00 2001 From: Wan Kai Date: Tue, 2 Jul 2024 18:58:47 +0800 Subject: [PATCH] Support BanyanDB internal metrics query execution tracing. (#12400) * Support BanyanDB internal metrics query execution tracing. * BanyanDB client config: rise the default `maxBulkSize` to 10000, add `flushTimeout` and set default to 10s. --- docs/en/changes/changes.md | 2 ++ .../setup/backend/configuration-vocabulary.md | 3 +- docs/en/setup/backend/storages/banyandb.md | 6 ++-- oap-server-bom/pom.xml | 2 +- .../core/query/MetricsQueryService.java | 8 +++--- .../server/core/query/RecordQueryService.java | 4 +-- .../query/type/debugging/DebuggingSpan.java | 11 +++----- .../type/debugging/DebuggingTraceContext.java | 13 ++++++++- .../src/main/resources/application.yml | 4 ++- .../banyandb/BanyanDBStorageClient.java | 8 ++++-- .../banyandb/BanyanDBStorageConfig.java | 4 +++ .../banyandb/BanyanDBStorageProvider.java | 2 +- .../banyandb/stream/AbstractBanyanDBDAO.java | 28 ++++++++++++++++++- .../src/main/resources/application.yml | 4 ++- test/e2e-v2/script/env | 2 +- 15 files changed, 75 insertions(+), 26 deletions(-) diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 000dd37544..49730287ca 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -24,6 +24,8 @@ * Fix BanyanDB metrics query: used the wrong `Downsampling` type to find the schema. * Support fetch cilium flow to monitoring network traffic between cilium services. * Support `labelCount` function in the OAL engine. +* Support BanyanDB internal metrics query execution tracing. +* BanyanDB client config: rise the default `maxBulkSize` to 10000, add `flushTimeout` and set default to 10s. #### UI * Highlight search log keywords. diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md index 8f89debe52..461cf105a7 100644 --- a/docs/en/setup/backend/configuration-vocabulary.md +++ b/docs/en/setup/backend/configuration-vocabulary.md @@ -140,8 +140,9 @@ The Configuration Vocabulary lists all available configurations provided by `app | - | - | asyncBatchPersistentPoolSize | async flush data into database thread size | SW_STORAGE_ASYNC_BATCH_PERSISTENT_POOL_SIZE | 4 | | - | banyandb | - | BanyanDB storage. | - | - | | - | - | targets | Hosts with ports of the BanyanDB. | SW_STORAGE_BANYANDB_TARGETS | 127.0.0.1:17912 | -| - | - | maxBulkSize | The maximum size of write entities in a single batch write call. | SW_STORAGE_BANYANDB_MAX_BULK_SIZE | 5000 | +| - | - | maxBulkSize | The maximum size of write entities in a single batch write call. | SW_STORAGE_BANYANDB_MAX_BULK_SIZE | 10000 | | - | - | flushInterval | Period of flush interval. In the timeunit of seconds. | SW_STORAGE_BANYANDB_FLUSH_INTERVAL | 15 | +| - | - | flushTimeout | The timeout seconds of a bulk flush. | SW_STORAGE_BANYANDB_FLUSH_TIMEOUT | 10 | | - | - | metricsShardsNumber | Shards Number for measure/metrics. | SW_STORAGE_BANYANDB_METRICS_SHARDS_NUMBER | 1 | | - | - | recordShardsNumber | Shards Number for a normal record. | SW_STORAGE_BANYANDB_RECORD_SHARDS_NUMBER | 1 | | - | - | superDatasetShardsFactor | Shards Factor for a super dataset record, i.e. Shard number of a super dataset is recordShardsNumber*superDatasetShardsFactor. | SW_STORAGE_BANYANDB_SUPERDATASET_SHARDS_FACTOR | 2 | diff --git a/docs/en/setup/backend/storages/banyandb.md b/docs/en/setup/backend/storages/banyandb.md index 10d1234358..a37d225c5e 100644 --- a/docs/en/setup/backend/storages/banyandb.md +++ b/docs/en/setup/backend/storages/banyandb.md @@ -3,7 +3,7 @@ [BanyanDB](https://github.com/apache/skywalking-banyandb) is a dedicated storage implementation developed by the SkyWalking Team and the community. Activate BanyanDB as the storage, and set storage provider to **banyandb**. -The OAP requires BanyanDB 0.6 server. From this version, BanyanDB provides general compatibility. +The OAP requires BanyanDB 0.7 server. From this version, BanyanDB provides general compatibility. ```yaml storage: @@ -14,12 +14,14 @@ storage: targets: ${SW_STORAGE_BANYANDB_TARGETS:127.0.0.1:17912} # The max number of records in a bulk write request. # Bigger value can improve the write performance, but also increase the OAP and BanyanDB Server memory usage. - maxBulkSize: ${SW_STORAGE_BANYANDB_MAX_BULK_SIZE:5000} + maxBulkSize: ${SW_STORAGE_BANYANDB_MAX_BULK_SIZE:10000} # The minimum seconds between two bulk flushes. # If the data in a bulk is less than maxBulkSize, the data will be flushed after this period. # If the data in a bulk is more than maxBulkSize, the data will be flushed immediately. # Bigger value can reduce the write pressure on BanyanDB Server, but also increase the latency of the data. flushInterval: ${SW_STORAGE_BANYANDB_FLUSH_INTERVAL:15} + # The timeout seconds of a bulk flush. + flushTimeout: ${SW_STORAGE_BANYANDB_FLUSH_TIMEOUT:10} # The shard number of `measure` groups that store the metrics data. metricsShardsNumber: ${SW_STORAGE_BANYANDB_METRICS_SHARDS_NUMBER:1} # The shard number of `stream` groups that store the trace, log and profile data. diff --git a/oap-server-bom/pom.xml b/oap-server-bom/pom.xml index 7a8d5571e2..6f4e6845cb 100644 --- a/oap-server-bom/pom.xml +++ b/oap-server-bom/pom.xml @@ -73,7 +73,7 @@ 4.4.13 4.1.5 1.21 - 0.6.0 + 0.7.0-rc1 3.4.0 2.4.6.RELEASE 1.5.3 diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetricsQueryService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetricsQueryService.java index fba490a510..41844b362c 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetricsQueryService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetricsQueryService.java @@ -92,8 +92,8 @@ public class MetricsQueryService implements Service { DebuggingSpan span = null; try { if (traceContext != null) { - span = traceContext.createSpan("Query Service"); - span.setMsg("readMetricsValues, MetricsCondition: " + condition + ", Duration: " + duration); + span = traceContext.createSpan("Query Service: readMetricsValues"); + span.setMsg("MetricsCondition: " + condition + ", Duration: " + duration); } return invokeReadMetricsValues(condition, duration); } finally { @@ -125,8 +125,8 @@ public class MetricsQueryService implements Service { DebuggingSpan span = null; try { if (traceContext != null) { - span = traceContext.createSpan("Query Service"); - span.setMsg("readLabeledMetricsValues, MetricsCondition: " + condition + ", Labels: " + labels + ", Duration: " + duration); + span = traceContext.createSpan("Query Service: readLabeledMetricsValues"); + span.setMsg("MetricsCondition: " + condition + ", Labels: " + labels + ", Duration: " + duration); } return invokeReadLabeledMetricsValues(condition, labels, duration); } finally { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/RecordQueryService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/RecordQueryService.java index cd6e0abca9..63adea79b6 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/RecordQueryService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/RecordQueryService.java @@ -65,8 +65,8 @@ public class RecordQueryService implements Service { DebuggingSpan span = null; try { if (traceContext != null) { - span = traceContext.createSpan("Query Service"); - span.setMsg("readRecords, RecordCondition: " + condition + ", Duration: " + duration); + span = traceContext.createSpan("Query Service: readRecords"); + span.setMsg("RecordCondition: " + condition + ", Duration: " + duration); } return invokeReadRecords(condition, duration); } finally { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/debugging/DebuggingSpan.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/debugging/DebuggingSpan.java index 7b836aca0a..5905c2466d 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/debugging/DebuggingSpan.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/debugging/DebuggingSpan.java @@ -28,10 +28,13 @@ public class DebuggingSpan { private int parentSpanId; private final String operation; //nano seconds - private final long startTime; + @Setter + private long startTime; //nano seconds + @Setter private long endTime; //nano seconds + @Setter private long duration; @Setter private String msg; @@ -41,11 +44,5 @@ public class DebuggingSpan { public DebuggingSpan(int spanId, String operation) { this.spanId = spanId; this.operation = operation; - this.startTime = System.nanoTime(); - } - - public void stopSpan() { - this.endTime = System.nanoTime(); - this.duration = endTime - startTime; } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/debugging/DebuggingTraceContext.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/debugging/DebuggingTraceContext.java index 830c89cdc8..704fc5dfc2 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/debugging/DebuggingTraceContext.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/debugging/DebuggingTraceContext.java @@ -39,8 +39,11 @@ public class DebuggingTraceContext { public DebuggingSpan createSpan(String operation) { DebuggingSpan span = new DebuggingSpan(spanIdGenerator++, operation); if (debug) { + //default start time, could be overwritten by setStartTime (BanyanDB Trace) + span.setStartTime(System.nanoTime()); DebuggingSpan parentSpan = spanStack.isEmpty() ? null : spanStack.peek(); if (parentSpan != null) { + //default parent span id, could be overwritten by setParentSpanId (BanyanDB Trace) span.setParentSpanId(parentSpan.getSpanId()); } else { span.setParentSpanId(-1); @@ -51,9 +54,17 @@ public class DebuggingTraceContext { return span; } + public DebuggingSpan getParentSpan() { + if (spanStack.isEmpty()) { + return null; + } + return spanStack.peek(); + } + public void stopSpan(DebuggingSpan span) { if (debug) { - span.stopSpan(); + span.setEndTime(System.nanoTime()); + span.setDuration(span.getEndTime() - span.getStartTime()); if (spanStack.isEmpty()) { return; } diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 26eda39bbd..75a0b2fd45 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -222,12 +222,14 @@ storage: targets: ${SW_STORAGE_BANYANDB_TARGETS:127.0.0.1:17912} # The max number of records in a bulk write request. # Bigger value can improve the write performance, but also increase the OAP and BanyanDB Server memory usage. - maxBulkSize: ${SW_STORAGE_BANYANDB_MAX_BULK_SIZE:5000} + maxBulkSize: ${SW_STORAGE_BANYANDB_MAX_BULK_SIZE:10000} # The minimum seconds between two bulk flushes. # If the data in a bulk is less than maxBulkSize, the data will be flushed after this period. # If the data in a bulk is more than maxBulkSize, the data will be flushed immediately. # Bigger value can reduce the write pressure on BanyanDB Server, but also increase the latency of the data. flushInterval: ${SW_STORAGE_BANYANDB_FLUSH_INTERVAL:15} + # The timeout seconds of a bulk flush. + flushTimeout: ${SW_STORAGE_BANYANDB_FLUSH_TIMEOUT:10} # The shard number of `measure` groups that store the metrics data. metricsShardsNumber: ${SW_STORAGE_BANYANDB_METRICS_SHARDS_NUMBER:1} # The shard number of `stream` groups that store the trace, log and profile data. diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageClient.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageClient.java index f5f6eef636..25b6cb2c5b 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageClient.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageClient.java @@ -54,9 +54,11 @@ import java.util.List; public class BanyanDBStorageClient implements Client, HealthCheckable { final BanyanDBClient client; private final DelegatedHealthChecker healthChecker = new DelegatedHealthChecker(); + private final int flushTimeout; - public BanyanDBStorageClient(String... targets) { + public BanyanDBStorageClient(int flushTimeout, String... targets) { this.client = new BanyanDBClient(targets); + this.flushTimeout = flushTimeout; } @Override @@ -234,11 +236,11 @@ public class BanyanDBStorageClient implements Client, HealthCheckable { } public StreamBulkWriteProcessor createStreamBulkProcessor(int maxBulkSize, int flushInterval, int concurrency) { - return this.client.buildStreamWriteProcessor(maxBulkSize, flushInterval, concurrency); + return this.client.buildStreamWriteProcessor(maxBulkSize, flushInterval, concurrency, flushTimeout); } public MeasureBulkWriteProcessor createMeasureBulkProcessor(int maxBulkSize, int flushInterval, int concurrency) { - return this.client.buildMeasureWriteProcessor(maxBulkSize, flushInterval, concurrency); + return this.client.buildMeasureWriteProcessor(maxBulkSize, flushInterval, concurrency, flushTimeout); } @Override diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java index 148290f487..793e5ada1f 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java @@ -41,6 +41,10 @@ public class BanyanDBStorageConfig extends ModuleConfig { * Period of flush interval. In the timeunit of seconds. */ private int flushInterval = 15; + /** + * Timeout of flush. In the timeunit of seconds. + */ + private int flushTimeout = 10; /** * Concurrent consumer threads for batch writing. */ diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java index 2dee1ba62b..5a6c443d7a 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java @@ -117,7 +117,7 @@ public class BanyanDBStorageProvider extends ModuleProvider { public void prepare() throws ServiceNotProvidedException, ModuleStartException { this.registerServiceImplementation(StorageBuilderFactory.class, new StorageBuilderFactory.Default()); - this.client = new BanyanDBStorageClient(config.getTargetArray()); + this.client = new BanyanDBStorageClient(config.getFlushTimeout(), config.getTargetArray()); this.modelInstaller = new BanyanDBIndexInstaller(client, getManager(), this.config); // Stream diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java index 466077dd1d..aa510226ca 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java @@ -26,11 +26,13 @@ import org.apache.skywalking.banyandb.v1.client.MeasureQuery; import org.apache.skywalking.banyandb.v1.client.MeasureQueryResponse; import org.apache.skywalking.banyandb.v1.client.Or; import org.apache.skywalking.banyandb.v1.client.PairQueryCondition; +import org.apache.skywalking.banyandb.v1.client.Span; import org.apache.skywalking.banyandb.v1.client.StreamQuery; import org.apache.skywalking.banyandb.v1.client.StreamQueryResponse; import org.apache.skywalking.banyandb.v1.client.TimestampRange; import org.apache.skywalking.banyandb.v1.client.TopNQuery; import org.apache.skywalking.banyandb.v1.client.TopNQueryResponse; +import org.apache.skywalking.banyandb.v1.client.Trace; import org.apache.skywalking.oap.server.core.query.type.KeyValue; import org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingSpan; import org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTraceContext; @@ -196,6 +198,7 @@ public abstract class AbstractBanyanDBDAO extends AbstractDAO addDBSpan2DebuggingTrace(span, traceContext, parentSpan)); + } + + private void addDBSpan2DebuggingTrace(Span span, DebuggingTraceContext traceContext, DebuggingSpan parentSpan) { + DebuggingSpan debuggingSpan = traceContext.createSpan("BanyanDB: " + span.getMessage()); + debuggingSpan.setStartTime(span.getStartTime().getSeconds() * 1000_000_000 + span.getStartTime().getNanos()); + debuggingSpan.setEndTime(span.getEndTime().getSeconds() * 1000_000_000 + span.getEndTime().getNanos()); + debuggingSpan.setDuration(span.getDuration()); + debuggingSpan.setParentSpanId(parentSpan.getSpanId()); + debuggingSpan.setMsg(span.getTags().toString()); + if (span.isError()) { + debuggingSpan.setError("BanyanDB occurs error."); + } + span.getChildren().forEach(child -> addDBSpan2DebuggingTrace(child, traceContext, debuggingSpan)); + } + protected static QueryBuilder emptyMeasureQuery() { return new QueryBuilder() { @Override diff --git a/oap-server/server-tools/data-generator/src/main/resources/application.yml b/oap-server/server-tools/data-generator/src/main/resources/application.yml index 613192ac30..f7b1d5ea1c 100755 --- a/oap-server/server-tools/data-generator/src/main/resources/application.yml +++ b/oap-server/server-tools/data-generator/src/main/resources/application.yml @@ -170,12 +170,14 @@ storage: targets: ${SW_STORAGE_BANYANDB_TARGETS:127.0.0.1:17912} # The max number of records in a bulk write request. # Bigger value can improve the write performance, but also increase the OAP and BanyanDB Server memory usage. - maxBulkSize: ${SW_STORAGE_BANYANDB_MAX_BULK_SIZE:5000} + maxBulkSize: ${SW_STORAGE_BANYANDB_MAX_BULK_SIZE:10000} # The minimum seconds between two bulk flushes. # If the data in a bulk is less than maxBulkSize, the data will be flushed after this period. # If the data in a bulk is more than maxBulkSize, the data will be flushed immediately. # Bigger value can reduce the write pressure on BanyanDB Server, but also increase the latency of the data. flushInterval: ${SW_STORAGE_BANYANDB_FLUSH_INTERVAL:15} + # The timeout seconds of a bulk flush. + flushTimeout: ${SW_STORAGE_BANYANDB_FLUSH_TIMEOUT:10} # The shard number of `measure` groups that store the metrics data. metricsShardsNumber: ${SW_STORAGE_BANYANDB_METRICS_SHARDS_NUMBER:1} # The shard number of `stream` groups that store the trace, log and profile data. diff --git a/test/e2e-v2/script/env b/test/e2e-v2/script/env index 9b8059ef4b..e906d727c7 100644 --- a/test/e2e-v2/script/env +++ b/test/e2e-v2/script/env @@ -23,7 +23,7 @@ SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449 SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016 SW_KUBERNETES_COMMIT_SHA=1741f5a0959b85faaf0a2fc6bbd7b550ec330280 SW_ROVER_COMMIT=6bbd39aa701984482330d9dfb4dbaaff0527d55c -SW_BANYANDB_COMMIT=7a12fd03339fcab8e4d65286d2f65319e5576ba2 +SW_BANYANDB_COMMIT=e7210733022566cd0cee7ea2dc12cfb2f30fa8f3 SW_AGENT_PHP_COMMIT=3192c553002707d344bd6774cfab5bc61f67a1d3 SW_CTL_COMMIT=d5f3597733aa5217373986d776a3ee5ee8b3c468