diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 497296faac..e5175f0457 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -42,6 +42,8 @@ * Replace workaround with Armeria native supported context path. * Add an http endpoint wrapper for health check. * Bump up Armeria and transitive dependencies. +* BanyanDB: if the model column is already a `@BanyanDB.TimestampColumn`, set `@BanyanDB.NoIndexing` on it to reduce indexes. +* BanyanDB: stream sort-by `time` query, use internal time-series rather than `index` to improve the query performance. #### UI diff --git a/oap-server-bom/pom.xml b/oap-server-bom/pom.xml index 96e40db308..083372ee87 100644 --- a/oap-server-bom/pom.xml +++ b/oap-server-bom/pom.xml @@ -73,7 +73,7 @@ 4.4.13 4.1.5 1.21 - 0.7.0-rc1 + 0.7.0-rc2 3.4.0 2.4.6.RELEASE 1.5.3 diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java index 4b8966a6ea..7daa15c062 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java @@ -85,6 +85,7 @@ public class SegmentRecord extends Record { @Setter @Getter @Column(name = START_TIME) + @BanyanDB.NoIndexing private long startTime; @Setter @Getter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/spanattach/SpanAttachedEventRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/spanattach/SpanAttachedEventRecord.java index fb681be8bc..d8263b5357 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/spanattach/SpanAttachedEventRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/spanattach/SpanAttachedEventRecord.java @@ -77,6 +77,7 @@ public class SpanAttachedEventRecord extends Record { @Setter @Getter @Column(name = TIMESTAMP) + @BanyanDB.NoIndexing private long timestamp; @Override diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingTaskRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingTaskRecord.java index 00d33db75a..7889b65aa5 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingTaskRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingTaskRecord.java @@ -79,6 +79,7 @@ public class EBPFProfilingTaskRecord extends NoneStream { @Column(name = TARGET_TYPE) private int targetType = EBPFProfilingTargetType.UNKNOWN.value(); @Column(name = CREATE_TIME) + @BanyanDB.NoIndexing private long createTime; @Column(name = LAST_UPDATE_TIME) private long lastUpdateTime; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/trace/ProfileTaskRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/trace/ProfileTaskRecord.java index b1fdc16349..80c57850d8 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/trace/ProfileTaskRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/trace/ProfileTaskRecord.java @@ -67,6 +67,7 @@ public class ProfileTaskRecord extends NoneStream { @Column(name = TASK_ID) private String taskId; @Column(name = START_TIME) + @BanyanDB.NoIndexing private long startTime; @Column(name = DURATION) private int duration; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/trace/ProfileThreadSnapshotRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/trace/ProfileThreadSnapshotRecord.java index db8eebfa76..6ce04666b0 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/trace/ProfileThreadSnapshotRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/trace/ProfileThreadSnapshotRecord.java @@ -60,6 +60,7 @@ public class ProfileThreadSnapshotRecord extends Record { @BanyanDB.SeriesID(index = 0) private String segmentId; @Column(name = DUMP_TIME) + @BanyanDB.NoIndexing private long dumpTime; @Column(name = SEQUENCE) private int sequence; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/zipkin/ZipkinSpanRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/zipkin/ZipkinSpanRecord.java index c576cb1f22..fdd1dce246 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/zipkin/ZipkinSpanRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/zipkin/ZipkinSpanRecord.java @@ -107,10 +107,12 @@ public class ZipkinSpanRecord extends Record { @Setter @Getter @Column(name = TIMESTAMP_MILLIS) + @BanyanDB.NoIndexing private long timestampMillis; @Setter @Getter @Column(name = TIMESTAMP) + @BanyanDB.NoIndexing private long timestamp; @Setter @Getter diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBZipkinQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBZipkinQueryDAO.java index f11ccb92fd..3d1cf7d7cc 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBZipkinQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBZipkinQueryDAO.java @@ -241,7 +241,7 @@ public class BanyanDBZipkinQueryDAO extends AbstractBanyanDBDAO implements IZipk if (request.maxDuration() != null) { query.and(lte(ZipkinSpanRecord.DURATION, request.maxDuration())); } - query.setOrderBy(new StreamQuery.OrderBy(ZipkinSpanRecord.TIMESTAMP_MILLIS, AbstractQuery.Sort.DESC)); + query.setOrderBy(new StreamQuery.OrderBy(AbstractQuery.Sort.DESC)); query.setLimit(limit); } }; @@ -276,7 +276,7 @@ public class BanyanDBZipkinQueryDAO extends AbstractBanyanDBDAO implements IZipk query.criteria(or(conditions)); } query.setOrderBy( - new StreamQuery.OrderBy(ZipkinSpanRecord.TIMESTAMP_MILLIS, AbstractQuery.Sort.DESC)); + new StreamQuery.OrderBy(AbstractQuery.Sort.DESC)); query.setLimit(QUERY_MAX_SIZE); } } diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBEventQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBEventQueryDAO.java index 5bb76b90f6..7b56d35026 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBEventQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBEventQueryDAO.java @@ -25,6 +25,7 @@ import java.util.Set; import com.google.common.collect.ImmutableSet; import org.apache.skywalking.banyandb.v1.client.AbstractCriteria; +import org.apache.skywalking.banyandb.v1.client.AbstractQuery; import org.apache.skywalking.banyandb.v1.client.DataPoint; import org.apache.skywalking.banyandb.v1.client.MeasureQuery; import org.apache.skywalking.banyandb.v1.client.MeasureQueryResponse; @@ -98,9 +99,9 @@ public class BanyanDBEventQueryDAO extends AbstractBanyanDBDAO implements IEvent query.limit(page.getLimit()); query.offset(page.getFrom()); if (queryOrder == Order.ASC) { - query.setOrderBy(asc(Event.START_TIME)); + query.setOrderBy(new AbstractQuery.OrderBy(Event.START_TIME, AbstractQuery.Sort.ASC)); } else { - query.setOrderBy(desc(Event.START_TIME)); + query.setOrderBy(new AbstractQuery.OrderBy(Event.START_TIME, AbstractQuery.Sort.DESC)); } for (final EventQueryCondition condition : conditionList) { List> queryConditions = new ArrayList<>(); diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java index 7607a07d60..8d8d053388 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java @@ -52,6 +52,10 @@ public abstract class AbstractBanyanDBDAO extends AbstractDAO conditions) { if (conditions.isEmpty()) { return null; diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBEBPFProfilingTaskDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBEBPFProfilingTaskDAO.java index 422d9d9dbe..8880e34bd7 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBEBPFProfilingTaskDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBEBPFProfilingTaskDAO.java @@ -71,7 +71,7 @@ public class BanyanDBEBPFProfilingTaskDAO extends AbstractBanyanDBDAO implements if (triggerType != null) { query.and(eq(EBPFProfilingTaskRecord.TRIGGER_TYPE, triggerType.value())); } - query.setOrderBy(new AbstractQuery.OrderBy(EBPFProfilingTaskRecord.CREATE_TIME, AbstractQuery.Sort.DESC)); + query.setOrderBy(new AbstractQuery.OrderBy(AbstractQuery.Sort.DESC)); } }); tasks.addAll(resp.getElements().stream().map(this::buildTask).collect(Collectors.toList())); @@ -100,7 +100,7 @@ public class BanyanDBEBPFProfilingTaskDAO extends AbstractBanyanDBDAO implements } query.and(eq(EBPFProfilingTaskRecord.TARGET_TYPE, targetType.value())); appendTimeQuery(this, query, taskStartTime, latestUpdateTime); - query.setOrderBy(new AbstractQuery.OrderBy(EBPFProfilingTaskRecord.CREATE_TIME, AbstractQuery.Sort.DESC)); + query.setOrderBy(new AbstractQuery.OrderBy(AbstractQuery.Sort.DESC)); } }); tasks.addAll(resp.getElements().stream().map(this::buildTask).collect(Collectors.toList())); diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBProfileTaskQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBProfileTaskQueryDAO.java index a120e957fe..3d35cd693c 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBProfileTaskQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBProfileTaskQueryDAO.java @@ -23,6 +23,7 @@ import org.apache.skywalking.banyandb.v1.client.AbstractQuery; import org.apache.skywalking.banyandb.v1.client.RowEntity; import org.apache.skywalking.banyandb.v1.client.StreamQuery; import org.apache.skywalking.banyandb.v1.client.StreamQueryResponse; +import org.apache.skywalking.banyandb.v1.client.TimestampRange; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskRecord; import org.apache.skywalking.oap.server.core.query.type.ProfileTask; @@ -58,7 +59,15 @@ public class BanyanDBProfileTaskQueryDAO extends AbstractBanyanDBDAO implements @Override public List getTaskList(String serviceId, String endpointName, Long startTimeBucket, Long endTimeBucket, Integer limit) throws IOException { - StreamQueryResponse resp = query(ProfileTaskRecord.INDEX_NAME, TAGS, + long startTS = LOWER_BOUND_TIME; + long endTS = UPPER_BOUND_TIME; + if (startTimeBucket != null) { + startTS = TimeBucket.getTimestamp(startTimeBucket); + } + if (endTimeBucket != null) { + endTS = TimeBucket.getTimestamp(endTimeBucket); + } + StreamQueryResponse resp = query(ProfileTaskRecord.INDEX_NAME, TAGS, new TimestampRange(startTS, endTS), new QueryBuilder() { @Override protected void apply(StreamQuery query) { @@ -68,19 +77,13 @@ public class BanyanDBProfileTaskQueryDAO extends AbstractBanyanDBDAO implements if (StringUtil.isNotEmpty(endpointName)) { query.and(eq(ProfileTaskRecord.ENDPOINT_NAME, endpointName)); } - if (startTimeBucket != null) { - query.and(gte(ProfileTaskRecord.START_TIME, TimeBucket.getTimestamp(startTimeBucket))); - } - if (endTimeBucket != null) { - query.and(lte(ProfileTaskRecord.START_TIME, TimeBucket.getTimestamp(endTimeBucket))); - } if (limit != null) { query.setLimit(limit); } else { query.setLimit(BanyanDBProfileTaskQueryDAO.this.queryMaxSize); } - query.setOrderBy(new AbstractQuery.OrderBy(ProfileTaskRecord.START_TIME, AbstractQuery.Sort.DESC)); + query.setOrderBy(new AbstractQuery.OrderBy(AbstractQuery.Sort.DESC)); } }); diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBProfileThreadSnapshotQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBProfileThreadSnapshotQueryDAO.java index d688a84686..573ac72959 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBProfileThreadSnapshotQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBProfileThreadSnapshotQueryDAO.java @@ -19,9 +19,11 @@ package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream; import com.google.common.collect.ImmutableSet; +import org.apache.skywalking.banyandb.v1.client.AbstractQuery; import org.apache.skywalking.banyandb.v1.client.RowEntity; import org.apache.skywalking.banyandb.v1.client.StreamQuery; import org.apache.skywalking.banyandb.v1.client.StreamQueryResponse; +import org.apache.skywalking.banyandb.v1.client.TimestampRange; import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileThreadSnapshotRecord; import org.apache.skywalking.oap.server.core.storage.profiling.trace.IProfileThreadSnapshotQueryDAO; @@ -84,7 +86,7 @@ public class BanyanDBProfileThreadSnapshotQueryDAO extends AbstractBanyanDBDAO i public void apply(StreamQuery query) { query.and(eq(ProfileThreadSnapshotRecord.TASK_ID, taskId)) .and(eq(ProfileThreadSnapshotRecord.SEQUENCE, 0L)); - query.setOrderBy(desc(ProfileThreadSnapshotRecord.DUMP_TIME)); + query.setOrderBy(new StreamQuery.OrderBy(AbstractQuery.Sort.DESC)); query.setLimit(querySegmentMaxSize); } }); @@ -135,13 +137,11 @@ public class BanyanDBProfileThreadSnapshotQueryDAO extends AbstractBanyanDBDAO i private int querySequenceWithAgg(AggType aggType, String segmentId, long start, long end) throws IOException { StreamQueryResponse resp = query(ProfileThreadSnapshotRecord.INDEX_NAME, - TAGS_ALL, + TAGS_ALL, new TimestampRange(start, end), new QueryBuilder() { @Override public void apply(StreamQuery query) { - query.and(eq(ProfileThreadSnapshotRecord.SEGMENT_ID, segmentId)) - .and(lte(ProfileThreadSnapshotRecord.DUMP_TIME, end)) - .and(gte(ProfileThreadSnapshotRecord.DUMP_TIME, start)); + query.and(eq(ProfileThreadSnapshotRecord.SEGMENT_ID, segmentId)); } }); @@ -176,4 +176,4 @@ public class BanyanDBProfileThreadSnapshotQueryDAO extends AbstractBanyanDBDAO i enum AggType { MIN, MAX } -} \ No newline at end of file +} diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBSpanAttachedEventQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBSpanAttachedEventQueryDAO.java index e616338f8a..984bc43801 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBSpanAttachedEventQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBSpanAttachedEventQueryDAO.java @@ -58,7 +58,7 @@ public class BanyanDBSpanAttachedEventQueryDAO extends AbstractBanyanDBDAO imple protected void apply(StreamQuery query) { query.and(in(SpanAttachedEventRecord.RELATED_TRACE_ID, traceIds)); query.and(eq(SpanAttachedEventRecord.TRACE_REF_TYPE, type.value())); - query.setOrderBy(new StreamQuery.OrderBy(SpanAttachedEventRecord.START_TIME_SECOND, AbstractQuery.Sort.ASC)); + query.setOrderBy(new StreamQuery.OrderBy(AbstractQuery.Sort.ASC)); } }); diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBTraceQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBTraceQueryDAO.java index e01ff35320..b63baad81b 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBTraceQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBTraceQueryDAO.java @@ -118,7 +118,7 @@ public class BanyanDBTraceQueryDAO extends AbstractBanyanDBDAO implements ITrace switch (queryOrder) { case BY_START_TIME: - query.setOrderBy(new StreamQuery.OrderBy(SegmentRecord.START_TIME, AbstractQuery.Sort.DESC)); + query.setOrderBy(new StreamQuery.OrderBy(AbstractQuery.Sort.DESC)); break; case BY_DURATION: query.setOrderBy(new StreamQuery.OrderBy(SegmentRecord.LATENCY, AbstractQuery.Sort.DESC)); diff --git a/test/e2e-v2/script/env b/test/e2e-v2/script/env index b495437dde..c461eeddaa 100644 --- a/test/e2e-v2/script/env +++ b/test/e2e-v2/script/env @@ -23,7 +23,7 @@ SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449 SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016 SW_KUBERNETES_COMMIT_SHA=1335f15bf821a40a7cd71448fa805f0be265afcc SW_ROVER_COMMIT=6bbd39aa701984482330d9dfb4dbaaff0527d55c -SW_BANYANDB_COMMIT=6f938ccf812e2183b4bd891fb90b2124aa65c170 +SW_BANYANDB_COMMIT=d48a810f8cca8b66d7b3b179f36090d78f46e12c SW_AGENT_PHP_COMMIT=3192c553002707d344bd6774cfab5bc61f67a1d3 SW_CTL_COMMIT=d5f3597733aa5217373986d776a3ee5ee8b3c468