From 80da79ebfa9ad3a2de08a79e2832fbbd600b9201 Mon Sep 17 00:00:00 2001
From: Wan Kai <wankai123@foxmail.com>
Date: Thu, 22 May 2025 20:36:47 +0800
Subject: [PATCH] BanyanDB: add topN cold stage data query. (#13259)

---
 .github/workflows/skywalking.yaml             |  2 +-
 docs/en/changes/changes.md                    |  1 +
 oap-server-bom/pom.xml                        |  2 +-
 .../promql/handler/PromQLApiHandler.java      |  2 +-
 .../banyandb/BanyanDBAggregationQueryDAO.java |  1 -
 .../banyandb/stream/AbstractBanyanDBDAO.java  | 24 ++------
 skywalking-ui                                 |  2 +-
 .../cases/storage/banyandb/stages/e2e.yaml    |  5 ++
 .../expected/cold/dependency-instance.yml     |  1 +
 .../expected/cold/topN-OP-endpoint.yml        | 58 +++++++++++++++++++
 .../storage/expected/cold/topN-OP-service.yml | 58 +++++++++++++++++++
 .../apache/skywalking/mock/AgentDataMock.java |  1 +
 12 files changed, 134 insertions(+), 23 deletions(-)
 create mode 100644 test/e2e-v2/cases/storage/expected/cold/topN-OP-endpoint.yml
 create mode 100644 test/e2e-v2/cases/storage/expected/cold/topN-OP-service.yml

diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml
index 1bde7c328a..4cc6fc8448 100644
--- a/.github/workflows/skywalking.yaml
+++ b/.github/workflows/skywalking.yaml
@@ -1021,7 +1021,7 @@ jobs:
             # check if segment files exist
             if docker exec $CONTAINER_ID sh -c '[ -n "$(ls /tmp/measure-data/measure/data/day/seg* 2>/dev/null)" ]'; then
               echo "✅ found segment files"
-              sleep 30
+              sleep 180
               # create and copy files
               docker cp $CONTAINER_ID:/tmp ${BANYANDB_DATA_GENERATE_ROOT}
               docker cp $CONTAINER_ID:/tmp/measure-data/measure/data/index ${BANYANDB_DATA_GENERATE_ROOT}
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index 920a7a99e1..09a199d4aa 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -26,6 +26,7 @@
 * Enhance the trace `List/Tree/Table` graph to support displaying multiple refs of spans and distinguishing different parents.
 * Fix: correct the same labels for metrics.
 * Refactor: use the Fetch API to instead of Axios.
+* Support cold stage data for metrics, trace and log.
 
 #### Documentation
 
diff --git a/oap-server-bom/pom.xml b/oap-server-bom/pom.xml
index efb2d8fa2f..fc1ea0f655 100644
--- a/oap-server-bom/pom.xml
+++ b/oap-server-bom/pom.xml
@@ -72,7 +72,7 @@
         <httpcore.version>4.4.16</httpcore.version>
         <httpasyncclient.version>4.1.5</httpasyncclient.version>
         <commons-compress.version>1.21</commons-compress.version>
-        <banyandb-java-client.version>0.9.0-rc0</banyandb-java-client.version>
+        <banyandb-java-client.version>0.9.0-rc2</banyandb-java-client.version>
         <kafka-clients.version>3.4.0</kafka-clients.version>
         <spring-kafka-test.version>2.4.6.RELEASE</spring-kafka-test.version>
         <consul.client.version>1.5.3</consul.client.version>
diff --git a/oap-server/server-query-plugin/promql-plugin/src/main/java/org/apache/skywalking/oap/query/promql/handler/PromQLApiHandler.java b/oap-server/server-query-plugin/promql-plugin/src/main/java/org/apache/skywalking/oap/query/promql/handler/PromQLApiHandler.java
index 203ede8105..e97594be03 100644
--- a/oap-server/server-query-plugin/promql-plugin/src/main/java/org/apache/skywalking/oap/query/promql/handler/PromQLApiHandler.java
+++ b/oap-server/server-query-plugin/promql-plugin/src/main/java/org/apache/skywalking/oap/query/promql/handler/PromQLApiHandler.java
@@ -511,7 +511,7 @@ public class PromQLApiHandler {
     @Get
     @Post
     @Path("/api/v1/format_query")
-    public HttpResponse query_range(@Param("query") String query) throws IOException {
+    public HttpResponse format_query(@Param("query") String query) throws IOException {
         QueryFormatRsp rsp = new QueryFormatRsp();
         rsp.setData(query.replaceAll("\\s", ""));
         return jsonResponse(rsp);
diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBAggregationQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBAggregationQueryDAO.java
index 008bced45a..0fd25a9fa8 100644
--- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBAggregationQueryDAO.java
+++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBAggregationQueryDAO.java
@@ -78,7 +78,6 @@ public class BanyanDBAggregationQueryDAO extends AbstractBanyanDBDAO implements
         return directMetricsTopN(isColdStage, condition, schema, valueColumnName, spec, getTimestampRange(duration), additionalConditions);
     }
 
-    //todo: query cold stage
     List<SelectedRecord> serverSideTopN(boolean isColdStage, TopNCondition condition, MetadataRegistry.Schema schema, MetadataRegistry.ColumnSpec valueColumnSpec,
                                         TimestampRange timestampRange, List<KeyValue> additionalConditions) throws IOException {
         TopNQueryResponse resp = null;
diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java
index 6be527421e..2cdb5b158a 100644
--- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java
+++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java
@@ -139,22 +139,6 @@ public abstract class AbstractBanyanDBDAO extends AbstractDAO<BanyanDBStorageCli
         }
     }
 
-    protected TopNQueryResponse topN(MetadataRegistry.Schema schema,
-                                     TimestampRange timestampRange,
-                                     int number,
-                                     List<KeyValue> additionalConditions,
-                                     List<AttrCondition> attributes) throws IOException {
-        return topNQuery(schema, timestampRange, number, AbstractQuery.Sort.DESC, additionalConditions, attributes);
-    }
-
-    protected TopNQueryResponse bottomN(MetadataRegistry.Schema schema,
-                                        TimestampRange timestampRange,
-                                        int number,
-                                        List<KeyValue> additionalConditions,
-                                        List<AttrCondition> attributes) throws IOException {
-        return topNQuery(schema, timestampRange, number, AbstractQuery.Sort.ASC, additionalConditions, attributes);
-    }
-
     protected TopNQueryResponse topNQueryDebuggable(boolean isColdStage,
                                                     MetadataRegistry.Schema schema,
                                                     TimestampRange timestampRange,
@@ -185,7 +169,7 @@ public abstract class AbstractBanyanDBDAO extends AbstractDAO<BanyanDBStorageCli
                        .append(isColdStage);
                 span.setMsg(builder.toString());
             }
-            TopNQueryResponse response = topNQuery(schema, timestampRange, number, sort, additionalConditions, attributes);
+            TopNQueryResponse response = topNQuery(isColdStage, schema, timestampRange, number, sort, additionalConditions, attributes);
             if (traceContext != null && traceContext.isDumpStorageRsp()) {
                 builder.append("\n").append(" Response: ").append(new Gson().toJson(response.getTopNLists()));
                 span.setMsg(builder.toString());
@@ -198,7 +182,8 @@ public abstract class AbstractBanyanDBDAO extends AbstractDAO<BanyanDBStorageCli
         }
     }
 
-    private TopNQueryResponse topNQuery(MetadataRegistry.Schema schema,
+    private TopNQueryResponse topNQuery(boolean isColdStage,
+                                        MetadataRegistry.Schema schema,
                                         TimestampRange timestampRange,
                                         int number,
                                         AbstractQuery.Sort sort,
@@ -225,6 +210,9 @@ public abstract class AbstractBanyanDBDAO extends AbstractDAO<BanyanDBStorageCli
             });
         }
         q.setConditions(conditions);
+        if (isColdStage) {
+            q.setStages(List.of(BanyanDBStorageConfig.StageName.cold.name()));
+        }
 
         return getClient().query(q);
     }
diff --git a/skywalking-ui b/skywalking-ui
index 5e6e5aa737..a28972bc5c 160000
--- a/skywalking-ui
+++ b/skywalking-ui
@@ -1 +1 @@
-Subproject commit 5e6e5aa737979bc82846b555f6ea2e57689af088
+Subproject commit a28972bc5cefc09ada594b514bee4f3bfc156655
diff --git a/test/e2e-v2/cases/storage/banyandb/stages/e2e.yaml b/test/e2e-v2/cases/storage/banyandb/stages/e2e.yaml
index d02da95cc7..0e223372fb 100644
--- a/test/e2e-v2/cases/storage/banyandb/stages/e2e.yaml
+++ b/test/e2e-v2/cases/storage/banyandb/stages/e2e.yaml
@@ -69,3 +69,8 @@ verify:
             | yq e '.traces | select(.[].endpointnames[0]=="/dubbox-case/case/dubbox-rest/404-test") | .[2].traceids[0]' -
         ) --start="-96h" --end="-48h" --cold=true
       expected: ../../expected/cold/trace-detail.yml
+    ## topN
+    - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="top_n(service_resp_time,3,des,attr0='GENERAL')" --start="-96h" --end="-48h" --cold=true --step=DAY
+      expected: ../../expected/cold/topN-OP-service.yml
+    - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="top_n(endpoint_resp_time,3,des)" --start="-96h" --end="-48h" --cold=true --step=DAY
+      expected: ../../expected/cold/topN-OP-endpoint.yml
\ No newline at end of file
diff --git a/test/e2e-v2/cases/storage/expected/cold/dependency-instance.yml b/test/e2e-v2/cases/storage/expected/cold/dependency-instance.yml
index 90d583f3a2..7792498d7f 100644
--- a/test/e2e-v2/cases/storage/expected/cold/dependency-instance.yml
+++ b/test/e2e-v2/cases/storage/expected/cold/dependency-instance.yml
@@ -37,5 +37,6 @@ calls:
   targetcomponents: []
   id: {{ b64enc "mock_a_service" }}.1_{{ b64enc "mock_a_service_instance" }}-{{ b64enc "mock_b_service" }}.1_{{ b64enc "mock_b_service_instance" }}
   detectpoints:
+    - CLIENT
     - SERVER
 {{- end }}
diff --git a/test/e2e-v2/cases/storage/expected/cold/topN-OP-endpoint.yml b/test/e2e-v2/cases/storage/expected/cold/topN-OP-endpoint.yml
new file mode 100644
index 0000000000..c2bca18476
--- /dev/null
+++ b/test/e2e-v2/cases/storage/expected/cold/topN-OP-endpoint.yml
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+debuggingtrace: null
+type: SORTED_LIST
+results:
+  - metric:
+      labels: []
+    values:
+      - id: "mock_a_service - /dubbox-case/case/dubbox-rest/404-test"
+        value: "{{ (index (index .results 0).values 0).value }}"
+        traceid: null
+        owner:
+          scope: Endpoint
+          serviceid: {{ b64enc "mock_a_service"}}.1
+          servicename: mock_a_service
+          normal: true
+          serviceinstanceid: null
+          serviceinstancename: null
+          endpointid: {{ b64enc "mock_a_service" }}.1_{{ b64enc "/dubbox-case/case/dubbox-rest/404-test" }}
+          endpointname: /dubbox-case/case/dubbox-rest/404-test
+      - id: "mock_b_service - org.skywaking.apm.testcase.dubbo.services.GreetServiceImpl.doBusiness()"
+        value: "{{ (index (index .results 0).values 1).value }}"
+        traceid: null
+        owner:
+          scope: Endpoint
+          serviceid: {{ b64enc "mock_b_service" }}.1
+          servicename: mock_b_service
+          normal: true
+          serviceinstanceid: null
+          serviceinstancename: null
+          endpointid: {{ b64enc "mock_b_service" }}.1_{{ b64enc "org.skywaking.apm.testcase.dubbo.services.GreetServiceImpl.doBusiness()" }}
+          endpointname: org.skywaking.apm.testcase.dubbo.services.GreetServiceImpl.doBusiness()
+      - id: "mock_c_service - org.apache.skywalking.RocketMQ"
+        value: "{{ (index (index .results 0).values 2).value }}"
+        traceid: null
+        owner:
+          scope: Endpoint
+          serviceid: {{ b64enc "mock_c_service" }}.1
+          servicename: mock_c_service
+          normal: true
+          serviceinstanceid: null
+          serviceinstancename: null
+          endpointid: {{ b64enc "mock_c_service" }}.1_{{ b64enc "org.apache.skywalking.RocketMQ" }}
+          endpointname: org.apache.skywalking.RocketMQ
+error: null
diff --git a/test/e2e-v2/cases/storage/expected/cold/topN-OP-service.yml b/test/e2e-v2/cases/storage/expected/cold/topN-OP-service.yml
new file mode 100644
index 0000000000..6f9f0786ce
--- /dev/null
+++ b/test/e2e-v2/cases/storage/expected/cold/topN-OP-service.yml
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+debuggingtrace: null
+type: SORTED_LIST
+results:
+  - metric:
+      labels: []
+    values:
+      - id: mock_a_service
+        value: "{{ (index (index .results 0).values 0).value }}"
+        traceid: null
+        owner:
+          scope: Service
+          serviceid: {{ b64enc "mock_a_service"}}.1
+          servicename: mock_a_service
+          normal: true
+          serviceinstanceid: null
+          serviceinstancename: null
+          endpointid: null
+          endpointname: null
+      - id: mock_b_service
+        value: "{{ (index (index .results 0).values 1).value }}"
+        traceid: null
+        owner:
+          scope: Service
+          serviceid: {{ b64enc "mock_b_service"}}.1
+          servicename: mock_b_service
+          normal: true
+          serviceinstanceid: null
+          serviceinstancename: null
+          endpointid: null
+          endpointname: null
+      - id: mock_c_service
+        value: "{{ (index (index .results 0).values 2).value }}"
+        traceid: null
+        owner:
+          scope: Service
+          serviceid: {{ b64enc "mock_c_service" }}.1
+          servicename: mock_c_service
+          normal: true
+          serviceinstanceid: null
+          serviceinstancename: null
+          endpointid: null
+          endpointname: null
+error: null
diff --git a/test/e2e-v2/java-test-service/trace-mocker/src/main/java/org/apache/skywalking/mock/AgentDataMock.java b/test/e2e-v2/java-test-service/trace-mocker/src/main/java/org/apache/skywalking/mock/AgentDataMock.java
index 645ee6e1c4..3be0d80640 100644
--- a/test/e2e-v2/java-test-service/trace-mocker/src/main/java/org/apache/skywalking/mock/AgentDataMock.java
+++ b/test/e2e-v2/java-test-service/trace-mocker/src/main/java/org/apache/skywalking/mock/AgentDataMock.java
@@ -73,6 +73,7 @@ public class AgentDataMock {
                 streamObserver, traceId, serviceBSegmentId, serviceASegmentId, startTimestamp);
             serviceCMock.mock(
                 streamObserver, traceId, serviceCSegmentId, serviceBSegmentId, startTimestamp);
+            TimeUnit.SECONDS.sleep(30);
         }
 
         streamObserver.onCompleted();