kazusa
/
skywalking
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

BanyanDB: add topN cold stage data query. (#13259)

This commit is contained in:
Wan Kai 2025-05-22 20:36:47 +08:00 committed by GitHub
parent 5622cb9f71
commit 80da79ebfa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 134 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/skywalking.yaml vendored
View File

@ -1021,7 +1021,7 @@ jobs:
# check if segment files exist
if docker exec $CONTAINER_ID sh -c '[ -n "$(ls /tmp/measure-data/measure/data/day/seg* 2>/dev/null)" ]'; then
echo "✅ found segment files"
sleep 30
sleep 180
# create and copy files
docker cp $CONTAINER_ID:/tmp ${BANYANDB_DATA_GENERATE_ROOT}
docker cp $CONTAINER_ID:/tmp/measure-data/measure/data/index ${BANYANDB_DATA_GENERATE_ROOT}

1
docs/en/changes/changes.md
View File

@ -26,6 +26,7 @@
* Enhance the trace `List/Tree/Table` graph to support displaying multiple refs of spans and distinguishing different parents.
* Fix: correct the same labels for metrics.
* Refactor: use the Fetch API to instead of Axios.
* Support cold stage data for metrics, trace and log.
#### Documentation

2
oap-server-bom/pom.xml
View File

@ -72,7 +72,7 @@
<httpcore.version>4.4.16</httpcore.version>
<httpasyncclient.version>4.1.5</httpasyncclient.version>
<commons-compress.version>1.21</commons-compress.version>
<banyandb-java-client.version>0.9.0-rc0</banyandb-java-client.version>
<banyandb-java-client.version>0.9.0-rc2</banyandb-java-client.version>
<kafka-clients.version>3.4.0</kafka-clients.version>
<spring-kafka-test.version>2.4.6.RELEASE</spring-kafka-test.version>
<consul.client.version>1.5.3</consul.client.version>

2
oap-server/server-query-plugin/promql-plugin/src/main/java/org/apache/skywalking/oap/query/promql/handler/PromQLApiHandler.java
View File

@ -511,7 +511,7 @@ public class PromQLApiHandler {
@Get
@Post
@Path("/api/v1/format_query")
public HttpResponse query_range(@Param("query") String query) throws IOException {
public HttpResponse format_query(@Param("query") String query) throws IOException {
QueryFormatRsp rsp = new QueryFormatRsp();
rsp.setData(query.replaceAll("\\s", ""));
return jsonResponse(rsp);

1
oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBAggregationQueryDAO.java
View File

@ -78,7 +78,6 @@ public class BanyanDBAggregationQueryDAO extends AbstractBanyanDBDAO implements
return directMetricsTopN(isColdStage, condition, schema, valueColumnName, spec, getTimestampRange(duration), additionalConditions);
}
//todo: query cold stage
List<SelectedRecord> serverSideTopN(boolean isColdStage, TopNCondition condition, MetadataRegistry.Schema schema, MetadataRegistry.ColumnSpec valueColumnSpec,
TimestampRange timestampRange, List<KeyValue> additionalConditions) throws IOException {
TopNQueryResponse resp = null;

24
oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java
View File

@ -139,22 +139,6 @@ public abstract class AbstractBanyanDBDAO extends AbstractDAO<BanyanDBStorageCli
}
}
protected TopNQueryResponse topN(MetadataRegistry.Schema schema,
TimestampRange timestampRange,
int number,
List<KeyValue> additionalConditions,
List<AttrCondition> attributes) throws IOException {
return topNQuery(schema, timestampRange, number, AbstractQuery.Sort.DESC, additionalConditions, attributes);
}
protected TopNQueryResponse bottomN(MetadataRegistry.Schema schema,
TimestampRange timestampRange,
int number,
List<KeyValue> additionalConditions,
List<AttrCondition> attributes) throws IOException {
return topNQuery(schema, timestampRange, number, AbstractQuery.Sort.ASC, additionalConditions, attributes);
}
protected TopNQueryResponse topNQueryDebuggable(boolean isColdStage,
MetadataRegistry.Schema schema,
TimestampRange timestampRange,
@ -185,7 +169,7 @@ public abstract class AbstractBanyanDBDAO extends AbstractDAO<BanyanDBStorageCli
.append(isColdStage);
span.setMsg(builder.toString());
}
TopNQueryResponse response = topNQuery(schema, timestampRange, number, sort, additionalConditions, attributes);
TopNQueryResponse response = topNQuery(isColdStage, schema, timestampRange, number, sort, additionalConditions, attributes);
if (traceContext != null && traceContext.isDumpStorageRsp()) {
builder.append("\n").append(" Response: ").append(new Gson().toJson(response.getTopNLists()));
span.setMsg(builder.toString());
@ -198,7 +182,8 @@ public abstract class AbstractBanyanDBDAO extends AbstractDAO<BanyanDBStorageCli
}
}
private TopNQueryResponse topNQuery(MetadataRegistry.Schema schema,
private TopNQueryResponse topNQuery(boolean isColdStage,
MetadataRegistry.Schema schema,
TimestampRange timestampRange,
int number,
AbstractQuery.Sort sort,
@ -225,6 +210,9 @@ public abstract class AbstractBanyanDBDAO extends AbstractDAO<BanyanDBStorageCli
});
}
q.setConditions(conditions);
if (isColdStage) {
q.setStages(List.of(BanyanDBStorageConfig.StageName.cold.name()));
}
return getClient().query(q);
}

2
skywalking-ui

@ -1 +1 @@
Subproject commit 5e6e5aa737979bc82846b555f6ea2e57689af088
Subproject commit a28972bc5cefc09ada594b514bee4f3bfc156655

5
test/e2e-v2/cases/storage/banyandb/stages/e2e.yaml
View File

@ -69,3 +69,8 @@ verify:
| yq e '.traces | select(.[].endpointnames[0]=="/dubbox-case/case/dubbox-rest/404-test") | .[2].traceids[0]' -
) --start="-96h" --end="-48h" --cold=true
expected: ../../expected/cold/trace-detail.yml
## topN
- query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="top_n(service_resp_time,3,des,attr0='GENERAL')" --start="-96h" --end="-48h" --cold=true --step=DAY
expected: ../../expected/cold/topN-OP-service.yml
- query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="top_n(endpoint_resp_time,3,des)" --start="-96h" --end="-48h" --cold=true --step=DAY
expected: ../../expected/cold/topN-OP-endpoint.yml

1
test/e2e-v2/cases/storage/expected/cold/dependency-instance.yml
View File

@ -37,5 +37,6 @@ calls:
targetcomponents: []
id: {{ b64enc "mock_a_service" }}.1_{{ b64enc "mock_a_service_instance" }}-{{ b64enc "mock_b_service" }}.1_{{ b64enc "mock_b_service_instance" }}
detectpoints:
- CLIENT
- SERVER
{{- end }}

58
test/e2e-v2/cases/storage/expected/cold/topN-OP-endpoint.yml Normal file
View File

@ -0,0 +1,58 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
debuggingtrace: null
type: SORTED_LIST
results:
- metric:
labels: []
values:
- id: "mock_a_service - /dubbox-case/case/dubbox-rest/404-test"
value: "{{ (index (index .results 0).values 0).value }}"
traceid: null
owner:
scope: Endpoint
serviceid: {{ b64enc "mock_a_service"}}.1
servicename: mock_a_service
normal: true
serviceinstanceid: null
serviceinstancename: null
endpointid: {{ b64enc "mock_a_service" }}.1_{{ b64enc "/dubbox-case/case/dubbox-rest/404-test" }}
endpointname: /dubbox-case/case/dubbox-rest/404-test
- id: "mock_b_service - org.skywaking.apm.testcase.dubbo.services.GreetServiceImpl.doBusiness()"
value: "{{ (index (index .results 0).values 1).value }}"
traceid: null
owner:
scope: Endpoint
serviceid: {{ b64enc "mock_b_service" }}.1
servicename: mock_b_service
normal: true
serviceinstanceid: null
serviceinstancename: null
endpointid: {{ b64enc "mock_b_service" }}.1_{{ b64enc "org.skywaking.apm.testcase.dubbo.services.GreetServiceImpl.doBusiness()" }}
endpointname: org.skywaking.apm.testcase.dubbo.services.GreetServiceImpl.doBusiness()
- id: "mock_c_service - org.apache.skywalking.RocketMQ"
value: "{{ (index (index .results 0).values 2).value }}"
traceid: null
owner:
scope: Endpoint
serviceid: {{ b64enc "mock_c_service" }}.1
servicename: mock_c_service
normal: true
serviceinstanceid: null
serviceinstancename: null
endpointid: {{ b64enc "mock_c_service" }}.1_{{ b64enc "org.apache.skywalking.RocketMQ" }}
endpointname: org.apache.skywalking.RocketMQ
error: null

58
test/e2e-v2/cases/storage/expected/cold/topN-OP-service.yml Normal file
View File

@ -0,0 +1,58 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
debuggingtrace: null
type: SORTED_LIST
results:
- metric:
labels: []
values:
- id: mock_a_service
value: "{{ (index (index .results 0).values 0).value }}"
traceid: null
owner:
scope: Service
serviceid: {{ b64enc "mock_a_service"}}.1
servicename: mock_a_service
normal: true
serviceinstanceid: null
serviceinstancename: null
endpointid: null
endpointname: null
- id: mock_b_service
value: "{{ (index (index .results 0).values 1).value }}"
traceid: null
owner:
scope: Service
serviceid: {{ b64enc "mock_b_service"}}.1
servicename: mock_b_service
normal: true
serviceinstanceid: null
serviceinstancename: null
endpointid: null
endpointname: null
- id: mock_c_service
value: "{{ (index (index .results 0).values 2).value }}"
traceid: null
owner:
scope: Service
serviceid: {{ b64enc "mock_c_service" }}.1
servicename: mock_c_service
normal: true
serviceinstanceid: null
serviceinstancename: null
endpointid: null
endpointname: null
error: null

1
test/e2e-v2/java-test-service/trace-mocker/src/main/java/org/apache/skywalking/mock/AgentDataMock.java
View File

@ -73,6 +73,7 @@ public class AgentDataMock {
streamObserver, traceId, serviceBSegmentId, serviceASegmentId, startTimestamp);
serviceCMock.mock(
streamObserver, traceId, serviceCSegmentId, serviceBSegmentId, startTimestamp);
TimeUnit.SECONDS.sleep(30);
}
streamObserver.onCompleted();