kazusa
/
skywalking
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add limit config for query autocomplete tags (#9140)

This commit is contained in:
Wan Kai 2022-05-27 20:34:49 +08:00 committed by GitHub
parent 9786ed838c
commit d69213ec6b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 40 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/en/setup/backend/configuration-vocabulary.md
View File

@ -35,6 +35,8 @@ The Configuration Vocabulary lists all available configurations provided by `app
| - | - | searchableTracesTags | Defines a set of span tag keys which are searchable through GraphQL. Multiple values are separated by commas. | SW_SEARCHABLE_TAG_KEYS | http.method,http.status_code,rpc.status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker |
| - | - | searchableLogsTags | Defines a set of log tag keys which are searchable through GraphQL. Multiple values are separated by commas. | SW_SEARCHABLE_LOGS_TAG_KEYS | level |
| - | - | searchableAlarmTags | Defines a set of alarm tag keys which are searchable through GraphQL. Multiple values are separated by commas. | SW_SEARCHABLE_ALARM_TAG_KEYS | level |
| - | - | autocompleteTagKeysQueryMaxSize | The max size of tags keys for autocomplete select. | SW_AUTOCOMPLETE_TAG_KEYS_QUERY_MAX_SIZE | 100 |
| - | - | autocompleteTagValuesQueryMaxSize | The max size of tags values for autocomplete select. | SW_AUTOCOMPLETE_TAG_VALUES_QUERY_MAX_SIZE | 100 |
| - | - | gRPCThreadPoolSize | Pool size of gRPC server. | SW_CORE_GRPC_THREAD_POOL_SIZE | CPU core * 4 |
| - | - | gRPCThreadPoolQueueSize | Queue size of gRPC server. | SW_CORE_GRPC_POOL_QUEUE_SIZE | 10000 |
| - | - | maxConcurrentCallsPerConnection | The maximum number of concurrent calls permitted for each incoming connection. Defaults to no limit. | SW_CORE_GRPC_MAX_CONCURRENT_CALL | - |
@ -227,7 +229,7 @@ The Configuration Vocabulary lists all available configurations provided by `app
| - | - | enableLogTestTool | Enable the log testing API to test the LAL. **NOTE**: This API evaluates untrusted code on the OAP server. A malicious script can do significant damage (steal keys and secrets, remove files and directories, install malware, etc). As such, please enable this API only when you completely trust your users. | SW_QUERY_GRAPHQL_ENABLE_LOG_TEST_TOOL | false |
| - | - | maxQueryComplexity | Maximum complexity allowed for the GraphQL query that can be used to abort a query if the total number of data fields queried exceeds the defined threshold. | SW_QUERY_MAX_QUERY_COMPLEXITY | 1000 |
| - | - | enableUpdateUITemplate | Allow user add，disable and update UI template. | SW_ENABLE_UPDATE_UI_TEMPLATE | false |
| - | - | enableOnDemandPodLog | Ondemand Pod log: fetch the Pod logs on users' demand, the logs are fetched and displayed in real time, and are not persisted in any kind. This is helpful when users want to do some experiments and monitor the logs and see what's happing inside the service. Note: if you print secrets in the logs, they are also visible to the UI, so for the sake of security, this feature is disabled by default, please set this configuration to enable the feature manually. | SW_ENABLE_ON_DEMAND_POD_LOG | false |
| - | - | enableOnDemandPodLog | Ondemand Pod log: fetch the Pod logs on users' demand, the logs are fetched and displayed in real time, and are not persisted in any kind. This is helpful when users want to do some experiments and monitor the logs and see what's happing inside the service. Note: if you print secrets in the logs, they are also visible to the UI, so for the sake of security, this feature is disabled by default, please set this configuration to enable the feature manually. | SW_ENABLE_ON_DEMAND_POD_LOG | false |
| alarm | default | - | Read [alarm doc](backend-alarm.md) for more details. | - | |
| telemetry | - | - | Read [telemetry doc](backend-telemetry.md) for more details. | - | |
| - | none | - | No op implementation. | - | |

17
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java
View File

@ -158,7 +158,22 @@ public class CoreModuleConfig extends ModuleConfig {
@Setter
@Getter
private String searchableAlarmTags = "";
/**
* The max size of tags keys for autocomplete select.
*
* @since 9.1.0
*/
@Setter
@Getter
private int autocompleteTagKeysQueryMaxSize = 100;
/**
* The max size of tags values for autocomplete select.
*
* @since 9.1.0
*/
@Setter
@Getter
private int autocompleteTagValuesQueryMaxSize = 100;
/**
* The number of threads used to prepare metrics data to the storage.
*

2
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java
View File

@ -264,7 +264,7 @@ public class CoreModuleProvider extends ModuleProvider {
this.registerServiceImplementation(AlarmQueryService.class, new AlarmQueryService(getManager()));
this.registerServiceImplementation(TopNRecordsQueryService.class, new TopNRecordsQueryService(getManager()));
this.registerServiceImplementation(EventQueryService.class, new EventQueryService(getManager()));
this.registerServiceImplementation(TagAutoCompleteQueryService.class, new TagAutoCompleteQueryService(getManager()));
this.registerServiceImplementation(TagAutoCompleteQueryService.class, new TagAutoCompleteQueryService(getManager(), moduleConfig));
// add profile service implementations
this.registerServiceImplementation(

10
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TagAutoCompleteQueryService.java
View File

@ -20,6 +20,7 @@ package org.apache.skywalking.oap.server.core.query;
import java.io.IOException;
import java.util.Set;
import org.apache.skywalking.oap.server.core.CoreModuleConfig;
import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType;
import org.apache.skywalking.oap.server.core.storage.StorageModule;
import org.apache.skywalking.oap.server.core.storage.query.ITagAutoCompleteQueryDAO;
@ -28,10 +29,12 @@ import org.apache.skywalking.oap.server.library.module.Service;
public class TagAutoCompleteQueryService implements Service {
private final ModuleManager moduleManager;
private final CoreModuleConfig config;
private ITagAutoCompleteQueryDAO tagAutoCompleteQueryDAO;
public TagAutoCompleteQueryService(ModuleManager moduleManager) {
public TagAutoCompleteQueryService(ModuleManager moduleManager, CoreModuleConfig config) {
this.moduleManager = moduleManager;
this.config = config;
}
private ITagAutoCompleteQueryDAO getTagAutoCompleteQueryDAO() {
@ -44,15 +47,14 @@ public class TagAutoCompleteQueryService implements Service {
public Set<String> queryTagAutocompleteKeys(final TagType tagType,
final long startSecondTB,
final long endSecondTB) throws IOException {
return getTagAutoCompleteQueryDAO().queryTagAutocompleteKeys(tagType, startSecondTB, endSecondTB);
return getTagAutoCompleteQueryDAO().queryTagAutocompleteKeys(tagType, config.getAutocompleteTagKeysQueryMaxSize(), startSecondTB, endSecondTB);
}
public Set<String> queryTagAutocompleteValues(final TagType tagType,
final String tagKey,
final int limit,
final long startSecondTB,
final long endSecondTB) throws IOException {
return getTagAutoCompleteQueryDAO().queryTagAutocompleteValues(
tagType, tagKey, limit, startSecondTB, endSecondTB);
tagType, tagKey, config.getAutocompleteTagValuesQueryMaxSize(), startSecondTB, endSecondTB);
}
}

1
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/ITagAutoCompleteQueryDAO.java
View File

@ -25,6 +25,7 @@ import org.apache.skywalking.oap.server.library.module.Service;
public interface ITagAutoCompleteQueryDAO extends Service {
Set<String> queryTagAutocompleteKeys(final TagType tagType,
final int limit,
final long startSecondTB,
final long endSecondTB) throws IOException;

2
oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/LogQuery.java
View File

@ -106,6 +106,6 @@ public class LogQuery implements GraphQLQueryResolver {
}
public Set<String> queryLogTagAutocompleteValues(final String tagKey, final Duration queryDuration) throws IOException {
return getTagQueryService().queryTagAutocompleteValues(TagType.LOG, tagKey, 100, queryDuration.getStartTimeBucketInSec(), queryDuration.getEndTimeBucketInSec());
return getTagQueryService().queryTagAutocompleteValues(TagType.LOG, tagKey, queryDuration.getStartTimeBucketInSec(), queryDuration.getEndTimeBucketInSec());
}
}

2
oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/TraceQuery.java
View File

@ -99,6 +99,6 @@ public class TraceQuery implements GraphQLQueryResolver {
}
public Set<String> queryTraceTagAutocompleteValues(final String tagKey, final Duration queryDuration) throws IOException {
return getTagQueryService().queryTagAutocompleteValues(TagType.TRACE, tagKey, 100, queryDuration.getStartTimeBucketInSec(), queryDuration.getEndTimeBucketInSec());
return getTagQueryService().queryTagAutocompleteValues(TagType.TRACE, tagKey, queryDuration.getStartTimeBucketInSec(), queryDuration.getEndTimeBucketInSec());
}
}

4
oap-server/server-starter/src/main/resources/application.yml
View File

@ -121,6 +121,10 @@ core:
searchableLogsTags: ${SW_SEARCHABLE_LOGS_TAG_KEYS:level}
# Define the set of alarm tag keys, which should be searchable through the GraphQL.
searchableAlarmTags: ${SW_SEARCHABLE_ALARM_TAG_KEYS:level}
# The max size of tags keys for autocomplete select.
autocompleteTagKeysQueryMaxSize: ${SW_AUTOCOMPLETE_TAG_KEYS_QUERY_MAX_SIZE:100}
# The max size of tags values for autocomplete select.
autocompleteTagValuesQueryMaxSize: ${SW_AUTOCOMPLETE_TAG_VALUES_QUERY_MAX_SIZE:100}
# The number of threads used to prepare metrics data to the storage.
prepareThreads: ${SW_CORE_PREPARE_THREADS:2}
# Turn it on then automatically grouping endpoint by the given OpenAPI definitions.

3
oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTagAutocompleteQueryDAO.java
View File

@ -47,7 +47,7 @@ public class BanyanDBTagAutocompleteQueryDAO extends AbstractBanyanDBDAO impleme
}
@Override
public Set<String> queryTagAutocompleteKeys(TagType tagType, long startSecondTB, long endSecondTB) throws IOException {
public Set<String> queryTagAutocompleteKeys(TagType tagType, int limit, long startSecondTB, long endSecondTB) throws IOException {
TimestampRange range = null;
if (startSecondTB > 0 && endSecondTB > 0) {
range = new TimestampRange(TimeBucket.getTimestamp(startSecondTB), TimeBucket.getTimestamp(endSecondTB));
@ -59,6 +59,7 @@ public class BanyanDBTagAutocompleteQueryDAO extends AbstractBanyanDBDAO impleme
@Override
protected void apply(MeasureQuery query) {
query.groupBy(ImmutableSet.of(TagAutocompleteData.TAG_KEY));
query.setLimit(limit);
query.and(eq(TagAutocompleteData.TAG_TYPE, tagType.name()));
}
}

4
oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java
View File

@ -47,13 +47,15 @@ public class TagAutoCompleteQueryDAO extends EsDAO implements ITagAutoCompleteQu
@Override
public Set<String> queryTagAutocompleteKeys(final TagType tagType,
final int limit,
final long startSecondTB,
final long endSecondTB) throws IOException {
BoolQueryBuilder query = Query.bool();
query.must(Query.term(TagAutocompleteData.TAG_TYPE, tagType.name()));
final SearchBuilder search = Search.builder().query(query);
search.aggregation(Aggregation.terms(TagAutocompleteData.TAG_KEY)
.field(TagAutocompleteData.TAG_KEY));
.field(TagAutocompleteData.TAG_KEY)
.size(limit));
final SearchResponse response = getClient().search(
new TimeRangeIndexNameGenerator(

2
oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TagAutoCompleteQueryDAO.java
View File

@ -38,6 +38,7 @@ public class H2TagAutoCompleteQueryDAO implements ITagAutoCompleteQueryDAO {
@Override
public Set<String> queryTagAutocompleteKeys(final TagType tagType,
final int limit,
final long startSecondTB,
final long endSecondTB) throws IOException {
StringBuilder sql = new StringBuilder();
@ -47,6 +48,7 @@ public class H2TagAutoCompleteQueryDAO implements ITagAutoCompleteQueryDAO {
.append(TagAutocompleteData.INDEX_NAME).append(" where ");
sql.append(" 1=1 ");
appendTagAutocompleteCondition(tagType, startSecondTB, endSecondTB, sql, condition);
sql.append(" limit ").append(limit);
try (Connection connection = h2Client.getConnection()) {
ResultSet resultSet = h2Client.executeQuery(connection, sql.toString(), condition.toArray(new Object[0]));
Set<String> tagKeys = new HashSet<>();