diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md index 921915de1d..c6952d2063 100644 --- a/docs/en/setup/backend/configuration-vocabulary.md +++ b/docs/en/setup/backend/configuration-vocabulary.md @@ -35,6 +35,8 @@ The Configuration Vocabulary lists all available configurations provided by `app | - | - | searchableTracesTags | Defines a set of span tag keys which are searchable through GraphQL. Multiple values are separated by commas. | SW_SEARCHABLE_TAG_KEYS | http.method,http.status_code,rpc.status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker | | - | - | searchableLogsTags | Defines a set of log tag keys which are searchable through GraphQL. Multiple values are separated by commas. | SW_SEARCHABLE_LOGS_TAG_KEYS | level | | - | - | searchableAlarmTags | Defines a set of alarm tag keys which are searchable through GraphQL. Multiple values are separated by commas. | SW_SEARCHABLE_ALARM_TAG_KEYS | level | +| - | - | autocompleteTagKeysQueryMaxSize | The max size of tags keys for autocomplete select. | SW_AUTOCOMPLETE_TAG_KEYS_QUERY_MAX_SIZE | 100 | +| - | - | autocompleteTagValuesQueryMaxSize | The max size of tags values for autocomplete select. | SW_AUTOCOMPLETE_TAG_VALUES_QUERY_MAX_SIZE | 100 | | - | - | gRPCThreadPoolSize | Pool size of gRPC server. | SW_CORE_GRPC_THREAD_POOL_SIZE | CPU core * 4 | | - | - | gRPCThreadPoolQueueSize | Queue size of gRPC server. | SW_CORE_GRPC_POOL_QUEUE_SIZE | 10000 | | - | - | maxConcurrentCallsPerConnection | The maximum number of concurrent calls permitted for each incoming connection. Defaults to no limit. | SW_CORE_GRPC_MAX_CONCURRENT_CALL | - | @@ -227,7 +229,7 @@ The Configuration Vocabulary lists all available configurations provided by `app | - | - | enableLogTestTool | Enable the log testing API to test the LAL. **NOTE**: This API evaluates untrusted code on the OAP server. A malicious script can do significant damage (steal keys and secrets, remove files and directories, install malware, etc). As such, please enable this API only when you completely trust your users. | SW_QUERY_GRAPHQL_ENABLE_LOG_TEST_TOOL | false | | - | - | maxQueryComplexity | Maximum complexity allowed for the GraphQL query that can be used to abort a query if the total number of data fields queried exceeds the defined threshold. | SW_QUERY_MAX_QUERY_COMPLEXITY | 1000 | | - | - | enableUpdateUITemplate | Allow user add，disable and update UI template. | SW_ENABLE_UPDATE_UI_TEMPLATE | false | -| - | - | enableOnDemandPodLog | Ondemand Pod log: fetch the Pod logs on users' demand, the logs are fetched and displayed in real time, and are not persisted in any kind. This is helpful when users want to do some experiments and monitor the logs and see what's happing inside the service. Note: if you print secrets in the logs, they are also visible to the UI, so for the sake of security, this feature is disabled by default, please set this configuration to enable the feature manually. | SW_ENABLE_ON_DEMAND_POD_LOG | false | +| - | - | enableOnDemandPodLog | Ondemand Pod log: fetch the Pod logs on users' demand, the logs are fetched and displayed in real time, and are not persisted in any kind. This is helpful when users want to do some experiments and monitor the logs and see what's happing inside the service. Note: if you print secrets in the logs, they are also visible to the UI, so for the sake of security, this feature is disabled by default, please set this configuration to enable the feature manually. | SW_ENABLE_ON_DEMAND_POD_LOG | false | | alarm | default | - | Read [alarm doc](backend-alarm.md) for more details. | - | | | telemetry | - | - | Read [telemetry doc](backend-telemetry.md) for more details. | - | | | - | none | - | No op implementation. | - | | diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java index 2686c0a652..51d33db88a 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java @@ -158,7 +158,22 @@ public class CoreModuleConfig extends ModuleConfig { @Setter @Getter private String searchableAlarmTags = ""; - + /** + * The max size of tags keys for autocomplete select. + * + * @since 9.1.0 + */ + @Setter + @Getter + private int autocompleteTagKeysQueryMaxSize = 100; + /** + * The max size of tags values for autocomplete select. + * + * @since 9.1.0 + */ + @Setter + @Getter + private int autocompleteTagValuesQueryMaxSize = 100; /** * The number of threads used to prepare metrics data to the storage. * diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java index 074980b563..de79ac0d17 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java @@ -264,7 +264,7 @@ public class CoreModuleProvider extends ModuleProvider { this.registerServiceImplementation(AlarmQueryService.class, new AlarmQueryService(getManager())); this.registerServiceImplementation(TopNRecordsQueryService.class, new TopNRecordsQueryService(getManager())); this.registerServiceImplementation(EventQueryService.class, new EventQueryService(getManager())); - this.registerServiceImplementation(TagAutoCompleteQueryService.class, new TagAutoCompleteQueryService(getManager())); + this.registerServiceImplementation(TagAutoCompleteQueryService.class, new TagAutoCompleteQueryService(getManager(), moduleConfig)); // add profile service implementations this.registerServiceImplementation( diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TagAutoCompleteQueryService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TagAutoCompleteQueryService.java index ce4c9ebe06..79992dd1dc 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TagAutoCompleteQueryService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TagAutoCompleteQueryService.java @@ -20,6 +20,7 @@ package org.apache.skywalking.oap.server.core.query; import java.io.IOException; import java.util.Set; +import org.apache.skywalking.oap.server.core.CoreModuleConfig; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; import org.apache.skywalking.oap.server.core.storage.StorageModule; import org.apache.skywalking.oap.server.core.storage.query.ITagAutoCompleteQueryDAO; @@ -28,10 +29,12 @@ import org.apache.skywalking.oap.server.library.module.Service; public class TagAutoCompleteQueryService implements Service { private final ModuleManager moduleManager; + private final CoreModuleConfig config; private ITagAutoCompleteQueryDAO tagAutoCompleteQueryDAO; - public TagAutoCompleteQueryService(ModuleManager moduleManager) { + public TagAutoCompleteQueryService(ModuleManager moduleManager, CoreModuleConfig config) { this.moduleManager = moduleManager; + this.config = config; } private ITagAutoCompleteQueryDAO getTagAutoCompleteQueryDAO() { @@ -44,15 +47,14 @@ public class TagAutoCompleteQueryService implements Service { public Set queryTagAutocompleteKeys(final TagType tagType, final long startSecondTB, final long endSecondTB) throws IOException { - return getTagAutoCompleteQueryDAO().queryTagAutocompleteKeys(tagType, startSecondTB, endSecondTB); + return getTagAutoCompleteQueryDAO().queryTagAutocompleteKeys(tagType, config.getAutocompleteTagKeysQueryMaxSize(), startSecondTB, endSecondTB); } public Set queryTagAutocompleteValues(final TagType tagType, final String tagKey, - final int limit, final long startSecondTB, final long endSecondTB) throws IOException { return getTagAutoCompleteQueryDAO().queryTagAutocompleteValues( - tagType, tagKey, limit, startSecondTB, endSecondTB); + tagType, tagKey, config.getAutocompleteTagValuesQueryMaxSize(), startSecondTB, endSecondTB); } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/ITagAutoCompleteQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/ITagAutoCompleteQueryDAO.java index fc0d1c68b5..e54802a949 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/ITagAutoCompleteQueryDAO.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/ITagAutoCompleteQueryDAO.java @@ -25,6 +25,7 @@ import org.apache.skywalking.oap.server.library.module.Service; public interface ITagAutoCompleteQueryDAO extends Service { Set queryTagAutocompleteKeys(final TagType tagType, + final int limit, final long startSecondTB, final long endSecondTB) throws IOException; diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/LogQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/LogQuery.java index 28c8b47d3b..4c0b132895 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/LogQuery.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/LogQuery.java @@ -106,6 +106,6 @@ public class LogQuery implements GraphQLQueryResolver { } public Set queryLogTagAutocompleteValues(final String tagKey, final Duration queryDuration) throws IOException { - return getTagQueryService().queryTagAutocompleteValues(TagType.LOG, tagKey, 100, queryDuration.getStartTimeBucketInSec(), queryDuration.getEndTimeBucketInSec()); + return getTagQueryService().queryTagAutocompleteValues(TagType.LOG, tagKey, queryDuration.getStartTimeBucketInSec(), queryDuration.getEndTimeBucketInSec()); } } diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/TraceQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/TraceQuery.java index 045154eb44..4f1202cf05 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/TraceQuery.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/TraceQuery.java @@ -99,6 +99,6 @@ public class TraceQuery implements GraphQLQueryResolver { } public Set queryTraceTagAutocompleteValues(final String tagKey, final Duration queryDuration) throws IOException { - return getTagQueryService().queryTagAutocompleteValues(TagType.TRACE, tagKey, 100, queryDuration.getStartTimeBucketInSec(), queryDuration.getEndTimeBucketInSec()); + return getTagQueryService().queryTagAutocompleteValues(TagType.TRACE, tagKey, queryDuration.getStartTimeBucketInSec(), queryDuration.getEndTimeBucketInSec()); } } diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 5571c101ca..a65fe59064 100755 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -121,6 +121,10 @@ core: searchableLogsTags: ${SW_SEARCHABLE_LOGS_TAG_KEYS:level} # Define the set of alarm tag keys, which should be searchable through the GraphQL. searchableAlarmTags: ${SW_SEARCHABLE_ALARM_TAG_KEYS:level} + # The max size of tags keys for autocomplete select. + autocompleteTagKeysQueryMaxSize: ${SW_AUTOCOMPLETE_TAG_KEYS_QUERY_MAX_SIZE:100} + # The max size of tags values for autocomplete select. + autocompleteTagValuesQueryMaxSize: ${SW_AUTOCOMPLETE_TAG_VALUES_QUERY_MAX_SIZE:100} # The number of threads used to prepare metrics data to the storage. prepareThreads: ${SW_CORE_PREPARE_THREADS:2} # Turn it on then automatically grouping endpoint by the given OpenAPI definitions. diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTagAutocompleteQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTagAutocompleteQueryDAO.java index ab154529b5..2e330a64d7 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTagAutocompleteQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTagAutocompleteQueryDAO.java @@ -47,7 +47,7 @@ public class BanyanDBTagAutocompleteQueryDAO extends AbstractBanyanDBDAO impleme } @Override - public Set queryTagAutocompleteKeys(TagType tagType, long startSecondTB, long endSecondTB) throws IOException { + public Set queryTagAutocompleteKeys(TagType tagType, int limit, long startSecondTB, long endSecondTB) throws IOException { TimestampRange range = null; if (startSecondTB > 0 && endSecondTB > 0) { range = new TimestampRange(TimeBucket.getTimestamp(startSecondTB), TimeBucket.getTimestamp(endSecondTB)); @@ -59,6 +59,7 @@ public class BanyanDBTagAutocompleteQueryDAO extends AbstractBanyanDBDAO impleme @Override protected void apply(MeasureQuery query) { query.groupBy(ImmutableSet.of(TagAutocompleteData.TAG_KEY)); + query.setLimit(limit); query.and(eq(TagAutocompleteData.TAG_TYPE, tagType.name())); } } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java index f9a90dea83..b9872cac41 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java @@ -47,13 +47,15 @@ public class TagAutoCompleteQueryDAO extends EsDAO implements ITagAutoCompleteQu @Override public Set queryTagAutocompleteKeys(final TagType tagType, + final int limit, final long startSecondTB, final long endSecondTB) throws IOException { BoolQueryBuilder query = Query.bool(); query.must(Query.term(TagAutocompleteData.TAG_TYPE, tagType.name())); final SearchBuilder search = Search.builder().query(query); search.aggregation(Aggregation.terms(TagAutocompleteData.TAG_KEY) - .field(TagAutocompleteData.TAG_KEY)); + .field(TagAutocompleteData.TAG_KEY) + .size(limit)); final SearchResponse response = getClient().search( new TimeRangeIndexNameGenerator( diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TagAutoCompleteQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TagAutoCompleteQueryDAO.java index f41f31d34f..b73414c71d 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TagAutoCompleteQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TagAutoCompleteQueryDAO.java @@ -38,6 +38,7 @@ public class H2TagAutoCompleteQueryDAO implements ITagAutoCompleteQueryDAO { @Override public Set queryTagAutocompleteKeys(final TagType tagType, + final int limit, final long startSecondTB, final long endSecondTB) throws IOException { StringBuilder sql = new StringBuilder(); @@ -47,6 +48,7 @@ public class H2TagAutoCompleteQueryDAO implements ITagAutoCompleteQueryDAO { .append(TagAutocompleteData.INDEX_NAME).append(" where "); sql.append(" 1=1 "); appendTagAutocompleteCondition(tagType, startSecondTB, endSecondTB, sql, condition); + sql.append(" limit ").append(limit); try (Connection connection = h2Client.getConnection()) { ResultSet resultSet = h2Client.executeQuery(connection, sql.toString(), condition.toArray(new Object[0])); Set tagKeys = new HashSet<>();