From ffc69dc6f369730819a9a1420ccdb69e369d6d28 Mon Sep 17 00:00:00 2001 From: pkxiuluo <616197526@qq.com> Date: Tue, 27 Jul 2021 20:13:31 +0800 Subject: [PATCH] ES storage support set order of index template. (#7378) --- CHANGES.md | 2 +- docs/en/setup/backend/backend-storage.md | 1 + docs/en/setup/backend/configuration-vocabulary.md | 2 ++ .../server-bootstrap/src/main/resources/application.yml | 3 +++ .../library/client/elasticsearch/ElasticSearchClient.java | 3 ++- .../library/client/elasticsearch/ITElasticSearchClient.java | 4 ++-- .../elasticsearch/StorageModuleElasticsearchConfig.java | 6 ++++++ .../plugin/elasticsearch/base/StorageEsInstaller.java | 2 +- .../plugin/elasticsearch7/client/ElasticSearch7Client.java | 5 +++-- 9 files changed, 21 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2296942cf..ee30a04da 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -84,7 +84,7 @@ Release Notes. * Fix InfluxDB storage-plugin Metrics#multiGet issue. * Replace zuul proxy with spring cloud gateway 2.x. in webapp module. * Upgrade etcd cluster coordinator and dynamic configuration to v3.x. -* Configuration: Allow configuring server maximum request header size. +* Configuration: Allow configuring server maximum request header size and ES index template order. * Add thread state metric and class loaded info metric to JVMMetric. * Performance: compile LAL DSL statically and run with type checked. * Add pagination to event query protocol. diff --git a/docs/en/setup/backend/backend-storage.md b/docs/en/setup/backend/backend-storage.md index 4be32ca0f..9f56f9ccc 100644 --- a/docs/en/setup/backend/backend-storage.md +++ b/docs/en/setup/backend/backend-storage.md @@ -73,6 +73,7 @@ storage: superDatasetDayStep: ${SW_SUPERDATASET_STORAGE_DAY_STEP:-1} # Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0 superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces. superDatasetIndexReplicasNumber: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_REPLICAS_NUMBER:0} # Represent the replicas number in the super size dataset record index, the default value is 0. + indexTemplateOrder: ${SW_STORAGE_ES_INDEX_TEMPLATE_ORDER:0} # the order of index template bulkActions: ${SW_STORAGE_ES_BULK_ACTIONS:1000} # Execute the async bulk record data every ${SW_STORAGE_ES_BULK_ACTIONS} requests flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md index 0eb412843..e671774a5 100644 --- a/docs/en/setup/backend/configuration-vocabulary.md +++ b/docs/en/setup/backend/configuration-vocabulary.md @@ -96,6 +96,7 @@ core|default|role|Option values, `Mixed/Receiver/Aggregator`. **Receiver** mode | - | - | superDatasetDayStep | Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0.|SW_SUPERDATASET_STORAGE_DAY_STEP|-1 | | - | - | superDatasetIndexShardsFactor | Super data set has been defined in the codes, such as trace segments. This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces.|SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR|5 | | - | - | superDatasetIndexReplicasNumber | Represent the replicas number in the super size dataset record index.|SW_STORAGE_ES_SUPER_DATASET_INDEX_REPLICAS_NUMBER|0 | +| - | - | indexTemplateOrder| The order of index template. | SW_STORAGE_ES_INDEX_TEMPLATE_ORDER| 0| | - | - | bulkActions| Async bulk size of the record data batch execution. | SW_STORAGE_ES_BULK_ACTIONS| 5000| | - | - | flushInterval| Period of flush, no matter `bulkActions` reached or not. Unit is second. INT(flushInterval * 2/3) would be used for index refresh period.| SW_STORAGE_ES_FLUSH_INTERVAL | 15 (index refresh period = 10)| | - | - | concurrentRequests| The number of concurrent requests allowed to be executed. | SW_STORAGE_ES_CONCURRENT_REQUESTS| 2 | @@ -121,6 +122,7 @@ core|default|role|Option values, `Mixed/Receiver/Aggregator`. **Receiver** mode | - | - | superDatasetDayStep | Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0.|SW_SUPERDATASET_STORAGE_DAY_STEP|-1 | | - | - | superDatasetIndexShardsFactor | Super data set has been defined in the codes, such as trace segments. This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces.|SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR|5 | | - | - | superDatasetIndexReplicasNumber | Represent the replicas number in the super size dataset record index.|SW_STORAGE_ES_SUPER_DATASET_INDEX_REPLICAS_NUMBER|0 | +| - | - | indexTemplateOrder| The order of index template. | SW_STORAGE_ES_INDEX_TEMPLATE_ORDER| 0| | - | - | bulkActions| Async bulk size of data batch execution. | SW_STORAGE_ES_BULK_ACTIONS| 5000| | - | - | flushInterval| Period of flush, no matter `bulkActions` reached or not. Unit is second. INT(flushInterval * 2/3) would be used for index refresh period.| SW_STORAGE_ES_FLUSH_INTERVAL | 15 (index refresh period = 10)| | - | - | concurrentRequests| The number of concurrent requests allowed to be executed. | SW_STORAGE_ES_CONCURRENT_REQUESTS| 2 | diff --git a/oap-server/server-bootstrap/src/main/resources/application.yml b/oap-server/server-bootstrap/src/main/resources/application.yml index d79a40cc6..3cb87b326 100755 --- a/oap-server/server-bootstrap/src/main/resources/application.yml +++ b/oap-server/server-bootstrap/src/main/resources/application.yml @@ -137,6 +137,7 @@ storage: superDatasetDayStep: ${SW_SUPERDATASET_STORAGE_DAY_STEP:-1} # Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0 superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces. superDatasetIndexReplicasNumber: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_REPLICAS_NUMBER:0} # Represent the replicas number in the super size dataset record index, the default value is 0. + indexTemplateOrder: ${SW_STORAGE_ES_INDEX_TEMPLATE_ORDER:0} # the order of index template bulkActions: ${SW_STORAGE_ES_BULK_ACTIONS:5000} # Execute the async bulk record data every ${SW_STORAGE_ES_BULK_ACTIONS} requests # flush the bulk every 10 seconds whatever the number of requests # INT(flushInterval * 2/3) would be used for index refresh period. @@ -164,6 +165,7 @@ storage: superDatasetDayStep: ${SW_SUPERDATASET_STORAGE_DAY_STEP:-1} # Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0 superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces. superDatasetIndexReplicasNumber: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_REPLICAS_NUMBER:0} # Represent the replicas number in the super size dataset record index, the default value is 0. + indexTemplateOrder: ${SW_STORAGE_ES_INDEX_TEMPLATE_ORDER:0} # the order of index template user: ${SW_ES_USER:""} password: ${SW_ES_PASSWORD:""} secretsManagementFile: ${SW_ES_SECRETS_MANAGEMENT_FILE:""} # Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool. @@ -179,6 +181,7 @@ storage: oapAnalyzer: ${SW_STORAGE_ES_OAP_ANALYZER:"{\"analyzer\":{\"oap_analyzer\":{\"type\":\"stop\"}}}"} # the oap analyzer. oapLogAnalyzer: ${SW_STORAGE_ES_OAP_LOG_ANALYZER:"{\"analyzer\":{\"oap_log_analyzer\":{\"type\":\"standard\"}}}"} # the oap log analyzer. It could be customized by the ES analyzer configuration to support more language log formats, such as Chinese log, Japanese log and etc. advanced: ${SW_STORAGE_ES_ADVANCED:""} + h2: driver: ${SW_STORAGE_H2_DRIVER:org.h2.jdbcx.JdbcDataSource} url: ${SW_STORAGE_H2_URL:jdbc:h2:mem:skywalking-oap-db;DB_CLOSE_DELAY=-1} diff --git a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java index 4bc5bab48..38e3af680 100644 --- a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java +++ b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java @@ -398,7 +398,7 @@ public class ElasticSearchClient implements Client, HealthCheckable { } public boolean createOrUpdateTemplate(String indexName, Map settings, - Map mapping) throws IOException { + Map mapping, int order) throws IOException { indexName = formatIndexName(indexName); String[] patterns = new String[] {indexName + "-*"}; @@ -411,6 +411,7 @@ public class ElasticSearchClient implements Client, HealthCheckable { template.put("aliases", aliases); template.put("settings", settings); template.put("mappings", mapping); + template.put("order", order); HttpEntity entity = new NStringEntity(new Gson().toJson(template), ContentType.APPLICATION_JSON); diff --git a/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ITElasticSearchClient.java b/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ITElasticSearchClient.java index 44d97e8bb..ae9a9a254 100644 --- a/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ITElasticSearchClient.java +++ b/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ITElasticSearchClient.java @@ -176,7 +176,7 @@ public class ITElasticSearchClient { String indexName = "template_operate"; - client.createOrUpdateTemplate(indexName, settings, mapping); + client.createOrUpdateTemplate(indexName, settings, mapping, 0); Assert.assertTrue(client.isExistsTemplate(indexName)); @@ -233,7 +233,7 @@ public class ITElasticSearchClient { column.addProperty("type", "text"); properties.add("name", column); - client.createOrUpdateTemplate(indexName, new HashMap<>(), mapping); + client.createOrUpdateTemplate(indexName, new HashMap<>(), mapping, 0); XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field("name", "pengys").endObject(); client.forceInsert(timeSeriesIndexName, "testid", builder); diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java index 62411f8b2..069436495 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java @@ -64,6 +64,12 @@ public class StorageModuleElasticsearchConfig extends ModuleConfig { private int superDatasetIndexReplicasNumber = 0; private int superDatasetIndexShardsFactor = 5; private int indexRefreshInterval = 2; + + /** + * @since 8.7.0 The order of index template. + */ + private int indexTemplateOrder = 0; + /** * @since 8.7.0 This setting affects all traces/logs/metrics/metadata flush policy. */ diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java index 7fd9f22b4..442c0759d 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java @@ -121,7 +121,7 @@ public class StorageEsInstaller extends ModelInstaller { if (shouldUpdateTemplate) { structures.putStructure(tableName, mapping); boolean isAcknowledged = esClient.createOrUpdateTemplate( - tableName, settings, structures.getMapping(tableName)); + tableName, settings, structures.getMapping(tableName), config.getIndexTemplateOrder()); log.info("create {} index template finished, isAcknowledged: {}", tableName, isAcknowledged); if (!isAcknowledged) { throw new IOException("create " + tableName + " index template failure, "); diff --git a/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/client/ElasticSearch7Client.java b/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/client/ElasticSearch7Client.java index 4129a8815..f8b68cae8 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/client/ElasticSearch7Client.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/client/ElasticSearch7Client.java @@ -264,7 +264,7 @@ public class ElasticSearch7Client extends ElasticSearchClient { @Override public boolean createOrUpdateTemplate(String indexName, Map settings, - Map mapping) throws IOException { + Map mapping, int order) throws IOException { indexName = formatIndexName(indexName); PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest(indexName).patterns( @@ -272,7 +272,8 @@ public class ElasticSearch7Client extends ElasticSearchClient { .alias(new Alias( indexName)) .settings(settings) - .mapping(mapping); + .mapping(mapping) + .order(order); AcknowledgedResponse acknowledgedResponse = client.indices() .putTemplate(putIndexTemplateRequest, RequestOptions.DEFAULT);