From 15b0068fdf8df2c610abc58dd763679ca1a7f22b Mon Sep 17 00:00:00 2001 From: Evan <31562192+EvanLjp@users.noreply.github.com> Date: Tue, 1 Sep 2020 23:06:21 +0800 Subject: [PATCH] Add superdataset replicas number config (#5424) * distinct the number of metrics or common record index with super dataset index. --- docs/en/setup/backend/backend-storage.md | 11 +-- .../setup/backend/configuration-vocabulary.md | 11 ++- .../src/main/resources/application.yml | 20 +++-- .../StorageModuleElasticsearchConfig.java | 82 +++++++------------ .../base/StorageEsInstaller.java | 4 +- 5 files changed, 59 insertions(+), 69 deletions(-) diff --git a/docs/en/setup/backend/backend-storage.md b/docs/en/setup/backend/backend-storage.md index bd48a565c..732c90bb2 100644 --- a/docs/en/setup/backend/backend-storage.md +++ b/docs/en/setup/backend/backend-storage.md @@ -53,15 +53,16 @@ storage: protocol: ${SW_STORAGE_ES_HTTP_PROTOCOL:"http"} trustStorePath: ${SW_STORAGE_ES_SSL_JKS_PATH:""} trustStorePass: ${SW_STORAGE_ES_SSL_JKS_PASS:""} - dayStep: ${SW_STORAGE_DAY_STEP:1} # Represent the number of days in the one minute/hour/day index. - superDatasetDayStep: ${SW_SUPERDATASET_STORAGE_DAY_STEP:-1} # Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value lt 0 user: ${SW_ES_USER:""} password: ${SW_ES_PASSWORD:""} secretsManagementFile: ${SW_ES_SECRETS_MANAGEMENT_FILE:""} # Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool. + dayStep: ${SW_STORAGE_DAY_STEP:1} # Represent the number of days in the one minute/hour/day index. indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:1} # Shard number of new indexes - superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # Super data set has been defined in the codes, such as trace segments. This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces. - indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:0} - # Batch process setting, refer to https://www.elastic.co/guide/en/elasticsearch/client/java-api/5.5/java-docs-bulk-processor.html + indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:1} # Replicas number of new indexes + # Super data set has been defined in the codes, such as trace segments.The following 3 config would be improve es performance when storage super size data in es. + superDatasetDayStep: ${SW_SUPERDATASET_STORAGE_DAY_STEP:-1} # Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0 + superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces. + superDatasetIndexReplicasNumber: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_REPLICAS_NUMBER:0} # Represent the replicas number in the super size dataset record index, the default value is 0. bulkActions: ${SW_STORAGE_ES_BULK_ACTIONS:1000} # Execute the bulk every 1000 requests flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md index 87df0b39e..5ae1f3c20 100644 --- a/docs/en/setup/backend/configuration-vocabulary.md +++ b/docs/en/setup/backend/configuration-vocabulary.md @@ -74,12 +74,13 @@ core|default|role|Option values, `Mixed/Receiver/Aggregator`. **Receiver** mode | - | - | password | Password of ElasticSearch cluster | SW_ES_PASSWORD | - | | - | - | trustStorePath | Trust JKS file path. Only work when user name and password opened | SW_STORAGE_ES_SSL_JKS_PATH | - | | - | - | trustStorePass | Trust JKS file password. Only work when user name and password opened | SW_STORAGE_ES_SSL_JKS_PASS | - | -| - | - | indexShardsNumber | Shard number of new indexes | SW_STORAGE_ES_INDEX_SHARDS_NUMBER | 1 | | - | - | secretsManagementFile| Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool. Provide the capability to update them in the runtime.|SW_ES_SECRETS_MANAGEMENT_FILE | - | | - | - | dayStep| Represent the number of days in the one minute/hour/day index.| SW_STORAGE_DAY_STEP | 1| | - | - | indexShardsNumber | Shard number of new indexes | SW_STORAGE_ES_INDEX_SHARDS_NUMBER | 1 | -| - | - | superDatasetIndexShardsFactor | Super data set has been defined in the codes, such as trace segments. This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces.|SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR|5 | | - | - | indexReplicasNumber | Replicas number of new indexes | SW_STORAGE_ES_INDEX_REPLICAS_NUMBER | 0 | +| - | - | superDatasetDayStep | Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0.|SW_SUPERDATASET_STORAGE_DAY_STEP|-1 | +| - | - | superDatasetIndexShardsFactor | Super data set has been defined in the codes, such as trace segments. This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces.|SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR|5 | +| - | - | superDatasetIndexReplicasNumber | Represent the replicas number in the super size dataset record index.|SW_STORAGE_ES_SUPER_DATASET_INDEX_REPLICAS_NUMBER|0 | | - | - | bulkActions| Bulk size of the batch execution. | SW_STORAGE_ES_BULK_ACTIONS| 1000| | - | - | flushInterval| Period of flush, no matter `bulkActions` reached or not. Unit is second.| SW_STORAGE_ES_FLUSH_INTERVAL | 10| | - | - | concurrentRequests| The number of concurrent requests allowed to be executed. | SW_STORAGE_ES_CONCURRENT_REQUESTS| 2 | @@ -99,11 +100,13 @@ core|default|role|Option values, `Mixed/Receiver/Aggregator`. **Receiver** mode | - | - | secretsManagementFile| Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool. Provide the capability to update them in the runtime.|SW_ES_SECRETS_MANAGEMENT_FILE | - | | - | - | dayStep| Represent the number of days in the one minute/hour/day index.| SW_STORAGE_DAY_STEP | 1| | - | - | indexShardsNumber | Shard number of new indexes | SW_STORAGE_ES_INDEX_SHARDS_NUMBER | 1 | -| - | - | superDatasetIndexShardsFactor | Super data set has been defined in the codes, such as trace segments. This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces.|SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR|5 | | - | - | indexReplicasNumber | Replicas number of new indexes | SW_STORAGE_ES_INDEX_REPLICAS_NUMBER | 0 | +| - | - | superDatasetDayStep | Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0.|SW_SUPERDATASET_STORAGE_DAY_STEP|-1 | +| - | - | superDatasetIndexShardsFactor | Super data set has been defined in the codes, such as trace segments. This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces.|SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR|5 | +| - | - | superDatasetIndexReplicasNumber | Represent the replicas number in the super size dataset record index.|SW_STORAGE_ES_SUPER_DATASET_INDEX_REPLICAS_NUMBER|0 | | - | - | bulkActions| Bulk size of the batch execution. | SW_STORAGE_ES_BULK_ACTIONS| 1000| | - | - | flushInterval| Period of flush, no matter `bulkActions` reached or not. Unit is second.| SW_STORAGE_ES_FLUSH_INTERVAL | 10| -| - | - | concurrentRequests| The number of concurrent requests allowed to be executed. |SW_STORAGE_ES_CONCURRENT_REQUESTS| 2 | +| - | - | concurrentRequests| The number of concurrent requests allowed to be executed. | SW_STORAGE_ES_CONCURRENT_REQUESTS| 2 | | - | - | resultWindowMaxSize | The max size of dataset when OAP loading cache, such as network alias. | SW_STORAGE_ES_QUERY_MAX_WINDOW_SIZE | 10000| | - | - | metadataQueryMaxSize | The max size of metadata per query. | SW_STORAGE_ES_QUERY_MAX_SIZE | 5000 | | - | - | segmentQueryMaxSize | The max size of trace segments per query. | SW_STORAGE_ES_QUERY_SEGMENT_SIZE | 200| diff --git a/oap-server/server-bootstrap/src/main/resources/application.yml b/oap-server/server-bootstrap/src/main/resources/application.yml index f35e3a2e3..f026062ed 100755 --- a/oap-server/server-bootstrap/src/main/resources/application.yml +++ b/oap-server/server-bootstrap/src/main/resources/application.yml @@ -101,16 +101,18 @@ storage: nameSpace: ${SW_NAMESPACE:""} clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:localhost:9200} protocol: ${SW_STORAGE_ES_HTTP_PROTOCOL:"http"} - trustStorePath: ${SW_STORAGE_ES_SSL_JKS_PATH:""} - trustStorePass: ${SW_STORAGE_ES_SSL_JKS_PASS:""} user: ${SW_ES_USER:""} password: ${SW_ES_PASSWORD:""} + trustStorePath: ${SW_STORAGE_ES_SSL_JKS_PATH:""} + trustStorePass: ${SW_STORAGE_ES_SSL_JKS_PASS:""} secretsManagementFile: ${SW_ES_SECRETS_MANAGEMENT_FILE:""} # Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool. dayStep: ${SW_STORAGE_DAY_STEP:1} # Represent the number of days in the one minute/hour/day index. - superDatasetDayStep: ${SW_SUPERDATASET_STORAGE_DAY_STEP:-1} # Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0 indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:1} # Shard number of new indexes - superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # Super data set has been defined in the codes, such as trace segments. This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces. - indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:0} + indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:1} # Replicas number of new indexes + # Super data set has been defined in the codes, such as trace segments.The following 3 config would be improve es performance when storage super size data in es. + superDatasetDayStep: ${SW_SUPERDATASET_STORAGE_DAY_STEP:-1} # Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0 + superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces. + superDatasetIndexReplicasNumber: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_REPLICAS_NUMBER:0} # Represent the replicas number in the super size dataset record index, the default value is 0. bulkActions: ${SW_STORAGE_ES_BULK_ACTIONS:1000} # Execute the bulk every 1000 requests flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests @@ -126,13 +128,15 @@ storage: trustStorePath: ${SW_STORAGE_ES_SSL_JKS_PATH:""} trustStorePass: ${SW_STORAGE_ES_SSL_JKS_PASS:""} dayStep: ${SW_STORAGE_DAY_STEP:1} # Represent the number of days in the one minute/hour/day index. + indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:1} # Shard number of new indexes + indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:1} # Replicas number of new indexes + # Super data set has been defined in the codes, such as trace segments.The following 3 config would be improve es performance when storage super size data in es. superDatasetDayStep: ${SW_SUPERDATASET_STORAGE_DAY_STEP:-1} # Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0 + superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces. + superDatasetIndexReplicasNumber: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_REPLICAS_NUMBER:0} # Represent the replicas number in the super size dataset record index, the default value is 0. user: ${SW_ES_USER:""} password: ${SW_ES_PASSWORD:""} secretsManagementFile: ${SW_ES_SECRETS_MANAGEMENT_FILE:""} # Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool. - indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:1} # Shard number of new indexes - superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # Super data set has been defined in the codes, such as trace segments. This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces. - indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:0} bulkActions: ${SW_STORAGE_ES_BULK_ACTIONS:1000} # Execute the bulk every 1000 requests flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java index 4fa466d65..02cfc2fa0 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java @@ -20,56 +20,15 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch; import lombok.Getter; import lombok.Setter; - +import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset; import org.apache.skywalking.oap.server.library.module.ModuleConfig; @Getter +@Setter public class StorageModuleElasticsearchConfig extends ModuleConfig { - @Setter private String nameSpace; - @Setter private String clusterNodes; - @Getter - @Setter String protocol = "http"; - @Setter - private int indexShardsNumber = 1; - @Setter - private int superDatasetIndexShardsFactor = 5; - @Setter - private int indexReplicasNumber = 0; - @Setter - private int indexRefreshInterval = 2; - @Setter - private int bulkActions = 2000; - @Setter - private int flushInterval = 10; - @Setter - private int concurrentRequests = 2; - /** - * @since 7.0.0 This could be managed inside {@link #secretsManagementFile} - */ - @Setter - private String user; - /** - * @since 7.0.0 This could be managed inside {@link #secretsManagementFile} - */ - @Setter - private String password; - /** - * Secrets management file includes the username, password, which are managed by 3rd party tool. - */ - @Getter - private String secretsManagementFile; - @Getter - @Setter - private String trustStorePath; - /** - * @since 7.0.0 This could be managed inside {@link #secretsManagementFile} - */ - @Getter - @Setter - private String trustStorePass; /** * Since 6.4.0, the index of metrics and traces data in minute/hour/month precision are organized in days. ES * storage creates new indexes in every day. @@ -78,22 +37,43 @@ public class StorageModuleElasticsearchConfig extends ModuleConfig { * with previous versions. But if there isn't much traffic for single one day, user could set the step larger to * reduce the number of indexes, and keep the TTL longer. */ - @Getter private int dayStep = 1; + private int indexReplicasNumber = 0; + private int indexShardsNumber = 1; /** * @since 8.2.0, the record day step is for super size dataset record index rolling when the value of it is greater than 0 */ - @Getter private int superDatasetDayStep = -1; - @Setter + /** + * @see SuperDataset + * @since 8.2.0, the replicas number is for super size dataset record replicas number + */ + private int superDatasetIndexReplicasNumber = 0; + private int superDatasetIndexShardsFactor = 5; + private int indexRefreshInterval = 2; + private int bulkActions = 2000; + private int flushInterval = 10; + private int concurrentRequests = 2; + /** + * @since 7.0.0 This could be managed inside {@link #secretsManagementFile} + */ + private String user; + /** + * @since 7.0.0 This could be managed inside {@link #secretsManagementFile} + */ + private String password; + /** + * Secrets management file includes the username, password, which are managed by 3rd party tool. + */ + private String secretsManagementFile; + private String trustStorePath; + /** + * @since 7.0.0 This could be managed inside {@link #secretsManagementFile} + */ + private String trustStorePass; private int resultWindowMaxSize = 10000; - @Setter private int metadataQueryMaxSize = 5000; - @Setter private int segmentQueryMaxSize = 200; - @Setter private int profileTaskQueryMaxSize = 200; - @Setter private String advanced; - } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java index 9dc331b8e..43358fc5c 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java @@ -104,7 +104,9 @@ public class StorageEsInstaller extends ModelInstaller { protected Map createSetting(Model model) { Map setting = new HashMap<>(); - setting.put("index.number_of_replicas", config.getIndexReplicasNumber()); + setting.put("index.number_of_replicas", model.isSuperDataset() + ? config.getSuperDatasetIndexReplicasNumber() + : config.getIndexReplicasNumber()); setting.put("index.number_of_shards", model.isSuperDataset() ? config.getIndexShardsNumber() * config.getSuperDatasetIndexShardsFactor() : config.getIndexShardsNumber());