Add superdataset replicas number config (#5424)
* distinct the number of metrics or common record index with super dataset index.
This commit is contained in:
Evan
GitHub
parent
246ea6b38b
commit
15b0068fdf
|
|
@ -53,15 +53,16 @@ storage:
|
|||
protocol: ${SW_STORAGE_ES_HTTP_PROTOCOL:"http"}
|
||||
trustStorePath: ${SW_STORAGE_ES_SSL_JKS_PATH:""}
|
||||
trustStorePass: ${SW_STORAGE_ES_SSL_JKS_PASS:""}
|
||||
dayStep: ${SW_STORAGE_DAY_STEP:1} # Represent the number of days in the one minute/hour/day index.
|
||||
superDatasetDayStep: ${SW_SUPERDATASET_STORAGE_DAY_STEP:-1} # Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value lt 0
|
||||
user: ${SW_ES_USER:""}
|
||||
password: ${SW_ES_PASSWORD:""}
|
||||
secretsManagementFile: ${SW_ES_SECRETS_MANAGEMENT_FILE:""} # Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool.
|
||||
dayStep: ${SW_STORAGE_DAY_STEP:1} # Represent the number of days in the one minute/hour/day index.
|
||||
indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:1} # Shard number of new indexes
|
||||
superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # Super data set has been defined in the codes, such as trace segments. This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces.
|
||||
indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:0}
|
||||
# Batch process setting, refer to https://www.elastic.co/guide/en/elasticsearch/client/java-api/5.5/java-docs-bulk-processor.html
|
||||
indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:1} # Replicas number of new indexes
|
||||
# Super data set has been defined in the codes, such as trace segments.The following 3 config would be improve es performance when storage super size data in es.
|
||||
superDatasetDayStep: ${SW_SUPERDATASET_STORAGE_DAY_STEP:-1} # Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0
|
||||
superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces.
|
||||
superDatasetIndexReplicasNumber: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_REPLICAS_NUMBER:0} # Represent the replicas number in the super size dataset record index, the default value is 0.
|
||||
bulkActions: ${SW_STORAGE_ES_BULK_ACTIONS:1000} # Execute the bulk every 1000 requests
|
||||
flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests
|
||||
concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests
|
||||
|
|
|
|||
|
|
@ -74,12 +74,13 @@ core|default|role|Option values, `Mixed/Receiver/Aggregator`. **Receiver** mode
|
|||
| - | - | password | Password of ElasticSearch cluster | SW_ES_PASSWORD | - |
|
||||
| - | - | trustStorePath | Trust JKS file path. Only work when user name and password opened | SW_STORAGE_ES_SSL_JKS_PATH | - |
|
||||
| - | - | trustStorePass | Trust JKS file password. Only work when user name and password opened | SW_STORAGE_ES_SSL_JKS_PASS | - |
|
||||
| - | - | indexShardsNumber | Shard number of new indexes | SW_STORAGE_ES_INDEX_SHARDS_NUMBER | 1 |
|
||||
| - | - | secretsManagementFile| Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool. Provide the capability to update them in the runtime.|SW_ES_SECRETS_MANAGEMENT_FILE | - |
|
||||
| - | - | dayStep| Represent the number of days in the one minute/hour/day index.| SW_STORAGE_DAY_STEP | 1|
|
||||
| - | - | indexShardsNumber | Shard number of new indexes | SW_STORAGE_ES_INDEX_SHARDS_NUMBER | 1 |
|
||||
| - | - | superDatasetIndexShardsFactor | Super data set has been defined in the codes, such as trace segments. This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces.|SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR|5 |
|
||||
| - | - | indexReplicasNumber | Replicas number of new indexes | SW_STORAGE_ES_INDEX_REPLICAS_NUMBER | 0 |
|
||||
| - | - | superDatasetDayStep | Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0.|SW_SUPERDATASET_STORAGE_DAY_STEP|-1 |
|
||||
| - | - | superDatasetIndexShardsFactor | Super data set has been defined in the codes, such as trace segments. This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces.|SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR|5 |
|
||||
| - | - | superDatasetIndexReplicasNumber | Represent the replicas number in the super size dataset record index.|SW_STORAGE_ES_SUPER_DATASET_INDEX_REPLICAS_NUMBER|0 |
|
||||
| - | - | bulkActions| Bulk size of the batch execution. | SW_STORAGE_ES_BULK_ACTIONS| 1000|
|
||||
| - | - | flushInterval| Period of flush, no matter `bulkActions` reached or not. Unit is second.| SW_STORAGE_ES_FLUSH_INTERVAL | 10|
|
||||
| - | - | concurrentRequests| The number of concurrent requests allowed to be executed. | SW_STORAGE_ES_CONCURRENT_REQUESTS| 2 |
|
||||
|
|
@ -99,11 +100,13 @@ core|default|role|Option values, `Mixed/Receiver/Aggregator`. **Receiver** mode
|
|||
| - | - | secretsManagementFile| Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool. Provide the capability to update them in the runtime.|SW_ES_SECRETS_MANAGEMENT_FILE | - |
|
||||
| - | - | dayStep| Represent the number of days in the one minute/hour/day index.| SW_STORAGE_DAY_STEP | 1|
|
||||
| - | - | indexShardsNumber | Shard number of new indexes | SW_STORAGE_ES_INDEX_SHARDS_NUMBER | 1 |
|
||||
| - | - | superDatasetIndexShardsFactor | Super data set has been defined in the codes, such as trace segments. This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces.|SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR|5 |
|
||||
| - | - | indexReplicasNumber | Replicas number of new indexes | SW_STORAGE_ES_INDEX_REPLICAS_NUMBER | 0 |
|
||||
| - | - | superDatasetDayStep | Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0.|SW_SUPERDATASET_STORAGE_DAY_STEP|-1 |
|
||||
| - | - | superDatasetIndexShardsFactor | Super data set has been defined in the codes, such as trace segments. This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces.|SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR|5 |
|
||||
| - | - | superDatasetIndexReplicasNumber | Represent the replicas number in the super size dataset record index.|SW_STORAGE_ES_SUPER_DATASET_INDEX_REPLICAS_NUMBER|0 |
|
||||
| - | - | bulkActions| Bulk size of the batch execution. | SW_STORAGE_ES_BULK_ACTIONS| 1000|
|
||||
| - | - | flushInterval| Period of flush, no matter `bulkActions` reached or not. Unit is second.| SW_STORAGE_ES_FLUSH_INTERVAL | 10|
|
||||
| - | - | concurrentRequests| The number of concurrent requests allowed to be executed. |SW_STORAGE_ES_CONCURRENT_REQUESTS| 2 |
|
||||
| - | - | concurrentRequests| The number of concurrent requests allowed to be executed. | SW_STORAGE_ES_CONCURRENT_REQUESTS| 2 |
|
||||
| - | - | resultWindowMaxSize | The max size of dataset when OAP loading cache, such as network alias. | SW_STORAGE_ES_QUERY_MAX_WINDOW_SIZE | 10000|
|
||||
| - | - | metadataQueryMaxSize | The max size of metadata per query. | SW_STORAGE_ES_QUERY_MAX_SIZE | 5000 |
|
||||
| - | - | segmentQueryMaxSize | The max size of trace segments per query. | SW_STORAGE_ES_QUERY_SEGMENT_SIZE | 200|
|
||||
|
|
|
|||
|
|
@ -101,16 +101,18 @@ storage:
|
|||
nameSpace: ${SW_NAMESPACE:""}
|
||||
clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:localhost:9200}
|
||||
protocol: ${SW_STORAGE_ES_HTTP_PROTOCOL:"http"}
|
||||
trustStorePath: ${SW_STORAGE_ES_SSL_JKS_PATH:""}
|
||||
trustStorePass: ${SW_STORAGE_ES_SSL_JKS_PASS:""}
|
||||
user: ${SW_ES_USER:""}
|
||||
password: ${SW_ES_PASSWORD:""}
|
||||
trustStorePath: ${SW_STORAGE_ES_SSL_JKS_PATH:""}
|
||||
trustStorePass: ${SW_STORAGE_ES_SSL_JKS_PASS:""}
|
||||
secretsManagementFile: ${SW_ES_SECRETS_MANAGEMENT_FILE:""} # Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool.
|
||||
dayStep: ${SW_STORAGE_DAY_STEP:1} # Represent the number of days in the one minute/hour/day index.
|
||||
superDatasetDayStep: ${SW_SUPERDATASET_STORAGE_DAY_STEP:-1} # Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0
|
||||
indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:1} # Shard number of new indexes
|
||||
superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # Super data set has been defined in the codes, such as trace segments. This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces.
|
||||
indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:0}
|
||||
indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:1} # Replicas number of new indexes
|
||||
# Super data set has been defined in the codes, such as trace segments.The following 3 config would be improve es performance when storage super size data in es.
|
||||
superDatasetDayStep: ${SW_SUPERDATASET_STORAGE_DAY_STEP:-1} # Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0
|
||||
superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces.
|
||||
superDatasetIndexReplicasNumber: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_REPLICAS_NUMBER:0} # Represent the replicas number in the super size dataset record index, the default value is 0.
|
||||
bulkActions: ${SW_STORAGE_ES_BULK_ACTIONS:1000} # Execute the bulk every 1000 requests
|
||||
flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests
|
||||
concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests
|
||||
|
|
@ -126,13 +128,15 @@ storage:
|
|||
trustStorePath: ${SW_STORAGE_ES_SSL_JKS_PATH:""}
|
||||
trustStorePass: ${SW_STORAGE_ES_SSL_JKS_PASS:""}
|
||||
dayStep: ${SW_STORAGE_DAY_STEP:1} # Represent the number of days in the one minute/hour/day index.
|
||||
indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:1} # Shard number of new indexes
|
||||
indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:1} # Replicas number of new indexes
|
||||
# Super data set has been defined in the codes, such as trace segments.The following 3 config would be improve es performance when storage super size data in es.
|
||||
superDatasetDayStep: ${SW_SUPERDATASET_STORAGE_DAY_STEP:-1} # Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0
|
||||
superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces.
|
||||
superDatasetIndexReplicasNumber: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_REPLICAS_NUMBER:0} # Represent the replicas number in the super size dataset record index, the default value is 0.
|
||||
user: ${SW_ES_USER:""}
|
||||
password: ${SW_ES_PASSWORD:""}
|
||||
secretsManagementFile: ${SW_ES_SECRETS_MANAGEMENT_FILE:""} # Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool.
|
||||
indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:1} # Shard number of new indexes
|
||||
superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # Super data set has been defined in the codes, such as trace segments. This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces.
|
||||
indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:0}
|
||||
bulkActions: ${SW_STORAGE_ES_BULK_ACTIONS:1000} # Execute the bulk every 1000 requests
|
||||
flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests
|
||||
concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests
|
||||
|
|
|
|||
|
|
@ -20,56 +20,15 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch;
|
|||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset;
|
||||
import org.apache.skywalking.oap.server.library.module.ModuleConfig;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public class StorageModuleElasticsearchConfig extends ModuleConfig {
|
||||
@Setter
|
||||
private String nameSpace;
|
||||
@Setter
|
||||
private String clusterNodes;
|
||||
@Getter
|
||||
@Setter
|
||||
String protocol = "http";
|
||||
@Setter
|
||||
private int indexShardsNumber = 1;
|
||||
@Setter
|
||||
private int superDatasetIndexShardsFactor = 5;
|
||||
@Setter
|
||||
private int indexReplicasNumber = 0;
|
||||
@Setter
|
||||
private int indexRefreshInterval = 2;
|
||||
@Setter
|
||||
private int bulkActions = 2000;
|
||||
@Setter
|
||||
private int flushInterval = 10;
|
||||
@Setter
|
||||
private int concurrentRequests = 2;
|
||||
/**
|
||||
* @since 7.0.0 This could be managed inside {@link #secretsManagementFile}
|
||||
*/
|
||||
@Setter
|
||||
private String user;
|
||||
/**
|
||||
* @since 7.0.0 This could be managed inside {@link #secretsManagementFile}
|
||||
*/
|
||||
@Setter
|
||||
private String password;
|
||||
/**
|
||||
* Secrets management file includes the username, password, which are managed by 3rd party tool.
|
||||
*/
|
||||
@Getter
|
||||
private String secretsManagementFile;
|
||||
@Getter
|
||||
@Setter
|
||||
private String trustStorePath;
|
||||
/**
|
||||
* @since 7.0.0 This could be managed inside {@link #secretsManagementFile}
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
private String trustStorePass;
|
||||
/**
|
||||
* Since 6.4.0, the index of metrics and traces data in minute/hour/month precision are organized in days. ES
|
||||
* storage creates new indexes in every day.
|
||||
|
|
@ -78,22 +37,43 @@ public class StorageModuleElasticsearchConfig extends ModuleConfig {
|
|||
* with previous versions. But if there isn't much traffic for single one day, user could set the step larger to
|
||||
* reduce the number of indexes, and keep the TTL longer.
|
||||
*/
|
||||
@Getter
|
||||
private int dayStep = 1;
|
||||
private int indexReplicasNumber = 0;
|
||||
private int indexShardsNumber = 1;
|
||||
/**
|
||||
* @since 8.2.0, the record day step is for super size dataset record index rolling when the value of it is greater than 0
|
||||
*/
|
||||
@Getter
|
||||
private int superDatasetDayStep = -1;
|
||||
@Setter
|
||||
/**
|
||||
* @see SuperDataset
|
||||
* @since 8.2.0, the replicas number is for super size dataset record replicas number
|
||||
*/
|
||||
private int superDatasetIndexReplicasNumber = 0;
|
||||
private int superDatasetIndexShardsFactor = 5;
|
||||
private int indexRefreshInterval = 2;
|
||||
private int bulkActions = 2000;
|
||||
private int flushInterval = 10;
|
||||
private int concurrentRequests = 2;
|
||||
/**
|
||||
* @since 7.0.0 This could be managed inside {@link #secretsManagementFile}
|
||||
*/
|
||||
private String user;
|
||||
/**
|
||||
* @since 7.0.0 This could be managed inside {@link #secretsManagementFile}
|
||||
*/
|
||||
private String password;
|
||||
/**
|
||||
* Secrets management file includes the username, password, which are managed by 3rd party tool.
|
||||
*/
|
||||
private String secretsManagementFile;
|
||||
private String trustStorePath;
|
||||
/**
|
||||
* @since 7.0.0 This could be managed inside {@link #secretsManagementFile}
|
||||
*/
|
||||
private String trustStorePass;
|
||||
private int resultWindowMaxSize = 10000;
|
||||
@Setter
|
||||
private int metadataQueryMaxSize = 5000;
|
||||
@Setter
|
||||
private int segmentQueryMaxSize = 200;
|
||||
@Setter
|
||||
private int profileTaskQueryMaxSize = 200;
|
||||
@Setter
|
||||
private String advanced;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -104,7 +104,9 @@ public class StorageEsInstaller extends ModelInstaller {
|
|||
protected Map<String, Object> createSetting(Model model) {
|
||||
Map<String, Object> setting = new HashMap<>();
|
||||
|
||||
setting.put("index.number_of_replicas", config.getIndexReplicasNumber());
|
||||
setting.put("index.number_of_replicas", model.isSuperDataset()
|
||||
? config.getSuperDatasetIndexReplicasNumber()
|
||||
: config.getIndexReplicasNumber());
|
||||
setting.put("index.number_of_shards", model.isSuperDataset()
|
||||
? config.getIndexShardsNumber() * config.getSuperDatasetIndexShardsFactor()
|
||||
: config.getIndexShardsNumber());
|
||||
|
|
|
|||
Loading…
Reference in New Issue