From e697385b5380c775b02ab46d6d5cefe2c2775c18 Mon Sep 17 00:00:00 2001 From: "Jared.Tan" Date: Fri, 23 Nov 2018 14:52:37 +0800 Subject: [PATCH] feat:support set config from system.envs. (#1919) * support set config from system.envs. --- .../setup/backend/backend-setting-override.md | 36 ++- .../server-library/library-util/pom.xml | 16 +- .../util/PlaceholderConfigurerSupport.java | 34 +++ .../util/PropertyPlaceholderHelper.java | 208 ++++++++++++++++++ .../oap/server/library/util/StringUtils.java | 12 + .../util/PropertyPlaceholderHelperTest.java | 90 ++++++++ .../src/test/resources/application.yml | 82 +++++++ .../src/main/assembly/application.yml | 82 +++---- .../config/ApplicationConfigLoader.java | 24 +- .../src/main/resources/application.yml | 78 +++---- 10 files changed, 567 insertions(+), 95 deletions(-) create mode 100644 oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/PlaceholderConfigurerSupport.java create mode 100644 oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/PropertyPlaceholderHelper.java create mode 100644 oap-server/server-library/library-util/src/test/java/org/apache/skywalking/oap/server/library/util/PropertyPlaceholderHelperTest.java create mode 100644 oap-server/server-library/library-util/src/test/resources/application.yml diff --git a/docs/en/setup/backend/backend-setting-override.md b/docs/en/setup/backend/backend-setting-override.md index 7207cc379..8714aed54 100644 --- a/docs/en/setup/backend/backend-setting-override.md +++ b/docs/en/setup/backend/backend-setting-override.md @@ -1,13 +1,14 @@ # Setting Override -SkyWalking backend supports setting overrides by system properties. +SkyWalking backend supports setting overrides by system properties and system environment variables. You could override the settings in `application.yml` -- System properties key rule +## System properties key rule **ModuleName**.**ProviderName**.**SettingKey**. - Example Override `restHost` in this setting segment + ```yaml core: default: @@ -20,5 +21,32 @@ core: Use command arg ``` --core.default.restHost=172.0.4.12 -``` \ No newline at end of file +-Dcore.default.restHost=172.0.4.12 +``` + +## System environment variables +- Example + + Override `restHost` in this setting segment through environment variables + +```yaml +core: + default: + restHost: ${REST_HOST:0.0.0.0} + restPort: 12800 + restContextPath: / + gRPCHost: 0.0.0. + gRPCPort: 11800 +``` + +If the `REST_HOST ` environment variable exists in your operating system and its value is `172.0.4.12`, +then the value of `restHost` here will be overwritten to `172.0.4.12`, otherwise, it will be set to `0.0.0.0`. + +By the way, Placeholder nesting is also supported, like `${REST_HOST:${ANOTHER_REST_HOST:127.0.0.1}}`. +In this case, if the `REST_HOST ` environment variable not exists, but the ```REST_ANOTHER_REST_HOSTHOST``` +environment variable exists and its value is `172.0.4.12`, then the value of `restHost` here will be overwritten to `172.0.4.12`, +otherwise, it will be set to `127.0.0.1`. + + + + diff --git a/oap-server/server-library/library-util/pom.xml b/oap-server/server-library/library-util/pom.xml index c4a8d0d96..87ca944fb 100644 --- a/oap-server/server-library/library-util/pom.xml +++ b/oap-server/server-library/library-util/pom.xml @@ -17,7 +17,8 @@ ~ --> - + server-library org.apache.skywalking @@ -34,5 +35,16 @@ joda-time ${joda-time.version} + + org.yaml + snakeyaml + test + + + com.github.stefanbirkner + system-rules + 1.18.0 + test + - \ No newline at end of file + diff --git a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/PlaceholderConfigurerSupport.java b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/PlaceholderConfigurerSupport.java new file mode 100644 index 000000000..4ef02e95d --- /dev/null +++ b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/PlaceholderConfigurerSupport.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.util; + +/** + * @author jian.tan + */ +public class PlaceholderConfigurerSupport { + + /** Default placeholder prefix: {@value} */ + public static final String DEFAULT_PLACEHOLDER_PREFIX = "${"; + + /** Default placeholder suffix: {@value} */ + public static final String DEFAULT_PLACEHOLDER_SUFFIX = "}"; + + /** Default value separator: {@value} */ + public static final String DEFAULT_VALUE_SEPARATOR = ":"; +} diff --git a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/PropertyPlaceholderHelper.java b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/PropertyPlaceholderHelper.java new file mode 100644 index 000000000..60d6c4a17 --- /dev/null +++ b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/PropertyPlaceholderHelper.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.util; + +import com.google.common.base.Strings; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import org.apache.logging.log4j.core.util.Assert; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class for working with Strings that have placeholder values in them. A placeholder takes the form {@code + * ${name}}. Using {@code PropertyPlaceholderHelper} these placeholders can be substituted for user-supplied values.

+ * Values for substitution can be supplied using a {@link Properties} instance or using a {@link PlaceholderResolver}. + */ +public class PropertyPlaceholderHelper { + private static final Logger logger = LoggerFactory.getLogger(PropertyPlaceholderHelper.class); + + private static final Map WELL_KNOWN_SIMPLE_PREFIXES = new HashMap<>(4); + + static { + WELL_KNOWN_SIMPLE_PREFIXES.put("}", "{"); + WELL_KNOWN_SIMPLE_PREFIXES.put("]", "["); + WELL_KNOWN_SIMPLE_PREFIXES.put(")", "("); + } + + private final String placeholderPrefix; + + private final String placeholderSuffix; + + private final String simplePrefix; + + private final String valueSeparator; + + private final boolean ignoreUnresolvablePlaceholders; + + /** + * Creates a new {@code PropertyPlaceholderHelper} that uses the supplied prefix and suffix. + * + * @param placeholderPrefix the prefix that denotes the start of a placeholder + * @param placeholderSuffix the suffix that denotes the end of a placeholder + * @param valueSeparator the separating character between the placeholder variable and the associated default value, + * if any + * @param ignoreUnresolvablePlaceholders indicates whether unresolvable placeholders should be ignored ({@code + * true}) or cause an exception ({@code false}) + */ + public PropertyPlaceholderHelper(String placeholderPrefix, String placeholderSuffix, + String valueSeparator, boolean ignoreUnresolvablePlaceholders) { + Assert.requireNonEmpty(placeholderPrefix, "'placeholderPrefix' must not be null"); + Assert.requireNonEmpty(placeholderSuffix, "'placeholderSuffix' must not be null"); + this.placeholderPrefix = placeholderPrefix; + this.placeholderSuffix = placeholderSuffix; + String simplePrefixForSuffix = WELL_KNOWN_SIMPLE_PREFIXES.get(this.placeholderSuffix); + if (simplePrefixForSuffix != null && this.placeholderPrefix.endsWith(simplePrefixForSuffix)) { + this.simplePrefix = simplePrefixForSuffix; + } else { + this.simplePrefix = this.placeholderPrefix; + } + this.valueSeparator = valueSeparator; + this.ignoreUnresolvablePlaceholders = ignoreUnresolvablePlaceholders; + } + + /** + * Replaces all placeholders of format {@code ${name}} with the corresponding property from the supplied {@link + * Properties}. + * + * @param value the value containing the placeholders to be replaced + * @param properties the {@code Properties} to use for replacement + * @return the supplied value with placeholders replaced inline + */ + public String replacePlaceholders(String value, final Properties properties) { + Assert.requireNonEmpty(properties, "'properties' must not be null"); + return replacePlaceholders(value, placeholderName -> this.getConfigValue(placeholderName, properties)); + } + + private String getConfigValue(String key, final Properties properties) { + String value = System.getProperty(key); + if (Strings.isNullOrEmpty(value)) { + value = System.getenv(key); + } + if (Strings.isNullOrEmpty(value)) { + value = properties.getProperty(key); + } + return value; + } + + /** + * Replaces all placeholders of format {@code ${name}} with the value returned from the supplied {@link + * PlaceholderResolver}. + * + * @param value the value containing the placeholders to be replaced + * @param placeholderResolver the {@code PlaceholderResolver} to use for replacement + * @return the supplied value with placeholders replaced inline + */ + public String replacePlaceholders(String value, PlaceholderResolver placeholderResolver) { + Assert.requireNonEmpty(value, "'value' must not be null"); + return parseStringValue(value, placeholderResolver, new HashSet()); + } + + protected String parseStringValue(String value, PlaceholderResolver placeholderResolver, + Set visitedPlaceholders) { + + StringBuilder result = new StringBuilder(value); + + int startIndex = value.indexOf(this.placeholderPrefix); + while (startIndex != -1) { + int endIndex = findPlaceholderEndIndex(result, startIndex); + if (endIndex != -1) { + String placeholder = result.substring(startIndex + this.placeholderPrefix.length(), endIndex); + String originalPlaceholder = placeholder; + if (!visitedPlaceholders.add(originalPlaceholder)) { + throw new IllegalArgumentException( + "Circular placeholder reference '" + originalPlaceholder + "' in property definitions"); + } + // Recursive invocation, parsing placeholders contained in the placeholder key. + placeholder = parseStringValue(placeholder, placeholderResolver, visitedPlaceholders); + // Now obtain the value for the fully resolved key... + String propVal = placeholderResolver.resolvePlaceholder(placeholder); + if (propVal == null && this.valueSeparator != null) { + int separatorIndex = placeholder.indexOf(this.valueSeparator); + if (separatorIndex != -1) { + String actualPlaceholder = placeholder.substring(0, separatorIndex); + String defaultValue = placeholder.substring(separatorIndex + this.valueSeparator.length()); + propVal = placeholderResolver.resolvePlaceholder(actualPlaceholder); + if (propVal == null) { + propVal = defaultValue; + } + } + } + if (propVal != null) { + // Recursive invocation, parsing placeholders contained in the + // previously resolved placeholder value. + propVal = parseStringValue(propVal, placeholderResolver, visitedPlaceholders); + result.replace(startIndex, endIndex + this.placeholderSuffix.length(), propVal); + if (logger.isTraceEnabled()) { + logger.trace("Resolved placeholder '" + placeholder + "'"); + } + startIndex = result.indexOf(this.placeholderPrefix, startIndex + propVal.length()); + } else if (this.ignoreUnresolvablePlaceholders) { + // Proceed with unprocessed value. + startIndex = result.indexOf(this.placeholderPrefix, endIndex + this.placeholderSuffix.length()); + } else { + throw new IllegalArgumentException("Could not resolve placeholder '" + + placeholder + "'" + " in value \"" + value + "\""); + } + visitedPlaceholders.remove(originalPlaceholder); + } else { + startIndex = -1; + } + } + return result.toString(); + } + + private int findPlaceholderEndIndex(CharSequence buf, int startIndex) { + int index = startIndex + this.placeholderPrefix.length(); + int withinNestedPlaceholder = 0; + while (index < buf.length()) { + if (StringUtils.substringMatch(buf, index, this.placeholderSuffix)) { + if (withinNestedPlaceholder > 0) { + withinNestedPlaceholder--; + index = index + this.placeholderSuffix.length(); + } else { + return index; + } + } else if (StringUtils.substringMatch(buf, index, this.simplePrefix)) { + withinNestedPlaceholder++; + index = index + this.simplePrefix.length(); + } else { + index++; + } + } + return -1; + } + + /** + * Strategy interface used to resolve replacement values for placeholders contained in Strings. + */ + public interface PlaceholderResolver { + + /** + * Resolve the supplied placeholder name to the replacement value. + * + * @param placeholderName the name of the placeholder to resolve + * @return the replacement value, or {@code null} if no replacement is to be made + */ + String resolvePlaceholder(String placeholderName); + } +} diff --git a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/StringUtils.java b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/StringUtils.java index 2d425c46b..1375c8098 100644 --- a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/StringUtils.java +++ b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/StringUtils.java @@ -36,4 +36,16 @@ public class StringUtils { public static String getOrDefault(String value, String defaultValue) { return value == null ? defaultValue : value; } + + public static boolean substringMatch(CharSequence str, int index, CharSequence substring) { + if (index + substring.length() > str.length()) { + return false; + } + for (int i = 0; i < substring.length(); i++) { + if (str.charAt(index + i) != substring.charAt(i)) { + return false; + } + } + return true; + } } diff --git a/oap-server/server-library/library-util/src/test/java/org/apache/skywalking/oap/server/library/util/PropertyPlaceholderHelperTest.java b/oap-server/server-library/library-util/src/test/java/org/apache/skywalking/oap/server/library/util/PropertyPlaceholderHelperTest.java new file mode 100644 index 000000000..81fb6d7c7 --- /dev/null +++ b/oap-server/server-library/library-util/src/test/java/org/apache/skywalking/oap/server/library/util/PropertyPlaceholderHelperTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.util; + +import java.io.FileNotFoundException; +import java.io.Reader; +import java.util.Map; +import java.util.Properties; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.contrib.java.lang.system.EnvironmentVariables; +import org.yaml.snakeyaml.Yaml; + +/** + * @author jian.tan + */ +public class PropertyPlaceholderHelperTest { + private PropertyPlaceholderHelper placeholderHelper; + private Properties properties = new Properties(); + private final Yaml yaml = new Yaml(); + + /** + * The EnvironmentVariables rule allows you to set environment variables for your test. All changes to environment + * variables are reverted after the test. + */ + @Rule + public final EnvironmentVariables environmentVariables = new EnvironmentVariables().set("REST_PORT", "12801"); + + @SuppressWarnings("unchecked") + @Before + public void init() throws FileNotFoundException { + Reader applicationReader = ResourceUtils.read("application.yml"); + Map>> moduleConfig = yaml.loadAs(applicationReader, Map.class); + if (CollectionUtils.isNotEmpty(moduleConfig)) { + moduleConfig.forEach((moduleName, providerConfig) -> { + if (providerConfig.size() > 0) { + providerConfig.forEach((name, propertiesConfig) -> { + if (propertiesConfig != null) { + propertiesConfig.forEach((key, value) -> properties.put(key, value)); + } + }); + } + }); + } + placeholderHelper = + new PropertyPlaceholderHelper(PlaceholderConfigurerSupport.DEFAULT_PLACEHOLDER_PREFIX, + PlaceholderConfigurerSupport.DEFAULT_PLACEHOLDER_SUFFIX, + PlaceholderConfigurerSupport.DEFAULT_VALUE_SEPARATOR, true); + } + + @Test + public void testDataType() { + //tests that do not use ${name} to set config. + Assert.assertEquals("grpc.skywalking.incubator.apache.org", + yaml.load(placeholderHelper.replacePlaceholders(properties.getProperty("gRPCHost"), properties))); + + //tests that use ${REST_HOST:0.0.0.0} but not set REST_HOST in environmentVariables. + Assert.assertEquals("0.0.0.0", + yaml.load(placeholderHelper.replacePlaceholders(properties.getProperty("restHost"), properties))); + + //tests that use ${REST_PORT:12800} and set REST_PORT in environmentVariables. + Assert.assertEquals(12801, + yaml.load(placeholderHelper.replacePlaceholders(properties.getProperty("restPort"), properties))); + } + + @After + public void afterTest() { + //revert environment variables changes after the test for safe. + environmentVariables.clear("REST_HOST"); + } +} diff --git a/oap-server/server-library/library-util/src/test/resources/application.yml b/oap-server/server-library/library-util/src/test/resources/application.yml new file mode 100644 index 000000000..aa1855a72 --- /dev/null +++ b/oap-server/server-library/library-util/src/test/resources/application.yml @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cluster: + standalone: +# zookeeper: +# hostPort: localhost:2181 +# # Retry Policy +# baseSleepTimeMs: 1000 # initial amount of time to wait between retries +# maxRetries: 3 # max number of times to retry +# kubernetes: +# watchTimeoutSeconds: 60 +# namespace: default +# labelSelector: app=collector,release=skywalking +# uidEnvName: SKYWALKING_COLLECTOR_UID +core: + default: + restHost: ${REST_HOST:0.0.0.0} + restPort: ${REST_PORT:12800} + restContextPath: ${REST_CONTEXT_PATH:/} + gRPCHost: grpc.skywalking.incubator.apache.org + gRPCPort: ${GRPC_PORT:11800} + downsampling: + - Hour + - Day + - Month + # Set a timeout on metric data. After the timeout has expired, the metric data will automatically be deleted. + recordDataTTL: ${RECORD_DATA_TTL:90} # Unit is minute + minuteMetricsDataTTL: ${MINUTE_METRIC_DATA_TTL:90} # Unit is minute + hourMetricsDataTTL: ${HOUR_METRIC_DATA_TTL:36} # Unit is hour + dayMetricsDataTTL: ${DAY_METRIC_DATA_TTL:45} # Unit is day + monthMetricsDataTTL: ${MONTH_METRIC_DATA_TTL:18} # Unit is month +storage: + elasticsearch: + clusterNodes: ${ES_CLUSTER_ADDRESS:localhost:9200} + indexShardsNumber: ${ES_INDEX_SHARDS_NUMBER:2} + indexReplicasNumber: ${ES_INDEX_REPLICAS_NUMBER:0} + # Batch process setting, refer to https://www.elastic.co/guide/en/elasticsearch/client/java-api/5.5/java-docs-bulk-processor.html + bulkActions: ${ES_BULK_ACTIONS:2000} # Execute the bulk every 2000 requests + bulkSize: ${ES_BULK_SIZE:20} # flush the bulk every 20mb + flushInterval: ${ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests + concurrentRequests: ${ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests +# h2: +# driver: org.h2.jdbcx.JdbcDataSource +# url: jdbc:h2:mem:skywalking-oap-db +# user: sa +receiver-register: + default: +receiver-trace: + default: + bufferPath: ${RECEIVER_BUFFER_PATH:../trace-buffer/} # Path to trace buffer files, suggest to use absolute path + bufferOffsetMaxFileSize: ${RECEIVER_BUFFER_OFFSET_MAX_FILE_SIZE:100} # Unit is MB + bufferDataMaxFileSize: ${RECEIVER_BUFFER_DATA_MAX_FILE_SIZE:500} # Unit is MB + bufferFileCleanWhenRestart: ${RECEIVER_BUFFER_FILE_CLEAN_WHEN_RESTART:false} +receiver-jvm: + default: +service-mesh: + default: + bufferPath: ${SERVICE_MESH_BUFFER_PATH:../mesh-buffer/} # Path to trace buffer files, suggest to use absolute path + bufferOffsetMaxFileSize: ${SERVICE_MESH_OFFSET_MAX_FILE_SIZE:100} # Unit is MB + bufferDataMaxFileSize: ${SERVICE_MESH_BUFFER_DATA_MAX_FILE_SIZE:500} # Unit is MB + bufferFileCleanWhenRestart: ${SERVICE_MESH_BUFFER_FILE_CLEAN_WHEN_RESTART:false} +istio-telemetry: + default: +query: + graphql: + path: ${QUERY_GRAPHQL_PATH:/graphql} +alarm: + default: diff --git a/oap-server/server-starter/src/main/assembly/application.yml b/oap-server/server-starter/src/main/assembly/application.yml index f6612620a..f97c9e8ff 100644 --- a/oap-server/server-starter/src/main/assembly/application.yml +++ b/oap-server/server-starter/src/main/assembly/application.yml @@ -17,73 +17,73 @@ cluster: standalone: # Please check your ZooKeeper is 3.5+, However, it is also compatible with ZooKeeper 3.4.x. Replace the ZooKeeper 3.5+ - # library in the oap-libs folder with your ZooKeeper 3.4.x libraru. + # library the oap-libs folder with your ZooKeeper 3.4.x library. # zookeeper: -# hostPort: localhost:2181 -# # Retry Policy -# baseSleepTimeMs: 1000 # initial amount of time to wait between retries -# maxRetries: 3 # max number of times to retry +# hostPort: ${SW_CLUSTER_ZK_HOST_PORT:localhost:2181} +# #Retry Policy +# baseSleepTimeMs: ${SW_CLUSTER_ZK_SLEEP_TIME:1000} # initial amount of time to wait between retries +# maxRetries: ${SW_CLUSTER_ZK_MAX_RETRIES:3} # max number of times to retry # kubernetes: -# watchTimeoutSeconds: 60 -# namespace: default -# labelSelector: app=collector,release=skywalking -# uidEnvName: SKYWALKING_COLLECTOR_UID +# watchTimeoutSeconds: ${SW_CLUSTER_K8S_WATCH_TIMEOUT:60} +# namespace: ${SW_CLUSTER_K8S_NAMESPACE:default} +# labelSelector: ${SW_CLUSTER_K8S_LABEL:app=collector,release=skywalking} +# uidEnvName: ${SW_CLUSTER_K8S_UID:SKYWALKING_COLLECTOR_UID} core: default: - restHost: 0.0.0.0 - restPort: 12800 - restContextPath: / - gRPCHost: 0.0.0.0 - gRPCPort: 11800 + restHost: ${SW_CORE_REST_HOST:0.0.0.0} + restPort: ${SW_CORE_REST_PORT:12800} + restContextPath: ${SW_CORE_REST_CONTEXT_PATH:/} + gRPCHost: ${SW_CORE_GRPC_HOST:0.0.0.0} + gRPCPort: ${SW_CORE_GRPC_PORT:11800} downsampling: - Hour - Day - Month # Set a timeout on metric data. After the timeout has expired, the metric data will automatically be deleted. - recordDataTTL: 90 # Unit is minute - minuteMetricsDataTTL: 90 # Unit is minute - hourMetricsDataTTL: 36 # Unit is hour - dayMetricsDataTTL: 45 # Unit is day - monthMetricsDataTTL: 18 # Unit is month + recordDataTTL: ${SW_CORE_RECORD_DATA_TTL:90} # Unit is minute + minuteMetricsDataTTL: ${SW_CORE_MINUTE_METRIC_DATA_TTL:90} # Unit is minute + hourMetricsDataTTL: ${SW_CORE_HOUR_METRIC_DATA_TTL:36} # Unit is hour + dayMetricsDataTTL: ${SW_CORE_DAY_METRIC_DATA_TTL:45} # Unit is day + monthMetricsDataTTL: ${SW_CORE_MONTH_METRIC_DATA_TTL:18} # Unit is month storage: h2: - driver: org.h2.jdbcx.JdbcDataSource - url: jdbc:h2:mem:skywalking-oap-db - user: sa + driver: ${SW_STORAGE_H2_DRIVER:org.h2.jdbcx.JdbcDataSource} + url: ${SW_STORAGE_H2_URL:jdbc:h2:mem:skywalking-oap-db} + user: ${SW_STORAGE_H2_USER:sa} # elasticsearch: -# clusterNodes: localhost:9200 -# indexShardsNumber: 2 -# indexReplicasNumber: 0 +# clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:localhost:9200} +# indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:2} +# indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:0} # # Batch process setting, refer to https://www.elastic.co/guide/en/elasticsearch/client/java-api/5.5/java-docs-bulk-processor.html -# bulkActions: 2000 # Execute the bulk every 2000 requests -# bulkSize: 20 # flush the bulk every 20mb -# flushInterval: 10 # flush the bulk every 10 seconds whatever the number of requests -# concurrentRequests: 2 # the number of concurrent requests +# bulkActions: ${SW_STORAGE_ES_BULK_ACTIONS:2000} # Execute the bulk every 2000 requests +# bulkSize: ${SW_STORAGE_ES_BULK_SIZE:20} # flush the bulk every 20mb +# flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests +# concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests receiver-register: default: receiver-trace: default: - bufferPath: ../trace-buffer/ # Path to trace buffer files, suggest to use absolute path - bufferOffsetMaxFileSize: 100 # Unit is MB - bufferDataMaxFileSize: 500 # Unit is MB - bufferFileCleanWhenRestart: false + bufferPath: ${SW_RECEIVER_BUFFER_PATH:../trace-buffer/} # Path to trace buffer files, suggest to use absolute path + bufferOffsetMaxFileSize: ${SW_RECEIVER_BUFFER_OFFSET_MAX_FILE_SIZE:100} # Unit is MB + bufferDataMaxFileSize: ${SW_RECEIVER_BUFFER_DATA_MAX_FILE_SIZE:500} # Unit is MB + bufferFileCleanWhenRestart: ${SW_RECEIVER_BUFFER_FILE_CLEAN_WHEN_RESTART:false} receiver-jvm: default: service-mesh: default: - bufferPath: ../mesh-buffer/ # Path to mesh telemetry data buffer files, suggest to use absolute path - bufferOffsetMaxFileSize: 100 # Unit is MB - bufferDataMaxFileSize: 500 # Unit is MB - bufferFileCleanWhenRestart: false + bufferPath: ${SW_SERVICE_MESH_BUFFER_PATH:../mesh-buffer/} # Path to trace buffer files, suggest to use absolute path + bufferOffsetMaxFileSize: ${SW_SERVICE_MESH_OFFSET_MAX_FILE_SIZE:100} # Unit is MB + bufferDataMaxFileSize: ${SW_SERVICE_MESH_BUFFER_DATA_MAX_FILE_SIZE:500} # Unit is MB + bufferFileCleanWhenRestart: ${SW_SERVICE_MESH_BUFFER_FILE_CLEAN_WHEN_RESTART:false} istio-telemetry: default: #receiver_zipkin: # default: -# host: 0.0.0.0 -# port: 9411 -# contextPath: / +# host: ${SW_RECEIVER_ZIPKIN_HOST:0.0.0.0} +# port: ${SW_RECEIVER_ZIPKIN_PORT:9411} +# contextPath: ${SW_RECEIVER_ZIPKIN_CONTEXT_PATH:/} query: graphql: - path: /graphql + path: ${SW_QUERY_GRAPHQL_PATH:/graphql} alarm: default: diff --git a/oap-server/server-starter/src/main/java/org/apache/skywalking/oap/server/starter/config/ApplicationConfigLoader.java b/oap-server/server-starter/src/main/java/org/apache/skywalking/oap/server/starter/config/ApplicationConfigLoader.java index dbedbf6a4..5d93503d9 100644 --- a/oap-server/server-starter/src/main/java/org/apache/skywalking/oap/server/starter/config/ApplicationConfigLoader.java +++ b/oap-server/server-starter/src/main/java/org/apache/skywalking/oap/server/starter/config/ApplicationConfigLoader.java @@ -18,22 +18,22 @@ package org.apache.skywalking.oap.server.starter.config; +import java.io.FileNotFoundException; +import java.io.Reader; +import java.util.Map; +import java.util.Properties; import org.apache.skywalking.oap.server.library.module.ApplicationConfiguration; import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.library.util.PlaceholderConfigurerSupport; +import org.apache.skywalking.oap.server.library.util.PropertyPlaceholderHelper; import org.apache.skywalking.oap.server.library.util.ResourceUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml; -import java.io.FileNotFoundException; -import java.io.Reader; -import java.util.Map; -import java.util.Properties; - /** - * Initialize collector settings with following sources. - * Use application.yml as primary setting, - * and fix missing setting by default settings in application-default.yml. + * Initialize collector settings with following sources. Use application.yml as primary setting, and fix missing setting + * by default settings in application-default.yml. * * At last, override setting by system.properties and system.envs if the key matches moduleName.provideName.settingKey. * @@ -68,7 +68,13 @@ public class ApplicationConfigLoader implements ConfigLoader { properties.put(key, value); - logger.info("The property with key: {}, value: {}, in {} provider", key, value, name); + PropertyPlaceholderHelper helper = + new PropertyPlaceholderHelper(PlaceholderConfigurerSupport.DEFAULT_PLACEHOLDER_PREFIX, + PlaceholderConfigurerSupport.DEFAULT_PLACEHOLDER_SUFFIX, + PlaceholderConfigurerSupport.DEFAULT_VALUE_SEPARATOR, true); + final Object replaceValue = yaml.load(helper.replacePlaceholders(value + "", properties)); + properties.replace(key, replaceValue); + logger.info("The property with key: {}, value: {}, in {} provider", key, replaceValue.toString(), name); }); } moduleConfiguration.addProviderConfiguration(name, properties); diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 47f0463d0..d94a82ae4 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -19,71 +19,71 @@ cluster: # Please check your ZooKeeper is 3.5+, However, it is also compatible with ZooKeeper 3.4.x. Replace the ZooKeeper 3.5+ # library the oap-libs folder with your ZooKeeper 3.4.x library. # zookeeper: -# hostPort: localhost:2181 +# hostPort: ${SW_CLUSTER_ZK_HOST_PORT:localhost:2181} # #Retry Policy -# baseSleepTimeMs: 1000 # initial amount of time to wait between retries -# maxRetries: 3 # max number of times to retry +# baseSleepTimeMs: ${SW_CLUSTER_ZK_SLEEP_TIME:1000} # initial amount of time to wait between retries +# maxRetries: ${SW_CLUSTER_ZK_MAX_RETRIES:3} # max number of times to retry # kubernetes: -# watchTimeoutSeconds: 60 -# namespace: default -# labelSelector: app=collector,release=skywalking -# uidEnvName: SKYWALKING_COLLECTOR_UID +# watchTimeoutSeconds: ${SW_CLUSTER_K8S_WATCH_TIMEOUT:60} +# namespace: ${SW_CLUSTER_K8S_NAMESPACE:default} +# labelSelector: ${SW_CLUSTER_K8S_LABEL:app=collector,release=skywalking} +# uidEnvName: ${SW_CLUSTER_K8S_UID:SKYWALKING_COLLECTOR_UID} core: default: - restHost: 0.0.0.0 - restPort: 12800 - restContextPath: / - gRPCHost: 0.0.0.0 - gRPCPort: 11800 + restHost: ${SW_CORE_REST_HOST:0.0.0.0} + restPort: ${SW_CORE_REST_PORT:12800} + restContextPath: ${SW_CORE_REST_CONTEXT_PATH:/} + gRPCHost: ${SW_CORE_GRPC_HOST:0.0.0.0} + gRPCPort: ${SW_CORE_GRPC_PORT:11800} downsampling: - Hour - Day - Month # Set a timeout on metric data. After the timeout has expired, the metric data will automatically be deleted. - recordDataTTL: 90 # Unit is minute - minuteMetricsDataTTL: 90 # Unit is minute - hourMetricsDataTTL: 36 # Unit is hour - dayMetricsDataTTL: 45 # Unit is day - monthMetricsDataTTL: 18 # Unit is month + recordDataTTL: ${SW_CORE_RECORD_DATA_TTL:90} # Unit is minute + minuteMetricsDataTTL: ${SW_CORE_MINUTE_METRIC_DATA_TTL:90} # Unit is minute + hourMetricsDataTTL: ${SW_CORE_HOUR_METRIC_DATA_TTL:36} # Unit is hour + dayMetricsDataTTL: ${SW_CORE_DAY_METRIC_DATA_TTL:45} # Unit is day + monthMetricsDataTTL: ${SW_CORE_MONTH_METRIC_DATA_TTL:18} # Unit is month storage: elasticsearch: - clusterNodes: localhost:9200 - indexShardsNumber: 2 - indexReplicasNumber: 0 + clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:localhost:9200} + indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:2} + indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:0} # Batch process setting, refer to https://www.elastic.co/guide/en/elasticsearch/client/java-api/5.5/java-docs-bulk-processor.html - bulkActions: 2000 # Execute the bulk every 2000 requests - bulkSize: 20 # flush the bulk every 20mb - flushInterval: 10 # flush the bulk every 10 seconds whatever the number of requests - concurrentRequests: 2 # the number of concurrent requests + bulkActions: ${SW_STORAGE_ES_BULK_ACTIONS:2000} # Execute the bulk every 2000 requests + bulkSize: ${SW_STORAGE_ES_BULK_SIZE:20} # flush the bulk every 20mb + flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests + concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests # h2: -# driver: org.h2.jdbcx.JdbcDataSource -# url: jdbc:h2:mem:skywalking-oap-db -# user: sa +# driver: ${SW_STORAGE_H2_DRIVER:org.h2.jdbcx.JdbcDataSource} +# url: ${SW_STORAGE_H2_URL:jdbc:h2:mem:skywalking-oap-db} +# user: ${SW_STORAGE_H2_USER:sa} receiver-register: default: receiver-trace: default: - bufferPath: ../trace-buffer/ # Path to trace buffer files, suggest to use absolute path - bufferOffsetMaxFileSize: 100 # Unit is MB - bufferDataMaxFileSize: 500 # Unit is MB - bufferFileCleanWhenRestart: false + bufferPath: ${SW_RECEIVER_BUFFER_PATH:../trace-buffer/} # Path to trace buffer files, suggest to use absolute path + bufferOffsetMaxFileSize: ${SW_RECEIVER_BUFFER_OFFSET_MAX_FILE_SIZE:100} # Unit is MB + bufferDataMaxFileSize: ${SW_RECEIVER_BUFFER_DATA_MAX_FILE_SIZE:500} # Unit is MB + bufferFileCleanWhenRestart: ${SW_RECEIVER_BUFFER_FILE_CLEAN_WHEN_RESTART:false} receiver-jvm: default: service-mesh: default: - bufferPath: ../mesh-buffer/ # Path to trace buffer files, suggest to use absolute path - bufferOffsetMaxFileSize: 100 # Unit is MB - bufferDataMaxFileSize: 500 # Unit is MB - bufferFileCleanWhenRestart: false + bufferPath: ${SW_SERVICE_MESH_BUFFER_PATH:../mesh-buffer/} # Path to trace buffer files, suggest to use absolute path + bufferOffsetMaxFileSize: ${SW_SERVICE_MESH_OFFSET_MAX_FILE_SIZE:100} # Unit is MB + bufferDataMaxFileSize: ${SW_SERVICE_MESH_BUFFER_DATA_MAX_FILE_SIZE:500} # Unit is MB + bufferFileCleanWhenRestart: ${SW_SERVICE_MESH_BUFFER_FILE_CLEAN_WHEN_RESTART:false} istio-telemetry: default: receiver_zipkin: default: - host: 0.0.0.0 - port: 9411 - contextPath: / + host: ${SW_RECEIVER_ZIPKIN_HOST:0.0.0.0} + port: ${SW_RECEIVER_ZIPKIN_PORT:9411} + contextPath: ${SW_RECEIVER_ZIPKIN_CONTEXT_PATH:/} query: graphql: - path: /graphql + path: ${SW_QUERY_GRAPHQL_PATH:/graphql} alarm: default: