feat:support set config from system.envs. (#1919)

* support set config from system.envs.

docs/en/setup/backend/backend-setting-override.md

@@ -1,13 +1,14 @@
 # Setting Override
-SkyWalking backend supports setting overrides by system properties. 
+SkyWalking backend supports setting overrides by system properties and system environment variables. 
 You could override the settings in `application.yml`
 
-- System properties key rule
+## System properties key rule
 **ModuleName**.**ProviderName**.**SettingKey**.
 
 - Example
 
   Override `restHost` in this setting segment
+  
 ```yaml
 core:
   default:
@@ -20,5 +21,32 @@ core:
 
 Use command arg
 ```
--core.default.restHost=172.0.4.12
-```
+-Dcore.default.restHost=172.0.4.12
+```
+
+## System environment variables
+- Example
+
+  Override `restHost` in this setting segment through environment variables
+  
+```yaml
+core:
+  default:
+    restHost: ${REST_HOST:0.0.0.0}
+    restPort: 12800
+    restContextPath: /
+    gRPCHost: 0.0.0.
+    gRPCPort: 11800
+```
+
+If the `REST_HOST ` environment variable exists in your operating system and its value is `172.0.4.12`, 
+then the value of `restHost` here will be overwritten to `172.0.4.12`, otherwise, it will be set to `0.0.0.0`.
+
+By the way, Placeholder nesting is also supported, like `${REST_HOST:${ANOTHER_REST_HOST:127.0.0.1}}`.
+In this case, if the `REST_HOST ` environment variable not exists, but the ```REST_ANOTHER_REST_HOSTHOST``` 
+environment variable exists and its value is `172.0.4.12`, then the value of `restHost` here will be overwritten to `172.0.4.12`,
+otherwise, it will be set to `127.0.0.1`.
+
+
+
+

oap-server/server-library/library-util/pom.xml

@@ -17,7 +17,8 @@
   ~
   -->
 
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>server-library</artifactId>
         <groupId>org.apache.skywalking</groupId>
@@ -34,5 +35,16 @@
             <artifactId>joda-time</artifactId>
             <version>${joda-time.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.yaml</groupId>
+            <artifactId>snakeyaml</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.github.stefanbirkner</groupId>
+            <artifactId>system-rules</artifactId>
+            <version>1.18.0</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
-</project>
+</project>

oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/PlaceholderConfigurerSupport.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.library.util;
+
+/**
+ * @author jian.tan
+ */
+public class PlaceholderConfigurerSupport {
+
+    /** Default placeholder prefix: {@value} */
+    public static final String DEFAULT_PLACEHOLDER_PREFIX = "${";
+
+    /** Default placeholder suffix: {@value} */
+    public static final String DEFAULT_PLACEHOLDER_SUFFIX = "}";
+
+    /** Default value separator: {@value} */
+    public static final String DEFAULT_VALUE_SEPARATOR = ":";
+}

oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/PropertyPlaceholderHelper.java

@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.library.util;
+
+import com.google.common.base.Strings;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import org.apache.logging.log4j.core.util.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility class for working with Strings that have placeholder values in them. A placeholder takes the form {@code
+ * ${name}}. Using {@code PropertyPlaceholderHelper} these placeholders can be substituted for user-supplied values. <p>
+ * Values for substitution can be supplied using a {@link Properties} instance or using a {@link PlaceholderResolver}.
+ */
+public class PropertyPlaceholderHelper {
+    private static final Logger logger = LoggerFactory.getLogger(PropertyPlaceholderHelper.class);
+
+    private static final Map<String, String> WELL_KNOWN_SIMPLE_PREFIXES = new HashMap<>(4);
+
+    static {
+        WELL_KNOWN_SIMPLE_PREFIXES.put("}", "{");
+        WELL_KNOWN_SIMPLE_PREFIXES.put("]", "[");
+        WELL_KNOWN_SIMPLE_PREFIXES.put(")", "(");
+    }
+
+    private final String placeholderPrefix;
+
+    private final String placeholderSuffix;
+
+    private final String simplePrefix;
+
+    private final String valueSeparator;
+
+    private final boolean ignoreUnresolvablePlaceholders;
+
+    /**
+     * Creates a new {@code PropertyPlaceholderHelper} that uses the supplied prefix and suffix.
+     *
+     * @param placeholderPrefix the prefix that denotes the start of a placeholder
+     * @param placeholderSuffix the suffix that denotes the end of a placeholder
+     * @param valueSeparator the separating character between the placeholder variable and the associated default value,
+     * if any
+     * @param ignoreUnresolvablePlaceholders indicates whether unresolvable placeholders should be ignored ({@code
+     * true}) or cause an exception ({@code false})
+     */
+    public PropertyPlaceholderHelper(String placeholderPrefix, String placeholderSuffix,
+        String valueSeparator, boolean ignoreUnresolvablePlaceholders) {
+        Assert.requireNonEmpty(placeholderPrefix, "'placeholderPrefix' must not be null");
+        Assert.requireNonEmpty(placeholderSuffix, "'placeholderSuffix' must not be null");
+        this.placeholderPrefix = placeholderPrefix;
+        this.placeholderSuffix = placeholderSuffix;
+        String simplePrefixForSuffix = WELL_KNOWN_SIMPLE_PREFIXES.get(this.placeholderSuffix);
+        if (simplePrefixForSuffix != null && this.placeholderPrefix.endsWith(simplePrefixForSuffix)) {
+            this.simplePrefix = simplePrefixForSuffix;
+        } else {
+            this.simplePrefix = this.placeholderPrefix;
+        }
+        this.valueSeparator = valueSeparator;
+        this.ignoreUnresolvablePlaceholders = ignoreUnresolvablePlaceholders;
+    }
+
+    /**
+     * Replaces all placeholders of format {@code ${name}} with the corresponding property from the supplied {@link
+     * Properties}.
+     *
+     * @param value the value containing the placeholders to be replaced
+     * @param properties the {@code Properties} to use for replacement
+     * @return the supplied value with placeholders replaced inline
+     */
+    public String replacePlaceholders(String value, final Properties properties) {
+        Assert.requireNonEmpty(properties, "'properties' must not be null");
+        return replacePlaceholders(value, placeholderName -> this.getConfigValue(placeholderName, properties));
+    }
+
+    private String getConfigValue(String key, final Properties properties) {
+        String value = System.getProperty(key);
+        if (Strings.isNullOrEmpty(value)) {
+            value = System.getenv(key);
+        }
+        if (Strings.isNullOrEmpty(value)) {
+            value = properties.getProperty(key);
+        }
+        return value;
+    }
+
+    /**
+     * Replaces all placeholders of format {@code ${name}} with the value returned from the supplied {@link
+     * PlaceholderResolver}.
+     *
+     * @param value the value containing the placeholders to be replaced
+     * @param placeholderResolver the {@code PlaceholderResolver} to use for replacement
+     * @return the supplied value with placeholders replaced inline
+     */
+    public String replacePlaceholders(String value, PlaceholderResolver placeholderResolver) {
+        Assert.requireNonEmpty(value, "'value' must not be null");
+        return parseStringValue(value, placeholderResolver, new HashSet<String>());
+    }
+
+    protected String parseStringValue(String value, PlaceholderResolver placeholderResolver,
+        Set<String> visitedPlaceholders) {
+
+        StringBuilder result = new StringBuilder(value);
+
+        int startIndex = value.indexOf(this.placeholderPrefix);
+        while (startIndex != -1) {
+            int endIndex = findPlaceholderEndIndex(result, startIndex);
+            if (endIndex != -1) {
+                String placeholder = result.substring(startIndex + this.placeholderPrefix.length(), endIndex);
+                String originalPlaceholder = placeholder;
+                if (!visitedPlaceholders.add(originalPlaceholder)) {
+                    throw new IllegalArgumentException(
+                        "Circular placeholder reference '" + originalPlaceholder + "' in property definitions");
+                }
+                // Recursive invocation, parsing placeholders contained in the placeholder key.
+                placeholder = parseStringValue(placeholder, placeholderResolver, visitedPlaceholders);
+                // Now obtain the value for the fully resolved key...
+                String propVal = placeholderResolver.resolvePlaceholder(placeholder);
+                if (propVal == null && this.valueSeparator != null) {
+                    int separatorIndex = placeholder.indexOf(this.valueSeparator);
+                    if (separatorIndex != -1) {
+                        String actualPlaceholder = placeholder.substring(0, separatorIndex);
+                        String defaultValue = placeholder.substring(separatorIndex + this.valueSeparator.length());
+                        propVal = placeholderResolver.resolvePlaceholder(actualPlaceholder);
+                        if (propVal == null) {
+                            propVal = defaultValue;
+                        }
+                    }
+                }
+                if (propVal != null) {
+                    // Recursive invocation, parsing placeholders contained in the
+                    // previously resolved placeholder value.
+                    propVal = parseStringValue(propVal, placeholderResolver, visitedPlaceholders);
+                    result.replace(startIndex, endIndex + this.placeholderSuffix.length(), propVal);
+                    if (logger.isTraceEnabled()) {
+                        logger.trace("Resolved placeholder '" + placeholder + "'");
+                    }
+                    startIndex = result.indexOf(this.placeholderPrefix, startIndex + propVal.length());
+                } else if (this.ignoreUnresolvablePlaceholders) {
+                    // Proceed with unprocessed value.
+                    startIndex = result.indexOf(this.placeholderPrefix, endIndex + this.placeholderSuffix.length());
+                } else {
+                    throw new IllegalArgumentException("Could not resolve placeholder '" +
+                        placeholder + "'" + " in value \"" + value + "\"");
+                }
+                visitedPlaceholders.remove(originalPlaceholder);
+            } else {
+                startIndex = -1;
+            }
+        }
+        return result.toString();
+    }
+
+    private int findPlaceholderEndIndex(CharSequence buf, int startIndex) {
+        int index = startIndex + this.placeholderPrefix.length();
+        int withinNestedPlaceholder = 0;
+        while (index < buf.length()) {
+            if (StringUtils.substringMatch(buf, index, this.placeholderSuffix)) {
+                if (withinNestedPlaceholder > 0) {
+                    withinNestedPlaceholder--;
+                    index = index + this.placeholderSuffix.length();
+                } else {
+                    return index;
+                }
+            } else if (StringUtils.substringMatch(buf, index, this.simplePrefix)) {
+                withinNestedPlaceholder++;
+                index = index + this.simplePrefix.length();
+            } else {
+                index++;
+            }
+        }
+        return -1;
+    }
+
+    /**
+     * Strategy interface used to resolve replacement values for placeholders contained in Strings.
+     */
+    public interface PlaceholderResolver {
+
+        /**
+         * Resolve the supplied placeholder name to the replacement value.
+         *
+         * @param placeholderName the name of the placeholder to resolve
+         * @return the replacement value, or {@code null} if no replacement is to be made
+         */
+        String resolvePlaceholder(String placeholderName);
+    }
+}

oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/StringUtils.java

@@ -36,4 +36,16 @@ public class StringUtils {
     public static String getOrDefault(String value, String defaultValue) {
         return value == null ? defaultValue : value;
     }
+
+    public static boolean substringMatch(CharSequence str, int index, CharSequence substring) {
+        if (index + substring.length() > str.length()) {
+            return false;
+        }
+        for (int i = 0; i < substring.length(); i++) {
+            if (str.charAt(index + i) != substring.charAt(i)) {
+                return false;
+            }
+        }
+        return true;
+    }
 }

oap-server/server-library/library-util/src/test/java/org/apache/skywalking/oap/server/library/util/PropertyPlaceholderHelperTest.java

@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.library.util;
+
+import java.io.FileNotFoundException;
+import java.io.Reader;
+import java.util.Map;
+import java.util.Properties;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.contrib.java.lang.system.EnvironmentVariables;
+import org.yaml.snakeyaml.Yaml;
+
+/**
+ * @author jian.tan
+ */
+public class PropertyPlaceholderHelperTest {
+    private PropertyPlaceholderHelper placeholderHelper;
+    private Properties properties = new Properties();
+    private final Yaml yaml = new Yaml();
+
+    /**
+     * The EnvironmentVariables rule allows you to set environment variables for your test. All changes to environment
+     * variables are reverted after the test.
+     */
+    @Rule
+    public final EnvironmentVariables environmentVariables = new EnvironmentVariables().set("REST_PORT", "12801");
+
+    @SuppressWarnings("unchecked")
+    @Before
+    public void init() throws FileNotFoundException {
+        Reader applicationReader = ResourceUtils.read("application.yml");
+        Map<String, Map<String, Map<String, ?>>> moduleConfig = yaml.loadAs(applicationReader, Map.class);
+        if (CollectionUtils.isNotEmpty(moduleConfig)) {
+            moduleConfig.forEach((moduleName, providerConfig) -> {
+                if (providerConfig.size() > 0) {
+                    providerConfig.forEach((name, propertiesConfig) -> {
+                        if (propertiesConfig != null) {
+                            propertiesConfig.forEach((key, value) -> properties.put(key, value));
+                        }
+                    });
+                }
+            });
+        }
+        placeholderHelper =
+            new PropertyPlaceholderHelper(PlaceholderConfigurerSupport.DEFAULT_PLACEHOLDER_PREFIX,
+                PlaceholderConfigurerSupport.DEFAULT_PLACEHOLDER_SUFFIX,
+                PlaceholderConfigurerSupport.DEFAULT_VALUE_SEPARATOR, true);
+    }
+
+    @Test
+    public void testDataType() {
+        //tests that do not use ${name} to set config.
+        Assert.assertEquals("grpc.skywalking.incubator.apache.org",
+            yaml.load(placeholderHelper.replacePlaceholders(properties.getProperty("gRPCHost"), properties)));
+
+        //tests that use ${REST_HOST:0.0.0.0} but not set REST_HOST in environmentVariables.
+        Assert.assertEquals("0.0.0.0",
+            yaml.load(placeholderHelper.replacePlaceholders(properties.getProperty("restHost"), properties)));
+
+        //tests that use ${REST_PORT:12800} and set REST_PORT in environmentVariables.
+        Assert.assertEquals(12801,
+            yaml.load(placeholderHelper.replacePlaceholders(properties.getProperty("restPort"), properties)));
+    }
+
+    @After
+    public void afterTest() {
+        //revert environment variables changes after the test for safe.
+        environmentVariables.clear("REST_HOST");
+    }
+}

oap-server/server-library/library-util/src/test/resources/application.yml

@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cluster:
+  standalone:
+#  zookeeper:
+#    hostPort: localhost:2181
+#    # Retry Policy
+#    baseSleepTimeMs: 1000 # initial amount of time to wait between retries
+#    maxRetries: 3 # max number of times to retry
+#  kubernetes:
+#    watchTimeoutSeconds: 60
+#    namespace: default
+#    labelSelector: app=collector,release=skywalking
+#    uidEnvName: SKYWALKING_COLLECTOR_UID
+core:
+  default:
+    restHost: ${REST_HOST:0.0.0.0}
+    restPort: ${REST_PORT:12800}
+    restContextPath: ${REST_CONTEXT_PATH:/}
+    gRPCHost: grpc.skywalking.incubator.apache.org
+    gRPCPort: ${GRPC_PORT:11800}
+    downsampling:
+    - Hour
+    - Day
+    - Month
+    # Set a timeout on metric data. After the timeout has expired, the metric data will automatically be deleted.
+    recordDataTTL: ${RECORD_DATA_TTL:90} # Unit is minute
+    minuteMetricsDataTTL: ${MINUTE_METRIC_DATA_TTL:90} # Unit is minute
+    hourMetricsDataTTL: ${HOUR_METRIC_DATA_TTL:36} # Unit is hour
+    dayMetricsDataTTL: ${DAY_METRIC_DATA_TTL:45} # Unit is day
+    monthMetricsDataTTL: ${MONTH_METRIC_DATA_TTL:18} # Unit is month
+storage:
+  elasticsearch:
+    clusterNodes: ${ES_CLUSTER_ADDRESS:localhost:9200}
+    indexShardsNumber: ${ES_INDEX_SHARDS_NUMBER:2}
+    indexReplicasNumber: ${ES_INDEX_REPLICAS_NUMBER:0}
+    # Batch process setting, refer to https://www.elastic.co/guide/en/elasticsearch/client/java-api/5.5/java-docs-bulk-processor.html
+    bulkActions: ${ES_BULK_ACTIONS:2000} # Execute the bulk every 2000 requests
+    bulkSize: ${ES_BULK_SIZE:20} # flush the bulk every 20mb
+    flushInterval: ${ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests
+    concurrentRequests: ${ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests
+#  h2:
+#    driver: org.h2.jdbcx.JdbcDataSource
+#    url: jdbc:h2:mem:skywalking-oap-db
+#    user: sa
+receiver-register:
+  default:
+receiver-trace:
+  default:
+    bufferPath: ${RECEIVER_BUFFER_PATH:../trace-buffer/}  # Path to trace buffer files, suggest to use absolute path
+    bufferOffsetMaxFileSize: ${RECEIVER_BUFFER_OFFSET_MAX_FILE_SIZE:100} # Unit is MB
+    bufferDataMaxFileSize: ${RECEIVER_BUFFER_DATA_MAX_FILE_SIZE:500} # Unit is MB
+    bufferFileCleanWhenRestart: ${RECEIVER_BUFFER_FILE_CLEAN_WHEN_RESTART:false}
+receiver-jvm:
+  default:
+service-mesh:
+  default:
+    bufferPath: ${SERVICE_MESH_BUFFER_PATH:../mesh-buffer/}  # Path to trace buffer files, suggest to use absolute path
+    bufferOffsetMaxFileSize: ${SERVICE_MESH_OFFSET_MAX_FILE_SIZE:100} # Unit is MB
+    bufferDataMaxFileSize: ${SERVICE_MESH_BUFFER_DATA_MAX_FILE_SIZE:500} # Unit is MB
+    bufferFileCleanWhenRestart: ${SERVICE_MESH_BUFFER_FILE_CLEAN_WHEN_RESTART:false}
+istio-telemetry:
+  default:
+query:
+  graphql:
+    path: ${QUERY_GRAPHQL_PATH:/graphql}
+alarm:
+  default:

oap-server/server-starter/src/main/assembly/application.yml

@@ -17,73 +17,73 @@
 cluster:
   standalone:
   # Please check your ZooKeeper is 3.5+, However, it is also compatible with ZooKeeper 3.4.x. Replace the ZooKeeper 3.5+
-  # library in the oap-libs folder with your ZooKeeper 3.4.x libraru.
+  # library the oap-libs folder with your ZooKeeper 3.4.x library.
 #  zookeeper:
-#    hostPort: localhost:2181
-#    # Retry Policy
-#    baseSleepTimeMs: 1000 # initial amount of time to wait between retries
-#    maxRetries: 3 # max number of times to retry
+#    hostPort: ${SW_CLUSTER_ZK_HOST_PORT:localhost:2181}
+#    #Retry Policy
+#    baseSleepTimeMs: ${SW_CLUSTER_ZK_SLEEP_TIME:1000} # initial amount of time to wait between retries
+#    maxRetries: ${SW_CLUSTER_ZK_MAX_RETRIES:3} # max number of times to retry
 #  kubernetes:
-#    watchTimeoutSeconds: 60
-#    namespace: default
-#    labelSelector: app=collector,release=skywalking
-#    uidEnvName: SKYWALKING_COLLECTOR_UID
+#    watchTimeoutSeconds: ${SW_CLUSTER_K8S_WATCH_TIMEOUT:60}
+#    namespace: ${SW_CLUSTER_K8S_NAMESPACE:default}
+#    labelSelector: ${SW_CLUSTER_K8S_LABEL:app=collector,release=skywalking}
+#    uidEnvName: ${SW_CLUSTER_K8S_UID:SKYWALKING_COLLECTOR_UID}
 core:
   default:
-    restHost: 0.0.0.0
-    restPort: 12800
-    restContextPath: /
-    gRPCHost: 0.0.0.0
-    gRPCPort: 11800
+    restHost: ${SW_CORE_REST_HOST:0.0.0.0}
+    restPort: ${SW_CORE_REST_PORT:12800}
+    restContextPath: ${SW_CORE_REST_CONTEXT_PATH:/}
+    gRPCHost: ${SW_CORE_GRPC_HOST:0.0.0.0}
+    gRPCPort: ${SW_CORE_GRPC_PORT:11800}
     downsampling:
     - Hour
     - Day
     - Month
     # Set a timeout on metric data. After the timeout has expired, the metric data will automatically be deleted.
-    recordDataTTL: 90 # Unit is minute
-    minuteMetricsDataTTL: 90 # Unit is minute
-    hourMetricsDataTTL: 36 # Unit is hour
-    dayMetricsDataTTL: 45 # Unit is day
-    monthMetricsDataTTL: 18 # Unit is month
+    recordDataTTL: ${SW_CORE_RECORD_DATA_TTL:90} # Unit is minute
+    minuteMetricsDataTTL: ${SW_CORE_MINUTE_METRIC_DATA_TTL:90} # Unit is minute
+    hourMetricsDataTTL: ${SW_CORE_HOUR_METRIC_DATA_TTL:36} # Unit is hour
+    dayMetricsDataTTL: ${SW_CORE_DAY_METRIC_DATA_TTL:45} # Unit is day
+    monthMetricsDataTTL: ${SW_CORE_MONTH_METRIC_DATA_TTL:18} # Unit is month
 storage:
   h2:
-    driver: org.h2.jdbcx.JdbcDataSource
-    url: jdbc:h2:mem:skywalking-oap-db
-    user: sa
+    driver: ${SW_STORAGE_H2_DRIVER:org.h2.jdbcx.JdbcDataSource}
+    url: ${SW_STORAGE_H2_URL:jdbc:h2:mem:skywalking-oap-db}
+    user: ${SW_STORAGE_H2_USER:sa}
 #  elasticsearch:
-#    clusterNodes: localhost:9200
-#    indexShardsNumber: 2
-#    indexReplicasNumber: 0
+#    clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:localhost:9200}
+#    indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:2}
+#    indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:0}
 #    # Batch process setting, refer to https://www.elastic.co/guide/en/elasticsearch/client/java-api/5.5/java-docs-bulk-processor.html
-#    bulkActions: 2000 # Execute the bulk every 2000 requests
-#    bulkSize: 20 # flush the bulk every 20mb
-#    flushInterval: 10 # flush the bulk every 10 seconds whatever the number of requests
-#    concurrentRequests: 2 # the number of concurrent requests
+#    bulkActions: ${SW_STORAGE_ES_BULK_ACTIONS:2000} # Execute the bulk every 2000 requests
+#    bulkSize: ${SW_STORAGE_ES_BULK_SIZE:20} # flush the bulk every 20mb
+#    flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests
+#    concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests
 receiver-register:
   default:
 receiver-trace:
   default:
-    bufferPath: ../trace-buffer/  # Path to trace buffer files, suggest to use absolute path
-    bufferOffsetMaxFileSize: 100 # Unit is MB
-    bufferDataMaxFileSize: 500 # Unit is MB
-    bufferFileCleanWhenRestart: false
+    bufferPath: ${SW_RECEIVER_BUFFER_PATH:../trace-buffer/}  # Path to trace buffer files, suggest to use absolute path
+    bufferOffsetMaxFileSize: ${SW_RECEIVER_BUFFER_OFFSET_MAX_FILE_SIZE:100} # Unit is MB
+    bufferDataMaxFileSize: ${SW_RECEIVER_BUFFER_DATA_MAX_FILE_SIZE:500} # Unit is MB
+    bufferFileCleanWhenRestart: ${SW_RECEIVER_BUFFER_FILE_CLEAN_WHEN_RESTART:false}
 receiver-jvm:
   default:
 service-mesh:
   default:
-    bufferPath: ../mesh-buffer/  # Path to mesh telemetry data buffer files, suggest to use absolute path
-    bufferOffsetMaxFileSize: 100 # Unit is MB
-    bufferDataMaxFileSize: 500 # Unit is MB
-    bufferFileCleanWhenRestart: false
+    bufferPath: ${SW_SERVICE_MESH_BUFFER_PATH:../mesh-buffer/}  # Path to trace buffer files, suggest to use absolute path
+    bufferOffsetMaxFileSize: ${SW_SERVICE_MESH_OFFSET_MAX_FILE_SIZE:100} # Unit is MB
+    bufferDataMaxFileSize: ${SW_SERVICE_MESH_BUFFER_DATA_MAX_FILE_SIZE:500} # Unit is MB
+    bufferFileCleanWhenRestart: ${SW_SERVICE_MESH_BUFFER_FILE_CLEAN_WHEN_RESTART:false}
 istio-telemetry:
   default:
 #receiver_zipkin:
 #  default:
-#    host: 0.0.0.0
-#    port: 9411
-#    contextPath: /
+#    host: ${SW_RECEIVER_ZIPKIN_HOST:0.0.0.0}
+#    port: ${SW_RECEIVER_ZIPKIN_PORT:9411}
+#    contextPath: ${SW_RECEIVER_ZIPKIN_CONTEXT_PATH:/}
 query:
   graphql:
-    path: /graphql
+    path: ${SW_QUERY_GRAPHQL_PATH:/graphql}
 alarm:
   default:

oap-server/server-starter/src/main/java/org/apache/skywalking/oap/server/starter/config/ApplicationConfigLoader.java

@@ -18,22 +18,22 @@
 
 package org.apache.skywalking.oap.server.starter.config;
 
+import java.io.FileNotFoundException;
+import java.io.Reader;
+import java.util.Map;
+import java.util.Properties;
 import org.apache.skywalking.oap.server.library.module.ApplicationConfiguration;
 import org.apache.skywalking.oap.server.library.util.CollectionUtils;
+import org.apache.skywalking.oap.server.library.util.PlaceholderConfigurerSupport;
+import org.apache.skywalking.oap.server.library.util.PropertyPlaceholderHelper;
 import org.apache.skywalking.oap.server.library.util.ResourceUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.yaml.snakeyaml.Yaml;
 
-import java.io.FileNotFoundException;
-import java.io.Reader;
-import java.util.Map;
-import java.util.Properties;
-
 /**
- * Initialize collector settings with following sources.
- * Use application.yml as primary setting,
- * and fix missing setting by default settings in application-default.yml.
+ * Initialize collector settings with following sources. Use application.yml as primary setting, and fix missing setting
+ * by default settings in application-default.yml.
  *
  * At last, override setting by system.properties and system.envs if the key matches moduleName.provideName.settingKey.
  *
@@ -68,7 +68,13 @@ public class ApplicationConfigLoader implements ConfigLoader<ApplicationConfigur
                             if (propertiesConfig != null) {
                                 propertiesConfig.forEach((key, value) -> {
                                     properties.put(key, value);
-                                    logger.info("The property with key: {}, value: {}, in {} provider", key, value, name);
+                                    PropertyPlaceholderHelper helper =
+                                        new PropertyPlaceholderHelper(PlaceholderConfigurerSupport.DEFAULT_PLACEHOLDER_PREFIX,
+                                            PlaceholderConfigurerSupport.DEFAULT_PLACEHOLDER_SUFFIX,
+                                            PlaceholderConfigurerSupport.DEFAULT_VALUE_SEPARATOR, true);
+                                    final Object replaceValue = yaml.load(helper.replacePlaceholders(value + "", properties));
+                                    properties.replace(key, replaceValue);
+                                    logger.info("The property with key: {}, value: {}, in {} provider", key, replaceValue.toString(), name);
                                 });
                             }
                             moduleConfiguration.addProviderConfiguration(name, properties);

oap-server/server-starter/src/main/resources/application.yml

@@ -19,71 +19,71 @@ cluster:
    # Please check your ZooKeeper is 3.5+, However, it is also compatible with ZooKeeper 3.4.x. Replace the ZooKeeper 3.5+
    # library the oap-libs folder with your ZooKeeper 3.4.x library.
 #  zookeeper:
-#    hostPort: localhost:2181
+#    hostPort: ${SW_CLUSTER_ZK_HOST_PORT:localhost:2181}
 #    #Retry Policy
-#    baseSleepTimeMs: 1000 # initial amount of time to wait between retries
-#    maxRetries: 3 # max number of times to retry
+#    baseSleepTimeMs: ${SW_CLUSTER_ZK_SLEEP_TIME:1000} # initial amount of time to wait between retries
+#    maxRetries: ${SW_CLUSTER_ZK_MAX_RETRIES:3} # max number of times to retry
 #  kubernetes:
-#    watchTimeoutSeconds: 60
-#    namespace: default
-#    labelSelector: app=collector,release=skywalking
-#    uidEnvName: SKYWALKING_COLLECTOR_UID
+#    watchTimeoutSeconds: ${SW_CLUSTER_K8S_WATCH_TIMEOUT:60}
+#    namespace: ${SW_CLUSTER_K8S_NAMESPACE:default}
+#    labelSelector: ${SW_CLUSTER_K8S_LABEL:app=collector,release=skywalking}
+#    uidEnvName: ${SW_CLUSTER_K8S_UID:SKYWALKING_COLLECTOR_UID}
 core:
   default:
-    restHost: 0.0.0.0
-    restPort: 12800
-    restContextPath: /
-    gRPCHost: 0.0.0.0
-    gRPCPort: 11800
+    restHost: ${SW_CORE_REST_HOST:0.0.0.0}
+    restPort: ${SW_CORE_REST_PORT:12800}
+    restContextPath: ${SW_CORE_REST_CONTEXT_PATH:/}
+    gRPCHost: ${SW_CORE_GRPC_HOST:0.0.0.0}
+    gRPCPort: ${SW_CORE_GRPC_PORT:11800}
     downsampling:
     - Hour
     - Day
     - Month
     # Set a timeout on metric data. After the timeout has expired, the metric data will automatically be deleted.
-    recordDataTTL: 90 # Unit is minute
-    minuteMetricsDataTTL: 90 # Unit is minute
-    hourMetricsDataTTL: 36 # Unit is hour
-    dayMetricsDataTTL: 45 # Unit is day
-    monthMetricsDataTTL: 18 # Unit is month
+    recordDataTTL: ${SW_CORE_RECORD_DATA_TTL:90} # Unit is minute
+    minuteMetricsDataTTL: ${SW_CORE_MINUTE_METRIC_DATA_TTL:90} # Unit is minute
+    hourMetricsDataTTL: ${SW_CORE_HOUR_METRIC_DATA_TTL:36} # Unit is hour
+    dayMetricsDataTTL: ${SW_CORE_DAY_METRIC_DATA_TTL:45} # Unit is day
+    monthMetricsDataTTL: ${SW_CORE_MONTH_METRIC_DATA_TTL:18} # Unit is month
 storage:
   elasticsearch:
-    clusterNodes: localhost:9200
-    indexShardsNumber: 2
-    indexReplicasNumber: 0
+    clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:localhost:9200}
+    indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:2}
+    indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:0}
     # Batch process setting, refer to https://www.elastic.co/guide/en/elasticsearch/client/java-api/5.5/java-docs-bulk-processor.html
-    bulkActions: 2000 # Execute the bulk every 2000 requests
-    bulkSize: 20 # flush the bulk every 20mb
-    flushInterval: 10 # flush the bulk every 10 seconds whatever the number of requests
-    concurrentRequests: 2 # the number of concurrent requests
+    bulkActions: ${SW_STORAGE_ES_BULK_ACTIONS:2000} # Execute the bulk every 2000 requests
+    bulkSize: ${SW_STORAGE_ES_BULK_SIZE:20} # flush the bulk every 20mb
+    flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests
+    concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests
 #  h2:
-#    driver: org.h2.jdbcx.JdbcDataSource
-#    url: jdbc:h2:mem:skywalking-oap-db
-#    user: sa
+#    driver: ${SW_STORAGE_H2_DRIVER:org.h2.jdbcx.JdbcDataSource}
+#    url: ${SW_STORAGE_H2_URL:jdbc:h2:mem:skywalking-oap-db}
+#    user: ${SW_STORAGE_H2_USER:sa}
 receiver-register:
   default:
 receiver-trace:
   default:
-    bufferPath: ../trace-buffer/  # Path to trace buffer files, suggest to use absolute path
-    bufferOffsetMaxFileSize: 100 # Unit is MB
-    bufferDataMaxFileSize: 500 # Unit is MB
-    bufferFileCleanWhenRestart: false
+    bufferPath: ${SW_RECEIVER_BUFFER_PATH:../trace-buffer/}  # Path to trace buffer files, suggest to use absolute path
+    bufferOffsetMaxFileSize: ${SW_RECEIVER_BUFFER_OFFSET_MAX_FILE_SIZE:100} # Unit is MB
+    bufferDataMaxFileSize: ${SW_RECEIVER_BUFFER_DATA_MAX_FILE_SIZE:500} # Unit is MB
+    bufferFileCleanWhenRestart: ${SW_RECEIVER_BUFFER_FILE_CLEAN_WHEN_RESTART:false}
 receiver-jvm:
   default:
 service-mesh:
   default:
-    bufferPath: ../mesh-buffer/  # Path to trace buffer files, suggest to use absolute path
-    bufferOffsetMaxFileSize: 100 # Unit is MB
-    bufferDataMaxFileSize: 500 # Unit is MB
-    bufferFileCleanWhenRestart: false
+    bufferPath: ${SW_SERVICE_MESH_BUFFER_PATH:../mesh-buffer/}  # Path to trace buffer files, suggest to use absolute path
+    bufferOffsetMaxFileSize: ${SW_SERVICE_MESH_OFFSET_MAX_FILE_SIZE:100} # Unit is MB
+    bufferDataMaxFileSize: ${SW_SERVICE_MESH_BUFFER_DATA_MAX_FILE_SIZE:500} # Unit is MB
+    bufferFileCleanWhenRestart: ${SW_SERVICE_MESH_BUFFER_FILE_CLEAN_WHEN_RESTART:false}
 istio-telemetry:
   default:
 receiver_zipkin:
   default:
-    host: 0.0.0.0
-    port: 9411
-    contextPath: /
+    host: ${SW_RECEIVER_ZIPKIN_HOST:0.0.0.0}
+    port: ${SW_RECEIVER_ZIPKIN_PORT:9411}
+    contextPath: ${SW_RECEIVER_ZIPKIN_CONTEXT_PATH:/}
 query:
   graphql:
-    path: /graphql
+    path: ${SW_QUERY_GRAPHQL_PATH:/graphql}
 alarm:
   default: