kazusa
/
skywalking-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Configmap configuration (#4959)

This commit is contained in:
Evan 2020-07-20 12:35:35 +08:00 committed by GitHub
parent b9892ab1d4
commit e0325a4407
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 586 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
docs/en/setup/backend/dynamic-config.md
View File

@ -103,3 +103,20 @@ configuration:
appId: ${SW_CONFIG_APOLLO_APP_ID:skywalking}
period: ${SW_CONFIG_APOLLO_PERIOD:5}
```
## Dynamic Configuration Kuberbetes Configmap Implementation
[configmap](https://kubernetes.io/docs/concepts/configuration/configmap/) is also supported as DCC(Dynamic Configuration Center), to use it, just configured as follows:
```yaml
configuration:
selector: ${SW_CONFIGURATION:k8s-configmap}
# [example] (../../../../oap-server/server-configuration/configuration-k8s-configmap/src/test/resources/skywalking-dynamic-configmap.example.yaml)
k8s-configmap:
# Sync period in seconds. Defaults to 60 seconds.
period: ${SW_CONFIG_CONFIGMAP_PERIOD:60}
# Which namespace is confiigmap deployed in.
namespace: ${SW_CLUSTER_K8S_NAMESPACE:default}
# Labelselector is used to locate specific configmap
labelSelector: ${SW_CLUSTER_K8S_LABEL:app=collector,release=skywalking}
```

5
oap-server/server-bootstrap/pom.xml
View File

@ -195,6 +195,11 @@
<artifactId>configuration-consul</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.skywalking</groupId>
<artifactId>configuration-k8s-configmap</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<build>

4
oap-server/server-bootstrap/src/main/resources/application.yml
View File

@ -264,6 +264,10 @@ configuration:
period: ${SW_CONFIG_CONSUL_PERIOD:1}
# Consul aclToken
aclToken: ${SW_CONFIG_CONSUL_ACL_TOKEN:""}
k8s-configmap:
period: ${SW_CONFIG_CONFIGMAP_PERIOD:60}
namespace: ${SW_CLUSTER_K8S_NAMESPACE:default}
labelSelector: ${SW_CLUSTER_K8S_LABEL:app=collector,release=skywalking}
exporter:
selector: ${SW_EXPORTER:-}

51
oap-server/server-configuration/configuration-k8s-configmap/pom.xml Normal file
View File

@ -0,0 +1,51 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
~
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>server-configuration</artifactId>
<groupId>org.apache.skywalking</groupId>
<version>8.1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>configuration-k8s-configmap</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.skywalking</groupId>
<artifactId>configuration-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.skywalking</groupId>
<artifactId>server-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.kubernetes</groupId>
<artifactId>client-java</artifactId>
</dependency>
<dependency>
<groupId>org.apache.skywalking</groupId>
<artifactId>library-client</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</project>

54
oap-server/server-configuration/configuration-k8s-configmap/src/main/java/org/apache/skywalking/oap/server/configuration/configmap/ConfigmapConfigurationProvider.java Normal file
View File

@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.configuration.configmap;
import com.google.common.base.Strings;
import org.apache.skywalking.oap.server.configuration.api.AbstractConfigurationProvider;
import org.apache.skywalking.oap.server.configuration.api.ConfigWatcherRegister;
import org.apache.skywalking.oap.server.library.module.ModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
public class ConfigmapConfigurationProvider extends AbstractConfigurationProvider {
private final ConfigmapConfigurationSettings settings;
public ConfigmapConfigurationProvider() {
this.settings = new ConfigmapConfigurationSettings();
}
@Override
public String name() {
return "k8s-configmap";
}
@Override
public ModuleConfig createConfigBeanIfAbsent() {
return settings;
}
@Override
protected ConfigWatcherRegister initConfigReader() throws ModuleStartException {
if (Strings.isNullOrEmpty(settings.getLabelSelector()) || Strings.isNullOrEmpty(settings.getNamespace())) {
throw new ModuleStartException("the settings of configmap configuration is illegal.");
}
ConfigurationConfigmapInformer informer = new ConfigurationConfigmapInformer(settings);
return new ConfigmapConfigurationWatcherRegister(settings, informer);
}
}

40
oap-server/server-configuration/configuration-k8s-configmap/src/main/java/org/apache/skywalking/oap/server/configuration/configmap/ConfigmapConfigurationSettings.java Normal file
View File

@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.configuration.configmap;
import lombok.Getter;
import lombok.Setter;
import org.apache.skywalking.oap.server.library.module.ModuleConfig;
@Setter
@Getter
public class ConfigmapConfigurationSettings extends ModuleConfig {
/**
* namespace for deployment
*/
private String namespace;
/**
* how to find the configmap
*/
private String labelSelector;
/**
* the period for skywalking configuration reloading
*/
private Integer period;
}

56
oap-server/server-configuration/configuration-k8s-configmap/src/main/java/org/apache/skywalking/oap/server/configuration/configmap/ConfigmapConfigurationWatcherRegister.java Normal file
View File

@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.configuration.configmap;
import io.kubernetes.client.openapi.models.V1ConfigMap;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.oap.server.configuration.api.ConfigTable;
import org.apache.skywalking.oap.server.configuration.api.ConfigWatcherRegister;
@Slf4j
public class ConfigmapConfigurationWatcherRegister extends ConfigWatcherRegister {
private final ConfigurationConfigmapInformer informer;
public ConfigmapConfigurationWatcherRegister(ConfigmapConfigurationSettings settings,
ConfigurationConfigmapInformer informer) {
super(settings.getPeriod());
this.informer = informer;
}
@Override
public Optional<ConfigTable> readConfig(Set<String> keys) {
final ConfigTable configTable = new ConfigTable();
Optional<V1ConfigMap> v1ConfigMap = informer.configMap();
for (final String name : keys) {
final String value = v1ConfigMap.map(configMap -> configMap.getData().get(name)).orElse(null);
if (log.isDebugEnabled()) {
log.debug("read config: name:{} ,value:{}", name, value);
}
if (Objects.nonNull(value)) {
configTable.add(new ConfigTable.ConfigItem(name, value));
}
}
return Optional.of(configTable);
}
}

89
oap-server/server-configuration/configuration-k8s-configmap/src/main/java/org/apache/skywalking/oap/server/configuration/configmap/ConfigurationConfigmapInformer.java Normal file
View File

@ -0,0 +1,89 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.configuration.configmap;
import io.kubernetes.client.informer.SharedIndexInformer;
import io.kubernetes.client.informer.SharedInformerFactory;
import io.kubernetes.client.informer.cache.Lister;
import io.kubernetes.client.openapi.ApiClient;
import io.kubernetes.client.openapi.apis.CoreV1Api;
import io.kubernetes.client.openapi.models.V1ConfigMap;
import io.kubernetes.client.openapi.models.V1ConfigMapList;
import io.kubernetes.client.util.Config;
import java.io.IOException;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class ConfigurationConfigmapInformer {
private Lister<V1ConfigMap> configMapLister;
private SharedInformerFactory factory;
private final ExecutorService executorService = Executors.newSingleThreadExecutor(r -> {
Thread thread = new Thread(r, "SKYWALKING_KUBERNETES_CONFIGURATION_INFORMER");
thread.setDaemon(true);
return thread;
});
public ConfigurationConfigmapInformer(ConfigmapConfigurationSettings settings) {
try {
doStartConfigMapInformer(settings);
doAddShutdownHook();
} catch (IOException e) {
log.error("cannot connect with api server in kubernetes", e);
}
}
private void doAddShutdownHook() {
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
if (Objects.nonNull(factory)) {
factory.stopAllRegisteredInformers();
}
}));
}
private void doStartConfigMapInformer(final ConfigmapConfigurationSettings settings) throws IOException {
ApiClient apiClient = Config.defaultClient();
apiClient.setHttpClient(apiClient.getHttpClient().newBuilder().readTimeout(0, TimeUnit.SECONDS).build());
CoreV1Api coreV1Api = new CoreV1Api(apiClient);
factory = new SharedInformerFactory(executorService);
SharedIndexInformer<V1ConfigMap> configMapSharedIndexInformer = factory.sharedIndexInformerFor(
params -> coreV1Api.listNamespacedConfigMapCall(
settings.getNamespace(), null, null, null, null, settings.getLabelSelector()
, 1, params.resourceVersion, params.timeoutSeconds, params.watch, null
),
V1ConfigMap.class, V1ConfigMapList.class
);
factory.startAllRegisteredInformers();
configMapLister = new Lister<>(configMapSharedIndexInformer.getIndexer());
}
public Optional<V1ConfigMap> configMap() {
return Optional.ofNullable(configMapLister.list().size() == 1 ? configMapLister.list().get(0) : null);
}
}

19
oap-server/server-configuration/configuration-k8s-configmap/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider Normal file
View File

@ -0,0 +1,19 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
org.apache.skywalking.oap.server.configuration.configmap.ConfigmapConfigurationProvider

94
oap-server/server-configuration/configuration-k8s-configmap/src/test/java/org/apache/skywalking/oap/server/configuration/configmap/ConfigmapConfigWatcherRegisterTest.java Normal file
View File

@ -0,0 +1,94 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.configuration.configmap;
import io.kubernetes.client.openapi.models.V1ConfigMap;
import java.io.Reader;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
import org.apache.skywalking.oap.server.configuration.api.ConfigTable;
import org.apache.skywalking.oap.server.library.util.ResourceUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.yaml.snakeyaml.Yaml;
@RunWith(PowerMockRunner.class)
@PowerMockIgnore("javax.management.*")
@PrepareForTest({ConfigurationConfigmapInformer.class})
public class ConfigmapConfigWatcherRegisterTest {
private ConfigmapConfigurationWatcherRegister register;
private ConfigurationConfigmapInformer informer;
private final Yaml yaml = new Yaml();
@Before
public void prepare() throws IllegalAccessException {
ConfigmapConfigurationSettings settings = new ConfigmapConfigurationSettings();
settings.setPeriod(60);
informer = PowerMockito.mock(ConfigurationConfigmapInformer.class);
register = new ConfigmapConfigurationWatcherRegister(settings, informer);
}
@Test
public void readConfigWhenInformerNotwork() throws Exception {
PowerMockito.doReturn(Optional.empty()).when(informer).configMap();
Optional<ConfigTable> optionalConfigTable = register.readConfig(new HashSet<String>() {{
add("key1");
}});
Assert.assertTrue(optionalConfigTable.isPresent());
ConfigTable configTable = optionalConfigTable.get();
Assert.assertEquals(configTable.getItems().size(), 0);
}
@Test
public void readConfigWhenInformerWork() throws Exception {
Reader configmapReader = ResourceUtils.read("skywalking-dynamic-configmap.example.yaml");
Map<String, Map<String, String>> configmapMap = yaml.loadAs(configmapReader, Map.class);
V1ConfigMap v1ConfigMap = new V1ConfigMap();
v1ConfigMap.data(configmapMap.get("data"));
PowerMockito.doReturn(Optional.of(v1ConfigMap)).when(informer).configMap();
Optional<ConfigTable> optionalConfigTable = register.readConfig(new HashSet<String>() {{
add("receiver-trace.default.slowDBAccessThreshold");
add("alarm.default.alarm-settings");
add("core.default.apdexThreshold");
add("receiver-trace.default.uninstrumentedGateways");
}});
Assert.assertTrue(optionalConfigTable.isPresent());
ConfigTable configTable = optionalConfigTable.get();
List<String> list = configTable.getItems().stream()
.map(ConfigTable.ConfigItem::getValue)
.filter(Objects::nonNull)
.collect(Collectors.toList());
Assert.assertEquals(list.size(), 4);
}
}

47
oap-server/server-configuration/configuration-k8s-configmap/src/test/java/org/apache/skywalking/oap/server/configuration/configmap/ConfigmapConfigurationProviderTest.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.configuration.configmap;
import org.apache.skywalking.oap.server.library.module.ModuleConfig;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.modules.junit4.PowerMockRunner;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@RunWith(PowerMockRunner.class)
@PowerMockIgnore("javax.management.*")
public class ConfigmapConfigurationProviderTest {
private final ConfigmapConfigurationProvider provider = new ConfigmapConfigurationProvider();
@Test
public void name() {
assertEquals("k8s-configmap", provider.name());
}
@Test
public void module() {
ModuleConfig moduleConfig = provider.createConfigBeanIfAbsent();
assertTrue(moduleConfig instanceof ConfigmapConfigurationSettings);
}
}

109
oap-server/server-configuration/configuration-k8s-configmap/src/test/resources/skywalking-dynamic-configmap.example.yaml Normal file
View File

@ -0,0 +1,109 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
apiVersion: v1
kind: ConfigMap
metadata:
name: skywalking-dynamic-config
labels:
app: skywalking-alarm
release: skywalking
data:
receiver-trace.default.slowDBAccessThreshold: default:200,mongodb:50
alarm.default.alarm-settings: |-
rules:
# Rule unique name, must be ended with `_rule`.
service_resp_time_rule:
metrics-name: service_resp_time
op: ">"
threshold: 1000
period: 10
count: 3
silence-period: 5
message: Response time of service {name} is more than 1000ms in 3 minutes of last 10 minutes.
service_sla_rule:
# Metrics value need to be long, double or int
metrics-name: service_sla
op: "<"
threshold: 8000
# The length of time to evaluate the metrics
period: 10
# How many times after the metrics match the condition, will trigger alarm
count: 2
# How many times of checks, the alarm keeps silence after alarm triggered, default as same as period.
silence-period: 3
message: Successful rate of service {name} is lower than 80% in 2 minutes of last 10 minutes
service_resp_time_percentile_rule:
# Metrics value need to be long, double or int
metrics-name: service_percentile
op: ">"
threshold: 1000,1000,1000,1000,1000
period: 10
count: 3
silence-period: 5
message: Percentile response time of service {name} alarm in 3 minutes of last 10 minutes, due to more than one condition of p50 > 1000, p75 > 1000, p90 > 1000, p95 > 1000, p99 > 1000
service_instance_resp_time_rule:
metrics-name: service_instance_resp_time
op: ">"
threshold: 1000
period: 10
count: 2
silence-period: 5
message: Response time of service instance {name} is more than 1000ms in 2 minutes of last 10 minutes
database_access_resp_time_rule:
metrics-name: database_access_resp_time
threshold: 1000
op: ">"
period: 10
count: 2
message: Response time of database access {name} is more than 1000ms in 2 minutes of last 10 minutes
endpoint_relation_resp_time_rule:
metrics-name: endpoint_relation_resp_time
threshold: 1000
op: ">"
period: 10
count: 2
message: Response time of endpoint relation {name} is more than 1000ms in 2 minutes of last 10 minutes
# Active endpoint related metrics alarm will cost more memory than service and service instance metrics alarm.
# Because the number of endpoint is much more than service and instance.
#
# endpoint_avg_rule:
# metrics-name: endpoint_avg
# op: ">"
# threshold: 1000
# period: 10
# count: 2
# silence-period: 5
# message: Response time of endpoint {name} is more than 1000ms in 2 minutes of last 10 minutes
webhooks:
# - http://127.0.0.1/notify/
# - http://127.0.0.1/go-wechat/
core.default.apdexThreshold: |-
default: 500
# example:
# the threshold of service "tomcat" is 1s
# tomcat: 1000
# the threshold of service "springboot1" is 50ms
# springboot1: 50
receiver-trace.default.uninstrumentedGateways: |-
#gateways:
# - name: proxy0
# instances:
# - host: 127.0.0.1 # the host/ip of this gateway instance
# port: 9099 # the port of this gateway instance, defaults to 80

1
oap-server/server-configuration/pom.xml
View File

@ -35,6 +35,7 @@
<module>configuration-zookeeper</module>
<module>configuration-etcd</module>
<module>configuration-consul</module>
<module>configuration-k8s-configmap</module>
</modules>
</project>