diff --git a/Jenkinsfile b/Jenkinsfile index b63dc9e08..c11e646e1 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -25,6 +25,7 @@ pipeline { )) timestamps() skipStagesAfterUnstable() + timeout(time: 5, unit: 'HOURS') } environment { diff --git a/Jenkinsfile-E2E b/Jenkinsfile-E2E index 1bb9e89d9..6abe4cdda 100755 --- a/Jenkinsfile-E2E +++ b/Jenkinsfile-E2E @@ -21,6 +21,11 @@ pipeline { label 'skywalking' } + options { + timestamps() + timeout(time: 5, unit: 'HOURS') + } + tools { jdk 'JDK 1.8 (latest)' } diff --git a/docs/en/setup/backend/backend-alarm.md b/docs/en/setup/backend/backend-alarm.md index 5ab73909c..0a2ee5103 100644 --- a/docs/en/setup/backend/backend-alarm.md +++ b/docs/en/setup/backend/backend-alarm.md @@ -1,5 +1,5 @@ # Alarm -Alarm core is driven a collection of rules, which are defined in `config/alarm-settings.yml`. +Alarm core is driven by a collection of rules, which are defined in `config/alarm-settings.yml`. There are two parts in alarm rule definition. 1. [Alarm rules](#rules). They define how metrics alarm should be triggered, what conditions should be considered. 1. [Webhooks](#webhook). The list of web service endpoint, which should be called after the alarm is triggered. @@ -90,3 +90,11 @@ Example as following "startTime": 1560524171000 }] ``` + +## Update the settings dynamically +Since 6.5.0, the alarm settings can be updated dynamically at runtime by [Dynamic Configuration](dynamic-config.md), +which will override the settings in `alarm-settings.yml`. + +In order to determine that whether an alarm rule is triggered or not, SkyWalking needs to cache the metrics of a time window for +each alarm rule, if any attribute (`metrics-name`, `op`, `threshold`, `period`, `count`, etc.) of a rule is changed, +the sliding window will be destroyed and re-created, causing the alarm of this specific rule to restart again. \ No newline at end of file diff --git a/docs/en/setup/backend/dynamic-config.md b/docs/en/setup/backend/dynamic-config.md index 09a077ddd..0ddf1552c 100755 --- a/docs/en/setup/backend/dynamic-config.md +++ b/docs/en/setup/backend/dynamic-config.md @@ -7,7 +7,8 @@ Right now, SkyWalking supports following dynamic configurations. | Config Key | Value Description | Value Format Example | |:----:|:----:|:----:| |receiver-trace.default.slowDBAccessThreshold| Thresholds of slow Database statement, override `receiver-trace/default/slowDBAccessThreshold` of `applciation.yml`. | default:200,mongodb:50| -|receiver-trace.default.uninstrumentedGateways| The uninstrumented gateways, override `gateways.yml`. | not set | +|receiver-trace.default.uninstrumentedGateways| The uninstrumented gateways, override `gateways.yml`. | same as [`gateways.yml`](uninstrumented-gateways.md#configuration-format) | +|alarm.default.alarm-settings| The alarm settings, will override `alarm-settings.yml`. | same as [`alarm-settings.yml`](backend-alarm.md) | This feature depends on upstream service, so it is **OFF** as default. diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCore.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCore.java index 852629ded..5a4525e7e 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCore.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCore.java @@ -18,11 +18,17 @@ package org.apache.skywalking.oap.server.core.alarm.provider; -import java.util.*; -import java.util.concurrent.*; -import org.apache.skywalking.oap.server.core.alarm.*; -import org.joda.time.*; -import org.slf4j.*; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.apache.skywalking.oap.server.core.alarm.AlarmCallback; +import org.apache.skywalking.oap.server.core.alarm.AlarmMessage; +import org.joda.time.LocalDateTime; +import org.joda.time.Minutes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Alarm core includes metrics values in certain time windows based on alarm settings. By using its internal timer @@ -33,24 +39,15 @@ import org.slf4j.*; public class AlarmCore { private static final Logger logger = LoggerFactory.getLogger(AlarmCore.class); - private Map> runningContext; private LocalDateTime lastExecuteTime; + private AlarmRulesWatcher alarmRulesWatcher; - AlarmCore(Rules rules) { - runningContext = new HashMap<>(); - rules.getRules().forEach(rule -> { - RunningRule runningRule = new RunningRule(rule); - - String metricsName = rule.getMetricsName(); - - List runningRules = runningContext.computeIfAbsent(metricsName, key -> new ArrayList<>()); - - runningRules.add(runningRule); - }); + AlarmCore(AlarmRulesWatcher alarmRulesWatcher) { + this.alarmRulesWatcher = alarmRulesWatcher; } public List findRunningRule(String metricsName) { - return runningContext.get(metricsName); + return alarmRulesWatcher.getRunningContext().get(metricsName); } public void start(List allCallbacks) { @@ -62,10 +59,10 @@ public class AlarmCore { LocalDateTime checkTime = LocalDateTime.now(); int minutes = Minutes.minutesBetween(lastExecuteTime, checkTime).getMinutes(); boolean[] hasExecute = new boolean[] {false}; - runningContext.values().forEach(ruleList -> ruleList.forEach(runningRule -> { + alarmRulesWatcher.getRunningContext().values().forEach(ruleList -> ruleList.forEach(runningRule -> { if (minutes > 0) { runningRule.moveTo(checkTime); - /** + /* * Don't run in the first quarter per min, avoid to trigger false alarm. */ if (checkTime.getSecondOfMinute() > 15) { diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmModuleProvider.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmModuleProvider.java index 6039b8035..5cdfcf7f0 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmModuleProvider.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmModuleProvider.java @@ -19,6 +19,9 @@ package org.apache.skywalking.oap.server.core.alarm.provider; import java.io.*; + +import org.apache.skywalking.oap.server.configuration.api.ConfigurationModule; +import org.apache.skywalking.oap.server.configuration.api.DynamicConfigurationService; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.alarm.*; import org.apache.skywalking.oap.server.library.module.*; @@ -27,6 +30,7 @@ import org.apache.skywalking.oap.server.library.util.ResourceUtils; public class AlarmModuleProvider extends ModuleProvider { private NotifyHandler notifyHandler; + private AlarmRulesWatcher alarmRulesWatcher; @Override public String name() { return "default"; @@ -49,12 +53,17 @@ public class AlarmModuleProvider extends ModuleProvider { } RulesReader reader = new RulesReader(applicationReader); Rules rules = reader.readRules(); - notifyHandler = new NotifyHandler(rules); + + alarmRulesWatcher = new AlarmRulesWatcher(rules, this); + + notifyHandler = new NotifyHandler(alarmRulesWatcher); notifyHandler.init(new AlarmStandardPersistence()); this.registerServiceImplementation(MetricsNotify.class, notifyHandler); } @Override public void start() throws ServiceNotProvidedException, ModuleStartException { + DynamicConfigurationService dynamicConfigurationService = getManager().find(ConfigurationModule.NAME).provider().getService(DynamicConfigurationService.class); + dynamicConfigurationService.registerConfigChangeWatcher(alarmRulesWatcher); } @Override public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException { @@ -62,6 +71,6 @@ public class AlarmModuleProvider extends ModuleProvider { } @Override public String[] requiredModules() { - return new String[] {CoreModule.NAME}; + return new String[] {CoreModule.NAME, ConfigurationModule.NAME}; } } diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRule.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRule.java index 8d55aadd4..bf077cf42 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRule.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRule.java @@ -19,13 +19,21 @@ package org.apache.skywalking.oap.server.core.alarm.provider; import java.util.ArrayList; +import java.util.Objects; + import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; /** * @author wusheng */ +@Builder +@NoArgsConstructor +@AllArgsConstructor @Setter(AccessLevel.PUBLIC) @Getter(AccessLevel.PUBLIC) public class AlarmRule { @@ -39,4 +47,32 @@ public class AlarmRule { private int count; private int silencePeriod; private String message; + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + final AlarmRule alarmRule = (AlarmRule) o; + + return period == alarmRule.period + && count == alarmRule.count + && silencePeriod == alarmRule.silencePeriod + && Objects.equals(alarmRuleName, alarmRule.alarmRuleName) + && Objects.equals(metricsName, alarmRule.metricsName) + && Objects.equals(includeNames, alarmRule.includeNames) + && Objects.equals(threshold, alarmRule.threshold) + && Objects.equals(op, alarmRule.op) + && Objects.equals(message, alarmRule.message); + } + + @Override + public int hashCode() { + return Objects.hash(alarmRuleName, metricsName, includeNames, threshold, op, period, count, silencePeriod, message); + } } diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRulesWatcher.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRulesWatcher.java new file mode 100644 index 000000000..abfecc9f5 --- /dev/null +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRulesWatcher.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.alarm.provider; + +import lombok.Getter; +import org.apache.skywalking.oap.server.configuration.api.ConfigChangeWatcher; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.alarm.AlarmModule; +import org.apache.skywalking.oap.server.library.module.ModuleProvider; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Alarm rules' settings can be dynamically updated via configuration center(s), + * this class is responsible for monitoring the configuration and parsing them + * into {@link Rules} and {@link #runningContext}. + * + * @author kezhenxu94 + * @since 6.5.0 + */ +public class AlarmRulesWatcher extends ConfigChangeWatcher { + @Getter + private volatile Map> runningContext; + private volatile Map alarmRuleRunningRuleMap; + private volatile Rules rules; + private volatile String settingsString; + + public AlarmRulesWatcher(Rules defaultRules, ModuleProvider provider) { + super(AlarmModule.NAME, provider, "alarm-settings"); + this.runningContext = new HashMap<>(); + this.alarmRuleRunningRuleMap = new HashMap<>(); + this.settingsString = Const.EMPTY_STRING; + + notify(defaultRules); + } + + @Override + public void notify(ConfigChangeEvent value) { + if (value.getEventType() == EventType.DELETE) { + settingsString = Const.EMPTY_STRING; + notify(new Rules()); + } else { + settingsString = value.getNewValue(); + RulesReader rulesReader = new RulesReader(new StringReader(value.getNewValue())); + Rules rules = rulesReader.readRules(); + notify(rules); + } + } + + void notify(Rules newRules) { + Map newAlarmRuleRunningRuleMap = new HashMap<>(); + Map> newRunningContext = new HashMap<>(); + + newRules.getRules().forEach(rule -> { + /* + * If there is already an alarm rule that is the same as the new one, we'll reuse its + * corresponding runningRule, to keep its history metrics + */ + RunningRule runningRule = alarmRuleRunningRuleMap.getOrDefault(rule, new RunningRule(rule)); + + newAlarmRuleRunningRuleMap.put(rule, runningRule); + + String metricsName = rule.getMetricsName(); + + List runningRules = newRunningContext.computeIfAbsent(metricsName, key -> new ArrayList<>()); + + runningRules.add(runningRule); + }); + + this.rules = newRules; + this.runningContext = newRunningContext; + this.alarmRuleRunningRuleMap = newAlarmRuleRunningRuleMap; + } + + @Override + public String value() { + return settingsString; + } + + public List getRules() { + return this.rules.getRules(); + } + + public List getWebHooks() { + return this.rules.getWebhooks(); + } +} diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java index 3341cf895..8fbedcb8c 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java @@ -33,11 +33,11 @@ public class NotifyHandler implements MetricsNotify { private EndpointInventoryCache endpointInventoryCache; private final AlarmCore core; - private final Rules rules; + private final AlarmRulesWatcher alarmRulesWatcher; - public NotifyHandler(Rules rules) { - this.rules = rules; - core = new AlarmCore(rules); + public NotifyHandler(AlarmRulesWatcher alarmRulesWatcher) { + this.alarmRulesWatcher = alarmRulesWatcher; + core = new AlarmCore(alarmRulesWatcher); } @Override public void notify(Metrics metrics) { @@ -95,11 +95,8 @@ public class NotifyHandler implements MetricsNotify { } public void init(AlarmCallback... callbacks) { - List allCallbacks = new ArrayList<>(); - for (AlarmCallback callback : callbacks) { - allCallbacks.add(callback); - } - allCallbacks.add(new WebhookCallback(rules.getWebhooks())); + List allCallbacks = new ArrayList<>(Arrays.asList(callbacks)); + allCallbacks.add(new WebhookCallback(alarmRulesWatcher)); core.start(allCallbacks); } diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java index d9873e3ac..0d6f81e83 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java @@ -18,15 +18,11 @@ package org.apache.skywalking.oap.server.core.alarm.provider; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.ReentrantLock; import org.apache.skywalking.oap.server.core.alarm.AlarmMessage; import org.apache.skywalking.oap.server.core.alarm.MetaInAlarm; -import org.apache.skywalking.oap.server.core.analysis.metrics.*; +import org.apache.skywalking.oap.server.core.analysis.metrics.DoubleValueHolder; +import org.apache.skywalking.oap.server.core.analysis.metrics.IntValueHolder; +import org.apache.skywalking.oap.server.core.analysis.metrics.LongValueHolder; import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.library.util.CollectionUtils; import org.joda.time.LocalDateTime; @@ -36,6 +32,13 @@ import org.joda.time.format.DateTimeFormatter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantLock; + /** * RunningRule represents each rule in running status. Based on the {@link AlarmRule} definition, * @@ -157,6 +160,8 @@ public class RunningRule { return alarmMessageList; } + + /** * A metrics window, based on {@link AlarmRule#period}. This window slides with time, just keeps the recent * N(period) buckets. diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java index 2dff24178..6b04ac08e 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java @@ -18,17 +18,16 @@ package org.apache.skywalking.oap.server.core.alarm.provider; -import com.google.gson.Gson; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.nio.charset.StandardCharsets; import java.util.List; +import com.google.gson.Gson; import io.netty.handler.codec.http.HttpHeaderValues; import org.apache.http.HttpHeaders; import org.apache.http.HttpStatus; import org.apache.http.StatusLine; -import org.apache.http.client.ClientProtocolException; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; @@ -51,12 +50,12 @@ public class WebhookCallback implements AlarmCallback { private static final int HTTP_CONNECTION_REQUEST_TIMEOUT = 1000; private static final int HTTP_SOCKET_TIMEOUT = 10000; - private List remoteEndpoints; + private AlarmRulesWatcher alarmRulesWatcher; private RequestConfig requestConfig; private Gson gson = new Gson(); - public WebhookCallback(List remoteEndpoints) { - this.remoteEndpoints = remoteEndpoints; + public WebhookCallback(AlarmRulesWatcher alarmRulesWatcher) { + this.alarmRulesWatcher = alarmRulesWatcher; requestConfig = RequestConfig.custom() .setConnectTimeout(HTTP_CONNECT_TIMEOUT) .setConnectionRequestTimeout(HTTP_CONNECTION_REQUEST_TIMEOUT) @@ -65,13 +64,13 @@ public class WebhookCallback implements AlarmCallback { @Override public void doAlarm(List alarmMessage) { - if (remoteEndpoints.size() == 0) { + if (alarmRulesWatcher.getWebHooks().size() == 0) { return; } CloseableHttpClient httpClient = HttpClients.custom().build(); try { - remoteEndpoints.forEach(url -> { + alarmRulesWatcher.getWebHooks().forEach(url -> { HttpPost post = new HttpPost(url); post.setConfig(requestConfig); post.setHeader(HttpHeaders.ACCEPT, HttpHeaderValues.APPLICATION_JSON.toString()); @@ -88,8 +87,6 @@ public class WebhookCallback implements AlarmCallback { } } catch (UnsupportedEncodingException e) { logger.error("Alarm to JSON error, " + e.getMessage(), e); - } catch (ClientProtocolException e) { - logger.error("send alarm to " + url + " failure.", e); } catch (IOException e) { logger.error("send alarm to " + url + " failure.", e); } diff --git a/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCoreTest.java b/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCoreTest.java index e91b4224a..91402b5fc 100644 --- a/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCoreTest.java +++ b/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCoreTest.java @@ -50,7 +50,7 @@ public class AlarmCoreTest { Rules emptyRules = new Rules(); emptyRules.setRules(new ArrayList<>(0)); emptyRules.setWebhooks(new ArrayList<>(0)); - AlarmCore core = new AlarmCore(emptyRules); + AlarmCore core = new AlarmCore(new AlarmRulesWatcher(emptyRules, null)); Map> runningContext = Whitebox.getInternalState(core, "runningContext"); diff --git a/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmModuleProviderTest.java b/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmModuleProviderTest.java index 31bb8d53a..264036cbb 100644 --- a/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmModuleProviderTest.java +++ b/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmModuleProviderTest.java @@ -17,6 +17,7 @@ */ package org.apache.skywalking.oap.server.core.alarm.provider; +import org.apache.skywalking.oap.server.configuration.api.ConfigurationModule; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.alarm.AlarmModule; import org.apache.skywalking.oap.server.library.module.ModuleProvider; @@ -62,11 +63,6 @@ public class AlarmModuleProviderTest { assertEquals(AlarmModule.class, moduleProvider.module()); } - @Test - public void start() throws Exception { - moduleProvider.start(); - } - @Test public void notifyAfterCompleted() throws Exception { @@ -81,6 +77,6 @@ public class AlarmModuleProviderTest { @Test public void requiredModules() { String[] modules = moduleProvider.requiredModules(); - assertArrayEquals(new String[]{CoreModule.NAME}, modules); + assertArrayEquals(new String[]{CoreModule.NAME, ConfigurationModule.NAME}, modules); } } \ No newline at end of file diff --git a/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRulesWatcherTest.java b/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRulesWatcherTest.java new file mode 100644 index 000000000..572d590f8 --- /dev/null +++ b/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRulesWatcherTest.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.alarm.provider; + +import org.apache.skywalking.oap.server.configuration.api.ConfigChangeWatcher; +import org.apache.skywalking.oap.server.library.util.ResourceUtils; +import org.junit.Before; +import org.junit.Test; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; + +import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.spy; + +/** + * @author kezhenxu94 + */ +public class AlarmRulesWatcherTest { + @Spy + private AlarmRulesWatcher alarmRulesWatcher = new AlarmRulesWatcher(new Rules(), null); + + private AlarmRule.AlarmRuleBuilder rulePrototypeBuilder = AlarmRule.builder() + .alarmRuleName("name1") + .count(1) + .includeNames(new ArrayList() { + { + add("1"); + add("2"); + } + }) + .message("test") + .metricsName("metrics1") + .op(">") + .period(1) + .silencePeriod(2) + .threshold("2"); + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + } + + @Test + public void shouldSetAlarmRulesOnEventChanged() throws IOException { + assertTrue(alarmRulesWatcher.getRules().isEmpty()); + + Reader reader = ResourceUtils.read("alarm-settings.yml"); + char[] chars = new char[1024 * 1024]; + int length = reader.read(chars); + + alarmRulesWatcher.notify(new ConfigChangeWatcher.ConfigChangeEvent(new String(chars, 0, length), ConfigChangeWatcher.EventType.MODIFY)); + + assertEquals(2, alarmRulesWatcher.getRules().size()); + assertEquals(2, alarmRulesWatcher.getWebHooks().size()); + assertEquals(2, alarmRulesWatcher.getRunningContext().size()); + } + + @Test + public void shouldClearAlarmRulesOnEventDeleted() throws IOException { + Reader reader = ResourceUtils.read("alarm-settings.yml"); + Rules defaultRules = new RulesReader(reader).readRules(); + + alarmRulesWatcher = spy(new AlarmRulesWatcher(defaultRules, null)); + + alarmRulesWatcher.notify(new ConfigChangeWatcher.ConfigChangeEvent("whatever", ConfigChangeWatcher.EventType.DELETE)); + + assertEquals(0, alarmRulesWatcher.getRules().size()); + assertEquals(0, alarmRulesWatcher.getWebHooks().size()); + assertEquals(0, alarmRulesWatcher.getRunningContext().size()); + } + + @Test + public void shouldKeepExistedRunningRuleIfAlarmRuleExists() { + AlarmRule.AlarmRuleBuilder ruleBuilder = rulePrototypeBuilder; + + AlarmRule rule = ruleBuilder.build(); + Rules rules = new Rules(); + rules.getRules().add(rule); + + alarmRulesWatcher = spy(new AlarmRulesWatcher(rules, null)); + assertEquals(1, alarmRulesWatcher.getRunningContext().size()); + assertEquals(1, alarmRulesWatcher.getRunningContext().get(rule.getMetricsName()).size()); + + RunningRule runningRule = alarmRulesWatcher.getRunningContext().get(rule.getMetricsName()).get(0); + + Rules updatedRules = new Rules(); + updatedRules.getRules().addAll(Arrays.asList(rule, ruleBuilder.alarmRuleName("name2").build())); + + alarmRulesWatcher.notify(updatedRules); + + assertEquals(1, alarmRulesWatcher.getRunningContext().size()); + assertEquals(2, alarmRulesWatcher.getRunningContext().get(rule.getMetricsName()).size()); + assertEquals( + "The same alarm rule should map to the same existed running rule", + runningRule, alarmRulesWatcher.getRunningContext().get(rule.getMetricsName()).get(0) + ); + } + + @Test + public void shouldRemoveRunningRuleIfAlarmRuleIsRemoved() { + AlarmRule.AlarmRuleBuilder ruleBuilder = rulePrototypeBuilder; + + AlarmRule rule = ruleBuilder.build(); + Rules rules = new Rules(); + rules.getRules().add(rule); + + alarmRulesWatcher = spy(new AlarmRulesWatcher(rules, null)); + assertEquals(1, alarmRulesWatcher.getRunningContext().size()); + assertEquals(1, alarmRulesWatcher.getRunningContext().get(rule.getMetricsName()).size()); + + RunningRule runningRule = alarmRulesWatcher.getRunningContext().get(rule.getMetricsName()).get(0); + + Rules updatedRules = new Rules(); + updatedRules.getRules().add(ruleBuilder.alarmRuleName("name2").build()); + + alarmRulesWatcher.notify(updatedRules); + + assertEquals(1, alarmRulesWatcher.getRunningContext().size()); + assertEquals(1, alarmRulesWatcher.getRunningContext().get(rule.getMetricsName()).size()); + assertNotEquals( + "The new alarm rule should map to a different running rule", + runningRule, alarmRulesWatcher.getRunningContext().get(rule.getMetricsName()).get(0) + ); + } + + @Test + public void shouldReplaceRunningRuleIfAlarmRulesAreReplaced() { + AlarmRule.AlarmRuleBuilder ruleBuilder = rulePrototypeBuilder; + + AlarmRule rule = ruleBuilder.build(); + Rules rules = new Rules(); + rules.getRules().add(rule); + + alarmRulesWatcher = spy(new AlarmRulesWatcher(rules, null)); + assertEquals(1, alarmRulesWatcher.getRunningContext().size()); + assertEquals(1, alarmRulesWatcher.getRunningContext().get(rule.getMetricsName()).size()); + + Rules updatedRules = new Rules(); + // replace the original alarm rules + updatedRules.getRules().addAll( + Arrays.asList( + ruleBuilder.alarmRuleName("name2").metricsName("metrics2").build(), + ruleBuilder.alarmRuleName("name3").metricsName("metrics3").build() + ) + ); + + alarmRulesWatcher.notify(updatedRules); + + assertEquals(2, alarmRulesWatcher.getRunningContext().size()); + assertNull(alarmRulesWatcher.getRunningContext().get("metrics1")); + assertEquals(1, alarmRulesWatcher.getRunningContext().get("metrics2").size()); + assertEquals(1, alarmRulesWatcher.getRunningContext().get("metrics3").size()); + } +} \ No newline at end of file diff --git a/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandlerTest.java b/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandlerTest.java index 3e8b34b8b..8a7382a67 100644 --- a/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandlerTest.java +++ b/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandlerTest.java @@ -197,7 +197,7 @@ public class NotifyHandlerTest { Rules rules = new Rules(); - notifyHandler = new NotifyHandler(rules); + notifyHandler = new NotifyHandler(new AlarmRulesWatcher(rules, null)); notifyHandler.init(alarmMessageList -> { for (AlarmMessage message : alarmMessageList) { diff --git a/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallbackTest.java b/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallbackTest.java index 097a77fe2..bf56e6560 100644 --- a/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallbackTest.java +++ b/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallbackTest.java @@ -18,25 +18,43 @@ package org.apache.skywalking.oap.server.core.alarm.provider; -import com.google.gson.*; -import java.io.*; +import javax.servlet.Servlet; +import javax.servlet.ServletConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; import java.net.InetSocketAddress; -import java.util.*; -import javax.servlet.*; -import javax.servlet.http.*; +import java.util.ArrayList; +import java.util.List; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; import org.apache.skywalking.oap.server.core.alarm.AlarmMessage; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.servlet.*; -import org.junit.*; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; public class WebhookCallbackTest implements Servlet { private Server server; + private int port; private volatile boolean isSuccess = false; @Before public void init() throws Exception { - server = new Server(new InetSocketAddress("127.0.0.1", 8778)); + + server = new Server(new InetSocketAddress("127.0.0.1", 0)); ServletContextHandler servletContextHandler = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); servletContextHandler.setContextPath("/webhook"); @@ -47,6 +65,10 @@ public class WebhookCallbackTest implements Servlet { servletContextHandler.addServlet(servletHolder, "/receiveAlarm"); server.start(); + + port = server.getURI().getPort(); + + assertTrue(port > 0); } @After @@ -57,8 +79,11 @@ public class WebhookCallbackTest implements Servlet { @Test public void testWebhook() { List remoteEndpoints = new ArrayList<>(); - remoteEndpoints.add("http://127.0.0.1:8778/webhook/receiveAlarm"); - WebhookCallback webhookCallback = new WebhookCallback(remoteEndpoints); + remoteEndpoints.add("http://127.0.0.1:" + port + "/webhook/receiveAlarm"); + Rules rules = new Rules(); + rules.setWebhooks(remoteEndpoints); + AlarmRulesWatcher alarmRulesWatcher = new AlarmRulesWatcher(rules, null); + WebhookCallback webhookCallback = new WebhookCallback(alarmRulesWatcher); List alarmMessages = new ArrayList<>(2); AlarmMessage alarmMessage = new AlarmMessage(); alarmMessage.setScopeId(DefaultScopeDefine.ALL);