Support dynamically configure alarm settings (#3557)
* Support dynamically configure alarm settings * Update documentation
This commit is contained in:
kezhenxu94
吴晟 Wu Sheng
parent
81f4c087b0
commit
2801de752f
|
|
@ -25,6 +25,7 @@ pipeline {
|
|||
))
|
||||
timestamps()
|
||||
skipStagesAfterUnstable()
|
||||
timeout(time: 5, unit: 'HOURS')
|
||||
}
|
||||
|
||||
environment {
|
||||
|
|
|
|||
|
|
@ -21,6 +21,11 @@ pipeline {
|
|||
label 'skywalking'
|
||||
}
|
||||
|
||||
options {
|
||||
timestamps()
|
||||
timeout(time: 5, unit: 'HOURS')
|
||||
}
|
||||
|
||||
tools {
|
||||
jdk 'JDK 1.8 (latest)'
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# Alarm
|
||||
Alarm core is driven a collection of rules, which are defined in `config/alarm-settings.yml`.
|
||||
Alarm core is driven by a collection of rules, which are defined in `config/alarm-settings.yml`.
|
||||
There are two parts in alarm rule definition.
|
||||
1. [Alarm rules](#rules). They define how metrics alarm should be triggered, what conditions should be considered.
|
||||
1. [Webhooks](#webhook). The list of web service endpoint, which should be called after the alarm is triggered.
|
||||
|
|
@ -90,3 +90,11 @@ Example as following
|
|||
"startTime": 1560524171000
|
||||
}]
|
||||
```
|
||||
|
||||
## Update the settings dynamically
|
||||
Since 6.5.0, the alarm settings can be updated dynamically at runtime by [Dynamic Configuration](dynamic-config.md),
|
||||
which will override the settings in `alarm-settings.yml`.
|
||||
|
||||
In order to determine that whether an alarm rule is triggered or not, SkyWalking needs to cache the metrics of a time window for
|
||||
each alarm rule, if any attribute (`metrics-name`, `op`, `threshold`, `period`, `count`, etc.) of a rule is changed,
|
||||
the sliding window will be destroyed and re-created, causing the alarm of this specific rule to restart again.
|
||||
|
|
@ -7,7 +7,8 @@ Right now, SkyWalking supports following dynamic configurations.
|
|||
| Config Key | Value Description | Value Format Example |
|
||||
|:----:|:----:|:----:|
|
||||
|receiver-trace.default.slowDBAccessThreshold| Thresholds of slow Database statement, override `receiver-trace/default/slowDBAccessThreshold` of `applciation.yml`. | default:200,mongodb:50|
|
||||
|receiver-trace.default.uninstrumentedGateways| The uninstrumented gateways, override `gateways.yml`. | not set |
|
||||
|receiver-trace.default.uninstrumentedGateways| The uninstrumented gateways, override `gateways.yml`. | same as [`gateways.yml`](uninstrumented-gateways.md#configuration-format) |
|
||||
|alarm.default.alarm-settings| The alarm settings, will override `alarm-settings.yml`. | same as [`alarm-settings.yml`](backend-alarm.md) |
|
||||
|
||||
|
||||
This feature depends on upstream service, so it is **OFF** as default.
|
||||
|
|
|
|||
|
|
@ -18,11 +18,17 @@
|
|||
|
||||
package org.apache.skywalking.oap.server.core.alarm.provider;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.concurrent.*;
|
||||
import org.apache.skywalking.oap.server.core.alarm.*;
|
||||
import org.joda.time.*;
|
||||
import org.slf4j.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.skywalking.oap.server.core.alarm.AlarmCallback;
|
||||
import org.apache.skywalking.oap.server.core.alarm.AlarmMessage;
|
||||
import org.joda.time.LocalDateTime;
|
||||
import org.joda.time.Minutes;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Alarm core includes metrics values in certain time windows based on alarm settings. By using its internal timer
|
||||
|
|
@ -33,24 +39,15 @@ import org.slf4j.*;
|
|||
public class AlarmCore {
|
||||
private static final Logger logger = LoggerFactory.getLogger(AlarmCore.class);
|
||||
|
||||
private Map<String, List<RunningRule>> runningContext;
|
||||
private LocalDateTime lastExecuteTime;
|
||||
private AlarmRulesWatcher alarmRulesWatcher;
|
||||
|
||||
AlarmCore(Rules rules) {
|
||||
runningContext = new HashMap<>();
|
||||
rules.getRules().forEach(rule -> {
|
||||
RunningRule runningRule = new RunningRule(rule);
|
||||
|
||||
String metricsName = rule.getMetricsName();
|
||||
|
||||
List<RunningRule> runningRules = runningContext.computeIfAbsent(metricsName, key -> new ArrayList<>());
|
||||
|
||||
runningRules.add(runningRule);
|
||||
});
|
||||
AlarmCore(AlarmRulesWatcher alarmRulesWatcher) {
|
||||
this.alarmRulesWatcher = alarmRulesWatcher;
|
||||
}
|
||||
|
||||
public List<RunningRule> findRunningRule(String metricsName) {
|
||||
return runningContext.get(metricsName);
|
||||
return alarmRulesWatcher.getRunningContext().get(metricsName);
|
||||
}
|
||||
|
||||
public void start(List<AlarmCallback> allCallbacks) {
|
||||
|
|
@ -62,10 +59,10 @@ public class AlarmCore {
|
|||
LocalDateTime checkTime = LocalDateTime.now();
|
||||
int minutes = Minutes.minutesBetween(lastExecuteTime, checkTime).getMinutes();
|
||||
boolean[] hasExecute = new boolean[] {false};
|
||||
runningContext.values().forEach(ruleList -> ruleList.forEach(runningRule -> {
|
||||
alarmRulesWatcher.getRunningContext().values().forEach(ruleList -> ruleList.forEach(runningRule -> {
|
||||
if (minutes > 0) {
|
||||
runningRule.moveTo(checkTime);
|
||||
/**
|
||||
/*
|
||||
* Don't run in the first quarter per min, avoid to trigger false alarm.
|
||||
*/
|
||||
if (checkTime.getSecondOfMinute() > 15) {
|
||||
|
|
|
|||
|
|
@ -19,6 +19,9 @@
|
|||
package org.apache.skywalking.oap.server.core.alarm.provider;
|
||||
|
||||
import java.io.*;
|
||||
|
||||
import org.apache.skywalking.oap.server.configuration.api.ConfigurationModule;
|
||||
import org.apache.skywalking.oap.server.configuration.api.DynamicConfigurationService;
|
||||
import org.apache.skywalking.oap.server.core.CoreModule;
|
||||
import org.apache.skywalking.oap.server.core.alarm.*;
|
||||
import org.apache.skywalking.oap.server.library.module.*;
|
||||
|
|
@ -27,6 +30,7 @@ import org.apache.skywalking.oap.server.library.util.ResourceUtils;
|
|||
public class AlarmModuleProvider extends ModuleProvider {
|
||||
|
||||
private NotifyHandler notifyHandler;
|
||||
private AlarmRulesWatcher alarmRulesWatcher;
|
||||
|
||||
@Override public String name() {
|
||||
return "default";
|
||||
|
|
@ -49,12 +53,17 @@ public class AlarmModuleProvider extends ModuleProvider {
|
|||
}
|
||||
RulesReader reader = new RulesReader(applicationReader);
|
||||
Rules rules = reader.readRules();
|
||||
notifyHandler = new NotifyHandler(rules);
|
||||
|
||||
alarmRulesWatcher = new AlarmRulesWatcher(rules, this);
|
||||
|
||||
notifyHandler = new NotifyHandler(alarmRulesWatcher);
|
||||
notifyHandler.init(new AlarmStandardPersistence());
|
||||
this.registerServiceImplementation(MetricsNotify.class, notifyHandler);
|
||||
}
|
||||
|
||||
@Override public void start() throws ServiceNotProvidedException, ModuleStartException {
|
||||
DynamicConfigurationService dynamicConfigurationService = getManager().find(ConfigurationModule.NAME).provider().getService(DynamicConfigurationService.class);
|
||||
dynamicConfigurationService.registerConfigChangeWatcher(alarmRulesWatcher);
|
||||
}
|
||||
|
||||
@Override public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException {
|
||||
|
|
@ -62,6 +71,6 @@ public class AlarmModuleProvider extends ModuleProvider {
|
|||
}
|
||||
|
||||
@Override public String[] requiredModules() {
|
||||
return new String[] {CoreModule.NAME};
|
||||
return new String[] {CoreModule.NAME, ConfigurationModule.NAME};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,13 +19,21 @@
|
|||
package org.apache.skywalking.oap.server.core.alarm.provider;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Objects;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* @author wusheng
|
||||
*/
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Setter(AccessLevel.PUBLIC)
|
||||
@Getter(AccessLevel.PUBLIC)
|
||||
public class AlarmRule {
|
||||
|
|
@ -39,4 +47,32 @@ public class AlarmRule {
|
|||
private int count;
|
||||
private int silencePeriod;
|
||||
private String message;
|
||||
|
||||
@Override
|
||||
public boolean equals(final Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
final AlarmRule alarmRule = (AlarmRule) o;
|
||||
|
||||
return period == alarmRule.period
|
||||
&& count == alarmRule.count
|
||||
&& silencePeriod == alarmRule.silencePeriod
|
||||
&& Objects.equals(alarmRuleName, alarmRule.alarmRuleName)
|
||||
&& Objects.equals(metricsName, alarmRule.metricsName)
|
||||
&& Objects.equals(includeNames, alarmRule.includeNames)
|
||||
&& Objects.equals(threshold, alarmRule.threshold)
|
||||
&& Objects.equals(op, alarmRule.op)
|
||||
&& Objects.equals(message, alarmRule.message);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(alarmRuleName, metricsName, includeNames, threshold, op, period, count, silencePeriod, message);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,107 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.apache.skywalking.oap.server.core.alarm.provider;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.apache.skywalking.oap.server.configuration.api.ConfigChangeWatcher;
|
||||
import org.apache.skywalking.oap.server.core.Const;
|
||||
import org.apache.skywalking.oap.server.core.alarm.AlarmModule;
|
||||
import org.apache.skywalking.oap.server.library.module.ModuleProvider;
|
||||
|
||||
import java.io.StringReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Alarm rules' settings can be dynamically updated via configuration center(s),
|
||||
* this class is responsible for monitoring the configuration and parsing them
|
||||
* into {@link Rules} and {@link #runningContext}.
|
||||
*
|
||||
* @author kezhenxu94
|
||||
* @since 6.5.0
|
||||
*/
|
||||
public class AlarmRulesWatcher extends ConfigChangeWatcher {
|
||||
@Getter
|
||||
private volatile Map<String, List<RunningRule>> runningContext;
|
||||
private volatile Map<AlarmRule, RunningRule> alarmRuleRunningRuleMap;
|
||||
private volatile Rules rules;
|
||||
private volatile String settingsString;
|
||||
|
||||
public AlarmRulesWatcher(Rules defaultRules, ModuleProvider provider) {
|
||||
super(AlarmModule.NAME, provider, "alarm-settings");
|
||||
this.runningContext = new HashMap<>();
|
||||
this.alarmRuleRunningRuleMap = new HashMap<>();
|
||||
this.settingsString = Const.EMPTY_STRING;
|
||||
|
||||
notify(defaultRules);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void notify(ConfigChangeEvent value) {
|
||||
if (value.getEventType() == EventType.DELETE) {
|
||||
settingsString = Const.EMPTY_STRING;
|
||||
notify(new Rules());
|
||||
} else {
|
||||
settingsString = value.getNewValue();
|
||||
RulesReader rulesReader = new RulesReader(new StringReader(value.getNewValue()));
|
||||
Rules rules = rulesReader.readRules();
|
||||
notify(rules);
|
||||
}
|
||||
}
|
||||
|
||||
void notify(Rules newRules) {
|
||||
Map<AlarmRule, RunningRule> newAlarmRuleRunningRuleMap = new HashMap<>();
|
||||
Map<String, List<RunningRule>> newRunningContext = new HashMap<>();
|
||||
|
||||
newRules.getRules().forEach(rule -> {
|
||||
/*
|
||||
* If there is already an alarm rule that is the same as the new one, we'll reuse its
|
||||
* corresponding runningRule, to keep its history metrics
|
||||
*/
|
||||
RunningRule runningRule = alarmRuleRunningRuleMap.getOrDefault(rule, new RunningRule(rule));
|
||||
|
||||
newAlarmRuleRunningRuleMap.put(rule, runningRule);
|
||||
|
||||
String metricsName = rule.getMetricsName();
|
||||
|
||||
List<RunningRule> runningRules = newRunningContext.computeIfAbsent(metricsName, key -> new ArrayList<>());
|
||||
|
||||
runningRules.add(runningRule);
|
||||
});
|
||||
|
||||
this.rules = newRules;
|
||||
this.runningContext = newRunningContext;
|
||||
this.alarmRuleRunningRuleMap = newAlarmRuleRunningRuleMap;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String value() {
|
||||
return settingsString;
|
||||
}
|
||||
|
||||
public List<AlarmRule> getRules() {
|
||||
return this.rules.getRules();
|
||||
}
|
||||
|
||||
public List<String> getWebHooks() {
|
||||
return this.rules.getWebhooks();
|
||||
}
|
||||
}
|
||||
|
|
@ -33,11 +33,11 @@ public class NotifyHandler implements MetricsNotify {
|
|||
private EndpointInventoryCache endpointInventoryCache;
|
||||
|
||||
private final AlarmCore core;
|
||||
private final Rules rules;
|
||||
private final AlarmRulesWatcher alarmRulesWatcher;
|
||||
|
||||
public NotifyHandler(Rules rules) {
|
||||
this.rules = rules;
|
||||
core = new AlarmCore(rules);
|
||||
public NotifyHandler(AlarmRulesWatcher alarmRulesWatcher) {
|
||||
this.alarmRulesWatcher = alarmRulesWatcher;
|
||||
core = new AlarmCore(alarmRulesWatcher);
|
||||
}
|
||||
|
||||
@Override public void notify(Metrics metrics) {
|
||||
|
|
@ -95,11 +95,8 @@ public class NotifyHandler implements MetricsNotify {
|
|||
}
|
||||
|
||||
public void init(AlarmCallback... callbacks) {
|
||||
List<AlarmCallback> allCallbacks = new ArrayList<>();
|
||||
for (AlarmCallback callback : callbacks) {
|
||||
allCallbacks.add(callback);
|
||||
}
|
||||
allCallbacks.add(new WebhookCallback(rules.getWebhooks()));
|
||||
List<AlarmCallback> allCallbacks = new ArrayList<>(Arrays.asList(callbacks));
|
||||
allCallbacks.add(new WebhookCallback(alarmRulesWatcher));
|
||||
core.start(allCallbacks);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,15 +18,11 @@
|
|||
|
||||
package org.apache.skywalking.oap.server.core.alarm.provider;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import org.apache.skywalking.oap.server.core.alarm.AlarmMessage;
|
||||
import org.apache.skywalking.oap.server.core.alarm.MetaInAlarm;
|
||||
import org.apache.skywalking.oap.server.core.analysis.metrics.*;
|
||||
import org.apache.skywalking.oap.server.core.analysis.metrics.DoubleValueHolder;
|
||||
import org.apache.skywalking.oap.server.core.analysis.metrics.IntValueHolder;
|
||||
import org.apache.skywalking.oap.server.core.analysis.metrics.LongValueHolder;
|
||||
import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
|
||||
import org.apache.skywalking.oap.server.library.util.CollectionUtils;
|
||||
import org.joda.time.LocalDateTime;
|
||||
|
|
@ -36,6 +32,13 @@ import org.joda.time.format.DateTimeFormatter;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
/**
|
||||
* RunningRule represents each rule in running status. Based on the {@link AlarmRule} definition,
|
||||
*
|
||||
|
|
@ -157,6 +160,8 @@ public class RunningRule {
|
|||
return alarmMessageList;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* A metrics window, based on {@link AlarmRule#period}. This window slides with time, just keeps the recent
|
||||
* N(period) buckets.
|
||||
|
|
|
|||
|
|
@ -18,17 +18,16 @@
|
|||
|
||||
package org.apache.skywalking.oap.server.core.alarm.provider;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import io.netty.handler.codec.http.HttpHeaderValues;
|
||||
import org.apache.http.HttpHeaders;
|
||||
import org.apache.http.HttpStatus;
|
||||
import org.apache.http.StatusLine;
|
||||
import org.apache.http.client.ClientProtocolException;
|
||||
import org.apache.http.client.config.RequestConfig;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
|
|
@ -51,12 +50,12 @@ public class WebhookCallback implements AlarmCallback {
|
|||
private static final int HTTP_CONNECTION_REQUEST_TIMEOUT = 1000;
|
||||
private static final int HTTP_SOCKET_TIMEOUT = 10000;
|
||||
|
||||
private List<String> remoteEndpoints;
|
||||
private AlarmRulesWatcher alarmRulesWatcher;
|
||||
private RequestConfig requestConfig;
|
||||
private Gson gson = new Gson();
|
||||
|
||||
public WebhookCallback(List<String> remoteEndpoints) {
|
||||
this.remoteEndpoints = remoteEndpoints;
|
||||
public WebhookCallback(AlarmRulesWatcher alarmRulesWatcher) {
|
||||
this.alarmRulesWatcher = alarmRulesWatcher;
|
||||
requestConfig = RequestConfig.custom()
|
||||
.setConnectTimeout(HTTP_CONNECT_TIMEOUT)
|
||||
.setConnectionRequestTimeout(HTTP_CONNECTION_REQUEST_TIMEOUT)
|
||||
|
|
@ -65,13 +64,13 @@ public class WebhookCallback implements AlarmCallback {
|
|||
|
||||
@Override
|
||||
public void doAlarm(List<AlarmMessage> alarmMessage) {
|
||||
if (remoteEndpoints.size() == 0) {
|
||||
if (alarmRulesWatcher.getWebHooks().size() == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
CloseableHttpClient httpClient = HttpClients.custom().build();
|
||||
try {
|
||||
remoteEndpoints.forEach(url -> {
|
||||
alarmRulesWatcher.getWebHooks().forEach(url -> {
|
||||
HttpPost post = new HttpPost(url);
|
||||
post.setConfig(requestConfig);
|
||||
post.setHeader(HttpHeaders.ACCEPT, HttpHeaderValues.APPLICATION_JSON.toString());
|
||||
|
|
@ -88,8 +87,6 @@ public class WebhookCallback implements AlarmCallback {
|
|||
}
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
logger.error("Alarm to JSON error, " + e.getMessage(), e);
|
||||
} catch (ClientProtocolException e) {
|
||||
logger.error("send alarm to " + url + " failure.", e);
|
||||
} catch (IOException e) {
|
||||
logger.error("send alarm to " + url + " failure.", e);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ public class AlarmCoreTest {
|
|||
Rules emptyRules = new Rules();
|
||||
emptyRules.setRules(new ArrayList<>(0));
|
||||
emptyRules.setWebhooks(new ArrayList<>(0));
|
||||
AlarmCore core = new AlarmCore(emptyRules);
|
||||
AlarmCore core = new AlarmCore(new AlarmRulesWatcher(emptyRules, null));
|
||||
|
||||
Map<String, List<RunningRule>> runningContext = Whitebox.getInternalState(core, "runningContext");
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
*/
|
||||
package org.apache.skywalking.oap.server.core.alarm.provider;
|
||||
|
||||
import org.apache.skywalking.oap.server.configuration.api.ConfigurationModule;
|
||||
import org.apache.skywalking.oap.server.core.CoreModule;
|
||||
import org.apache.skywalking.oap.server.core.alarm.AlarmModule;
|
||||
import org.apache.skywalking.oap.server.library.module.ModuleProvider;
|
||||
|
|
@ -62,11 +63,6 @@ public class AlarmModuleProviderTest {
|
|||
assertEquals(AlarmModule.class, moduleProvider.module());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void start() throws Exception {
|
||||
moduleProvider.start();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void notifyAfterCompleted() throws Exception {
|
||||
|
||||
|
|
@ -81,6 +77,6 @@ public class AlarmModuleProviderTest {
|
|||
@Test
|
||||
public void requiredModules() {
|
||||
String[] modules = moduleProvider.requiredModules();
|
||||
assertArrayEquals(new String[]{CoreModule.NAME}, modules);
|
||||
assertArrayEquals(new String[]{CoreModule.NAME, ConfigurationModule.NAME}, modules);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,178 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.apache.skywalking.oap.server.core.alarm.provider;
|
||||
|
||||
import org.apache.skywalking.oap.server.configuration.api.ConfigChangeWatcher;
|
||||
import org.apache.skywalking.oap.server.library.util.ResourceUtils;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.mockito.MockitoAnnotations;
|
||||
import org.mockito.Spy;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotEquals;
|
||||
import static org.junit.Assert.assertNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.mockito.Mockito.spy;
|
||||
|
||||
/**
|
||||
* @author kezhenxu94
|
||||
*/
|
||||
public class AlarmRulesWatcherTest {
|
||||
@Spy
|
||||
private AlarmRulesWatcher alarmRulesWatcher = new AlarmRulesWatcher(new Rules(), null);
|
||||
|
||||
private AlarmRule.AlarmRuleBuilder rulePrototypeBuilder = AlarmRule.builder()
|
||||
.alarmRuleName("name1")
|
||||
.count(1)
|
||||
.includeNames(new ArrayList<String>() {
|
||||
{
|
||||
add("1");
|
||||
add("2");
|
||||
}
|
||||
})
|
||||
.message("test")
|
||||
.metricsName("metrics1")
|
||||
.op(">")
|
||||
.period(1)
|
||||
.silencePeriod(2)
|
||||
.threshold("2");
|
||||
|
||||
@Before
|
||||
public void setUp() {
|
||||
MockitoAnnotations.initMocks(this);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldSetAlarmRulesOnEventChanged() throws IOException {
|
||||
assertTrue(alarmRulesWatcher.getRules().isEmpty());
|
||||
|
||||
Reader reader = ResourceUtils.read("alarm-settings.yml");
|
||||
char[] chars = new char[1024 * 1024];
|
||||
int length = reader.read(chars);
|
||||
|
||||
alarmRulesWatcher.notify(new ConfigChangeWatcher.ConfigChangeEvent(new String(chars, 0, length), ConfigChangeWatcher.EventType.MODIFY));
|
||||
|
||||
assertEquals(2, alarmRulesWatcher.getRules().size());
|
||||
assertEquals(2, alarmRulesWatcher.getWebHooks().size());
|
||||
assertEquals(2, alarmRulesWatcher.getRunningContext().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldClearAlarmRulesOnEventDeleted() throws IOException {
|
||||
Reader reader = ResourceUtils.read("alarm-settings.yml");
|
||||
Rules defaultRules = new RulesReader(reader).readRules();
|
||||
|
||||
alarmRulesWatcher = spy(new AlarmRulesWatcher(defaultRules, null));
|
||||
|
||||
alarmRulesWatcher.notify(new ConfigChangeWatcher.ConfigChangeEvent("whatever", ConfigChangeWatcher.EventType.DELETE));
|
||||
|
||||
assertEquals(0, alarmRulesWatcher.getRules().size());
|
||||
assertEquals(0, alarmRulesWatcher.getWebHooks().size());
|
||||
assertEquals(0, alarmRulesWatcher.getRunningContext().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldKeepExistedRunningRuleIfAlarmRuleExists() {
|
||||
AlarmRule.AlarmRuleBuilder ruleBuilder = rulePrototypeBuilder;
|
||||
|
||||
AlarmRule rule = ruleBuilder.build();
|
||||
Rules rules = new Rules();
|
||||
rules.getRules().add(rule);
|
||||
|
||||
alarmRulesWatcher = spy(new AlarmRulesWatcher(rules, null));
|
||||
assertEquals(1, alarmRulesWatcher.getRunningContext().size());
|
||||
assertEquals(1, alarmRulesWatcher.getRunningContext().get(rule.getMetricsName()).size());
|
||||
|
||||
RunningRule runningRule = alarmRulesWatcher.getRunningContext().get(rule.getMetricsName()).get(0);
|
||||
|
||||
Rules updatedRules = new Rules();
|
||||
updatedRules.getRules().addAll(Arrays.asList(rule, ruleBuilder.alarmRuleName("name2").build()));
|
||||
|
||||
alarmRulesWatcher.notify(updatedRules);
|
||||
|
||||
assertEquals(1, alarmRulesWatcher.getRunningContext().size());
|
||||
assertEquals(2, alarmRulesWatcher.getRunningContext().get(rule.getMetricsName()).size());
|
||||
assertEquals(
|
||||
"The same alarm rule should map to the same existed running rule",
|
||||
runningRule, alarmRulesWatcher.getRunningContext().get(rule.getMetricsName()).get(0)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldRemoveRunningRuleIfAlarmRuleIsRemoved() {
|
||||
AlarmRule.AlarmRuleBuilder ruleBuilder = rulePrototypeBuilder;
|
||||
|
||||
AlarmRule rule = ruleBuilder.build();
|
||||
Rules rules = new Rules();
|
||||
rules.getRules().add(rule);
|
||||
|
||||
alarmRulesWatcher = spy(new AlarmRulesWatcher(rules, null));
|
||||
assertEquals(1, alarmRulesWatcher.getRunningContext().size());
|
||||
assertEquals(1, alarmRulesWatcher.getRunningContext().get(rule.getMetricsName()).size());
|
||||
|
||||
RunningRule runningRule = alarmRulesWatcher.getRunningContext().get(rule.getMetricsName()).get(0);
|
||||
|
||||
Rules updatedRules = new Rules();
|
||||
updatedRules.getRules().add(ruleBuilder.alarmRuleName("name2").build());
|
||||
|
||||
alarmRulesWatcher.notify(updatedRules);
|
||||
|
||||
assertEquals(1, alarmRulesWatcher.getRunningContext().size());
|
||||
assertEquals(1, alarmRulesWatcher.getRunningContext().get(rule.getMetricsName()).size());
|
||||
assertNotEquals(
|
||||
"The new alarm rule should map to a different running rule",
|
||||
runningRule, alarmRulesWatcher.getRunningContext().get(rule.getMetricsName()).get(0)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldReplaceRunningRuleIfAlarmRulesAreReplaced() {
|
||||
AlarmRule.AlarmRuleBuilder ruleBuilder = rulePrototypeBuilder;
|
||||
|
||||
AlarmRule rule = ruleBuilder.build();
|
||||
Rules rules = new Rules();
|
||||
rules.getRules().add(rule);
|
||||
|
||||
alarmRulesWatcher = spy(new AlarmRulesWatcher(rules, null));
|
||||
assertEquals(1, alarmRulesWatcher.getRunningContext().size());
|
||||
assertEquals(1, alarmRulesWatcher.getRunningContext().get(rule.getMetricsName()).size());
|
||||
|
||||
Rules updatedRules = new Rules();
|
||||
// replace the original alarm rules
|
||||
updatedRules.getRules().addAll(
|
||||
Arrays.asList(
|
||||
ruleBuilder.alarmRuleName("name2").metricsName("metrics2").build(),
|
||||
ruleBuilder.alarmRuleName("name3").metricsName("metrics3").build()
|
||||
)
|
||||
);
|
||||
|
||||
alarmRulesWatcher.notify(updatedRules);
|
||||
|
||||
assertEquals(2, alarmRulesWatcher.getRunningContext().size());
|
||||
assertNull(alarmRulesWatcher.getRunningContext().get("metrics1"));
|
||||
assertEquals(1, alarmRulesWatcher.getRunningContext().get("metrics2").size());
|
||||
assertEquals(1, alarmRulesWatcher.getRunningContext().get("metrics3").size());
|
||||
}
|
||||
}
|
||||
|
|
@ -197,7 +197,7 @@ public class NotifyHandlerTest {
|
|||
|
||||
Rules rules = new Rules();
|
||||
|
||||
notifyHandler = new NotifyHandler(rules);
|
||||
notifyHandler = new NotifyHandler(new AlarmRulesWatcher(rules, null));
|
||||
|
||||
notifyHandler.init(alarmMessageList -> {
|
||||
for (AlarmMessage message : alarmMessageList) {
|
||||
|
|
|
|||
|
|
@ -18,25 +18,43 @@
|
|||
|
||||
package org.apache.skywalking.oap.server.core.alarm.provider;
|
||||
|
||||
import com.google.gson.*;
|
||||
import java.io.*;
|
||||
import javax.servlet.Servlet;
|
||||
import javax.servlet.ServletConfig;
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.ServletRequest;
|
||||
import javax.servlet.ServletResponse;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.util.*;
|
||||
import javax.servlet.*;
|
||||
import javax.servlet.http.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.JsonArray;
|
||||
import org.apache.skywalking.oap.server.core.alarm.AlarmMessage;
|
||||
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
|
||||
import org.eclipse.jetty.server.Server;
|
||||
import org.eclipse.jetty.servlet.*;
|
||||
import org.junit.*;
|
||||
import org.eclipse.jetty.servlet.ServletContextHandler;
|
||||
import org.eclipse.jetty.servlet.ServletHolder;
|
||||
import org.junit.After;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
public class WebhookCallbackTest implements Servlet {
|
||||
private Server server;
|
||||
private int port;
|
||||
private volatile boolean isSuccess = false;
|
||||
|
||||
@Before
|
||||
public void init() throws Exception {
|
||||
server = new Server(new InetSocketAddress("127.0.0.1", 8778));
|
||||
|
||||
server = new Server(new InetSocketAddress("127.0.0.1", 0));
|
||||
ServletContextHandler servletContextHandler = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
|
||||
servletContextHandler.setContextPath("/webhook");
|
||||
|
||||
|
|
@ -47,6 +65,10 @@ public class WebhookCallbackTest implements Servlet {
|
|||
servletContextHandler.addServlet(servletHolder, "/receiveAlarm");
|
||||
|
||||
server.start();
|
||||
|
||||
port = server.getURI().getPort();
|
||||
|
||||
assertTrue(port > 0);
|
||||
}
|
||||
|
||||
@After
|
||||
|
|
@ -57,8 +79,11 @@ public class WebhookCallbackTest implements Servlet {
|
|||
@Test
|
||||
public void testWebhook() {
|
||||
List<String> remoteEndpoints = new ArrayList<>();
|
||||
remoteEndpoints.add("http://127.0.0.1:8778/webhook/receiveAlarm");
|
||||
WebhookCallback webhookCallback = new WebhookCallback(remoteEndpoints);
|
||||
remoteEndpoints.add("http://127.0.0.1:" + port + "/webhook/receiveAlarm");
|
||||
Rules rules = new Rules();
|
||||
rules.setWebhooks(remoteEndpoints);
|
||||
AlarmRulesWatcher alarmRulesWatcher = new AlarmRulesWatcher(rules, null);
|
||||
WebhookCallback webhookCallback = new WebhookCallback(alarmRulesWatcher);
|
||||
List<AlarmMessage> alarmMessages = new ArrayList<>(2);
|
||||
AlarmMessage alarmMessage = new AlarmMessage();
|
||||
alarmMessage.setScopeId(DefaultScopeDefine.ALL);
|
||||
|
|
|
|||
Loading…
Reference in New Issue