kazusa
/
skywalking-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make endpoint and instance alarm active (#1987) 

* Support alarm in service instance and endpoint.

* Fix alarm default settings and document.

* Fix wrong default setting

* Make service instance and endpoint scopes available in alarm core.

* Fix alarm bug.
This commit is contained in:
吴晟 Wu Sheng 2018-12-01 14:18:31 +08:00 committed by 彭勇升 pengys
parent 5777103a2b
commit 2d8f390351
8 changed files with 173 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
docs/en/setup/backend/backend-alarm.md
View File

@ -49,9 +49,18 @@ rules:
count: 4
```
## Default alarm rules
We provided a default `alarm-setting.yml` in our distribution only for convenience, which including following rules
1. Service average response time over 1s in last 3 minutes.
1. Service success rate lower than 80% in last 2 minutes.
1. Service 90% response time is lower than 1000ms in last 3 minutes
1. Service Instance average response time over 1s in last 2 minutes.
1. Endpoint average response time over 1s in last 2 minutes.
## List of all potential metric name
The metric names are defined in official [OAL scripts](../../guides/backend-oal-scripts.md), right now
only metric from **Service** scope could be used in Alarm, we will extend in further versions.
metric from **Service**, **Service Instance**, **Endpoint** scopes could be used in Alarm, we will extend in further versions.
Submit issue or pull request if you want to support any other scope in alarm.

12
oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCore.java
View File

@ -69,21 +69,25 @@ public class AlarmCore {
Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> {
try {
List<AlarmMessage> alarmMessageList = new ArrayList<>(30);
LocalDateTime checkTime = LocalDateTime.now();
int minutes = Minutes.minutesBetween(lastExecuteTime, checkTime).getMinutes();
boolean[] hasExecute = new boolean[] {false};
runningContext.values().forEach(ruleList -> ruleList.forEach(runningRule -> {
LocalDateTime checkTime = LocalDateTime.now();
int minutes = Minutes.minutesBetween(lastExecuteTime, checkTime).getMinutes();
if (minutes > 0) {
hasExecute[0] = true;
runningRule.moveTo(checkTime);
/**
* Don't run in the first quarter per min, avoid to trigger false alarm.
*/
if (checkTime.getSecondOfMinute() > 15) {
alarmMessageList.addAll(runningRule.check());
// Set the last execute time, and make sure the second is `00`, such as: 18:30:00
lastExecuteTime = checkTime.minusSeconds(checkTime.getSecondOfMinute());
}
}
}));
// Set the last execute time, and make sure the second is `00`, such as: 18:30:00
if (hasExecute[0]) {
lastExecuteTime = checkTime.minusSeconds(checkTime.getSecondOfMinute());
}
if (alarmMessageList.size() > 0) {
allCallbacks.forEach(callback -> callback.doAlarm(alarmMessageList));

4
oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java
View File

@ -38,6 +38,10 @@ public class NotifyHandler implements IndicatorNotify {
switch (meta.getScope()) {
case Service:
break;
case ServiceInstance:
break;
case Endpoint:
break;
default:
return;
}

11
oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java
View File

@ -118,14 +118,9 @@ public class RunningRule {
Window window = windows.get(meta);
if (window == null) {
window = new Window(period);
Window ifAbsent = windows.putIfAbsent(meta, window);
if (ifAbsent == null) {
LocalDateTime timebucket = TIME_BUCKET_FORMATTER.parseLocalDateTime(indicator.getTimeBucket() + "");
window.moveTo(timebucket);
} else {
window = windows.get(meta);
}
LocalDateTime timebucket = TIME_BUCKET_FORMATTER.parseLocalDateTime(indicator.getTimeBucket() + "");
window.moveTo(timebucket);
windows.put(meta, window);
}
window.add(indicator);

34
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmEntrance.java
View File

@ -21,7 +21,11 @@ package org.apache.skywalking.oap.server.core.alarm;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.analysis.indicator.Indicator;
import org.apache.skywalking.oap.server.core.cache.EndpointInventoryCache;
import org.apache.skywalking.oap.server.core.cache.ServiceInstanceInventoryCache;
import org.apache.skywalking.oap.server.core.cache.ServiceInventoryCache;
import org.apache.skywalking.oap.server.core.register.EndpointInventory;
import org.apache.skywalking.oap.server.core.register.ServiceInstanceInventory;
import org.apache.skywalking.oap.server.core.register.ServiceInventory;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
@ -31,6 +35,8 @@ import org.apache.skywalking.oap.server.library.module.ModuleManager;
public class AlarmEntrance {
private ModuleManager moduleManager;
private ServiceInventoryCache serviceInventoryCache;
private ServiceInstanceInventoryCache serviceInstanceInventoryCache;
private EndpointInventoryCache endpointInventoryCache;
private IndicatorNotify indicatorNotify;
private ReentrantLock initLock;
@ -48,7 +54,7 @@ public class AlarmEntrance {
AlarmMeta alarmMeta = ((AlarmSupported)indicator).getAlarmMeta();
MetaInAlarm metaInAlarm = null;
MetaInAlarm metaInAlarm;
switch (alarmMeta.getScope()) {
case Service:
int serviceId = Integer.parseInt(alarmMeta.getId());
@ -59,6 +65,30 @@ public class AlarmEntrance {
serviceMetaInAlarm.setName(serviceInventory.getName());
metaInAlarm = serviceMetaInAlarm;
break;
case ServiceInstance:
int serviceInstanceId = Integer.parseInt(alarmMeta.getId());
ServiceInstanceInventory serviceInstanceInventory = serviceInstanceInventoryCache.get(serviceInstanceId);
ServiceInstanceMetaInAlarm instanceMetaInAlarm = new ServiceInstanceMetaInAlarm();
instanceMetaInAlarm.setIndicatorName(alarmMeta.getIndicatorName());
instanceMetaInAlarm.setId(serviceInstanceId);
instanceMetaInAlarm.setName(serviceInstanceInventory.getName());
metaInAlarm = instanceMetaInAlarm;
break;
case Endpoint:
int endpointId = Integer.parseInt(alarmMeta.getId());
EndpointInventory endpointInventory = endpointInventoryCache.get(endpointId);
EndpointMetaInAlarm endpointMetaInAlarm = new EndpointMetaInAlarm();
endpointMetaInAlarm.setIndicatorName(alarmMeta.getIndicatorName());
endpointMetaInAlarm.setId(endpointId);
serviceId = endpointInventory.getServiceId();
serviceInventory = serviceInventoryCache.get(serviceId);
String textName = endpointInventory.getName() + " in " + serviceInventory.getName();
endpointMetaInAlarm.setName(textName);
metaInAlarm = endpointMetaInAlarm;
break;
default:
return;
}
@ -72,6 +102,8 @@ public class AlarmEntrance {
try {
if (serviceInventoryCache == null) {
serviceInventoryCache = moduleManager.find(CoreModule.NAME).provider().getService(ServiceInventoryCache.class);
serviceInstanceInventoryCache = moduleManager.find(CoreModule.NAME).provider().getService(ServiceInstanceInventoryCache.class);
endpointInventoryCache = moduleManager.find(CoreModule.NAME).provider().getService(EndpointInventoryCache.class);
indicatorNotify = moduleManager.find(AlarmModule.NAME).provider().getService(IndicatorNotify.class);
indicatorNotify.init(new AlarmStandardPersistence());
}

47
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/EndpointMetaInAlarm.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.alarm;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.Setter;
import org.apache.skywalking.oap.server.core.source.Scope;
@Getter(AccessLevel.PUBLIC)
@Setter(AccessLevel.PUBLIC)
public class EndpointMetaInAlarm extends MetaInAlarm {
private String indicatorName;
private int id;
private String name;
private String[] tags;
private String[] properties;
@Override public Scope getScope() {
return Scope.Endpoint;
}
@Override public int getId0() {
return id;
}
@Override public int getId1() {
return 0;
}
}

47
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/ServiceInstanceMetaInAlarm.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.alarm;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.Setter;
import org.apache.skywalking.oap.server.core.source.Scope;
@Getter(AccessLevel.PUBLIC)
@Setter(AccessLevel.PUBLIC)
public class ServiceInstanceMetaInAlarm extends MetaInAlarm {
private String indicatorName;
private int id;
private String name;
private String[] tags;
private String[] properties;
@Override public Scope getScope() {
return Scope.ServiceInstance;
}
@Override public int getId0() {
return id;
}
@Override public int getId1() {
return 0;
}
}

26
oap-server/server-starter/src/main/assembly/alarm-settings.yml
View File

@ -24,28 +24,44 @@ rules:
period: 10
count: 3
silence-period: 5
message: Response time of service {name} is more than 2000ms.
message: Response time of service {name} is more than 1000ms in last 3 minutes.
service_sla_rule:
# Indicator value need to be long, double or int
indicator-name: service_sla
op: "<"
threshold: 80
threshold: 8000
# The length of time to evaluate the metric
period: 10
# How many times after the metric match the condition, will trigger alarm
count: 2
# How many times of checks, the alarm keeps silence after alarm triggered, default as same as period.
silence-period: 3
message: Successful rate of service {name} is lower than 80%
message: Successful rate of service {name} is lower than 80% in last 2 minutes.
service_p90_sla_rule:
# Indicator value need to be long, double or int
indicator-name: service_sla
indicator-name: service_p90
op: ">"
threshold: 1000
period: 10
count: 3
silence-period: 5
message: 90% response time of service {name} is lower than 80%
message: 90% response time of service {name} is lower than 1000ms in last 3 minutes
service_instance_resp_time_rule:
indicator-name: service_instance_resp_time
op: ">"
threshold: 1000
period: 10
count: 2
silence-period: 5
message: Response time of service instance {name} is more than 1000ms in last 2 minutes.
endpoint_avg_rule:
indicator-name: endpoint_avg
op: ">"
threshold: 1000
period: 10
count: 2
silence-period: 5
message: Response time of endpoint {name} is more than 1000ms in last 2 minutes.
webhooks:
# - http://127.0.0.1/notify/