diff --git a/docs/en/setup/backend/backend-alarm.md b/docs/en/setup/backend/backend-alarm.md index c4ff6f57c..aa22b9f6f 100644 --- a/docs/en/setup/backend/backend-alarm.md +++ b/docs/en/setup/backend/backend-alarm.md @@ -49,9 +49,18 @@ rules: count: 4 ``` +## Default alarm rules +We provided a default `alarm-setting.yml` in our distribution only for convenience, which including following rules +1. Service average response time over 1s in last 3 minutes. +1. Service success rate lower than 80% in last 2 minutes. +1. Service 90% response time is lower than 1000ms in last 3 minutes +1. Service Instance average response time over 1s in last 2 minutes. +1. Endpoint average response time over 1s in last 2 minutes. + + ## List of all potential metric name The metric names are defined in official [OAL scripts](../../guides/backend-oal-scripts.md), right now -only metric from **Service** scope could be used in Alarm, we will extend in further versions. +metric from **Service**, **Service Instance**, **Endpoint** scopes could be used in Alarm, we will extend in further versions. Submit issue or pull request if you want to support any other scope in alarm. diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCore.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCore.java index e76591db5..b43e18cff 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCore.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCore.java @@ -69,21 +69,25 @@ public class AlarmCore { Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> { try { List alarmMessageList = new ArrayList<>(30); + LocalDateTime checkTime = LocalDateTime.now(); + int minutes = Minutes.minutesBetween(lastExecuteTime, checkTime).getMinutes(); + boolean[] hasExecute = new boolean[] {false}; runningContext.values().forEach(ruleList -> ruleList.forEach(runningRule -> { - LocalDateTime checkTime = LocalDateTime.now(); - int minutes = Minutes.minutesBetween(lastExecuteTime, checkTime).getMinutes(); if (minutes > 0) { + hasExecute[0] = true; runningRule.moveTo(checkTime); /** * Don't run in the first quarter per min, avoid to trigger false alarm. */ if (checkTime.getSecondOfMinute() > 15) { alarmMessageList.addAll(runningRule.check()); - // Set the last execute time, and make sure the second is `00`, such as: 18:30:00 - lastExecuteTime = checkTime.minusSeconds(checkTime.getSecondOfMinute()); } } })); + // Set the last execute time, and make sure the second is `00`, such as: 18:30:00 + if (hasExecute[0]) { + lastExecuteTime = checkTime.minusSeconds(checkTime.getSecondOfMinute()); + } if (alarmMessageList.size() > 0) { allCallbacks.forEach(callback -> callback.doAlarm(alarmMessageList)); diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java index 1d4aacc51..408f9eedc 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java @@ -38,6 +38,10 @@ public class NotifyHandler implements IndicatorNotify { switch (meta.getScope()) { case Service: break; + case ServiceInstance: + break; + case Endpoint: + break; default: return; } diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java index a010392fa..dd0659d5b 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java @@ -118,14 +118,9 @@ public class RunningRule { Window window = windows.get(meta); if (window == null) { window = new Window(period); - Window ifAbsent = windows.putIfAbsent(meta, window); - if (ifAbsent == null) { - LocalDateTime timebucket = TIME_BUCKET_FORMATTER.parseLocalDateTime(indicator.getTimeBucket() + ""); - window.moveTo(timebucket); - } else { - window = windows.get(meta); - } - + LocalDateTime timebucket = TIME_BUCKET_FORMATTER.parseLocalDateTime(indicator.getTimeBucket() + ""); + window.moveTo(timebucket); + windows.put(meta, window); } window.add(indicator); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmEntrance.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmEntrance.java index 3992841d7..b2d0e4114 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmEntrance.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmEntrance.java @@ -21,7 +21,11 @@ package org.apache.skywalking.oap.server.core.alarm; import java.util.concurrent.locks.ReentrantLock; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.analysis.indicator.Indicator; +import org.apache.skywalking.oap.server.core.cache.EndpointInventoryCache; +import org.apache.skywalking.oap.server.core.cache.ServiceInstanceInventoryCache; import org.apache.skywalking.oap.server.core.cache.ServiceInventoryCache; +import org.apache.skywalking.oap.server.core.register.EndpointInventory; +import org.apache.skywalking.oap.server.core.register.ServiceInstanceInventory; import org.apache.skywalking.oap.server.core.register.ServiceInventory; import org.apache.skywalking.oap.server.library.module.ModuleManager; @@ -31,6 +35,8 @@ import org.apache.skywalking.oap.server.library.module.ModuleManager; public class AlarmEntrance { private ModuleManager moduleManager; private ServiceInventoryCache serviceInventoryCache; + private ServiceInstanceInventoryCache serviceInstanceInventoryCache; + private EndpointInventoryCache endpointInventoryCache; private IndicatorNotify indicatorNotify; private ReentrantLock initLock; @@ -48,7 +54,7 @@ public class AlarmEntrance { AlarmMeta alarmMeta = ((AlarmSupported)indicator).getAlarmMeta(); - MetaInAlarm metaInAlarm = null; + MetaInAlarm metaInAlarm; switch (alarmMeta.getScope()) { case Service: int serviceId = Integer.parseInt(alarmMeta.getId()); @@ -59,6 +65,30 @@ public class AlarmEntrance { serviceMetaInAlarm.setName(serviceInventory.getName()); metaInAlarm = serviceMetaInAlarm; break; + case ServiceInstance: + int serviceInstanceId = Integer.parseInt(alarmMeta.getId()); + ServiceInstanceInventory serviceInstanceInventory = serviceInstanceInventoryCache.get(serviceInstanceId); + ServiceInstanceMetaInAlarm instanceMetaInAlarm = new ServiceInstanceMetaInAlarm(); + instanceMetaInAlarm.setIndicatorName(alarmMeta.getIndicatorName()); + instanceMetaInAlarm.setId(serviceInstanceId); + instanceMetaInAlarm.setName(serviceInstanceInventory.getName()); + metaInAlarm = instanceMetaInAlarm; + break; + case Endpoint: + int endpointId = Integer.parseInt(alarmMeta.getId()); + EndpointInventory endpointInventory = endpointInventoryCache.get(endpointId); + EndpointMetaInAlarm endpointMetaInAlarm = new EndpointMetaInAlarm(); + endpointMetaInAlarm.setIndicatorName(alarmMeta.getIndicatorName()); + endpointMetaInAlarm.setId(endpointId); + + serviceId = endpointInventory.getServiceId(); + serviceInventory = serviceInventoryCache.get(serviceId); + + String textName = endpointInventory.getName() + " in " + serviceInventory.getName(); + + endpointMetaInAlarm.setName(textName); + metaInAlarm = endpointMetaInAlarm; + break; default: return; } @@ -72,6 +102,8 @@ public class AlarmEntrance { try { if (serviceInventoryCache == null) { serviceInventoryCache = moduleManager.find(CoreModule.NAME).provider().getService(ServiceInventoryCache.class); + serviceInstanceInventoryCache = moduleManager.find(CoreModule.NAME).provider().getService(ServiceInstanceInventoryCache.class); + endpointInventoryCache = moduleManager.find(CoreModule.NAME).provider().getService(EndpointInventoryCache.class); indicatorNotify = moduleManager.find(AlarmModule.NAME).provider().getService(IndicatorNotify.class); indicatorNotify.init(new AlarmStandardPersistence()); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/EndpointMetaInAlarm.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/EndpointMetaInAlarm.java new file mode 100644 index 000000000..10b859fdc --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/EndpointMetaInAlarm.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.alarm; + +import lombok.AccessLevel; +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.source.Scope; + +@Getter(AccessLevel.PUBLIC) +@Setter(AccessLevel.PUBLIC) +public class EndpointMetaInAlarm extends MetaInAlarm { + private String indicatorName; + + private int id; + private String name; + private String[] tags; + private String[] properties; + + @Override public Scope getScope() { + return Scope.Endpoint; + } + + @Override public int getId0() { + return id; + } + + @Override public int getId1() { + return 0; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/ServiceInstanceMetaInAlarm.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/ServiceInstanceMetaInAlarm.java new file mode 100644 index 000000000..4c8862ae0 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/ServiceInstanceMetaInAlarm.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.alarm; + +import lombok.AccessLevel; +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.source.Scope; + +@Getter(AccessLevel.PUBLIC) +@Setter(AccessLevel.PUBLIC) +public class ServiceInstanceMetaInAlarm extends MetaInAlarm { + private String indicatorName; + + private int id; + private String name; + private String[] tags; + private String[] properties; + + @Override public Scope getScope() { + return Scope.ServiceInstance; + } + + @Override public int getId0() { + return id; + } + + @Override public int getId1() { + return 0; + } +} diff --git a/oap-server/server-starter/src/main/assembly/alarm-settings.yml b/oap-server/server-starter/src/main/assembly/alarm-settings.yml index 5b839fb67..9ee76e2fe 100644 --- a/oap-server/server-starter/src/main/assembly/alarm-settings.yml +++ b/oap-server/server-starter/src/main/assembly/alarm-settings.yml @@ -24,28 +24,44 @@ rules: period: 10 count: 3 silence-period: 5 - message: Response time of service {name} is more than 2000ms. + message: Response time of service {name} is more than 1000ms in last 3 minutes. service_sla_rule: # Indicator value need to be long, double or int indicator-name: service_sla op: "<" - threshold: 80 + threshold: 8000 # The length of time to evaluate the metric period: 10 # How many times after the metric match the condition, will trigger alarm count: 2 # How many times of checks, the alarm keeps silence after alarm triggered, default as same as period. silence-period: 3 - message: Successful rate of service {name} is lower than 80% + message: Successful rate of service {name} is lower than 80% in last 2 minutes. service_p90_sla_rule: # Indicator value need to be long, double or int - indicator-name: service_sla + indicator-name: service_p90 op: ">" threshold: 1000 period: 10 count: 3 silence-period: 5 - message: 90% response time of service {name} is lower than 80% + message: 90% response time of service {name} is lower than 1000ms in last 3 minutes + service_instance_resp_time_rule: + indicator-name: service_instance_resp_time + op: ">" + threshold: 1000 + period: 10 + count: 2 + silence-period: 5 + message: Response time of service instance {name} is more than 1000ms in last 2 minutes. + endpoint_avg_rule: + indicator-name: endpoint_avg + op: ">" + threshold: 1000 + period: 10 + count: 2 + silence-period: 5 + message: Response time of endpoint {name} is more than 1000ms in last 2 minutes. webhooks: # - http://127.0.0.1/notify/