diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index e226fb0424..befc5aad67 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -70,6 +70,8 @@ * Add type descriptor when converting Envoy logs to JSON for persistence, to avoid conversion error. * Bseline: Support query baseline with MQE and use in the Alarm Rule. * Bump up netty to 4.11.118 to fix CVE-2025-24970. +* Add `Get Alarm Runtime Status` API. +* Add `lock` when query the Alarm metrics window values. #### UI diff --git a/docs/en/status/query_alarm_runtime_status.md b/docs/en/status/query_alarm_runtime_status.md new file mode 100644 index 0000000000..e3678e0df2 --- /dev/null +++ b/docs/en/status/query_alarm_runtime_status.md @@ -0,0 +1,161 @@ +# Get Alarm Runtime Status + +OAP calculates the alarm conditions in the memory based on the alarm rules and the metrics data. +The following APIs are exposed to make the alerting running kernel visible. + +## Get Alarm Running Rules + +Return the list of alarm running rules. + +- URL, `http://{core restHost}:{core restPort}/status/alarm/rules` +- HTTP GET method. + +```json +{ + "ruleNames": [ + "service_percentile_rule", + "service_resp_time_rule" + ] +} +``` + +## Get Alarm Running Rule Info + +Return the detailed information of the alarm running rule. + +- URL, `http://{core restHost}:{core restPort}/status/alarm/rules/{ruleName}` +- HTTP GET method. + +```json +{ + "ruleName": "service_resp_time_rule", + "expression": "sum(service_resp_time > baseline(service_resp_time,upper)) >= 1", + "period": 10, + "silentPeriod": 10, + "additonalPeriod": 0, + "includeNames": [ + "mock_a_service", + "mock_b_service", + "mock_c_service" + ], + "excludeNames": [], + "includeNamesRegex": "", + "excludeNamesRegex": "", + "affectedEntities": [ + { + "scope": "SERVICE", + "name": "mock_b_service" + }, + { + "scope": "SERVICE", + "name": "mock_a_service" + }, + { + "scope": "SERVICE", + "name": "mock_c_service" + } + ], + "tags": [ + { + "key": "level", + "value": "WARNING" + } + ], + "hooks": [ + "webhook.default", + "wechat.default" + ], + "includeMetrics": [ + "service_resp_time" + ], + "formattedMessages": [ + { + "mock_b_service": "Response time of service mock_b_service is more than upper baseline in 1 minutes of last 10 minutes." + }, + { + "mock_a_service": "Response time of service mock_a_service is more than upper baseline in 1 minutes of last 10 minutes." + }, + { + "mock_c_service": "Response time of service mock_c_service is more than upper baseline in 1 minutes of last 10 minutes." + } + ] +} +``` + +- `additonalPeriod` is the additional period if the expression includes the [increase/rate function](../api/metrics-query-expression.md#trend-operation). +This additional period is used to enlarge window size for calculating the trend value. +- `affectedEntities` is the entities that have metrics data and being calculated by the alarm rule. +- `formattedMessages` is the result message according to the message template and the affected entities. + +## Get Alarm Running Context + +Return the running context of the alarm rule. + +- URL, `http://{core restHost}:{core restPort}/status/alarm/{ruleName}/{entityName}` +- HTTP GET method. + +```json +{ + "expression": "sum(service_resp_time > baseline(service_resp_time,upper)) >= 1", + "endTime": "2025-02-12T13:39:00.000", + "additionalPeriod": 0, + "size": 10, + "silenceCountdown": 10, + "windowValues": [ + { + "index": 0, + "metrics": [] + }, + { + "index": 1, + "metrics": [] + }, + { + "index": 2, + "metrics": [] + }, + { + "index": 3, + "metrics": [] + }, + { + "index": 4, + "metrics": [] + }, + { + "index": 5, + "metrics": [] + }, + { + "index": 6, + "metrics": [] + }, + { + "index": 7, + "metrics": [ + { + "timeBucket": 202502121437, + "name": "service_resp_time", + "value": "6000" + } + ] + }, + { + "index": 8, + "metrics": [] + }, + { + "index": 9, + "metrics": [] + } + ], + "mqeMetricsSnapshot": { + "service_resp_time": "[{\"metric\":{\"labels\":[]},\"values\":[{\"id\":\"202502121430\",\"doubleValue\":0.0,\"isEmptyValue\":true},{\"id\":\"202502121431\",\"doubleValue\":0.0,\"isEmptyValue\":true},{\"id\":\"202502121432\",\"doubleValue\":0.0,\"isEmptyValue\":true},{\"id\":\"202502121433\",\"doubleValue\":0.0,\"isEmptyValue\":true},{\"id\":\"202502121434\",\"doubleValue\":0.0,\"isEmptyValue\":true},{\"id\":\"202502121435\",\"doubleValue\":0.0,\"isEmptyValue\":true},{\"id\":\"202502121436\",\"doubleValue\":0.0,\"isEmptyValue\":true},{\"id\":\"202502121437\",\"doubleValue\":6000.0,\"isEmptyValue\":false},{\"id\":\"202502121438\",\"doubleValue\":0.0,\"isEmptyValue\":true},{\"id\":\"202502121439\",\"doubleValue\":0.0,\"isEmptyValue\":true}]}]", + "baseline(service_resp_time,upper)": "[{\"metric\":{\"labels\":[]},\"values\":[{\"id\":\"202502121430\",\"doubleValue\":10.0,\"isEmptyValue\":false},{\"id\":\"202502121431\",\"doubleValue\":10.0,\"isEmptyValue\":false},{\"id\":\"202502121432\",\"doubleValue\":10.0,\"isEmptyValue\":false},{\"id\":\"202502121433\",\"doubleValue\":10.0,\"isEmptyValue\":false},{\"id\":\"202502121434\",\"doubleValue\":10.0,\"isEmptyValue\":false},{\"id\":\"202502121435\",\"doubleValue\":10.0,\"isEmptyValue\":false},{\"id\":\"202502121436\",\"doubleValue\":10.0,\"isEmptyValue\":false},{\"id\":\"202502121437\",\"doubleValue\":10.0,\"isEmptyValue\":false},{\"id\":\"202502121438\",\"doubleValue\":10.0,\"isEmptyValue\":false},{\"id\":\"202502121439\",\"doubleValue\":10.0,\"isEmptyValue\":false}]}]" + } +} +``` +`size` is the window size. Equal to the `period + additionalPeriod`. +`silenceCountdown` is the countdown of the silence period. -1 means silence countdown is not running. +`windowValues` is the original metrics data. The `index` is the index of the window, starting from 0. +`mqeMetricsSnapshot` is the metrics data in the MQE format. When checking conditions, these data will be calculated according to the expression. diff --git a/docs/en/status/status_apis.md b/docs/en/status/status_apis.md index 01cc939e72..67e144fae0 100644 --- a/docs/en/status/status_apis.md +++ b/docs/en/status/status_apis.md @@ -11,6 +11,7 @@ logs and self-observability solutions. - [Tracing Query Execution APIs](../debugging/query-tracing.md) - [Get Effective TTL Configurations API](query_ttl_setup.md) - [Query Cluster Nodes API](query_cluster_nodes.md) +- [Get Alarm Runtime Status API](query_alarm_runtime_status.md) If you have a proposal about new status API, please don't hesitate to [create a discussion](https://github.com/apache/skywalking/discussions/new?category=ideas). diff --git a/docs/menu.yml b/docs/menu.yml index a1416469ad..6643b4013b 100644 --- a/docs/menu.yml +++ b/docs/menu.yml @@ -346,6 +346,8 @@ catalog: path: "/en/status/query_ttl_setup" - name: "Get Node List in the Cluster" path: "/en/status/query_cluster_nodes" + - name: "Get Alarm Runtime Status" + path: "/en/status/query_alarm_runtime_status" - name: "Customization" catalog: - name: "Overview" diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmMessageFormatter.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmMessageFormatter.java index 87e5dc5fa3..55aa0c7589 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmMessageFormatter.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmMessageFormatter.java @@ -20,6 +20,7 @@ package org.apache.skywalking.oap.server.core.alarm.provider; import java.util.ArrayList; import java.util.List; +import lombok.Getter; /** * This is a formatter especially for alarm message. @@ -28,6 +29,7 @@ import java.util.List; *

* - Successful rate of endpoint {name} is lower than 75% */ +@Getter public class AlarmMessageFormatter { private List formatSegments; private List valueFroms; @@ -88,7 +90,7 @@ public class AlarmMessageFormatter { return message.toString(); } - private enum ValueFrom { + public enum ValueFrom { ID, NAME } } diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmModuleProvider.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmModuleProvider.java index c73054e785..8acac27b30 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmModuleProvider.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmModuleProvider.java @@ -20,6 +20,7 @@ package org.apache.skywalking.oap.server.core.alarm.provider; import java.io.FileNotFoundException; import java.io.Reader; +import lombok.Getter; import org.apache.skywalking.oap.server.configuration.api.ConfigurationModule; import org.apache.skywalking.oap.server.configuration.api.DynamicConfigurationService; import org.apache.skywalking.oap.server.core.CoreModule; @@ -35,6 +36,7 @@ import org.apache.skywalking.oap.server.library.util.ResourceUtils; public class AlarmModuleProvider extends ModuleProvider { private NotifyHandler notifyHandler; + @Getter private AlarmRulesWatcher alarmRulesWatcher; @Override diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java index dcf10cb816..d270611b88 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java @@ -29,8 +29,10 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; import java.util.regex.Pattern; import java.util.stream.Collectors; +import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; import lombok.extern.slf4j.Slf4j; @@ -68,6 +70,7 @@ import static org.apache.skywalking.oap.server.core.query.type.debugging.Debuggi * RunningRule represents each rule in running status. Based on the {@link AlarmRule} definition, */ @Slf4j +@Getter public class RunningRule { private static DateTimeFormatter TIME_BUCKET_FORMATTER = DateTimeFormat.forPattern("yyyyMMddHHmm"); @@ -243,12 +246,17 @@ public class RunningRule { * buckets. */ public class Window { + @Getter private LocalDateTime endTime; + @Getter private final int additionalPeriod; + @Getter private final int size; + @Getter private int silenceCountdown; private LinkedList> values; private ReentrantLock lock = new ReentrantLock(); + @Getter private JsonObject mqeMetricsSnapshot; private AlarmEntity entity; @@ -356,6 +364,7 @@ public class RunningRule { } private boolean isMatch() { + this.lock.lock(); int isMatch = 0; try { TRACE_CONTEXT.set(new DebuggingTraceContext(expression, false, false)); @@ -407,6 +416,7 @@ public class RunningRule { this.mqeMetricsSnapshot = visitor.getMqeMetricsSnapshot(); return isMatch == 1; } finally { + this.lock.unlock(); TRACE_CONTEXT.remove(); } } @@ -422,6 +432,15 @@ public class RunningRule { return true; } + public void scanWindowValues(Consumer>> scanFunction) { + lock.lock(); + try { + scanFunction.accept(values); + } finally { + lock.unlock(); + } + } + private void init() { values = new LinkedList<>(); for (int i = 0; i < size; i++) { diff --git a/oap-server/server-query-plugin/status-query-plugin/pom.xml b/oap-server/server-query-plugin/status-query-plugin/pom.xml index 67c410d184..89be571bf5 100644 --- a/oap-server/server-query-plugin/status-query-plugin/pom.xml +++ b/oap-server/server-query-plugin/status-query-plugin/pom.xml @@ -44,5 +44,10 @@ zipkin-query-plugin ${project.version} + + org.apache.skywalking + server-alarm-plugin + ${project.version} + diff --git a/oap-server/server-query-plugin/status-query-plugin/src/main/java/org/apache/skywalking/oap/query/debug/AlarmStatusQueryHandler.java b/oap-server/server-query-plugin/status-query-plugin/src/main/java/org/apache/skywalking/oap/query/debug/AlarmStatusQueryHandler.java new file mode 100644 index 0000000000..b277970306 --- /dev/null +++ b/oap-server/server-query-plugin/status-query-plugin/src/main/java/org/apache/skywalking/oap/query/debug/AlarmStatusQueryHandler.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.query.debug; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.linecorp.armeria.common.HttpResponse; +import com.linecorp.armeria.common.MediaType; +import com.linecorp.armeria.server.annotation.ExceptionHandler; +import com.linecorp.armeria.server.annotation.Get; +import com.linecorp.armeria.server.annotation.Param; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.alarm.AlarmModule; +import org.apache.skywalking.oap.server.core.alarm.provider.AlarmEntity; +import org.apache.skywalking.oap.server.core.alarm.provider.AlarmModuleProvider; +import org.apache.skywalking.oap.server.core.alarm.provider.AlarmRulesWatcher; +import org.apache.skywalking.oap.server.core.alarm.provider.RunningRule; +import org.apache.skywalking.oap.server.core.analysis.metrics.DoubleValueHolder; +import org.apache.skywalking.oap.server.core.analysis.metrics.IntValueHolder; +import org.apache.skywalking.oap.server.core.analysis.metrics.LabeledValueHolder; +import org.apache.skywalking.oap.server.core.analysis.metrics.LongValueHolder; +import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.library.module.ModuleManager; + +@Slf4j +@ExceptionHandler(StatusQueryExceptionHandler.class) +public class AlarmStatusQueryHandler { + private final Gson gson = new Gson(); + private final ModuleManager moduleManager; + private AlarmRulesWatcher alarmRulesWatcher; + + public AlarmStatusQueryHandler(final ModuleManager manager) { + this.moduleManager = manager; + } + + private AlarmRulesWatcher getAlarmRulesWatcher() { + if (alarmRulesWatcher == null) { + + AlarmModuleProvider provider = (AlarmModuleProvider) moduleManager.find(AlarmModule.NAME) + .provider(); + alarmRulesWatcher = provider.getAlarmRulesWatcher(); + } + return alarmRulesWatcher; + } + + @Get("/status/alarm/rules") + public HttpResponse getAlarmRules() { + Map runningRules = getAlarmRulesWatcher().getRunningContext().values().stream().map(List::stream) + .flatMap(r -> r).collect(Collectors.toMap(RunningRule::getRuleName, r -> r)); + JsonObject runningRuleNames = new JsonObject(); + JsonArray nameList = new JsonArray(); + runningRuleNames.add("ruleNames", nameList); + runningRules.keySet().forEach(nameList::add); + return HttpResponse.of(MediaType.JSON_UTF_8, gson.toJson(runningRuleNames)); + } + + @Get("/status/alarm/{ruleName}") + public HttpResponse getAlarmRuleByName(@Param("ruleName") String ruleName) { + Map runningRules = getAlarmRulesWatcher().getRunningContext().values().stream().flatMap(List::stream) + .collect(Collectors.toMap(RunningRule::getRuleName, r -> r)); + RunningRule rule = runningRules.get(ruleName); + JsonObject runningRuleInfo = new JsonObject(); + runningRuleInfo.addProperty("ruleName", rule.getRuleName()); + runningRuleInfo.addProperty("expression", rule.getExpression()); + runningRuleInfo.addProperty("period", rule.getPeriod()); + runningRuleInfo.addProperty("silentPeriod", rule.getSilencePeriod()); + runningRuleInfo.addProperty("additonalPeriod", rule.getAdditionalPeriod()); + + JsonArray includeNameList = new JsonArray(); + runningRuleInfo.add("includeNames", includeNameList); + rule.getIncludeNames().forEach(includeNameList::add); + + JsonArray excludeNameList = new JsonArray(); + runningRuleInfo.add("excludeNames", excludeNameList); + rule.getExcludeNames().forEach(excludeNameList::add); + + runningRuleInfo.addProperty("includeNamesRegex", rule.getExcludeNamesRegex() == null ? "" : rule.getIncludeNamesRegex().toString()); + runningRuleInfo.addProperty("excludeNamesRegex", rule.getExcludeNamesRegex() == null ? "" : rule.getExcludeNamesRegex().toString()); + + JsonArray affectedEntities = new JsonArray(); + runningRuleInfo.add("affectedEntities", affectedEntities); + JsonArray msgFormatter = new JsonArray(); + rule.getWindows().keySet().forEach(e -> { + JsonObject entity = new JsonObject(); + entity.addProperty("scope", e.getScope()); + entity.addProperty("name", e.getName()); + affectedEntities.add(entity); + JsonObject msg = new JsonObject(); + msg.addProperty(e.getName(), rule.getFormatter().format(e)); + msgFormatter.add(msg); + }); + + JsonArray tagList = new JsonArray(); + runningRuleInfo.add("tags", tagList); + rule.getTags().forEach(tag -> { + JsonObject tagInfo = new JsonObject(); + tagInfo.addProperty("key", tag.getKey()); + tagInfo.addProperty("value", tag.getValue()); + tagList.add(tagInfo); + }); + + JsonArray hookList = new JsonArray(); + runningRuleInfo.add("hooks", hookList); + rule.getHooks().forEach(hookList::add); + + JsonArray includeMetricList = new JsonArray(); + runningRuleInfo.add("includeMetrics", includeMetricList); + rule.getIncludeMetrics().forEach(includeMetricList::add); + + runningRuleInfo.add("formattedMessages", msgFormatter); + + return HttpResponse.of(MediaType.JSON_UTF_8, runningRuleInfo.toString()); + } + + @Get("/status/alarm/{ruleName}/{entityName}") + public HttpResponse getAlarmRuleContext(@Param("ruleName") String ruleName, @Param("entityName") String entityName) { + Map runningRules = getAlarmRulesWatcher().getRunningContext().values().stream().flatMap(List::stream) + .collect(Collectors.toMap(RunningRule::getRuleName, r -> r)); + RunningRule rule = runningRules.get(ruleName); + Map windows = rule.getWindows(); + RunningRule.Window window = windows.keySet().stream().filter(e -> e.getName().equals(entityName)).map(windows::get) + .findFirst().orElse(null); + JsonObject runningContext = new JsonObject(); + if (window == null) { + return HttpResponse.of(MediaType.JSON_UTF_8, runningContext.toString()); + } + + runningContext.addProperty("expression", rule.getExpression()); + runningContext.addProperty("endTime", window.getEndTime().toString()); + runningContext.addProperty("additionalPeriod", window.getAdditionalPeriod()); + runningContext.addProperty("size", window.getSize()); + runningContext.addProperty("silenceCountdown", window.getSilenceCountdown()); + + JsonArray metricValues = new JsonArray(); + runningContext.add("windowValues", metricValues); + + window.scanWindowValues(values -> { + for (int i = 0; i < values.size(); i++) { + JsonObject index = new JsonObject(); + JsonArray metrics = new JsonArray(); + metricValues.add(index); + index.addProperty("index", i); + index.add("metrics", metrics); + Map m = values.get(i); + if (null != m) { + m.forEach((name, metric) -> { + JsonObject metricValue = new JsonObject(); + metricValue.addProperty("timeBucket", metric.getTimeBucket()); + metricValue.addProperty("name", name); + String value = ""; + if (metric instanceof LongValueHolder) { + value = Long.toString(((LongValueHolder) metric).getValue()); + } else if (metric instanceof IntValueHolder) { + value = Integer.toString(((IntValueHolder) metric).getValue()); + } else if (metric instanceof DoubleValueHolder) { + value = Double.toString(((DoubleValueHolder) metric).getValue()); + } else if (metric instanceof LabeledValueHolder) { + value = ((LabeledValueHolder) metric).getValue().toString(); + } + metricValue.addProperty("value", value); + metrics.add(metricValue); + }); + } + } + }); + + runningContext.add("mqeMetricsSnapshot", window.getMqeMetricsSnapshot()); + return HttpResponse.of(MediaType.JSON_UTF_8, gson.toJson(runningContext)); + } +} diff --git a/oap-server/server-query-plugin/status-query-plugin/src/main/java/org/apache/skywalking/oap/query/debug/StatusQueryProvider.java b/oap-server/server-query-plugin/status-query-plugin/src/main/java/org/apache/skywalking/oap/query/debug/StatusQueryProvider.java index ebeb939ab8..5d1f84fb74 100644 --- a/oap-server/server-query-plugin/status-query-plugin/src/main/java/org/apache/skywalking/oap/query/debug/StatusQueryProvider.java +++ b/oap-server/server-query-plugin/status-query-plugin/src/main/java/org/apache/skywalking/oap/query/debug/StatusQueryProvider.java @@ -75,6 +75,10 @@ public class StatusQueryProvider extends ModuleProvider { new ClusterStatusQueryHandler(getManager()), Collections.singletonList(HttpMethod.GET) ); + service.addHandler( + new AlarmStatusQueryHandler(getManager()), + Collections.singletonList(HttpMethod.GET) + ); } public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException {