kazusa
/
skywalking-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

support Slack alarm hooks (#5364)

This commit is contained in:
Jared Tan 2020-08-22 17:18:09 +08:00 committed by GitHub
parent 5bfac1b50f
commit 3dfa40ebb5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 227 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
docs/en/setup/backend/backend-alarm.md
View File

@ -162,6 +162,24 @@ message AlarmMessage {
}
```
## Slack Chat Hook
To do this you need to follow the [Getting Started with Incoming Webhooks guide](https://api.slack.com/messaging/webhooks) and create new Webhooks.
The alarm message will send through HTTP post by `application/json` content type if you configured Slack Incoming Webhooks as following:
```yml
slackHooks:
textTemplate: |-
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": ":alarm_clock: *Apache Skywalking Alarm* \n **%s**."
}
}
webhooks:
- https://hooks.slack.com/services/x/y/z
```
## Update the settings dynamically
Since 6.5.0, the alarm settings can be updated dynamically at runtime by [Dynamic Configuration](dynamic-config.md),
which will override the settings in `alarm-settings.yml`.

5
oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRulesWatcher.java
View File

@ -29,6 +29,7 @@ import org.apache.skywalking.oap.server.configuration.api.ConfigChangeWatcher;
import org.apache.skywalking.oap.server.core.Const;
import org.apache.skywalking.oap.server.core.alarm.AlarmModule;
import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCAlarmSetting;
import org.apache.skywalking.oap.server.core.alarm.provider.slack.SlackSettings;
import org.apache.skywalking.oap.server.library.module.ModuleProvider;
/**
@ -109,4 +110,8 @@ public class AlarmRulesWatcher extends ConfigChangeWatcher {
public GRPCAlarmSetting getGrpchookSetting() {
return this.rules.getGrpchookSetting();
}
public SlackSettings getSlackSettings() {
return this.rules.getSlacks();
}
}

2
oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java
View File

@ -32,6 +32,7 @@ import org.apache.skywalking.oap.server.core.alarm.MetricsNotify;
import org.apache.skywalking.oap.server.core.alarm.ServiceInstanceMetaInAlarm;
import org.apache.skywalking.oap.server.core.alarm.ServiceMetaInAlarm;
import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCCallback;
import org.apache.skywalking.oap.server.core.alarm.provider.slack.SlackhookCallback;
import org.apache.skywalking.oap.server.core.analysis.IDManager;
import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.analysis.metrics.MetricsMetaInfo;
@ -158,6 +159,7 @@ public class NotifyHandler implements MetricsNotify {
List<AlarmCallback> allCallbacks = new ArrayList<>(Arrays.asList(callbacks));
allCallbacks.add(new WebhookCallback(alarmRulesWatcher));
allCallbacks.add(new GRPCCallback(alarmRulesWatcher));
allCallbacks.add(new SlackhookCallback(alarmRulesWatcher));
core.start(allCallbacks);
}
}

2
oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/Rules.java
View File

@ -24,6 +24,7 @@ import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCAlarmSetting;
import org.apache.skywalking.oap.server.core.alarm.provider.slack.SlackSettings;
@Setter
@Getter
@ -32,6 +33,7 @@ public class Rules {
private List<AlarmRule> rules;
private List<String> webhooks;
private GRPCAlarmSetting grpchookSetting;
private SlackSettings slacks;
public Rules() {
this.rules = new ArrayList<>();

28
oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RulesReader.java
View File

@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCAlarmSetting;
import org.apache.skywalking.oap.server.core.alarm.provider.slack.SlackSettings;
import org.yaml.snakeyaml.Yaml;
/**
@ -65,8 +66,10 @@ public class RulesReader {
alarmRule.setExcludeNames((ArrayList) settings.getOrDefault("exclude-names", new ArrayList(0)));
alarmRule.setIncludeNamesRegex((String) settings.getOrDefault("include-names-regex", ""));
alarmRule.setExcludeNamesRegex((String) settings.getOrDefault("exclude-names-regex", ""));
alarmRule.setIncludeLabels((ArrayList) settings.getOrDefault("include-labels", new ArrayList(0)));
alarmRule.setExcludeLabels((ArrayList) settings.getOrDefault("exclude-labels", new ArrayList(0)));
alarmRule.setIncludeLabels(
(ArrayList) settings.getOrDefault("include-labels", new ArrayList(0)));
alarmRule.setExcludeLabels(
(ArrayList) settings.getOrDefault("exclude-labels", new ArrayList(0)));
alarmRule.setIncludeLabelsRegex((String) settings.getOrDefault("include-labels-regex", ""));
alarmRule.setExcludeLabelsRegex((String) settings.getOrDefault("exclude-labels-regex", ""));
alarmRule.setThreshold(settings.get("threshold").toString());
@ -74,8 +77,9 @@ public class RulesReader {
alarmRule.setPeriod((Integer) settings.getOrDefault("period", 1));
alarmRule.setCount((Integer) settings.getOrDefault("count", 1));
alarmRule.setSilencePeriod((Integer) settings.getOrDefault("silence-period", -1));
alarmRule.setMessage((String) settings.getOrDefault("message", "Alarm caused by Rule " + alarmRule
.getAlarmRuleName()));
alarmRule.setMessage(
(String) settings.getOrDefault("message", "Alarm caused by Rule " + alarmRule
.getAlarmRuleName()));
rules.getRules().add(alarmRule);
}
@ -104,6 +108,22 @@ public class RulesReader {
rules.setGrpchookSetting(grpcAlarmSetting);
}
Map slacks = (Map) yamlData.get("slackHooks");
if (slacks != null) {
SlackSettings slackSettings = new SlackSettings();
Object textTemplate = slacks.getOrDefault("textTemplate", "");
slackSettings.setTextTemplate((String) textTemplate);
List<String> slackWebhooks = (List<String>) slacks.get("webhooks");
if (slackWebhooks != null) {
slackWebhooks.forEach(
url -> slackSettings.getWebhooks().add(url)
);
}
rules.setSlacks(slackSettings);
}
}
return rules;

15
oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java
View File

@ -24,6 +24,7 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
@ -35,14 +36,12 @@ import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.skywalking.oap.server.core.alarm.AlarmCallback;
import org.apache.skywalking.oap.server.core.alarm.AlarmMessage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Use SkyWalking alarm webhook API call a remote endpoints.
*/
@Slf4j
public class WebhookCallback implements AlarmCallback {
private static final Logger LOGGER = LoggerFactory.getLogger(WebhookCallback.class);
private static final int HTTP_CONNECT_TIMEOUT = 1000;
private static final int HTTP_CONNECTION_REQUEST_TIMEOUT = 1000;
private static final int HTTP_SOCKET_TIMEOUT = 10000;
@ -62,7 +61,7 @@ public class WebhookCallback implements AlarmCallback {
@Override
public void doAlarm(List<AlarmMessage> alarmMessage) {
if (alarmRulesWatcher.getWebHooks().size() == 0) {
if (alarmRulesWatcher.getWebHooks().isEmpty()) {
return;
}
@ -81,19 +80,19 @@ public class WebhookCallback implements AlarmCallback {
CloseableHttpResponse httpResponse = httpClient.execute(post);
StatusLine statusLine = httpResponse.getStatusLine();
if (statusLine != null && statusLine.getStatusCode() != HttpStatus.SC_OK) {
LOGGER.error("send alarm to " + url + " failure. Response code: " + statusLine.getStatusCode());
log.error("send alarm to " + url + " failure. Response code: " + statusLine.getStatusCode());
}
} catch (UnsupportedEncodingException e) {
LOGGER.error("Alarm to JSON error, " + e.getMessage(), e);
log.error("Alarm to JSON error, " + e.getMessage(), e);
} catch (IOException e) {
LOGGER.error("send alarm to " + url + " failure.", e);
log.error("send alarm to " + url + " failure.", e);
}
});
} finally {
try {
httpClient.close();
} catch (IOException e) {
LOGGER.error(e.getMessage(), e);
log.error(e.getMessage(), e);
}
}
}

42
oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/slack/SlackSettings.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.alarm.provider.slack;
import java.util.ArrayList;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Setter
@Getter
@ToString
public class SlackSettings {
private String textTemplate;
@Builder.Default
private List<String> webhooks = new ArrayList<>();
}

115
oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/slack/SlackhookCallback.java Normal file
View File

@ -0,0 +1,115 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.alarm.provider.slack;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import io.netty.handler.codec.http.HttpHeaderValues;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.skywalking.oap.server.core.alarm.AlarmCallback;
import org.apache.skywalking.oap.server.core.alarm.AlarmMessage;
import org.apache.skywalking.oap.server.core.alarm.provider.AlarmRulesWatcher;
/**
* Use SkyWalking alarm slack webhook API call a remote endpoints.
*/
@Slf4j
public class SlackhookCallback implements AlarmCallback {
private static final int HTTP_CONNECT_TIMEOUT = 1000;
private static final int HTTP_CONNECTION_REQUEST_TIMEOUT = 1000;
private static final int HTTP_SOCKET_TIMEOUT = 10000;
private static final Gson GSON = new Gson();
private AlarmRulesWatcher alarmRulesWatcher;
private RequestConfig requestConfig;
public SlackhookCallback(final AlarmRulesWatcher alarmRulesWatcher) {
this.alarmRulesWatcher = alarmRulesWatcher;
this.requestConfig = RequestConfig.custom()
.setConnectTimeout(HTTP_CONNECT_TIMEOUT)
.setConnectionRequestTimeout(HTTP_CONNECTION_REQUEST_TIMEOUT)
.setSocketTimeout(HTTP_SOCKET_TIMEOUT)
.build();
}
@Override
public void doAlarm(List<AlarmMessage> alarmMessages) {
if (this.alarmRulesWatcher.getSlackSettings().getWebhooks().isEmpty()) {
return;
}
CloseableHttpClient httpClient = HttpClients.custom().build();
try {
this.alarmRulesWatcher.getSlackSettings().getWebhooks().forEach(url -> {
HttpPost post = new HttpPost(url);
post.setConfig(requestConfig);
post.setHeader(HttpHeaders.ACCEPT, HttpHeaderValues.APPLICATION_JSON.toString());
post.setHeader(HttpHeaders.CONTENT_TYPE, HttpHeaderValues.APPLICATION_JSON.toString());
StringEntity entity;
try {
JsonObject jsonObject = new JsonObject();
JsonArray jsonElements = new JsonArray();
alarmMessages.forEach(item -> {
jsonElements.add(GSON.fromJson(
String.format(
this.alarmRulesWatcher.getSlackSettings().getTextTemplate(), item.getAlarmMessage()
), JsonObject.class));
});
jsonObject.add("blocks", jsonElements);
entity = new StringEntity(GSON.toJson(jsonObject), ContentType.APPLICATION_JSON);
post.setEntity(entity);
CloseableHttpResponse httpResponse = httpClient.execute(post);
StatusLine statusLine = httpResponse.getStatusLine();
if (statusLine != null && statusLine.getStatusCode() != HttpStatus.SC_OK) {
log.error("send alarm to " + url + " failure. Response code: " + statusLine.getStatusCode());
}
} catch (UnsupportedEncodingException e) {
log.error("Alarm to JSON error, " + e.getMessage(), e);
} catch (IOException e) {
log.error("send alarm to " + url + " failure.", e);
}
});
} finally {
try {
httpClient.close();
} catch (IOException e) {
log.error(e.getMessage(), e);
}
}
}
}

12
oap-server/server-bootstrap/src/main/resources/alarm-settings.yml
View File

@ -47,3 +47,15 @@ gRPCHook:
# target_host: 127.0.0.1
# target_port: 9888
slackHooks:
textTemplate: |-
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": ":alarm_clock: *Apache Skywalking Alarm* \n **%s**."
}
}
webhooks:
# - https://hooks.slack.com/services/x/y/zssss