kazusa
/
skywalking
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Support PagerDuty Alarm Hook (#9247)

This commit is contained in:
Chen Ni 2022-06-18 18:45:25 +08:00 committed by GitHub
parent 0945b36dd8
commit 61278ddd0e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 329 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/en/changes/changes.md
View File

@ -7,6 +7,7 @@
* Add more entities for Zipkin to improve performance.
* ElasticSearch: scroll id should be updated when scrolling as it may change.
* Mesh: fix only last rule works when multiple rules are defined in metadata-service-mapping.yaml.
* Support sending alarm messages to PagerDuty.
#### UI

16
docs/en/setup/backend/backend-alarm.md
View File

@ -299,6 +299,22 @@ welinkHooks:
robot_name: robot
```
## PagerDuty Hook
The PagerDuty hook is based on [Events API v2](https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTgw-events-api-v2-overview).
Follow the [Getting Started](https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTgw-events-api-v2-overview#getting-started) section to create an **Events API v2** integration on your PagerDuty service and copy the integration key.
Then configure as follows:
```yml
pagerDutyHooks:
textTemplate: "Apache SkyWalking Alarm: \n %s."
integrationKeys:
- 5c6d805c9dcf4e03d09dfa81e8789ba1
```
You can also configure multiple integration keys.
## Update the settings dynamically
Since 6.5.0, the alerting settings can be updated dynamically at runtime by [Dynamic Configuration](dynamic-config.md),
which will override the settings in `alarm-settings.yml`.

5
oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRulesWatcher.java
View File

@ -32,6 +32,7 @@ import org.apache.skywalking.oap.server.core.alarm.provider.expression.Expressio
import org.apache.skywalking.oap.server.core.alarm.provider.expression.ExpressionContext;
import org.apache.skywalking.oap.server.core.alarm.provider.feishu.FeishuSettings;
import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCAlarmSetting;
import org.apache.skywalking.oap.server.core.alarm.provider.pagerduty.PagerDutySettings;
import org.apache.skywalking.oap.server.core.alarm.provider.slack.SlackSettings;
import org.apache.skywalking.oap.server.core.alarm.provider.wechat.WechatSettings;
import org.apache.skywalking.oap.server.core.alarm.provider.welink.WeLinkSettings;
@ -142,4 +143,8 @@ public class AlarmRulesWatcher extends ConfigChangeWatcher {
public WeLinkSettings getWeLinkSettings() {
return this.rules.getWelinks();
}
public PagerDutySettings getPagerDutySettings() {
return this.rules.getPagerDutySettings();
}
}

2
oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java
View File

@ -35,6 +35,7 @@ import org.apache.skywalking.oap.server.core.alarm.ServiceMetaInAlarm;
import org.apache.skywalking.oap.server.core.alarm.provider.dingtalk.DingtalkHookCallback;
import org.apache.skywalking.oap.server.core.alarm.provider.feishu.FeishuHookCallback;
import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCCallback;
import org.apache.skywalking.oap.server.core.alarm.provider.pagerduty.PagerDutyHookCallback;
import org.apache.skywalking.oap.server.core.alarm.provider.slack.SlackhookCallback;
import org.apache.skywalking.oap.server.core.alarm.provider.wechat.WechatHookCallback;
import org.apache.skywalking.oap.server.core.alarm.provider.welink.WeLinkHookCallback;
@ -185,6 +186,7 @@ public class NotifyHandler implements MetricsNotify {
allCallbacks.add(new FeishuHookCallback(alarmRulesWatcher));
allCallbacks.add(new EventHookCallback(this.manager));
allCallbacks.add(new WeLinkHookCallback(alarmRulesWatcher));
allCallbacks.add(new PagerDutyHookCallback(alarmRulesWatcher));
core.start(allCallbacks);
}
}

2
oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/Rules.java
View File

@ -26,6 +26,7 @@ import lombok.ToString;
import org.apache.skywalking.oap.server.core.alarm.provider.dingtalk.DingtalkSettings;
import org.apache.skywalking.oap.server.core.alarm.provider.feishu.FeishuSettings;
import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCAlarmSetting;
import org.apache.skywalking.oap.server.core.alarm.provider.pagerduty.PagerDutySettings;
import org.apache.skywalking.oap.server.core.alarm.provider.slack.SlackSettings;
import org.apache.skywalking.oap.server.core.alarm.provider.wechat.WechatSettings;
import org.apache.skywalking.oap.server.core.alarm.provider.welink.WeLinkSettings;
@ -43,6 +44,7 @@ public class Rules {
private DingtalkSettings dingtalks;
private FeishuSettings feishus;
private WeLinkSettings welinks;
private PagerDutySettings pagerDutySettings;
public Rules() {
this.rules = new ArrayList<>();

22
oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RulesReader.java
View File

@ -27,6 +27,8 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import org.apache.skywalking.oap.server.core.alarm.provider.pagerduty.PagerDutySettings;
import org.apache.skywalking.oap.server.library.util.StringUtil;
import org.apache.skywalking.oap.server.core.alarm.provider.dingtalk.DingtalkSettings;
import org.apache.skywalking.oap.server.core.alarm.provider.feishu.FeishuSettings;
@ -70,6 +72,7 @@ public class RulesReader {
readDingtalkConfig(rules);
readFeishuConfig(rules);
readWeLinkConfig(rules);
readPagerDutyConfig(rules);
}
return rules;
}
@ -281,4 +284,23 @@ public class RulesReader {
welinkSettings.setWebhooks(webHookUrls);
rules.setWelinks(welinkSettings);
}
/**
* Read PagerDuty hook config into {@link PagerDutySettings}
*/
private void readPagerDutyConfig(Rules rules) {
Map<String, Object> pagerDutyConfig = (Map<String, Object>) yamlData.get("pagerDutyHooks");
if (pagerDutyConfig != null) {
PagerDutySettings pagerDutySettings = new PagerDutySettings();
String textTemplate = (String) pagerDutyConfig.getOrDefault("textTemplate", "");
pagerDutySettings.setTextTemplate(textTemplate);
List<String> integrationKeys = (List<String>) pagerDutyConfig.get("integrationKeys");
if (integrationKeys != null) {
pagerDutySettings.getIntegrationKeys().addAll(integrationKeys);
}
rules.setPagerDutySettings(pagerDutySettings);
}
}
}

139
oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/pagerduty/PagerDutyHookCallback.java Normal file
View File

@ -0,0 +1,139 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.alarm.provider.pagerduty;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
import io.netty.handler.codec.http.HttpHeaderValues;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.apache.skywalking.oap.server.core.alarm.AlarmCallback;
import org.apache.skywalking.oap.server.core.alarm.AlarmMessage;
import org.apache.skywalking.oap.server.core.alarm.provider.AlarmRulesWatcher;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.UUID;
@Slf4j
public class PagerDutyHookCallback implements AlarmCallback {
private static final String PAGER_DUTY_EVENTS_API_V2_URL = "https://events.pagerduty.com/v2/enqueue";
private static final int HTTP_CONNECT_TIMEOUT = 1000;
private static final int HTTP_CONNECTION_REQUEST_TIMEOUT = 1000;
private static final int HTTP_SOCKET_TIMEOUT = 10000;
private static final Gson GSON = new Gson();
private AlarmRulesWatcher alarmRulesWatcher;
private RequestConfig requestConfig;
public PagerDutyHookCallback(final AlarmRulesWatcher alarmRulesWatcher) {
this.alarmRulesWatcher = alarmRulesWatcher;
this.requestConfig = RequestConfig.custom()
.setConnectTimeout(HTTP_CONNECT_TIMEOUT)
.setConnectionRequestTimeout(HTTP_CONNECTION_REQUEST_TIMEOUT)
.setSocketTimeout(HTTP_SOCKET_TIMEOUT)
.build();
}
@Override
public void doAlarm(List<AlarmMessage> alarmMessages) {
if (this.alarmRulesWatcher.getPagerDutySettings() == null || this.alarmRulesWatcher.getPagerDutySettings().getIntegrationKeys().isEmpty()) {
return;
}
CloseableHttpClient httpClient = HttpClients.custom().build();
try {
this.alarmRulesWatcher.getPagerDutySettings().getIntegrationKeys().forEach(integrationKey -> {
alarmMessages.forEach(alarmMessage -> {
sendAlarmMessage(httpClient, alarmMessage, integrationKey);
});
});
} finally {
try {
httpClient.close();
} catch (IOException e) {
log.error(e.getMessage(), e);
}
}
}
private void sendAlarmMessage(CloseableHttpClient httpClient, AlarmMessage alarmMessage, String integrationKey) {
CloseableHttpResponse httpResponse = null;
try {
HttpPost post = new HttpPost(PAGER_DUTY_EVENTS_API_V2_URL);
post.setConfig(requestConfig);
post.setHeader(HttpHeaders.ACCEPT, HttpHeaderValues.APPLICATION_JSON.toString());
post.setHeader(HttpHeaders.CONTENT_TYPE, HttpHeaderValues.APPLICATION_JSON.toString());
post.setEntity(
getStringEntity(alarmMessage, integrationKey)
);
httpResponse = httpClient.execute(post);
StatusLine statusLine = httpResponse.getStatusLine();
if (statusLine != null && statusLine.getStatusCode() != HttpStatus.SC_ACCEPTED) {
log.error("send PagerDuty alarm to {} failure. Response code: {}, message: {} ",
PAGER_DUTY_EVENTS_API_V2_URL, statusLine.getStatusCode(),
EntityUtils.toString(httpResponse.getEntity(), StandardCharsets.UTF_8)
);
}
} catch (Throwable e) {
log.error("send PagerDuty alarm to {} failure.", PAGER_DUTY_EVENTS_API_V2_URL, e);
} finally {
if (httpResponse != null) {
try {
httpResponse.close();
} catch (IOException e) {
log.error(e.getMessage(), e);
}
}
}
}
private StringEntity getStringEntity(AlarmMessage alarmMessage, String integrationKey) throws UnsupportedEncodingException {
JsonObject body = new JsonObject();
JsonObject payload = new JsonObject();
payload.add("summary", new JsonPrimitive(getFormattedMessage(alarmMessage)));
payload.add("severity", new JsonPrimitive("warning"));
payload.add("source", new JsonPrimitive("Skywalking"));
body.add("payload", payload);
body.add("routing_key", new JsonPrimitive(integrationKey));
body.add("dedup_key", new JsonPrimitive(UUID.randomUUID().toString()));
body.add("event_action", new JsonPrimitive("trigger"));
return new StringEntity(GSON.toJson(body), ContentType.APPLICATION_JSON);
}
private String getFormattedMessage(AlarmMessage alarmMessage) {
return String.format(
this.alarmRulesWatcher.getPagerDutySettings().getTextTemplate(), alarmMessage.getAlarmMessage()
);
}
}

43
oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/pagerduty/PagerDutySettings.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.alarm.provider.pagerduty;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import java.util.ArrayList;
import java.util.List;
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Setter
@Getter
@ToString
public class PagerDutySettings {
private String textTemplate;
@Builder.Default
private List<String> integrationKeys = new ArrayList<>();
}

9
oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/RulesReaderTest.java
View File

@ -21,6 +21,7 @@ package org.apache.skywalking.oap.server.core.alarm.provider;
import org.apache.skywalking.oap.server.core.alarm.provider.dingtalk.DingtalkSettings;
import org.apache.skywalking.oap.server.core.alarm.provider.feishu.FeishuSettings;
import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCAlarmSetting;
import org.apache.skywalking.oap.server.core.alarm.provider.pagerduty.PagerDutySettings;
import org.apache.skywalking.oap.server.core.alarm.provider.slack.SlackSettings;
import org.apache.skywalking.oap.server.core.alarm.provider.wechat.WechatSettings;
import org.junit.Assert;
@ -33,6 +34,7 @@ import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertEquals;
public class RulesReaderTest {
@Test
@ -96,5 +98,12 @@ public class RulesReaderTest {
assertThat(feishuSettingsWebhooks.get(0).getSecret(), is("dummysecret"));
assertThat(feishuSettingsWebhooks.get(1).getUrl(), is("https://open.feishu.cn/open-apis/bot/v2/hook/dummy_token2"));
assertNull(feishuSettingsWebhooks.get(1).getSecret());
PagerDutySettings pagerDutySettings = rules.getPagerDutySettings();
assertEquals("dummy_text_template", pagerDutySettings.getTextTemplate());
List<String> pagerDutyIntegrationKeys = pagerDutySettings.getIntegrationKeys();
assertEquals(2, pagerDutyIntegrationKeys.size());
assertEquals("dummy_key", pagerDutyIntegrationKeys.get(0));
assertEquals("dummy_key2", pagerDutyIntegrationKeys.get(1));
}
}

74
oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/pagerduty/PagerDutyHookCallbackTest.java Normal file
View File

@ -0,0 +1,74 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.alarm.provider.pagerduty;
import org.apache.skywalking.oap.server.core.alarm.AlarmMessage;
import org.apache.skywalking.oap.server.core.alarm.provider.AlarmRulesWatcher;
import org.apache.skywalking.oap.server.core.alarm.provider.Rules;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.junit.Ignore;
import org.junit.Test;
import java.util.Arrays;
import java.util.List;
public class PagerDutyHookCallbackTest {
@Ignore
@Test
public void testWithRealAccount() {
// replace this with your actual integration key(s) and run this test manually
List<String> integrationKeys = Arrays.asList(
"dummy-integration-key"
);
Rules rules = new Rules();
rules.setPagerDutySettings(
PagerDutySettings.builder()
.integrationKeys(integrationKeys)
.textTemplate("Apache SkyWalking Alarm: \n %s.")
.build()
);
PagerDutyHookCallback pagerDutyHookCallback = new PagerDutyHookCallback(
new AlarmRulesWatcher(rules, null)
);
pagerDutyHookCallback.doAlarm(getMockAlarmMessages());
// please check your pagerduty account to see if the alarm is sent
}
private List<AlarmMessage> getMockAlarmMessages() {
AlarmMessage alarmMessage = new AlarmMessage();
alarmMessage.setScopeId(DefaultScopeDefine.SERVICE);
alarmMessage.setRuleName("service_resp_time_rule");
alarmMessage.setAlarmMessage("alarmMessage with [DefaultScopeDefine.All]");
AlarmMessage anotherAlarmMessage = new AlarmMessage();
anotherAlarmMessage.setScopeId(DefaultScopeDefine.ENDPOINT);
anotherAlarmMessage.setRuleName("service_resp_time_rule_2");
anotherAlarmMessage.setAlarmMessage("anotherAlarmMessage with [DefaultScopeDefine.Endpoint]");
return Arrays.asList(
alarmMessage,
anotherAlarmMessage
);
}
}

8
oap-server/server-alarm-plugin/src/test/resources/alarm-settings.yml
View File

@ -133,4 +133,10 @@ welinkHooks:
# if you send to multi group at a time, separate group_ids with commas, e.g. "123xx","456xx"
group_ids: "dummy_group_id"
# make a name you like for the robot, it will display in group
robot_name: robot
robot_name: robot
pagerDutyHooks:
textTemplate: "dummy_text_template"
integrationKeys:
- dummy_key
- dummy_key2

9
oap-server/server-starter/src/main/resources/alarm-settings.yml
View File

@ -123,3 +123,12 @@ welinkHooks:
# group_ids: "dummy_group_id"
# # make a name you like for the robot, it will display in group
# robot_name: robot
pagerDutyHooks:
textTemplate: "Apache SkyWalking Alarm: \n %s."
integrationKeys:
# # you can find your integration key(s) on the Events API V2 integration page for your PagerDuty service(s).
# # (you may need to create an Events API V2 integration for your PagerDuty service if you don't have one yet)
# # below are dummy keys that should be replaced with your own integration keys.
# - dummy_key
# - dummy_key2