Add alarm tests in the e2e (#5961)

.github/workflows/e2e.yaml

@@ -152,3 +152,32 @@ jobs:
         with:
           name: logs
           path: logs
+
+  FeatureGroup05:
+    name: Alarm
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - name: Compile and Build
+        run: make docker
+      - name: Copy dist package
+        run: cp -R dist test/e2e/
+      - name: Meter receiver
+        run: ./mvnw --batch-mode -f test/e2e/pom.xml -am -DfailIfNoTests=false verify -Dit.test=org.apache.skywalking.e2e.alarm.AlarmE2E
+      - name: Report Coverage
+        run: bash -x tools/coverage/report.sh
+      - uses: actions/upload-artifact@v1
+        if: failure()
+        with:
+          name: logs
+          path: logs

CHANGES.md

@@ -5,6 +5,7 @@ Release Notes.
 8.4.0
 ------------------
 #### Project
+* Incompatible with previous releases when use H2/MySQL/TiDB storage options, due to support multiple alarm rules triggered for one entity.
 * Chore: adapt `create_source_release.sh` to make it runnable on Linux.
 
 #### Java Agent
@@ -15,6 +16,7 @@ Release Notes.
 * Make meter receiver support MAL.
 * Support influxDB connection response format option. Fix some error when use JSON as influxDB response format.
 * Support Kafka MirrorMaker 2.0 to replicate topics between Kafka clusters.
+* Add the rule name field to alarm record storage entity as a part of ID, to support multiple alarm rules triggered for one entity. The scope id has been removed from the ID.
 * Fix MAL concurrent execution issues
 
 #### UI

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java

@@ -18,8 +18,6 @@
 
 package org.apache.skywalking.oap.server.core.alarm;
 
-import java.util.HashMap;
-import java.util.Map;
 import lombok.Getter;
 import lombok.Setter;
 import org.apache.skywalking.oap.server.core.Const;
@@ -31,6 +29,9 @@ import org.apache.skywalking.oap.server.core.source.ScopeDeclaration;
 import org.apache.skywalking.oap.server.core.storage.StorageBuilder;
 import org.apache.skywalking.oap.server.core.storage.annotation.Column;
 
+import java.util.HashMap;
+import java.util.Map;
+
 import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.ALARM;
 
 @Getter
@@ -46,10 +47,11 @@ public class AlarmRecord extends Record {
     public static final String ID1 = "id1";
     public static final String START_TIME = "start_time";
     public static final String ALARM_MESSAGE = "alarm_message";
+    public static final String RULE_NAME = "rule_name";
 
     @Override
     public String id() {
-        return getTimeBucket() + Const.ID_CONNECTOR + scope + Const.ID_CONNECTOR + id0 + Const.ID_CONNECTOR + id1;
+        return getTimeBucket() + Const.ID_CONNECTOR + ruleName + Const.ID_CONNECTOR + id0 + Const.ID_CONNECTOR + id1;
     }
 
     @Column(columnName = SCOPE)
@@ -64,6 +66,8 @@ public class AlarmRecord extends Record {
     private long startTime;
     @Column(columnName = ALARM_MESSAGE, matchQuery = true)
     private String alarmMessage;
+    @Column(columnName = RULE_NAME)
+    private String ruleName;
 
     public static class Builder implements StorageBuilder<AlarmRecord> {
 
@@ -77,6 +81,7 @@ public class AlarmRecord extends Record {
             map.put(ALARM_MESSAGE, storageData.getAlarmMessage());
             map.put(START_TIME, storageData.getStartTime());
             map.put(TIME_BUCKET, storageData.getTimeBucket());
+            map.put(RULE_NAME, storageData.getRuleName());
             return map;
         }
 
@@ -90,6 +95,7 @@ public class AlarmRecord extends Record {
             record.setAlarmMessage((String) dbMap.get(ALARM_MESSAGE));
             record.setStartTime(((Number) dbMap.get(START_TIME)).longValue());
             record.setTimeBucket(((Number) dbMap.get(TIME_BUCKET)).longValue());
+            record.setRuleName((String) dbMap.get(RULE_NAME));
             return record;
         }
     }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmStandardPersistence.java

@@ -18,12 +18,13 @@
 
 package org.apache.skywalking.oap.server.core.alarm;
 
-import java.util.List;
 import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
 import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.List;
+
 /**
  * Save the alarm info into storage for UI query.
  */
@@ -46,6 +47,7 @@ public class AlarmStandardPersistence implements AlarmCallback {
             record.setAlarmMessage(message.getAlarmMessage());
             record.setStartTime(message.getStartTime());
             record.setTimeBucket(TimeBucket.getRecordTimeBucket(message.getStartTime()));
+            record.setRuleName(message.getRuleName());
 
             RecordStreamProcessor.getInstance().in(record);
         });

test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/SimpleQueryClient.java

@@ -19,13 +19,10 @@
 package org.apache.skywalking.e2e;
 
 import com.google.common.io.Resources;
-import java.net.URI;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.skywalking.e2e.alarm.AlarmQuery;
+import org.apache.skywalking.e2e.alarm.GetAlarm;
+import org.apache.skywalking.e2e.alarm.GetAlarmData;
 import org.apache.skywalking.e2e.browser.BrowserErrorLog;
 import org.apache.skywalking.e2e.browser.BrowserErrorLogQuery;
 import org.apache.skywalking.e2e.browser.BrowserErrorLogsData;
@@ -60,6 +57,13 @@ import org.springframework.http.RequestEntity;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.client.RestTemplate;
 
+import java.net.URI;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
 @SuppressWarnings("UnstableApiUsage")
 @Slf4j
 public class SimpleQueryClient {
@@ -358,4 +362,29 @@ public class SimpleQueryClient {
 
         return Objects.requireNonNull(responseEntity.getBody()).getData().getReadLabeledMetricsValues();
     }
+
+    public GetAlarm readAlarms(final AlarmQuery query) throws Exception {
+        final URL queryFileUrl = Resources.getResource("read-alarms.gql");
+        final String queryString = Resources.readLines(queryFileUrl, StandardCharsets.UTF_8)
+            .stream()
+            .filter(it -> !it.startsWith("#"))
+            .collect(Collectors.joining())
+            .replace("{step}", query.step())
+            .replace("{start}", query.start())
+            .replace("{end}", query.end())
+            .replace("{pageSize}", "20")
+            .replace("{needTotal}", "true");
+        LOGGER.info("Query: {}", queryString);
+        final ResponseEntity<GQLResponse<GetAlarmData>> responseEntity = restTemplate.exchange(
+            new RequestEntity<>(queryString, HttpMethod.POST, URI.create(endpointUrl)),
+            new ParameterizedTypeReference<GQLResponse<GetAlarmData>>() {
+            }
+        );
+
+        if (responseEntity.getStatusCode() != HttpStatus.OK) {
+            throw new RuntimeException("Response status != 200, actual: " + responseEntity.getStatusCode());
+        }
+
+        return Objects.requireNonNull(responseEntity.getBody()).getData().getGetAlarm();
+    }
 }

test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/alarm/AlarmMatcher.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.skywalking.e2e.alarm;
+
+import lombok.Data;
+import org.apache.skywalking.e2e.verification.AbstractMatcher;
+
+@Data
+public class AlarmMatcher extends AbstractMatcher<AlarmMatcher> {
+    private String startTime;
+    private String scope;
+    private String id;
+    private String message;
+
+    @Override
+    public void verify(AlarmMatcher alarmMatcher) {
+        doVerify(this.startTime, alarmMatcher.startTime);
+        doVerify(this.scope, alarmMatcher.scope);
+        doVerify(this.id, alarmMatcher.id);
+        doVerify(this.message, alarmMatcher.message);
+    }
+}

test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/alarm/AlarmQuery.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.skywalking.e2e.alarm;
+
+import org.apache.skywalking.e2e.AbstractQuery;
+
+public class AlarmQuery extends AbstractQuery<AlarmQuery> {
+}

test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/alarm/AlarmsMatcher.java

@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.skywalking.e2e.alarm;
+
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.Assert;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
+
+@Data
+@Slf4j
+public class AlarmsMatcher {
+    private GetAlarm alarm;
+
+    public void verify(final GetAlarm alarm) {
+        LOGGER.info("alarms:{} matchers:{}", alarm, this.alarm);
+        Assert.assertEquals(this.alarm.getTotal(), alarm.getTotal());
+
+        assertThat(this.alarm.getMsgs()).hasSameSizeAs(alarm.getMsgs());
+
+        for (int i = 0; i < this.alarm.getMsgs().size(); i++) {
+            boolean matched = false;
+            for (AlarmMatcher alarmMatcher : alarm.getMsgs()) {
+                try {
+                    this.alarm.getMsgs().get(i).verify(alarmMatcher);
+                    matched = true;
+                } catch (Throwable ignored) {
+                }
+            }
+            if (!matched) {
+                fail("\nExpected: %s\nActual: %s", this.alarm, alarm);
+            }
+        }
+    }
+
+}

test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/alarm/GetAlarm.java

@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.skywalking.e2e.alarm;
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class GetAlarm {
+    private int total;
+    private List<AlarmMatcher> msgs;
+}

test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/alarm/GetAlarmData.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.skywalking.e2e.alarm;
+
+import lombok.Data;
+
+@Data
+public class GetAlarmData {
+    private GetAlarm getAlarm;
+}

test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/alarm/HookAlarmMatcher.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.skywalking.e2e.alarm;
+
+import lombok.Data;
+import org.apache.skywalking.e2e.verification.AbstractMatcher;
+
+@Data
+public class HookAlarmMatcher extends AbstractMatcher<HookAlarmMatcher> {
+    private String scopeId;
+    private String scope;
+    private String name;
+    private String id0;
+    private String id1;
+    private String ruleName;
+    private String alarmMessage;
+    private String startTime;
+
+    @Override
+    public void verify(HookAlarmMatcher hookAlarmMatcher) {
+        doVerify(this.scopeId, hookAlarmMatcher.getScopeId());
+        doVerify(this.scope, hookAlarmMatcher.getScope());
+        doVerify(this.name, hookAlarmMatcher.getName());
+        doVerify(this.id0, hookAlarmMatcher.getId0());
+        doVerify(this.id1, hookAlarmMatcher.getId1());
+        doVerify(this.ruleName, hookAlarmMatcher.getRuleName());
+        doVerify(this.alarmMessage, hookAlarmMatcher.getAlarmMessage());
+        doVerify(this.startTime, hookAlarmMatcher.getStartTime());
+    }
+}

test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/alarm/HookAlarms.java

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.skywalking.e2e.alarm;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.apache.skywalking.e2e.verification.AbstractMatcher;
+
+import java.util.ArrayList;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class HookAlarms extends AbstractMatcher<HookAlarms> {
+    private ArrayList<HookAlarmMatcher> messages;
+
+    @Override
+    public void verify(HookAlarms hookAlarms) {
+        assertThat(this.messages).hasSameSizeAs(hookAlarms.getMessages());
+
+        for (int i = 0; i < this.messages.size(); i++) {
+            boolean matched = false;
+            for (HookAlarmMatcher alarmMatcher : hookAlarms.getMessages()) {
+                try {
+                    this.messages.get(i).verify(alarmMatcher);
+                    matched = true;
+                } catch (Throwable ignored) {
+                }
+            }
+            if (!matched) {
+                fail("\nExpected: %s\nActual: %s", this.messages.get(i), hookAlarms.getMessages());
+            }
+        }
+    }
+}

test/e2e/e2e-data/src/main/resources/read-alarms.gql

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+  "query":"query queryData($duration: Duration!, $paging: Pagination!) {
+    getAlarm: getAlarm(
+        duration: $duration,
+        paging: $paging
+    ) {
+        total
+        msgs {
+            startTime
+            scope
+            id
+            message
+        }
+   }}",
+  "variables":{
+    "duration":{
+      "start":"{start}",
+      "end":"{end}",
+      "step":"{step}"
+    },
+    "paging":{
+      "pageSize":"{pageSize}",
+      "needTotal":"{needTotal}"
+    }
+  }
+}

test/e2e/e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/AlarmController.java

@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.skywalking.e2e.controller;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@RestController
+@RequestMapping("/alarm")
+public class AlarmController {
+
+    // Save all received alarm message from oap
+    private List<AlarmMessage> alarmMessages = new ArrayList<>();
+
+    @PostMapping("/receive")
+    public String receiveAlarmMessage(@RequestBody List<AlarmMessage> data) {
+        alarmMessages.addAll(data);
+        return "success";
+    }
+
+    @PostMapping("/read")
+    public Alarms readMessages() {
+        return Alarms.builder().messages(alarmMessages).build();
+    }
+
+    /**
+     * Alarm message represents the details of each alarm.
+     */
+    @Setter
+    @Getter
+    @NoArgsConstructor
+    public static class AlarmMessage {
+        private int scopeId;
+        private String scope;
+        private String name;
+        private String id0;
+        private String id1;
+        private String ruleName;
+        private String alarmMessage;
+        private long startTime;
+    }
+
+    /**
+     * Alarm wrapper
+     */
+    @Data
+    @Builder
+    @NoArgsConstructor
+    @AllArgsConstructor
+    public static class Alarms {
+        private List<AlarmMessage> messages;
+    }
+
+}

test/e2e/e2e-test/docker/alarm/alarm-settings.yml

@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+rules:
+  # service response time > 10ms
+  service_resp_time_rule:
+    metrics-name: service_resp_time
+    op: ">"
+    threshold: 10
+    period: 10
+    count: 1
+    silence-period: 1
+    message: Response time of service {name} is more than 10ms in 1 minutes of last 10 minutes.
+    only-as-condition: false
+  # service sla > 1%
+  service_sla_rule:
+    metrics-name: service_sla
+    op: ">"
+    threshold: 100
+    period: 10
+    count: 1
+    silence-period: 1
+    message: Successful rate of service {name} is more than 1% in 1 minutes of last 10 minutes
+    only-as-condition: true
+composite-rules:
+  comp_rule:
+    expression: service_resp_time_rule && service_sla_rule
+    message: Service {name} response time is more than 10ms and sla is more than 1%.
+
+webhooks:
+  - http://provider:9090/alarm/receive

test/e2e/e2e-test/docker/alarm/docker-compose.yml

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '2.1'
+
+services:
+  oap:
+    extends:
+      file: ../base-compose.yml
+      service: oap
+    volumes:
+      - ./alarm-settings.yml:/skywalking/config/alarm-settings.yml
+
+  ui:
+    extends:
+      file: ../base-compose.yml
+      service: ui
+    depends_on:
+      oap:
+        condition: service_healthy
+
+  provider:
+    extends:
+      file: ../base-compose.yml
+      service: provider
+    environment:
+      SW_AGENT_NAME: e2e-service-provider
+    depends_on:
+      oap:
+        condition: service_healthy
+
+networks:
+  e2e:

test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/alarm/AlarmE2E.java

@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.skywalking.e2e.alarm;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.skywalking.e2e.annotation.ContainerHostAndPort;
+import org.apache.skywalking.e2e.annotation.DockerCompose;
+import org.apache.skywalking.e2e.base.SkyWalkingE2E;
+import org.apache.skywalking.e2e.base.SkyWalkingTestAdapter;
+import org.apache.skywalking.e2e.common.HostAndPort;
+import org.apache.skywalking.e2e.retryable.RetryableTest;
+import org.apache.skywalking.e2e.service.Service;
+import org.apache.skywalking.e2e.service.ServicesMatcher;
+import org.apache.skywalking.e2e.service.ServicesQuery;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.TestMethodOrder;
+import org.springframework.http.ResponseEntity;
+import org.testcontainers.containers.DockerComposeContainer;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.skywalking.e2e.utils.Times.now;
+import static org.apache.skywalking.e2e.utils.Yamls.load;
+
+@Slf4j
+@SkyWalkingE2E
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+public class AlarmE2E extends SkyWalkingTestAdapter {
+
+    @SuppressWarnings("unused")
+    @DockerCompose("docker/alarm/docker-compose.yml")
+    protected DockerComposeContainer<?> justForSideEffects;
+
+    @SuppressWarnings("unused")
+    @ContainerHostAndPort(name = "ui", port = 8080)
+    protected HostAndPort swWebappHostPort = HostAndPort.builder().host("localhost").port(12800).build();
+
+    @SuppressWarnings("unused")
+    @ContainerHostAndPort(name = "provider", port = 9090)
+    protected HostAndPort serviceHostPort = HostAndPort.builder().host("localhost").port(9090).build();
+
+    @BeforeAll
+    public void setUp() throws Exception {
+        queryClient(swWebappHostPort);
+
+        trafficController(serviceHostPort, "/users");
+    }
+
+    @AfterAll
+    public void tearDown() {
+        trafficController.stop();
+    }
+
+    @RetryableTest
+    @Order(1)
+    void services() throws Exception {
+        // Load services
+        List<Service> services = graphql.services(new ServicesQuery().start(startTime).end(now()));
+        LOGGER.info("services: {}", services);
+        load("expected/alarm/services.yml").as(ServicesMatcher.class).verify(services);
+    }
+
+    @RetryableTest
+    @Order(2)
+    void basicAlarm() throws Exception {
+        // Wait all alarm notified(single and compose)
+        validate("expected/alarm/silence-before-graphql.yml", "expected/alarm/silence-before-webhook.yml");
+
+        // Wait silence period finished
+        TimeUnit.SECONDS.sleep(90);
+    }
+
+    @RetryableTest
+    @Order(3)
+    void afterSilenceAlarm() throws Exception {
+        // Retry to send request and check silence config
+        validate("expected/alarm/silence-after-graphql.yml", "expected/alarm/silence-after-webhook.yml");
+    }
+
+    private void validate(String alarmFile, String hookFile) throws Exception {
+        // validate graphql
+        GetAlarm alarms = graphql.readAlarms(new AlarmQuery().start(startTime).end(now()));
+        LOGGER.info("alarms query: {}", alarms);
+        load(alarmFile).as(AlarmsMatcher.class).verify(alarms);
+
+        // validate web hook receiver
+        ResponseEntity<HookAlarms> responseEntity = restTemplate.postForEntity("http://" + serviceHostPort.host() + ":" + serviceHostPort.port() + "/alarm/read", null, HookAlarms.class);
+        LOGGER.info("alarms hook: {}", responseEntity.getBody());
+        load(hookFile).as(HookAlarms.class).verify(responseEntity.getBody());
+    }
+
+}

test/e2e/e2e-test/src/test/resources/expected/alarm/services.yml

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+services:
+  - key: not null
+    label: "e2e-service-provider"

test/e2e/e2e-test/src/test/resources/expected/alarm/silence-after-graphql.yml

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+alarm:
+  total: 4
+  msgs:
+    - startTime: gt 0
+      scope: Service
+      id: ZTJlLXNlcnZpY2UtcHJvdmlkZXI=.1
+      message: Service e2e-service-provider response time is more than 10ms and sla is more than 1%.
+    - startTime: gt 0
+      scope: Service
+      id: ZTJlLXNlcnZpY2UtcHJvdmlkZXI=.1
+      message: Response time of service e2e-service-provider is more than 10ms in 1 minutes of last 10 minutes.
+    - startTime: gt 0
+      scope: Service
+      id: ZTJlLXNlcnZpY2UtcHJvdmlkZXI=.1
+      message: Service e2e-service-provider response time is more than 10ms and sla is more than 1%.
+    - startTime: gt 0
+      scope: Service
+      id: ZTJlLXNlcnZpY2UtcHJvdmlkZXI=.1
+      message: Response time of service e2e-service-provider is more than 10ms in 1 minutes of last 10 minutes.

test/e2e/e2e-test/src/test/resources/expected/alarm/silence-after-webhook.yml

@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+messages:
+  - scopeId: 1
+    scope: SERVICE
+    name: e2e-service-provider
+    id0: ZTJlLXNlcnZpY2UtcHJvdmlkZXI=.1
+    id1: ""
+    ruleName: service_resp_time_rule
+    alarmMessage: Response time of service e2e-service-provider is more than 10ms in 1 minutes of last 10 minutes.
+    startTime: gt 0
+  - scopeId: 1
+    scope: SERVICE
+    name: e2e-service-provider
+    id0: ZTJlLXNlcnZpY2UtcHJvdmlkZXI=.1
+    id1: ""
+    ruleName: comp_rule
+    alarmMessage: Service e2e-service-provider response time is more than 10ms and sla is more than 1%.
+    startTime: gt 0
+  - scopeId: 1
+    scope: SERVICE
+    name: e2e-service-provider
+    id0: ZTJlLXNlcnZpY2UtcHJvdmlkZXI=.1
+    id1: ""
+    ruleName: service_resp_time_rule
+    alarmMessage: Response time of service e2e-service-provider is more than 10ms in 1 minutes of last 10 minutes.
+    startTime: gt 0
+  - scopeId: 1
+    scope: SERVICE
+    name: e2e-service-provider
+    id0: ZTJlLXNlcnZpY2UtcHJvdmlkZXI=.1
+    id1: ""
+    ruleName: comp_rule
+    alarmMessage: Service e2e-service-provider response time is more than 10ms and sla is more than 1%.
+    startTime: gt 0
+

test/e2e/e2e-test/src/test/resources/expected/alarm/silence-before-graphql.yml

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+alarm:
+  total: 2
+  msgs:
+    - startTime: gt 0
+      scope: Service
+      id: ZTJlLXNlcnZpY2UtcHJvdmlkZXI=.1
+      message: Service e2e-service-provider response time is more than 10ms and sla is more than 1%.
+    - startTime: gt 0
+      scope: Service
+      id: ZTJlLXNlcnZpY2UtcHJvdmlkZXI=.1
+      message: Response time of service e2e-service-provider is more than 10ms in 1 minutes of last 10 minutes.

test/e2e/e2e-test/src/test/resources/expected/alarm/silence-before-webhook.yml

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+messages:
+  - scopeId: 1
+    scope: SERVICE
+    name: e2e-service-provider
+    id0: ZTJlLXNlcnZpY2UtcHJvdmlkZXI=.1
+    id1: ""
+    ruleName: service_resp_time_rule
+    alarmMessage: Response time of service e2e-service-provider is more than 10ms in 1 minutes of last 10 minutes.
+    startTime: gt 0
+  - scopeId: 1
+    scope: SERVICE
+    name: e2e-service-provider
+    id0: ZTJlLXNlcnZpY2UtcHJvdmlkZXI=.1
+    id1: ""
+    ruleName: comp_rule
+    alarmMessage: Service e2e-service-provider response time is more than 10ms and sla is more than 1%.
+    startTime: gt 0