From 1781d700dd4a34c103c3378ed7bdc7d3050d6ed2 Mon Sep 17 00:00:00 2001 From: wusheng Date: Tue, 12 Apr 2016 15:30:41 +0800 Subject: [PATCH] =?UTF-8?q?*=20=E5=A2=9E=E5=8A=A0=20#43=20=E6=8F=8F?= =?UTF-8?q?=E8=BF=B0=E7=9A=84=E6=97=B6=E9=97=B4=E8=B6=85=E6=97=B6=E5=91=8A?= =?UTF-8?q?=E8=AD=A6=E8=83=BD=E5=8A=9B=20*=20=E9=92=88=E5=AF=B9=20#26=20?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E2=80=9C=E8=BF=90=E8=A1=8C=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E7=BC=93=E6=85=A2=E5=91=8A=E8=AD=A6=E2=80=9D=E7=9A=84=E6=9C=8D?= =?UTF-8?q?=E5=8A=A1=E7=AB=AF=E5=A4=84=E7=90=86=E8=83=BD=E5=8A=9B=EF=BC=8C?= =?UTF-8?q?=E9=9C=80=E8=A6=81alarm=E7=A8=8B=E5=BA=8F=E9=85=8D=E5=90=88?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cloud/skywalking/alarm/conf/Config.java | 6 ++ .../src/main/resources/config.properties | 5 ++ .../skywalking/reciever/conf/Config.java | 6 ++ .../reciever/storage/AlarmRedisConnector.java | 52 ++++++------- .../reciever/storage/chain/AlarmChain.java | 75 +++++++------------ .../storage/chain/ExecuteTimeAlarmChain.java | 17 ----- .../chain/alarm/AbstractSpanChecker.java | 30 ++++++++ .../storage/chain/alarm/ExceptionChecker.java | 31 ++++++++ .../chain/alarm/ExecuteTimeChecker.java | 36 +++++++++ .../storage/chain/alarm/ISpanChecker.java | 7 ++ .../src/main/resources/config.properties | 7 +- 11 files changed, 181 insertions(+), 91 deletions(-) delete mode 100644 skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/ExecuteTimeAlarmChain.java create mode 100644 skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/alarm/AbstractSpanChecker.java create mode 100644 skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/alarm/ExceptionChecker.java create mode 100644 skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/alarm/ExecuteTimeChecker.java create mode 100644 skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/alarm/ISpanChecker.java diff --git a/skywalking-alarm/src/main/java/com/ai/cloud/skywalking/alarm/conf/Config.java b/skywalking-alarm/src/main/java/com/ai/cloud/skywalking/alarm/conf/Config.java index 51d9f4c0e..aea6f2359 100644 --- a/skywalking-alarm/src/main/java/com/ai/cloud/skywalking/alarm/conf/Config.java +++ b/skywalking-alarm/src/main/java/com/ai/cloud/skywalking/alarm/conf/Config.java @@ -83,6 +83,12 @@ public class Config { public static int REDIS_MAX_TOTAL = 20; public static boolean ALARM_OFF_FLAG = false; + + public static class Checker { + public static boolean TURN_ON_EXCEPTION_CHECKER = true; + + public static boolean TURN_ON_EXECUTE_TIME_CHECKER = true; + } } public static class MailSenderInfo { diff --git a/skywalking-alarm/src/main/resources/config.properties b/skywalking-alarm/src/main/resources/config.properties index b8758fefc..d854596d6 100644 --- a/skywalking-alarm/src/main/resources/config.properties +++ b/skywalking-alarm/src/main/resources/config.properties @@ -60,6 +60,11 @@ alarm.redis_max_total=50 #是否关闭告警发送 alarm.alarm_off_flag=false +#告警检查器：异常告警检查 +alarm.checker.turn_on_exception_checker=true +#告警检查器：执行时间超时告警检查 +alarm.checker.turn_on_execute_time_checker=true + #邮件发送配置id mailsenderinfo.configid=1000 #邮件模板配置id diff --git a/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/conf/Config.java b/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/conf/Config.java index b893fbe41..f4b262ea9 100644 --- a/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/conf/Config.java +++ b/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/conf/Config.java @@ -103,6 +103,12 @@ public class Config { public static boolean ALARM_OFF_FLAG = false; public static long ALARM_REDIS_INSPECTOR_INTERVAL = 5 * 1000L; + + public static class Checker { + public static boolean TURN_ON_EXCEPTION_CHECKER = true; + + public static boolean TURN_ON_EXECUTE_TIME_CHECKER = true; + } } public static class HealthCollector { diff --git a/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/AlarmRedisConnector.java b/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/AlarmRedisConnector.java index 904d037d4..3d6ff1e7d 100644 --- a/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/AlarmRedisConnector.java +++ b/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/AlarmRedisConnector.java @@ -21,15 +21,12 @@ public class AlarmRedisConnector { private static JedisPool jedisPool; static { - new RedisInspector().start(); + new RedisInspector().connect().start(); } public static Jedis getJedis() { if (Config.Alarm.ALARM_OFF_FLAG) { return null; - } else if (jedisPool == null || jedisPool.isClosed()) { - reportJedisFailure(); - return null; } else { return jedisPool.getResource(); } @@ -62,6 +59,31 @@ public class AlarmRedisConnector { } } + private RedisInspector connect() { + if (jedisPool != null && !jedisPool.isClosed()) { + jedisPool.close(); + } + + GenericObjectPoolConfig genericObjectPoolConfig = buildGenericObjectPoolConfig(); + jedisPool = new JedisPool(genericObjectPoolConfig, config[0], + Integer.valueOf(config[1])); + // Test connect redis. + Jedis jedis = null; + try { + jedis = jedisPool.getResource(); + jedis.get("ok"); + needConnectInit = false; + } catch (Exception e) { + logger.error("can't connect to redis[" + + Config.Alarm.REDIS_SERVER + "]", e); + } finally { + if (jedis != null) { + jedis.close(); + } + } + return this; + } + @Override public void run() { if (Config.Alarm.ALARM_OFF_FLAG) @@ -70,27 +92,7 @@ public class AlarmRedisConnector { while (true) { try { if (needConnectInit) { - if (jedisPool != null && !jedisPool.isClosed()) { - jedisPool.close(); - } - - GenericObjectPoolConfig genericObjectPoolConfig = buildGenericObjectPoolConfig(); - jedisPool = new JedisPool(genericObjectPoolConfig, - config[0], Integer.valueOf(config[1])); - // Test connect redis. - Jedis jedis = null; - try { - jedis = jedisPool.getResource(); - jedis.get("ok"); - needConnectInit = false; - } catch (Exception e) { - logger.error("can't connect to redis[" - + Config.Alarm.REDIS_SERVER + "]", e); - } finally { - if (jedis != null) { - jedis.close(); - } - } + connect(); } if (needConnectInit) { diff --git a/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/AlarmChain.java b/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/AlarmChain.java index 827b7ae46..0618d9b3d 100644 --- a/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/AlarmChain.java +++ b/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/AlarmChain.java @@ -1,66 +1,45 @@ package com.ai.cloud.skywalking.reciever.storage.chain; -import static com.ai.cloud.skywalking.reciever.conf.Config.Alarm.ALARM_EXCEPTION_STACK_LENGTH; -import static com.ai.cloud.skywalking.reciever.conf.Config.Alarm.ALARM_EXPIRE_SECONDS; - +import java.util.ArrayList; import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import redis.clients.jedis.Jedis; - import com.ai.cloud.skywalking.protocol.Span; import com.ai.cloud.skywalking.reciever.conf.Config; -import com.ai.cloud.skywalking.reciever.storage.AlarmRedisConnector; import com.ai.cloud.skywalking.reciever.storage.Chain; import com.ai.cloud.skywalking.reciever.storage.IStorageChain; +import com.ai.cloud.skywalking.reciever.storage.chain.alarm.ExceptionChecker; +import com.ai.cloud.skywalking.reciever.storage.chain.alarm.ExecuteTimeChecker; +import com.ai.cloud.skywalking.reciever.storage.chain.alarm.ISpanChecker; + +import static com.ai.cloud.skywalking.reciever.conf.Config.Alarm.Checker.*; public class AlarmChain implements IStorageChain { - private static Logger logger = LogManager.getLogger(AlarmChain.class); + private static Logger logger = LogManager.getLogger(AlarmChain.class); - @Override - public void doChain(List spans, Chain chain) { - for (Span span : spans) { - if (span.getStatusCode() != 1) - continue; - String exceptionStack = span.getExceptionStack(); - if (exceptionStack == null) { - exceptionStack = ""; - } else if (exceptionStack.length() > ALARM_EXCEPTION_STACK_LENGTH) { - exceptionStack = exceptionStack.substring(0, ALARM_EXCEPTION_STACK_LENGTH); - } - saveAlarmMessage(generateAlarmKey(span), span.getTraceId(), exceptionStack); - } - chain.doChain(spans); - } + private List checkList = new ArrayList(); - private String generateAlarmKey(Span span) { - return span.getUserId() + "-" + span.getApplicationId() + "-" - + (System.currentTimeMillis() / (10000 * 6)); - } + public AlarmChain() { + if (TURN_ON_EXCEPTION_CHECKER) + checkList.add(new ExceptionChecker()); + if (TURN_ON_EXECUTE_TIME_CHECKER) + checkList.add(new ExecuteTimeChecker()); + } - private void saveAlarmMessage(String key, String traceId, String exceptionMsgOutline) { - if (Config.Alarm.ALARM_OFF_FLAG) { - return; - } + @Override + public void doChain(List spans, Chain chain) { + if (Config.Alarm.ALARM_OFF_FLAG) { + return; + } + + for (Span span : spans) { + for (ISpanChecker checker : checkList) { + checker.check(span); + } + } + chain.doChain(spans); + } - Jedis jedis = null; - try { - jedis = AlarmRedisConnector.getJedis(); - if(jedis == null){ - logger.error("Failed to set data. can't get jedis."); - return; - } - jedis.hsetnx(key, traceId, exceptionMsgOutline); - jedis.expire(key, ALARM_EXPIRE_SECONDS); - } catch (Exception e) { - AlarmRedisConnector.reportJedisFailure(); - logger.error("Failed to set data.", e); - } finally { - if (jedis != null) { - jedis.close(); - } - } - } } diff --git a/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/ExecuteTimeAlarmChain.java b/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/ExecuteTimeAlarmChain.java deleted file mode 100644 index 233fcc0f8..000000000 --- a/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/ExecuteTimeAlarmChain.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.ai.cloud.skywalking.reciever.storage.chain; - -import java.util.List; - -import com.ai.cloud.skywalking.protocol.Span; -import com.ai.cloud.skywalking.reciever.storage.Chain; -import com.ai.cloud.skywalking.reciever.storage.IStorageChain; - -public class ExecuteTimeAlarmChain implements IStorageChain { - - @Override - public void doChain(List spans, Chain chain) { - // TODO Auto-generated method stub - - } - -} diff --git a/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/alarm/AbstractSpanChecker.java b/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/alarm/AbstractSpanChecker.java new file mode 100644 index 000000000..e6567aec0 --- /dev/null +++ b/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/alarm/AbstractSpanChecker.java @@ -0,0 +1,30 @@ +package com.ai.cloud.skywalking.reciever.storage.chain.alarm; + +import static com.ai.cloud.skywalking.reciever.conf.Config.Alarm.ALARM_EXPIRE_SECONDS; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import redis.clients.jedis.Jedis; + +import com.ai.cloud.skywalking.reciever.storage.AlarmRedisConnector; + +public abstract class AbstractSpanChecker implements ISpanChecker { + private static Logger logger = LogManager.getLogger(AbstractSpanChecker.class); + + protected void saveAlarmMessage(String key, String traceId, String alarmMsg) { + Jedis jedis = null; + try { + jedis = AlarmRedisConnector.getJedis(); + jedis.hsetnx(key, traceId, alarmMsg); + jedis.expire(key, ALARM_EXPIRE_SECONDS); + } catch (Exception e) { + AlarmRedisConnector.reportJedisFailure(); + logger.error("Failed to set data.", e); + } finally { + if (jedis != null) { + jedis.close(); + } + } + } +} diff --git a/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/alarm/ExceptionChecker.java b/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/alarm/ExceptionChecker.java new file mode 100644 index 000000000..23d42f764 --- /dev/null +++ b/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/alarm/ExceptionChecker.java @@ -0,0 +1,31 @@ +package com.ai.cloud.skywalking.reciever.storage.chain.alarm; + +import static com.ai.cloud.skywalking.reciever.conf.Config.Alarm.ALARM_EXCEPTION_STACK_LENGTH; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ai.cloud.skywalking.protocol.Span; + +public class ExceptionChecker extends AbstractSpanChecker { + private static Logger logger = LogManager.getLogger(ExceptionChecker.class); + + @Override + public void check(Span span) { + if (span.getStatusCode() != 1) + return; + String exceptionStack = span.getExceptionStack(); + if (exceptionStack == null) { + exceptionStack = ""; + } else if (exceptionStack.length() > ALARM_EXCEPTION_STACK_LENGTH) { + exceptionStack = exceptionStack.substring(0, ALARM_EXCEPTION_STACK_LENGTH); + } + saveAlarmMessage(generateAlarmKey(span), span.getTraceId(), exceptionStack); + } + + private String generateAlarmKey(Span span) { + return span.getUserId() + "-" + span.getApplicationId() + "-" + + (System.currentTimeMillis() / (10000 * 6)); + } + +} diff --git a/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/alarm/ExecuteTimeChecker.java b/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/alarm/ExecuteTimeChecker.java new file mode 100644 index 000000000..8c4d15861 --- /dev/null +++ b/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/alarm/ExecuteTimeChecker.java @@ -0,0 +1,36 @@ +package com.ai.cloud.skywalking.reciever.storage.chain.alarm; + +import com.ai.cloud.skywalking.protocol.Span; + +public class ExecuteTimeChecker extends AbstractSpanChecker { + + @Override + public void check(Span span) { + long cost = span.getCost(); + + if (cost > 500 && cost < 3000) { + /** + * Issue #43
+ * 单埋点调用时间超过500ms的进行预警 + */ + saveAlarmMessage(generateWarningAlarmKey(span), span.getTraceId(), span.getViewPointId() + " cost " + cost + " ms."); + } + if (cost >= 3000) { + /** + * Issue #43
+ * 单埋点调用时间超过3S的进行告警 + */ + saveAlarmMessage(generatePossibleErrorAlarmKey(span), span.getTraceId(), span.getViewPointId() + " cost " + cost + " ms."); + } + } + + private String generateWarningAlarmKey(Span span) { + return span.getUserId() + "-" + span.getApplicationId() + "-" + + (System.currentTimeMillis() / (10000 * 6)) + "-ExecuteTime-Warning"; + } + + private String generatePossibleErrorAlarmKey(Span span) { + return span.getUserId() + "-" + span.getApplicationId() + "-" + + (System.currentTimeMillis() / (10000 * 6)) + "-ExecuteTime-PossibleError"; + } +} diff --git a/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/alarm/ISpanChecker.java b/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/alarm/ISpanChecker.java new file mode 100644 index 000000000..85ced5c13 --- /dev/null +++ b/skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/alarm/ISpanChecker.java @@ -0,0 +1,7 @@ +package com.ai.cloud.skywalking.reciever.storage.chain.alarm; + +import com.ai.cloud.skywalking.protocol.Span; + +public interface ISpanChecker { + void check(Span span); +} diff --git a/skywalking-server/src/main/resources/config.properties b/skywalking-server/src/main/resources/config.properties index 04acb0a76..9442627e0 100644 --- a/skywalking-server/src/main/resources/config.properties +++ b/skywalking-server/src/main/resources/config.properties @@ -62,4 +62,9 @@ alarm.edis_max_total=20 #是否关闭告警 alarm.larm_off_flag=false #告警redis检测器检测周期 -alarm.alarm_redis_inspector_interval=5000 \ No newline at end of file +alarm.alarm_redis_inspector_interval=5000 + +#告警检查器：异常告警检查 +alarm.checker.turn_on_exception_checker=true +#告警检查器：执行时间超时告警检查 +alarm.checker.turn_on_execute_time_checker=true \ No newline at end of file