kazusa
/
skywalking-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* 增加 #43 描述的时间超时告警能力 

* 针对 #26 增加“运行时间缓慢告警”的服务端处理能力，需要alarm程序配合实现
This commit is contained in:
wusheng 2016-04-12 15:30:41 +08:00
parent 2a49c5939e
commit 1781d700dd
11 changed files with 181 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
skywalking-alarm/src/main/java/com/ai/cloud/skywalking/alarm/conf/Config.java
View File

@ -83,6 +83,12 @@ public class Config {
public static int REDIS_MAX_TOTAL = 20;
public static boolean ALARM_OFF_FLAG = false;
public static class Checker {
public static boolean TURN_ON_EXCEPTION_CHECKER = true;
public static boolean TURN_ON_EXECUTE_TIME_CHECKER = true;
}
}
public static class MailSenderInfo {

5
skywalking-alarm/src/main/resources/config.properties
View File

@ -60,6 +60,11 @@ alarm.redis_max_total=50
#是否关闭告警发送
alarm.alarm_off_flag=false
#告警检查器：异常告警检查
alarm.checker.turn_on_exception_checker=true
#告警检查器：执行时间超时告警检查
alarm.checker.turn_on_execute_time_checker=true
#邮件发送配置id
mailsenderinfo.configid=1000
#邮件模板配置id

6
skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/conf/Config.java
View File

@ -103,6 +103,12 @@ public class Config {
public static boolean ALARM_OFF_FLAG = false;
public static long ALARM_REDIS_INSPECTOR_INTERVAL = 5 * 1000L;
public static class Checker {
public static boolean TURN_ON_EXCEPTION_CHECKER = true;
public static boolean TURN_ON_EXECUTE_TIME_CHECKER = true;
}
}
public static class HealthCollector {

52
skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/AlarmRedisConnector.java
View File

@ -21,15 +21,12 @@ public class AlarmRedisConnector {
private static JedisPool jedisPool;
static {
new RedisInspector().start();
new RedisInspector().connect().start();
}
public static Jedis getJedis() {
if (Config.Alarm.ALARM_OFF_FLAG) {
return null;
} else if (jedisPool == null || jedisPool.isClosed()) {
reportJedisFailure();
return null;
} else {
return jedisPool.getResource();
}
@ -62,6 +59,31 @@ public class AlarmRedisConnector {
}
}
private RedisInspector connect() {
if (jedisPool != null && !jedisPool.isClosed()) {
jedisPool.close();
}
GenericObjectPoolConfig genericObjectPoolConfig = buildGenericObjectPoolConfig();
jedisPool = new JedisPool(genericObjectPoolConfig, config[0],
Integer.valueOf(config[1]));
// Test connect redis.
Jedis jedis = null;
try {
jedis = jedisPool.getResource();
jedis.get("ok");
needConnectInit = false;
} catch (Exception e) {
logger.error("can't connect to redis["
+ Config.Alarm.REDIS_SERVER + "]", e);
} finally {
if (jedis != null) {
jedis.close();
}
}
return this;
}
@Override
public void run() {
if (Config.Alarm.ALARM_OFF_FLAG)
@ -70,27 +92,7 @@ public class AlarmRedisConnector {
while (true) {
try {
if (needConnectInit) {
if (jedisPool != null && !jedisPool.isClosed()) {
jedisPool.close();
}
GenericObjectPoolConfig genericObjectPoolConfig = buildGenericObjectPoolConfig();
jedisPool = new JedisPool(genericObjectPoolConfig,
config[0], Integer.valueOf(config[1]));
// Test connect redis.
Jedis jedis = null;
try {
jedis = jedisPool.getResource();
jedis.get("ok");
needConnectInit = false;
} catch (Exception e) {
logger.error("can't connect to redis["
+ Config.Alarm.REDIS_SERVER + "]", e);
} finally {
if (jedis != null) {
jedis.close();
}
}
connect();
}
if (needConnectInit) {

75
skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/AlarmChain.java
View File

@ -1,66 +1,45 @@
package com.ai.cloud.skywalking.reciever.storage.chain;
import static com.ai.cloud.skywalking.reciever.conf.Config.Alarm.ALARM_EXCEPTION_STACK_LENGTH;
import static com.ai.cloud.skywalking.reciever.conf.Config.Alarm.ALARM_EXPIRE_SECONDS;
import java.util.ArrayList;
import java.util.List;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import redis.clients.jedis.Jedis;
import com.ai.cloud.skywalking.protocol.Span;
import com.ai.cloud.skywalking.reciever.conf.Config;
import com.ai.cloud.skywalking.reciever.storage.AlarmRedisConnector;
import com.ai.cloud.skywalking.reciever.storage.Chain;
import com.ai.cloud.skywalking.reciever.storage.IStorageChain;
import com.ai.cloud.skywalking.reciever.storage.chain.alarm.ExceptionChecker;
import com.ai.cloud.skywalking.reciever.storage.chain.alarm.ExecuteTimeChecker;
import com.ai.cloud.skywalking.reciever.storage.chain.alarm.ISpanChecker;
import static com.ai.cloud.skywalking.reciever.conf.Config.Alarm.Checker.*;
public class AlarmChain implements IStorageChain {
private static Logger logger = LogManager.getLogger(AlarmChain.class);
private static Logger logger = LogManager.getLogger(AlarmChain.class);
@Override
public void doChain(List<Span> spans, Chain chain) {
for (Span span : spans) {
if (span.getStatusCode() != 1)
continue;
String exceptionStack = span.getExceptionStack();
if (exceptionStack == null) {
exceptionStack = "";
} else if (exceptionStack.length() > ALARM_EXCEPTION_STACK_LENGTH) {
exceptionStack = exceptionStack.substring(0, ALARM_EXCEPTION_STACK_LENGTH);
}
saveAlarmMessage(generateAlarmKey(span), span.getTraceId(), exceptionStack);
}
chain.doChain(spans);
}
private List<ISpanChecker> checkList = new ArrayList<ISpanChecker>();
private String generateAlarmKey(Span span) {
return span.getUserId() + "-" + span.getApplicationId() + "-"
+ (System.currentTimeMillis() / (10000 * 6));
}
public AlarmChain() {
if (TURN_ON_EXCEPTION_CHECKER)
checkList.add(new ExceptionChecker());
if (TURN_ON_EXECUTE_TIME_CHECKER)
checkList.add(new ExecuteTimeChecker());
}
private void saveAlarmMessage(String key, String traceId, String exceptionMsgOutline) {
if (Config.Alarm.ALARM_OFF_FLAG) {
return;
}
@Override
public void doChain(List<Span> spans, Chain chain) {
if (Config.Alarm.ALARM_OFF_FLAG) {
return;
}
for (Span span : spans) {
for (ISpanChecker checker : checkList) {
checker.check(span);
}
}
chain.doChain(spans);
}
Jedis jedis = null;
try {
jedis = AlarmRedisConnector.getJedis();
if(jedis == null){
logger.error("Failed to set data. can't get jedis.");
return;
}
jedis.hsetnx(key, traceId, exceptionMsgOutline);
jedis.expire(key, ALARM_EXPIRE_SECONDS);
} catch (Exception e) {
AlarmRedisConnector.reportJedisFailure();
logger.error("Failed to set data.", e);
} finally {
if (jedis != null) {
jedis.close();
}
}
}
}

17
skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/ExecuteTimeAlarmChain.java
View File

@ -1,17 +0,0 @@
package com.ai.cloud.skywalking.reciever.storage.chain;
import java.util.List;
import com.ai.cloud.skywalking.protocol.Span;
import com.ai.cloud.skywalking.reciever.storage.Chain;
import com.ai.cloud.skywalking.reciever.storage.IStorageChain;
public class ExecuteTimeAlarmChain implements IStorageChain {
@Override
public void doChain(List<Span> spans, Chain chain) {
// TODO Auto-generated method stub
}
}

30
skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/alarm/AbstractSpanChecker.java Normal file
View File

@ -0,0 +1,30 @@
package com.ai.cloud.skywalking.reciever.storage.chain.alarm;
import static com.ai.cloud.skywalking.reciever.conf.Config.Alarm.ALARM_EXPIRE_SECONDS;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import redis.clients.jedis.Jedis;
import com.ai.cloud.skywalking.reciever.storage.AlarmRedisConnector;
public abstract class AbstractSpanChecker implements ISpanChecker {
private static Logger logger = LogManager.getLogger(AbstractSpanChecker.class);
protected void saveAlarmMessage(String key, String traceId, String alarmMsg) {
Jedis jedis = null;
try {
jedis = AlarmRedisConnector.getJedis();
jedis.hsetnx(key, traceId, alarmMsg);
jedis.expire(key, ALARM_EXPIRE_SECONDS);
} catch (Exception e) {
AlarmRedisConnector.reportJedisFailure();
logger.error("Failed to set data.", e);
} finally {
if (jedis != null) {
jedis.close();
}
}
}
}

31
skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/alarm/ExceptionChecker.java Normal file
View File

@ -0,0 +1,31 @@
package com.ai.cloud.skywalking.reciever.storage.chain.alarm;
import static com.ai.cloud.skywalking.reciever.conf.Config.Alarm.ALARM_EXCEPTION_STACK_LENGTH;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.ai.cloud.skywalking.protocol.Span;
public class ExceptionChecker extends AbstractSpanChecker {
private static Logger logger = LogManager.getLogger(ExceptionChecker.class);
@Override
public void check(Span span) {
if (span.getStatusCode() != 1)
return;
String exceptionStack = span.getExceptionStack();
if (exceptionStack == null) {
exceptionStack = "";
} else if (exceptionStack.length() > ALARM_EXCEPTION_STACK_LENGTH) {
exceptionStack = exceptionStack.substring(0, ALARM_EXCEPTION_STACK_LENGTH);
}
saveAlarmMessage(generateAlarmKey(span), span.getTraceId(), exceptionStack);
}
private String generateAlarmKey(Span span) {
return span.getUserId() + "-" + span.getApplicationId() + "-"
+ (System.currentTimeMillis() / (10000 * 6));
}
}

36
skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/alarm/ExecuteTimeChecker.java Normal file
View File

@ -0,0 +1,36 @@
package com.ai.cloud.skywalking.reciever.storage.chain.alarm;
import com.ai.cloud.skywalking.protocol.Span;
public class ExecuteTimeChecker extends AbstractSpanChecker {
@Override
public void check(Span span) {
long cost = span.getCost();
if (cost > 500 && cost < 3000) {
/**
* Issue #43 <br/>
* 单埋点调用时间超过500ms的进行预警
*/
saveAlarmMessage(generateWarningAlarmKey(span), span.getTraceId(), span.getViewPointId() + " cost " + cost + " ms.");
}
if (cost >= 3000) {
/**
* Issue #43 <br/>
* 单埋点调用时间超过3S的进行告警
*/
saveAlarmMessage(generatePossibleErrorAlarmKey(span), span.getTraceId(), span.getViewPointId() + " cost " + cost + " ms.");
}
}
private String generateWarningAlarmKey(Span span) {
return span.getUserId() + "-" + span.getApplicationId() + "-"
+ (System.currentTimeMillis() / (10000 * 6)) + "-ExecuteTime-Warning";
}
private String generatePossibleErrorAlarmKey(Span span) {
return span.getUserId() + "-" + span.getApplicationId() + "-"
+ (System.currentTimeMillis() / (10000 * 6)) + "-ExecuteTime-PossibleError";
}
}

7
skywalking-server/src/main/java/com/ai/cloud/skywalking/reciever/storage/chain/alarm/ISpanChecker.java Normal file
View File

@ -0,0 +1,7 @@
package com.ai.cloud.skywalking.reciever.storage.chain.alarm;
import com.ai.cloud.skywalking.protocol.Span;
public interface ISpanChecker {
void check(Span span);
}

7
skywalking-server/src/main/resources/config.properties
View File

@ -62,4 +62,9 @@ alarm.edis_max_total=20
#是否关闭告警
alarm.larm_off_flag=false
#告警redis检测器检测周期
alarm.alarm_redis_inspector_interval=5000
alarm.alarm_redis_inspector_interval=5000
#告警检查器：异常告警检查
alarm.checker.turn_on_exception_checker=true
#告警检查器：执行时间超时告警检查
alarm.checker.turn_on_execute_time_checker=true