1. 增加MapReduce的java_opts的配置,在有些情况下会导致内存溢出,无法进行MapReduce

2. 没有EntranceNode不进行Reduce
This commit is contained in:
ascrutae 2016-04-11 19:02:57 +08:00
parent e9a1b4cfab
commit 2b6377fb3d
4 changed files with 14 additions and 4 deletions

View File

@ -12,6 +12,7 @@ import com.ai.cloud.skywalking.analysis.config.HBaseTableMetaData;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.io.Text;
@ -55,6 +56,8 @@ public class AnalysisServerDriver extends Configured implements Tool {
conf.set("skywalking.analysis.mode", String.valueOf(Config.AnalysisServer.IS_ACCUMULATE_MODE));
conf.set("hbase.zookeeper.quorum", Config.HBase.ZK_QUORUM);
conf.set("hbase.zookeeper.property.clientPort", Config.HBase.ZK_CLIENT_PORT);
//-XX:+UseParallelGC -XX:ParallelGCThreads=4 -XX:GCTimeRatio=10 -XX:YoungGenerationSizeIncrement=20 -XX:TenuredGenerationSizeIncrement=20 -XX:AdaptiveSizeDecrementScaleFactor=2
conf.set("mapred.child.java.opts",Config.MapReduce.java_opts);
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if (otherArgs.length != 2) {
System.err.println("Usage: AnalysisServer yyyy-MM-dd/HH:mm:ss yyyy-MM-dd/HH:mm:ss");

View File

@ -19,7 +19,6 @@ import org.apache.hadoop.io.Text;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.util.*;
@ -56,8 +55,10 @@ public class ChainBuildMapper extends TableMapper<Text, Text> {
chainInfo = spanToChainInfo(Bytes.toString(key.get()), spanList);
logger.debug("convert tid[" + Bytes.toString(key.get())
+ "] to chain with cid[" + chainInfo.getCID() + "].");
context.write(
new Text(chainInfo.getCallEntrance()), new Text(new Gson().toJson(chainInfo)));
if (chainInfo.getCallEntrance() != null && chainInfo.getCallEntrance().length() > 0) {
context.write(
new Text(chainInfo.getCallEntrance()), new Text(new Gson().toJson(chainInfo)));
}
} catch (Exception e) {
logger.error("Failed to mapper call chain[" + key.toString() + "]",
e);

View File

@ -29,4 +29,8 @@ public class Config {
public static class AnalysisServer{
public static boolean IS_ACCUMULATE_MODE = true;
}
public static class MapReduce{
public static String java_opts = "-Xmx200m";
}
}

View File

@ -15,4 +15,6 @@ filter.filter_package_name=com.ai.cloud.skywalking.analysis.chainbuild.filter.im
chainnodesummary.interval=1
reducer.reducer_number=4
reducer.reducer_number=4
mapreduce.java_opts=-Xmx512m