1. 增加MapReduce的java_opts的配置,在有些情况下会导致内存溢出,无法进行MapReduce
2. 没有EntranceNode不进行Reduce
This commit is contained in:
parent
e9a1b4cfab
commit
2b6377fb3d
|
|
@ -12,6 +12,7 @@ import com.ai.cloud.skywalking.analysis.config.HBaseTableMetaData;
|
|||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.conf.Configured;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.client.Scan;
|
||||
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
|
||||
import org.apache.hadoop.io.Text;
|
||||
|
|
@ -55,6 +56,8 @@ public class AnalysisServerDriver extends Configured implements Tool {
|
|||
conf.set("skywalking.analysis.mode", String.valueOf(Config.AnalysisServer.IS_ACCUMULATE_MODE));
|
||||
conf.set("hbase.zookeeper.quorum", Config.HBase.ZK_QUORUM);
|
||||
conf.set("hbase.zookeeper.property.clientPort", Config.HBase.ZK_CLIENT_PORT);
|
||||
//-XX:+UseParallelGC -XX:ParallelGCThreads=4 -XX:GCTimeRatio=10 -XX:YoungGenerationSizeIncrement=20 -XX:TenuredGenerationSizeIncrement=20 -XX:AdaptiveSizeDecrementScaleFactor=2
|
||||
conf.set("mapred.child.java.opts",Config.MapReduce.java_opts);
|
||||
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
|
||||
if (otherArgs.length != 2) {
|
||||
System.err.println("Usage: AnalysisServer yyyy-MM-dd/HH:mm:ss yyyy-MM-dd/HH:mm:ss");
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ import org.apache.hadoop.io.Text;
|
|||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
|
||||
|
|
@ -56,8 +55,10 @@ public class ChainBuildMapper extends TableMapper<Text, Text> {
|
|||
chainInfo = spanToChainInfo(Bytes.toString(key.get()), spanList);
|
||||
logger.debug("convert tid[" + Bytes.toString(key.get())
|
||||
+ "] to chain with cid[" + chainInfo.getCID() + "].");
|
||||
context.write(
|
||||
new Text(chainInfo.getCallEntrance()), new Text(new Gson().toJson(chainInfo)));
|
||||
if (chainInfo.getCallEntrance() != null && chainInfo.getCallEntrance().length() > 0) {
|
||||
context.write(
|
||||
new Text(chainInfo.getCallEntrance()), new Text(new Gson().toJson(chainInfo)));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error("Failed to mapper call chain[" + key.toString() + "]",
|
||||
e);
|
||||
|
|
|
|||
|
|
@ -29,4 +29,8 @@ public class Config {
|
|||
public static class AnalysisServer{
|
||||
public static boolean IS_ACCUMULATE_MODE = true;
|
||||
}
|
||||
|
||||
public static class MapReduce{
|
||||
public static String java_opts = "-Xmx200m";
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,4 +15,6 @@ filter.filter_package_name=com.ai.cloud.skywalking.analysis.chainbuild.filter.im
|
|||
|
||||
chainnodesummary.interval=1
|
||||
|
||||
reducer.reducer_number=4
|
||||
reducer.reducer_number=4
|
||||
|
||||
mapreduce.java_opts=-Xmx512m
|
||||
Loading…
Reference in New Issue