From 1dd349037b3e9105f867bcba7a0d158b8cde1e2e Mon Sep 17 00:00:00 2001 From: wusheng Date: Thu, 5 May 2016 23:48:14 +0800 Subject: [PATCH] =?UTF-8?q?1.=E6=8F=90=E4=BE=9B=E5=A4=A7=E9=87=8F=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=E6=96=B9=E6=B3=95=EF=BC=8C=E4=BB=A5=E6=8F=90=E9=AB=98?= =?UTF-8?q?mr=E4=BB=BB=E5=8A=A1=E7=9A=84=E6=95=88=E7=8E=87=EF=BC=8C?= =?UTF-8?q?=E9=99=8D=E4=BD=8E=E5=A4=84=E7=90=86=E5=A4=8D=E6=9D=82=E5=BA=A6?= =?UTF-8?q?=E5=92=8C=E6=95=B0=E6=8D=AE=E9=87=8F=202.=E9=9C=80=E8=A6=81?= =?UTF-8?q?=E5=A4=A7=E8=A7=84=E6=A8=A1=E9=99=8D=E4=BD=8E=E5=AF=B9=E8=B1=A1?= =?UTF-8?q?=E7=AC=A6=E5=90=88=E5=85=B3=E7=B3=BB=EF=BC=8C=E9=99=8D=E4=BD=8E?= =?UTF-8?q?GC=E5=8E=8B=E5=8A=9B=E5=92=8C=E5=AF=B9=E8=B1=A1=E5=88=9B?= =?UTF-8?q?=E5=BB=BA=E5=8E=8B=E5=8A=9B=203.=E9=80=9A=E8=BF=87=E6=9B=B4?= =?UTF-8?q?=E7=BB=86=E7=B2=92=E5=BA=A6=E7=9A=84=E4=BB=BB=E5=8A=A1=EF=BC=8C?= =?UTF-8?q?=E6=8F=90=E9=AB=98mr=E8=BF=9B=E5=BA=A6=E7=9A=84=E7=B2=BE?= =?UTF-8?q?=E5=BA=A6=EF=BC=8C=E5=B9=B6=E5=A4=A7=E5=B9=85=E5=BA=A6=E5=87=8F?= =?UTF-8?q?=E5=B0=91=E5=86=85=E5=AD=98=E9=9C=80=E6=B1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../analysis/chainbuild/ChainBuildMapper.java | 27 +++++++++++++ .../impl/NumberOfCalledStatisticsAction.java | 40 +++++++++++++++++-- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/chainbuild/ChainBuildMapper.java b/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/chainbuild/ChainBuildMapper.java index cec2acd5c..67e3971f7 100644 --- a/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/chainbuild/ChainBuildMapper.java +++ b/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/chainbuild/ChainBuildMapper.java @@ -68,6 +68,22 @@ public class ChainBuildMapper extends TableMapper { if (chainInfo.getCallEntrance() != null && chainInfo.getCallEntrance().length() > 0) { for (ChainNode chainNode : chainInfo.getNodes()) { + /** + * TODO: 进一步提高运行速度所需的性能提升 + * 此处修改原因， + * 1.更细粒度划分reduce任务，提高性能。 + * 2.减少数据传输量，以及处理复杂度。 + * 3.请避免使用gson序列化，提高程序处理性能 + * + * hour/day/month/year， + * key 修改为：类型+时间字符+callEntrance+levelId+viewpoint, + * value 为ChainNodeSpecificTimeWindowSummaryValue中所需的明确的值组成的简单串 + * value包含： + * 1.是否正确调用，由NodeStatus获取，值为N/A/I + * 2.调用所需时间，由cost获取 + * + */ + context.write(new Text(SummaryType.HOUR.getValue() + "-" + hourSimpleDateFormat.format( new Date(chainNode.getStartDate()) ) + ":" + chainInfo.getCallEntrance()), new Text(new Gson().toJson(chainNode))); @@ -84,6 +100,17 @@ public class ChainBuildMapper extends TableMapper { new Date(chainNode.getStartDate()) ) + ":" + chainInfo.getCallEntrance()), new Text(new Gson().toJson(chainNode))); } + + /** + * TODO：通过对本地的调用链进行缓存，每个map任务中的调用链，在一个JVM内只会被传递一次，大幅度降低reduce任务的数据量。 + * + * 1.使用静态变量，缓存MAP中的调用链。每个典型调用链ID只传递一次 + * 2.注意缓存需要限制容量，初期规划，缓存1W个典型调用链KEY（可通过配置扩展）。仅缓存典型调用链ID，非链路明细。 + * 3.对于节点数量大于2K条的调用量，暂不进行关系处理 + * + * 注意：CallChainRelationshipAction暂时不做修改，此处的修改会大规模降低reduce的处理数据量，提高总体运行速度 + * + */ // Reduce key : R-CallEntrance context.write(new Text(SummaryType.RELATIONSHIP + "-" + TokenGenerator.generateTreeToken(chainInfo.getCallEntrance()) + ":" + chainInfo.getCallEntrance()), diff --git a/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/chainbuild/action/impl/NumberOfCalledStatisticsAction.java b/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/chainbuild/action/impl/NumberOfCalledStatisticsAction.java index f96f76082..322993f8f 100644 --- a/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/chainbuild/action/impl/NumberOfCalledStatisticsAction.java +++ b/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/chainbuild/action/impl/NumberOfCalledStatisticsAction.java @@ -1,19 +1,28 @@ package com.ai.cloud.skywalking.analysis.chainbuild.action.impl; +import java.io.IOException; +import java.sql.SQLException; + import com.ai.cloud.skywalking.analysis.chainbuild.action.IStatisticsAction; import com.ai.cloud.skywalking.analysis.chainbuild.entity.CallChainTree; import com.ai.cloud.skywalking.analysis.chainbuild.entity.CallChainTreeNode; +import com.ai.cloud.skywalking.analysis.chainbuild.entity.ChainNodeSpecificTimeWindowSummaryValue; import com.ai.cloud.skywalking.analysis.chainbuild.po.ChainNode; import com.ai.cloud.skywalking.analysis.chainbuild.po.SummaryType; +import com.ai.cloud.skywalking.analysis.config.Config; import com.google.gson.Gson; -import java.io.IOException; -import java.sql.SQLException; - public class NumberOfCalledStatisticsAction implements IStatisticsAction { private CallChainTree callChainTree; private String summaryDate; private SummaryType summaryType; + + /** + * 统计任务的主要存储值 + * + * TODO：配置本次修改 + */ + private ChainNodeSpecificTimeWindowSummaryValue summaryValue = new ChainNodeSpecificTimeWindowSummaryValue(); public NumberOfCalledStatisticsAction(String entryKey, String summaryDate) throws IOException { callChainTree = CallChainTree.create(entryKey); @@ -22,6 +31,21 @@ public class NumberOfCalledStatisticsAction implements IStatisticsAction { @Override public void doAction(String summaryData) throws IOException { + /** + * TODO:根据MAP的优化，大幅度减少对象逻辑复杂度 + * + * 每次传递来的数据，根据NodeStatus的值在内存中快速累加到ChainNodeSpecificTimeWindowSummaryValue中 + * + * ChainNodeSpecificTimeWindowSummaryValue执行以下逻辑： + * 1.totalCall默认+1 + * 2.totalCostTime根据cost累加 + * 3.correctNumber和humanInterruptionNumber根据状态累加 + * + * 注意： + * 1.减少对象创建和嵌套 + * 2.此过程中不要操作任何hbase数据 + */ + ChainNode chainNode = new Gson().fromJson(summaryData, ChainNode.class); CallChainTreeNode newCallChainTreeNode = new CallChainTreeNode(chainNode); CallChainTreeNode callChainTreeNode = callChainTree.getNodes().get(newCallChainTreeNode.getTreeNodeId()); @@ -34,6 +58,16 @@ public class NumberOfCalledStatisticsAction implements IStatisticsAction { @Override public void doSave() throws InterruptedException, SQLException, IOException { + /** + * TODO:新的写入逻辑，快速完成 + * + * 根据Config.AnalysisServer.IS_ACCUMULATE_MODE决定，在需要时，精确读取指定rowkey的指定列值。 + * 和ChainNodeSpecificTimeWindowSummaryValue值进行简单相加，然后汇总写入。 + * + * 注意： + * 1.减少对象创建和嵌套 + * 2。确保任务的告诉高效完成 + */ callChainTree.saveToHBase(summaryType); }