kazusa
/
skywalking-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

1. 修改NodeToken的规则，增加前缀CID 

2. 提交部分Reduce的代码
This commit is contained in:
zhangxin10 2016-01-19 10:16:07 +08:00
parent 5d224c3fd0
commit ab08ff1226
10 changed files with 233 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
skywalking-analysis/skywalking-analysis.iml Normal file
View File

@ -0,0 +1,117 @@
<?xml version="1.0" encoding="UTF-8"?>
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_6" inherit-compiler-output="false">
<output url="file://$MODULE_DIR$/target/classes" />
<output-test url="file://$MODULE_DIR$/target/test-classes" />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.12" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
<orderEntry type="library" name="Maven: org.apache.hbase:hbase-client:1.1.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.hbase:hbase-annotations:1.1.2" level="project" />
<orderEntry type="module-library">
<library name="Maven: jdk.tools:jdk.tools:1.7">
<CLASSES>
<root url="jar://D:/Programs/Java/jdk1.7.0_75/lib/tools.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
<orderEntry type="library" name="Maven: org.apache.hbase:hbase-common:1.1.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.hbase:hbase-protocol:1.1.2" level="project" />
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.9" level="project" />
<orderEntry type="library" name="Maven: commons-io:commons-io:2.4" level="project" />
<orderEntry type="library" name="Maven: commons-lang:commons-lang:2.6" level="project" />
<orderEntry type="library" name="Maven: commons-logging:commons-logging:1.2" level="project" />
<orderEntry type="library" name="Maven: com.google.guava:guava:12.0.1" level="project" />
<orderEntry type="library" name="Maven: com.google.code.findbugs:jsr305:1.3.9" level="project" />
<orderEntry type="library" name="Maven: com.google.protobuf:protobuf-java:2.5.0" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-all:4.0.23.Final" level="project" />
<orderEntry type="library" name="Maven: org.apache.zookeeper:zookeeper:3.4.6" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.6.1" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:slf4j-log4j12:1.6.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.htrace:htrace-core:3.1.0-incubating" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-mapper-asl:1.9.13" level="project" />
<orderEntry type="library" name="Maven: org.jruby.jcodings:jcodings:1.0.8" level="project" />
<orderEntry type="library" name="Maven: org.jruby.joni:joni:2.1.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-auth:2.5.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.2.5" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.2.4" level="project" />
<orderEntry type="library" name="Maven: org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15" level="project" />
<orderEntry type="library" name="Maven: org.apache.directory.server:apacheds-i18n:2.0.0-M15" level="project" />
<orderEntry type="library" name="Maven: org.apache.directory.api:api-asn1-api:1.0.0-M20" level="project" />
<orderEntry type="library" name="Maven: org.apache.directory.api:api-util:1.0.0-M20" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-common:2.5.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-annotations:2.5.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-math3:3.1.1" level="project" />
<orderEntry type="library" name="Maven: xmlenc:xmlenc:0.52" level="project" />
<orderEntry type="library" name="Maven: commons-net:commons-net:3.1" level="project" />
<orderEntry type="library" name="Maven: commons-el:commons-el:1.0" level="project" />
<orderEntry type="library" name="Maven: commons-configuration:commons-configuration:1.6" level="project" />
<orderEntry type="library" name="Maven: commons-digester:commons-digester:1.8" level="project" />
<orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils:1.7.0" level="project" />
<orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils-core:1.8.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.avro:avro:1.7.4" level="project" />
<orderEntry type="library" name="Maven: com.thoughtworks.paranamer:paranamer:2.3" level="project" />
<orderEntry type="library" name="Maven: org.xerial.snappy:snappy-java:1.0.4.1" level="project" />
<orderEntry type="library" name="Maven: com.jcraft:jsch:0.1.42" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-compress:1.4.1" level="project" />
<orderEntry type="library" name="Maven: org.tukaani:xz:1.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-core:2.5.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-common:2.5.1" level="project" />
<orderEntry type="library" name="Maven: javax.xml.bind:jaxb-api:2.2.2" level="project" />
<orderEntry type="library" name="Maven: javax.xml.stream:stax-api:1.0-2" level="project" />
<orderEntry type="library" name="Maven: javax.activation:activation:1.1" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty:3.6.2.Final" level="project" />
<orderEntry type="library" name="Maven: com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hbase:hbase-server:1.1.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.hbase:hbase-procedure:1.1.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.hbase:hbase-common:tests:1.1.2" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.hbase:hbase-prefix-tree:1.1.2" level="project" />
<orderEntry type="library" name="Maven: commons-httpclient:commons-httpclient:3.1" level="project" />
<orderEntry type="library" name="Maven: commons-collections:commons-collections:3.2.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hbase:hbase-hadoop-compat:1.1.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.hbase:hbase-hadoop2-compat:1.1.2" level="project" />
<orderEntry type="library" name="Maven: com.yammer.metrics:metrics-core:2.2.0" level="project" />
<orderEntry type="library" name="Maven: com.sun.jersey:jersey-core:1.9" level="project" />
<orderEntry type="library" name="Maven: com.sun.jersey:jersey-server:1.9" level="project" />
<orderEntry type="library" name="Maven: asm:asm:3.1" level="project" />
<orderEntry type="library" name="Maven: commons-cli:commons-cli:1.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-math:2.2" level="project" />
<orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />
<orderEntry type="library" name="Maven: org.mortbay.jetty:jetty:6.1.26" level="project" />
<orderEntry type="library" name="Maven: org.mortbay.jetty:jetty-util:6.1.26" level="project" />
<orderEntry type="library" name="Maven: org.mortbay.jetty:jetty-sslengine:6.1.26" level="project" />
<orderEntry type="library" name="Maven: org.mortbay.jetty:jsp-2.1:6.1.14" level="project" />
<orderEntry type="library" name="Maven: org.mortbay.jetty:jsp-api-2.1:6.1.14" level="project" />
<orderEntry type="library" name="Maven: org.mortbay.jetty:servlet-api-2.5:6.1.14" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-core-asl:1.9.13" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-jaxrs:1.9.13" level="project" />
<orderEntry type="library" name="Maven: tomcat:jasper-compiler:5.5.23" level="project" />
<orderEntry type="library" name="Maven: tomcat:jasper-runtime:5.5.23" level="project" />
<orderEntry type="library" name="Maven: org.jamon:jamon-runtime:2.3.1" level="project" />
<orderEntry type="library" name="Maven: com.lmax:disruptor:3.3.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-client:2.5.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-app:2.5.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-common:2.5.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-client:2.5.1" level="project" />
<orderEntry type="library" name="Maven: com.sun.jersey:jersey-client:1.9" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-server-common:2.5.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-shuffle:2.5.1" level="project" />
<orderEntry type="library" name="Maven: org.fusesource.leveldbjni:leveldbjni-all:1.8" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-api:2.5.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-jobclient:2.5.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-hdfs:2.5.1" level="project" />
<orderEntry type="library" name="Maven: commons-daemon:commons-daemon:1.0.13" level="project" />
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-core:2.4.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-api:2.4.1" level="project" />
<orderEntry type="module" module-name="skywalking-protocol" />
</component>
</module>

1
skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/AnalysisServerDriver.java
View File

@ -43,6 +43,7 @@ public class AnalysisServerDriver extends Configured implements Tool {
TableMapReduceUtil.initTableMapperJob(Config.HBase.CALL_CHAIN_TABLE_NAME, scan, CallChainMapper.class,
String.class, ChainInfo.class, job);
//TableMapReduceUtil.initTableReducerJob("sw-call-chain-model", CallChainReducer.class, job);
return job.waitForCompletion(true) ? 0 : 1;
}

21
skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/config/Config.java
View File

@ -13,24 +13,23 @@ public class Config {
public static String TRACE_INFO_TABLE_NAME;
public static String TABLE_CALL_CHAIN_RELATIONSHIP = "sw_chain_relationship";
}
public static class TraceInfo {
public static String PARENT_LEVEL_ID = "parentLevelId";
public static String LEVEL_ID = "levelId";
public static String BUSINESS_KEY = "businessKey";
public static String COST = "cost";
public static String TRACE_INFO_COLUMN_CID = "cid";
}
public static String STATUS = "status";
public static class MySql {
public static String USER_ID = "UId";
public static String url;
public static String userName;
public static String password;
public static String driverClass;
}
public static class Filter {

38
skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/dao/CallChainInfoDao.java Normal file
View File

@ -0,0 +1,38 @@
package com.ai.cloud.skywalking.analysis.dao;
import com.ai.cloud.skywalking.analysis.config.Config;
import com.ai.cloud.skywalking.analysis.model.ChainInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.*;
public class CallChainInfoDao {
private static Logger logger = LoggerFactory.getLogger(CallChainInfoDao.class.getName());
private static Connection connection;
static {
try {
Class.forName(Config.MySql.driverClass);
connection = DriverManager.getConnection(Config.MySql.url, Config.MySql.userName, Config.MySql.password);
} catch (ClassNotFoundException e) {
logger.error("Failed to find jdbc driver class[" + Config.MySql.driverClass + "]", e);
System.exit(-1);
} catch (SQLException e) {
logger.error("Failed to connection database.", e);
System.exit(-1);
}
}
public static boolean existCallChainInfo(String cid, String userId) throws SQLException {
final String sql = "SELECT COUNT(cid) as TOTAL_SIZE FROM sw_chain_info WHERE cid = ? AND uid = ?";
PreparedStatement ps = connection.prepareStatement(sql);
ps.setString(1, cid);
ps.setString(2, userId);
ResultSet resultSet = ps.executeQuery();
resultSet.next();
return resultSet.getInt("TOTAL_SIZE") > 0 ? true : false;
}
}

56
skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/model/ChainInfo.java
View File

@ -1,8 +1,7 @@
package com.ai.cloud.skywalking.analysis.model;
import org.apache.hadoop.io.Writable;
import com.ai.cloud.skywalking.analysis.util.TokenGenerator;
import org.apache.hadoop.io.Writable;
import java.io.DataInput;
import java.io.DataOutput;
@ -25,6 +24,7 @@ public class ChainInfo implements Writable {
public void write(DataOutput out) throws IOException {
out.write(chainToken.getBytes());
out.writeChar(chainStatus.getValue());
out.write(userId.getBytes());
out.writeInt(nodes.size());
for (ChainNode chainNode : nodes) {
@ -34,6 +34,7 @@ public class ChainInfo implements Writable {
out.writeLong(chainNode.getCost());
out.write(chainNode.getParentLevelId().getBytes());
out.writeInt(chainNode.getLevelId());
out.write(chainNode.getBusinessKey().getBytes());
}
}
@ -41,6 +42,7 @@ public class ChainInfo implements Writable {
public void readFields(DataInput in) throws IOException {
chainToken = in.readLine();
chainStatus = ChainStatus.convert(in.readChar());
userId = in.readLine();
int nodeSize = in.readInt();
this.nodes = new ArrayList<ChainNode>();
@ -52,6 +54,8 @@ public class ChainInfo implements Writable {
chainNode.setCost(in.readLong());
chainNode.setParentLevelId(in.readLine());
chainNode.setLevelId(in.readInt());
chainNode.setBusinessKey(in.readLine());
nodes.add(chainNode);
}
}
@ -59,20 +63,20 @@ public class ChainInfo implements Writable {
public String getChainToken() {
return chainToken;
}
public String getEntranceNodeToken(){
if(firstChainNode == null){
return "";
}else{
return firstChainNode.getNodeToken();
}
public String getEntranceNodeToken() {
if (firstChainNode == null) {
return "";
} else {
return firstChainNode.getNodeToken();
}
}
public void generateChainToken() {
StringBuilder chainTokenDesc = new StringBuilder();
for(ChainNode node: nodes){
chainTokenDesc.append(node.getParentLevelId() + "." + node.getLevelId() + "-" + node.getNodeToken() + ";");
}
StringBuilder chainTokenDesc = new StringBuilder();
for (ChainNode node : nodes) {
chainTokenDesc.append(node.getParentLevelId() + "." + node.getLevelId() + "-" + node.getNodeToken() + ";");
}
this.chainToken = TokenGenerator.generate(chainTokenDesc.toString());
}
@ -83,19 +87,19 @@ public class ChainInfo implements Writable {
public void setChainStatus(ChainStatus chainStatus) {
this.chainStatus = chainStatus;
}
public void addNodes(ChainNode chainNode){
this.nodes.add(0, chainNode);
if (chainNode.getStatus() == ChainNode.NodeStatus.ABNORMAL) {
chainStatus = ChainStatus.ABNORMAL;
}
if(userId == null){
userId = chainNode.getUserId();
}
if ((chainNode.getParentLevelId() == null || chainNode.getParentLevelId().length() == 0)
public void addNodes(ChainNode chainNode) {
this.nodes.add(0, chainNode);
if (chainNode.getStatus() == ChainNode.NodeStatus.ABNORMAL) {
chainStatus = ChainStatus.ABNORMAL;
}
if (userId == null) {
userId = chainNode.getUserId();
}
if ((chainNode.getParentLevelId() == null || chainNode.getParentLevelId().length() == 0)
&& chainNode.getLevelId() == 0) {
firstChainNode = chainNode;
}
firstChainNode = chainNode;
}
}
public List<ChainNode> getNodes() {
@ -109,7 +113,7 @@ public class ChainInfo implements Writable {
public void setUserId(String userId) {
this.userId = userId;
}
public enum ChainStatus {
NORMAL('N'), ABNORMAL('A');
private char value;

8
skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/model/ChainNode.java
View File

@ -79,6 +79,14 @@ public class ChainNode {
this.businessKey = businessKey;
}
public String getCallType() {
return callType;
}
public String getBusinessKey() {
return businessKey;
}
public enum NodeStatus {
NORMAL('N'), ABNORMAL('A');
private char value;

9
skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/reduce/ChainInfoReduce.java
View File

@ -7,9 +7,14 @@ import org.apache.hadoop.io.Text;
import java.io.IOException;
public class ChainInfoReduce extends TableReducer<Text, ChainInfo, Put> {
public class ChainInfoReduce extends TableReducer<Text, ChainInfo, Put> {
@Override
protected void reduce(Text key, Iterable<ChainInfo> values, Context context) throws IOException, InterruptedException {
super.reduce(key, values, context);
String[] keyArray = key.toString().split(":");
String userId = keyArray[0];
String firstNode = keyArray[1];
//
}
}

20
skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/util/HBaseUtil.java
View File

@ -2,17 +2,17 @@ package com.ai.cloud.skywalking.analysis.util;
import com.ai.cloud.skywalking.analysis.config.Config;
import com.ai.cloud.skywalking.analysis.model.ChainInfo;
import com.ai.cloud.skywalking.protocol.Span;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class HBaseUtil {
private static Logger logger = LoggerFactory.getLogger(HBaseUtil.class.getName());
@ -75,4 +75,16 @@ public class HBaseUtil {
}
}
public static void selectById(String id) throws IOException {
List<Span> entries = new ArrayList<Span>();
Table table = connection.getTable(TableName.valueOf(Config.HBase.TABLE_CALL_CHAIN_RELATIONSHIP));
Get g = new Get(Bytes.toBytes(id));
Result r = table.get(g);
for (Cell cell : r.rawCells()) {
if (cell.getValueArray().length > 0)
entries.add(new Span(Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())));
}
}
}

2
skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/util/TokenGenerator.java
View File

@ -31,6 +31,6 @@ public class TokenGenerator {
System.exit(-1);
}
}
return result.toString().toUpperCase();
return "CID:" + result.toString().toUpperCase();
}
}

11
skywalking-analysis/src/main/resources/config.properties
View File

@ -6,10 +6,9 @@ hbase.zk_client_port=29181
hbase.trace_info_table_name=sw-trace-info
hbase.trace_info_column_family=trace_info
traceinfo.parent_level_id=parentLevelId
traceinfo.level_id=levelId
traceinfo.business_key=businessKey
traceinfo.cost=cost
traceinfo.trace_info_column_cid=cid
traceinfo.status=status
traceinfo.user_id=UId
mysql.url=
mysql.username=
mysql.password=
mysql.driverclass=