From dbe8aeb8834e58b1ae14ff4e122745691bdb160f Mon Sep 17 00:00:00 2001 From: ascrutae Date: Tue, 17 May 2016 20:04:13 +0800 Subject: [PATCH] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E9=83=A8=E7=BD=B2=E6=96=87?= =?UTF-8?q?=E6=A1=A3=EF=BC=8C=E4=BB=A5=E5=8F=8A=E4=BF=AE=E6=94=B9=E8=84=9A?= =?UTF-8?q?=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bin/How-to-deploy-sw_analysis.md | 64 +++++++++++++++++++ .../bin/deploy-sw_analysis-problem.md | 36 +++++++++++ skywalking-analysis/bin/start-analysis.sh | 2 +- 3 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 skywalking-analysis/bin/How-to-deploy-sw_analysis.md create mode 100644 skywalking-analysis/bin/deploy-sw_analysis-problem.md diff --git a/skywalking-analysis/bin/How-to-deploy-sw_analysis.md b/skywalking-analysis/bin/How-to-deploy-sw_analysis.md new file mode 100644 index 000000000..d6a102b05 --- /dev/null +++ b/skywalking-analysis/bin/How-to-deploy-sw_analysis.md @@ -0,0 +1,64 @@ +# 怎么部署分析模块 How to deploy skywalking-analysis + +## 前置步骤 Prepared deploy skywalking-analysis +1. 将HBase安装包拷贝到Hadoop安装目录下. Copy HBase installation package to the Hadoop installation directory. +2. 用HBase的主节点的配置覆盖HBase的安装包里面的配置. Use the configuration of the HBase master node converting the new Hbase package +3. 在.bash_profile文件添加下面的配置,(需要根据实际情况进行配置). Add the following configuration to .base_profile +``` +export HBASE_HOME=/aifs01/users/hdpusr01/hbase-1.1.2 +export PATH=$HBASE_HOME/bin:$PATH +``` +4. 运行以下命令. Run the command as follow. +``` +source .bash_profile +echo ${HBASE_HOME} +``` + + +## 部署模块 Deploy package +1. git clone analysis工程. Clone the analysis code from github +2. 根据实际情况修改src/main/resources目录下的analysis.conf. Modify the analysis.conf file that inside src/main/resources directory. +``` +hbase.zk_quorum=10.1.235.197,10.1.235.198,10.1.235.199 +hbase.zk_client_port=29181 + +mysql.url=jdbc:mysql://10.1.228.202:31316/test +mysql.username=devrdbusr21 +mysql.password=devrdbusr21 +``` +3. 运行skywalking-webui包下的创建Table的命令. Run the create table sql script. +``` +# 新增sw_chain_detail，用于调用链分析表 +CREATE TABLE `sw_chain_detail` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `viewpoint` varchar(8192) COLLATE utf8_bin DEFAULT NULL, + `treeId` varchar(40) COLLATE utf8_bin DEFAULT NULL, + `uid` varchar(32) COLLATE utf8_bin DEFAULT NULL, + `traceLevelId` varchar(32) COLLATE utf8_bin DEFAULT NULL, + `create_time` timestamp NULL DEFAULT NULL, + `update_time` timestamp NULL DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `id_UNIQUE` (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=8146 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; +SELECT * FROM sw_chain_detail; +``` +4. 打包. Package the project + +``` +package -Dmaven.test.skip=true +``` +5. 上传skywalking-analysis-1.0-SNAPSHOT.jar. Upload the skywalking-analysis-1.0-SNAPSHOT.jar +6. 上传start-analysis.sh. Upload the start-analysis.sh +7. 修改权限. Change mode start-analysis.sh +``` +> chmod +x start-analysis.sh +``` +8. 运行脚本. Run the command. +``` +>./start-analysis.sh +``` +9. 查看日志. tail the log +``` +skywalking-analysis/log> tail -1f map-reduce.log +``` + diff --git a/skywalking-analysis/bin/deploy-sw_analysis-problem.md b/skywalking-analysis/bin/deploy-sw_analysis-problem.md new file mode 100644 index 000000000..141655b9e --- /dev/null +++ b/skywalking-analysis/bin/deploy-sw_analysis-problem.md @@ -0,0 +1,36 @@ +#FAQ + +#常见命令 Common Commands： +``` +#上传所有jar包. Upload all jar file to hdfs +./hdfs dfs -put /aifs01/users/devhdp01/hadoop-2.6.0/share/hadoop/common/lib/*.jar /aifs01/users/devhdp01/hadoop-2.6.0/share/hadoop/common/lib/ + +#创建目录. Make directory into hdfs +./hdfs dfs -mkdir -p /aifs01/users/devhdp01/hbase-1.1.2/lib/ +``` + +## Jar包找不到. Jar File cannot be find. +###日志. Detail +``` +Exception in thread "main" java.io.FileNotFoundException: File does not exist: hdfs://host-10-1-241-18:9000/aifs01/users/devhdp01/hbase-1.1.2/lib/hbase-hadoop-compat-1.1.2.jar + at org.apache.hadoop.hdfs.DistributedFileSystem$18.doCall(DistributedFileSystem.java:1122) + at org.apache.hadoop.hdfs.DistributedFileSystem$18.doCall(DistributedFileSystem.java:1114) + at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) + at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1114) + at org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager.getFileStatus(ClientDistributedCacheManager.java:288) + at org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager.getFileStatus(ClientDistributedCacheManager.java:224) + at org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager.determineTimestamps(ClientDistributedCacheManager.java:93) + at org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager.determineTimestampsAndCacheVisibilities(ClientDistributedCacheManager.java:57) +``` +###解决.Resolve +1. 创建目录. Make directory into hdfs +``` +${HADOOP_HOME}/bin>./hdfs dfs -mkdir -p /aifs01/users/devhdp01/hbase-1.1.2/lib/ +``` +2. 上传文件. Upload jar file to hdfs +``` +${HADOOP_HOME}/bin>./hdfs dfs -put /aifs01/users/devhdp01/hbase-1.1.2/lib/hbase-hadoop-compat-1.1.2.jar /aifs01/users/devhdp01/hbase-1.1.2/lib/ +``` + +## 分析失败, 如何分析之前的数据. After analyis failed. Cannot analysis the data before now. +1. 删除start-analysis.sh脚本中${SW_ANALYSIS_HOME}对应的目录，默认分析三个月之前的数据 \ No newline at end of file diff --git a/skywalking-analysis/bin/start-analysis.sh b/skywalking-analysis/bin/start-analysis.sh index e0b0a5956..6956805b6 100755 --- a/skywalking-analysis/bin/start-analysis.sh +++ b/skywalking-analysis/bin/start-analysis.sh @@ -84,7 +84,7 @@ if [ "${SW_ANALYSIS_MODE}" != "REWRITE" ]; then START_TIME=`cat ${PRE_TIME_OF_ACCUMULATE_FILE}` if [ "$START_TIME" = "" ]; then - START_TIME=`date --date='1 month ago' "+%Y-%m-%d/%H:%M:%S"` + START_TIME=`date --date='3 month ago' "+%Y-%m-%d/%H:%M:%S"` fi SW_ANALYSIS_MODE=ACCUMULATE echo "skywalking analysis process will execute accumulate mode. start time: ${START_TIME}."