kazusa
/
skywalking-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

增加启动DataIndex服务的工程，调整pom文件

This commit is contained in:
ascrutae 2016-11-21 16:51:48 +08:00
parent 0e309da054
commit 7eb5379116
14 changed files with 353 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
skywalking-storage-center/pom.xml
View File

@ -19,11 +19,6 @@
</modules>
<dependencies>
<dependency>
<groupId>com.a.eye</groupId>
<artifactId>skywalking-logging-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.a.eye</groupId>
<artifactId>skywalking-network</artifactId>
@ -36,18 +31,18 @@
</dependency>
<dependency>
<groupId>com.a.eye</groupId>
<artifactId>skywalking-health-report</artifactId>
<version>${project.version}</version>
<artifactId>skywalking-registry</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>com.a.eye</groupId>
<artifactId>skywalking-logging-impl-log4j2</artifactId>
<version>${project.version}</version>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>com.a.eye</groupId>
<artifactId>skywalking-registry</artifactId>
<version>${project.version}</version>
<artifactId>skywalking-health-report</artifactId>
<version>${project.parent.version}</version>
</dependency>
</dependencies>

28
skywalking-storage-center/skywalking-storage/bin/storageServer.sh Normal file
View File

@ -0,0 +1,28 @@
#!/usr/bin/env bash
STORAGE_SERVER_BIN="$0"
# Get standard environment variables
STORAGE_SERVER_BIN_DIR=`dirname "$STORAGE_SERVER_BIN"`
STORAGE_PREFIX="${STORAGE_SERVER_BIN_DIR}/.."
STORAGE_LOG_DIR="${STORAGE_SERVER_BIN_DIR}/../log"
STORAGE_CFG_DIR="${STORAGE_SERVER_BIN_DIR}/../config"
#echo $STORAGE_SERVER_BIN_DIR
#set java home
if [ "$JAVA_HOME" != "" ]; then
JAVA="$JAVA_HOME/bin/java"
else
JAVA=java
fi
CLASSPATH="$STORAGE_CFG_DIR:$CLASSPATH"
for i in "${STORAGE_SERVER_BIN_DIR}"/../lib/*.jar
do
CLASSPATH="$i:$CLASSPATH"
done
echo "CLASSPATH=$CLASSPATH"
$JAVA ${JAVA_OPTS} -DDATA_INDEX_HOME=${STORAGE_SERVER_BIN_DIR}/../data/index -classpath $CLASSPATH com.a.eye.skywalking.storage.Main >> ${STORAGE_SERVER_BIN_DIR}/../logs/storage-server.log & 2>&1&

BIN
skywalking-storage-center/skywalking-storage/lib/dataindex-es-5.0.1.zip Normal file → Executable file
View File

Binary file not shown.

131
skywalking-storage-center/skywalking-storage/pom.xml
View File

@ -25,6 +25,12 @@
<artifactId>transport</artifactId>
<version>5.0.1</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.17</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
@ -39,8 +45,129 @@
<encoding>${project.build.sourceEncoding}</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>2.6</version>
<executions>
<execution>
<id>copy-resources</id>
<phase>package</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/install/config</outputDirectory>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>false</filtering>
</resource>
</resources>
</configuration>
</execution>
<execution>
<id>copy-start-script</id>
<phase>package</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/install/bin</outputDirectory>
<resources>
<resource>
<directory>bin</directory>
<filtering>false</filtering>
</resource>
</resources>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
</execution>
</executions>
<configuration>
<outputDirectory>${project.build.directory}/install/lib</outputDirectory>
<excludeTransitive>false</excludeTransitive>
<stripVersion>true</stripVersion>
<includeScope>runtime</includeScope>
<includeScope>compile</includeScope>
</configuration>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<excludes>
<exclude>*.properties</exclude>
<exclude>*.xml</exclude>
</excludes>
<outputDirectory>${project.build.directory}/install/lib</outputDirectory>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
<executions>
<execution>
<id>unpack</id>
<phase>compile</phase>
<configuration>
<target>
<mkdir dir="${project.build.directory}/install/bin"/>
<mkdir dir="${project.build.directory}/install/lib"/>
<mkdir dir="${project.build.directory}/install/logs"/>
<mkdir dir="${project.build.directory}/install/bin"/>
<mkdir dir="${project.build.directory}/install/data/index"/>
<echo message="unzip elastic"/>
<unzip src="${project.basedir}/lib/dataindex-es-5.0.1.zip"
dest="${project.build.directory}/install/data/index"/>
<chmod dir="${project.build.directory}/install/data/index/bin" perm="ugo+rx"
includes="**"/>
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
<execution>
<id>clean</id>
<phase>pre-clean</phase>
<configuration>
<target>
<delete dir="${project.build.directory}/install/data/index"/>
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
<execution>
<id>tar</id>
<phase>package</phase>
<configuration>
<target>
<chmod dir="${project.build.directory}/install/bin" perm="ugo+rx"
includes="**"/>
<tar destfile="${project.build.directory}/install.tar" basedir="${project.build.directory}/install">
</tar>
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

8
skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/Main.java
View File

@ -9,6 +9,7 @@ import com.a.eye.skywalking.registry.RegistryCenterFactory;
import com.a.eye.skywalking.registry.api.CenterType;
import com.a.eye.skywalking.registry.api.RegistryCenter;
import com.a.eye.skywalking.registry.impl.zookeeper.ZookeeperConfig;
import com.a.eye.skywalking.storage.boot.ElasticBooter;
import com.a.eye.skywalking.storage.config.Config;
import com.a.eye.skywalking.storage.config.ConfigInitializer;
import com.a.eye.skywalking.storage.data.file.DataFilesManager;
@ -28,7 +29,7 @@ import static com.a.eye.skywalking.storage.config.Config.RegistryCenter.PATH_PRE
*/
public class Main {
private static final ILog logger = LogManager.getLogger(Main.class);
private static final ILog logger = LogManager.getLogger(Main.class);
private static final String SERVER_REPORTER_NAME = "DataConsumer Server";
static {
@ -42,6 +43,7 @@ public class Main {
initializeParam();
HealthCollector.init(SERVER_REPORTER_NAME);
new ElasticBooter().boot(NetUtils.getIndexServerPort());
IndexOperatorFactory.initOperatorPool();
DataFilesManager.init();
@ -59,6 +61,7 @@ public class Main {
logger.info("SkyWalking storage server started.");
Thread.currentThread().join();
} catch (Throwable e) {
e.printStackTrace();
logger.error("SkyWalking storage server start failure.", e);
} finally {
provider.stop();
@ -73,8 +76,7 @@ public class Main {
registerConfig.setProperty(ZookeeperConfig.AUTH_SCHEMA, Config.RegistryCenter.AUTH_SCHEMA);
registerConfig.setProperty(ZookeeperConfig.AUTH_INFO, Config.RegistryCenter.AUTH_INFO);
registryCenter.start(registerConfig);
registryCenter.register(
PATH_PREFIX + NetUtils.getLocalAddress().getHostAddress() + ":" + Config.Server.PORT);
registryCenter.register(PATH_PREFIX + NetUtils.getLocalAddress().getHostAddress() + ":" + Config.Server.PORT);
}
private static void initializeParam() throws IllegalAccessException, IOException {

28
skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/boot/ElasticBooter.java
View File

@ -1,21 +1,30 @@
package com.a.eye.skywalking.storage.boot;
import com.a.eye.skywalking.logging.api.ILog;
import com.a.eye.skywalking.logging.api.LogManager;
import java.io.File;
import java.io.IOException;
/**
* Created by xin on 2016/11/20.
*/
public class ElasticBooter {
private static ILog logger = LogManager.getLogger(ElasticBooter.class);
public static final String DATA_INDEX_HOME = "DATA_INDEX_HOME";
private static final String DEVELOP_RUNTIME_ELASTIC_HOME =
ElasticBooter.class.getResource("/").getPath() + ".." + File.separator + "install" + File.separator + "data"
+ File.separator + "index";
private String elasticHome;
public ElasticBooter(String elasticHome) {
this.elasticHome = elasticHome;
public ElasticBooter() {
this.elasticHome = fetchElasticHome();
}
public void boot(int port) {
public void boot(int port) throws IOException {
ElasticConfigModifier modifier = new ElasticConfigModifier(elasticHome);
modifier.append(port).replaceConfig();
modifier.replaceConfig(port);
ElasticServer elasticServer = new ElasticServer(elasticHome);
@ -26,8 +35,17 @@ public class ElasticBooter {
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
elasticServer.stop();
try {
elasticServer.stop();
} catch (IOException e) {
logger.error("Failed to stop elastic server.", e);
}
}
});
}
public String fetchElasticHome() {
return System.getProperty(DATA_INDEX_HOME, DEVELOP_RUNTIME_ELASTIC_HOME);
}
}

38
skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/boot/ElasticConfigModifier.java
View File

@ -1,16 +1,44 @@
package com.a.eye.skywalking.storage.boot;
import com.a.eye.skywalking.logging.api.ILog;
import com.a.eye.skywalking.logging.api.LogManager;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
class ElasticConfigModifier {
private static ILog logger = LogManager.getLogger(ElasticConfigModifier.class);
private File elasticConfigDir = null;
public ElasticConfigModifier(String elasticHome) {
this.elasticConfigDir = new File(elasticHome, "config");
if (!elasticConfigDir.exists()) {
logger.warn("Elastic search config dir is not exists. Will create it");
elasticConfigDir.mkdirs();
}
}
public ElasticConfigModifier append(int port) {
return null;
public void replaceConfig(int port) throws IOException {
File newConfigFile = new File(elasticConfigDir, "elasticsearch.yml");
Files.copy(ElasticConfigModifier.class.getResourceAsStream("/elasticsearch.yml"), newConfigFile.toPath(),
REPLACE_EXISTING);
appendingNewConfig(port, newConfigFile);
return;
}
public void replaceConfig() {
private void appendingNewConfig(int port, File newConfigFile) throws IOException {
FileWriter writer = null;
try {
writer = new FileWriter(newConfigFile, true);
writer.write("transport.tcp.port: " + port);
writer.flush();
} finally {
writer.close();
}
}
}

41
skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/boot/ElasticServer.java
View File

@ -1,21 +1,48 @@
package com.a.eye.skywalking.storage.boot;
import com.a.eye.skywalking.logging.api.ILog;
import com.a.eye.skywalking.logging.api.LogManager;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
class ElasticServer {
private boolean started;
private static ILog logger = LogManager.getLogger(ElasticServer.class);
private String elasticBinDir;
public ElasticServer(String elasticHome) {
this.elasticBinDir = elasticHome + File.separator + "bin" + File.separator;
}
public void stop() {
public void stop() throws IOException {
int pid = readServerPID();
if (pid == -1) {
return;
}
Runtime.getRuntime().exec("kill -9 " + pid);
}
public boolean isStarted() {
return started;
public boolean isStarted() throws IOException {
return false;
}
public void start() {
public void start() throws IOException {
Runtime.getRuntime().exec(elasticBinDir + "elasticsearch -p " + elasticBinDir + "elastic.pid -d");
}
private int readServerPID() {
try {
BufferedReader reader = new BufferedReader(new FileReader(elasticBinDir + "elastic.pid"));
return Integer.parseInt(reader.readLine());
} catch (IOException e) {
logger.error("Failed to elastic server pid", e);
}
return -1;
}
}

10
skywalking-storage-center/skywalking-storage/src/main/resources/log4j.properties
View File

@ -1,19 +1,13 @@
# logger #
log4j.rootLogger=CONSOLE,Rolling_File
log4j.rootLogger=Rolling_File
log4j.logger.org.apache=OFF
log4j.logger.io.netty=OFF
log4j.org.elasticsearch=OFF
log4j.logger.com.a.eye.skywalking.network.dependencies.io.netty=OFF
# Console Appender #
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
log4j.appender.CONSOLE.Target=System.out
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%d %-5p %c{1}:%L - %m%n
log4j.appender.Rolling_File=org.apache.log4j.RollingFileAppender
log4j.appender.Rolling_File.Threshold=WARN
log4j.appender.Rolling_File.File=logs/storage-server-log4j.log
log4j.appender.Rolling_File.File=../logs/storage-server-log4j.log
log4j.appender.Rolling_File.Append=true
log4j.appender.Rolling_File.MaxFileSize=100MB
log4j.appender.Rolling_File.MaxBackupIndex=5

3
skywalking-storage-center/skywalking-storage/src/main/resources/log4j2.xml
View File

@ -4,7 +4,7 @@
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d %-5p %c{1}:%L - %m%n"/>
</Console>
<RollingFile name="RollingFile" fileName="logs/storage-server.log"
<RollingFile name="RollingFile" fileName="../logs/storage-server.log"
filePattern="logs/storage-server-%d{MM-dd-yyyy}.log.gz"
ignoreExceptions="false">
<PatternLayout>
@ -17,7 +17,6 @@
<Loggers>
<Root level="DEBUG">
<AppenderRef ref="RollingFile" level="INFO"/>
<appender-ref ref="Console" level="DEBUG"/>
</Root>
</Loggers>
</Configuration>

29
skywalking-storage-center/skywalking-storage/src/test/java/StorageClient.java
View File

@ -8,8 +8,9 @@ import com.a.eye.skywalking.network.grpc.*;
import static com.a.eye.skywalking.network.grpc.SpanStorageServiceGrpc.newStub;
public class StorageClient {
private static ManagedChannel channel =
ManagedChannelBuilder.forAddress("10.128.35.79", 34000).usePlaintext(true).build();
ManagedChannelBuilder.forAddress("127.0.0.1", 34000).usePlaintext(true).build();
private static SpanStorageServiceGrpc.SpanStorageServiceStub spanStorageServiceStub = newStub(channel);
@ -19,28 +20,26 @@ public class StorageClient {
public static void main(String[] args) throws InterruptedException {
long startTime = System.currentTimeMillis();
for (int i = 0; i < 1000000; i++) {
for (int i = 0; i < 1; i++) {
long value = System.currentTimeMillis();
RequestSpan requestSpan =
RequestSpan.newBuilder().setSpanType(1).setAddress("127.0.0.1").setApplicationId("1").setCallType("1")
.setLevelId(0).setProcessNo("19287").setStartDate(System.currentTimeMillis()).setTraceId(
TraceId.newBuilder().addSegments(201611).addSegments(System.currentTimeMillis()).addSegments(8504828)
.addSegments(2277).addSegments(53).addSegments(3).build()).setUserId("1")
.setViewPointId("http://localhost:8080/wwww/test/helloWorld").build();
RequestSpan.newBuilder().setSpanType(1).setAddress("127.0.0.1").setApplicationId("1")
.setCallType("1").setLevelId(0).setProcessNo("19287")
.setStartDate(System.currentTimeMillis()).setTraceId(
TraceId.newBuilder().addSegments(201611).addSegments(value)
.addSegments(8504828).addSegments(2277).addSegments(53).addSegments(3).build())
.setUserId("1").setViewPointId("http://localhost:8080/wwww/test/helloWorld").build();
AckSpan ackSpan = AckSpan.newBuilder().setLevelId(0).setCost(10).setTraceId(
TraceId.newBuilder().addSegments(201611).addSegments(System.currentTimeMillis()).addSegments(8504828)
.addSegments(2277).addSegments(53).addSegments(3).build()).setStatusCode(0)
TraceId.newBuilder().addSegments(201611).addSegments(value).addSegments(8504828).addSegments(2277)
.addSegments(53).addSegments(3).build()).setStatusCode(0)
.setViewpointId("http://localhost:8080/wwww/test/helloWorld").build();
StreamObserver<AckSpan> ackSpanStreamObserver =
spanStorageServiceStub.storageACKSpan(new StreamObserver<SendResult>() {
@Override
public void onNext(SendResult sendResult) {
}
@Override
@ -70,7 +69,7 @@ public class StorageClient {
endTime2 = System.currentTimeMillis();
}
});
for (int j = 0; j < 10; j++) {
for (int j = 0; j < 1; j++) {
requestSpanStreamObserver.onNext(requestSpan);
ackSpanStreamObserver.onNext(ackSpan);
@ -87,7 +86,7 @@ public class StorageClient {
requestSpanStreamObserver.onCompleted();
if(i % 1_000 == 0){
if (i % 1_000 == 0) {
System.out.println(i);
}

2
skywalking-storage-center/skywalking-storage/src/test/java/com/a/eye/skywalking/storage/SearchClient.java
View File

@ -36,7 +36,7 @@ public class SearchClient {
StreamObserver<QueryTask> searchResult = searchServiceStub.search(serverStreamObserver);
searchResult.onNext(QueryTask.newBuilder().setTraceId(
TraceId.newBuilder().addSegments(201611).addSegments(1479267274243L).addSegments(8504828)
TraceId.newBuilder().addSegments(201611).addSegments(1479717228982L).addSegments(8504828)
.addSegments(2277).addSegments(53).addSegments(3).build()).setTaskId(1).build());
searchResult.onCompleted();

32
skywalking-storage-center/skywalking-storage/src/test/java/com/a/eye/skywalking/storage/boot/ElasticBooterTest.java Normal file
View File

@ -0,0 +1,32 @@
package com.a.eye.skywalking.storage.boot;
import org.junit.After;
import org.junit.Test;
import java.io.File;
import static com.a.eye.skywalking.storage.boot.ElasticBooter.DATA_INDEX_HOME;
import static org.junit.Assert.assertEquals;
public class ElasticBooterTest {
private String bastPath = ElasticBooterTest.class.getResource("/").getPath() + ".." + File.separator;
@Test
public void fetchElasticHomeWithoutProperty() {
ElasticBooter booter = new ElasticBooter();
assertEquals("Elastic Home :", booter.fetchElasticHome(), bastPath + "install/data/index");
}
@Test
public void fetchElasticHomeWithProperty() {
System.setProperty(DATA_INDEX_HOME, "/test/test");
ElasticBooter fetcher = new ElasticBooter();
assertEquals("Elastic Home :", fetcher.fetchElasticHome(), "/test/test");
}
@After
public void tearUp() {
System.clearProperty(DATA_INDEX_HOME);
}
}

46
skywalking-storage-center/skywalking-storage/src/test/java/com/a/eye/skywalking/storage/boot/ElasticConfigModifierTest.java Normal file
View File

@ -0,0 +1,46 @@
package com.a.eye.skywalking.storage.boot;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.yaml.snakeyaml.Yaml;
import java.io.File;
import java.io.FileInputStream;
import java.nio.file.Files;
import java.util.HashMap;
import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class ElasticConfigModifierTest {
private File configDir = new File(ElasticConfigModifierTest.class.getResource("/").getFile(), "test");
File elasticSearchConfigFile = new File(configDir + File.separator + "config", "elasticsearch.yml");
@Before
public void setUp() throws Exception {
}
@After
public void tearDown() throws Exception {
Files.delete(elasticSearchConfigFile.toPath());
Files.delete(new File(configDir, "config").toPath());
Files.delete(configDir.toPath());
}
@Test
public void testReplaceConfig() throws Exception {
ElasticConfigModifier modifier = new ElasticConfigModifier(configDir.getPath());
modifier.replaceConfig(18080);
assertTrue(elasticSearchConfigFile.exists());
Yaml yaml = new Yaml();
HashMap<String, Integer> config = (HashMap<String, Integer>) yaml.load(new FileInputStream
(elasticSearchConfigFile));
assertEquals(18080,config.get("transport.tcp.port").intValue());
}
}