kazusa
/
skywalking-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

修复编译问题，完成Elastic启动部分逻辑，修改pom依赖

This commit is contained in:
ascrutae 2016-11-20 21:50:15 +08:00
parent 5b8d5e83fb
commit bf9db407f0
19 changed files with 246 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
skywalking-commons/pom.xml
View File

@ -26,7 +26,9 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.6.0</version>
<configuration>
<source>1.6</source>
<target>1.6</target>

7
skywalking-commons/skywalking-logging/skywalking-logging-impl-log4j2/pom.xml
View File

@ -18,10 +18,15 @@
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.6.2</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.4.1</version>
<version>2.6.2</version>
</dependency>
</dependencies>
</project>

15
skywalking-storage-center/pom.xml
View File

@ -34,6 +34,21 @@
<artifactId>data-carrier</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>com.a.eye</groupId>
<artifactId>skywalking-health-report</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.a.eye</groupId>
<artifactId>skywalking-logging-impl-log4j2</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.a.eye</groupId>
<artifactId>skywalking-registry</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</project>

115
skywalking-storage-center/skywalking-storage/pom.xml
View File

@ -15,32 +15,6 @@
</properties>
<dependencies>
<dependency>
<groupId>com.a.eye</groupId>
<artifactId>skywalking-registry</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.a.eye</groupId>
<artifactId>skywalking-logging-impl-log4j2</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.a.eye</groupId>
<artifactId>skywalking-health-report</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.hsqldb</groupId>
<artifactId>hsqldb</artifactId>
<version>2.3.4</version>
</dependency>
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
<version>2.4.3</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-pool2</artifactId>
@ -52,94 +26,21 @@
<version>5.0.1</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>2.6</version>
<executions>
<execution>
<id>copy-resources</id>
<phase>package</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/installer/config</outputDirectory>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>false</filtering>
</resource>
</resources>
</configuration>
</execution>
<execution>
<id>copy-start-script</id>
<phase>package</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/installer/bin</outputDirectory>
<resources>
<resource>
<directory>bin</directory>
<filtering>false</filtering>
</resource>
</resources>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
</execution>
</executions>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.6.0</version>
<configuration>
<outputDirectory>${project.build.directory}/installer/lib</outputDirectory>
<excludeTransitive>false</excludeTransitive>
<stripVersion>true</stripVersion>
<source>1.8</source>
<target>1.8</target>
<encoding>${project.build.sourceEncoding}</encoding>
</configuration>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<excludes>
<exclude>*.properties</exclude>
<exclude>*.xml</exclude>
</excludes>
<outputDirectory>${project.build.directory}/installer/lib</outputDirectory>
</configuration>
</plugin>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>clean</id>
<phase>package</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<tasks>
<mkdir dir="${project.build.directory}/installer/data/buffer"/>
<mkdir dir="${project.build.directory}/installer/data/offset"/>
<mkdir dir="${project.build.directory}/installer/log"/>
<mkdir dir="${project.build.directory}/installer/bin"/>
</tasks>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

2
skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/Main.java
View File

@ -44,7 +44,6 @@ public class Main {
DataFilesManager.init();
provider = ServiceProvider.newBuilder(Config.Server.PORT).addSpanStorageService(new StorageListener())
.addAsyncTraceSearchService(new SearchListener()).build();
provider.start();
@ -58,7 +57,6 @@ public class Main {
logger.info("SkyWalking storage server started.");
Thread.currentThread().join();
} catch (Throwable e) {
e.printStackTrace();
logger.error("SkyWalking storage server start failure.", e);
} finally {
provider.stop();

33
skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/boot/ElasticBooter.java Normal file
View File

@ -0,0 +1,33 @@
package com.a.eye.skywalking.storage.boot;
import java.io.File;
/**
* Created by xin on 2016/11/20.
*/
public class ElasticBooter {
private String elasticHome;
public ElasticBooter(String elasticHome) {
this.elasticHome = elasticHome;
}
public void boot(int port) {
ElasticConfigModifier modifier = new ElasticConfigModifier(elasticHome);
modifier.append(port).replaceConfig();
ElasticServer elasticServer = new ElasticServer(elasticHome);
if (elasticServer.isStarted()) {
elasticServer.stop();
}
elasticServer.start();
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
elasticServer.stop();
}
});
}
}

16
skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/boot/ElasticConfigModifier.java Normal file
View File

@ -0,0 +1,16 @@
package com.a.eye.skywalking.storage.boot;
class ElasticConfigModifier {
public ElasticConfigModifier(String elasticHome) {
}
public ElasticConfigModifier append(int port) {
return null;
}
public void replaceConfig() {
}
}

21
skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/boot/ElasticServer.java Normal file
View File

@ -0,0 +1,21 @@
package com.a.eye.skywalking.storage.boot;
class ElasticServer {
private boolean started;
public ElasticServer(String elasticHome) {
}
public void stop() {
}
public boolean isStarted() {
return started;
}
public void start() {
}
}

4
skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexMetaCollection.java
View File

@ -21,4 +21,8 @@ public class IndexMetaCollection implements Iterable<IndexMetaInfo> {
public Iterator<IndexMetaInfo> iterator() {
return metaInfo.iterator();
}
public int size() {
return metaInfo.size();
}
}

12
skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexMetaInfo.java
View File

@ -8,6 +8,8 @@ public class IndexMetaInfo {
private SpanData spanData;
private SpanType spanType;
private DataFileNameDesc nameDesc;
private long offset;
@ -17,10 +19,18 @@ public class IndexMetaInfo {
public IndexMetaInfo(SpanData data, DataFileNameDesc fileNameDesc, long offset, int length) {
this.spanData = data;
this.nameDesc = fileNameDesc;
this.spanType = data.getSpanType();
this.offset = offset;
this.length = length;
}
public IndexMetaInfo(DataFileNameDesc fileNameDesc, long offset, int length, SpanType spanType) {
this.nameDesc = fileNameDesc;
this.offset = offset;
this.length = length;
this.spanType = spanType;
}
public DataFileNameDesc getFileName() {
return nameDesc;
}
@ -46,6 +56,6 @@ public class IndexMetaInfo {
}
public SpanType getSpanType() {
return spanData.getSpanType();
return spanType;
}
}

2
skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/Executor.java
View File

@ -1,7 +1,5 @@
package com.a.eye.skywalking.storage.data.index.operator;
import com.a.eye.skywalking.storage.data.index.IndexMetaCollection;
interface Executor<T> {
T execute(IndexOperator indexOperator);
}

2
skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/IndexOperator.java
View File

@ -3,7 +3,7 @@ package com.a.eye.skywalking.storage.data.index.operator;
import com.a.eye.skywalking.storage.data.index.IndexMetaCollection;
public interface IndexOperator {
void batchUpdate(IndexMetaCollection metaInfos);
int batchUpdate(IndexMetaCollection metaInfos);
IndexMetaCollection findIndex(Long[] traceId);
}

36
skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/IndexOperatorImpl.java
View File

@ -4,16 +4,24 @@ import com.a.eye.skywalking.health.report.HealthCollector;
import com.a.eye.skywalking.health.report.HeathReading;
import com.a.eye.skywalking.logging.api.ILog;
import com.a.eye.skywalking.logging.api.LogManager;
import com.a.eye.skywalking.storage.data.file.DataFileNameDesc;
import com.a.eye.skywalking.storage.data.index.IndexMetaCollection;
import com.a.eye.skywalking.storage.data.index.IndexMetaInfo;
import com.a.eye.skywalking.storage.data.spandata.SpanType;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import java.io.IOException;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
public class IndexOperatorImpl implements IndexOperator {
@ -26,7 +34,7 @@ public class IndexOperatorImpl implements IndexOperator {
}
@Override
public void batchUpdate(IndexMetaCollection metaInfos) {
public int batchUpdate(IndexMetaCollection metaInfos) {
BulkRequestBuilder requestBuilder = client.prepareBulk();
for (IndexMetaInfo indexMetaInfo : metaInfos) {
try {
@ -43,20 +51,42 @@ public class IndexOperatorImpl implements IndexOperator {
HealthCollector.getCurrentHeathReading("IndexOperator").updateData(HeathReading.ERROR,
"Failed to " + "update index. Error message : " + bulkRequest.buildFailureMessage());
}
return metaInfos.size();
}
private XContentBuilder buildSource(IndexMetaInfo indexMetaInfo) throws IOException {
XContentBuilder xContentBuilder = jsonBuilder().startObject().field("traceid_s0", indexMetaInfo.getTraceId()[0])
.field("traceid_s1", indexMetaInfo.getTraceId()[1]).field("traceid_s2", indexMetaInfo.getTraceId()[2])
.field("traceid_s3", indexMetaInfo.getTraceId()[3]).field("traceid_s4", indexMetaInfo.getTraceId()[4])
.field("traceid_s5", indexMetaInfo.getTraceId()[5]).field("fileName", indexMetaInfo.getFileName())
.field("traceid_s5", indexMetaInfo.getTraceId()[5])
.field("span_type", indexMetaInfo.getSpanType().getValue())
.field("fileName", indexMetaInfo.getFileName().getName())
.field("fileName_suffix", indexMetaInfo.getFileName().getSuffix())
.field("offset", indexMetaInfo.getOffset()).field("length", indexMetaInfo.getLength()).endObject();
return xContentBuilder;
}
@Override
public IndexMetaCollection findIndex(Long[] traceId) {
return null;
int index = 0;
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
for (Long traceIdSegment : traceId) {
queryBuilder.must(termQuery("traceid_s" + index++, traceIdSegment));
}
IndexMetaCollection collection = new IndexMetaCollection();
SearchResponse response = client.prepareSearch("skywalking").setQuery(queryBuilder).execute().actionGet();
for (SearchHit hit : response.getHits()) {
DataFileNameDesc desc = new DataFileNameDesc(Long.parseLong(hit.getSource().get("fileName").toString()),
Integer.parseInt(hit.getSource().get("fileName_suffix").toString()));
int length = Integer.parseInt(hit.getSource().get("length").toString());
long offset = Long.parseLong(hit.getSource().get("offset").toString());
SpanType spanType = SpanType.convert(Integer.parseInt(hit.getSource().get("span_type").toString()));
collection.add(new IndexMetaInfo(desc, offset, length, spanType));
}
return collection;
}
}

2
skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/OperatorFactory.java
View File

@ -15,7 +15,7 @@ public class OperatorFactory {
return new IndexOperatorImpl(new PreBuiltTransportClient(Settings.EMPTY).addTransportAddress(
new InetSocketTransportAddress(InetAddress.getLocalHost(), Config.DataIndex.INDEX_LISTEN_PORT)));
} catch (Exception e) {
throw new IndexOperatorInitializeFailedException("Failed to initialze operator.", e);
throw new IndexOperatorInitializeFailedException("Failed to initialize operator.", e);
}
}

7
skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/UpdateExecutor.java
View File

@ -2,9 +2,6 @@ package com.a.eye.skywalking.storage.data.index.operator;
import com.a.eye.skywalking.storage.data.index.IndexMetaCollection;
/**
* Created by xin on 2016/11/20.
*/
public class UpdateExecutor implements Executor<Integer> {
private IndexMetaCollection metaCollection;
@ -14,7 +11,7 @@ public class UpdateExecutor implements Executor<Integer> {
}
@Override
public IndexMetaCollection execute(IndexOperator indexOperator) {
return null;
public Integer execute(IndexOperator indexOperator) {
return indexOperator.batchUpdate(metaCollection);
}
}

2
skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/pool/IndexOperatorPooledObjectFactory.java
View File

@ -20,7 +20,7 @@ public class IndexOperatorPooledObjectFactory extends BasePooledObjectFactory<Tr
}
@Override
public PooledObject<TransportClient> wrap(org.elasticsearch.client.transport.TransportClient client) {
public PooledObject<TransportClient> wrap(TransportClient client) {
return new DefaultPooledObject<>(client);
}

91
skywalking-storage-center/skywalking-storage/src/main/resources/elasticsearch.yml Normal file
View File

@ -0,0 +1,91 @@
# ======================== Elasticsearch Configuration =========================
#
# NOTE: Elasticsearch comes with reasonable defaults for most settings.
# Before you set out to tweak and tune the configuration, make sure you
# understand what are you trying to accomplish and the consequences.
#
# The primary way of configuring a node is via this file. This template lists
# the most important settings you may want to configure for a production cluster.
#
# Please see the documentation for further information on configuration options:
# <https://www.elastic.co/guide/en/elasticsearch/reference/5.0/settings.html>
#
# ---------------------------------- Cluster -----------------------------------
#
# Use a descriptive name for your cluster:
#
#cluster.name: my-application
#
# ------------------------------------ Node ------------------------------------
#
# Use a descriptive name for the node:
#
#node.name: node-1
#
# Add custom attributes to the node:
#
#node.attr.rack: r1
#
# ----------------------------------- Paths ------------------------------------
#
# Path to directory where to store the data (separate multiple locations by comma):
#
#path.data: /path/to/data
#
# Path to log files:
#
#path.logs: /path/to/logs
#
# ----------------------------------- Memory -----------------------------------
#
# Lock the memory on startup:
#
#bootstrap.memory_lock: true
#
# Make sure that the heap size is set to about half the memory available
# on the system and that the owner of the process is allowed to use this
# limit.
#
# Elasticsearch performs poorly when the system is swapping the memory.
#
# ---------------------------------- Network -----------------------------------
#
# Set the bind address to a specific IP (IPv4 or IPv6):
#
#network.host: 192.168.0.1
#
# Set a custom port for HTTP:
#
#http.port: 9200
#
# For more information, see the documentation at:
# <https://www.elastic.co/guide/en/elasticsearch/reference/5.0/modules-network.html>
#
# --------------------------------- Discovery ----------------------------------
#
# Pass an initial list of hosts to perform discovery when new node is started:
# The default list of hosts is ["127.0.0.1", "[::1]"]
#
#discovery.zen.ping.unicast.hosts: ["host1", "host2"]
#
# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1):
#
#discovery.zen.minimum_master_nodes: 3
#
# For more information, see the documentation at:
# <https://www.elastic.co/guide/en/elasticsearch/reference/5.0/modules-discovery-zen.html>
#
# ---------------------------------- Gateway -----------------------------------
#
# Block initial recovery after a full cluster restart until N nodes are started:
#
#gateway.recover_after_nodes: 3
#
# For more information, see the documentation at:
# <https://www.elastic.co/guide/en/elasticsearch/reference/5.0/modules-gateway.html>
#
# ---------------------------------- Various -----------------------------------
#
# Require explicit names when deleting indices:
#
#action.destructive_requires_name: true

1
skywalking-storage-center/skywalking-storage/src/main/resources/log4j.properties
View File

@ -2,6 +2,7 @@
log4j.rootLogger=CONSOLE,Rolling_File
log4j.logger.org.apache=OFF
log4j.logger.io.netty=OFF
log4j.org.elasticsearch=OFF
log4j.logger.com.a.eye.skywalking.network.dependencies.io.netty=OFF
# Console Appender #

58
skywalking-storage-center/skywalking-storage/src/test/java/com/a/eye/skywalking/storage/data/file/DataFileWriterTest.java
View File

@ -1,58 +0,0 @@
package com.a.eye.skywalking.storage.data.file;
import com.a.eye.skywalking.network.grpc.RequestSpan;
import com.a.eye.skywalking.storage.config.Config;
import com.a.eye.skywalking.storage.data.spandata.RequestSpanData;
import com.a.eye.skywalking.storage.data.spandata.SpanData;
import com.a.eye.skywalking.storage.util.PathResolver;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
import static org.junit.Assert.assertEquals;
/**
* Created by xin on 2016/11/15.
*/
public class DataFileWriterTest {
private DataFileWriter writer;
@Before
public void setUp() {
Config.DataFile.PATH = "/tmp";
Config.DataFile.SIZE = 10;
writer = new DataFileWriter();
}
@Test
public void testConvertFile() throws Exception {
List<SpanData> spanData = new ArrayList<>();
spanData.add(new RequestSpanData(
RequestSpan.newBuilder()/*.setTraceId("test-traceId")*/.setStartDate(System.currentTimeMillis())
.setProcessNo("7777").setLevelId(10).setParentLevel("0.0.0").setAddress("127.0.0.1").build()));
writer.write(spanData);
writer.write(spanData);
File dir = new File(PathResolver.getAbsolutePath(Config.DataFile.PATH));
assertEquals(2, dir.listFiles().length);
}
@After
public void tearUp() throws IOException {
File dir = new File(PathResolver.getAbsolutePath(Config.DataFile.PATH));
for (File file : dir.listFiles()) {
file.delete();
}
dir.delete();
}
}