diff --git a/skywalking-commons/pom.xml b/skywalking-commons/pom.xml index 9cae90dff..b674ecef4 100644 --- a/skywalking-commons/pom.xml +++ b/skywalking-commons/pom.xml @@ -26,7 +26,9 @@ + org.apache.maven.plugins maven-compiler-plugin + 3.6.0 1.6 1.6 diff --git a/skywalking-commons/skywalking-logging/skywalking-logging-impl-log4j2/pom.xml b/skywalking-commons/skywalking-logging/skywalking-logging-impl-log4j2/pom.xml index 26751f770..f01053079 100644 --- a/skywalking-commons/skywalking-logging/skywalking-logging-impl-log4j2/pom.xml +++ b/skywalking-commons/skywalking-logging/skywalking-logging-impl-log4j2/pom.xml @@ -18,10 +18,15 @@ ${project.version} + + org.apache.logging.log4j + log4j-api + 2.6.2 + org.apache.logging.log4j log4j-core - 2.4.1 + 2.6.2 diff --git a/skywalking-storage-center/pom.xml b/skywalking-storage-center/pom.xml index 814bc3b5c..399d80bc1 100644 --- a/skywalking-storage-center/pom.xml +++ b/skywalking-storage-center/pom.xml @@ -34,6 +34,21 @@ data-carrier 1.2 + + com.a.eye + skywalking-health-report + ${project.version} + + + com.a.eye + skywalking-logging-impl-log4j2 + ${project.version} + + + com.a.eye + skywalking-registry + ${project.version} + diff --git a/skywalking-storage-center/skywalking-storage/pom.xml b/skywalking-storage-center/skywalking-storage/pom.xml index 0fd0aad04..f318aa003 100644 --- a/skywalking-storage-center/skywalking-storage/pom.xml +++ b/skywalking-storage-center/skywalking-storage/pom.xml @@ -15,32 +15,6 @@ - - com.a.eye - skywalking-registry - ${project.version} - - - com.a.eye - skywalking-logging-impl-log4j2 - ${project.version} - - - com.a.eye - skywalking-health-report - ${project.version} - - - org.hsqldb - hsqldb - 2.3.4 - - - com.zaxxer - HikariCP - 2.4.3 - - org.apache.commons commons-pool2 @@ -52,94 +26,21 @@ 5.0.1 + org.apache.maven.plugins - maven-resources-plugin - 2.6 - - - copy-resources - package - - copy-resources - - - ${project.build.directory}/installer/config - - - src/main/resources - false - - - - - - copy-start-script - package - - copy-resources - - - ${project.build.directory}/installer/bin - - - bin - false - - - - - - - - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - + maven-compiler-plugin + 3.6.0 - ${project.build.directory}/installer/lib - false - true + 1.8 + 1.8 + ${project.build.sourceEncoding} - - maven-jar-plugin - - - *.properties - *.xml - - ${project.build.directory}/installer/lib - - - - maven-antrun-plugin - - - clean - package - - run - - - - - - - - - - - - + + diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/Main.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/Main.java index 0fe6b9321..7f3eff697 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/Main.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/Main.java @@ -44,7 +44,6 @@ public class Main { DataFilesManager.init(); - provider = ServiceProvider.newBuilder(Config.Server.PORT).addSpanStorageService(new StorageListener()) .addAsyncTraceSearchService(new SearchListener()).build(); provider.start(); @@ -58,7 +57,6 @@ public class Main { logger.info("SkyWalking storage server started."); Thread.currentThread().join(); } catch (Throwable e) { - e.printStackTrace(); logger.error("SkyWalking storage server start failure.", e); } finally { provider.stop(); diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/boot/ElasticBooter.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/boot/ElasticBooter.java new file mode 100644 index 000000000..eb2c43572 --- /dev/null +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/boot/ElasticBooter.java @@ -0,0 +1,33 @@ +package com.a.eye.skywalking.storage.boot; + +import java.io.File; + +/** + * Created by xin on 2016/11/20. + */ +public class ElasticBooter { + + private String elasticHome; + + public ElasticBooter(String elasticHome) { + this.elasticHome = elasticHome; + } + + public void boot(int port) { + ElasticConfigModifier modifier = new ElasticConfigModifier(elasticHome); + modifier.append(port).replaceConfig(); + + ElasticServer elasticServer = new ElasticServer(elasticHome); + + if (elasticServer.isStarted()) { + elasticServer.stop(); + } + elasticServer.start(); + + Runtime.getRuntime().addShutdownHook(new Thread() { + public void run() { + elasticServer.stop(); + } + }); + } +} diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/boot/ElasticConfigModifier.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/boot/ElasticConfigModifier.java new file mode 100644 index 000000000..d094eb4c1 --- /dev/null +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/boot/ElasticConfigModifier.java @@ -0,0 +1,16 @@ +package com.a.eye.skywalking.storage.boot; + +class ElasticConfigModifier { + + public ElasticConfigModifier(String elasticHome) { + + } + + public ElasticConfigModifier append(int port) { + return null; + } + + public void replaceConfig() { + + } +} diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/boot/ElasticServer.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/boot/ElasticServer.java new file mode 100644 index 000000000..4820efac5 --- /dev/null +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/boot/ElasticServer.java @@ -0,0 +1,21 @@ +package com.a.eye.skywalking.storage.boot; + +class ElasticServer { + private boolean started; + + public ElasticServer(String elasticHome) { + + } + + public void stop() { + + } + + public boolean isStarted() { + return started; + } + + public void start() { + + } +} diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexMetaCollection.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexMetaCollection.java index 401d2c54d..f92a3caf9 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexMetaCollection.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexMetaCollection.java @@ -21,4 +21,8 @@ public class IndexMetaCollection implements Iterable { public Iterator iterator() { return metaInfo.iterator(); } + + public int size() { + return metaInfo.size(); + } } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexMetaInfo.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexMetaInfo.java index af4d6f675..e3a825712 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexMetaInfo.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexMetaInfo.java @@ -8,6 +8,8 @@ public class IndexMetaInfo { private SpanData spanData; + private SpanType spanType; + private DataFileNameDesc nameDesc; private long offset; @@ -17,10 +19,18 @@ public class IndexMetaInfo { public IndexMetaInfo(SpanData data, DataFileNameDesc fileNameDesc, long offset, int length) { this.spanData = data; this.nameDesc = fileNameDesc; + this.spanType = data.getSpanType(); this.offset = offset; this.length = length; } + public IndexMetaInfo(DataFileNameDesc fileNameDesc, long offset, int length, SpanType spanType) { + this.nameDesc = fileNameDesc; + this.offset = offset; + this.length = length; + this.spanType = spanType; + } + public DataFileNameDesc getFileName() { return nameDesc; } @@ -46,6 +56,6 @@ public class IndexMetaInfo { } public SpanType getSpanType() { - return spanData.getSpanType(); + return spanType; } } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/Executor.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/Executor.java index a61133f7f..cca25918d 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/Executor.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/Executor.java @@ -1,7 +1,5 @@ package com.a.eye.skywalking.storage.data.index.operator; -import com.a.eye.skywalking.storage.data.index.IndexMetaCollection; - interface Executor { T execute(IndexOperator indexOperator); } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/IndexOperator.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/IndexOperator.java index 8ecd0b831..b767ea715 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/IndexOperator.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/IndexOperator.java @@ -3,7 +3,7 @@ package com.a.eye.skywalking.storage.data.index.operator; import com.a.eye.skywalking.storage.data.index.IndexMetaCollection; public interface IndexOperator { - void batchUpdate(IndexMetaCollection metaInfos); + int batchUpdate(IndexMetaCollection metaInfos); IndexMetaCollection findIndex(Long[] traceId); } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/IndexOperatorImpl.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/IndexOperatorImpl.java index cd050f2bf..0b3fa6c6f 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/IndexOperatorImpl.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/IndexOperatorImpl.java @@ -4,16 +4,24 @@ import com.a.eye.skywalking.health.report.HealthCollector; import com.a.eye.skywalking.health.report.HeathReading; import com.a.eye.skywalking.logging.api.ILog; import com.a.eye.skywalking.logging.api.LogManager; +import com.a.eye.skywalking.storage.data.file.DataFileNameDesc; import com.a.eye.skywalking.storage.data.index.IndexMetaCollection; import com.a.eye.skywalking.storage.data.index.IndexMetaInfo; +import com.a.eye.skywalking.storage.data.spandata.SpanType; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; import java.io.IOException; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; public class IndexOperatorImpl implements IndexOperator { @@ -26,7 +34,7 @@ public class IndexOperatorImpl implements IndexOperator { } @Override - public void batchUpdate(IndexMetaCollection metaInfos) { + public int batchUpdate(IndexMetaCollection metaInfos) { BulkRequestBuilder requestBuilder = client.prepareBulk(); for (IndexMetaInfo indexMetaInfo : metaInfos) { try { @@ -43,20 +51,42 @@ public class IndexOperatorImpl implements IndexOperator { HealthCollector.getCurrentHeathReading("IndexOperator").updateData(HeathReading.ERROR, "Failed to " + "update index. Error message : " + bulkRequest.buildFailureMessage()); } + + return metaInfos.size(); } private XContentBuilder buildSource(IndexMetaInfo indexMetaInfo) throws IOException { XContentBuilder xContentBuilder = jsonBuilder().startObject().field("traceid_s0", indexMetaInfo.getTraceId()[0]) .field("traceid_s1", indexMetaInfo.getTraceId()[1]).field("traceid_s2", indexMetaInfo.getTraceId()[2]) .field("traceid_s3", indexMetaInfo.getTraceId()[3]).field("traceid_s4", indexMetaInfo.getTraceId()[4]) - .field("traceid_s5", indexMetaInfo.getTraceId()[5]).field("fileName", indexMetaInfo.getFileName()) + .field("traceid_s5", indexMetaInfo.getTraceId()[5]) + .field("span_type", indexMetaInfo.getSpanType().getValue()) + .field("fileName", indexMetaInfo.getFileName().getName()) + .field("fileName_suffix", indexMetaInfo.getFileName().getSuffix()) .field("offset", indexMetaInfo.getOffset()).field("length", indexMetaInfo.getLength()).endObject(); return xContentBuilder; } @Override public IndexMetaCollection findIndex(Long[] traceId) { - return null; + int index = 0; + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Long traceIdSegment : traceId) { + queryBuilder.must(termQuery("traceid_s" + index++, traceIdSegment)); + } + + IndexMetaCollection collection = new IndexMetaCollection(); + SearchResponse response = client.prepareSearch("skywalking").setQuery(queryBuilder).execute().actionGet(); + for (SearchHit hit : response.getHits()) { + DataFileNameDesc desc = new DataFileNameDesc(Long.parseLong(hit.getSource().get("fileName").toString()), + Integer.parseInt(hit.getSource().get("fileName_suffix").toString())); + int length = Integer.parseInt(hit.getSource().get("length").toString()); + long offset = Long.parseLong(hit.getSource().get("offset").toString()); + SpanType spanType = SpanType.convert(Integer.parseInt(hit.getSource().get("span_type").toString())); + collection.add(new IndexMetaInfo(desc, offset, length, spanType)); + } + + return collection; } } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/OperatorFactory.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/OperatorFactory.java index 9e654f1e9..a229a0d6d 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/OperatorFactory.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/OperatorFactory.java @@ -15,7 +15,7 @@ public class OperatorFactory { return new IndexOperatorImpl(new PreBuiltTransportClient(Settings.EMPTY).addTransportAddress( new InetSocketTransportAddress(InetAddress.getLocalHost(), Config.DataIndex.INDEX_LISTEN_PORT))); } catch (Exception e) { - throw new IndexOperatorInitializeFailedException("Failed to initialze operator.", e); + throw new IndexOperatorInitializeFailedException("Failed to initialize operator.", e); } } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/UpdateExecutor.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/UpdateExecutor.java index 67d69a9cb..e779267e1 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/UpdateExecutor.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/UpdateExecutor.java @@ -2,9 +2,6 @@ package com.a.eye.skywalking.storage.data.index.operator; import com.a.eye.skywalking.storage.data.index.IndexMetaCollection; -/** - * Created by xin on 2016/11/20. - */ public class UpdateExecutor implements Executor { private IndexMetaCollection metaCollection; @@ -14,7 +11,7 @@ public class UpdateExecutor implements Executor { } @Override - public IndexMetaCollection execute(IndexOperator indexOperator) { - return null; + public Integer execute(IndexOperator indexOperator) { + return indexOperator.batchUpdate(metaCollection); } } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/pool/IndexOperatorPooledObjectFactory.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/pool/IndexOperatorPooledObjectFactory.java index 326582512..85fa34318 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/pool/IndexOperatorPooledObjectFactory.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/operator/pool/IndexOperatorPooledObjectFactory.java @@ -20,7 +20,7 @@ public class IndexOperatorPooledObjectFactory extends BasePooledObjectFactory wrap(org.elasticsearch.client.transport.TransportClient client) { + public PooledObject wrap(TransportClient client) { return new DefaultPooledObject<>(client); } diff --git a/skywalking-storage-center/skywalking-storage/src/main/resources/elasticsearch.yml b/skywalking-storage-center/skywalking-storage/src/main/resources/elasticsearch.yml new file mode 100644 index 000000000..bca863555 --- /dev/null +++ b/skywalking-storage-center/skywalking-storage/src/main/resources/elasticsearch.yml @@ -0,0 +1,91 @@ +# ======================== Elasticsearch Configuration ========================= +# +# NOTE: Elasticsearch comes with reasonable defaults for most settings. +# Before you set out to tweak and tune the configuration, make sure you +# understand what are you trying to accomplish and the consequences. +# +# The primary way of configuring a node is via this file. This template lists +# the most important settings you may want to configure for a production cluster. +# +# Please see the documentation for further information on configuration options: +# +# +# ---------------------------------- Cluster ----------------------------------- +# +# Use a descriptive name for your cluster: +# +#cluster.name: my-application +# +# ------------------------------------ Node ------------------------------------ +# +# Use a descriptive name for the node: +# +#node.name: node-1 +# +# Add custom attributes to the node: +# +#node.attr.rack: r1 +# +# ----------------------------------- Paths ------------------------------------ +# +# Path to directory where to store the data (separate multiple locations by comma): +# +#path.data: /path/to/data +# +# Path to log files: +# +#path.logs: /path/to/logs +# +# ----------------------------------- Memory ----------------------------------- +# +# Lock the memory on startup: +# +#bootstrap.memory_lock: true +# +# Make sure that the heap size is set to about half the memory available +# on the system and that the owner of the process is allowed to use this +# limit. +# +# Elasticsearch performs poorly when the system is swapping the memory. +# +# ---------------------------------- Network ----------------------------------- +# +# Set the bind address to a specific IP (IPv4 or IPv6): +# +#network.host: 192.168.0.1 +# +# Set a custom port for HTTP: +# +#http.port: 9200 +# +# For more information, see the documentation at: +# +# +# --------------------------------- Discovery ---------------------------------- +# +# Pass an initial list of hosts to perform discovery when new node is started: +# The default list of hosts is ["127.0.0.1", "[::1]"] +# +#discovery.zen.ping.unicast.hosts: ["host1", "host2"] +# +# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1): +# +#discovery.zen.minimum_master_nodes: 3 +# +# For more information, see the documentation at: +# +# +# ---------------------------------- Gateway ----------------------------------- +# +# Block initial recovery after a full cluster restart until N nodes are started: +# +#gateway.recover_after_nodes: 3 +# +# For more information, see the documentation at: +# +# +# ---------------------------------- Various ----------------------------------- +# +# Require explicit names when deleting indices: +# +#action.destructive_requires_name: true diff --git a/skywalking-storage-center/skywalking-storage/src/main/resources/log4j.properties b/skywalking-storage-center/skywalking-storage/src/main/resources/log4j.properties index be0760950..17c194cc6 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/resources/log4j.properties +++ b/skywalking-storage-center/skywalking-storage/src/main/resources/log4j.properties @@ -2,6 +2,7 @@ log4j.rootLogger=CONSOLE,Rolling_File log4j.logger.org.apache=OFF log4j.logger.io.netty=OFF +log4j.org.elasticsearch=OFF log4j.logger.com.a.eye.skywalking.network.dependencies.io.netty=OFF # Console Appender # diff --git a/skywalking-storage-center/skywalking-storage/src/test/java/com/a/eye/skywalking/storage/data/file/DataFileWriterTest.java b/skywalking-storage-center/skywalking-storage/src/test/java/com/a/eye/skywalking/storage/data/file/DataFileWriterTest.java deleted file mode 100644 index 86e3a9c2c..000000000 --- a/skywalking-storage-center/skywalking-storage/src/test/java/com/a/eye/skywalking/storage/data/file/DataFileWriterTest.java +++ /dev/null @@ -1,58 +0,0 @@ -package com.a.eye.skywalking.storage.data.file; - -import com.a.eye.skywalking.network.grpc.RequestSpan; -import com.a.eye.skywalking.storage.config.Config; -import com.a.eye.skywalking.storage.data.spandata.RequestSpanData; -import com.a.eye.skywalking.storage.data.spandata.SpanData; -import com.a.eye.skywalking.storage.util.PathResolver; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; - -/** - * Created by xin on 2016/11/15. - */ -public class DataFileWriterTest { - - private DataFileWriter writer; - - @Before - public void setUp() { - Config.DataFile.PATH = "/tmp"; - Config.DataFile.SIZE = 10; - writer = new DataFileWriter(); - } - - @Test - public void testConvertFile() throws Exception { - List spanData = new ArrayList<>(); - spanData.add(new RequestSpanData( - RequestSpan.newBuilder()/*.setTraceId("test-traceId")*/.setStartDate(System.currentTimeMillis()) - .setProcessNo("7777").setLevelId(10).setParentLevel("0.0.0").setAddress("127.0.0.1").build())); - writer.write(spanData); - - writer.write(spanData); - File dir = new File(PathResolver.getAbsolutePath(Config.DataFile.PATH)); - assertEquals(2, dir.listFiles().length); - } - - - @After - public void tearUp() throws IOException { - File dir = new File(PathResolver.getAbsolutePath(Config.DataFile.PATH)); - for (File file : dir.listFiles()) { - file.delete(); - } - - dir.delete(); - } - -}