diff --git a/pom.xml b/pom.xml index 4408bc1c1..70c003e9b 100644 --- a/pom.xml +++ b/pom.xml @@ -12,10 +12,9 @@ skywalking-webui skywalking-sniffer skywalking-storage-center - skywalking-registry samples/skywalking-auth samples/skywalking-example - skywalking-logging + skywalking-commons pom diff --git a/skywalking-commons/pom.xml b/skywalking-commons/pom.xml new file mode 100644 index 000000000..5f0e50b92 --- /dev/null +++ b/skywalking-commons/pom.xml @@ -0,0 +1,26 @@ + + + skywalking + com.a.eye + 2.0-2016 + + 4.0.0 + + skywalking-commons + pom + + + skywalking-logging + skywalking-registry + skywalking-health-report + + + skywalking-commons + http://maven.apache.org + + + UTF-8 + 2.0-2016 + + diff --git a/skywalking-commons/skywalking-health-report/pom.xml b/skywalking-commons/skywalking-health-report/pom.xml new file mode 100644 index 000000000..a229641f7 --- /dev/null +++ b/skywalking-commons/skywalking-health-report/pom.xml @@ -0,0 +1,23 @@ + + + skywalking-commons + com.a.eye + 2.0-2016 + + 4.0.0 + + skywalking-health-report + jar + + skywalking-health-report + http://maven.apache.org + + + + com.a.eye + skywalking-logging-api + 2.0-2016 + + + diff --git a/skywalking-commons/skywalking-health-report/src/main/java/com/a/eye/skywalking/health/report/HealthCollector.java b/skywalking-commons/skywalking-health-report/src/main/java/com/a/eye/skywalking/health/report/HealthCollector.java new file mode 100644 index 000000000..d010d98f3 --- /dev/null +++ b/skywalking-commons/skywalking-health-report/src/main/java/com/a/eye/skywalking/health/report/HealthCollector.java @@ -0,0 +1,82 @@ +package com.a.eye.skywalking.health.report; + + +import com.a.eye.skywalking.health.report.util.MachineUtil; +import com.a.eye.skywalking.logging.api.ILog; +import com.a.eye.skywalking.logging.api.LogManager; + +import java.util.Arrays; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class HealthCollector extends Thread { + private static ILog logger = + LogManager.getLogger(HealthCollector.class); + private static Map heathReadings = + new ConcurrentHashMap(); + private static final long DEFAULT_REPORT_INTERVAL = 60 * 1000; + private final long reportInterval; + + private HealthCollector() { + this(DEFAULT_REPORT_INTERVAL); + } + + private HealthCollector(long reportInterval) { + super("HealthCollector"); + this.setDaemon(true); + this.reportInterval = reportInterval; + } + + public static void init() { + new HealthCollector().start(); + } + + public static HeathReading getCurrentHeathReading(String extraId) { + String id = getId(extraId); + if (!heathReadings.containsKey(id)) { + synchronized (heathReadings) { + if (!heathReadings.containsKey(id)) { + if (heathReadings.keySet().size() > 5000) { + throw new RuntimeException( + "use HealthCollector illegal. There is an overflow trend of Server Health Collector Report Data."); + } + heathReadings.put(id, new HeathReading(id)); + } + } + } + return heathReadings.get(id); + } + + private static String getId(String extraId) { + return "SkyWalking Health Report,P:" + MachineUtil.getProcessNo() + ",T:" + Thread.currentThread().getName() + + "(" + Thread.currentThread().getId() + ")" + (extraId == null ? "" : ",extra:" + extraId); + } + + @Override + public void run() { + while (true) { + try { + Map heathReadingsSnapshot = heathReadings; + heathReadings = new ConcurrentHashMap(); + String[] keyList = heathReadingsSnapshot.keySet().toArray(new String[0]); + Arrays.sort(keyList); + StringBuilder log = new StringBuilder(); + log.append("\n---------Server Health Collector Report---------\n"); + for (String key : keyList) { + log.append(heathReadingsSnapshot.get(key)).append("\n"); + } + log.append("------------------------------------------------\n"); + + logger.info(log.toString()); + + try { + Thread.sleep(reportInterval); + } catch (InterruptedException e) { + logger.warn("sleep error.", e); + } + } catch (Throwable t) { + logger.error("HealthCollector report error.", t); + } + } + } +} diff --git a/skywalking-commons/skywalking-health-report/src/main/java/com/a/eye/skywalking/health/report/HeathReading.java b/skywalking-commons/skywalking-health-report/src/main/java/com/a/eye/skywalking/health/report/HeathReading.java new file mode 100644 index 000000000..4d0ec0ab4 --- /dev/null +++ b/skywalking-commons/skywalking-health-report/src/main/java/com/a/eye/skywalking/health/report/HeathReading.java @@ -0,0 +1,77 @@ +package com.a.eye.skywalking.health.report; + + +import java.util.HashMap; +import java.util.Map; + +public class HeathReading { + public static final String ERROR = "[ERROR]"; + public static final String WARNING = "[WARNING]"; + public static final String INFO = "[INFO]"; + + private String id; + + private Map datas = new HashMap(); + + /** + * 健康读数，只应该在工作线程中创建 + */ + public HeathReading(String id) { + this.id = id; + } + + public void updateData(String key, String message) { + updateData(key, message, new Object[0]); + } + + public void updateData(String key, String newData, Object... arguments) { + if (datas.containsKey(key)) { + datas.get(key).updateData(newData, arguments); + } else { + datas.put(key, new HeathDetailData(newData)); + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("id<").append(this.id).append(">\n"); + for (Map.Entry data : datas.entrySet()) { + sb.append(data.getKey()).append(data.getValue().toString()).append("\n"); + } + + datas = new HashMap(); + return sb.toString(); + } + + class HeathDetailData { + private String data; + + private long statusTime; + + HeathDetailData(String initialData) { + data = initialData; + statusTime = System.currentTimeMillis(); + } + + void updateData(String newData, Object... arguments) { + data = newData; + if (arguments.length > 0) + data = String.format(newData, arguments); + statusTime = System.currentTimeMillis(); + } + + String getData() { + return data; + } + + long getStatusTime() { + return statusTime; + } + + @Override + public String toString() { + return data + "(t:" + statusTime + ")"; + } + } +} diff --git a/skywalking-commons/skywalking-health-report/src/main/java/com/a/eye/skywalking/health/report/util/MachineUtil.java b/skywalking-commons/skywalking-health-report/src/main/java/com/a/eye/skywalking/health/report/util/MachineUtil.java new file mode 100644 index 000000000..63f65c01a --- /dev/null +++ b/skywalking-commons/skywalking-health-report/src/main/java/com/a/eye/skywalking/health/report/util/MachineUtil.java @@ -0,0 +1,27 @@ +package com.a.eye.skywalking.health.report.util; + +import java.lang.management.ManagementFactory; + +/** + * Created by xin on 2016/11/14. + */ + +public class MachineUtil { + private static String processNo; + + static { + processNo = getProcessNo(); + } + + public static String getProcessNo() { + if (processNo == null) { + String name = ManagementFactory.getRuntimeMXBean().getName(); + processNo = name.split("@")[0]; + } + return processNo; + } + + private MachineUtil() { + // Non + } +} diff --git a/skywalking-logging/pom.xml b/skywalking-commons/skywalking-logging/pom.xml similarity index 92% rename from skywalking-logging/pom.xml rename to skywalking-commons/skywalking-logging/pom.xml index d6ae3605b..6802cab9a 100644 --- a/skywalking-logging/pom.xml +++ b/skywalking-commons/skywalking-logging/pom.xml @@ -1,7 +1,7 @@ - skywalking + skywalking-commons com.a.eye 2.0-2016 diff --git a/skywalking-logging/skywalking-logging-api/pom.xml b/skywalking-commons/skywalking-logging/skywalking-logging-api/pom.xml similarity index 100% rename from skywalking-logging/skywalking-logging-api/pom.xml rename to skywalking-commons/skywalking-logging/skywalking-logging-api/pom.xml diff --git a/skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/ILog.java b/skywalking-commons/skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/ILog.java similarity index 74% rename from skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/ILog.java rename to skywalking-commons/skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/ILog.java index e978b78a1..54956ef14 100644 --- a/skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/ILog.java +++ b/skywalking-commons/skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/ILog.java @@ -1,5 +1,7 @@ package com.a.eye.skywalking.logging.api; +import java.util.Objects; + /** * Created by xin on 2016/11/10. */ @@ -16,5 +18,9 @@ public interface ILog { void error(String format, Object argument, Throwable e); + boolean isDebugEnable(); + void debug(String format); + + void debug(String format, Object... arguments); } diff --git a/skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/LogManager.java b/skywalking-commons/skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/LogManager.java similarity index 100% rename from skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/LogManager.java rename to skywalking-commons/skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/LogManager.java diff --git a/skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/LogResolver.java b/skywalking-commons/skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/LogResolver.java similarity index 100% rename from skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/LogResolver.java rename to skywalking-commons/skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/LogResolver.java diff --git a/skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/NoopLogger.java b/skywalking-commons/skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/NoopLogger.java similarity index 74% rename from skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/NoopLogger.java rename to skywalking-commons/skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/NoopLogger.java index 83479d813..20b7dc85c 100644 --- a/skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/NoopLogger.java +++ b/skywalking-commons/skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/NoopLogger.java @@ -35,4 +35,19 @@ public class NoopLogger implements ILog{ @Override public void error(String format, Object argument, Throwable e) { } + + @Override + public boolean isDebugEnable() { + return false; + } + + @Override + public void debug(String format) { + + } + + @Override + public void debug(String format, Object... arguments) { + + } } diff --git a/skywalking-logging/skywalking-logging-impl-log4j2/pom.xml b/skywalking-commons/skywalking-logging/skywalking-logging-impl-log4j2/pom.xml similarity index 100% rename from skywalking-logging/skywalking-logging-impl-log4j2/pom.xml rename to skywalking-commons/skywalking-logging/skywalking-logging-impl-log4j2/pom.xml diff --git a/skywalking-logging/skywalking-logging-impl-log4j2/src/main/java/com/a/eye/skywalking/logging/impl/log4j2/Log4j2Logger.java b/skywalking-commons/skywalking-logging/skywalking-logging-impl-log4j2/src/main/java/com/a/eye/skywalking/logging/impl/log4j2/Log4j2Logger.java similarity index 77% rename from skywalking-logging/skywalking-logging-impl-log4j2/src/main/java/com/a/eye/skywalking/logging/impl/log4j2/Log4j2Logger.java rename to skywalking-commons/skywalking-logging/skywalking-logging-impl-log4j2/src/main/java/com/a/eye/skywalking/logging/impl/log4j2/Log4j2Logger.java index a4aaee754..cb6327c64 100644 --- a/skywalking-logging/skywalking-logging-impl-log4j2/src/main/java/com/a/eye/skywalking/logging/impl/log4j2/Log4j2Logger.java +++ b/skywalking-commons/skywalking-logging/skywalking-logging-impl-log4j2/src/main/java/com/a/eye/skywalking/logging/impl/log4j2/Log4j2Logger.java @@ -42,4 +42,19 @@ public class Log4j2Logger implements ILog { public void error(String message, Object argument, Throwable e) { logger.error(message, argument, e); } + + @Override + public boolean isDebugEnable() { + return logger.isDebugEnabled(); + } + + @Override + public void debug(String format) { + logger.debug(format); + } + + @Override + public void debug(String format, Object... arguments) { + logger.debug(format, arguments); + } } diff --git a/skywalking-logging/skywalking-logging-impl-log4j2/src/main/java/com/a/eye/skywalking/logging/impl/log4j2/Log4j2Resolver.java b/skywalking-commons/skywalking-logging/skywalking-logging-impl-log4j2/src/main/java/com/a/eye/skywalking/logging/impl/log4j2/Log4j2Resolver.java similarity index 100% rename from skywalking-logging/skywalking-logging-impl-log4j2/src/main/java/com/a/eye/skywalking/logging/impl/log4j2/Log4j2Resolver.java rename to skywalking-commons/skywalking-logging/skywalking-logging-impl-log4j2/src/main/java/com/a/eye/skywalking/logging/impl/log4j2/Log4j2Resolver.java diff --git a/skywalking-registry/pom.xml b/skywalking-commons/skywalking-registry/pom.xml similarity index 88% rename from skywalking-registry/pom.xml rename to skywalking-commons/skywalking-registry/pom.xml index 5d7423cd9..88e67cf3b 100644 --- a/skywalking-registry/pom.xml +++ b/skywalking-commons/skywalking-registry/pom.xml @@ -1,7 +1,7 @@ - skywalking + skywalking-commons com.a.eye 2.0-2016 @@ -26,12 +26,12 @@ com.a.eye skywalking-logging-api - ${project.version} + 2.0-2016 com.a.eye skywalking-logging-impl-log4j2 - ${project.version} + 2.0-2016 diff --git a/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/RegistryCenterFactory.java b/skywalking-commons/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/RegistryCenterFactory.java similarity index 100% rename from skywalking-registry/src/main/java/com/a/eye/skywalking/registry/RegistryCenterFactory.java rename to skywalking-commons/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/RegistryCenterFactory.java diff --git a/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/api/Center.java b/skywalking-commons/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/api/Center.java similarity index 100% rename from skywalking-registry/src/main/java/com/a/eye/skywalking/registry/api/Center.java rename to skywalking-commons/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/api/Center.java diff --git a/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/api/CenterType.java b/skywalking-commons/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/api/CenterType.java similarity index 100% rename from skywalking-registry/src/main/java/com/a/eye/skywalking/registry/api/CenterType.java rename to skywalking-commons/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/api/CenterType.java diff --git a/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/api/EventType.java b/skywalking-commons/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/api/EventType.java similarity index 100% rename from skywalking-registry/src/main/java/com/a/eye/skywalking/registry/api/EventType.java rename to skywalking-commons/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/api/EventType.java diff --git a/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/api/NotifyListener.java b/skywalking-commons/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/api/NotifyListener.java similarity index 100% rename from skywalking-registry/src/main/java/com/a/eye/skywalking/registry/api/NotifyListener.java rename to skywalking-commons/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/api/NotifyListener.java diff --git a/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/api/RegistryCenter.java b/skywalking-commons/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/api/RegistryCenter.java similarity index 100% rename from skywalking-registry/src/main/java/com/a/eye/skywalking/registry/api/RegistryCenter.java rename to skywalking-commons/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/api/RegistryCenter.java diff --git a/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/impl/zookeeper/ZookeeperConfig.java b/skywalking-commons/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/impl/zookeeper/ZookeeperConfig.java similarity index 100% rename from skywalking-registry/src/main/java/com/a/eye/skywalking/registry/impl/zookeeper/ZookeeperConfig.java rename to skywalking-commons/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/impl/zookeeper/ZookeeperConfig.java diff --git a/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/impl/zookeeper/ZookeeperRegistryCenter.java b/skywalking-commons/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/impl/zookeeper/ZookeeperRegistryCenter.java similarity index 100% rename from skywalking-registry/src/main/java/com/a/eye/skywalking/registry/impl/zookeeper/ZookeeperRegistryCenter.java rename to skywalking-commons/skywalking-registry/src/main/java/com/a/eye/skywalking/registry/impl/zookeeper/ZookeeperRegistryCenter.java diff --git a/skywalking-registry/src/main/resources/META-INF/services/com.a.eye.skywalking.registry.api.RegistryCenter b/skywalking-commons/skywalking-registry/src/main/resources/META-INF/services/com.a.eye.skywalking.registry.api.RegistryCenter similarity index 100% rename from skywalking-registry/src/main/resources/META-INF/services/com.a.eye.skywalking.registry.api.RegistryCenter rename to skywalking-commons/skywalking-registry/src/main/resources/META-INF/services/com.a.eye.skywalking.registry.api.RegistryCenter diff --git a/skywalking-registry/src/test/java/com/a/eye/skywalking/registry/RegistryCenterFactoryTest.java b/skywalking-commons/skywalking-registry/src/test/java/com/a/eye/skywalking/registry/RegistryCenterFactoryTest.java similarity index 100% rename from skywalking-registry/src/test/java/com/a/eye/skywalking/registry/RegistryCenterFactoryTest.java rename to skywalking-commons/skywalking-registry/src/test/java/com/a/eye/skywalking/registry/RegistryCenterFactoryTest.java diff --git a/skywalking-registry/src/test/resources/log4j.properties b/skywalking-commons/skywalking-registry/src/test/resources/log4j.properties similarity index 100% rename from skywalking-registry/src/test/resources/log4j.properties rename to skywalking-commons/skywalking-registry/src/test/resources/log4j.properties diff --git a/skywalking-registry/src/test/resources/log4j2.xml b/skywalking-commons/skywalking-registry/src/test/resources/log4j2.xml similarity index 100% rename from skywalking-registry/src/test/resources/log4j2.xml rename to skywalking-commons/skywalking-registry/src/test/resources/log4j2.xml diff --git a/skywalking-network/pom.xml b/skywalking-network/pom.xml index 6e4d2a0b6..69f74e998 100644 --- a/skywalking-network/pom.xml +++ b/skywalking-network/pom.xml @@ -120,7 +120,7 @@ - + org.apache.maven.plugins maven-antrun-plugin diff --git a/skywalking-network/src/main/java/com/a/eye/skywalking/network/TransferService.java b/skywalking-network/src/main/java/com/a/eye/skywalking/network/ServiceProvider.java similarity index 51% rename from skywalking-network/src/main/java/com/a/eye/skywalking/network/TransferService.java rename to skywalking-network/src/main/java/com/a/eye/skywalking/network/ServiceProvider.java index 3049635c8..98d0da770 100644 --- a/skywalking-network/src/main/java/com/a/eye/skywalking/network/TransferService.java +++ b/skywalking-network/src/main/java/com/a/eye/skywalking/network/ServiceProvider.java @@ -1,18 +1,18 @@ package com.a.eye.skywalking.network; -import com.a.eye.skywalking.network.grpc.services.SpanStorageService; -import com.a.eye.skywalking.network.grpc.services.TraceSearchService; -import com.a.eye.skywalking.network.listener.SpanStorageNotifier; -import com.a.eye.skywalking.network.listener.TraceSearchNotifier; +import com.a.eye.skywalking.network.grpc.provider.SpanStorageService; +import com.a.eye.skywalking.network.grpc.provider.TraceSearchService; +import com.a.eye.skywalking.network.listener.SpanStorageListener; +import com.a.eye.skywalking.network.listener.TraceSearchListener; import io.grpc.Server; import io.grpc.ServerBuilder; import java.io.IOException; -public class TransferService { +public class ServiceProvider { private Server server; - private TransferService(Server server) { + private ServiceProvider(Server server) { this.server = server; } @@ -21,7 +21,7 @@ public class TransferService { // 当JVM停止之后，Server也需要停止 Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { - TransferService.this.stop(); + ServiceProvider.this.stop(); } }); } @@ -32,10 +32,8 @@ public class TransferService { } } - private void blockUntilShutdown() throws InterruptedException { - if (server != null) { - server.awaitTermination(); - } + public static TransferServiceBuilder newBuilder(int port) { + return new TransferServiceBuilder(port); } public static class TransferServiceBuilder { @@ -45,21 +43,17 @@ public class TransferService { private ServerBuilder serverBuilder; - public static TransferServiceBuilder newBuilder(int port) { - return new TransferServiceBuilder(port); + public ServiceProvider build() { + return new ServiceProvider(serverBuilder.build()); } - public TransferService build() { - return new TransferService(serverBuilder.build()); - } - - public TransferServiceBuilder startSpanStorageService(SpanStorageNotifier spanStorageListener) { + public TransferServiceBuilder addSpanStorageService(SpanStorageListener spanStorageListener) { serverBuilder.addService(new SpanStorageService(spanStorageListener)); return this; } - public TransferServiceBuilder startTraceSearchService(TraceSearchNotifier traceSearchNotifier) { - serverBuilder.addService(new TraceSearchService(traceSearchNotifier)); + public TransferServiceBuilder addTraceSearchService(TraceSearchListener traceSearchListener) { + serverBuilder.addService(new TraceSearchService(traceSearchListener)); return this; } } diff --git a/skywalking-network/src/main/java/com/a/eye/skywalking/network/grpc/services/SpanStorageService.java b/skywalking-network/src/main/java/com/a/eye/skywalking/network/grpc/provider/SpanStorageService.java similarity index 88% rename from skywalking-network/src/main/java/com/a/eye/skywalking/network/grpc/services/SpanStorageService.java rename to skywalking-network/src/main/java/com/a/eye/skywalking/network/grpc/provider/SpanStorageService.java index a9038bdd8..29928a130 100644 --- a/skywalking-network/src/main/java/com/a/eye/skywalking/network/grpc/services/SpanStorageService.java +++ b/skywalking-network/src/main/java/com/a/eye/skywalking/network/grpc/provider/SpanStorageService.java @@ -1,17 +1,16 @@ -package com.a.eye.skywalking.network.grpc.services; +package com.a.eye.skywalking.network.grpc.provider; import com.a.eye.skywalking.network.grpc.AckSpan; import com.a.eye.skywalking.network.grpc.RequestSpan; import com.a.eye.skywalking.network.grpc.SendResult; import com.a.eye.skywalking.network.grpc.SpanStorageServiceGrpc; -import com.a.eye.skywalking.network.listener.SpanStorageNotifier; +import com.a.eye.skywalking.network.listener.SpanStorageListener; import io.grpc.stub.StreamObserver; public class SpanStorageService extends SpanStorageServiceGrpc.SpanStorageServiceImplBase { - private SpanStorageNotifier listener; - - public SpanStorageService(SpanStorageNotifier listener) { + private SpanStorageListener listener; + public SpanStorageService(SpanStorageListener listener) { this.listener = listener; } @@ -20,12 +19,12 @@ public class SpanStorageService extends SpanStorageServiceGrpc.SpanStorageServic return new StreamObserver() { @Override public void onNext(AckSpan value) { + listener.storage(value); } @Override public void onError(Throwable t) { - } @Override @@ -47,7 +46,6 @@ public class SpanStorageService extends SpanStorageServiceGrpc.SpanStorageServic @Override public void onError(Throwable t) { - } @Override diff --git a/skywalking-network/src/main/java/com/a/eye/skywalking/network/grpc/services/TraceSearchService.java b/skywalking-network/src/main/java/com/a/eye/skywalking/network/grpc/provider/TraceSearchService.java similarity index 65% rename from skywalking-network/src/main/java/com/a/eye/skywalking/network/grpc/services/TraceSearchService.java rename to skywalking-network/src/main/java/com/a/eye/skywalking/network/grpc/provider/TraceSearchService.java index 23f384595..effbfc19d 100644 --- a/skywalking-network/src/main/java/com/a/eye/skywalking/network/grpc/services/TraceSearchService.java +++ b/skywalking-network/src/main/java/com/a/eye/skywalking/network/grpc/provider/TraceSearchService.java @@ -1,10 +1,10 @@ -package com.a.eye.skywalking.network.grpc.services; +package com.a.eye.skywalking.network.grpc.provider; import com.a.eye.skywalking.network.grpc.SearchResult; import com.a.eye.skywalking.network.grpc.Span; import com.a.eye.skywalking.network.grpc.TraceSearchCondition; import com.a.eye.skywalking.network.grpc.TraceSearchServiceGrpc; -import com.a.eye.skywalking.network.listener.TraceSearchNotifier; +import com.a.eye.skywalking.network.listener.TraceSearchListener; import io.grpc.stub.StreamObserver; import java.util.List; @@ -14,15 +14,15 @@ import java.util.List; */ public class TraceSearchService extends TraceSearchServiceGrpc.TraceSearchServiceImplBase { - private TraceSearchNotifier traceSearchNotifier; + private TraceSearchListener traceSearchListener; - public TraceSearchService(TraceSearchNotifier traceSearchNotifier) { - this.traceSearchNotifier = traceSearchNotifier; + public TraceSearchService(TraceSearchListener traceSearchListener) { + this.traceSearchListener = traceSearchListener; } @Override public void search(TraceSearchCondition request, StreamObserver responseObserver) { - List spans = traceSearchNotifier.search(request.getTraceid()); + List spans = traceSearchListener.search(request.getTraceid()); responseObserver.onNext(SearchResult.newBuilder().addAllSpans(spans).build()); responseObserver.onCompleted(); } diff --git a/skywalking-network/src/main/java/com/a/eye/skywalking/network/listener/SpanStorageNotifier.java b/skywalking-network/src/main/java/com/a/eye/skywalking/network/listener/SpanStorageListener.java similarity index 86% rename from skywalking-network/src/main/java/com/a/eye/skywalking/network/listener/SpanStorageNotifier.java rename to skywalking-network/src/main/java/com/a/eye/skywalking/network/listener/SpanStorageListener.java index 3ae23a1f1..71c26794b 100644 --- a/skywalking-network/src/main/java/com/a/eye/skywalking/network/listener/SpanStorageNotifier.java +++ b/skywalking-network/src/main/java/com/a/eye/skywalking/network/listener/SpanStorageListener.java @@ -3,7 +3,7 @@ package com.a.eye.skywalking.network.listener; import com.a.eye.skywalking.network.grpc.AckSpan; import com.a.eye.skywalking.network.grpc.RequestSpan; -public interface SpanStorageNotifier { +public interface SpanStorageListener{ boolean storage(RequestSpan requestSpan); boolean storage(AckSpan ackSpan); diff --git a/skywalking-network/src/main/java/com/a/eye/skywalking/network/listener/TraceSearchNotifier.java b/skywalking-network/src/main/java/com/a/eye/skywalking/network/listener/TraceSearchListener.java similarity index 83% rename from skywalking-network/src/main/java/com/a/eye/skywalking/network/listener/TraceSearchNotifier.java rename to skywalking-network/src/main/java/com/a/eye/skywalking/network/listener/TraceSearchListener.java index b05528b7b..d08b2cb2c 100644 --- a/skywalking-network/src/main/java/com/a/eye/skywalking/network/listener/TraceSearchNotifier.java +++ b/skywalking-network/src/main/java/com/a/eye/skywalking/network/listener/TraceSearchListener.java @@ -7,6 +7,6 @@ import java.util.List; /** * Created by xin on 2016/11/12. */ -public interface TraceSearchNotifier { +public interface TraceSearchListener{ List search(String traceId); } diff --git a/skywalking-storage-center/skywalking-storage/pom.xml b/skywalking-storage-center/skywalking-storage/pom.xml index cfac8066d..8939995fc 100644 --- a/skywalking-storage-center/skywalking-storage/pom.xml +++ b/skywalking-storage-center/skywalking-storage/pom.xml @@ -23,7 +23,7 @@ com.a.eye skywalking-registry - ${project.version} + 2.0-2016 com.a.eye @@ -51,6 +51,11 @@ HikariCP 2.4.3 + + com.a.eye + skywalking-health-report + 2.0-2016 + diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/Main.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/Main.java index 77139db45..3cc27ba63 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/Main.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/Main.java @@ -1,10 +1,10 @@ package com.a.eye.skywalking.storage; +import com.a.eye.skywalking.health.report.HealthCollector; import com.a.eye.skywalking.logging.api.ILog; import com.a.eye.skywalking.logging.api.LogManager; import com.a.eye.skywalking.logging.impl.log4j2.Log4j2Resolver; -import com.a.eye.skywalking.network.TransferService; -import com.a.eye.skywalking.network.TransferService.TransferServiceBuilder; +import com.a.eye.skywalking.network.ServiceProvider; import com.a.eye.skywalking.registry.RegistryCenterFactory; import com.a.eye.skywalking.registry.api.CenterType; import com.a.eye.skywalking.registry.api.RegistryCenter; @@ -13,14 +13,16 @@ import com.a.eye.skywalking.storage.block.index.BlockIndexEngine; import com.a.eye.skywalking.storage.config.Config; import com.a.eye.skywalking.storage.config.ConfigInitializer; import com.a.eye.skywalking.storage.data.IndexDataCapacityMonitor; -import com.a.eye.skywalking.storage.notifier.SearchNotifier; -import com.a.eye.skywalking.storage.notifier.StorageNotifier; +import com.a.eye.skywalking.storage.data.file.DataFilesManager; +import com.a.eye.skywalking.storage.listener.SearchListener; +import com.a.eye.skywalking.storage.listener.StorageListener; import com.a.eye.skywalking.storage.util.NetUtils; import java.io.IOException; +import java.util.Map; import java.util.Properties; -import static com.a.eye.skywalking.storage.config.Config.RegistryCenter.REGISTRY_PATH_PREFIX; +import static com.a.eye.skywalking.storage.config.Config.RegistryCenter.PATH_PREFIX; /** * Created by xin on 2016/11/12. @@ -33,28 +35,36 @@ public class Main { LogManager.setLogResolver(new Log4j2Resolver()); } - private static TransferService transferService; + private static ServiceProvider provider; public static void main(String[] args) { try { initializeParam(); + HealthCollector.init(); + + DataFilesManager.init(); + BlockIndexEngine.start(); + IndexDataCapacityMonitor.start(); - transferService = - TransferServiceBuilder.newBuilder(Config.Server.PORT).startSpanStorageService(new StorageNotifier()) - .startTraceSearchService(new SearchNotifier()).build(); - transferService.start(); - logger.info("transfer service started successfully."); + + provider = ServiceProvider.newBuilder(Config.Server.PORT).addSpanStorageService(new StorageListener()) + .addTraceSearchService(new SearchListener()).build(); + provider.start(); + + if (logger.isDebugEnable()) { + logger.debug("Service provider started successfully."); + } registryNode(); - logger.info("storage service started successfully."); + + logger.info("Storage service started successfully."); Thread.currentThread().join(); } catch (Throwable e) { - e.printStackTrace(); logger.error("Failed to start service.", e); } finally { - transferService.stop(); + provider.stop(); IndexDataCapacityMonitor.stop(); } } @@ -68,13 +78,14 @@ public class Main { registerConfig.setProperty(ZookeeperConfig.AUTH_INFO, Config.RegistryCenter.AUTH_INFO); registryCenter.start(registerConfig); registryCenter.register( - REGISTRY_PATH_PREFIX + NetUtils.getLocalAddress().getHostAddress() + ":" + Config.Server.PORT); + PATH_PREFIX + NetUtils.getLocalAddress().getHostAddress() + ":" + Config.Server.PORT); } private static void initializeParam() throws IllegalAccessException, IOException { Properties properties = new Properties(); try { properties.load(Main.class.getResourceAsStream("/config.properties")); + printStorageConfig(properties); ConfigInitializer.initialize(properties, Config.class); } catch (IllegalAccessException e) { logger.error("Initialize the collect server configuration failed", e); @@ -84,4 +95,11 @@ public class Main { throw e; } } + + + private static void printStorageConfig(Properties config) { + for (Map.Entry entry : config.entrySet()) { + logger.info("{} = {}", entry.getKey(), entry.getValue()); + } + } } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/block/index/BlockFinder.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/block/index/BlockFinder.java index 440aaf2cc..90f1fe690 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/block/index/BlockFinder.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/block/index/BlockFinder.java @@ -1,10 +1,15 @@ package com.a.eye.skywalking.storage.block.index; +import com.a.eye.skywalking.logging.api.ILog; +import com.a.eye.skywalking.logging.api.LogManager; + /** * Created by xin on 2016/11/2. */ public class BlockFinder { + private static ILog logger = LogManager.getLogger(BlockFinder.class); + private L1Cache l1Cache; private L2Cache l2Cache; @@ -19,11 +24,15 @@ public class BlockFinder { index = l2Cache.find(timestamp); } + if (logger.isDebugEnable()) { + logger.debug("Time stamp : {} is mapping with block Index {}.", timestamp, index); + } + return index; } - public long findLastBlockIndex(){ + public long findLastBlockIndex() { return l2Cache.getLastBlockIndex(); } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/block/index/BlockIndexUpdator.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/block/index/BlockIndexUpdator.java index d1061339c..753b60a50 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/block/index/BlockIndexUpdator.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/block/index/BlockIndexUpdator.java @@ -9,8 +9,9 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import static com.a.eye.skywalking.storage.config.Config.BlockIndex.DATA_FILE_INDEX_FILE_NAME; -import static com.a.eye.skywalking.storage.config.Config.BlockIndex.STORAGE_BASE_PATH; +import static com.a.eye.skywalking.storage.config.Config.BlockIndex.FILE_NAME; +import static com.a.eye.skywalking.storage.config.Config.BlockIndex.PATH; +import static com.a.eye.skywalking.storage.util.PathResolver.getAbsolutePath; public class BlockIndexUpdator { @@ -24,12 +25,12 @@ public class BlockIndexUpdator { } public void addRecord(long timestamp) { - logger.info("Updating index. timestamp:{}", timestamp); + logger.info("Updating block index. index key:{}", timestamp); try { updateFile(timestamp); updateCache(timestamp); } catch (Exception e) { - logger.error("Failed to add index record", e); + logger.error("Failed to add block index record", e); } } @@ -83,19 +84,30 @@ public class BlockIndexUpdator { } } + if (indexData.size() == 0) { + if (logger.isDebugEnable()) { + logger.debug("Any block index was not founded. will add new block index.", indexData.size()); + } //如果此前没有记录，则取之前五分钟到目前的数据 addRecord(System.currentTimeMillis() - 5 * 60 * 1000); return; } + if (logger.isDebugEnable()) { + logger.debug("There are {} block index was founded. Begin to init L1Cache and L2Cache", indexData.size()); + } + Collections.reverse(indexData); l1Cache.init(indexData); l2Cache.init(indexData); } public File getOrCreateBlockIndexFile() throws IOException { - File blockIndexFile = new File(STORAGE_BASE_PATH, DATA_FILE_INDEX_FILE_NAME); + if (logger.isDebugEnable()) { + + } + File blockIndexFile = new File(getAbsolutePath(PATH), FILE_NAME); if (!blockIndexFile.getParentFile().exists()) { blockIndexFile.getParentFile().mkdirs(); diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/block/index/L1Cache.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/block/index/L1Cache.java index 0107eaac2..ad9e5b1ca 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/block/index/L1Cache.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/block/index/L1Cache.java @@ -9,6 +9,8 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import static com.a.eye.skywalking.storage.config.Config.BlockIndexEngine.L1_CACHE_SIZE; + /** * 块索引的一级缓存 *

@@ -16,15 +18,20 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; */ public class L1Cache { - private static final int MAX_DATA_KEEP_SIZE = 30; - private static ILog logger = LogManager.getLogger(L1Cache.class); - private TreeSet cacheData = new TreeSet(); - private final ReadWriteLock updateLock = new ReentrantReadWriteLock(); + private static ILog logger = LogManager.getLogger(L1Cache.class); + private TreeSet cacheData = new TreeSet(); + private final ReadWriteLock updateLock = new ReentrantReadWriteLock(); void init(List data) { - int size = data.size() > MAX_DATA_KEEP_SIZE ? MAX_DATA_KEEP_SIZE : data.size(); + StringBuilder initData = new StringBuilder(); + int size = data.size() > L1_CACHE_SIZE ? L1_CACHE_SIZE : data.size(); for (int i = 0; i < size; i++) { this.cacheData.add(data.get(i)); + initData.append(data.get(i)).append(","); + } + + if (logger.isDebugEnable()) { + logger.info("L1 cache init data : [{}]", initData.deleteCharAt(initData.length() - 1)); } } @@ -42,7 +49,7 @@ public class L1Cache { TreeSet newCacheData = new TreeSet<>(cacheData); newCacheData.add(timestamp); - if (newCacheData.size() >= MAX_DATA_KEEP_SIZE + 1) { + if (newCacheData.size() >= L1_CACHE_SIZE + 1) { long removedData = newCacheData.pollFirst(); logger.info("Add cache data : {}, removed cache Data:{}", timestamp, removedData); } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/block/index/L2Cache.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/block/index/L2Cache.java index f9ebcac47..81975f04d 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/block/index/L2Cache.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/block/index/L2Cache.java @@ -1,5 +1,8 @@ package com.a.eye.skywalking.storage.block.index; +import com.a.eye.skywalking.logging.api.ILog; +import com.a.eye.skywalking.logging.api.LogManager; + import java.util.List; import java.util.TreeSet; import java.util.concurrent.locks.Lock; @@ -11,11 +14,16 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; */ public class L2Cache { + private ILog logger = LogManager.getLogger(L2Cache.class); + private TreeSet cacheData = new TreeSet(); private ReadWriteLock updateLock = new ReentrantReadWriteLock(); void init(List data) { this.cacheData.addAll(data); + if (logger.isDebugEnable()) { + logger.info("L2 cache init data : {}", data); + } } public Long find(long timestamp) { diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/config/Config.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/config/Config.java index aa498e994..3740b57ae 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/config/Config.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/config/Config.java @@ -6,34 +6,44 @@ package com.a.eye.skywalking.storage.config; public class Config { public static class Server { public static int PORT = 34000; + + public static int CHANNEL_SIZE = 10; + + public static int BUFFER_SIZE = 1000; } public static class BlockIndex { + public static String PATH = "/block_index"; - public static String STORAGE_BASE_PATH = "/tmp/skywalking/block_index"; - - public static String DATA_FILE_INDEX_FILE_NAME = "data_file.index"; + public static String FILE_NAME = "data_file.index"; } public static class DataFile { - public static String BASE_PATH = "/tmp/skywalking/data/file"; + public static String PATH = "/data/file"; - public static long MAX_LENGTH = 3 * 1024 * 1024 * 1024; + public static long SIZE = 3 * 1024 * 1024 * 1024L; } public static class DataIndex { - public static String TABLE_NAME = "data_index"; + public static String PATH = "/data/index"; - public static String BASE_PATH = "/tmp/skywalking/data/index"; + public static String FILE_NAME = "dataIndex"; - public static String STORAGE_INDEX_FILE_NAME = "dataIndex"; + public static long SIZE = 1000 * 1000 * 1000; - public static long MAX_CAPACITY_PER_INDEX = 1000 * 1000 * 1000 * 1000; + public static class Operator { + public static int CACHE_SIZE = 5; + } + } + + + public static class BlockIndexEngine { + public static int L1_CACHE_SIZE = 10; } @@ -45,11 +55,17 @@ public class Config { public static String CONNECT_URL = "127.0.0.1:2181"; - public static String REGISTRY_PATH_PREFIX = "/storage_list/"; + public static String PATH_PREFIX = "/skywalking/storage_list/"; } - public static class SpanFinder { - public static int MAX_CACHE_SIZE = 10; + public static class Finder { + public static int CACHED_SIZE = 10; + + + public static class DataSource { + public static int MAX_POOL_SIZE = 20; + public static int MIN_IDLE = 5; + } } } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/config/Constants.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/config/Constants.java index 38d97709f..dfbd0275e 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/config/Constants.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/config/Constants.java @@ -1,14 +1,15 @@ package com.a.eye.skywalking.storage.config; -import static com.a.eye.skywalking.storage.config.Config.DataIndex.TABLE_NAME; - public class Constants { + public final static String TABLE_NAME = "data_index"; + + public static int MAX_BATCH_SIZE = 50; + public static class SQL { public static final String CREATE_TABLE = "CREATE TABLE " + TABLE_NAME + "\n" + "(\n" + " id INT PRIMARY KEY NOT NULL IDENTITY,\n" + " trace_id VARCHAR(64) NOT NULL,\n" - + " levelId VARCHAR(1024) NOT NULL,\n" + " span_type INT NOT NULL, \n" + " file_name VARCHAR(32) NOT NULL,\n" + " offset BIGINT NOT NULL,\n" @@ -16,8 +17,8 @@ public class Constants { public static final String CREATE_INDEX = "CREATE INDEX \"index_data_trace_id_index\" ON " + TABLE_NAME + " (trace_id);"; - public static final String INSERT_INDEX = "INSERT INTO " +TABLE_NAME + "(trace_id,levelId,span_type" - + ",file_name,offset,length) VALUES(?,?,?,?,?,?)"; + public static final String INSERT_INDEX = "INSERT INTO " +TABLE_NAME + "(trace_id,span_type" + + ",file_name,offset,length) VALUES(?,?,?,?,?)"; public static final String QUERY_TABLES = "SELECT count(1) AS TABLE_COUNT FROM INFORMATION_SCHEMA.TABLES " + "WHERE TABLE_NAME= '" + TABLE_NAME.toUpperCase() + "';"; diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/IndexDataCapacityMonitor.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/IndexDataCapacityMonitor.java index 7b74c2f1e..309456306 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/IndexDataCapacityMonitor.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/IndexDataCapacityMonitor.java @@ -1,5 +1,7 @@ package com.a.eye.skywalking.storage.data; +import com.a.eye.skywalking.health.report.HealthCollector; +import com.a.eye.skywalking.health.report.HeathReading; import com.a.eye.skywalking.logging.api.ILog; import com.a.eye.skywalking.logging.api.LogManager; import com.a.eye.skywalking.storage.block.index.BlockIndexEngine; @@ -11,7 +13,7 @@ import java.util.TimerTask; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import static com.a.eye.skywalking.storage.config.Config.DataIndex.MAX_CAPACITY_PER_INDEX; +import static com.a.eye.skywalking.storage.config.Config.DataIndex.SIZE; /** * Created by xin on 2016/11/6. @@ -63,9 +65,12 @@ public class IndexDataCapacityMonitor { @Override public void run() { - if (currentSize.get() > MAX_CAPACITY_PER_INDEX * 0.8) { + if (currentSize.get() > SIZE * 0.8) { notificationAddNewBlockIndexAndCreateNewIndexDB(); stopTimer(); + HealthCollector.getCurrentHeathReading("Index Data Capacity Detector").updateData(HeathReading.INFO, + "Detector is detecting the index [%d]. and the capacity of {} is %d", timestamp, + currentSize.get()); } } } @@ -73,6 +78,7 @@ public class IndexDataCapacityMonitor { private static void notificationAddNewBlockIndexAndCreateNewIndexDB() { long timestamp = System.currentTimeMillis() + 5 * 60 * 1000; BlockIndexEngine.newUpdator().addRecord(timestamp); + logger.info("Create a new Index DB [{}]", timestamp); createNewIndexDB(timestamp); detector = new Detector(timestamp); } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/SpanDataConsumer.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/SpanDataConsumer.java index aab9fc9f6..11d69edae 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/SpanDataConsumer.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/SpanDataConsumer.java @@ -1,6 +1,8 @@ package com.a.eye.skywalking.storage.data; import com.a.eye.datacarrier.consumer.IConsumer; +import com.a.eye.skywalking.health.report.HealthCollector; +import com.a.eye.skywalking.health.report.HeathReading; import com.a.eye.skywalking.storage.block.index.BlockIndexEngine; import com.a.eye.skywalking.storage.data.file.DataFileWriter; import com.a.eye.skywalking.storage.data.index.*; @@ -34,6 +36,8 @@ public class SpanDataConsumer implements IConsumer { IndexMetaGroup metaGroup = iterator.next(); IndexOperator indexOperator = IndexOperator.newOperator(getDBConnector(metaGroup)); indexOperator.batchUpdate(metaGroup); + HealthCollector.getCurrentHeathReading("SpanDataConsumer") + .updateData(HeathReading.INFO, "%s messages were successful consumed .", data.size()); } } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/SpanDataFinder.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/SpanDataFinder.java index ece072a9b..aa2b8bdfc 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/SpanDataFinder.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/SpanDataFinder.java @@ -15,10 +15,12 @@ import java.util.*; import static com.a.eye.skywalking.storage.config.Constants.SQL.DEFAULT_PASSWORD; import static com.a.eye.skywalking.storage.config.Constants.SQL.DEFAULT_USER; +import static com.a.eye.skywalking.storage.util.PathResolver.getAbsolutePath; public class SpanDataFinder { - private static ILog logger = LogManager.getLogger(SpanDataFinder.class); - private static IndexDataSourceCache datasourceCache = new IndexDataSourceCache(Config.SpanFinder.MAX_CACHE_SIZE); + private static ILog logger = LogManager.getLogger(SpanDataFinder.class); + private static IndexDataSourceCache datasourceCache = + new IndexDataSourceCache(Config.Finder.CACHED_SIZE); public static List find(String traceId) { long blockIndex = BlockIndexEngine.newFinder().find(fetchStartTimeFromTraceId(traceId)); @@ -71,13 +73,13 @@ public class SpanDataFinder { private static HikariConfig generateDatasourceConfig(long blockIndex) { HikariConfig config = new HikariConfig(); - config.setJdbcUrl(new ConnectURLGenerator(Config.DataIndex.BASE_PATH, Config.DataIndex.STORAGE_INDEX_FILE_NAME) - .generate(blockIndex)); + config.setJdbcUrl(new ConnectURLGenerator(getAbsolutePath(Config.DataIndex.PATH), + Config.DataIndex.FILE_NAME).generate(blockIndex)); config.setDriverClassName("org.hsqldb.jdbc.JDBCDriver"); config.setUsername(DEFAULT_USER); config.setPassword(DEFAULT_PASSWORD); - config.setMaximumPoolSize(20); - config.setMinimumIdle(5); + config.setMaximumPoolSize(Config.Finder.DataSource.MAX_POOL_SIZE); + config.setMinimumIdle(Config.Finder.DataSource.MIN_IDLE); return config; } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/exception/SpanConvertFailedException.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/exception/SpanConvertFailedException.java new file mode 100644 index 000000000..dec6bf06d --- /dev/null +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/exception/SpanConvertFailedException.java @@ -0,0 +1,10 @@ +package com.a.eye.skywalking.storage.data.exception; + +/** + * Created by xin on 2016/11/14. + */ +public class SpanConvertFailedException extends RuntimeException { + public SpanConvertFailedException(String message, Exception e) { + super(message, e); + } +} diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/DataFile.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/DataFile.java index f8c84c9fc..e005a85d6 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/DataFile.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/DataFile.java @@ -14,6 +14,8 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import static com.a.eye.skywalking.storage.util.PathResolver.getAbsolutePath; + /** * 数据文件 */ @@ -25,7 +27,7 @@ public class DataFile { private DataFileOperator operator; static { - File dataFileDir = new File(Config.DataFile.BASE_PATH); + File dataFileDir = new File(getAbsolutePath(Config.DataFile.PATH)); if (!dataFileDir.exists()) { dataFileDir.mkdirs(); } @@ -52,10 +54,13 @@ public class DataFile { } private void createFile() { - File dataFile = new File(Config.DataFile.BASE_PATH, fileName); + File dataFile = getDataFile(); if (!dataFile.exists()) { try { dataFile.createNewFile(); + if (logger.isDebugEnable()) { + logger.debug("Create an new data file[{}].", fileName); + } } catch (IOException e) { logger.error("Failed to create data file.", e); throw new DataFileOperatorCreateFailedException("Failed to create data file", e); @@ -64,7 +69,11 @@ public class DataFile { } public boolean overLimitLength() { - return currentOffset >= Config.DataFile.MAX_LENGTH; + boolean isOverLimitLength = currentOffset >= Config.DataFile.SIZE; + if (isOverLimitLength) { + logger.info("Data File[{}] is over limit length.", fileName); + } + return isOverLimitLength; } public IndexMetaInfo write(SpanData data) { @@ -107,7 +116,7 @@ public class DataFile { if (writer == null) { try { - writer = new FileOutputStream(new File(Config.DataFile.BASE_PATH, fileName), true); + writer = new FileOutputStream(getDataFile(), true); } catch (IOException e) { throw new DataFileOperatorCreateFailedException("Failed to create datafile output stream", e); } @@ -119,7 +128,7 @@ public class DataFile { public FileInputStream getReader() { if (reader == null) { try { - reader = new FileInputStream(new File(Config.DataFile.BASE_PATH, fileName)); + reader = new FileInputStream(getDataFile()); } catch (IOException e) { throw new DataFileOperatorCreateFailedException("Failed to create datafile input stream", e); } @@ -128,4 +137,13 @@ public class DataFile { return reader; } } + + private File getDataFile() { + return new File(getAbsolutePath(Config.DataFile.PATH), fileName); + } + + @Override + public String toString() { + return "DataFile{" + "fileName='" + fileName + '\'' + '}'; + } } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/DataFileLoader.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/DataFileLoader.java index 32973177d..1fd7971bd 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/DataFileLoader.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/DataFileLoader.java @@ -14,6 +14,11 @@ public class DataFileLoader { public List load() { File dataFileDir = new File(basePath); + + if (!dataFileDir.exists()) { + dataFileDir.mkdirs(); + } + List allDataFile = new ArrayList(); for (File fileEntry : dataFileDir.listFiles()) { allDataFile.add(new DataFile(fileEntry)); diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/DataFileReader.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/DataFileReader.java index e237401c7..34b6836e5 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/DataFileReader.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/DataFileReader.java @@ -1,5 +1,7 @@ package com.a.eye.skywalking.storage.data.file; +import com.a.eye.skywalking.health.report.HealthCollector; +import com.a.eye.skywalking.health.report.HeathReading; import com.a.eye.skywalking.logging.api.ILog; import com.a.eye.skywalking.logging.api.LogManager; import com.a.eye.skywalking.network.grpc.AckSpan; @@ -13,6 +15,9 @@ import com.a.eye.skywalking.storage.data.spandata.SpanType; import java.util.ArrayList; import java.util.List; +import static com.a.eye.skywalking.storage.data.spandata.SpanDataBuilder.buildAckSpan; +import static com.a.eye.skywalking.storage.data.spandata.SpanDataBuilder.buildRequestSpan; + /** * Created by xin on 2016/11/6. */ @@ -31,12 +36,12 @@ public class DataFileReader { byte[] dataByte = dataFile.read(indexMetaInfo.getOffset(), indexMetaInfo.getLength()); try { if (indexMetaInfo.getSpanType() == SpanType.RequestSpan) { - metaData.add(new RequestSpanData(RequestSpan.parseFrom(dataByte))); + metaData.add(new RequestSpanData(buildRequestSpan(dataByte))); } else { - metaData.add(new AckSpanData(AckSpan.parseFrom(dataByte))); + metaData.add(new AckSpanData(buildAckSpan(dataByte))); } } catch (Exception e) { - logger.error("Failed to conver to data", e); + logger.error("Failed to convert to data", e); } } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/DataFilesManager.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/DataFilesManager.java index 04adb9341..a8e44d2cd 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/DataFilesManager.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/DataFilesManager.java @@ -5,6 +5,8 @@ import com.a.eye.skywalking.storage.config.Config; import java.util.List; import java.util.concurrent.ConcurrentLinkedDeque; +import static com.a.eye.skywalking.storage.util.PathResolver.getAbsolutePath; + /** * 管理数据文件，目前主要是用来加载所有未满的数据文件，以及创建数据文件 */ @@ -13,7 +15,7 @@ public class DataFilesManager { private static ConcurrentLinkedDeque unFinishedDataFiles = new ConcurrentLinkedDeque<>(); public static void init() { - List allDataFile = new DataFileLoader(Config.DataFile.BASE_PATH).load(); + List allDataFile = new DataFileLoader(getAbsolutePath(Config.DataFile.PATH)).load(); unFinishedDataFiles = new ConcurrentLinkedDeque<>(new UnFinishedDataFilePicker(allDataFile).pickUp()); } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/UnFinishedDataFilePicker.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/UnFinishedDataFilePicker.java index 9df47927e..2a4360dce 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/UnFinishedDataFilePicker.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/UnFinishedDataFilePicker.java @@ -1,5 +1,8 @@ package com.a.eye.skywalking.storage.data.file; +import com.a.eye.skywalking.logging.api.ILog; +import com.a.eye.skywalking.logging.api.LogManager; + import java.util.ArrayList; import java.util.List; @@ -8,6 +11,8 @@ import java.util.List; */ public class UnFinishedDataFilePicker { + private static ILog logger = LogManager.getLogger(UnFinishedDataFilePicker.class); + private List dataFiles; public UnFinishedDataFilePicker(List dataFiles) { @@ -17,10 +22,11 @@ public class UnFinishedDataFilePicker { public List pickUp() { List result = new ArrayList(); for (DataFile file : dataFiles) { - if (file.overLimitLength()){ + if (!file.overLimitLength()) { result.add(file); } } + logger.info("Unfinished files: [{}].", result); return result; } } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexDBConnector.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexDBConnector.java index aedaa2ed0..737902015 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexDBConnector.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexDBConnector.java @@ -3,6 +3,7 @@ package com.a.eye.skywalking.storage.data.index; import com.a.eye.skywalking.logging.api.ILog; import com.a.eye.skywalking.logging.api.LogManager; import com.a.eye.skywalking.storage.config.Config; +import com.a.eye.skywalking.storage.config.Constants; import com.a.eye.skywalking.storage.data.exception.ConnectorInitializeFailedException; import com.a.eye.skywalking.storage.data.spandata.AckSpanData; import com.a.eye.skywalking.storage.data.spandata.RequestSpanData; @@ -12,14 +13,13 @@ import com.a.eye.skywalking.storage.data.spandata.SpanType; import java.sql.*; import static com.a.eye.skywalking.storage.config.Constants.SQL.*; +import static com.a.eye.skywalking.storage.util.PathResolver.getAbsolutePath; /** * Created by xin on 2016/11/4. */ public class IndexDBConnector { - private static final int MAX_BATCH_SIZE = 20; - private static ILog logger = LogManager.getLogger(IndexDBConnector.class); static { @@ -32,8 +32,8 @@ public class IndexDBConnector { private long timestamp; private Connection connection; - private ConnectURLGenerator generator = - new ConnectURLGenerator(Config.DataIndex.BASE_PATH, Config.DataIndex.STORAGE_INDEX_FILE_NAME); + private ConnectURLGenerator generator = new ConnectURLGenerator(getAbsolutePath(Config.DataIndex.PATH), + Config.DataIndex.FILE_NAME); public IndexDBConnector(long timestamp) { this.timestamp = timestamp; @@ -41,7 +41,7 @@ public class IndexDBConnector { createTableAndIndexIfNecessary(); } - public IndexDBConnector(Connection connection){ + public IndexDBConnector(Connection connection) { this.connection = connection; createTableAndIndexIfNecessary(); } @@ -100,13 +100,12 @@ public class IndexDBConnector { PreparedStatement ps = connection.prepareStatement(INSERT_INDEX); for (IndexMetaInfo metaInfo : metaGroup.getMetaInfo()) { ps.setString(1, metaInfo.getTraceId()); - ps.setString(2, metaInfo.getLevelId()); - ps.setInt(3, metaInfo.getSpanType().getValue()); - ps.setString(4, metaInfo.getFileName()); - ps.setLong(5, metaInfo.getOffset()); - ps.setInt(6, metaInfo.getLength()); + ps.setInt(2, metaInfo.getSpanType().getValue()); + ps.setString(3, metaInfo.getFileName()); + ps.setLong(4, metaInfo.getOffset()); + ps.setInt(5, metaInfo.getLength()); ps.addBatch(); - if (++currentIndex > MAX_BATCH_SIZE) { + if (++currentIndex > Constants.MAX_BATCH_SIZE) { ps.executeBatch(); } } @@ -126,7 +125,7 @@ public class IndexDBConnector { return indexSize; } - public IndexMetaCollection queryByTraceId(String traceId){ + public IndexMetaCollection queryByTraceId(String traceId) { try { PreparedStatement ps = connection.prepareStatement(QUERY_TRACE_ID); ps.setString(1, traceId); @@ -143,10 +142,12 @@ public class IndexDBConnector { spanData = new RequestSpanData(); } - collection.add(new IndexMetaInfo(spanData, rs.getString("file_name"), rs.getLong("offset"), rs.getInt("length"))); + collection.add(new IndexMetaInfo(spanData, rs.getString("file_name"), rs.getLong("offset"), + rs.getInt("length"))); } return collection; - }catch(SQLException e){ + } catch (SQLException e) { + logger.error("Failed to query trace Id [{}]", traceId, e); return new IndexMetaCollection(); } } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexDBConnectorCache.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexDBConnectorCache.java index fa52d96fd..589c826d1 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexDBConnectorCache.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexDBConnectorCache.java @@ -1,16 +1,16 @@ package com.a.eye.skywalking.storage.data.index; +import com.a.eye.skywalking.storage.config.Config; + import java.util.LinkedHashMap; import java.util.Map; public class IndexDBConnectorCache { - private static final int MAX_CACHE_SIZE = 5; - private LRUCache cachedOperators; public IndexDBConnectorCache() { - cachedOperators = new LRUCache(MAX_CACHE_SIZE); + cachedOperators = new LRUCache(Config.DataIndex.Operator.CACHE_SIZE); } public IndexDBConnector get(long timestamp) { @@ -33,7 +33,7 @@ public class IndexDBConnectorCache { private class LRUCache extends LinkedHashMap { @Override protected boolean removeEldestEntry(Map.Entry eldest) { - boolean isNeedRemove = size() > MAX_CACHE_SIZE; + boolean isNeedRemove = size() > Config.DataIndex.Operator.CACHE_SIZE; if (isNeedRemove) { removedCache(eldest.getValue()); } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexOperator.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexOperator.java index 279de6435..9770ebf5b 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexOperator.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexOperator.java @@ -16,11 +16,6 @@ public class IndexOperator { timestamp = connector.getTimestamp(); } - public List find(String taceId) { - return new ArrayList(); - } - - public void batchUpdate(IndexMetaGroup metaGroup) { try { connector.batchUpdate(metaGroup); diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/spandata/SpanDataBuilder.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/spandata/SpanDataBuilder.java index 0d524167c..ee8687023 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/spandata/SpanDataBuilder.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/spandata/SpanDataBuilder.java @@ -5,9 +5,7 @@ import com.a.eye.skywalking.logging.api.LogManager; import com.a.eye.skywalking.network.dependencies.com.google.protobuf.InvalidProtocolBufferException; import com.a.eye.skywalking.network.grpc.AckSpan; import com.a.eye.skywalking.network.grpc.RequestSpan; -import com.a.eye.skywalking.storage.data.spandata.AckSpanData; -import com.a.eye.skywalking.storage.data.spandata.RequestSpanData; -import com.a.eye.skywalking.storage.data.spandata.SpanData; +import com.a.eye.skywalking.storage.data.exception.SpanConvertFailedException; /** * Created by xin on 2016/11/12. @@ -29,7 +27,7 @@ public class SpanDataBuilder { return AckSpan.parseFrom(data); } catch (InvalidProtocolBufferException e) { logger.error("Failed to convert data to ack span.", e); - return null; + throw new SpanConvertFailedException("Failed to convert byte data to ack span", e); } } @@ -38,7 +36,7 @@ public class SpanDataBuilder { return RequestSpan.parseFrom(data); } catch (InvalidProtocolBufferException e) { logger.error("Failed to convert data to request span.", e); - return null; + throw new SpanConvertFailedException("Failed to convert byte data to request span", e); } } } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/notifier/SpanDataHelper.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/spandata/SpanDataHelper.java similarity index 90% rename from skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/notifier/SpanDataHelper.java rename to skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/spandata/SpanDataHelper.java index b685f7920..194f6d5da 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/notifier/SpanDataHelper.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/spandata/SpanDataHelper.java @@ -1,9 +1,6 @@ -package com.a.eye.skywalking.storage.notifier; +package com.a.eye.skywalking.storage.data.spandata; import com.a.eye.skywalking.network.grpc.Span; -import com.a.eye.skywalking.storage.data.spandata.AckSpanData; -import com.a.eye.skywalking.storage.data.spandata.RequestSpanData; -import com.a.eye.skywalking.storage.data.spandata.SpanData; import java.util.ArrayList; import java.util.HashMap; diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/notifier/SearchNotifier.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/listener/SearchListener.java similarity index 63% rename from skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/notifier/SearchNotifier.java rename to skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/listener/SearchListener.java index 61967d79e..2fdce57ae 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/notifier/SearchNotifier.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/listener/SearchListener.java @@ -1,13 +1,14 @@ -package com.a.eye.skywalking.storage.notifier; +package com.a.eye.skywalking.storage.listener; import com.a.eye.skywalking.network.grpc.Span; -import com.a.eye.skywalking.network.listener.TraceSearchNotifier; +import com.a.eye.skywalking.network.listener.TraceSearchListener; import com.a.eye.skywalking.storage.data.SpanDataFinder; import com.a.eye.skywalking.storage.data.spandata.SpanData; +import com.a.eye.skywalking.storage.data.spandata.SpanDataHelper; import java.util.List; -public class SearchNotifier implements TraceSearchNotifier { +public class SearchListener implements TraceSearchListener { @Override public List search(String s) { diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/notifier/StorageNotifier.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/listener/StorageListener.java similarity index 75% rename from skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/notifier/StorageNotifier.java rename to skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/listener/StorageListener.java index d309449a3..7cef89232 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/notifier/StorageNotifier.java +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/listener/StorageListener.java @@ -1,23 +1,24 @@ -package com.a.eye.skywalking.storage.notifier; +package com.a.eye.skywalking.storage.listener; import com.a.eye.datacarrier.DataCarrier; import com.a.eye.skywalking.logging.api.ILog; import com.a.eye.skywalking.logging.api.LogManager; import com.a.eye.skywalking.network.grpc.AckSpan; import com.a.eye.skywalking.network.grpc.RequestSpan; -import com.a.eye.skywalking.network.listener.SpanStorageNotifier; +import com.a.eye.skywalking.network.listener.SpanStorageListener; +import com.a.eye.skywalking.storage.config.Config; +import com.a.eye.skywalking.storage.data.SpanDataConsumer; import com.a.eye.skywalking.storage.data.spandata.SpanData; import com.a.eye.skywalking.storage.data.spandata.SpanDataBuilder; -import com.a.eye.skywalking.storage.data.SpanDataConsumer; -public class StorageNotifier implements SpanStorageNotifier { +public class StorageListener implements SpanStorageListener { - private ILog logger = LogManager.getLogger(StorageNotifier.class); + private ILog logger = LogManager.getLogger(StorageListener.class); private DataCarrier spanDataDataCarrier; - public StorageNotifier() { - spanDataDataCarrier = new DataCarrier<>(10, 1000); + public StorageListener() { + spanDataDataCarrier = new DataCarrier<>(Config.Server.CHANNEL_SIZE, Config.Server.BUFFER_SIZE); spanDataDataCarrier.consume(new SpanDataConsumer(), 5, true); } diff --git a/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/util/PathResolver.java b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/util/PathResolver.java new file mode 100644 index 000000000..9b04decc8 --- /dev/null +++ b/skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/util/PathResolver.java @@ -0,0 +1,20 @@ +package com.a.eye.skywalking.storage.util; + +import java.io.File; + +public class PathResolver { + + private final static String STORAGE_HOME; + + static { + STORAGE_HOME = System.getProperty("user.dir") + File.separator + ".."; + } + + public static String getAbsolutePath(String path) { + if (path.charAt(0) != '/') { + path = '/' + path; + } + return STORAGE_HOME + path; + } + +} diff --git a/skywalking-storage-center/skywalking-storage/src/main/resources/config.properties b/skywalking-storage-center/skywalking-storage/src/main/resources/config.properties index 586d7262e..12ef02c05 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/resources/config.properties +++ b/skywalking-storage-center/skywalking-storage/src/main/resources/config.properties @@ -1,35 +1,50 @@ -# +# the port which storage server listening server.port=34000 # +# the path that storage block index +#blockindex.path=/block-index # +# the name of file which storage block index +#blockindex.file_name=data_file.index # -blockindex.storage_base_path=/tmp/skywalking/block-index +# the path that storage data file +#datafile.path=/data/file # -blockindex.data_file_index_file_name=data_file.index +# the max size of data file (byte) +#datafile.size=3221225472 # +# the path that storage data index +#dataindex.path=/data/index # +# the name of the file which storage data index +#dataindex.file_name=dataIndex # -datafile.base_path=/tmp/skywalking/data/file +# the max size of data index +#dataindex.size=100000000 # -datafile.max_length=3221225472 +# the cached size of data index operator +#dataindex.operator.cache_size=10 # +# the auth info which registry center +#registrycenter.auth_info= # -#存放数据文件索引表名 -dataindex.table_name=data_index -#数据文件索引存储位置 -dataindex.base_path=/tmp/skywalking/data/index +# the auth schema which registry center +#registrycenter.auth_schema= # -dataindex.storage_index_file_name=dataIndex +# the registry center connect url (Default: zookeeper) +registrycenter.connect_url=127.0.0.1:2181 # -dataindex.max_capacity_per_index=1000000000 +# the prefix of path that each storage node register +#registrycenter.path_prefix=/skywalking/storage_list/ # +# the level1 size of block index engine +#blockindexengine.l1_cache_size=10 # +# the cached size of finder +#finder.cached_size=10 # -registrycenter.auth_info= -# -registrycenter.auth_schema= -# -registrycenter.connect_url= 127.0.0.1:2181 -# -registrycenter.registry_path_prefix= +# the max pool size of data source that finder operate +#finder.datasource.max_pool_size=20 # +# the min idle of data source that finder operate +#finder.datasource.min_idle=5 diff --git a/skywalking-storage-center/skywalking-storage/src/main/resources/log4j.properties b/skywalking-storage-center/skywalking-storage/src/main/resources/log4j.properties index 14cd87187..be0760950 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/resources/log4j.properties +++ b/skywalking-storage-center/skywalking-storage/src/main/resources/log4j.properties @@ -1,9 +1,8 @@ -# log4j properties # - # logger # -log4j.rootLogger=DEBUG,CONSOLE -log4j.logger.org=ON -#log4j.logger.org.systemgo.devframework=DEBUG +log4j.rootLogger=CONSOLE,Rolling_File +log4j.logger.org.apache=OFF +log4j.logger.io.netty=OFF +log4j.logger.com.a.eye.skywalking.network.dependencies.io.netty=OFF # Console Appender # log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender @@ -11,3 +10,12 @@ log4j.appender.CONSOLE.Target=System.out log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout log4j.appender.CONSOLE.layout.ConversionPattern=%d %-5p %c{1}:%L - %m%n +log4j.appender.Rolling_File=org.apache.log4j.RollingFileAppender +log4j.appender.Rolling_File.Threshold=WARN +log4j.appender.Rolling_File.File=logs/storage-server-log4j.log +log4j.appender.Rolling_File.Append=true +log4j.appender.Rolling_File.MaxFileSize=100MB +log4j.appender.Rolling_File.MaxBackupIndex=5 +log4j.appender.Rolling_File.layout=org.apache.log4j.PatternLayout +log4j.appender.Rolling_File.layout.ConversionPattern=%d %-5p %c{1}:%L - %m%n + diff --git a/skywalking-storage-center/skywalking-storage/src/main/resources/log4j2.xml b/skywalking-storage-center/skywalking-storage/src/main/resources/log4j2.xml index 91e20d37a..0d5cd5ce7 100644 --- a/skywalking-storage-center/skywalking-storage/src/main/resources/log4j2.xml +++ b/skywalking-storage-center/skywalking-storage/src/main/resources/log4j2.xml @@ -1,13 +1,23 @@ - - - - - - - - - - - - \ No newline at end of file + + + + + + + + %d %-5p %c{1}:%L - %m%n + + + + + + + + + + + +