lastData);
+}
diff --git a/apm-collector/apm-collector-remote/pom.xml b/apm-collector/apm-collector-remote/pom.xml
index 81c877d81..f20ebd870 100644
--- a/apm-collector/apm-collector-remote/pom.xml
+++ b/apm-collector/apm-collector-remote/pom.xml
@@ -29,6 +29,11 @@
apm-collector-server
${project.version}
+
+ org.skywalking
+ apm-collector-cluster
+ ${project.version}
+
io.grpc
grpc-netty
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleDefine.java b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleDefine.java
index ad6875615..87796d38c 100644
--- a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleDefine.java
+++ b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleDefine.java
@@ -1,9 +1,41 @@
package org.skywalking.apm.collector.remote;
+import java.util.List;
+import java.util.Map;
+import org.skywalking.apm.collector.cluster.ClusterModuleGroupDefine;
+import org.skywalking.apm.collector.core.client.ClientException;
+import org.skywalking.apm.collector.core.cluster.ClusterDataListenerDefine;
+import org.skywalking.apm.collector.core.cluster.ClusterModuleContext;
+import org.skywalking.apm.collector.core.config.ConfigParseException;
+import org.skywalking.apm.collector.core.framework.CollectorContextHelper;
+import org.skywalking.apm.collector.core.framework.DefineException;
+import org.skywalking.apm.collector.core.framework.Handler;
import org.skywalking.apm.collector.core.module.ModuleDefine;
+import org.skywalking.apm.collector.core.server.Server;
+import org.skywalking.apm.collector.core.server.ServerException;
+import org.skywalking.apm.collector.core.server.ServerHolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* @author pengys5
*/
-public abstract class RemoteModuleDefine extends ModuleDefine {
+public abstract class RemoteModuleDefine extends ModuleDefine implements ClusterDataListenerDefine {
+
+ private final Logger logger = LoggerFactory.getLogger(RemoteModuleDefine.class);
+
+ @Override
+ public final void initialize(Map config, ServerHolder serverHolder) throws DefineException, ClientException {
+ try {
+ configParser().parse(config);
+ Server server = server();
+ serverHolder.holdServer(server, handlerList());
+
+ ((ClusterModuleContext)CollectorContextHelper.INSTANCE.getContext(ClusterModuleGroupDefine.GROUP_NAME)).getDataMonitor().addListener(listener(), registration());
+ } catch (ConfigParseException | ServerException e) {
+ throw new RemoteModuleException(e.getMessage(), e);
+ }
+ }
+
+ public abstract List handlerList() throws DefineException;
}
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleException.java b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleException.java
new file mode 100644
index 000000000..87af6e986
--- /dev/null
+++ b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleException.java
@@ -0,0 +1,17 @@
+package org.skywalking.apm.collector.remote;
+
+import org.skywalking.apm.collector.core.module.ModuleException;
+
+/**
+ * @author pengys5
+ */
+public class RemoteModuleException extends ModuleException {
+
+ public RemoteModuleException(String message) {
+ super(message);
+ }
+
+ public RemoteModuleException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCConfig.java b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCConfig.java
new file mode 100644
index 000000000..6b465917a
--- /dev/null
+++ b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCConfig.java
@@ -0,0 +1,9 @@
+package org.skywalking.apm.collector.remote.grpc;
+
+/**
+ * @author pengys5
+ */
+public class RemoteGRPCConfig {
+ public static String HOST;
+ public static int PORT;
+}
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCConfigParser.java b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCConfigParser.java
new file mode 100644
index 000000000..73beb8a49
--- /dev/null
+++ b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCConfigParser.java
@@ -0,0 +1,27 @@
+package org.skywalking.apm.collector.remote.grpc;
+
+import java.util.Map;
+import org.skywalking.apm.collector.core.config.ConfigParseException;
+import org.skywalking.apm.collector.core.module.ModuleConfigParser;
+import org.skywalking.apm.collector.core.util.ObjectUtils;
+import org.skywalking.apm.collector.core.util.StringUtils;
+
+/**
+ * @author pengys5
+ */
+public class RemoteGRPCConfigParser implements ModuleConfigParser {
+
+ private static final String HOST = "host";
+ private static final String PORT = "port";
+
+ @Override public void parse(Map config) throws ConfigParseException {
+ if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(HOST))) {
+ RemoteGRPCConfig.HOST = "localhost";
+ }
+ if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(PORT))) {
+ RemoteGRPCConfig.PORT = 11800;
+ } else {
+ RemoteGRPCConfig.PORT = (Integer)config.get(PORT);
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCDataListener.java b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCDataListener.java
new file mode 100644
index 000000000..7d2e96c84
--- /dev/null
+++ b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCDataListener.java
@@ -0,0 +1,17 @@
+package org.skywalking.apm.collector.remote.grpc;
+
+import org.skywalking.apm.collector.cluster.ClusterModuleDefine;
+import org.skywalking.apm.collector.core.cluster.ClusterDataListener;
+import org.skywalking.apm.collector.remote.RemoteModuleGroupDefine;
+
+/**
+ * @author pengys5
+ */
+public class RemoteGRPCDataListener extends ClusterDataListener {
+
+ public static final String PATH = ClusterModuleDefine.BASE_CATALOG + "." + RemoteModuleGroupDefine.GROUP_NAME + "." + RemoteGRPCModuleDefine.MODULE_NAME;
+
+ @Override public String path() {
+ return PATH;
+ }
+}
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCModuleDefine.java b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCModuleDefine.java
index 877952cff..22b884e43 100644
--- a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCModuleDefine.java
+++ b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCModuleDefine.java
@@ -1,31 +1,42 @@
package org.skywalking.apm.collector.remote.grpc;
-import java.util.Map;
+import java.util.List;
import org.skywalking.apm.collector.core.client.Client;
-import org.skywalking.apm.collector.core.client.ClientException;
import org.skywalking.apm.collector.core.client.DataMonitor;
+import org.skywalking.apm.collector.core.cluster.ClusterDataListener;
+import org.skywalking.apm.collector.core.config.ConfigException;
import org.skywalking.apm.collector.core.framework.DefineException;
+import org.skywalking.apm.collector.core.framework.Handler;
import org.skywalking.apm.collector.core.module.ModuleConfigParser;
import org.skywalking.apm.collector.core.module.ModuleRegistration;
import org.skywalking.apm.collector.core.server.Server;
-import org.skywalking.apm.collector.core.server.ServerHolder;
import org.skywalking.apm.collector.remote.RemoteModuleDefine;
import org.skywalking.apm.collector.remote.RemoteModuleGroupDefine;
+import org.skywalking.apm.collector.remote.grpc.handler.RemoteHandlerDefineException;
+import org.skywalking.apm.collector.remote.grpc.handler.RemoteHandlerDefineLoader;
+import org.skywalking.apm.collector.server.grpc.GRPCServer;
/**
* @author pengys5
*/
public class RemoteGRPCModuleDefine extends RemoteModuleDefine {
+
+ public static final String MODULE_NAME = "remote";
+
+ @Override public String name() {
+ return MODULE_NAME;
+ }
+
@Override protected String group() {
return RemoteModuleGroupDefine.GROUP_NAME;
}
@Override public boolean defaultModule() {
- return false;
+ return true;
}
@Override protected ModuleConfigParser configParser() {
- return null;
+ return new RemoteGRPCConfigParser();
}
@Override protected Client createClient(DataMonitor dataMonitor) {
@@ -33,18 +44,25 @@ public class RemoteGRPCModuleDefine extends RemoteModuleDefine {
}
@Override protected Server server() {
- return null;
+ return new GRPCServer(RemoteGRPCConfig.HOST, RemoteGRPCConfig.PORT);
}
@Override protected ModuleRegistration registration() {
- return null;
+ return new RemoteGRPCModuleRegistration();
}
- @Override public void initialize(Map config, ServerHolder serverHolder) throws DefineException, ClientException {
-
+ @Override public ClusterDataListener listener() {
+ return new RemoteGRPCDataListener();
}
- @Override public String name() {
- return null;
+ @Override public List handlerList() throws DefineException {
+ RemoteHandlerDefineLoader loader = new RemoteHandlerDefineLoader();
+ List handlers = null;
+ try {
+ handlers = loader.load();
+ } catch (ConfigException e) {
+ throw new RemoteHandlerDefineException(e.getMessage(), e);
+ }
+ return handlers;
}
}
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCModuleRegistration.java b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCModuleRegistration.java
new file mode 100644
index 000000000..4f5a371d2
--- /dev/null
+++ b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCModuleRegistration.java
@@ -0,0 +1,13 @@
+package org.skywalking.apm.collector.remote.grpc;
+
+import org.skywalking.apm.collector.core.module.ModuleRegistration;
+
+/**
+ * @author pengys5
+ */
+public class RemoteGRPCModuleRegistration extends ModuleRegistration {
+
+ @Override public Value buildValue() {
+ return new Value(RemoteGRPCConfig.HOST, RemoteGRPCConfig.PORT, null);
+ }
+}
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefineException.java b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefineException.java
new file mode 100644
index 000000000..7a5c6063b
--- /dev/null
+++ b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefineException.java
@@ -0,0 +1,17 @@
+package org.skywalking.apm.collector.remote.grpc.handler;
+
+import org.skywalking.apm.collector.core.framework.DefineException;
+
+/**
+ * @author pengys5
+ */
+public class RemoteHandlerDefineException extends DefineException {
+
+ public RemoteHandlerDefineException(String message) {
+ super(message);
+ }
+
+ public RemoteHandlerDefineException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefineLoader.java b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefineLoader.java
new file mode 100644
index 000000000..b125bc090
--- /dev/null
+++ b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefineLoader.java
@@ -0,0 +1,30 @@
+package org.skywalking.apm.collector.remote.grpc.handler;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.skywalking.apm.collector.core.config.ConfigException;
+import org.skywalking.apm.collector.core.framework.Handler;
+import org.skywalking.apm.collector.core.framework.Loader;
+import org.skywalking.apm.collector.core.util.DefinitionLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class RemoteHandlerDefineLoader implements Loader> {
+
+ private final Logger logger = LoggerFactory.getLogger(RemoteHandlerDefineLoader.class);
+
+ @Override public List load() throws ConfigException {
+ List handlers = new ArrayList<>();
+
+ RemoteHandlerDefinitionFile definitionFile = new RemoteHandlerDefinitionFile();
+ DefinitionLoader definitionLoader = DefinitionLoader.load(Handler.class, definitionFile);
+ for (Handler handler : definitionLoader) {
+ logger.info("loaded remote handler definition class: {}", handler.getClass().getName());
+ handlers.add(handler);
+ }
+ return handlers;
+ }
+}
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefinitionFile.java b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefinitionFile.java
new file mode 100644
index 000000000..4231229c4
--- /dev/null
+++ b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefinitionFile.java
@@ -0,0 +1,13 @@
+package org.skywalking.apm.collector.remote.grpc.handler;
+
+import org.skywalking.apm.collector.core.framework.DefinitionFile;
+
+/**
+ * @author pengys5
+ */
+public class RemoteHandlerDefinitionFile extends DefinitionFile {
+
+ @Override protected String fileName() {
+ return "remote_handler.define";
+ }
+}
diff --git a/apm-collector/apm-collector-remote/src/main/proto/RemoteCommonService.proto b/apm-collector/apm-collector-remote/src/main/proto/RemoteCommonService.proto
index ae1da50f0..bf0f2d8bc 100644
--- a/apm-collector/apm-collector-remote/src/main/proto/RemoteCommonService.proto
+++ b/apm-collector/apm-collector-remote/src/main/proto/RemoteCommonService.proto
@@ -10,8 +10,8 @@ service RemoteCommonService {
message Message {
string workerRole = 1;
- int32 objectId = 2;
- bytes objectBytes = 3; // the byte array of data object
+ int32 dataDefineId = 2;
+ bytes dataBytes = 3;
}
message Empty {
diff --git a/apm-collector/apm-collector-stream/pom.xml b/apm-collector/apm-collector-stream/pom.xml
index e9be3a610..9ace78e53 100644
--- a/apm-collector/apm-collector-stream/pom.xml
+++ b/apm-collector/apm-collector-stream/pom.xml
@@ -23,5 +23,10 @@
apm-collector-queue
${project.version}
+
+ org.skywalking
+ apm-collector-remote
+ ${project.version}
+
\ No newline at end of file
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractClusterWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractRemoteWorker.java
similarity index 79%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractClusterWorker.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractRemoteWorker.java
index cc2991b4d..20c552bee 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractClusterWorker.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractRemoteWorker.java
@@ -1,7 +1,7 @@
package org.skywalking.apm.collector.stream;
/**
- * The AbstractClusterWorker implementations represent workers,
+ * The AbstractRemoteWorker implementations represent workers,
* which receive remote messages.
*
* Usually, the implementations are doing persistent, or aggregate works.
@@ -9,17 +9,17 @@ package org.skywalking.apm.collector.stream;
* @author pengys5
* @since v3.0-2017
*/
-public abstract class AbstractClusterWorker extends AbstractWorker {
+public abstract class AbstractRemoteWorker extends AbstractWorker {
/**
- * Construct an AbstractClusterWorker with the worker role and context.
+ * Construct an AbstractRemoteWorker with the worker role and context.
*
* @param role If multi-workers are for load balance, they should be more likely called worker instance. Meaning,
* each worker have multi instances.
* @param clusterContext See {@link ClusterWorkerContext}
* @param selfContext See {@link LocalWorkerContext}
*/
- protected AbstractClusterWorker(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
+ protected AbstractRemoteWorker(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
super(role, clusterContext, selfContext);
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractClusterWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractRemoteWorkerProvider.java
similarity index 66%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractClusterWorkerProvider.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractRemoteWorkerProvider.java
index 3e1fd1337..3774be6cb 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractClusterWorkerProvider.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractRemoteWorkerProvider.java
@@ -1,17 +1,17 @@
package org.skywalking.apm.collector.stream;
/**
- * The AbstractClusterWorkerProvider implementations represent providers,
- * which create instance of cluster workers whose implemented {@link AbstractClusterWorker}.
+ * The AbstractRemoteWorkerProvider implementations represent providers,
+ * which create instance of cluster workers whose implemented {@link AbstractRemoteWorker}.
*
*
* @author pengys5
* @since v3.0-2017
*/
-public abstract class AbstractClusterWorkerProvider extends AbstractWorkerProvider {
+public abstract class AbstractRemoteWorkerProvider extends AbstractWorkerProvider {
/**
- * Create how many worker instance of {@link AbstractClusterWorker} in one jvm.
+ * Create how many worker instance of {@link AbstractRemoteWorker} in one jvm.
*
* @return The worker instance number.
*/
@@ -21,7 +21,7 @@ public abstract class AbstractClusterWorkerProvider dataDefineMap;
+
private Map> roleWorkers;
public WorkerContext() {
@@ -20,8 +23,11 @@ public abstract class WorkerContext implements Context {
return this.roleWorkers;
}
- @Override
- final public WorkerRefs lookup(Role role) throws WorkerNotFoundException {
+ public final DataDefine getDataDefine(int defineId) {
+ return dataDefineMap.get(defineId);
+ }
+
+ @Override final public WorkerRefs lookup(Role role) throws WorkerNotFoundException {
if (getRoleWorkers().containsKey(role.roleName())) {
WorkerRefs refs = new WorkerRefs(getRoleWorkers().get(role.roleName()), role.workerSelector());
return refs;
@@ -30,16 +36,14 @@ public abstract class WorkerContext implements Context {
}
}
- @Override
- final public void put(WorkerRef workerRef) {
+ @Override final public void put(WorkerRef workerRef) {
if (!getRoleWorkers().containsKey(workerRef.getRole().roleName())) {
getRoleWorkers().putIfAbsent(workerRef.getRole().roleName(), new ArrayList());
}
getRoleWorkers().get(workerRef.getRole().roleName()).add(workerRef);
}
- @Override
- final public void remove(WorkerRef workerRef) {
+ @Override final public void remove(WorkerRef workerRef) {
getRoleWorkers().remove(workerRef.getRole().roleName());
}
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerModuleInstaller.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerModuleInstaller.java
deleted file mode 100644
index 5dd6095ca..000000000
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerModuleInstaller.java
+++ /dev/null
@@ -1,25 +0,0 @@
-package org.skywalking.apm.collector.stream;
-
-import java.util.Map;
-import org.skywalking.apm.collector.core.client.ClientException;
-import org.skywalking.apm.collector.core.framework.DefineException;
-import org.skywalking.apm.collector.core.module.ModuleDefine;
-import org.skywalking.apm.collector.core.module.ModuleInstaller;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author pengys5
- */
-public class WorkerModuleInstaller implements ModuleInstaller {
-
- private final Logger logger = LoggerFactory.getLogger(WorkerModuleInstaller.class);
-
- @Override public void install(Map moduleConfig,
- Map moduleDefineMap) throws DefineException, ClientException {
- logger.info("beginning worker module install");
- Map.Entry workerConfigEntry = moduleConfig.entrySet().iterator().next();
- ModuleDefine moduleDefine = moduleDefineMap.get(workerConfigEntry.getKey());
- moduleDefine.initialize(workerConfigEntry.getValue());
- }
-}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/AggregationWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/AggregationWorker.java
new file mode 100644
index 000000000..bd892035c
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/AggregationWorker.java
@@ -0,0 +1,55 @@
+package org.skywalking.apm.collector.stream.impl;
+
+import org.skywalking.apm.collector.core.queue.EndOfBatchCommand;
+import org.skywalking.apm.collector.stream.AbstractLocalAsyncWorker;
+import org.skywalking.apm.collector.stream.ClusterWorkerContext;
+import org.skywalking.apm.collector.stream.LocalWorkerContext;
+import org.skywalking.apm.collector.stream.ProviderNotFoundException;
+import org.skywalking.apm.collector.stream.Role;
+import org.skywalking.apm.collector.stream.WorkerException;
+import org.skywalking.apm.collector.stream.impl.data.Data;
+import org.skywalking.apm.collector.stream.impl.data.DataCache;
+
+/**
+ * @author pengys5
+ */
+public abstract class AggregationWorker extends AbstractLocalAsyncWorker {
+
+ private DataCache dataCache;
+
+ public AggregationWorker(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
+ super(role, clusterContext, selfContext);
+ dataCache = new DataCache();
+ }
+
+ private int messageNum;
+
+ @Override public void preStart() throws ProviderNotFoundException {
+ super.preStart();
+ }
+
+ @Override protected final void onWork(Object message) throws WorkerException {
+ if (message instanceof EndOfBatchCommand) {
+ sendToNext();
+ } else {
+ messageNum++;
+ aggregate(message);
+
+ if (messageNum >= 100) {
+ sendToNext();
+ messageNum = 0;
+ }
+ }
+ }
+
+ protected abstract void sendToNext();
+
+ protected final void aggregate(Object message) {
+ Data data = (Data)message;
+ if (dataCache.containsKey(data.id())) {
+ getClusterContext().getDataDefine(data.getDefineId()).mergeData(data, dataCache.get(data.id()));
+ } else {
+ dataCache.put(data.id(), data);
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/Const.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/Const.java
new file mode 100644
index 000000000..1850a2ee2
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/Const.java
@@ -0,0 +1,13 @@
+package org.skywalking.apm.collector.stream.impl;
+
+/**
+ * @author pengys5
+ */
+public class Const {
+ public static final String ID_SPLIT = "..-..";
+ public static final String IDS_SPLIT = "\\.\\.-\\.\\.";
+ public static final String PEERS_FRONT_SPLIT = "[";
+ public static final String PEERS_BEHIND_SPLIT = "]";
+ public static final String USER_CODE = "User";
+ public static final String RESULT = "result";
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/GRPCRemoteWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/GRPCRemoteWorker.java
new file mode 100644
index 000000000..3bd9a33fa
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/GRPCRemoteWorker.java
@@ -0,0 +1,26 @@
+package org.skywalking.apm.collector.stream.impl;
+
+import org.skywalking.apm.collector.stream.AbstractRemoteWorker;
+import org.skywalking.apm.collector.stream.ClusterWorkerContext;
+import org.skywalking.apm.collector.stream.LocalWorkerContext;
+import org.skywalking.apm.collector.stream.ProviderNotFoundException;
+import org.skywalking.apm.collector.stream.Role;
+import org.skywalking.apm.collector.stream.WorkerException;
+
+/**
+ * @author pengys5
+ */
+public class GRPCRemoteWorker extends AbstractRemoteWorker {
+
+ protected GRPCRemoteWorker(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
+ super(role, clusterContext, selfContext);
+ }
+
+ @Override public void preStart() throws ProviderNotFoundException {
+
+ }
+
+ @Override protected final void onWork(Object message) throws WorkerException {
+
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/RemoteCommonServiceHandler.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/RemoteCommonServiceHandler.java
new file mode 100644
index 000000000..ea560ef12
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/RemoteCommonServiceHandler.java
@@ -0,0 +1,24 @@
+package org.skywalking.apm.collector.stream.impl;
+
+import com.google.protobuf.ByteString;
+import io.grpc.stub.StreamObserver;
+import org.skywalking.apm.collector.remote.grpc.proto.Empty;
+import org.skywalking.apm.collector.remote.grpc.proto.Message;
+import org.skywalking.apm.collector.remote.grpc.proto.RemoteCommonServiceGrpc;
+import org.skywalking.apm.collector.server.grpc.GRPCHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class RemoteCommonServiceHandler extends RemoteCommonServiceGrpc.RemoteCommonServiceImplBase implements GRPCHandler {
+
+ private final Logger logger = LoggerFactory.getLogger(RemoteCommonServiceHandler.class);
+
+ @Override public void call(Message request, StreamObserver responseObserver) {
+ String workerRole = request.getWorkerRole();
+ int dataDefineId = request.getDataDefineId();
+ ByteString bytesData = request.getDataBytes();
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Attribute.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Attribute.java
new file mode 100644
index 000000000..3075517de
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Attribute.java
@@ -0,0 +1,28 @@
+package org.skywalking.apm.collector.stream.impl.data;
+
+/**
+ * @author pengys5
+ */
+public class Attribute {
+ private final String name;
+ private final AttributeType type;
+ private final Operation operation;
+
+ public Attribute(String name, AttributeType type, Operation operation) {
+ this.name = name;
+ this.type = type;
+ this.operation = operation;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public AttributeType getType() {
+ return type;
+ }
+
+ public Operation getOperation() {
+ return operation;
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/AttributeType.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/AttributeType.java
new file mode 100644
index 000000000..84e317ec0
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/AttributeType.java
@@ -0,0 +1,8 @@
+package org.skywalking.apm.collector.stream.impl.data;
+
+/**
+ * @author pengys5
+ */
+public enum AttributeType {
+ STRING, LONG, FLOAT
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Data.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Data.java
new file mode 100644
index 000000000..be8d4eb21
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Data.java
@@ -0,0 +1,50 @@
+package org.skywalking.apm.collector.stream.impl.data;
+
+/**
+ * @author pengys5
+ */
+public class Data {
+ private int defineId;
+ private String[] dataStrings;
+ private Long[] dataLongs;
+ private Float[] dataFloats;
+
+ public Data(int defineId, int stringCapacity, int longCapacity, int floatCapacity) {
+ this.defineId = defineId;
+ this.dataStrings = new String[stringCapacity];
+ this.dataLongs = new Long[longCapacity];
+ this.dataFloats = new Float[floatCapacity];
+ }
+
+ public void setDataString(int position, String value) {
+ dataStrings[position] = value;
+ }
+
+ public void setDataLong(int position, Long value) {
+ dataLongs[position] = value;
+ }
+
+ public void setDataFloat(int position, Float value) {
+ dataFloats[position] = value;
+ }
+
+ public String getDataString(int position) {
+ return dataStrings[position];
+ }
+
+ public Long getDataLong(int position) {
+ return dataLongs[position];
+ }
+
+ public Float getDataFloat(int position) {
+ return dataFloats[position];
+ }
+
+ public String id() {
+ return dataStrings[0];
+ }
+
+ public int getDefineId() {
+ return defineId;
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataCache.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataCache.java
new file mode 100644
index 000000000..db51cf91d
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataCache.java
@@ -0,0 +1,30 @@
+package org.skywalking.apm.collector.stream.impl.data;
+
+/**
+ * @author pengys5
+ */
+public class DataCache extends Window {
+
+ private DataCollection lockedDataCollection;
+
+ public boolean containsKey(String id) {
+ return lockedDataCollection.containsKey(id);
+ }
+
+ public Data get(String id) {
+ return lockedDataCollection.get(id);
+ }
+
+ public void put(String id, Data data) {
+ lockedDataCollection.put(id, data);
+ }
+
+ public void hold() {
+ lockedDataCollection = getCurrentAndHold();
+ }
+
+ public void release() {
+ lockedDataCollection.release();
+ lockedDataCollection = null;
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataCollection.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataCollection.java
new file mode 100644
index 000000000..725dcca71
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataCollection.java
@@ -0,0 +1,53 @@
+package org.skywalking.apm.collector.stream.impl.data;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author pengys5
+ */
+public class DataCollection {
+ private Map data;
+ private volatile boolean isHold;
+
+ public DataCollection() {
+ this.data = new HashMap<>();
+ this.isHold = false;
+ }
+
+ public void release() {
+ isHold = false;
+ }
+
+ public void hold() {
+ isHold = true;
+ }
+
+ public boolean isHolding() {
+ return isHold;
+ }
+
+ public boolean containsKey(String key) {
+ return data.containsKey(key);
+ }
+
+ public void put(String key, Data value) {
+ data.put(key, value);
+ }
+
+ public Data get(String key) {
+ return data.get(key);
+ }
+
+ public int size() {
+ return data.size();
+ }
+
+ public void clear() {
+ data.clear();
+ }
+
+ public Map asMap() {
+ return data;
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataDefine.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataDefine.java
new file mode 100644
index 000000000..5571ac264
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataDefine.java
@@ -0,0 +1,78 @@
+package org.skywalking.apm.collector.stream.impl.data;
+
+/**
+ * @author pengys5
+ */
+public abstract class DataDefine {
+ private Attribute[] attributes;
+ private int stringCapacity;
+ private int longCapacity;
+ private int floatCapacity;
+
+ public DataDefine() {
+ stringCapacity = 0;
+ longCapacity = 0;
+ floatCapacity = 0;
+ }
+
+ public final void initial() {
+ for (Attribute attribute : attributes) {
+ if (AttributeType.STRING.equals(attribute.getType())) {
+ stringCapacity++;
+ } else if (AttributeType.LONG.equals(attribute.getType())) {
+ longCapacity++;
+ } else if (AttributeType.FLOAT.equals(attribute.getType())) {
+ floatCapacity++;
+ }
+ }
+ }
+
+ public final void addAttribute(int position, Attribute attribute) {
+ attributes[position] = attribute;
+ }
+
+ public final void define() {
+ attributes = new Attribute[initialCapacity()];
+ }
+
+ protected abstract int defineId();
+
+ protected abstract int initialCapacity();
+
+ protected abstract void attributeDefine();
+
+ public int getStringCapacity() {
+ return stringCapacity;
+ }
+
+ public int getLongCapacity() {
+ return longCapacity;
+ }
+
+ public int getFloatCapacity() {
+ return floatCapacity;
+ }
+
+ public Data build() {
+ return new Data(defineId(), getStringCapacity(), getLongCapacity(), getFloatCapacity());
+ }
+
+ public void mergeData(Data newData, Data oldData) {
+ int stringPosition = 0;
+ int longPosition = 0;
+ int floatPosition = 0;
+ for (int i = 0; i < initialCapacity(); i++) {
+ Attribute attribute = attributes[i];
+ if (AttributeType.STRING.equals(attribute.getType())) {
+ attribute.getOperation().operate(newData.getDataString(stringPosition), oldData.getDataString(stringPosition));
+ stringPosition++;
+ } else if (AttributeType.LONG.equals(attribute.getType())) {
+ attribute.getOperation().operate(newData.getDataLong(longPosition), oldData.getDataLong(longPosition));
+ longPosition++;
+ } else if (AttributeType.FLOAT.equals(attribute.getType())) {
+ attribute.getOperation().operate(newData.getDataFloat(floatPosition), oldData.getDataFloat(floatPosition));
+ floatPosition++;
+ }
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataDefineLoader.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataDefineLoader.java
new file mode 100644
index 000000000..8ce454440
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataDefineLoader.java
@@ -0,0 +1,29 @@
+package org.skywalking.apm.collector.stream.impl.data;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.skywalking.apm.collector.core.config.ConfigException;
+import org.skywalking.apm.collector.core.framework.Loader;
+import org.skywalking.apm.collector.core.util.DefinitionLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class DataDefineLoader implements Loader> {
+
+ private final Logger logger = LoggerFactory.getLogger(DataDefineLoader.class);
+
+ @Override public Map load() throws ConfigException {
+ Map dataDefineMap = new HashMap<>();
+
+ DataDefinitionFile definitionFile = new DataDefinitionFile();
+ DefinitionLoader definitionLoader = DefinitionLoader.load(DataDefine.class, definitionFile);
+ for (DataDefine dataDefine : definitionLoader) {
+ logger.info("loaded data definition class: {}", dataDefine.getClass().getName());
+ dataDefineMap.put(dataDefine.defineId(), dataDefine);
+ }
+ return dataDefineMap;
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataDefinitionFile.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataDefinitionFile.java
new file mode 100644
index 000000000..93df98019
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataDefinitionFile.java
@@ -0,0 +1,13 @@
+package org.skywalking.apm.collector.stream.impl.data;
+
+import org.skywalking.apm.collector.core.framework.DefinitionFile;
+
+/**
+ * @author pengys5
+ */
+public class DataDefinitionFile extends DefinitionFile {
+
+ @Override protected String fileName() {
+ return "data.define";
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Operation.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Operation.java
new file mode 100644
index 000000000..b94e16b11
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Operation.java
@@ -0,0 +1,12 @@
+package org.skywalking.apm.collector.stream.impl.data;
+
+/**
+ * @author pengys5
+ */
+public interface Operation {
+ String operate(String newValue, String oldValue);
+
+ Long operate(Long newValue, Long oldValue);
+
+ Float operate(Float newValue, Float oldValue);
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Window.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Window.java
new file mode 100644
index 000000000..05b48c197
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Window.java
@@ -0,0 +1,44 @@
+package org.skywalking.apm.collector.stream.impl.data;
+
+/**
+ * @author pengys5
+ */
+public abstract class Window {
+
+ private DataCollection pointer;
+
+ private DataCollection windowDataA;
+ private DataCollection windowDataB;
+
+ public Window() {
+ windowDataA = new DataCollection();
+ windowDataB = new DataCollection();
+ pointer = windowDataA;
+ }
+
+ public void switchPointer() {
+ if (pointer == windowDataA) {
+ pointer = windowDataB;
+ } else {
+ pointer = windowDataA;
+ }
+ }
+
+ protected DataCollection getCurrentAndHold() {
+ if (pointer == windowDataA) {
+ windowDataA.hold();
+ return windowDataA;
+ } else {
+ windowDataB.hold();
+ return windowDataB;
+ }
+ }
+
+ public DataCollection getLast() {
+ if (pointer == windowDataA) {
+ return windowDataB;
+ } else {
+ return windowDataA;
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/operate/CoverOperation.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/operate/CoverOperation.java
new file mode 100644
index 000000000..bae2f6308
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/operate/CoverOperation.java
@@ -0,0 +1,20 @@
+package org.skywalking.apm.collector.stream.impl.data.operate;
+
+import org.skywalking.apm.collector.stream.impl.data.Operation;
+
+/**
+ * @author pengys5
+ */
+public class CoverOperation implements Operation {
+ @Override public String operate(String newValue, String oldValue) {
+ return newValue;
+ }
+
+ @Override public Long operate(Long newValue, Long oldValue) {
+ return newValue;
+ }
+
+ @Override public Float operate(Float newValue, Float oldValue) {
+ return newValue;
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/operate/NonOperation.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/operate/NonOperation.java
new file mode 100644
index 000000000..b33950073
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/operate/NonOperation.java
@@ -0,0 +1,20 @@
+package org.skywalking.apm.collector.stream.impl.data.operate;
+
+import org.skywalking.apm.collector.stream.impl.data.Operation;
+
+/**
+ * @author pengys5
+ */
+public class NonOperation implements Operation {
+ @Override public String operate(String newValue, String oldValue) {
+ return oldValue;
+ }
+
+ @Override public Long operate(Long newValue, Long oldValue) {
+ return oldValue;
+ }
+
+ @Override public Float operate(Float newValue, Float oldValue) {
+ return oldValue;
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/remote_handler.define b/apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/remote_handler.define
new file mode 100644
index 000000000..389dfb80e
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/remote_handler.define
@@ -0,0 +1 @@
+org.skywalking.apm.collector.stream.impl.RemoteCommonServiceHandler
\ No newline at end of file
From b53a3c5b536524aa2c095148ccd747bd0b42c0e5 Mon Sep 17 00:00:00 2001
From: pengys5 <8082209@qq.com>
Date: Mon, 24 Jul 2017 11:42:39 +0800
Subject: [PATCH 09/28] 1. remove remote module 2. make stream module as a
collector define module, contains grpc remote service for call remote workers
3. worker framework is ok 4. Collector start ok
---
.../apm-collector-agentstream/pom.xml | 45 +++++++
.../agentstream/AgentStreamModuleContext.java | 1 +
.../AgentStreamModuleDefineException.java | 17 +++
.../NodeComponentAggDayWorker.java | 25 ----
.../component/NodeComponentAggWorker.java | 57 +++++++++
.../component/NodeComponentDataDefine.java | 42 +++++++
.../component/NodeComponentRemoteWorker.java | 25 ++++
.../NodeComponentTable.java | 2 +-
.../node/define/NodeComponentDataDefine.java | 28 -----
.../define/NodeComponentEsTableDefine.java | 1 +
.../define/NodeComponentH2TableDefine.java | 1 +
.../src/main/proto/NodeComponent.proto | 2 +-
.../resources/META-INF/defines/data.define | 2 +-
.../local_async_worker_provider.define | 1 +
.../defines/remote_worker_provider.define | 0
.../apm/collector/boot/CollectorStarter.java | 4 -
.../src/main/resources/application.yml | 3 -
apm-collector/apm-collector-client/pom.xml | 4 -
.../core/module/SingleModuleInstaller.java | 6 +-
.../core/remote/SerializedDefine.java | 10 --
.../core/remote/SerializedDefineLoader.java | 33 -----
.../core/remote/SerializedDefinitionFile.java | 12 --
.../collector/queue/QueueModuleInstaller.java | 8 +-
.../QueueDataCarrierModuleDefine.java | 2 +-
apm-collector/apm-collector-remote/pom.xml | 118 ------------------
.../collector/remote/RemoteModuleContext.java | 12 --
.../remote/RemoteModuleException.java | 17 ---
.../remote/RemoteModuleInstaller.java | 18 ---
.../grpc/RemoteGRPCModuleRegistration.java | 13 --
.../handler/RemoteHandlerDefineException.java | 17 ---
.../handler/RemoteHandlerDefineLoader.java | 30 -----
.../handler/RemoteHandlerDefinitionFile.java | 13 --
.../resources/META-INF/defines/group.define | 1 -
.../resources/META-INF/defines/module.define | 1 -
apm-collector/apm-collector-stream/pom.xml | 52 +++++++-
.../AbstractLocalSyncWorkerProvider.java | 21 ----
.../stream/AbstractWorkerProvider.java | 36 ------
.../collector/stream/LocalWorkerContext.java | 17 ---
.../apm/collector/stream/Provider.java | 9 --
.../skywalking/apm/collector/stream/Role.java | 13 --
.../collector/stream/StreamModuleContext.java | 37 ++++++
.../collector/stream/StreamModuleDefine.java} | 12 +-
.../stream/StreamModuleException.java | 17 +++
.../stream/StreamModuleGroupDefine.java} | 10 +-
.../stream/StreamModuleInstaller.java | 81 ++++++++++++
.../stream/grpc/StreamGRPCConfig.java} | 4 +-
.../stream/grpc/StreamGRPCConfigParser.java} | 10 +-
.../stream/grpc/StreamGRPCDataListener.java} | 8 +-
.../stream/grpc/StreamGRPCModuleDefine.java} | 38 +++---
.../grpc/StreamGRPCModuleRegistration.java | 13 ++
.../handler/RemoteCommonServiceHandler.java | 42 +++++++
.../stream/impl/GRPCRemoteWorker.java | 26 ----
.../impl/RemoteCommonServiceHandler.java | 24 ----
.../AbstractLocalAsyncWorker.java | 11 +-
.../AbstractLocalAsyncWorkerProvider.java | 10 +-
.../{ => worker}/AbstractLocalSyncWorker.java | 6 +-
.../AbstractLocalSyncWorkerProvider.java | 15 +++
.../{ => worker}/AbstractLocalWorker.java | 6 +-
.../AbstractLocalWorkerProvider.java | 2 +-
.../{ => worker}/AbstractRemoteWorker.java | 7 +-
.../AbstractRemoteWorkerProvider.java | 6 +-
.../stream/{ => worker}/AbstractWorker.java | 11 +-
.../stream/worker/AbstractWorkerProvider.java | 21 ++++
.../{ => worker}/ClusterWorkerContext.java | 2 +-
.../{ => worker}/ClusterWorkerRefCounter.java | 2 +-
.../stream/{ => worker}/Context.java | 2 +-
.../LocalAsyncWorkerProviderDefineLoader.java | 32 +++++
...ocalAsyncWorkerProviderDefinitionFile.java | 12 ++
.../{ => worker}/LocalAsyncWorkerRef.java | 2 +-
.../{ => worker}/LocalSyncWorkerRef.java | 2 +-
.../collector/stream/{ => worker}/LookUp.java | 2 +-
.../apm/collector/stream/worker/Provider.java | 9 ++
.../ProviderNotFoundException.java | 2 +-
.../RemoteWorkerProviderDefineLoader.java | 31 +++++
.../RemoteWorkerProviderDefinitionFile.java | 12 ++
.../stream/{ => worker}/RemoteWorkerRef.java | 2 +-
.../apm/collector/stream/worker/Role.java | 13 ++
.../{ => worker}/UsedRoleNameException.java | 2 +-
.../stream/{ => worker}/WorkerContext.java | 29 +++--
.../stream/{ => worker}/WorkerException.java | 2 +-
.../{ => worker}/WorkerInvokeException.java | 2 +-
.../{ => worker}/WorkerNotFoundException.java | 2 +-
.../stream/{ => worker}/WorkerRef.java | 2 +-
.../stream/{ => worker}/WorkerRefs.java | 4 +-
.../{ => worker}/impl/AggregationWorker.java | 21 ++--
.../stream/{ => worker}/impl/Const.java | 2 +-
.../stream/worker/impl/GRPCRemoteWorker.java | 25 ++++
.../{ => worker}/impl/data/Attribute.java | 2 +-
.../{ => worker}/impl/data/AttributeType.java | 2 +-
.../stream/{ => worker}/impl/data/Data.java | 2 +-
.../{ => worker}/impl/data/DataCache.java | 2 +-
.../impl/data/DataCollection.java | 2 +-
.../{ => worker}/impl/data/DataDefine.java | 7 +-
.../impl/data/DataDefineLoader.java | 2 +-
.../impl/data/DataDefinitionFile.java | 2 +-
.../{ => worker}/impl/data/Operation.java | 2 +-
.../stream/{ => worker}/impl/data/Window.java | 2 +-
.../impl/data/operate/CoverOperation.java | 4 +-
.../impl/data/operate/NonOperation.java | 4 +-
.../selector/AbstractHashMessage.java | 2 +-
.../selector/HashCodeSelector.java | 7 +-
.../selector/RollingSelector.java | 7 +-
.../{ => worker}/selector/WorkerSelector.java | 7 +-
.../src/main/proto/RemoteCommonService.proto | 0
.../resources/META-INF/defines/group.define | 1 +
.../resources/META-INF/defines/module.define | 1 +
.../META-INF/defines/remote_handler.define | 1 -
apm-collector/pom.xml | 1 -
108 files changed, 753 insertions(+), 659 deletions(-)
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/AgentStreamModuleDefineException.java
delete mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/aggregation/NodeComponentAggDayWorker.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggWorker.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentDataDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentRemoteWorker.java
rename apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/{define => component}/NodeComponentTable.java (81%)
delete mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentDataDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define
create mode 100644 apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define
delete mode 100644 apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefine.java
delete mode 100644 apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefineLoader.java
delete mode 100644 apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefinitionFile.java
delete mode 100644 apm-collector/apm-collector-remote/pom.xml
delete mode 100644 apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleContext.java
delete mode 100644 apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleException.java
delete mode 100644 apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleInstaller.java
delete mode 100644 apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCModuleRegistration.java
delete mode 100644 apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefineException.java
delete mode 100644 apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefineLoader.java
delete mode 100644 apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefinitionFile.java
delete mode 100644 apm-collector/apm-collector-remote/src/main/resources/META-INF/defines/group.define
delete mode 100644 apm-collector/apm-collector-remote/src/main/resources/META-INF/defines/module.define
delete mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalSyncWorkerProvider.java
delete mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractWorkerProvider.java
delete mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/LocalWorkerContext.java
delete mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/Provider.java
delete mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/Role.java
create mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleContext.java
rename apm-collector/{apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleDefine.java => apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleDefine.java} (83%)
create mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleException.java
rename apm-collector/{apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleGroupDefine.java => apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleGroupDefine.java} (61%)
create mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java
rename apm-collector/{apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCConfig.java => apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCConfig.java} (52%)
rename apm-collector/{apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCConfigParser.java => apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCConfigParser.java} (72%)
rename apm-collector/{apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCDataListener.java => apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCDataListener.java} (57%)
rename apm-collector/{apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCModuleDefine.java => apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCModuleDefine.java} (51%)
create mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCModuleRegistration.java
create mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/handler/RemoteCommonServiceHandler.java
delete mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/GRPCRemoteWorker.java
delete mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/RemoteCommonServiceHandler.java
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/AbstractLocalAsyncWorker.java (88%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/AbstractLocalAsyncWorkerProvider.java (80%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/AbstractLocalSyncWorker.java (90%)
create mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalSyncWorkerProvider.java
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/AbstractLocalWorker.java (53%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/AbstractLocalWorkerProvider.java (73%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/AbstractRemoteWorker.java (87%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/AbstractRemoteWorkerProvider.java (84%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/AbstractWorker.java (71%)
create mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractWorkerProvider.java
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/ClusterWorkerContext.java (96%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/ClusterWorkerRefCounter.java (91%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/Context.java (85%)
create mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefineLoader.java
create mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefinitionFile.java
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/LocalAsyncWorkerRef.java (90%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/LocalSyncWorkerRef.java (91%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/LookUp.java (78%)
create mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Provider.java
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/ProviderNotFoundException.java (73%)
create mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerProviderDefineLoader.java
create mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerProviderDefinitionFile.java
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/RemoteWorkerRef.java (88%)
create mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Role.java
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/UsedRoleNameException.java (72%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/WorkerContext.java (75%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/WorkerException.java (87%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/WorkerInvokeException.java (88%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/WorkerNotFoundException.java (74%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/WorkerRef.java (84%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/WorkerRefs.java (89%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/impl/AggregationWorker.java (64%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/impl/Const.java (86%)
create mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/GRPCRemoteWorker.java
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/impl/data/Attribute.java (89%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/impl/data/AttributeType.java (56%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/impl/data/Data.java (94%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/impl/data/DataCache.java (90%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/impl/data/DataCollection.java (93%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/impl/data/DataDefine.java (90%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/impl/data/DataDefineLoader.java (94%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/impl/data/DataDefinitionFile.java (79%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/impl/data/Operation.java (77%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/impl/data/Window.java (93%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/impl/data/operate/CoverOperation.java (73%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/impl/data/operate/NonOperation.java (73%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/selector/AbstractHashMessage.java (88%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/selector/HashCodeSelector.java (82%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/selector/RollingSelector.java (74%)
rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/{ => worker}/selector/WorkerSelector.java (72%)
rename apm-collector/{apm-collector-remote => apm-collector-stream}/src/main/proto/RemoteCommonService.proto (100%)
create mode 100644 apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/group.define
create mode 100644 apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/module.define
delete mode 100644 apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/remote_handler.define
diff --git a/apm-collector/apm-collector-agentstream/pom.xml b/apm-collector/apm-collector-agentstream/pom.xml
index 7cf497179..e76c1b793 100644
--- a/apm-collector/apm-collector-agentstream/pom.xml
+++ b/apm-collector/apm-collector-agentstream/pom.xml
@@ -34,4 +34,49 @@
${project.version}
+
+
+
+
+ kr.motd.maven
+ os-maven-plugin
+ 1.4.1.Final
+
+
+
+
+ org.apache.maven.plugins
+ maven-resources-plugin
+ 2.4.3
+
+ ${project.build.sourceEncoding}
+
+
+
+ org.xolstice.maven.plugins
+ protobuf-maven-plugin
+ 0.5.0
+
+
+ com.google.protobuf:protoc:3.3.0:exe:${os.detected.classifier}
+
+ grpc-java
+ io.grpc:protoc-gen-grpc-java:1.4.0:exe:${os.detected.classifier}
+
+
+
+
+
+ compile
+ compile-custom
+
+
+
+
+
+
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleContext.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleContext.java
index 8d6219c04..f80a15ba5 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleContext.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleContext.java
@@ -6,6 +6,7 @@ import org.skywalking.apm.collector.core.framework.Context;
* @author pengys5
*/
public class AgentStreamModuleContext extends Context {
+
public AgentStreamModuleContext(String groupName) {
super(groupName);
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/AgentStreamModuleDefineException.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/AgentStreamModuleDefineException.java
new file mode 100644
index 000000000..c57b297fb
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/AgentStreamModuleDefineException.java
@@ -0,0 +1,17 @@
+package org.skywalking.apm.collector.agentstream.worker;
+
+import org.skywalking.apm.collector.core.framework.DefineException;
+
+/**
+ * @author pengys5
+ */
+public class AgentStreamModuleDefineException extends DefineException {
+
+ public AgentStreamModuleDefineException(String message) {
+ super(message);
+ }
+
+ public AgentStreamModuleDefineException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/aggregation/NodeComponentAggDayWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/aggregation/NodeComponentAggDayWorker.java
deleted file mode 100644
index f73e122c1..000000000
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/aggregation/NodeComponentAggDayWorker.java
+++ /dev/null
@@ -1,25 +0,0 @@
-package org.skywalking.apm.collector.agentstream.worker.node.aggregation;
-
-import org.skywalking.apm.collector.stream.ClusterWorkerContext;
-import org.skywalking.apm.collector.stream.LocalWorkerContext;
-import org.skywalking.apm.collector.stream.ProviderNotFoundException;
-import org.skywalking.apm.collector.stream.Role;
-import org.skywalking.apm.collector.stream.impl.AggregationWorker;
-
-/**
- * @author pengys5
- */
-public class NodeComponentAggDayWorker extends AggregationWorker {
-
- public NodeComponentAggDayWorker(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
- super(role, clusterContext, selfContext);
- }
-
- @Override public void preStart() throws ProviderNotFoundException {
- super.preStart();
- }
-
- @Override protected void sendToNext() {
-
- }
-}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggWorker.java
new file mode 100644
index 000000000..89fc75805
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggWorker.java
@@ -0,0 +1,57 @@
+package org.skywalking.apm.collector.agentstream.worker.node.component;
+
+import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider;
+import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
+import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
+import org.skywalking.apm.collector.stream.worker.impl.AggregationWorker;
+import org.skywalking.apm.collector.stream.worker.selector.RollingSelector;
+import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector;
+
+/**
+ * @author pengys5
+ */
+public class NodeComponentAggWorker extends AggregationWorker {
+
+ public NodeComponentAggWorker(Role role, ClusterWorkerContext clusterContext) {
+ super(role, clusterContext);
+ }
+
+ @Override public void preStart() throws ProviderNotFoundException {
+ super.preStart();
+ }
+
+ @Override protected void sendToNext() {
+
+ }
+
+ public static class Factory extends AbstractLocalAsyncWorkerProvider {
+ @Override
+ public Role role() {
+ return Role.INSTANCE;
+ }
+
+ @Override
+ public NodeComponentAggWorker workerInstance(ClusterWorkerContext clusterContext) {
+ return new NodeComponentAggWorker(role(), clusterContext);
+ }
+
+ @Override
+ public int queueSize() {
+ return 1024;
+ }
+ }
+
+ public enum Role implements org.skywalking.apm.collector.stream.worker.Role {
+ INSTANCE;
+
+ @Override
+ public String roleName() {
+ return NodeComponentAggWorker.class.getSimpleName();
+ }
+
+ @Override
+ public WorkerSelector workerSelector() {
+ return new RollingSelector();
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentDataDefine.java
new file mode 100644
index 000000000..34336b0d6
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentDataDefine.java
@@ -0,0 +1,42 @@
+package org.skywalking.apm.collector.agentstream.worker.node.component;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.InvalidProtocolBufferException;
+import org.skywalking.apm.collector.agentstream.worker.node.define.proto.NodeComponent;
+import org.skywalking.apm.collector.stream.worker.impl.data.Attribute;
+import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType;
+import org.skywalking.apm.collector.stream.worker.impl.data.Data;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
+import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation;
+import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation;
+
+/**
+ * @author pengys5
+ */
+public class NodeComponentDataDefine extends DataDefine {
+
+ @Override protected int defineId() {
+ return 0;
+ }
+
+ @Override protected int initialCapacity() {
+ return 4;
+ }
+
+ @Override protected void attributeDefine() {
+ addAttribute(0, new Attribute("id", AttributeType.STRING, new NonOperation()));
+ addAttribute(1, new Attribute("name", AttributeType.STRING, new CoverOperation()));
+ addAttribute(2, new Attribute("peers", AttributeType.STRING, new CoverOperation()));
+ addAttribute(3, new Attribute("aggregation", AttributeType.STRING, new CoverOperation()));
+ }
+
+ @Override public Data parseFrom(ByteString bytesData) throws InvalidProtocolBufferException {
+ NodeComponent.Message message = NodeComponent.Message.parseFrom(bytesData);
+ Data data = build();
+ data.setDataString(0, message.getId());
+ data.setDataString(1, message.getName());
+ data.setDataString(2, message.getPeers());
+ data.setDataString(3, message.getAggregation());
+ return data;
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentRemoteWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentRemoteWorker.java
new file mode 100644
index 000000000..41b9096ce
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentRemoteWorker.java
@@ -0,0 +1,25 @@
+package org.skywalking.apm.collector.agentstream.worker.node.component;
+
+import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorker;
+import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
+import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
+import org.skywalking.apm.collector.stream.worker.Role;
+import org.skywalking.apm.collector.stream.worker.WorkerException;
+
+/**
+ * @author pengys5
+ */
+public class NodeComponentRemoteWorker extends AbstractRemoteWorker {
+
+ protected NodeComponentRemoteWorker(Role role, ClusterWorkerContext clusterContext) {
+ super(role, clusterContext);
+ }
+
+ @Override public void preStart() throws ProviderNotFoundException {
+
+ }
+
+ @Override protected void onWork(Object message) throws WorkerException {
+
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentTable.java
similarity index 81%
rename from apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentTable.java
rename to apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentTable.java
index 9d479cd10..211c8b5e6 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentTable.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentTable.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.agentstream.worker.node.define;
+package org.skywalking.apm.collector.agentstream.worker.node.component;
import org.skywalking.apm.collector.agentstream.worker.CommonTable;
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentDataDefine.java
deleted file mode 100644
index 4bc3e36ce..000000000
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentDataDefine.java
+++ /dev/null
@@ -1,28 +0,0 @@
-package org.skywalking.apm.collector.agentstream.worker.node.define;
-
-import org.skywalking.apm.collector.stream.impl.data.Attribute;
-import org.skywalking.apm.collector.stream.impl.data.AttributeType;
-import org.skywalking.apm.collector.stream.impl.data.DataDefine;
-import org.skywalking.apm.collector.stream.impl.data.operate.CoverOperation;
-import org.skywalking.apm.collector.stream.impl.data.operate.NonOperation;
-
-/**
- * @author pengys5
- */
-public class NodeComponentDataDefine extends DataDefine {
-
- @Override protected int defineId() {
- return 0;
- }
-
- @Override protected int initialCapacity() {
- return 4;
- }
-
- @Override protected void attributeDefine() {
- addAttribute(0, new Attribute("id", AttributeType.STRING, new NonOperation()));
- addAttribute(1, new Attribute("name", AttributeType.STRING, new CoverOperation()));
- addAttribute(2, new Attribute("peers", AttributeType.STRING, new CoverOperation()));
- addAttribute(3, new Attribute("aggregation", AttributeType.STRING, new CoverOperation()));
- }
-}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentEsTableDefine.java
index a2b4fe89e..01f76b785 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentEsTableDefine.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentEsTableDefine.java
@@ -1,5 +1,6 @@
package org.skywalking.apm.collector.agentstream.worker.node.define;
+import org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentTable;
import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine;
import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine;
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentH2TableDefine.java
index 8b6ab6c67..550890ba2 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentH2TableDefine.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentH2TableDefine.java
@@ -1,5 +1,6 @@
package org.skywalking.apm.collector.agentstream.worker.node.define;
+import org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentTable;
import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine;
import org.skywalking.apm.collector.storage.h2.define.H2TableDefine;
diff --git a/apm-collector/apm-collector-agentstream/src/main/proto/NodeComponent.proto b/apm-collector/apm-collector-agentstream/src/main/proto/NodeComponent.proto
index 4ad963754..df981e85d 100644
--- a/apm-collector/apm-collector-agentstream/src/main/proto/NodeComponent.proto
+++ b/apm-collector/apm-collector-agentstream/src/main/proto/NodeComponent.proto
@@ -1,6 +1,6 @@
syntax = "proto3";
-option java_multiple_files = true;
+option java_multiple_files = false;
option java_package = "org.skywalking.apm.collector.agentstream.worker.node.define.proto";
message Message {
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define
index a25812b85..d87c1fdd9 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define
@@ -1 +1 @@
-org.skywalking.apm.collector.agentstream.worker.node.define.NodeComponentDataDefine
\ No newline at end of file
+org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentDataDefine
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define
new file mode 100644
index 000000000..33897fbdc
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define
@@ -0,0 +1 @@
+org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentAggWorker$Factory
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define
new file mode 100644
index 000000000..e69de29bb
diff --git a/apm-collector/apm-collector-boot/src/main/java/org/skywalking/apm/collector/boot/CollectorStarter.java b/apm-collector/apm-collector-boot/src/main/java/org/skywalking/apm/collector/boot/CollectorStarter.java
index de22a4cf1..74d041c27 100644
--- a/apm-collector/apm-collector-boot/src/main/java/org/skywalking/apm/collector/boot/CollectorStarter.java
+++ b/apm-collector/apm-collector-boot/src/main/java/org/skywalking/apm/collector/boot/CollectorStarter.java
@@ -11,7 +11,6 @@ import org.skywalking.apm.collector.core.module.ModuleDefine;
import org.skywalking.apm.collector.core.module.ModuleDefineLoader;
import org.skywalking.apm.collector.core.module.ModuleGroupDefine;
import org.skywalking.apm.collector.core.module.ModuleGroupDefineLoader;
-import org.skywalking.apm.collector.core.remote.SerializedDefineLoader;
import org.skywalking.apm.collector.core.server.ServerException;
import org.skywalking.apm.collector.core.server.ServerHolder;
import org.slf4j.Logger;
@@ -28,9 +27,6 @@ public class CollectorStarter implements Starter {
ModuleConfigLoader configLoader = new ModuleConfigLoader();
Map configuration = configLoader.load();
- SerializedDefineLoader serializedDefineLoader = new SerializedDefineLoader();
- serializedDefineLoader.load();
-
ModuleGroupDefineLoader groupDefineLoader = new ModuleGroupDefineLoader();
Map moduleGroupDefineMap = groupDefineLoader.load();
diff --git a/apm-collector/apm-collector-boot/src/main/resources/application.yml b/apm-collector/apm-collector-boot/src/main/resources/application.yml
index c5d7f5897..5e1a6de33 100644
--- a/apm-collector/apm-collector-boot/src/main/resources/application.yml
+++ b/apm-collector/apm-collector-boot/src/main/resources/application.yml
@@ -5,9 +5,6 @@ cluster:
# redis:
# host: localhost
# port: 6379
-queue:
- disruptor: on
- data_carrier: off
agentstream:
grpc:
host: localhost
diff --git a/apm-collector/apm-collector-client/pom.xml b/apm-collector/apm-collector-client/pom.xml
index 01bd8c710..31f9fdad0 100644
--- a/apm-collector/apm-collector-client/pom.xml
+++ b/apm-collector/apm-collector-client/pom.xml
@@ -37,10 +37,6 @@
snakeyaml
org.yaml
-
- log4j-api
- org.apache.logging.log4j
-
diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/SingleModuleInstaller.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/SingleModuleInstaller.java
index 4d28b2f55..247282402 100644
--- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/SingleModuleInstaller.java
+++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/SingleModuleInstaller.java
@@ -31,9 +31,9 @@ public abstract class SingleModuleInstaller implements ModuleInstaller {
}
}
} else {
- Map.Entry clusterConfigEntry = moduleConfig.entrySet().iterator().next();
- moduleDefine = moduleDefineMap.get(clusterConfigEntry.getKey());
- moduleDefine.initialize(clusterConfigEntry.getValue(), serverHolder);
+ Map.Entry configEntry = moduleConfig.entrySet().iterator().next();
+ moduleDefine = moduleDefineMap.get(configEntry.getKey());
+ moduleDefine.initialize(configEntry.getValue(), serverHolder);
}
}
}
diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefine.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefine.java
deleted file mode 100644
index bafafd87e..000000000
--- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefine.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package org.skywalking.apm.collector.core.remote;
-
-/**
- * @author pengys5
- */
-public interface SerializedDefine {
- int ID();
-
- Class clazz();
-}
diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefineLoader.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefineLoader.java
deleted file mode 100644
index c07386260..000000000
--- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefineLoader.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package org.skywalking.apm.collector.core.remote;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-import org.skywalking.apm.collector.core.config.ConfigException;
-import org.skywalking.apm.collector.core.framework.Loader;
-import org.skywalking.apm.collector.core.module.ModuleGroupDefineLoader;
-import org.skywalking.apm.collector.core.util.DefinitionLoader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author pengys5
- */
-public class SerializedDefineLoader implements Loader> {
-
- private final Logger logger = LoggerFactory.getLogger(ModuleGroupDefineLoader.class);
-
- @Override public Map load() throws ConfigException {
- Map serializedDefineMap = new LinkedHashMap<>();
- SerializedDefinitionFile definitionFile = new SerializedDefinitionFile();
- logger.info("serialized definition file name: {}", definitionFile.fileName());
-
- DefinitionLoader definitionLoader = DefinitionLoader.load(SerializedDefine.class, definitionFile);
-
- int id = 1;
- for (SerializedDefine serializedDefine : definitionLoader) {
- logger.info("loaded serialized definition class: {}", serializedDefine.getClass().getName());
- serializedDefineMap.put(id, serializedDefine);
- }
- return serializedDefineMap;
- }
-}
diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefinitionFile.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefinitionFile.java
deleted file mode 100644
index a8ee1988a..000000000
--- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefinitionFile.java
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.skywalking.apm.collector.core.remote;
-
-import org.skywalking.apm.collector.core.framework.DefinitionFile;
-
-/**
- * @author pengys5
- */
-public class SerializedDefinitionFile extends DefinitionFile {
- @Override protected String fileName() {
- return "serialized.define";
- }
-}
diff --git a/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/QueueModuleInstaller.java b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/QueueModuleInstaller.java
index 590c32a66..b0ed3238a 100644
--- a/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/QueueModuleInstaller.java
+++ b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/QueueModuleInstaller.java
@@ -2,9 +2,10 @@ package org.skywalking.apm.collector.queue;
import java.util.Map;
import org.skywalking.apm.collector.core.client.ClientException;
+import org.skywalking.apm.collector.core.framework.CollectorContextHelper;
import org.skywalking.apm.collector.core.framework.DefineException;
import org.skywalking.apm.collector.core.module.ModuleDefine;
-import org.skywalking.apm.collector.core.module.ModuleInstaller;
+import org.skywalking.apm.collector.core.module.SingleModuleInstaller;
import org.skywalking.apm.collector.core.server.ServerHolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -12,13 +13,16 @@ import org.slf4j.LoggerFactory;
/**
* @author pengys5
*/
-public class QueueModuleInstaller implements ModuleInstaller {
+public class QueueModuleInstaller extends SingleModuleInstaller {
private final Logger logger = LoggerFactory.getLogger(QueueModuleInstaller.class);
@Override public void install(Map moduleConfig,
Map moduleDefineMap, ServerHolder serverHolder) throws DefineException, ClientException {
logger.info("beginning queue module install");
+ QueueModuleContext context = new QueueModuleContext(QueueModuleGroupDefine.GROUP_NAME);
+ CollectorContextHelper.INSTANCE.putContext(context);
+ installSingle(moduleConfig, moduleDefineMap, serverHolder);
}
}
diff --git a/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/datacarrier/QueueDataCarrierModuleDefine.java b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/datacarrier/QueueDataCarrierModuleDefine.java
index 2bef023b4..af5db748a 100644
--- a/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/datacarrier/QueueDataCarrierModuleDefine.java
+++ b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/datacarrier/QueueDataCarrierModuleDefine.java
@@ -23,7 +23,7 @@ public class QueueDataCarrierModuleDefine extends QueueModuleDefine {
}
@Override public boolean defaultModule() {
- return true;
+ return false;
}
@Override
diff --git a/apm-collector/apm-collector-remote/pom.xml b/apm-collector/apm-collector-remote/pom.xml
deleted file mode 100644
index f20ebd870..000000000
--- a/apm-collector/apm-collector-remote/pom.xml
+++ /dev/null
@@ -1,118 +0,0 @@
-
-
-
- apm-collector
- org.skywalking
- 3.2-2017
-
- 4.0.0
-
- apm-collector-remote
- jar
-
-
- UTF-8
- 1.4.0
- 4.1.12.Final
-
-
-
-
- org.skywalking
- apm-collector-core
- ${project.version}
-
-
- org.skywalking
- apm-collector-server
- ${project.version}
-
-
- org.skywalking
- apm-collector-cluster
- ${project.version}
-
-
- io.grpc
- grpc-netty
- ${grpc.version}
-
-
- io.netty
- netty-codec-http2
-
-
- io.netty
- netty-handler-proxy
-
-
-
-
- io.grpc
- grpc-protobuf
- ${grpc.version}
-
-
- io.grpc
- grpc-stub
- ${grpc.version}
-
-
- io.netty
- netty-codec-http2
- ${netty.version}
-
-
- io.netty
- netty-handler-proxy
- ${netty.version}
-
-
-
-
-
-
- kr.motd.maven
- os-maven-plugin
- 1.4.1.Final
-
-
-
-
- org.apache.maven.plugins
- maven-resources-plugin
- 2.4.3
-
- ${project.build.sourceEncoding}
-
-
-
- org.xolstice.maven.plugins
- protobuf-maven-plugin
- 0.5.0
-
-
- com.google.protobuf:protoc:3.3.0:exe:${os.detected.classifier}
-
- grpc-java
- io.grpc:protoc-gen-grpc-java:1.4.0:exe:${os.detected.classifier}
-
-
-
-
-
- compile
- compile-custom
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleContext.java b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleContext.java
deleted file mode 100644
index f5bb5b805..000000000
--- a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleContext.java
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.skywalking.apm.collector.remote;
-
-import org.skywalking.apm.collector.core.framework.Context;
-
-/**
- * @author pengys5
- */
-public class RemoteModuleContext extends Context {
- public RemoteModuleContext(String groupName) {
- super(groupName);
- }
-}
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleException.java b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleException.java
deleted file mode 100644
index 87af6e986..000000000
--- a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleException.java
+++ /dev/null
@@ -1,17 +0,0 @@
-package org.skywalking.apm.collector.remote;
-
-import org.skywalking.apm.collector.core.module.ModuleException;
-
-/**
- * @author pengys5
- */
-public class RemoteModuleException extends ModuleException {
-
- public RemoteModuleException(String message) {
- super(message);
- }
-
- public RemoteModuleException(String message, Throwable cause) {
- super(message, cause);
- }
-}
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleInstaller.java b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleInstaller.java
deleted file mode 100644
index 18f1b8f2b..000000000
--- a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleInstaller.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package org.skywalking.apm.collector.remote;
-
-import java.util.Map;
-import org.skywalking.apm.collector.core.client.ClientException;
-import org.skywalking.apm.collector.core.framework.DefineException;
-import org.skywalking.apm.collector.core.module.ModuleDefine;
-import org.skywalking.apm.collector.core.module.ModuleInstaller;
-import org.skywalking.apm.collector.core.server.ServerHolder;
-
-/**
- * @author pengys5
- */
-public class RemoteModuleInstaller implements ModuleInstaller {
- @Override public void install(Map moduleConfig, Map moduleDefineMap,
- ServerHolder serverHolder) throws DefineException, ClientException {
-
- }
-}
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCModuleRegistration.java b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCModuleRegistration.java
deleted file mode 100644
index 4f5a371d2..000000000
--- a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCModuleRegistration.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package org.skywalking.apm.collector.remote.grpc;
-
-import org.skywalking.apm.collector.core.module.ModuleRegistration;
-
-/**
- * @author pengys5
- */
-public class RemoteGRPCModuleRegistration extends ModuleRegistration {
-
- @Override public Value buildValue() {
- return new Value(RemoteGRPCConfig.HOST, RemoteGRPCConfig.PORT, null);
- }
-}
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefineException.java b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefineException.java
deleted file mode 100644
index 7a5c6063b..000000000
--- a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefineException.java
+++ /dev/null
@@ -1,17 +0,0 @@
-package org.skywalking.apm.collector.remote.grpc.handler;
-
-import org.skywalking.apm.collector.core.framework.DefineException;
-
-/**
- * @author pengys5
- */
-public class RemoteHandlerDefineException extends DefineException {
-
- public RemoteHandlerDefineException(String message) {
- super(message);
- }
-
- public RemoteHandlerDefineException(String message, Throwable cause) {
- super(message, cause);
- }
-}
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefineLoader.java b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefineLoader.java
deleted file mode 100644
index b125bc090..000000000
--- a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefineLoader.java
+++ /dev/null
@@ -1,30 +0,0 @@
-package org.skywalking.apm.collector.remote.grpc.handler;
-
-import java.util.ArrayList;
-import java.util.List;
-import org.skywalking.apm.collector.core.config.ConfigException;
-import org.skywalking.apm.collector.core.framework.Handler;
-import org.skywalking.apm.collector.core.framework.Loader;
-import org.skywalking.apm.collector.core.util.DefinitionLoader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author pengys5
- */
-public class RemoteHandlerDefineLoader implements Loader> {
-
- private final Logger logger = LoggerFactory.getLogger(RemoteHandlerDefineLoader.class);
-
- @Override public List load() throws ConfigException {
- List handlers = new ArrayList<>();
-
- RemoteHandlerDefinitionFile definitionFile = new RemoteHandlerDefinitionFile();
- DefinitionLoader definitionLoader = DefinitionLoader.load(Handler.class, definitionFile);
- for (Handler handler : definitionLoader) {
- logger.info("loaded remote handler definition class: {}", handler.getClass().getName());
- handlers.add(handler);
- }
- return handlers;
- }
-}
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefinitionFile.java b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefinitionFile.java
deleted file mode 100644
index 4231229c4..000000000
--- a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/handler/RemoteHandlerDefinitionFile.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package org.skywalking.apm.collector.remote.grpc.handler;
-
-import org.skywalking.apm.collector.core.framework.DefinitionFile;
-
-/**
- * @author pengys5
- */
-public class RemoteHandlerDefinitionFile extends DefinitionFile {
-
- @Override protected String fileName() {
- return "remote_handler.define";
- }
-}
diff --git a/apm-collector/apm-collector-remote/src/main/resources/META-INF/defines/group.define b/apm-collector/apm-collector-remote/src/main/resources/META-INF/defines/group.define
deleted file mode 100644
index 141478aab..000000000
--- a/apm-collector/apm-collector-remote/src/main/resources/META-INF/defines/group.define
+++ /dev/null
@@ -1 +0,0 @@
-org.skywalking.apm.collector.remote.RemoteModuleGroupDefine
\ No newline at end of file
diff --git a/apm-collector/apm-collector-remote/src/main/resources/META-INF/defines/module.define b/apm-collector/apm-collector-remote/src/main/resources/META-INF/defines/module.define
deleted file mode 100644
index 02abab5e6..000000000
--- a/apm-collector/apm-collector-remote/src/main/resources/META-INF/defines/module.define
+++ /dev/null
@@ -1 +0,0 @@
-org.skywalking.apm.collector.remote.grpc.RemoteGRPCModuleDefine
\ No newline at end of file
diff --git a/apm-collector/apm-collector-stream/pom.xml b/apm-collector/apm-collector-stream/pom.xml
index 9ace78e53..cc73ad3ca 100644
--- a/apm-collector/apm-collector-stream/pom.xml
+++ b/apm-collector/apm-collector-stream/pom.xml
@@ -18,6 +18,11 @@
apm-collector-core
${project.version}
+
+ org.skywalking
+ apm-collector-cluster
+ ${project.version}
+
org.skywalking
apm-collector-queue
@@ -25,8 +30,53 @@
org.skywalking
- apm-collector-remote
+ apm-collector-server
${project.version}
+
+
+
+
+ kr.motd.maven
+ os-maven-plugin
+ 1.4.1.Final
+
+
+
+
+ org.apache.maven.plugins
+ maven-resources-plugin
+ 2.4.3
+
+ ${project.build.sourceEncoding}
+
+
+
+ org.xolstice.maven.plugins
+ protobuf-maven-plugin
+ 0.5.0
+
+
+ com.google.protobuf:protoc:3.3.0:exe:${os.detected.classifier}
+
+ grpc-java
+ io.grpc:protoc-gen-grpc-java:1.4.0:exe:${os.detected.classifier}
+
+
+
+
+
+ compile
+ compile-custom
+
+
+
+
+
+
\ No newline at end of file
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalSyncWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalSyncWorkerProvider.java
deleted file mode 100644
index 30ab8713c..000000000
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalSyncWorkerProvider.java
+++ /dev/null
@@ -1,21 +0,0 @@
-package org.skywalking.apm.collector.stream;
-
-/**
- * @author pengys5
- */
-public abstract class AbstractLocalSyncWorkerProvider extends AbstractLocalWorkerProvider {
-
- @Override
- final public WorkerRef onCreate(
- LocalWorkerContext localContext) throws ProviderNotFoundException {
- T localSyncWorker = (T) workerInstance(getClusterContext());
- localSyncWorker.preStart();
-
- LocalSyncWorkerRef workerRef = new LocalSyncWorkerRef(role(), localSyncWorker);
-
- if (localContext != null) {
- localContext.put(workerRef);
- }
- return workerRef;
- }
-}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractWorkerProvider.java
deleted file mode 100644
index 4e50c2d5c..000000000
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractWorkerProvider.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package org.skywalking.apm.collector.stream;
-
-/**
- * @author pengys5
- */
-public abstract class AbstractWorkerProvider implements Provider {
-
- private ClusterWorkerContext clusterContext;
-
- public abstract Role role();
-
- public abstract T workerInstance(ClusterWorkerContext clusterContext);
-
- public abstract WorkerRef onCreate(
- LocalWorkerContext localContext) throws ProviderNotFoundException;
-
- final public void setClusterContext(ClusterWorkerContext clusterContext) {
- this.clusterContext = clusterContext;
- }
-
- final protected ClusterWorkerContext getClusterContext() {
- return clusterContext;
- }
-
- final public WorkerRef create(
- AbstractWorker workerOwner) throws ProviderNotFoundException {
-
- if (workerOwner == null) {
- return onCreate(null);
- } else if (workerOwner.getSelfContext() instanceof LocalWorkerContext) {
- return onCreate((LocalWorkerContext)workerOwner.getSelfContext());
- } else {
- throw new IllegalArgumentException("the argument of workerOwner is Illegal");
- }
- }
-}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/LocalWorkerContext.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/LocalWorkerContext.java
deleted file mode 100644
index a5e642f4d..000000000
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/LocalWorkerContext.java
+++ /dev/null
@@ -1,17 +0,0 @@
-package org.skywalking.apm.collector.stream;
-
-/**
- * @author pengys5
- */
-public class LocalWorkerContext extends WorkerContext {
-
- @Override
- final public AbstractWorkerProvider findProvider(Role role) throws ProviderNotFoundException {
- return null;
- }
-
- @Override
- final public void putProvider(AbstractWorkerProvider provider) throws UsedRoleNameException {
-
- }
-}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/Provider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/Provider.java
deleted file mode 100644
index e81a561f9..000000000
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/Provider.java
+++ /dev/null
@@ -1,9 +0,0 @@
-package org.skywalking.apm.collector.stream;
-
-/**
- * @author pengys5
- */
-public interface Provider {
-
- WorkerRef create(AbstractWorker workerOwner) throws ProviderNotFoundException;
-}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/Role.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/Role.java
deleted file mode 100644
index 23df1c250..000000000
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/Role.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package org.skywalking.apm.collector.stream;
-
-import org.skywalking.apm.collector.stream.selector.WorkerSelector;
-
-/**
- * @author pengys5
- */
-public interface Role {
-
- String roleName();
-
- WorkerSelector workerSelector();
-}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleContext.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleContext.java
new file mode 100644
index 000000000..a68f89341
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleContext.java
@@ -0,0 +1,37 @@
+package org.skywalking.apm.collector.stream;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.skywalking.apm.collector.core.framework.Context;
+import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
+
+/**
+ * @author pengys5
+ */
+public class StreamModuleContext extends Context {
+
+ private Map dataDefineMap;
+ private ClusterWorkerContext clusterWorkerContext;
+
+ public StreamModuleContext(String groupName) {
+ super(groupName);
+ dataDefineMap = new HashMap<>();
+ }
+
+ public void putAllDataDefine(Map dataDefineMap) {
+ this.dataDefineMap.putAll(dataDefineMap);
+ }
+
+ public DataDefine getDataDefine(int dataDefineId) {
+ return this.dataDefineMap.get(dataDefineId);
+ }
+
+ public ClusterWorkerContext getClusterWorkerContext() {
+ return clusterWorkerContext;
+ }
+
+ public void setClusterWorkerContext(ClusterWorkerContext clusterWorkerContext) {
+ this.clusterWorkerContext = clusterWorkerContext;
+ }
+}
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleDefine.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleDefine.java
similarity index 83%
rename from apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleDefine.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleDefine.java
index 87796d38c..89dad56db 100644
--- a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleDefine.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleDefine.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.remote;
+package org.skywalking.apm.collector.stream;
import java.util.List;
import java.util.Map;
@@ -20,9 +20,9 @@ import org.slf4j.LoggerFactory;
/**
* @author pengys5
*/
-public abstract class RemoteModuleDefine extends ModuleDefine implements ClusterDataListenerDefine {
+public abstract class StreamModuleDefine extends ModuleDefine implements ClusterDataListenerDefine {
- private final Logger logger = LoggerFactory.getLogger(RemoteModuleDefine.class);
+ private final Logger logger = LoggerFactory.getLogger(StreamModuleDefine.class);
@Override
public final void initialize(Map config, ServerHolder serverHolder) throws DefineException, ClientException {
@@ -33,9 +33,13 @@ public abstract class RemoteModuleDefine extends ModuleDefine implements Cluster
((ClusterModuleContext)CollectorContextHelper.INSTANCE.getContext(ClusterModuleGroupDefine.GROUP_NAME)).getDataMonitor().addListener(listener(), registration());
} catch (ConfigParseException | ServerException e) {
- throw new RemoteModuleException(e.getMessage(), e);
+ throw new StreamModuleException(e.getMessage(), e);
}
}
+ @Override public final boolean defaultModule() {
+ return true;
+ }
+
public abstract List handlerList() throws DefineException;
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleException.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleException.java
new file mode 100644
index 000000000..81249ad74
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleException.java
@@ -0,0 +1,17 @@
+package org.skywalking.apm.collector.stream;
+
+import org.skywalking.apm.collector.core.module.ModuleException;
+
+/**
+ * @author pengys5
+ */
+public class StreamModuleException extends ModuleException {
+
+ public StreamModuleException(String message) {
+ super(message);
+ }
+
+ public StreamModuleException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleGroupDefine.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleGroupDefine.java
similarity index 61%
rename from apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleGroupDefine.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleGroupDefine.java
index 2271155de..e320a817d 100644
--- a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/RemoteModuleGroupDefine.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleGroupDefine.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.remote;
+package org.skywalking.apm.collector.stream;
import org.skywalking.apm.collector.core.framework.Context;
import org.skywalking.apm.collector.core.module.ModuleGroupDefine;
@@ -7,19 +7,19 @@ import org.skywalking.apm.collector.core.module.ModuleInstaller;
/**
* @author pengys5
*/
-public class RemoteModuleGroupDefine implements ModuleGroupDefine {
+public class StreamModuleGroupDefine implements ModuleGroupDefine {
- public static final String GROUP_NAME = "remote";
+ public static final String GROUP_NAME = "stream";
@Override public String name() {
return GROUP_NAME;
}
@Override public Context groupContext() {
- return new RemoteModuleContext(GROUP_NAME);
+ return new StreamModuleContext(GROUP_NAME);
}
@Override public ModuleInstaller moduleInstaller() {
- return new RemoteModuleInstaller();
+ return new StreamModuleInstaller();
}
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java
new file mode 100644
index 000000000..4328ff67b
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java
@@ -0,0 +1,81 @@
+package org.skywalking.apm.collector.stream;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import org.skywalking.apm.collector.core.client.ClientException;
+import org.skywalking.apm.collector.core.config.ConfigException;
+import org.skywalking.apm.collector.core.framework.CollectorContextHelper;
+import org.skywalking.apm.collector.core.framework.DefineException;
+import org.skywalking.apm.collector.core.module.ModuleDefine;
+import org.skywalking.apm.collector.core.module.ModuleInstaller;
+import org.skywalking.apm.collector.core.server.ServerHolder;
+import org.skywalking.apm.collector.core.util.ObjectUtils;
+import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider;
+import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorkerProvider;
+import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
+import org.skywalking.apm.collector.stream.worker.LocalAsyncWorkerProviderDefineLoader;
+import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
+import org.skywalking.apm.collector.stream.worker.RemoteWorkerProviderDefineLoader;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefineLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class StreamModuleInstaller implements ModuleInstaller {
+
+ private final Logger logger = LoggerFactory.getLogger(StreamModuleInstaller.class);
+
+ @Override public void install(Map moduleConfig, Map moduleDefineMap,
+ ServerHolder serverHolder) throws DefineException, ClientException {
+ logger.info("beginning stream module install");
+ StreamModuleContext context = new StreamModuleContext(StreamModuleGroupDefine.GROUP_NAME);
+ CollectorContextHelper.INSTANCE.putContext(context);
+
+ DataDefineLoader dataDefineLoader = new DataDefineLoader();
+ try {
+ Map dataDefineMap = dataDefineLoader.load();
+ context.putAllDataDefine(dataDefineMap);
+ } catch (ConfigException e) {
+ logger.error(e.getMessage(), e);
+ }
+
+ initializeWorker(context);
+
+ logger.info("could not configure cluster module, use the default");
+ Iterator> moduleDefineEntry = moduleDefineMap.entrySet().iterator();
+ while (moduleDefineEntry.hasNext()) {
+ ModuleDefine moduleDefine = moduleDefineEntry.next().getValue();
+ logger.info("module {} initialize", moduleDefine.getClass().getName());
+ moduleDefine.initialize((ObjectUtils.isNotEmpty(moduleConfig) && moduleConfig.containsKey(moduleDefine.name())) ? moduleConfig.get(moduleDefine.name()) : null, serverHolder);
+ }
+ }
+
+ private void initializeWorker(StreamModuleContext context) {
+ ClusterWorkerContext clusterWorkerContext = new ClusterWorkerContext();
+ context.setClusterWorkerContext(clusterWorkerContext);
+
+ LocalAsyncWorkerProviderDefineLoader localAsyncProviderLoader = new LocalAsyncWorkerProviderDefineLoader();
+ RemoteWorkerProviderDefineLoader remoteProviderLoader = new RemoteWorkerProviderDefineLoader();
+ try {
+ List localAsyncProviders = localAsyncProviderLoader.load();
+ for (AbstractLocalAsyncWorkerProvider provider : localAsyncProviders) {
+ provider.setClusterContext(clusterWorkerContext);
+ provider.create();
+ clusterWorkerContext.putRole(provider.role());
+ }
+
+ List remoteProviders = remoteProviderLoader.load();
+ for (AbstractRemoteWorkerProvider provider : remoteProviders) {
+ provider.setClusterContext(clusterWorkerContext);
+ provider.create();
+ clusterWorkerContext.putRole(provider.role());
+ }
+ } catch (ConfigException | ProviderNotFoundException e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCConfig.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCConfig.java
similarity index 52%
rename from apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCConfig.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCConfig.java
index 6b465917a..5d05cbd57 100644
--- a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCConfig.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCConfig.java
@@ -1,9 +1,9 @@
-package org.skywalking.apm.collector.remote.grpc;
+package org.skywalking.apm.collector.stream.grpc;
/**
* @author pengys5
*/
-public class RemoteGRPCConfig {
+public class StreamGRPCConfig {
public static String HOST;
public static int PORT;
}
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCConfigParser.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCConfigParser.java
similarity index 72%
rename from apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCConfigParser.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCConfigParser.java
index 73beb8a49..3a4899f1c 100644
--- a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCConfigParser.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCConfigParser.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.remote.grpc;
+package org.skywalking.apm.collector.stream.grpc;
import java.util.Map;
import org.skywalking.apm.collector.core.config.ConfigParseException;
@@ -9,19 +9,19 @@ import org.skywalking.apm.collector.core.util.StringUtils;
/**
* @author pengys5
*/
-public class RemoteGRPCConfigParser implements ModuleConfigParser {
+public class StreamGRPCConfigParser implements ModuleConfigParser {
private static final String HOST = "host";
private static final String PORT = "port";
@Override public void parse(Map config) throws ConfigParseException {
if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(HOST))) {
- RemoteGRPCConfig.HOST = "localhost";
+ StreamGRPCConfig.HOST = "localhost";
}
if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(PORT))) {
- RemoteGRPCConfig.PORT = 11800;
+ StreamGRPCConfig.PORT = 11800;
} else {
- RemoteGRPCConfig.PORT = (Integer)config.get(PORT);
+ StreamGRPCConfig.PORT = (Integer)config.get(PORT);
}
}
}
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCDataListener.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCDataListener.java
similarity index 57%
rename from apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCDataListener.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCDataListener.java
index 7d2e96c84..1b6e863ab 100644
--- a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCDataListener.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCDataListener.java
@@ -1,15 +1,15 @@
-package org.skywalking.apm.collector.remote.grpc;
+package org.skywalking.apm.collector.stream.grpc;
import org.skywalking.apm.collector.cluster.ClusterModuleDefine;
import org.skywalking.apm.collector.core.cluster.ClusterDataListener;
-import org.skywalking.apm.collector.remote.RemoteModuleGroupDefine;
+import org.skywalking.apm.collector.stream.StreamModuleGroupDefine;
/**
* @author pengys5
*/
-public class RemoteGRPCDataListener extends ClusterDataListener {
+public class StreamGRPCDataListener extends ClusterDataListener {
- public static final String PATH = ClusterModuleDefine.BASE_CATALOG + "." + RemoteModuleGroupDefine.GROUP_NAME + "." + RemoteGRPCModuleDefine.MODULE_NAME;
+ public static final String PATH = ClusterModuleDefine.BASE_CATALOG + "." + StreamModuleGroupDefine.GROUP_NAME + "." + StreamGRPCModuleDefine.MODULE_NAME;
@Override public String path() {
return PATH;
diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCModuleDefine.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCModuleDefine.java
similarity index 51%
rename from apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCModuleDefine.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCModuleDefine.java
index 22b884e43..3a8166476 100644
--- a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCModuleDefine.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCModuleDefine.java
@@ -1,42 +1,37 @@
-package org.skywalking.apm.collector.remote.grpc;
+package org.skywalking.apm.collector.stream.grpc;
+import java.util.ArrayList;
import java.util.List;
import org.skywalking.apm.collector.core.client.Client;
import org.skywalking.apm.collector.core.client.DataMonitor;
import org.skywalking.apm.collector.core.cluster.ClusterDataListener;
-import org.skywalking.apm.collector.core.config.ConfigException;
import org.skywalking.apm.collector.core.framework.DefineException;
import org.skywalking.apm.collector.core.framework.Handler;
import org.skywalking.apm.collector.core.module.ModuleConfigParser;
import org.skywalking.apm.collector.core.module.ModuleRegistration;
import org.skywalking.apm.collector.core.server.Server;
-import org.skywalking.apm.collector.remote.RemoteModuleDefine;
-import org.skywalking.apm.collector.remote.RemoteModuleGroupDefine;
-import org.skywalking.apm.collector.remote.grpc.handler.RemoteHandlerDefineException;
-import org.skywalking.apm.collector.remote.grpc.handler.RemoteHandlerDefineLoader;
import org.skywalking.apm.collector.server.grpc.GRPCServer;
+import org.skywalking.apm.collector.stream.StreamModuleDefine;
+import org.skywalking.apm.collector.stream.StreamModuleGroupDefine;
+import org.skywalking.apm.collector.stream.grpc.handler.RemoteCommonServiceHandler;
/**
* @author pengys5
*/
-public class RemoteGRPCModuleDefine extends RemoteModuleDefine {
+public class StreamGRPCModuleDefine extends StreamModuleDefine {
- public static final String MODULE_NAME = "remote";
+ public static final String MODULE_NAME = "stream";
@Override public String name() {
return MODULE_NAME;
}
@Override protected String group() {
- return RemoteModuleGroupDefine.GROUP_NAME;
- }
-
- @Override public boolean defaultModule() {
- return true;
+ return StreamModuleGroupDefine.GROUP_NAME;
}
@Override protected ModuleConfigParser configParser() {
- return new RemoteGRPCConfigParser();
+ return new StreamGRPCConfigParser();
}
@Override protected Client createClient(DataMonitor dataMonitor) {
@@ -44,25 +39,20 @@ public class RemoteGRPCModuleDefine extends RemoteModuleDefine {
}
@Override protected Server server() {
- return new GRPCServer(RemoteGRPCConfig.HOST, RemoteGRPCConfig.PORT);
+ return new GRPCServer(StreamGRPCConfig.HOST, StreamGRPCConfig.PORT);
}
@Override protected ModuleRegistration registration() {
- return new RemoteGRPCModuleRegistration();
+ return new StreamGRPCModuleRegistration();
}
@Override public ClusterDataListener listener() {
- return new RemoteGRPCDataListener();
+ return new StreamGRPCDataListener();
}
@Override public List handlerList() throws DefineException {
- RemoteHandlerDefineLoader loader = new RemoteHandlerDefineLoader();
- List handlers = null;
- try {
- handlers = loader.load();
- } catch (ConfigException e) {
- throw new RemoteHandlerDefineException(e.getMessage(), e);
- }
+ List handlers = new ArrayList<>();
+ handlers.add(new RemoteCommonServiceHandler());
return handlers;
}
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCModuleRegistration.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCModuleRegistration.java
new file mode 100644
index 000000000..1c74d45c3
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCModuleRegistration.java
@@ -0,0 +1,13 @@
+package org.skywalking.apm.collector.stream.grpc;
+
+import org.skywalking.apm.collector.core.module.ModuleRegistration;
+
+/**
+ * @author pengys5
+ */
+public class StreamGRPCModuleRegistration extends ModuleRegistration {
+
+ @Override public Value buildValue() {
+ return new Value(StreamGRPCConfig.HOST, StreamGRPCConfig.PORT, null);
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/handler/RemoteCommonServiceHandler.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/handler/RemoteCommonServiceHandler.java
new file mode 100644
index 000000000..f0294db24
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/handler/RemoteCommonServiceHandler.java
@@ -0,0 +1,42 @@
+package org.skywalking.apm.collector.stream.grpc.handler;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.InvalidProtocolBufferException;
+import io.grpc.stub.StreamObserver;
+import org.skywalking.apm.collector.cluster.ClusterModuleGroupDefine;
+import org.skywalking.apm.collector.core.framework.CollectorContextHelper;
+import org.skywalking.apm.collector.remote.grpc.proto.Empty;
+import org.skywalking.apm.collector.remote.grpc.proto.Message;
+import org.skywalking.apm.collector.remote.grpc.proto.RemoteCommonServiceGrpc;
+import org.skywalking.apm.collector.server.grpc.GRPCHandler;
+import org.skywalking.apm.collector.stream.StreamModuleContext;
+import org.skywalking.apm.collector.stream.worker.WorkerInvokeException;
+import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException;
+import org.skywalking.apm.collector.stream.worker.impl.data.Data;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class RemoteCommonServiceHandler extends RemoteCommonServiceGrpc.RemoteCommonServiceImplBase implements GRPCHandler {
+
+ private final Logger logger = LoggerFactory.getLogger(RemoteCommonServiceHandler.class);
+
+ @Override public void call(Message request, StreamObserver responseObserver) {
+ String roleName = request.getWorkerRole();
+ int dataDefineId = request.getDataDefineId();
+ ByteString bytesData = request.getDataBytes();
+
+ StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(ClusterModuleGroupDefine.GROUP_NAME);
+ DataDefine dataDefine = context.getDataDefine(dataDefineId);
+
+ try {
+ Data data = dataDefine.parseFrom(bytesData);
+ context.getClusterWorkerContext().lookup(context.getClusterWorkerContext().getRole(roleName)).tell(data);
+ } catch (InvalidProtocolBufferException | WorkerNotFoundException | WorkerInvokeException e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/GRPCRemoteWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/GRPCRemoteWorker.java
deleted file mode 100644
index 3bd9a33fa..000000000
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/GRPCRemoteWorker.java
+++ /dev/null
@@ -1,26 +0,0 @@
-package org.skywalking.apm.collector.stream.impl;
-
-import org.skywalking.apm.collector.stream.AbstractRemoteWorker;
-import org.skywalking.apm.collector.stream.ClusterWorkerContext;
-import org.skywalking.apm.collector.stream.LocalWorkerContext;
-import org.skywalking.apm.collector.stream.ProviderNotFoundException;
-import org.skywalking.apm.collector.stream.Role;
-import org.skywalking.apm.collector.stream.WorkerException;
-
-/**
- * @author pengys5
- */
-public class GRPCRemoteWorker extends AbstractRemoteWorker {
-
- protected GRPCRemoteWorker(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
- super(role, clusterContext, selfContext);
- }
-
- @Override public void preStart() throws ProviderNotFoundException {
-
- }
-
- @Override protected final void onWork(Object message) throws WorkerException {
-
- }
-}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/RemoteCommonServiceHandler.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/RemoteCommonServiceHandler.java
deleted file mode 100644
index ea560ef12..000000000
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/RemoteCommonServiceHandler.java
+++ /dev/null
@@ -1,24 +0,0 @@
-package org.skywalking.apm.collector.stream.impl;
-
-import com.google.protobuf.ByteString;
-import io.grpc.stub.StreamObserver;
-import org.skywalking.apm.collector.remote.grpc.proto.Empty;
-import org.skywalking.apm.collector.remote.grpc.proto.Message;
-import org.skywalking.apm.collector.remote.grpc.proto.RemoteCommonServiceGrpc;
-import org.skywalking.apm.collector.server.grpc.GRPCHandler;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author pengys5
- */
-public class RemoteCommonServiceHandler extends RemoteCommonServiceGrpc.RemoteCommonServiceImplBase implements GRPCHandler {
-
- private final Logger logger = LoggerFactory.getLogger(RemoteCommonServiceHandler.class);
-
- @Override public void call(Message request, StreamObserver responseObserver) {
- String workerRole = request.getWorkerRole();
- int dataDefineId = request.getDataDefineId();
- ByteString bytesData = request.getDataBytes();
- }
-}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalAsyncWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorker.java
similarity index 88%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalAsyncWorker.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorker.java
index 01ce9b8c7..17fd2edfd 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalAsyncWorker.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorker.java
@@ -1,4 +1,6 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
+
+import org.skywalking.apm.collector.core.queue.QueueExecutor;
/**
* The AbstractLocalAsyncWorker implementations represent workers,
@@ -7,7 +9,7 @@ package org.skywalking.apm.collector.stream;
* @author pengys5
* @since v3.0-2017
*/
-public abstract class AbstractLocalAsyncWorker extends AbstractLocalWorker {
+public abstract class AbstractLocalAsyncWorker extends AbstractLocalWorker implements QueueExecutor {
/**
* Construct an AbstractLocalAsyncWorker with the worker role and context.
@@ -15,10 +17,9 @@ public abstract class AbstractLocalAsyncWorker extends AbstractLocalWorker {
* @param role The responsibility of worker in cluster, more than one workers can have same responsibility which use
* to provide load balancing ability.
* @param clusterContext See {@link ClusterWorkerContext}
- * @param selfContext See {@link LocalWorkerContext}
*/
- public AbstractLocalAsyncWorker(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
- super(role, clusterContext, selfContext);
+ public AbstractLocalAsyncWorker(Role role, ClusterWorkerContext clusterContext) {
+ super(role, clusterContext);
}
/**
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalAsyncWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java
similarity index 80%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalAsyncWorkerProvider.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java
index 21fe8b100..712b13a76 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalAsyncWorkerProvider.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
import org.skywalking.apm.collector.core.framework.CollectorContextHelper;
import org.skywalking.apm.collector.core.queue.QueueCreator;
@@ -14,8 +14,7 @@ public abstract class AbstractLocalAsyncWorkerProviderAbstractLocalSyncWorker defines workers who receive data from jvm inside call and response in real
@@ -8,8 +8,8 @@ package org.skywalking.apm.collector.stream;
* @since v3.0-2017
*/
public abstract class AbstractLocalSyncWorker extends AbstractLocalWorker {
- public AbstractLocalSyncWorker(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
- super(role, clusterContext, selfContext);
+ public AbstractLocalSyncWorker(Role role, ClusterWorkerContext clusterContext) {
+ super(role, clusterContext);
}
/**
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalSyncWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalSyncWorkerProvider.java
new file mode 100644
index 000000000..d3a542a61
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalSyncWorkerProvider.java
@@ -0,0 +1,15 @@
+package org.skywalking.apm.collector.stream.worker;
+
+/**
+ * @author pengys5
+ */
+public abstract class AbstractLocalSyncWorkerProvider extends AbstractLocalWorkerProvider {
+
+ @Override final public WorkerRef create() throws ProviderNotFoundException {
+ T localSyncWorker = workerInstance(getClusterContext());
+ localSyncWorker.preStart();
+
+ LocalSyncWorkerRef workerRef = new LocalSyncWorkerRef(role(), localSyncWorker);
+ return workerRef;
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorker.java
similarity index 53%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalWorker.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorker.java
index e94b6119e..c92a23fc7 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalWorker.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorker.java
@@ -1,10 +1,10 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
/**
* @author pengys5
*/
public abstract class AbstractLocalWorker extends AbstractWorker {
- public AbstractLocalWorker(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
- super(role, clusterContext, selfContext);
+ public AbstractLocalWorker(Role role, ClusterWorkerContext clusterContext) {
+ super(role, clusterContext);
}
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorkerProvider.java
similarity index 73%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalWorkerProvider.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorkerProvider.java
index b0e155c93..8cc96b71f 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalWorkerProvider.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorkerProvider.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
/**
* @author pengys5
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractRemoteWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorker.java
similarity index 87%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractRemoteWorker.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorker.java
index 20c552bee..9eb31bbbf 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractRemoteWorker.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorker.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
/**
* The AbstractRemoteWorker implementations represent workers,
@@ -17,10 +17,9 @@ public abstract class AbstractRemoteWorker extends AbstractWorker {
* @param role If multi-workers are for load balance, they should be more likely called worker instance. Meaning,
* each worker have multi instances.
* @param clusterContext See {@link ClusterWorkerContext}
- * @param selfContext See {@link LocalWorkerContext}
*/
- protected AbstractRemoteWorker(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
- super(role, clusterContext, selfContext);
+ protected AbstractRemoteWorker(Role role, ClusterWorkerContext clusterContext) {
+ super(role, clusterContext);
}
/**
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractRemoteWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorkerProvider.java
similarity index 84%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractRemoteWorkerProvider.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorkerProvider.java
index 3774be6cb..5c4b25a8b 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/AbstractRemoteWorkerProvider.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorkerProvider.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
/**
* The AbstractRemoteWorkerProvider implementations represent providers,
@@ -20,13 +20,11 @@ public abstract class AbstractRemoteWorkerProvider implements Provider {
+
+ private ClusterWorkerContext clusterContext;
+
+ public abstract Role role();
+
+ public abstract T workerInstance(ClusterWorkerContext clusterContext);
+
+ final public void setClusterContext(ClusterWorkerContext clusterContext) {
+ this.clusterContext = clusterContext;
+ }
+
+ final protected ClusterWorkerContext getClusterContext() {
+ return clusterContext;
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/ClusterWorkerContext.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/ClusterWorkerContext.java
similarity index 96%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/ClusterWorkerContext.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/ClusterWorkerContext.java
index d049ec6c0..3db95ff65 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/ClusterWorkerContext.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/ClusterWorkerContext.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/ClusterWorkerRefCounter.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/ClusterWorkerRefCounter.java
similarity index 91%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/ClusterWorkerRefCounter.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/ClusterWorkerRefCounter.java
index 85bd2773c..3456dbb53 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/ClusterWorkerRefCounter.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/ClusterWorkerRefCounter.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/Context.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Context.java
similarity index 85%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/Context.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Context.java
index 9790d4084..218c78c28 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/Context.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Context.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
/**
* @author pengys5
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefineLoader.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefineLoader.java
new file mode 100644
index 000000000..0d6dfb566
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefineLoader.java
@@ -0,0 +1,32 @@
+package org.skywalking.apm.collector.stream.worker;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.skywalking.apm.collector.core.config.ConfigException;
+import org.skywalking.apm.collector.core.framework.Loader;
+import org.skywalking.apm.collector.core.util.DefinitionLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class LocalAsyncWorkerProviderDefineLoader implements Loader> {
+
+ private final Logger logger = LoggerFactory.getLogger(LocalAsyncWorkerProviderDefineLoader.class);
+
+ @Override public List load() throws ConfigException {
+ List providers = new ArrayList<>();
+ LocalAsyncWorkerProviderDefinitionFile definitionFile = new LocalAsyncWorkerProviderDefinitionFile();
+ logger.info("local async worker provider definition file name: {}", definitionFile.fileName());
+
+ DefinitionLoader definitionLoader = DefinitionLoader.load(AbstractLocalAsyncWorkerProvider.class, definitionFile);
+
+ int id = 1;
+ for (AbstractLocalAsyncWorkerProvider provider : definitionLoader) {
+ logger.info("loaded local async worker provider definition class: {}", provider.getClass().getName());
+ providers.add(provider);
+ }
+ return providers;
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefinitionFile.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefinitionFile.java
new file mode 100644
index 000000000..356cf9ffb
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefinitionFile.java
@@ -0,0 +1,12 @@
+package org.skywalking.apm.collector.stream.worker;
+
+import org.skywalking.apm.collector.core.framework.DefinitionFile;
+
+/**
+ * @author pengys5
+ */
+public class LocalAsyncWorkerProviderDefinitionFile extends DefinitionFile {
+ @Override protected String fileName() {
+ return "local_async_worker_provider.define";
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/LocalAsyncWorkerRef.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerRef.java
similarity index 90%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/LocalAsyncWorkerRef.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerRef.java
index 286fde0f6..06ee2f0b6 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/LocalAsyncWorkerRef.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerRef.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
import org.skywalking.apm.collector.core.queue.QueueEventHandler;
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/LocalSyncWorkerRef.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalSyncWorkerRef.java
similarity index 91%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/LocalSyncWorkerRef.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalSyncWorkerRef.java
index e40cfe0f3..d24d07779 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/LocalSyncWorkerRef.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalSyncWorkerRef.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
/**
* @author pengys5
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/LookUp.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LookUp.java
similarity index 78%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/LookUp.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LookUp.java
index 802de88e2..1e5ad6acb 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/LookUp.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LookUp.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
/**
* @author pengys5
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Provider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Provider.java
new file mode 100644
index 000000000..b0f4bdc05
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Provider.java
@@ -0,0 +1,9 @@
+package org.skywalking.apm.collector.stream.worker;
+
+/**
+ * @author pengys5
+ */
+public interface Provider {
+
+ WorkerRef create() throws ProviderNotFoundException;
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/ProviderNotFoundException.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/ProviderNotFoundException.java
similarity index 73%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/ProviderNotFoundException.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/ProviderNotFoundException.java
index f6c8f3a72..9b5327e72 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/ProviderNotFoundException.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/ProviderNotFoundException.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
public class ProviderNotFoundException extends Exception {
public ProviderNotFoundException(String message) {
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerProviderDefineLoader.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerProviderDefineLoader.java
new file mode 100644
index 000000000..383f63d31
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerProviderDefineLoader.java
@@ -0,0 +1,31 @@
+package org.skywalking.apm.collector.stream.worker;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.skywalking.apm.collector.core.config.ConfigException;
+import org.skywalking.apm.collector.core.framework.Loader;
+import org.skywalking.apm.collector.core.util.DefinitionLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class RemoteWorkerProviderDefineLoader implements Loader> {
+
+ private final Logger logger = LoggerFactory.getLogger(RemoteWorkerProviderDefineLoader.class);
+
+ @Override public List load() throws ConfigException {
+ List providers = new ArrayList<>();
+ RemoteWorkerProviderDefinitionFile definitionFile = new RemoteWorkerProviderDefinitionFile();
+ logger.info("remote worker provider definition file name: {}", definitionFile.fileName());
+
+ DefinitionLoader definitionLoader = DefinitionLoader.load(AbstractRemoteWorkerProvider.class, definitionFile);
+
+ for (AbstractRemoteWorkerProvider provider : definitionLoader) {
+ logger.info("loaded remote worker provider definition class: {}", provider.getClass().getName());
+ providers.add(provider);
+ }
+ return providers;
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerProviderDefinitionFile.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerProviderDefinitionFile.java
new file mode 100644
index 000000000..ed301cc92
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerProviderDefinitionFile.java
@@ -0,0 +1,12 @@
+package org.skywalking.apm.collector.stream.worker;
+
+import org.skywalking.apm.collector.core.framework.DefinitionFile;
+
+/**
+ * @author pengys5
+ */
+public class RemoteWorkerProviderDefinitionFile extends DefinitionFile {
+ @Override protected String fileName() {
+ return "remote_worker_provider.define";
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/RemoteWorkerRef.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerRef.java
similarity index 88%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/RemoteWorkerRef.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerRef.java
index 79a2abe03..f653c2f62 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/RemoteWorkerRef.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerRef.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
/**
* @author pengys5
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Role.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Role.java
new file mode 100644
index 000000000..e9c49d521
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Role.java
@@ -0,0 +1,13 @@
+package org.skywalking.apm.collector.stream.worker;
+
+import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector;
+
+/**
+ * @author pengys5
+ */
+public interface Role {
+
+ String roleName();
+
+ WorkerSelector workerSelector();
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/UsedRoleNameException.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/UsedRoleNameException.java
similarity index 72%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/UsedRoleNameException.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/UsedRoleNameException.java
index 13cd45d70..15df1fc82 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/UsedRoleNameException.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/UsedRoleNameException.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
public class UsedRoleNameException extends Exception {
public UsedRoleNameException(String message) {
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerContext.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerContext.java
similarity index 75%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerContext.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerContext.java
index 4d5617566..986397a2d 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerContext.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerContext.java
@@ -1,32 +1,29 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import org.skywalking.apm.collector.stream.impl.data.DataDefine;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
/**
* @author pengys5
*/
public abstract class WorkerContext implements Context {
+ private Map> roleWorkers;
+ private Map roles;
private Map dataDefineMap;
- private Map> roleWorkers;
-
public WorkerContext() {
- this.roleWorkers = new ConcurrentHashMap<>();
+ this.roleWorkers = new HashMap<>();
+ this.roles = new HashMap<>();
}
private Map> getRoleWorkers() {
return this.roleWorkers;
}
- public final DataDefine getDataDefine(int defineId) {
- return dataDefineMap.get(defineId);
- }
-
@Override final public WorkerRefs lookup(Role role) throws WorkerNotFoundException {
if (getRoleWorkers().containsKey(role.roleName())) {
WorkerRefs refs = new WorkerRefs(getRoleWorkers().get(role.roleName()), role.workerSelector());
@@ -36,6 +33,18 @@ public abstract class WorkerContext implements Context {
}
}
+ public final void putRole(Role role) {
+ roles.put(role.roleName(), role);
+ }
+
+ public final Role getRole(String roleName) {
+ return roles.get(roleName);
+ }
+
+ public final DataDefine getDataDefine(int defineId) {
+ return dataDefineMap.get(defineId);
+ }
+
@Override final public void put(WorkerRef workerRef) {
if (!getRoleWorkers().containsKey(workerRef.getRole().roleName())) {
getRoleWorkers().putIfAbsent(workerRef.getRole().roleName(), new ArrayList());
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerException.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerException.java
similarity index 87%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerException.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerException.java
index 1729e9449..ada554175 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerException.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerException.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
/**
* Defines a general exception a worker can throw when it
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerInvokeException.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerInvokeException.java
similarity index 88%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerInvokeException.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerInvokeException.java
index a93a80a37..0fe9b6de4 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerInvokeException.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerInvokeException.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
/**
* This exception is raised when worker fails to process job during "call" or "ask"
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerNotFoundException.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerNotFoundException.java
similarity index 74%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerNotFoundException.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerNotFoundException.java
index 2f454e061..ce231e1a1 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerNotFoundException.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerNotFoundException.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
public class WorkerNotFoundException extends WorkerException {
public WorkerNotFoundException(String message) {
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerRef.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerRef.java
similarity index 84%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerRef.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerRef.java
index 195950c01..899e3978e 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerRef.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerRef.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
/**
* @author pengys5
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerRefs.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerRefs.java
similarity index 89%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerRefs.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerRefs.java
index 80cc6fdd9..a178802c4 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/WorkerRefs.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerRefs.java
@@ -1,7 +1,7 @@
-package org.skywalking.apm.collector.stream;
+package org.skywalking.apm.collector.stream.worker;
import java.util.List;
-import org.skywalking.apm.collector.stream.selector.WorkerSelector;
+import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/AggregationWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/AggregationWorker.java
similarity index 64%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/AggregationWorker.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/AggregationWorker.java
index bd892035c..222caa4b6 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/AggregationWorker.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/AggregationWorker.java
@@ -1,14 +1,13 @@
-package org.skywalking.apm.collector.stream.impl;
+package org.skywalking.apm.collector.stream.worker.impl;
import org.skywalking.apm.collector.core.queue.EndOfBatchCommand;
-import org.skywalking.apm.collector.stream.AbstractLocalAsyncWorker;
-import org.skywalking.apm.collector.stream.ClusterWorkerContext;
-import org.skywalking.apm.collector.stream.LocalWorkerContext;
-import org.skywalking.apm.collector.stream.ProviderNotFoundException;
-import org.skywalking.apm.collector.stream.Role;
-import org.skywalking.apm.collector.stream.WorkerException;
-import org.skywalking.apm.collector.stream.impl.data.Data;
-import org.skywalking.apm.collector.stream.impl.data.DataCache;
+import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorker;
+import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
+import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
+import org.skywalking.apm.collector.stream.worker.Role;
+import org.skywalking.apm.collector.stream.worker.WorkerException;
+import org.skywalking.apm.collector.stream.worker.impl.data.Data;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataCache;
/**
* @author pengys5
@@ -17,8 +16,8 @@ public abstract class AggregationWorker extends AbstractLocalAsyncWorker {
private DataCache dataCache;
- public AggregationWorker(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
- super(role, clusterContext, selfContext);
+ public AggregationWorker(Role role, ClusterWorkerContext clusterContext) {
+ super(role, clusterContext);
dataCache = new DataCache();
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/Const.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/Const.java
similarity index 86%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/Const.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/Const.java
index 1850a2ee2..c3acf882b 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/Const.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/Const.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream.impl;
+package org.skywalking.apm.collector.stream.worker.impl;
/**
* @author pengys5
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/GRPCRemoteWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/GRPCRemoteWorker.java
new file mode 100644
index 000000000..4c76735f8
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/GRPCRemoteWorker.java
@@ -0,0 +1,25 @@
+package org.skywalking.apm.collector.stream.worker.impl;
+
+import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorker;
+import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
+import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
+import org.skywalking.apm.collector.stream.worker.Role;
+import org.skywalking.apm.collector.stream.worker.WorkerException;
+
+/**
+ * @author pengys5
+ */
+public class GRPCRemoteWorker extends AbstractRemoteWorker {
+
+ protected GRPCRemoteWorker(Role role, ClusterWorkerContext clusterContext) {
+ super(role, clusterContext);
+ }
+
+ @Override public void preStart() throws ProviderNotFoundException {
+
+ }
+
+ @Override protected final void onWork(Object message) throws WorkerException {
+
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Attribute.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Attribute.java
similarity index 89%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Attribute.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Attribute.java
index 3075517de..bba24de2c 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Attribute.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Attribute.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream.impl.data;
+package org.skywalking.apm.collector.stream.worker.impl.data;
/**
* @author pengys5
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/AttributeType.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/AttributeType.java
similarity index 56%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/AttributeType.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/AttributeType.java
index 84e317ec0..3706017b6 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/AttributeType.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/AttributeType.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream.impl.data;
+package org.skywalking.apm.collector.stream.worker.impl.data;
/**
* @author pengys5
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Data.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Data.java
similarity index 94%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Data.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Data.java
index be8d4eb21..b8318ca2e 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Data.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Data.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream.impl.data;
+package org.skywalking.apm.collector.stream.worker.impl.data;
/**
* @author pengys5
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataCache.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataCache.java
similarity index 90%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataCache.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataCache.java
index db51cf91d..8db3ba70f 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataCache.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataCache.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream.impl.data;
+package org.skywalking.apm.collector.stream.worker.impl.data;
/**
* @author pengys5
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataCollection.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataCollection.java
similarity index 93%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataCollection.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataCollection.java
index 725dcca71..12cc2f282 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataCollection.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataCollection.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream.impl.data;
+package org.skywalking.apm.collector.stream.worker.impl.data;
import java.util.HashMap;
import java.util.Map;
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataDefine.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefine.java
similarity index 90%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataDefine.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefine.java
index 5571ac264..2c3f3d620 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataDefine.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefine.java
@@ -1,4 +1,7 @@
-package org.skywalking.apm.collector.stream.impl.data;
+package org.skywalking.apm.collector.stream.worker.impl.data;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.InvalidProtocolBufferException;
/**
* @author pengys5
@@ -75,4 +78,6 @@ public abstract class DataDefine {
}
}
}
+
+ public abstract Data parseFrom(ByteString bytesData) throws InvalidProtocolBufferException;
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataDefineLoader.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java
similarity index 94%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataDefineLoader.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java
index 8ce454440..67e2cff5d 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataDefineLoader.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream.impl.data;
+package org.skywalking.apm.collector.stream.worker.impl.data;
import java.util.HashMap;
import java.util.Map;
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataDefinitionFile.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefinitionFile.java
similarity index 79%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataDefinitionFile.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefinitionFile.java
index 93df98019..e7875d8b8 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/DataDefinitionFile.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefinitionFile.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream.impl.data;
+package org.skywalking.apm.collector.stream.worker.impl.data;
import org.skywalking.apm.collector.core.framework.DefinitionFile;
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Operation.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Operation.java
similarity index 77%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Operation.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Operation.java
index b94e16b11..05bb5ed2a 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Operation.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Operation.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream.impl.data;
+package org.skywalking.apm.collector.stream.worker.impl.data;
/**
* @author pengys5
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Window.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Window.java
similarity index 93%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Window.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Window.java
index 05b48c197..d292dd5ef 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/Window.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Window.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream.impl.data;
+package org.skywalking.apm.collector.stream.worker.impl.data;
/**
* @author pengys5
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/operate/CoverOperation.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/CoverOperation.java
similarity index 73%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/operate/CoverOperation.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/CoverOperation.java
index bae2f6308..c0a48d077 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/operate/CoverOperation.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/CoverOperation.java
@@ -1,6 +1,6 @@
-package org.skywalking.apm.collector.stream.impl.data.operate;
+package org.skywalking.apm.collector.stream.worker.impl.data.operate;
-import org.skywalking.apm.collector.stream.impl.data.Operation;
+import org.skywalking.apm.collector.stream.worker.impl.data.Operation;
/**
* @author pengys5
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/operate/NonOperation.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/NonOperation.java
similarity index 73%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/operate/NonOperation.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/NonOperation.java
index b33950073..b306f1c99 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/impl/data/operate/NonOperation.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/NonOperation.java
@@ -1,6 +1,6 @@
-package org.skywalking.apm.collector.stream.impl.data.operate;
+package org.skywalking.apm.collector.stream.worker.impl.data.operate;
-import org.skywalking.apm.collector.stream.impl.data.Operation;
+import org.skywalking.apm.collector.stream.worker.impl.data.Operation;
/**
* @author pengys5
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/selector/AbstractHashMessage.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/AbstractHashMessage.java
similarity index 88%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/selector/AbstractHashMessage.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/AbstractHashMessage.java
index ef8aedbba..e6181510c 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/selector/AbstractHashMessage.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/AbstractHashMessage.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.stream.selector;
+package org.skywalking.apm.collector.stream.worker.selector;
/**
* The AbstractHashMessage implementations represent aggregate message,
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/selector/HashCodeSelector.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/HashCodeSelector.java
similarity index 82%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/selector/HashCodeSelector.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/HashCodeSelector.java
index e1fdc6cf2..cb3434110 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/selector/HashCodeSelector.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/HashCodeSelector.java
@@ -1,7 +1,8 @@
-package org.skywalking.apm.collector.stream.selector;
+package org.skywalking.apm.collector.stream.worker.selector;
import java.util.List;
-import org.skywalking.apm.collector.stream.WorkerRef;
+import org.skywalking.apm.collector.stream.worker.WorkerRef;
+import org.skywalking.apm.collector.stream.worker.AbstractWorker;
/**
* The HashCodeSelector is a simple implementation of {@link WorkerSelector}. It choose {@link WorkerRef}
@@ -17,7 +18,7 @@ public class HashCodeSelector implements WorkerSelector {
* Use message hashcode to select {@link WorkerRef}.
*
* @param members given {@link WorkerRef} list, which size is greater than 0;
- * @param message the {@link org.skywalking.apm.collector.stream.AbstractWorker} is going to send.
+ * @param message the {@link AbstractWorker} is going to send.
* @return the selected {@link WorkerRef}
*/
@Override
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/selector/RollingSelector.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/RollingSelector.java
similarity index 74%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/selector/RollingSelector.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/RollingSelector.java
index c68fb3319..f81e64bae 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/selector/RollingSelector.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/RollingSelector.java
@@ -1,7 +1,8 @@
-package org.skywalking.apm.collector.stream.selector;
+package org.skywalking.apm.collector.stream.worker.selector;
import java.util.List;
-import org.skywalking.apm.collector.stream.WorkerRef;
+import org.skywalking.apm.collector.stream.worker.WorkerRef;
+import org.skywalking.apm.collector.stream.worker.AbstractWorker;
/**
* The RollingSelector is a simple implementation of {@link WorkerSelector}.
@@ -18,7 +19,7 @@ public class RollingSelector implements WorkerSelector {
* Use round-robin to select {@link WorkerRef}.
*
* @param members given {@link WorkerRef} list, which size is greater than 0;
- * @param message message the {@link org.skywalking.apm.collector.stream.AbstractWorker} is going to send.
+ * @param message message the {@link AbstractWorker} is going to send.
* @return the selected {@link WorkerRef}
*/
@Override
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/selector/WorkerSelector.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/WorkerSelector.java
similarity index 72%
rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/selector/WorkerSelector.java
rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/WorkerSelector.java
index 45a217003..1409ef746 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/selector/WorkerSelector.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/WorkerSelector.java
@@ -1,7 +1,8 @@
-package org.skywalking.apm.collector.stream.selector;
+package org.skywalking.apm.collector.stream.worker.selector;
import java.util.List;
-import org.skywalking.apm.collector.stream.WorkerRef;
+import org.skywalking.apm.collector.stream.worker.WorkerRef;
+import org.skywalking.apm.collector.stream.worker.AbstractWorker;
/**
* The WorkerSelector should be implemented by any class whose instances
@@ -18,7 +19,7 @@ public interface WorkerSelector {
* select a {@link WorkerRef} from a {@link WorkerRef} list.
*
* @param members given {@link WorkerRef} list, which size is greater than 0;
- * @param message the {@link org.skywalking.apm.collector.stream.AbstractWorker} is going to send.
+ * @param message the {@link AbstractWorker} is going to send.
* @return the selected {@link WorkerRef}
*/
T select(List members, Object message);
diff --git a/apm-collector/apm-collector-remote/src/main/proto/RemoteCommonService.proto b/apm-collector/apm-collector-stream/src/main/proto/RemoteCommonService.proto
similarity index 100%
rename from apm-collector/apm-collector-remote/src/main/proto/RemoteCommonService.proto
rename to apm-collector/apm-collector-stream/src/main/proto/RemoteCommonService.proto
diff --git a/apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/group.define b/apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/group.define
new file mode 100644
index 000000000..dfcb24058
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/group.define
@@ -0,0 +1 @@
+org.skywalking.apm.collector.stream.StreamModuleGroupDefine
\ No newline at end of file
diff --git a/apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/module.define b/apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/module.define
new file mode 100644
index 000000000..b58f3f133
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/module.define
@@ -0,0 +1 @@
+org.skywalking.apm.collector.stream.grpc.StreamGRPCModuleDefine
\ No newline at end of file
diff --git a/apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/remote_handler.define b/apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/remote_handler.define
deleted file mode 100644
index 389dfb80e..000000000
--- a/apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/remote_handler.define
+++ /dev/null
@@ -1 +0,0 @@
-org.skywalking.apm.collector.stream.impl.RemoteCommonServiceHandler
\ No newline at end of file
diff --git a/apm-collector/pom.xml b/apm-collector/pom.xml
index a73fc46cd..7eda101c6 100644
--- a/apm-collector/pom.xml
+++ b/apm-collector/pom.xml
@@ -12,7 +12,6 @@
apm-collector-agentstream
apm-collector-ui
apm-collector-boot
- apm-collector-remote
apm-collector-stream
apm-collector-agentserver
apm-collector-agentregister
From 5bd5a69b49120d94d94329a7a3ff7fa1441f4cf7 Mon Sep 17 00:00:00 2001
From: pengys5 <8082209@qq.com>
Date: Tue, 25 Jul 2017 10:39:58 +0800
Subject: [PATCH 10/28] Dao loader and client inject
---
.../apm-collector-agentregister/pom.xml | 5 ++
.../application/ApplicationEsDAO.java | 15 +++++
.../application/ApplicationH2DAO.java | 15 +++++
.../application/ApplicationIDGetOrCreate.java | 13 +++++
.../application/IApplicationDAO.java | 8 +++
.../ApplicationRegisterServiceHandler.java | 17 +++++-
.../handler/TraceSegmentServiceHandler.java | 10 ++++
.../ApplicationRegisterDataDefine.java | 38 ++++++++++++
.../ApplicationRegisterRemoteWorker.java | 58 +++++++++++++++++++
.../ApplicationRegisterSerialWorker.java | 58 +++++++++++++++++++
.../application/ApplicationRegisterTable.java | 10 ++++
.../worker/segment/SegmentParse.java | 8 +++
.../src/main/proto/ApplicationRegister.proto | 8 +++
.../resources/META-INF/defines/data.define | 3 +-
.../local_async_worker_provider.define | 4 +-
.../defines/remote_worker_provider.define | 1 +
.../apm/collector/core/framework/Loader.java | 4 +-
.../core/module/ModuleConfigLoader.java | 5 +-
.../module/ModuleConfigLoaderException.java | 16 -----
.../core/module/ModuleDefineException.java | 16 +++++
.../core/module/ModuleDefineLoader.java | 4 +-
.../core/module/ModuleGroupDefineLoader.java | 4 +-
.../core/storage/StorageDefineLoader.java | 4 +-
.../core/storage/StorageInstaller.java | 4 +-
.../config/ModuleConfigLoaderTestCase.java | 5 +-
.../storage/StorageModuleDefine.java | 3 +
.../apm/collector/storage/dao/DAO.java | 18 ++++++
.../StorageElasticSearchModuleDefine.java | 12 ++++
.../storage/elasticsearch/dao/EsDAO.java | 10 ++++
.../elasticsearch/dao/EsDAODefineLoader.java | 30 ++++++++++
.../dao/EsDAODefinitionFile.java | 13 +++++
.../storage/h2/StorageH2ModuleDefine.java | 12 ++++
.../apm/collector/storage/h2/dao/H2DAO.java | 10 ++++
.../storage/h2/dao/H2DAODefineLoader.java | 30 ++++++++++
.../storage/h2/dao/H2DAODefinitionFile.java | 13 +++++
.../stream/StreamModuleInstaller.java | 13 ++---
.../LocalAsyncWorkerProviderDefineLoader.java | 4 +-
.../RemoteWorkerProviderDefineLoader.java | 4 +-
.../worker/impl/data/AttributeType.java | 2 +-
.../worker/impl/data/DataDefineLoader.java | 4 +-
.../worker/selector/ForeverFirstSelector.java | 14 +++++
41 files changed, 474 insertions(+), 51 deletions(-)
create mode 100644 apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationEsDAO.java
create mode 100644 apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationH2DAO.java
create mode 100644 apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDGetOrCreate.java
create mode 100644 apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/IApplicationDAO.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterDataDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterRemoteWorker.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterTable.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/proto/ApplicationRegister.proto
delete mode 100644 apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleConfigLoaderException.java
create mode 100644 apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleDefineException.java
create mode 100644 apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/DAO.java
create mode 100644 apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAO.java
create mode 100644 apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAODefineLoader.java
create mode 100644 apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAODefinitionFile.java
create mode 100644 apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAO.java
create mode 100644 apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAODefineLoader.java
create mode 100644 apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAODefinitionFile.java
create mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/ForeverFirstSelector.java
diff --git a/apm-collector/apm-collector-agentregister/pom.xml b/apm-collector/apm-collector-agentregister/pom.xml
index e704f264e..258ffa456 100644
--- a/apm-collector/apm-collector-agentregister/pom.xml
+++ b/apm-collector/apm-collector-agentregister/pom.xml
@@ -28,5 +28,10 @@
apm-network
${project.version}
+
+ org.skywalking
+ apm-collector-agentstream
+ ${project.version}
+
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationEsDAO.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationEsDAO.java
new file mode 100644
index 000000000..893c1f22a
--- /dev/null
+++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationEsDAO.java
@@ -0,0 +1,15 @@
+package org.skywalking.apm.collector.agentregister.application;
+
+import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient;
+import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO;
+
+/**
+ * @author pengys5
+ */
+public class ApplicationEsDAO extends EsDAO implements IApplicationDAO {
+
+ @Override public int getApplicationId(String applicationCode) {
+ ElasticSearchClient client = getClient();
+ return 0;
+ }
+}
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationH2DAO.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationH2DAO.java
new file mode 100644
index 000000000..15faea45b
--- /dev/null
+++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationH2DAO.java
@@ -0,0 +1,15 @@
+package org.skywalking.apm.collector.agentregister.application;
+
+import org.skywalking.apm.collector.client.h2.H2Client;
+import org.skywalking.apm.collector.storage.h2.dao.H2DAO;
+
+/**
+ * @author pengys5
+ */
+public class ApplicationH2DAO extends H2DAO implements IApplicationDAO {
+
+ @Override public int getApplicationId(String applicationCode) {
+ H2Client client = getClient();
+ return 0;
+ }
+}
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDGetOrCreate.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDGetOrCreate.java
new file mode 100644
index 000000000..777cc72c8
--- /dev/null
+++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDGetOrCreate.java
@@ -0,0 +1,13 @@
+package org.skywalking.apm.collector.agentregister.application;
+
+/**
+ * @author pengys5
+ */
+public class ApplicationIDGetOrCreate {
+
+ private IApplicationDAO applicationDAO;
+
+ public int getOrCreate(String applicationCode) {
+ return 0;
+ }
+}
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/IApplicationDAO.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/IApplicationDAO.java
new file mode 100644
index 000000000..38b78ab5e
--- /dev/null
+++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/IApplicationDAO.java
@@ -0,0 +1,8 @@
+package org.skywalking.apm.collector.agentregister.application;
+
+/**
+ * @author pengys5
+ */
+public interface IApplicationDAO {
+ int getApplicationId(String applicationCode);
+}
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandler.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandler.java
index 705c2b74e..76878fae7 100644
--- a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandler.java
+++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandler.java
@@ -1,16 +1,31 @@
package org.skywalking.apm.collector.agentregister.grpc.handler;
+import com.google.protobuf.ProtocolStringList;
import io.grpc.stub.StreamObserver;
+import org.skywalking.apm.collector.agentregister.application.ApplicationIDGetOrCreate;
import org.skywalking.apm.collector.server.grpc.GRPCHandler;
import org.skywalking.apm.network.proto.Application;
import org.skywalking.apm.network.proto.ApplicationMapping;
import org.skywalking.apm.network.proto.ApplicationRegisterServiceGrpc;
+import org.skywalking.apm.network.proto.KeyWithIntegerValue;
/**
* @author pengys5
*/
public class ApplicationRegisterServiceHandler extends ApplicationRegisterServiceGrpc.ApplicationRegisterServiceImplBase implements GRPCHandler {
- @Override public void register(Application request, StreamObserver responseObserver) {
+ private ApplicationIDGetOrCreate applicationIDGetOrCreate = new ApplicationIDGetOrCreate();
+
+ @Override public void register(Application request, StreamObserver responseObserver) {
+ ProtocolStringList applicationCodes = request.getApplicationCodeList();
+ for (int i = 0; i < applicationCodes.size(); i++) {
+ String applicationCode = applicationCodes.get(i);
+ int applicationId = applicationIDGetOrCreate.getOrCreate(applicationCode);
+
+ KeyWithIntegerValue value = KeyWithIntegerValue.newBuilder().setKey(applicationCode).setValue(applicationId).build();
+ ApplicationMapping mapping = ApplicationMapping.newBuilder().setApplication(i, value).build();
+ responseObserver.onNext(mapping);
+ }
+ responseObserver.onCompleted();
}
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandler.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandler.java
index 6cffd5772..d778e921e 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandler.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandler.java
@@ -1,9 +1,13 @@
package org.skywalking.apm.collector.agentstream.grpc.handler;
+import com.google.protobuf.InvalidProtocolBufferException;
import io.grpc.stub.StreamObserver;
+import java.util.List;
import org.skywalking.apm.collector.server.grpc.GRPCHandler;
import org.skywalking.apm.network.proto.Downstream;
+import org.skywalking.apm.network.proto.TraceSegmentObject;
import org.skywalking.apm.network.proto.TraceSegmentServiceGrpc;
+import org.skywalking.apm.network.proto.UniqueId;
import org.skywalking.apm.network.proto.UpstreamSegment;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -18,6 +22,12 @@ public class TraceSegmentServiceHandler extends TraceSegmentServiceGrpc.TraceSeg
@Override public StreamObserver collect(StreamObserver responseObserver) {
return new StreamObserver() {
@Override public void onNext(UpstreamSegment segment) {
+ try {
+ List traceIds = segment.getGlobalTraceIdsList();
+ TraceSegmentObject segmentObject = TraceSegmentObject.parseFrom(segment.getSegment());
+ } catch (InvalidProtocolBufferException e) {
+ logger.error(e.getMessage(), e);
+ }
}
@Override public void onError(Throwable throwable) {
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterDataDefine.java
new file mode 100644
index 000000000..2106c37ae
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterDataDefine.java
@@ -0,0 +1,38 @@
+package org.skywalking.apm.collector.agentstream.worker.register.application;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.InvalidProtocolBufferException;
+import org.skywalking.apm.collector.agentstream.worker.register.application.proto.ApplicationRegisterOuterClass;
+import org.skywalking.apm.collector.stream.worker.impl.data.Attribute;
+import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType;
+import org.skywalking.apm.collector.stream.worker.impl.data.Data;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
+import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation;
+import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation;
+
+/**
+ * @author pengys5
+ */
+public class ApplicationRegisterDataDefine extends DataDefine {
+
+ @Override protected int defineId() {
+ return 101;
+ }
+
+ @Override protected int initialCapacity() {
+ return 3;
+ }
+
+ @Override protected void attributeDefine() {
+ addAttribute(0, new Attribute("id", AttributeType.STRING, new NonOperation()));
+ addAttribute(1, new Attribute(ApplicationRegisterTable.COLUMN_APPLICATION_CODE, AttributeType.STRING, new CoverOperation()));
+ addAttribute(2, new Attribute(ApplicationRegisterTable.COLUMN_APPLICATION_ID, AttributeType.INTEGER, new CoverOperation()));
+ }
+
+ @Override public Data parseFrom(ByteString bytesData) throws InvalidProtocolBufferException {
+ ApplicationRegisterOuterClass.ApplicationRegister applicationRegister = ApplicationRegisterOuterClass.ApplicationRegister.parseFrom(bytesData);
+ Data data = build();
+ data.setDataString(1, applicationRegister.getApplicationCode());
+ return data;
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterRemoteWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterRemoteWorker.java
new file mode 100644
index 000000000..d6e2327f5
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterRemoteWorker.java
@@ -0,0 +1,58 @@
+package org.skywalking.apm.collector.agentstream.worker.register.application;
+
+import org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentAggWorker;
+import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorker;
+import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorkerProvider;
+import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
+import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
+import org.skywalking.apm.collector.stream.worker.Role;
+import org.skywalking.apm.collector.stream.worker.WorkerException;
+import org.skywalking.apm.collector.stream.worker.selector.ForeverFirstSelector;
+import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector;
+
+/**
+ * @author pengys5
+ */
+public class ApplicationRegisterRemoteWorker extends AbstractRemoteWorker {
+
+ protected ApplicationRegisterRemoteWorker(Role role, ClusterWorkerContext clusterContext) {
+ super(role, clusterContext);
+ }
+
+ @Override public void preStart() throws ProviderNotFoundException {
+ }
+
+ @Override protected void onWork(Object message) throws WorkerException {
+
+ }
+
+ public static class Factory extends AbstractRemoteWorkerProvider {
+ @Override
+ public Role role() {
+ return WorkerRole.INSTANCE;
+ }
+
+ @Override
+ public ApplicationRegisterRemoteWorker workerInstance(ClusterWorkerContext clusterContext) {
+ return new ApplicationRegisterRemoteWorker(role(), clusterContext);
+ }
+
+ @Override public int workerNum() {
+ return 1;
+ }
+ }
+
+ public enum WorkerRole implements Role {
+ INSTANCE;
+
+ @Override
+ public String roleName() {
+ return NodeComponentAggWorker.class.getSimpleName();
+ }
+
+ @Override
+ public WorkerSelector workerSelector() {
+ return new ForeverFirstSelector();
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java
new file mode 100644
index 000000000..7f14301ea
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java
@@ -0,0 +1,58 @@
+package org.skywalking.apm.collector.agentstream.worker.register.application;
+
+import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorker;
+import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider;
+import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
+import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
+import org.skywalking.apm.collector.stream.worker.Role;
+import org.skywalking.apm.collector.stream.worker.WorkerException;
+import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector;
+import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector;
+
+/**
+ * @author pengys5
+ */
+public class ApplicationRegisterSerialWorker extends AbstractLocalAsyncWorker {
+
+ public ApplicationRegisterSerialWorker(Role role, ClusterWorkerContext clusterContext) {
+ super(role, clusterContext);
+ }
+
+ @Override public void preStart() throws ProviderNotFoundException {
+ super.preStart();
+ }
+
+ @Override protected void onWork(Object message) throws WorkerException {
+
+ }
+
+ public static class Factory extends AbstractLocalAsyncWorkerProvider {
+ @Override
+ public Role role() {
+ return WorkerRole.INSTANCE;
+ }
+
+ @Override
+ public ApplicationRegisterSerialWorker workerInstance(ClusterWorkerContext clusterContext) {
+ return new ApplicationRegisterSerialWorker(role(), clusterContext);
+ }
+
+ @Override public int queueSize() {
+ return 256;
+ }
+ }
+
+ public enum WorkerRole implements Role {
+ INSTANCE;
+
+ @Override
+ public String roleName() {
+ return ApplicationRegisterSerialWorker.class.getSimpleName();
+ }
+
+ @Override
+ public WorkerSelector workerSelector() {
+ return new HashCodeSelector();
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterTable.java
new file mode 100644
index 000000000..1a053f4e8
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterTable.java
@@ -0,0 +1,10 @@
+package org.skywalking.apm.collector.agentstream.worker.register.application;
+
+/**
+ * @author pengys5
+ */
+public class ApplicationRegisterTable {
+ public static final String TABLE = "application_register";
+ public static final String COLUMN_APPLICATION_CODE = "application_code";
+ public static final String COLUMN_APPLICATION_ID = "application_id";
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java
new file mode 100644
index 000000000..09a90bfd9
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java
@@ -0,0 +1,8 @@
+package org.skywalking.apm.collector.agentstream.worker.segment;
+
+/**
+ * @author pengys5
+ */
+public class SegmentParse {
+
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/proto/ApplicationRegister.proto b/apm-collector/apm-collector-agentstream/src/main/proto/ApplicationRegister.proto
new file mode 100644
index 000000000..572cfd362
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/proto/ApplicationRegister.proto
@@ -0,0 +1,8 @@
+syntax = "proto3";
+
+option java_multiple_files = false;
+option java_package = "org.skywalking.apm.collector.agentstream.worker.register.application.proto";
+
+message ApplicationRegister {
+ string application_code = 1;
+}
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define
index d87c1fdd9..a7a420f19 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define
@@ -1 +1,2 @@
-org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentDataDefine
\ No newline at end of file
+org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentDataDefine
+org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterDataDefine
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define
index 33897fbdc..418ecfce1 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define
@@ -1 +1,3 @@
-org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentAggWorker$Factory
\ No newline at end of file
+org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentAggWorker$Factory
+
+org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterSerialWorker$Factory
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define
index e69de29bb..84c9c058e 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define
@@ -0,0 +1 @@
+org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterRemoteWorker$Factory
\ No newline at end of file
diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Loader.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Loader.java
index 10c4be934..485925c4c 100644
--- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Loader.java
+++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Loader.java
@@ -1,10 +1,8 @@
package org.skywalking.apm.collector.core.framework;
-import org.skywalking.apm.collector.core.config.ConfigException;
-
/**
* @author pengys5
*/
public interface Loader {
- T load() throws ConfigException;
+ T load() throws DefineException;
}
diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleConfigLoader.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleConfigLoader.java
index 8e89ef152..2dee03f20 100644
--- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleConfigLoader.java
+++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleConfigLoader.java
@@ -4,6 +4,7 @@ import java.io.FileNotFoundException;
import java.io.FileReader;
import java.util.Map;
import org.skywalking.apm.collector.core.config.ConfigLoader;
+import org.skywalking.apm.collector.core.framework.DefineException;
import org.skywalking.apm.collector.core.util.ResourceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -16,7 +17,7 @@ public class ModuleConfigLoader implements ConfigLoader> {
private final Logger logger = LoggerFactory.getLogger(ModuleConfigLoader.class);
- @Override public Map load() throws ModuleConfigLoaderException {
+ @Override public Map load() throws DefineException {
Yaml yaml = new Yaml();
try {
try {
@@ -27,7 +28,7 @@ public class ModuleConfigLoader implements ConfigLoader> {
return (Map)yaml.load(ResourceUtils.read("application-default.yml"));
}
} catch (FileNotFoundException e) {
- throw new ModuleConfigLoaderException(e.getMessage(), e);
+ throw new ModuleDefineException(e.getMessage(), e);
}
}
}
diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleConfigLoaderException.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleConfigLoaderException.java
deleted file mode 100644
index 9143d0197..000000000
--- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleConfigLoaderException.java
+++ /dev/null
@@ -1,16 +0,0 @@
-package org.skywalking.apm.collector.core.module;
-
-import org.skywalking.apm.collector.core.config.ConfigLoaderException;
-
-/**
- * @author pengys5
- */
-public class ModuleConfigLoaderException extends ConfigLoaderException {
- public ModuleConfigLoaderException(String message) {
- super(message);
- }
-
- public ModuleConfigLoaderException(String message, Throwable cause) {
- super(message, cause);
- }
-}
diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleDefineException.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleDefineException.java
new file mode 100644
index 000000000..773f465ec
--- /dev/null
+++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleDefineException.java
@@ -0,0 +1,16 @@
+package org.skywalking.apm.collector.core.module;
+
+import org.skywalking.apm.collector.core.framework.DefineException;
+
+/**
+ * @author pengys5
+ */
+public class ModuleDefineException extends DefineException {
+ public ModuleDefineException(String message) {
+ super(message);
+ }
+
+ public ModuleDefineException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleDefineLoader.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleDefineLoader.java
index 67a3adb60..5808ce0df 100644
--- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleDefineLoader.java
+++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleDefineLoader.java
@@ -2,7 +2,7 @@ package org.skywalking.apm.collector.core.module;
import java.util.LinkedHashMap;
import java.util.Map;
-import org.skywalking.apm.collector.core.config.ConfigException;
+import org.skywalking.apm.collector.core.framework.DefineException;
import org.skywalking.apm.collector.core.framework.Loader;
import org.skywalking.apm.collector.core.util.DefinitionLoader;
import org.slf4j.Logger;
@@ -15,7 +15,7 @@ public class ModuleDefineLoader implements Loader> load() throws ConfigException {
+ @Override public Map> load() throws DefineException {
Map> moduleDefineMap = new LinkedHashMap<>();
ModuleDefinitionFile definitionFile = new ModuleDefinitionFile();
diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleGroupDefineLoader.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleGroupDefineLoader.java
index 6f4cb58aa..6a4e1387c 100644
--- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleGroupDefineLoader.java
+++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleGroupDefineLoader.java
@@ -2,7 +2,7 @@ package org.skywalking.apm.collector.core.module;
import java.util.LinkedHashMap;
import java.util.Map;
-import org.skywalking.apm.collector.core.config.ConfigException;
+import org.skywalking.apm.collector.core.framework.DefineException;
import org.skywalking.apm.collector.core.framework.Loader;
import org.skywalking.apm.collector.core.util.DefinitionLoader;
import org.slf4j.Logger;
@@ -15,7 +15,7 @@ public class ModuleGroupDefineLoader implements Loader load() throws ConfigException {
+ @Override public Map load() throws DefineException {
Map moduleGroupDefineMap = new LinkedHashMap<>();
ModuleGroupDefineFile definitionFile = new ModuleGroupDefineFile();
diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageDefineLoader.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageDefineLoader.java
index 5eb363b4d..618ed1c6a 100644
--- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageDefineLoader.java
+++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageDefineLoader.java
@@ -2,7 +2,7 @@ package org.skywalking.apm.collector.core.storage;
import java.util.LinkedList;
import java.util.List;
-import org.skywalking.apm.collector.core.config.ConfigException;
+import org.skywalking.apm.collector.core.framework.DefineException;
import org.skywalking.apm.collector.core.framework.Loader;
import org.skywalking.apm.collector.core.util.DefinitionLoader;
import org.slf4j.Logger;
@@ -15,7 +15,7 @@ public class StorageDefineLoader implements Loader> {
private final Logger logger = LoggerFactory.getLogger(StorageDefineLoader.class);
- @Override public List load() throws ConfigException {
+ @Override public List load() throws DefineException {
List tableDefines = new LinkedList<>();
StorageDefinitionFile definitionFile = new StorageDefinitionFile();
diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstaller.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstaller.java
index b6c79b927..beff974ac 100644
--- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstaller.java
+++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstaller.java
@@ -2,7 +2,7 @@ package org.skywalking.apm.collector.core.storage;
import java.util.List;
import org.skywalking.apm.collector.core.client.Client;
-import org.skywalking.apm.collector.core.config.ConfigException;
+import org.skywalking.apm.collector.core.framework.DefineException;
/**
* @author pengys5
@@ -22,7 +22,7 @@ public abstract class StorageInstaller {
createTable(client, tableDefine);
}
}
- } catch (ConfigException e) {
+ } catch (DefineException e) {
throw new StorageInstallException(e.getMessage(), e);
}
}
diff --git a/apm-collector/apm-collector-core/src/test/java/org/skywalking/apm/collector/core/config/ModuleConfigLoaderTestCase.java b/apm-collector/apm-collector-core/src/test/java/org/skywalking/apm/collector/core/config/ModuleConfigLoaderTestCase.java
index 85f802f23..734a407b6 100644
--- a/apm-collector/apm-collector-core/src/test/java/org/skywalking/apm/collector/core/config/ModuleConfigLoaderTestCase.java
+++ b/apm-collector/apm-collector-core/src/test/java/org/skywalking/apm/collector/core/config/ModuleConfigLoaderTestCase.java
@@ -1,9 +1,8 @@
package org.skywalking.apm.collector.core.config;
-import java.io.FileNotFoundException;
import org.junit.Test;
import org.skywalking.apm.collector.core.module.ModuleConfigLoader;
-import org.skywalking.apm.collector.core.module.ModuleConfigLoaderException;
+import org.skywalking.apm.collector.core.module.ModuleDefineException;
/**
* @author pengys5
@@ -11,7 +10,7 @@ import org.skywalking.apm.collector.core.module.ModuleConfigLoaderException;
public class ModuleConfigLoaderTestCase {
@Test
- public void testLoad() throws ModuleConfigLoaderException {
+ public void testLoad() throws ModuleDefineException {
ModuleConfigLoader loader = new ModuleConfigLoader();
loader.load();
}
diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/StorageModuleDefine.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/StorageModuleDefine.java
index 8f3258749..57ccb77e9 100644
--- a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/StorageModuleDefine.java
+++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/StorageModuleDefine.java
@@ -33,6 +33,7 @@ public abstract class StorageModuleDefine extends ModuleDefine implements Cluste
Client client = createClient(null);
client.initialize();
context.setClient(client);
+ injectClientIntoDAO(client);
storageInstaller().install(client);
} catch (ConfigParseException | StorageException e) {
@@ -57,4 +58,6 @@ public abstract class StorageModuleDefine extends ModuleDefine implements Cluste
}
public abstract StorageInstaller storageInstaller();
+
+ public abstract void injectClientIntoDAO(Client client) throws DefineException;
}
diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/DAO.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/DAO.java
new file mode 100644
index 000000000..3e2d14c36
--- /dev/null
+++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/DAO.java
@@ -0,0 +1,18 @@
+package org.skywalking.apm.collector.storage.dao;
+
+import org.skywalking.apm.collector.core.client.Client;
+
+/**
+ * @author pengys5
+ */
+public abstract class DAO {
+ private C client;
+
+ public C getClient() {
+ return client;
+ }
+
+ public void setClient(C client) {
+ this.client = client;
+ }
+}
diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/StorageElasticSearchModuleDefine.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/StorageElasticSearchModuleDefine.java
index 7d2d9e0c7..2c21f7107 100644
--- a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/StorageElasticSearchModuleDefine.java
+++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/StorageElasticSearchModuleDefine.java
@@ -1,12 +1,16 @@
package org.skywalking.apm.collector.storage.elasticsearch;
+import java.util.List;
import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient;
import org.skywalking.apm.collector.core.client.Client;
import org.skywalking.apm.collector.core.client.DataMonitor;
+import org.skywalking.apm.collector.core.framework.DefineException;
import org.skywalking.apm.collector.core.module.ModuleConfigParser;
import org.skywalking.apm.collector.core.storage.StorageInstaller;
import org.skywalking.apm.collector.storage.StorageModuleDefine;
import org.skywalking.apm.collector.storage.StorageModuleGroupDefine;
+import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO;
+import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAODefineLoader;
import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchStorageInstaller;
/**
@@ -35,4 +39,12 @@ public class StorageElasticSearchModuleDefine extends StorageModuleDefine {
@Override public StorageInstaller storageInstaller() {
return new ElasticSearchStorageInstaller();
}
+
+ @Override public void injectClientIntoDAO(Client client) throws DefineException {
+ EsDAODefineLoader loader = new EsDAODefineLoader();
+ List esDAOs = loader.load();
+ esDAOs.forEach(esDAO -> {
+ esDAO.setClient((ElasticSearchClient)client);
+ });
+ }
}
diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAO.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAO.java
new file mode 100644
index 000000000..a1a15ac0f
--- /dev/null
+++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAO.java
@@ -0,0 +1,10 @@
+package org.skywalking.apm.collector.storage.elasticsearch.dao;
+
+import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient;
+import org.skywalking.apm.collector.storage.dao.DAO;
+
+/**
+ * @author pengys5
+ */
+public abstract class EsDAO extends DAO {
+}
diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAODefineLoader.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAODefineLoader.java
new file mode 100644
index 000000000..de5213bff
--- /dev/null
+++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAODefineLoader.java
@@ -0,0 +1,30 @@
+package org.skywalking.apm.collector.storage.elasticsearch.dao;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.skywalking.apm.collector.core.framework.DefineException;
+import org.skywalking.apm.collector.core.framework.Loader;
+import org.skywalking.apm.collector.core.util.DefinitionLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class EsDAODefineLoader implements Loader> {
+
+ private final Logger logger = LoggerFactory.getLogger(EsDAODefineLoader.class);
+
+ @Override public List load() throws DefineException {
+ List esDAOs = new ArrayList<>();
+
+ EsDAODefinitionFile definitionFile = new EsDAODefinitionFile();
+ logger.info("elasticsearch dao definition file name: {}", definitionFile.fileName());
+ DefinitionLoader definitionLoader = DefinitionLoader.load(EsDAO.class, definitionFile);
+ for (EsDAO dao : definitionLoader) {
+ logger.info("loaded elasticsearch dao definition class: {}", dao.getClass().getName());
+ esDAOs.add(dao);
+ }
+ return esDAOs;
+ }
+}
diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAODefinitionFile.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAODefinitionFile.java
new file mode 100644
index 000000000..08791d866
--- /dev/null
+++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAODefinitionFile.java
@@ -0,0 +1,13 @@
+package org.skywalking.apm.collector.storage.elasticsearch.dao;
+
+import org.skywalking.apm.collector.core.framework.DefinitionFile;
+
+/**
+ * @author pengys5
+ */
+public class EsDAODefinitionFile extends DefinitionFile {
+
+ @Override protected String fileName() {
+ return "es_dao.define";
+ }
+}
diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/StorageH2ModuleDefine.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/StorageH2ModuleDefine.java
index ac5da7839..86c3e84b3 100644
--- a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/StorageH2ModuleDefine.java
+++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/StorageH2ModuleDefine.java
@@ -1,12 +1,16 @@
package org.skywalking.apm.collector.storage.h2;
+import java.util.List;
import org.skywalking.apm.collector.client.h2.H2Client;
import org.skywalking.apm.collector.core.client.Client;
import org.skywalking.apm.collector.core.client.DataMonitor;
+import org.skywalking.apm.collector.core.framework.DefineException;
import org.skywalking.apm.collector.core.module.ModuleConfigParser;
import org.skywalking.apm.collector.core.storage.StorageInstaller;
import org.skywalking.apm.collector.storage.StorageModuleDefine;
import org.skywalking.apm.collector.storage.StorageModuleGroupDefine;
+import org.skywalking.apm.collector.storage.h2.dao.H2DAO;
+import org.skywalking.apm.collector.storage.h2.dao.H2DAODefineLoader;
import org.skywalking.apm.collector.storage.h2.define.H2StorageInstaller;
/**
@@ -35,4 +39,12 @@ public class StorageH2ModuleDefine extends StorageModuleDefine {
@Override public StorageInstaller storageInstaller() {
return new H2StorageInstaller();
}
+
+ @Override public void injectClientIntoDAO(Client client) throws DefineException {
+ H2DAODefineLoader loader = new H2DAODefineLoader();
+ List h2DAOs = loader.load();
+ h2DAOs.forEach(h2DAO -> {
+ h2DAO.setClient((H2Client)client);
+ });
+ }
}
diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAO.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAO.java
new file mode 100644
index 000000000..a9c9ddb5a
--- /dev/null
+++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAO.java
@@ -0,0 +1,10 @@
+package org.skywalking.apm.collector.storage.h2.dao;
+
+import org.skywalking.apm.collector.client.h2.H2Client;
+import org.skywalking.apm.collector.storage.dao.DAO;
+
+/**
+ * @author pengys5
+ */
+public abstract class H2DAO extends DAO {
+}
diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAODefineLoader.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAODefineLoader.java
new file mode 100644
index 000000000..733eb3bdd
--- /dev/null
+++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAODefineLoader.java
@@ -0,0 +1,30 @@
+package org.skywalking.apm.collector.storage.h2.dao;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.skywalking.apm.collector.core.framework.DefineException;
+import org.skywalking.apm.collector.core.framework.Loader;
+import org.skywalking.apm.collector.core.util.DefinitionLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class H2DAODefineLoader implements Loader> {
+
+ private final Logger logger = LoggerFactory.getLogger(H2DAODefineLoader.class);
+
+ @Override public List load() throws DefineException {
+ List h2DAOs = new ArrayList<>();
+
+ H2DAODefinitionFile definitionFile = new H2DAODefinitionFile();
+ logger.info("h2 dao definition file name: {}", definitionFile.fileName());
+ DefinitionLoader definitionLoader = DefinitionLoader.load(H2DAO.class, definitionFile);
+ for (H2DAO dao : definitionLoader) {
+ logger.info("loaded h2 dao definition class: {}", dao.getClass().getName());
+ h2DAOs.add(dao);
+ }
+ return h2DAOs;
+ }
+}
diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAODefinitionFile.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAODefinitionFile.java
new file mode 100644
index 000000000..72d37883f
--- /dev/null
+++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAODefinitionFile.java
@@ -0,0 +1,13 @@
+package org.skywalking.apm.collector.storage.h2.dao;
+
+import org.skywalking.apm.collector.core.framework.DefinitionFile;
+
+/**
+ * @author pengys5
+ */
+public class H2DAODefinitionFile extends DefinitionFile {
+
+ @Override protected String fileName() {
+ return "h2_dao.define";
+ }
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java
index 4328ff67b..f834edb20 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java
@@ -4,7 +4,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.skywalking.apm.collector.core.client.ClientException;
-import org.skywalking.apm.collector.core.config.ConfigException;
import org.skywalking.apm.collector.core.framework.CollectorContextHelper;
import org.skywalking.apm.collector.core.framework.DefineException;
import org.skywalking.apm.collector.core.module.ModuleDefine;
@@ -36,12 +35,8 @@ public class StreamModuleInstaller implements ModuleInstaller {
CollectorContextHelper.INSTANCE.putContext(context);
DataDefineLoader dataDefineLoader = new DataDefineLoader();
- try {
- Map dataDefineMap = dataDefineLoader.load();
- context.putAllDataDefine(dataDefineMap);
- } catch (ConfigException e) {
- logger.error(e.getMessage(), e);
- }
+ Map dataDefineMap = dataDefineLoader.load();
+ context.putAllDataDefine(dataDefineMap);
initializeWorker(context);
@@ -54,7 +49,7 @@ public class StreamModuleInstaller implements ModuleInstaller {
}
}
- private void initializeWorker(StreamModuleContext context) {
+ private void initializeWorker(StreamModuleContext context) throws DefineException {
ClusterWorkerContext clusterWorkerContext = new ClusterWorkerContext();
context.setClusterWorkerContext(clusterWorkerContext);
@@ -74,7 +69,7 @@ public class StreamModuleInstaller implements ModuleInstaller {
provider.create();
clusterWorkerContext.putRole(provider.role());
}
- } catch (ConfigException | ProviderNotFoundException e) {
+ } catch (ProviderNotFoundException e) {
logger.error(e.getMessage(), e);
}
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefineLoader.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefineLoader.java
index 0d6dfb566..3eeb4de9e 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefineLoader.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefineLoader.java
@@ -2,7 +2,7 @@ package org.skywalking.apm.collector.stream.worker;
import java.util.ArrayList;
import java.util.List;
-import org.skywalking.apm.collector.core.config.ConfigException;
+import org.skywalking.apm.collector.core.framework.DefineException;
import org.skywalking.apm.collector.core.framework.Loader;
import org.skywalking.apm.collector.core.util.DefinitionLoader;
import org.slf4j.Logger;
@@ -15,7 +15,7 @@ public class LocalAsyncWorkerProviderDefineLoader implements Loader load() throws ConfigException {
+ @Override public List load() throws DefineException {
List providers = new ArrayList<>();
LocalAsyncWorkerProviderDefinitionFile definitionFile = new LocalAsyncWorkerProviderDefinitionFile();
logger.info("local async worker provider definition file name: {}", definitionFile.fileName());
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerProviderDefineLoader.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerProviderDefineLoader.java
index 383f63d31..001370c4d 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerProviderDefineLoader.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerProviderDefineLoader.java
@@ -2,7 +2,7 @@ package org.skywalking.apm.collector.stream.worker;
import java.util.ArrayList;
import java.util.List;
-import org.skywalking.apm.collector.core.config.ConfigException;
+import org.skywalking.apm.collector.core.framework.DefineException;
import org.skywalking.apm.collector.core.framework.Loader;
import org.skywalking.apm.collector.core.util.DefinitionLoader;
import org.slf4j.Logger;
@@ -15,7 +15,7 @@ public class RemoteWorkerProviderDefineLoader implements Loader load() throws ConfigException {
+ @Override public List load() throws DefineException {
List providers = new ArrayList<>();
RemoteWorkerProviderDefinitionFile definitionFile = new RemoteWorkerProviderDefinitionFile();
logger.info("remote worker provider definition file name: {}", definitionFile.fileName());
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/AttributeType.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/AttributeType.java
index 3706017b6..3b2856bd9 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/AttributeType.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/AttributeType.java
@@ -4,5 +4,5 @@ package org.skywalking.apm.collector.stream.worker.impl.data;
* @author pengys5
*/
public enum AttributeType {
- STRING, LONG, FLOAT
+ STRING, LONG, FLOAT, INTEGER
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java
index 67e2cff5d..425fcb427 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java
@@ -2,7 +2,7 @@ package org.skywalking.apm.collector.stream.worker.impl.data;
import java.util.HashMap;
import java.util.Map;
-import org.skywalking.apm.collector.core.config.ConfigException;
+import org.skywalking.apm.collector.core.framework.DefineException;
import org.skywalking.apm.collector.core.framework.Loader;
import org.skywalking.apm.collector.core.util.DefinitionLoader;
import org.slf4j.Logger;
@@ -15,7 +15,7 @@ public class DataDefineLoader implements Loader> {
private final Logger logger = LoggerFactory.getLogger(DataDefineLoader.class);
- @Override public Map load() throws ConfigException {
+ @Override public Map load() throws DefineException {
Map dataDefineMap = new HashMap<>();
DataDefinitionFile definitionFile = new DataDefinitionFile();
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/ForeverFirstSelector.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/ForeverFirstSelector.java
new file mode 100644
index 000000000..9a668d791
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/ForeverFirstSelector.java
@@ -0,0 +1,14 @@
+package org.skywalking.apm.collector.stream.worker.selector;
+
+import java.util.List;
+import org.skywalking.apm.collector.stream.worker.WorkerRef;
+
+/**
+ * @author pengys5
+ */
+public class ForeverFirstSelector implements WorkerSelector {
+
+ @Override public WorkerRef select(List members, Object message) {
+ return members.get(0);
+ }
+}
From e4f56ab1c70197223317c75845e73841bd094d0a Mon Sep 17 00:00:00 2001
From: pengys5 <8082209@qq.com>
Date: Tue, 25 Jul 2017 16:40:43 +0800
Subject: [PATCH 11/28] add Dao container to cache Dao
---
.../application/ApplicationEsDAO.java | 16 ++++++++++++
.../application/ApplicationH2DAO.java | 2 +-
.../application/ApplicationIDGetOrCreate.java | 7 ++---
.../ApplicationRegisterServiceHandler.java | 7 ++++-
.../resources/META-INF/defines/es_dao.define | 1 +
.../resources/META-INF/defines/h2_dao.define | 1 +
...icationRegisterServiceHandlerTestCase.java | 26 +++++++++++++++++++
.../elasticsearch/ElasticSearchClient.java | 5 ++++
.../config/ModuleConfigLoaderTestCase.java | 17 ------------
.../apm/collector/server/grpc/GRPCServer.java | 2 +-
.../collector/storage/dao/DAOContainer.java | 21 +++++++++++++++
.../StorageElasticSearchModuleDefine.java | 3 +++
.../storage/h2/StorageH2ModuleDefine.java | 3 +++
13 files changed, 88 insertions(+), 23 deletions(-)
create mode 100644 apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/es_dao.define
create mode 100644 apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/h2_dao.define
create mode 100644 apm-collector/apm-collector-agentregister/src/test/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandlerTestCase.java
delete mode 100644 apm-collector/apm-collector-core/src/test/java/org/skywalking/apm/collector/core/config/ModuleConfigLoaderTestCase.java
create mode 100644 apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/DAOContainer.java
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationEsDAO.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationEsDAO.java
index 893c1f22a..80d1d2ec0 100644
--- a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationEsDAO.java
+++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationEsDAO.java
@@ -1,5 +1,10 @@
package org.skywalking.apm.collector.agentregister.application;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterTable;
import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient;
import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO;
@@ -10,6 +15,17 @@ public class ApplicationEsDAO extends EsDAO implements IApplicationDAO {
@Override public int getApplicationId(String applicationCode) {
ElasticSearchClient client = getClient();
+
+ SearchRequestBuilder searchRequestBuilder = client.prepareSearch(ApplicationRegisterTable.TABLE);
+ searchRequestBuilder.setTypes("type");
+ searchRequestBuilder.setSearchType(SearchType.QUERY_THEN_FETCH);
+ searchRequestBuilder.setQuery(QueryBuilders.termQuery(ApplicationRegisterTable.COLUMN_APPLICATION_CODE, applicationCode));
+ searchRequestBuilder.setSize(10);
+
+ SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
+ if (searchResponse.getHits().totalHits > 0) {
+ return searchResponse.getHits().getAt(0).getField(ApplicationRegisterTable.COLUMN_APPLICATION_ID).getValue();
+ }
return 0;
}
}
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationH2DAO.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationH2DAO.java
index 15faea45b..3bf09f6d8 100644
--- a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationH2DAO.java
+++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationH2DAO.java
@@ -10,6 +10,6 @@ public class ApplicationH2DAO extends H2DAO implements IApplicationDAO {
@Override public int getApplicationId(String applicationCode) {
H2Client client = getClient();
- return 0;
+ return 100;
}
}
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDGetOrCreate.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDGetOrCreate.java
index 777cc72c8..8a949235f 100644
--- a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDGetOrCreate.java
+++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDGetOrCreate.java
@@ -1,13 +1,14 @@
package org.skywalking.apm.collector.agentregister.application;
+import org.skywalking.apm.collector.storage.dao.DAOContainer;
+
/**
* @author pengys5
*/
public class ApplicationIDGetOrCreate {
- private IApplicationDAO applicationDAO;
-
public int getOrCreate(String applicationCode) {
- return 0;
+ IApplicationDAO dao = (IApplicationDAO)DAOContainer.INSTANCE.get(IApplicationDAO.class.getName());
+ return dao.getApplicationId(applicationCode);
}
}
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandler.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandler.java
index 76878fae7..cd7329895 100644
--- a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandler.java
+++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandler.java
@@ -8,22 +8,27 @@ import org.skywalking.apm.network.proto.Application;
import org.skywalking.apm.network.proto.ApplicationMapping;
import org.skywalking.apm.network.proto.ApplicationRegisterServiceGrpc;
import org.skywalking.apm.network.proto.KeyWithIntegerValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* @author pengys5
*/
public class ApplicationRegisterServiceHandler extends ApplicationRegisterServiceGrpc.ApplicationRegisterServiceImplBase implements GRPCHandler {
+ private final Logger logger = LoggerFactory.getLogger(ApplicationRegisterServiceHandler.class);
+
private ApplicationIDGetOrCreate applicationIDGetOrCreate = new ApplicationIDGetOrCreate();
@Override public void register(Application request, StreamObserver responseObserver) {
+ logger.debug("register application");
ProtocolStringList applicationCodes = request.getApplicationCodeList();
for (int i = 0; i < applicationCodes.size(); i++) {
String applicationCode = applicationCodes.get(i);
int applicationId = applicationIDGetOrCreate.getOrCreate(applicationCode);
KeyWithIntegerValue value = KeyWithIntegerValue.newBuilder().setKey(applicationCode).setValue(applicationId).build();
- ApplicationMapping mapping = ApplicationMapping.newBuilder().setApplication(i, value).build();
+ ApplicationMapping mapping = ApplicationMapping.newBuilder().addApplication(i, value).build();
responseObserver.onNext(mapping);
}
responseObserver.onCompleted();
diff --git a/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/es_dao.define b/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/es_dao.define
new file mode 100644
index 000000000..36d1e8c1a
--- /dev/null
+++ b/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/es_dao.define
@@ -0,0 +1 @@
+org.skywalking.apm.collector.agentregister.application.ApplicationEsDAO
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/h2_dao.define b/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/h2_dao.define
new file mode 100644
index 000000000..c6024dcf1
--- /dev/null
+++ b/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/h2_dao.define
@@ -0,0 +1 @@
+org.skywalking.apm.collector.agentregister.application.ApplicationH2DAO
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentregister/src/test/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandlerTestCase.java b/apm-collector/apm-collector-agentregister/src/test/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandlerTestCase.java
new file mode 100644
index 000000000..cb6bbb414
--- /dev/null
+++ b/apm-collector/apm-collector-agentregister/src/test/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandlerTestCase.java
@@ -0,0 +1,26 @@
+package org.skywalking.apm.collector.agentregister.grpc.handler;
+
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import org.junit.Test;
+import org.skywalking.apm.network.proto.Application;
+import org.skywalking.apm.network.proto.ApplicationMapping;
+import org.skywalking.apm.network.proto.ApplicationRegisterServiceGrpc;
+
+/**
+ * @author pengys5
+ */
+public class ApplicationRegisterServiceHandlerTestCase {
+
+ private ApplicationRegisterServiceGrpc.ApplicationRegisterServiceBlockingStub stub;
+
+ @Test
+ public void testRegister() {
+ ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 11800).usePlaintext(true).build();
+ stub = ApplicationRegisterServiceGrpc.newBlockingStub(channel);
+
+ Application application = Application.newBuilder().addApplicationCode("test").build();
+ ApplicationMapping mapping = stub.register(application);
+ System.out.println(mapping.getApplication(0).getKey() + ", " + mapping.getApplication(0).getValue());
+ }
+}
diff --git a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java
index 73865df2c..884907803 100644
--- a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java
+++ b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java
@@ -8,6 +8,7 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.client.IndicesAdminClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
@@ -101,6 +102,10 @@ public class ElasticSearchClient implements Client {
return response.isExists();
}
+ public SearchRequestBuilder prepareSearch(String indexName) {
+ return client.prepareSearch(indexName);
+ }
+
public IndexRequestBuilder prepareIndex(String indexName) {
return null;
}
diff --git a/apm-collector/apm-collector-core/src/test/java/org/skywalking/apm/collector/core/config/ModuleConfigLoaderTestCase.java b/apm-collector/apm-collector-core/src/test/java/org/skywalking/apm/collector/core/config/ModuleConfigLoaderTestCase.java
deleted file mode 100644
index 734a407b6..000000000
--- a/apm-collector/apm-collector-core/src/test/java/org/skywalking/apm/collector/core/config/ModuleConfigLoaderTestCase.java
+++ /dev/null
@@ -1,17 +0,0 @@
-package org.skywalking.apm.collector.core.config;
-
-import org.junit.Test;
-import org.skywalking.apm.collector.core.module.ModuleConfigLoader;
-import org.skywalking.apm.collector.core.module.ModuleDefineException;
-
-/**
- * @author pengys5
- */
-public class ModuleConfigLoaderTestCase {
-
- @Test
- public void testLoad() throws ModuleDefineException {
- ModuleConfigLoader loader = new ModuleConfigLoader();
- loader.load();
- }
-}
diff --git a/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/grpc/GRPCServer.java b/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/grpc/GRPCServer.java
index 592b44278..6f9472317 100644
--- a/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/grpc/GRPCServer.java
+++ b/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/grpc/GRPCServer.java
@@ -37,12 +37,12 @@ public class GRPCServer implements Server {
@Override public void initialize() throws ServerException {
InetSocketAddress address = new InetSocketAddress(host, port);
nettyServerBuilder = NettyServerBuilder.forAddress(address);
- server = nettyServerBuilder.build();
logger.info("Server started, host {} listening on {}", host, port);
}
@Override public void start() throws ServerException {
try {
+ server = nettyServerBuilder.build();
server.start();
} catch (IOException e) {
throw new GRPCServerException(e.getMessage(), e);
diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/DAOContainer.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/DAOContainer.java
new file mode 100644
index 000000000..a4f884239
--- /dev/null
+++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/DAOContainer.java
@@ -0,0 +1,21 @@
+package org.skywalking.apm.collector.storage.dao;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author pengys5
+ */
+public enum DAOContainer {
+ INSTANCE;
+
+ private Map daos = new HashMap<>();
+
+ public void put(String interfaceName, DAO dao) {
+ daos.put(interfaceName, dao);
+ }
+
+ public DAO get(String interfaceName) {
+ return daos.get(interfaceName);
+ }
+}
diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/StorageElasticSearchModuleDefine.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/StorageElasticSearchModuleDefine.java
index 2c21f7107..826ad4cb0 100644
--- a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/StorageElasticSearchModuleDefine.java
+++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/StorageElasticSearchModuleDefine.java
@@ -9,6 +9,7 @@ import org.skywalking.apm.collector.core.module.ModuleConfigParser;
import org.skywalking.apm.collector.core.storage.StorageInstaller;
import org.skywalking.apm.collector.storage.StorageModuleDefine;
import org.skywalking.apm.collector.storage.StorageModuleGroupDefine;
+import org.skywalking.apm.collector.storage.dao.DAOContainer;
import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO;
import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAODefineLoader;
import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchStorageInstaller;
@@ -45,6 +46,8 @@ public class StorageElasticSearchModuleDefine extends StorageModuleDefine {
List esDAOs = loader.load();
esDAOs.forEach(esDAO -> {
esDAO.setClient((ElasticSearchClient)client);
+ String interFaceName = esDAO.getClass().getInterfaces()[0].getName();
+ DAOContainer.INSTANCE.put(interFaceName, esDAO);
});
}
}
diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/StorageH2ModuleDefine.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/StorageH2ModuleDefine.java
index 86c3e84b3..84f9b31a7 100644
--- a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/StorageH2ModuleDefine.java
+++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/StorageH2ModuleDefine.java
@@ -9,6 +9,7 @@ import org.skywalking.apm.collector.core.module.ModuleConfigParser;
import org.skywalking.apm.collector.core.storage.StorageInstaller;
import org.skywalking.apm.collector.storage.StorageModuleDefine;
import org.skywalking.apm.collector.storage.StorageModuleGroupDefine;
+import org.skywalking.apm.collector.storage.dao.DAOContainer;
import org.skywalking.apm.collector.storage.h2.dao.H2DAO;
import org.skywalking.apm.collector.storage.h2.dao.H2DAODefineLoader;
import org.skywalking.apm.collector.storage.h2.define.H2StorageInstaller;
@@ -45,6 +46,8 @@ public class StorageH2ModuleDefine extends StorageModuleDefine {
List h2DAOs = loader.load();
h2DAOs.forEach(h2DAO -> {
h2DAO.setClient((H2Client)client);
+ String interFaceName = h2DAO.getClass().getInterfaces()[0].getName();
+ DAOContainer.INSTANCE.put(interFaceName, h2DAO);
});
}
}
From 1bc7250e2716c64db52a3222cc3f7bef427952b0 Mon Sep 17 00:00:00 2001
From: pengys5 <8082209@qq.com>
Date: Wed, 26 Jul 2017 18:45:14 +0800
Subject: [PATCH 12/28] Remote worker call is successful
---
.../grpc/AgentJVMGRPCDataListener.java | 4 ++
.../application/ApplicationEsDAO.java | 10 +--
.../application/ApplicationIDGetOrCreate.java | 24 ++++++-
.../grpc/AgentRegisterGRPCDataListener.java | 3 +
.../jetty/AgentServerJettyDataListener.java | 4 ++
.../apm-collector-agentstream/pom.xml | 45 ------------
.../AgentStreamModuleInstaller.java | 1 -
.../grpc/AgentStreamGRPCDataListener.java | 3 +
.../jetty/AgentStreamJettyDataListener.java | 3 +
.../component/NodeComponentAggWorker.java | 5 ++
.../component/NodeComponentDataDefine.java | 21 +++---
.../application/ApplicationDataDefine.java | 70 +++++++++++++++++++
.../application/ApplicationEsTableDefine.java | 31 ++++++++
.../application/ApplicationH2TableDefine.java | 19 +++++
.../ApplicationRegisterDataDefine.java | 38 ----------
.../ApplicationRegisterRemoteWorker.java | 19 +++--
.../ApplicationRegisterSerialWorker.java | 6 ++
...gisterTable.java => ApplicationTable.java} | 4 +-
.../src/main/proto/ApplicationRegister.proto | 8 ---
.../src/main/proto/NodeComponent.proto | 11 ---
.../resources/META-INF/defines/data.define | 2 +-
.../resources/META-INF/defines/storage.define | 5 +-
apm-collector/apm-collector-client/pom.xml | 5 ++
.../apm/collector/client/grpc/GRPCClient.java | 35 ++++++++++
.../client/grpc/GRPCClientException.java | 17 +++++
.../zookeeper/ClusterZKDataMonitor.java | 1 +
.../core/cluster/ClusterDataListener.java | 2 +
.../core/storage/StorageInstaller.java | 9 ++-
.../collector/core/storage/TableDefine.java | 4 +-
.../define/ElasticSearchColumnDefine.java | 2 +-
.../storage/h2/define/H2ColumnDefine.java | 2 +-
.../stream/StreamModuleInstaller.java | 3 +-
.../stream/grpc/StreamGRPCDataListener.java | 57 +++++++++++++++
.../handler/RemoteCommonServiceHandler.java | 26 +++----
.../worker/AbstractRemoteWorkerProvider.java | 19 +++--
.../stream/worker/ClusterWorkerContext.java | 28 ++------
.../apm/collector/stream/worker/Context.java | 4 +-
.../apm/collector/stream/worker/LookUp.java | 2 -
.../stream/worker/RemoteWorkerRef.java | 37 ++++++++--
.../apm/collector/stream/worker/Role.java | 3 +
.../stream/worker/WorkerContext.java | 19 ++++-
.../collector/stream/worker/WorkerRefs.java | 18 +++--
.../stream/worker/impl/data/Data.java | 44 +++++++++++-
.../stream/worker/impl/data/DataDefine.java | 39 +++++------
.../worker/impl/data/DataDefineLoader.java | 1 +
.../stream/worker/impl/data/Operation.java | 2 +
.../impl/data/operate/CoverOperation.java | 4 ++
.../impl/data/operate/NonOperation.java | 4 ++
.../worker/selector/ForeverFirstSelector.java | 5 ++
.../src/main/proto/RemoteCommonService.proto | 18 +++--
50 files changed, 514 insertions(+), 232 deletions(-)
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationDataDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationEsTableDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationH2TableDefine.java
delete mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterDataDefine.java
rename apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/{ApplicationRegisterTable.java => ApplicationTable.java} (71%)
delete mode 100644 apm-collector/apm-collector-agentstream/src/main/proto/ApplicationRegister.proto
delete mode 100644 apm-collector/apm-collector-agentstream/src/main/proto/NodeComponent.proto
create mode 100644 apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/grpc/GRPCClient.java
create mode 100644 apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/grpc/GRPCClientException.java
diff --git a/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/AgentJVMGRPCDataListener.java b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/AgentJVMGRPCDataListener.java
index d0ea0aa61..b25fcdb8d 100644
--- a/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/AgentJVMGRPCDataListener.java
+++ b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/AgentJVMGRPCDataListener.java
@@ -14,4 +14,8 @@ public class AgentJVMGRPCDataListener extends ClusterDataListener {
@Override public String path() {
return PATH;
}
+
+ @Override public void addressChangedNotify() {
+
+ }
}
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationEsDAO.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationEsDAO.java
index 80d1d2ec0..24a9b5c5b 100644
--- a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationEsDAO.java
+++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationEsDAO.java
@@ -4,7 +4,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.index.query.QueryBuilders;
-import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterTable;
+import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationTable;
import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient;
import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO;
@@ -16,15 +16,15 @@ public class ApplicationEsDAO extends EsDAO implements IApplicationDAO {
@Override public int getApplicationId(String applicationCode) {
ElasticSearchClient client = getClient();
- SearchRequestBuilder searchRequestBuilder = client.prepareSearch(ApplicationRegisterTable.TABLE);
+ SearchRequestBuilder searchRequestBuilder = client.prepareSearch(ApplicationTable.TABLE);
searchRequestBuilder.setTypes("type");
searchRequestBuilder.setSearchType(SearchType.QUERY_THEN_FETCH);
- searchRequestBuilder.setQuery(QueryBuilders.termQuery(ApplicationRegisterTable.COLUMN_APPLICATION_CODE, applicationCode));
- searchRequestBuilder.setSize(10);
+ searchRequestBuilder.setQuery(QueryBuilders.termQuery(ApplicationTable.COLUMN_APPLICATION_CODE, applicationCode));
+ searchRequestBuilder.setSize(1);
SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
if (searchResponse.getHits().totalHits > 0) {
- return searchResponse.getHits().getAt(0).getField(ApplicationRegisterTable.COLUMN_APPLICATION_ID).getValue();
+ return searchResponse.getHits().getAt(0).getField(ApplicationTable.COLUMN_APPLICATION_ID).getValue();
}
return 0;
}
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDGetOrCreate.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDGetOrCreate.java
index 8a949235f..1535e57a0 100644
--- a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDGetOrCreate.java
+++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDGetOrCreate.java
@@ -1,14 +1,36 @@
package org.skywalking.apm.collector.agentregister.application;
+import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine;
+import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterRemoteWorker;
+import org.skywalking.apm.collector.core.framework.CollectorContextHelper;
import org.skywalking.apm.collector.storage.dao.DAOContainer;
+import org.skywalking.apm.collector.stream.StreamModuleContext;
+import org.skywalking.apm.collector.stream.StreamModuleGroupDefine;
+import org.skywalking.apm.collector.stream.worker.WorkerInvokeException;
+import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* @author pengys5
*/
public class ApplicationIDGetOrCreate {
+ private final Logger logger = LoggerFactory.getLogger(ApplicationIDGetOrCreate.class);
+
public int getOrCreate(String applicationCode) {
IApplicationDAO dao = (IApplicationDAO)DAOContainer.INSTANCE.get(IApplicationDAO.class.getName());
- return dao.getApplicationId(applicationCode);
+ int applicationId = dao.getApplicationId(applicationCode);
+
+ if (applicationId == 0) {
+ StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME);
+ ApplicationDataDefine.Application application = new ApplicationDataDefine.Application(applicationCode, applicationCode, 0);
+ try {
+ context.getClusterWorkerContext().lookup(ApplicationRegisterRemoteWorker.WorkerRole.INSTANCE).tell(application);
+ } catch (WorkerNotFoundException | WorkerInvokeException e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+ return applicationId;
}
}
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/AgentRegisterGRPCDataListener.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/AgentRegisterGRPCDataListener.java
index e54e324ae..6ff6004d9 100644
--- a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/AgentRegisterGRPCDataListener.java
+++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/AgentRegisterGRPCDataListener.java
@@ -14,4 +14,7 @@ public class AgentRegisterGRPCDataListener extends ClusterDataListener {
@Override public String path() {
return PATH;
}
+
+ @Override public void addressChangedNotify() {
+ }
}
diff --git a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/AgentServerJettyDataListener.java b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/AgentServerJettyDataListener.java
index 9035ca297..1ce44e9d8 100644
--- a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/AgentServerJettyDataListener.java
+++ b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/AgentServerJettyDataListener.java
@@ -12,4 +12,8 @@ public class AgentServerJettyDataListener extends ClusterDataListener {
@Override public String path() {
return ClusterModuleDefine.BASE_CATALOG + "." + AgentServerModuleGroupDefine.GROUP_NAME + "." + AgentServerJettyModuleDefine.MODULE_NAME;
}
+
+ @Override public void addressChangedNotify() {
+
+ }
}
diff --git a/apm-collector/apm-collector-agentstream/pom.xml b/apm-collector/apm-collector-agentstream/pom.xml
index e76c1b793..7cf497179 100644
--- a/apm-collector/apm-collector-agentstream/pom.xml
+++ b/apm-collector/apm-collector-agentstream/pom.xml
@@ -34,49 +34,4 @@
${project.version}
-
-
-
-
- kr.motd.maven
- os-maven-plugin
- 1.4.1.Final
-
-
-
-
- org.apache.maven.plugins
- maven-resources-plugin
- 2.4.3
-
- ${project.build.sourceEncoding}
-
-
-
- org.xolstice.maven.plugins
- protobuf-maven-plugin
- 0.5.0
-
-
- com.google.protobuf:protoc:3.3.0:exe:${os.detected.classifier}
-
- grpc-java
- io.grpc:protoc-gen-grpc-java:1.4.0:exe:${os.detected.classifier}
-
-
-
-
-
- compile
- compile-custom
-
-
-
-
-
-
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleInstaller.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleInstaller.java
index db4b3dc04..ace5f61c0 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleInstaller.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleInstaller.java
@@ -26,7 +26,6 @@ public class AgentStreamModuleInstaller implements ModuleInstaller {
AgentStreamModuleContext context = new AgentStreamModuleContext(AgentStreamModuleGroupDefine.GROUP_NAME);
CollectorContextHelper.INSTANCE.putContext(context);
- logger.info("could not configure cluster module, use the default");
Iterator> moduleDefineEntry = moduleDefineMap.entrySet().iterator();
while (moduleDefineEntry.hasNext()) {
ModuleDefine moduleDefine = moduleDefineEntry.next().getValue();
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/AgentStreamGRPCDataListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/AgentStreamGRPCDataListener.java
index 2fc79a258..dd3428ec3 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/AgentStreamGRPCDataListener.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/AgentStreamGRPCDataListener.java
@@ -14,4 +14,7 @@ public class AgentStreamGRPCDataListener extends ClusterDataListener {
@Override public String path() {
return PATH;
}
+
+ @Override public void addressChangedNotify() {
+ }
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyDataListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyDataListener.java
index 0ec18d14a..375c84754 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyDataListener.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyDataListener.java
@@ -14,4 +14,7 @@ public class AgentStreamJettyDataListener extends ClusterDataListener {
@Override public String path() {
return PATH;
}
+
+ @Override public void addressChangedNotify() {
+ }
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggWorker.java
index 89fc75805..005198b68 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggWorker.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggWorker.java
@@ -4,6 +4,7 @@ import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvid
import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
import org.skywalking.apm.collector.stream.worker.impl.AggregationWorker;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
import org.skywalking.apm.collector.stream.worker.selector.RollingSelector;
import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector;
@@ -53,5 +54,9 @@ public class NodeComponentAggWorker extends AggregationWorker {
public WorkerSelector workerSelector() {
return new RollingSelector();
}
+
+ @Override public DataDefine dataDefine() {
+ return new NodeComponentDataDefine();
+ }
}
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentDataDefine.java
index 34336b0d6..1d09a862e 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentDataDefine.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentDataDefine.java
@@ -1,11 +1,8 @@
package org.skywalking.apm.collector.agentstream.worker.node.component;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
-import org.skywalking.apm.collector.agentstream.worker.node.define.proto.NodeComponent;
+import org.skywalking.apm.collector.remote.grpc.proto.RemoteData;
import org.skywalking.apm.collector.stream.worker.impl.data.Attribute;
import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType;
-import org.skywalking.apm.collector.stream.worker.impl.data.Data;
import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation;
import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation;
@@ -15,7 +12,7 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation
*/
public class NodeComponentDataDefine extends DataDefine {
- @Override protected int defineId() {
+ @Override public int defineId() {
return 0;
}
@@ -30,13 +27,11 @@ public class NodeComponentDataDefine extends DataDefine {
addAttribute(3, new Attribute("aggregation", AttributeType.STRING, new CoverOperation()));
}
- @Override public Data parseFrom(ByteString bytesData) throws InvalidProtocolBufferException {
- NodeComponent.Message message = NodeComponent.Message.parseFrom(bytesData);
- Data data = build();
- data.setDataString(0, message.getId());
- data.setDataString(1, message.getName());
- data.setDataString(2, message.getPeers());
- data.setDataString(3, message.getAggregation());
- return data;
+ @Override public Object deserialize(RemoteData remoteData) {
+ return null;
+ }
+
+ @Override public RemoteData serialize(Object object) {
+ return null;
}
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationDataDefine.java
new file mode 100644
index 000000000..35bb52c7b
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationDataDefine.java
@@ -0,0 +1,70 @@
+package org.skywalking.apm.collector.agentstream.worker.register.application;
+
+import org.skywalking.apm.collector.remote.grpc.proto.RemoteData;
+import org.skywalking.apm.collector.stream.worker.impl.data.Attribute;
+import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
+import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation;
+import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation;
+
+/**
+ * @author pengys5
+ */
+public class ApplicationDataDefine extends DataDefine {
+
+ public static final int DEFINE_ID = 101;
+
+ @Override public int defineId() {
+ return DEFINE_ID;
+ }
+
+ @Override protected int initialCapacity() {
+ return 3;
+ }
+
+ @Override protected void attributeDefine() {
+ addAttribute(0, new Attribute("id", AttributeType.STRING, new NonOperation()));
+ addAttribute(1, new Attribute(ApplicationTable.COLUMN_APPLICATION_CODE, AttributeType.STRING, new CoverOperation()));
+ addAttribute(2, new Attribute(ApplicationTable.COLUMN_APPLICATION_ID, AttributeType.INTEGER, new CoverOperation()));
+ }
+
+ @Override public Object deserialize(RemoteData remoteData) {
+ String id = remoteData.getDataStrings(0);
+ String applicationCode = remoteData.getDataStrings(1);
+ int applicationId = remoteData.getDataIntegers(0);
+ return new Application(id, applicationCode, applicationId);
+ }
+
+ @Override public RemoteData serialize(Object object) {
+ Application application = (Application)object;
+ RemoteData.Builder builder = RemoteData.newBuilder();
+ builder.addDataStrings(application.getId());
+ builder.addDataStrings(application.getApplicationCode());
+ builder.addDataIntegers(application.getApplicationId());
+ return builder.build();
+ }
+
+ public static class Application {
+ private final String id;
+ private final String applicationCode;
+ private final int applicationId;
+
+ public Application(String id, String applicationCode, int applicationId) {
+ this.id = id;
+ this.applicationCode = applicationCode;
+ this.applicationId = applicationId;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public String getApplicationCode() {
+ return applicationCode;
+ }
+
+ public int getApplicationId() {
+ return applicationId;
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationEsTableDefine.java
new file mode 100644
index 000000000..e5e5faded
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationEsTableDefine.java
@@ -0,0 +1,31 @@
+package org.skywalking.apm.collector.agentstream.worker.register.application;
+
+import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine;
+import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine;
+
+/**
+ * @author pengys5
+ */
+public class ApplicationEsTableDefine extends ElasticSearchTableDefine {
+
+ public ApplicationEsTableDefine() {
+ super(ApplicationTable.TABLE);
+ }
+
+ @Override public int refreshInterval() {
+ return 0;
+ }
+
+ @Override public int numberOfShards() {
+ return 2;
+ }
+
+ @Override public int numberOfReplicas() {
+ return 0;
+ }
+
+ @Override public void initialize() {
+ addColumn(new ElasticSearchColumnDefine(ApplicationTable.COLUMN_APPLICATION_CODE, ElasticSearchColumnDefine.Type.Keyword.name()));
+ addColumn(new ElasticSearchColumnDefine(ApplicationTable.COLUMN_APPLICATION_ID, ElasticSearchColumnDefine.Type.Integer.name()));
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationH2TableDefine.java
new file mode 100644
index 000000000..ef77965c2
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationH2TableDefine.java
@@ -0,0 +1,19 @@
+package org.skywalking.apm.collector.agentstream.worker.register.application;
+
+import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine;
+import org.skywalking.apm.collector.storage.h2.define.H2TableDefine;
+
+/**
+ * @author pengys5
+ */
+public class ApplicationH2TableDefine extends H2TableDefine {
+
+ public ApplicationH2TableDefine() {
+ super(ApplicationTable.TABLE);
+ }
+
+ @Override public void initialize() {
+ addColumn(new H2ColumnDefine(ApplicationTable.COLUMN_APPLICATION_CODE, H2ColumnDefine.Type.Varchar.name()));
+ addColumn(new H2ColumnDefine(ApplicationTable.COLUMN_APPLICATION_ID, H2ColumnDefine.Type.Int.name()));
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterDataDefine.java
deleted file mode 100644
index 2106c37ae..000000000
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterDataDefine.java
+++ /dev/null
@@ -1,38 +0,0 @@
-package org.skywalking.apm.collector.agentstream.worker.register.application;
-
-import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
-import org.skywalking.apm.collector.agentstream.worker.register.application.proto.ApplicationRegisterOuterClass;
-import org.skywalking.apm.collector.stream.worker.impl.data.Attribute;
-import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType;
-import org.skywalking.apm.collector.stream.worker.impl.data.Data;
-import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
-import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation;
-import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation;
-
-/**
- * @author pengys5
- */
-public class ApplicationRegisterDataDefine extends DataDefine {
-
- @Override protected int defineId() {
- return 101;
- }
-
- @Override protected int initialCapacity() {
- return 3;
- }
-
- @Override protected void attributeDefine() {
- addAttribute(0, new Attribute("id", AttributeType.STRING, new NonOperation()));
- addAttribute(1, new Attribute(ApplicationRegisterTable.COLUMN_APPLICATION_CODE, AttributeType.STRING, new CoverOperation()));
- addAttribute(2, new Attribute(ApplicationRegisterTable.COLUMN_APPLICATION_ID, AttributeType.INTEGER, new CoverOperation()));
- }
-
- @Override public Data parseFrom(ByteString bytesData) throws InvalidProtocolBufferException {
- ApplicationRegisterOuterClass.ApplicationRegister applicationRegister = ApplicationRegisterOuterClass.ApplicationRegister.parseFrom(bytesData);
- Data data = build();
- data.setDataString(1, applicationRegister.getApplicationCode());
- return data;
- }
-}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterRemoteWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterRemoteWorker.java
index d6e2327f5..2ead835fa 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterRemoteWorker.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterRemoteWorker.java
@@ -1,20 +1,24 @@
package org.skywalking.apm.collector.agentstream.worker.register.application;
-import org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentAggWorker;
import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorker;
import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorkerProvider;
import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
import org.skywalking.apm.collector.stream.worker.Role;
import org.skywalking.apm.collector.stream.worker.WorkerException;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
import org.skywalking.apm.collector.stream.worker.selector.ForeverFirstSelector;
import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* @author pengys5
*/
public class ApplicationRegisterRemoteWorker extends AbstractRemoteWorker {
+ private final Logger logger = LoggerFactory.getLogger(ApplicationRegisterRemoteWorker.class);
+
protected ApplicationRegisterRemoteWorker(Role role, ClusterWorkerContext clusterContext) {
super(role, clusterContext);
}
@@ -23,7 +27,8 @@ public class ApplicationRegisterRemoteWorker extends AbstractRemoteWorker {
}
@Override protected void onWork(Object message) throws WorkerException {
-
+ ApplicationDataDefine.Application application = (ApplicationDataDefine.Application)message;
+ logger.debug("application code: {}", application.getApplicationCode());
}
public static class Factory extends AbstractRemoteWorkerProvider {
@@ -36,10 +41,6 @@ public class ApplicationRegisterRemoteWorker extends AbstractRemoteWorker {
public ApplicationRegisterRemoteWorker workerInstance(ClusterWorkerContext clusterContext) {
return new ApplicationRegisterRemoteWorker(role(), clusterContext);
}
-
- @Override public int workerNum() {
- return 1;
- }
}
public enum WorkerRole implements Role {
@@ -47,12 +48,16 @@ public class ApplicationRegisterRemoteWorker extends AbstractRemoteWorker {
@Override
public String roleName() {
- return NodeComponentAggWorker.class.getSimpleName();
+ return ApplicationRegisterRemoteWorker.class.getSimpleName();
}
@Override
public WorkerSelector workerSelector() {
return new ForeverFirstSelector();
}
+
+ @Override public DataDefine dataDefine() {
+ return new ApplicationDataDefine();
+ }
}
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java
index 7f14301ea..49b80a365 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java
@@ -6,6 +6,7 @@ import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
import org.skywalking.apm.collector.stream.worker.Role;
import org.skywalking.apm.collector.stream.worker.WorkerException;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector;
import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector;
@@ -54,5 +55,10 @@ public class ApplicationRegisterSerialWorker extends AbstractLocalAsyncWorker {
public WorkerSelector workerSelector() {
return new HashCodeSelector();
}
+
+ @Override public DataDefine dataDefine() {
+ return new ApplicationDataDefine();
+ }
+
}
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationTable.java
similarity index 71%
rename from apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterTable.java
rename to apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationTable.java
index 1a053f4e8..797b8d80a 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterTable.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationTable.java
@@ -3,8 +3,8 @@ package org.skywalking.apm.collector.agentstream.worker.register.application;
/**
* @author pengys5
*/
-public class ApplicationRegisterTable {
- public static final String TABLE = "application_register";
+public class ApplicationTable {
+ public static final String TABLE = "application";
public static final String COLUMN_APPLICATION_CODE = "application_code";
public static final String COLUMN_APPLICATION_ID = "application_id";
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/proto/ApplicationRegister.proto b/apm-collector/apm-collector-agentstream/src/main/proto/ApplicationRegister.proto
deleted file mode 100644
index 572cfd362..000000000
--- a/apm-collector/apm-collector-agentstream/src/main/proto/ApplicationRegister.proto
+++ /dev/null
@@ -1,8 +0,0 @@
-syntax = "proto3";
-
-option java_multiple_files = false;
-option java_package = "org.skywalking.apm.collector.agentstream.worker.register.application.proto";
-
-message ApplicationRegister {
- string application_code = 1;
-}
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/proto/NodeComponent.proto b/apm-collector/apm-collector-agentstream/src/main/proto/NodeComponent.proto
deleted file mode 100644
index df981e85d..000000000
--- a/apm-collector/apm-collector-agentstream/src/main/proto/NodeComponent.proto
+++ /dev/null
@@ -1,11 +0,0 @@
-syntax = "proto3";
-
-option java_multiple_files = false;
-option java_package = "org.skywalking.apm.collector.agentstream.worker.node.define.proto";
-
-message Message {
- string id = 1;
- string name = 2;
- string peers = 3;
- string aggregation = 4;
-}
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define
index a7a420f19..dce35f4df 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define
@@ -1,2 +1,2 @@
org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentDataDefine
-org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterDataDefine
\ No newline at end of file
+org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define
index 82a6af78a..12f4a13a6 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define
@@ -1,4 +1,7 @@
org.skywalking.apm.collector.agentstream.worker.node.define.NodeComponentEsTableDefine
org.skywalking.apm.collector.agentstream.worker.node.define.NodeComponentH2TableDefine
org.skywalking.apm.collector.agentstream.worker.node.define.NodeMappingEsTableDefine
-org.skywalking.apm.collector.agentstream.worker.node.define.NodeMappingH2TableDefine
\ No newline at end of file
+org.skywalking.apm.collector.agentstream.worker.node.define.NodeMappingH2TableDefine
+
+org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationEsTableDefine
+org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationH2TableDefine
\ No newline at end of file
diff --git a/apm-collector/apm-collector-client/pom.xml b/apm-collector/apm-collector-client/pom.xml
index 31f9fdad0..5fe914473 100644
--- a/apm-collector/apm-collector-client/pom.xml
+++ b/apm-collector/apm-collector-client/pom.xml
@@ -58,5 +58,10 @@
+
+ io.grpc
+ grpc-core
+ 1.4.0
+
\ No newline at end of file
diff --git a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/grpc/GRPCClient.java b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/grpc/GRPCClient.java
new file mode 100644
index 000000000..8788a5472
--- /dev/null
+++ b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/grpc/GRPCClient.java
@@ -0,0 +1,35 @@
+package org.skywalking.apm.collector.client.grpc;
+
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import org.skywalking.apm.collector.core.client.Client;
+import org.skywalking.apm.collector.core.client.ClientException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class GRPCClient implements Client {
+
+ private final Logger logger = LoggerFactory.getLogger(GRPCClient.class);
+
+ private final String host;
+
+ private final int port;
+
+ private ManagedChannel channel;
+
+ public GRPCClient(String host, int port) {
+ this.host = host;
+ this.port = port;
+ }
+
+ @Override public void initialize() throws ClientException {
+ channel = ManagedChannelBuilder.forAddress(host, port).usePlaintext(true).build();
+ }
+
+ public ManagedChannel getChannel() {
+ return channel;
+ }
+}
diff --git a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/grpc/GRPCClientException.java b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/grpc/GRPCClientException.java
new file mode 100644
index 000000000..ec2a018c6
--- /dev/null
+++ b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/grpc/GRPCClientException.java
@@ -0,0 +1,17 @@
+package org.skywalking.apm.collector.client.grpc;
+
+import org.skywalking.apm.collector.core.client.ClientException;
+
+/**
+ * @author pengys5
+ */
+public class GRPCClientException extends ClientException {
+
+ public GRPCClientException(String message) {
+ super(message);
+ }
+
+ public GRPCClientException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKDataMonitor.java b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKDataMonitor.java
index c6be3bbe8..7e973716a 100644
--- a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKDataMonitor.java
+++ b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKDataMonitor.java
@@ -47,6 +47,7 @@ public class ClusterZKDataMonitor implements DataMonitor, Watcher {
String dataStr = new String(data);
logger.debug("path children has been changed, path: {}, data: {}", event.getPath() + "/" + serverPath, dataStr);
listeners.get(event.getPath()).addAddress(serverPath + dataStr);
+ listeners.get(event.getPath()).addressChangedNotify();
}
}
} catch (ZookeeperClientException e) {
diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterDataListener.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterDataListener.java
index 7b14e7a33..463a33adf 100644
--- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterDataListener.java
+++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterDataListener.java
@@ -28,4 +28,6 @@ public abstract class ClusterDataListener implements Listener {
public final void clearData() {
addresses.clear();
}
+
+ public abstract void addressChangedNotify();
}
diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstaller.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstaller.java
index beff974ac..c90bb3db2 100644
--- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstaller.java
+++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstaller.java
@@ -3,12 +3,16 @@ package org.skywalking.apm.collector.core.storage;
import java.util.List;
import org.skywalking.apm.collector.core.client.Client;
import org.skywalking.apm.collector.core.framework.DefineException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* @author pengys5
*/
public abstract class StorageInstaller {
+ private final Logger logger = LoggerFactory.getLogger(StorageInstaller.class);
+
public final void install(Client client) throws StorageException {
StorageDefineLoader defineLoader = new StorageDefineLoader();
try {
@@ -16,9 +20,8 @@ public abstract class StorageInstaller {
defineFilter(tableDefines);
for (TableDefine tableDefine : tableDefines) {
- if (isExists(client, tableDefine)) {
- deleteIndex(client, tableDefine);
- } else {
+ if (!isExists(client, tableDefine)) {
+ logger.info("table: {} not exists", tableDefine.getName());
createTable(client, tableDefine);
}
}
diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/TableDefine.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/TableDefine.java
index 026f2fcf1..1fa7d88f8 100644
--- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/TableDefine.java
+++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/TableDefine.java
@@ -21,11 +21,11 @@ public abstract class TableDefine {
columnDefines.add(columnDefine);
}
- public String getName() {
+ public final String getName() {
return name;
}
- public List getColumnDefines() {
+ public final List getColumnDefines() {
return columnDefines;
}
}
diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchColumnDefine.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchColumnDefine.java
index f50ff9d0c..883480a44 100644
--- a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchColumnDefine.java
+++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchColumnDefine.java
@@ -11,6 +11,6 @@ public class ElasticSearchColumnDefine extends ColumnDefine {
}
public enum Type {
- Binary, Boolean, Date, Keyword, Long, Text
+ Binary, Boolean, Date, Keyword, Long, Integer
}
}
diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2ColumnDefine.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2ColumnDefine.java
index 46d3a1a50..fa7760a0d 100644
--- a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2ColumnDefine.java
+++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2ColumnDefine.java
@@ -12,6 +12,6 @@ public class H2ColumnDefine extends ColumnDefine {
}
public enum Type {
- Boolean, Varchar, Bigint, Date
+ Boolean, Varchar, Int, Bigint, Date
}
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java
index f834edb20..a714b6fdd 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java
@@ -66,8 +66,9 @@ public class StreamModuleInstaller implements ModuleInstaller {
List remoteProviders = remoteProviderLoader.load();
for (AbstractRemoteWorkerProvider provider : remoteProviders) {
provider.setClusterContext(clusterWorkerContext);
- provider.create();
+// provider.create();
clusterWorkerContext.putRole(provider.role());
+ clusterWorkerContext.putProvider(provider);
}
} catch (ProviderNotFoundException e) {
logger.error(e.getMessage(), e);
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCDataListener.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCDataListener.java
index 1b6e863ab..89f7e64a3 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCDataListener.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCDataListener.java
@@ -1,17 +1,74 @@
package org.skywalking.apm.collector.stream.grpc;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.skywalking.apm.collector.client.grpc.GRPCClient;
import org.skywalking.apm.collector.cluster.ClusterModuleDefine;
+import org.skywalking.apm.collector.core.client.ClientException;
import org.skywalking.apm.collector.core.cluster.ClusterDataListener;
+import org.skywalking.apm.collector.core.framework.CollectorContextHelper;
+import org.skywalking.apm.collector.stream.StreamModuleContext;
import org.skywalking.apm.collector.stream.StreamModuleGroupDefine;
+import org.skywalking.apm.collector.stream.worker.RemoteWorkerRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* @author pengys5
*/
public class StreamGRPCDataListener extends ClusterDataListener {
+ private final Logger logger = LoggerFactory.getLogger(StreamGRPCDataListener.class);
+
public static final String PATH = ClusterModuleDefine.BASE_CATALOG + "." + StreamModuleGroupDefine.GROUP_NAME + "." + StreamGRPCModuleDefine.MODULE_NAME;
@Override public String path() {
return PATH;
}
+
+ private Map clients = new HashMap<>();
+ private Map workerRefs = new HashMap<>();
+
+ @Override public void addressChangedNotify() {
+ String selfAddress = StreamGRPCConfig.HOST + ":" + StreamGRPCConfig.PORT;
+
+ StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME);
+
+ List addresses = getAddresses();
+ clients.keySet().forEach(address -> {
+ if (!addresses.contains(address)) {
+ context.getClusterWorkerContext().remove(workerRefs.get(address));
+ workerRefs.remove(address);
+ }
+ });
+
+ for (String address : addresses) {
+ if (!clients.containsKey(address)) {
+ logger.debug("new address: {}, create this address remote worker reference", address);
+ String[] hostPort = address.split(":");
+ GRPCClient client = new GRPCClient(hostPort[0], Integer.valueOf(hostPort[1]));
+ try {
+ client.initialize();
+ } catch (ClientException e) {
+ e.printStackTrace();
+ }
+ clients.put(address, client);
+
+ if (selfAddress.equals(address)) {
+ context.getClusterWorkerContext().getProviders().forEach(provider -> {
+ logger.debug("create remote worker self reference, role: {}", provider.role().roleName());
+ provider.create();
+ });
+ } else {
+ context.getClusterWorkerContext().getProviders().forEach(provider -> {
+ logger.debug("create remote worker reference, role: {}", provider.role().roleName());
+ RemoteWorkerRef workerRef = provider.create(client);
+ });
+ }
+ } else {
+ logger.debug("address: {} had remote worker reference, ignore", address);
+ }
+ }
+ }
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/handler/RemoteCommonServiceHandler.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/handler/RemoteCommonServiceHandler.java
index f0294db24..86b5dc2b0 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/handler/RemoteCommonServiceHandler.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/handler/RemoteCommonServiceHandler.java
@@ -1,19 +1,17 @@
package org.skywalking.apm.collector.stream.grpc.handler;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
import io.grpc.stub.StreamObserver;
-import org.skywalking.apm.collector.cluster.ClusterModuleGroupDefine;
import org.skywalking.apm.collector.core.framework.CollectorContextHelper;
import org.skywalking.apm.collector.remote.grpc.proto.Empty;
-import org.skywalking.apm.collector.remote.grpc.proto.Message;
import org.skywalking.apm.collector.remote.grpc.proto.RemoteCommonServiceGrpc;
+import org.skywalking.apm.collector.remote.grpc.proto.RemoteData;
+import org.skywalking.apm.collector.remote.grpc.proto.RemoteMessage;
import org.skywalking.apm.collector.server.grpc.GRPCHandler;
import org.skywalking.apm.collector.stream.StreamModuleContext;
+import org.skywalking.apm.collector.stream.StreamModuleGroupDefine;
+import org.skywalking.apm.collector.stream.worker.Role;
import org.skywalking.apm.collector.stream.worker.WorkerInvokeException;
import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException;
-import org.skywalking.apm.collector.stream.worker.impl.data.Data;
-import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -24,18 +22,16 @@ public class RemoteCommonServiceHandler extends RemoteCommonServiceGrpc.RemoteCo
private final Logger logger = LoggerFactory.getLogger(RemoteCommonServiceHandler.class);
- @Override public void call(Message request, StreamObserver responseObserver) {
+ @Override public void call(RemoteMessage request, StreamObserver responseObserver) {
String roleName = request.getWorkerRole();
- int dataDefineId = request.getDataDefineId();
- ByteString bytesData = request.getDataBytes();
-
- StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(ClusterModuleGroupDefine.GROUP_NAME);
- DataDefine dataDefine = context.getDataDefine(dataDefineId);
+ RemoteData remoteData = request.getRemoteData();
+ StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME);
+ Role role = context.getClusterWorkerContext().getRole(roleName);
+ Object object = role.dataDefine().deserialize(remoteData);
try {
- Data data = dataDefine.parseFrom(bytesData);
- context.getClusterWorkerContext().lookup(context.getClusterWorkerContext().getRole(roleName)).tell(data);
- } catch (InvalidProtocolBufferException | WorkerNotFoundException | WorkerInvokeException e) {
+ context.getClusterWorkerContext().lookupInSide(roleName).tell(object);
+ } catch (WorkerNotFoundException | WorkerInvokeException e) {
logger.error(e.getMessage(), e);
}
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorkerProvider.java
index 5c4b25a8b..33b97df63 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorkerProvider.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorkerProvider.java
@@ -1,5 +1,7 @@
package org.skywalking.apm.collector.stream.worker;
+import org.skywalking.apm.collector.client.grpc.GRPCClient;
+
/**
* The AbstractRemoteWorkerProvider implementations represent providers,
* which create instance of cluster workers whose implemented {@link AbstractRemoteWorker}.
@@ -10,13 +12,6 @@ package org.skywalking.apm.collector.stream.worker;
*/
public abstract class AbstractRemoteWorkerProvider extends AbstractWorkerProvider {
- /**
- * Create how many worker instance of {@link AbstractRemoteWorker} in one jvm.
- *
- * @return The worker instance number.
- */
- public abstract int workerNum();
-
/**
* Create the worker instance into akka system, the akka system will control the cluster worker life cycle.
*
@@ -24,12 +19,16 @@ public abstract class AbstractRemoteWorkerProvider providers = new ConcurrentHashMap<>();
+ private List providers = new ArrayList<>();
- @Override
- public AbstractWorkerProvider findProvider(Role role) throws ProviderNotFoundException {
- logger.debug("find role of %s provider from ClusterWorkerContext", role.roleName());
- if (providers.containsKey(role.roleName())) {
- return providers.get(role.roleName());
- } else {
- throw new ProviderNotFoundException("role=" + role.roleName() + ", no available provider.");
- }
+ public List getProviders() {
+ return providers;
}
@Override
- public void putProvider(AbstractWorkerProvider provider) throws UsedRoleNameException {
- logger.debug("put role of %s provider into ClusterWorkerContext", provider.role().roleName());
- if (providers.containsKey(provider.role().roleName())) {
- throw new UsedRoleNameException("provider with role=" + provider.role().roleName() + " duplicate each other.");
- } else {
- providers.put(provider.role().roleName(), provider);
- }
+ public void putProvider(AbstractRemoteWorkerProvider provider) {
+ providers.add(provider);
}
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Context.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Context.java
index 218c78c28..8b8d4e418 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Context.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Context.java
@@ -5,10 +5,12 @@ package org.skywalking.apm.collector.stream.worker;
*/
public interface Context extends LookUp {
- void putProvider(AbstractWorkerProvider provider) throws UsedRoleNameException;
+ void putProvider(AbstractRemoteWorkerProvider provider);
WorkerRefs lookup(Role role) throws WorkerNotFoundException;
+ RemoteWorkerRef lookupInSide(String roleName) throws WorkerNotFoundException;
+
void put(WorkerRef workerRef);
void remove(WorkerRef workerRef);
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LookUp.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LookUp.java
index 1e5ad6acb..a462632c5 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LookUp.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LookUp.java
@@ -6,6 +6,4 @@ package org.skywalking.apm.collector.stream.worker;
public interface LookUp {
WorkerRefs lookup(Role role) throws WorkerNotFoundException;
-
- Provider findProvider(Role role) throws ProviderNotFoundException;
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerRef.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerRef.java
index f653c2f62..5e99ab101 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerRef.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerRef.java
@@ -1,19 +1,48 @@
package org.skywalking.apm.collector.stream.worker;
+import org.skywalking.apm.collector.client.grpc.GRPCClient;
+import org.skywalking.apm.collector.remote.grpc.proto.RemoteCommonServiceGrpc;
+import org.skywalking.apm.collector.remote.grpc.proto.RemoteData;
+import org.skywalking.apm.collector.remote.grpc.proto.RemoteMessage;
+
/**
* @author pengys5
*/
public class RemoteWorkerRef extends WorkerRef {
- private AbstractRemoteWorker clusterWorker;
+ private final Boolean acrossJVM;
+ private final RemoteCommonServiceGrpc.RemoteCommonServiceBlockingStub stub;
+ private final AbstractRemoteWorker remoteWorker;
- public RemoteWorkerRef(Role role, AbstractRemoteWorker clusterWorker) {
+ public RemoteWorkerRef(Role role, AbstractRemoteWorker remoteWorker) {
super(role);
- this.clusterWorker = clusterWorker;
+ this.remoteWorker = remoteWorker;
+ this.acrossJVM = false;
+ this.stub = null;
+ }
+
+ public RemoteWorkerRef(Role role, GRPCClient client) {
+ super(role);
+ this.remoteWorker = null;
+ this.acrossJVM = true;
+ this.stub = RemoteCommonServiceGrpc.newBlockingStub(client.getChannel());
}
@Override
public void tell(Object message) throws WorkerInvokeException {
- clusterWorker.allocateJob(message);
+ if (acrossJVM) {
+ RemoteData remoteData = getRole().dataDefine().serialize(message);
+
+ RemoteMessage.Builder builder = RemoteMessage.newBuilder();
+ builder.setWorkerRole(getRole().roleName());
+ builder.setRemoteData(remoteData);
+ stub.call(builder.build());
+ } else {
+ remoteWorker.allocateJob(message);
+ }
+ }
+
+ public Boolean isAcrossJVM() {
+ return acrossJVM;
}
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Role.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Role.java
index e9c49d521..6d9ce1497 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Role.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Role.java
@@ -1,5 +1,6 @@
package org.skywalking.apm.collector.stream.worker;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector;
/**
@@ -10,4 +11,6 @@ public interface Role {
String roleName();
WorkerSelector workerSelector();
+
+ DataDefine dataDefine();
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerContext.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerContext.java
index 986397a2d..9c11acc52 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerContext.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerContext.java
@@ -11,6 +11,7 @@ import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
*/
public abstract class WorkerContext implements Context {
+ private Map remoteWorkerRefs;
private Map> roleWorkers;
private Map roles;
private Map dataDefineMap;
@@ -18,6 +19,7 @@ public abstract class WorkerContext implements Context {
public WorkerContext() {
this.roleWorkers = new HashMap<>();
this.roles = new HashMap<>();
+ this.remoteWorkerRefs = new HashMap<>();
}
private Map> getRoleWorkers() {
@@ -33,6 +35,14 @@ public abstract class WorkerContext implements Context {
}
}
+ @Override final public RemoteWorkerRef lookupInSide(String roleName) throws WorkerNotFoundException {
+ if (remoteWorkerRefs.containsKey(roleName)) {
+ return remoteWorkerRefs.get(roleName);
+ } else {
+ throw new WorkerNotFoundException("role=" + roleName + ", no available worker.");
+ }
+ }
+
public final void putRole(Role role) {
roles.put(role.roleName(), role);
}
@@ -47,9 +57,16 @@ public abstract class WorkerContext implements Context {
@Override final public void put(WorkerRef workerRef) {
if (!getRoleWorkers().containsKey(workerRef.getRole().roleName())) {
- getRoleWorkers().putIfAbsent(workerRef.getRole().roleName(), new ArrayList());
+ getRoleWorkers().putIfAbsent(workerRef.getRole().roleName(), new ArrayList<>());
}
getRoleWorkers().get(workerRef.getRole().roleName()).add(workerRef);
+
+ if (workerRef instanceof RemoteWorkerRef) {
+ RemoteWorkerRef remoteWorkerRef = (RemoteWorkerRef)workerRef;
+ if (!remoteWorkerRef.isAcrossJVM()) {
+ remoteWorkerRefs.put(workerRef.getRole().roleName(), remoteWorkerRef);
+ }
+ }
}
@Override final public void remove(WorkerRef workerRef) {
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerRefs.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerRefs.java
index a178802c4..1fb3dfeda 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerRefs.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerRefs.java
@@ -14,23 +14,21 @@ public class WorkerRefs {
private List workerRefs;
private WorkerSelector workerSelector;
+ private Role role;
protected WorkerRefs(List workerRefs, WorkerSelector workerSelector) {
this.workerRefs = workerRefs;
this.workerSelector = workerSelector;
}
- public void tell(Object message) throws WorkerInvokeException {
- logger.debug("WorkerSelector instance of %s", workerSelector.getClass());
- workerSelector.select(workerRefs, message).tell(message);
+ protected WorkerRefs(List workerRefs, WorkerSelector workerSelector, Role role) {
+ this.workerRefs = workerRefs;
+ this.workerSelector = workerSelector;
+ this.role = role;
}
- public void ask(Object request, Object response) throws WorkerInvokeException {
- WorkerRef workerRef = workerSelector.select(workerRefs, request);
- if (workerRef instanceof LocalSyncWorkerRef) {
- ((LocalSyncWorkerRef)workerRef).ask(request, response);
- } else {
- throw new IllegalAccessError("only local sync worker can ask");
- }
+ public void tell(Object message) throws WorkerInvokeException {
+ logger.debug("WorkerSelector instance of {}", workerSelector.getClass());
+ workerSelector.select(workerRefs, message).tell(message);
}
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Data.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Data.java
index b8318ca2e..28a2f3074 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Data.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Data.java
@@ -1,19 +1,31 @@
package org.skywalking.apm.collector.stream.worker.impl.data;
+import org.skywalking.apm.collector.remote.grpc.proto.RemoteData;
+
/**
* @author pengys5
*/
public class Data {
private int defineId;
+ private final int stringCapacity;
+ private final int longCapacity;
+ private final int floatCapacity;
+ private final int integerCapacity;
private String[] dataStrings;
private Long[] dataLongs;
private Float[] dataFloats;
+ private Integer[] dataIntegers;
- public Data(int defineId, int stringCapacity, int longCapacity, int floatCapacity) {
+ public Data(int defineId, int stringCapacity, int longCapacity, int floatCapacity, int integerCapacity) {
this.defineId = defineId;
this.dataStrings = new String[stringCapacity];
this.dataLongs = new Long[longCapacity];
this.dataFloats = new Float[floatCapacity];
+ this.dataIntegers = new Integer[integerCapacity];
+ this.stringCapacity = stringCapacity;
+ this.longCapacity = longCapacity;
+ this.floatCapacity = floatCapacity;
+ this.integerCapacity = integerCapacity;
}
public void setDataString(int position, String value) {
@@ -28,6 +40,10 @@ public class Data {
dataFloats[position] = value;
}
+ public void setDataInteger(int position, Integer value) {
+ dataIntegers[position] = value;
+ }
+
public String getDataString(int position) {
return dataStrings[position];
}
@@ -40,6 +56,10 @@ public class Data {
return dataFloats[position];
}
+ public Integer getDataInteger(int position) {
+ return dataIntegers[position];
+ }
+
public String id() {
return dataStrings[0];
}
@@ -47,4 +67,26 @@ public class Data {
public int getDefineId() {
return defineId;
}
+
+ public RemoteData serialize() {
+ RemoteData.Builder builder = RemoteData.newBuilder();
+ builder.setIntegerCapacity(integerCapacity);
+ builder.setFloatCapacity(floatCapacity);
+ builder.setStringCapacity(stringCapacity);
+ builder.setLongCapacity(longCapacity);
+
+ for (int i = 0; i < dataStrings.length; i++) {
+ builder.setDataStrings(i, dataStrings[i]);
+ }
+ for (int i = 0; i < dataIntegers.length; i++) {
+ builder.setDataIntegers(i, dataIntegers[i]);
+ }
+ for (int i = 0; i < dataFloats.length; i++) {
+ builder.setDataFloats(i, dataFloats[i]);
+ }
+ for (int i = 0; i < dataLongs.length; i++) {
+ builder.setDataLongs(i, dataLongs[i]);
+ }
+ return builder.build();
+ }
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefine.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefine.java
index 2c3f3d620..ebf571721 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefine.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefine.java
@@ -1,7 +1,6 @@
package org.skywalking.apm.collector.stream.worker.impl.data;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
+import org.skywalking.apm.collector.remote.grpc.proto.RemoteData;
/**
* @author pengys5
@@ -11,14 +10,18 @@ public abstract class DataDefine {
private int stringCapacity;
private int longCapacity;
private int floatCapacity;
+ private int integerCapacity;
public DataDefine() {
stringCapacity = 0;
longCapacity = 0;
floatCapacity = 0;
+ integerCapacity = 0;
}
public final void initial() {
+ attributes = new Attribute[initialCapacity()];
+ attributeDefine();
for (Attribute attribute : attributes) {
if (AttributeType.STRING.equals(attribute.getType())) {
stringCapacity++;
@@ -26,6 +29,8 @@ public abstract class DataDefine {
longCapacity++;
} else if (AttributeType.FLOAT.equals(attribute.getType())) {
floatCapacity++;
+ } else if (AttributeType.INTEGER.equals(attribute.getType())) {
+ integerCapacity++;
}
}
}
@@ -34,36 +39,21 @@ public abstract class DataDefine {
attributes[position] = attribute;
}
- public final void define() {
- attributes = new Attribute[initialCapacity()];
- }
-
- protected abstract int defineId();
+ public abstract int defineId();
protected abstract int initialCapacity();
protected abstract void attributeDefine();
- public int getStringCapacity() {
- return stringCapacity;
- }
-
- public int getLongCapacity() {
- return longCapacity;
- }
-
- public int getFloatCapacity() {
- return floatCapacity;
- }
-
- public Data build() {
- return new Data(defineId(), getStringCapacity(), getLongCapacity(), getFloatCapacity());
+ public final Data build() {
+ return new Data(defineId(), stringCapacity, longCapacity, floatCapacity, integerCapacity);
}
public void mergeData(Data newData, Data oldData) {
int stringPosition = 0;
int longPosition = 0;
int floatPosition = 0;
+ int integerPosition = 0;
for (int i = 0; i < initialCapacity(); i++) {
Attribute attribute = attributes[i];
if (AttributeType.STRING.equals(attribute.getType())) {
@@ -75,9 +65,14 @@ public abstract class DataDefine {
} else if (AttributeType.FLOAT.equals(attribute.getType())) {
attribute.getOperation().operate(newData.getDataFloat(floatPosition), oldData.getDataFloat(floatPosition));
floatPosition++;
+ } else if (AttributeType.FLOAT.equals(attribute.getType())) {
+ attribute.getOperation().operate(newData.getDataInteger(integerPosition), oldData.getDataInteger(integerPosition));
+ integerPosition++;
}
}
}
- public abstract Data parseFrom(ByteString bytesData) throws InvalidProtocolBufferException;
+ public abstract Object deserialize(RemoteData remoteData);
+
+ public abstract RemoteData serialize(Object object);
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java
index 425fcb427..cf39e6dd9 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java
@@ -22,6 +22,7 @@ public class DataDefineLoader implements Loader> {
DefinitionLoader definitionLoader = DefinitionLoader.load(DataDefine.class, definitionFile);
for (DataDefine dataDefine : definitionLoader) {
logger.info("loaded data definition class: {}", dataDefine.getClass().getName());
+ dataDefine.initial();
dataDefineMap.put(dataDefine.defineId(), dataDefine);
}
return dataDefineMap;
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Operation.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Operation.java
index 05bb5ed2a..fea3f9f87 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Operation.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Operation.java
@@ -9,4 +9,6 @@ public interface Operation {
Long operate(Long newValue, Long oldValue);
Float operate(Float newValue, Float oldValue);
+
+ Integer operate(Integer newValue, Integer oldValue);
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/CoverOperation.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/CoverOperation.java
index c0a48d077..99c4eb31b 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/CoverOperation.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/CoverOperation.java
@@ -17,4 +17,8 @@ public class CoverOperation implements Operation {
@Override public Float operate(Float newValue, Float oldValue) {
return newValue;
}
+
+ @Override public Integer operate(Integer newValue, Integer oldValue) {
+ return newValue;
+ }
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/NonOperation.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/NonOperation.java
index b306f1c99..7bc09ba17 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/NonOperation.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/NonOperation.java
@@ -17,4 +17,8 @@ public class NonOperation implements Operation {
@Override public Float operate(Float newValue, Float oldValue) {
return oldValue;
}
+
+ @Override public Integer operate(Integer newValue, Integer oldValue) {
+ return oldValue;
+ }
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/ForeverFirstSelector.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/ForeverFirstSelector.java
index 9a668d791..f4af9237f 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/ForeverFirstSelector.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/ForeverFirstSelector.java
@@ -2,13 +2,18 @@ package org.skywalking.apm.collector.stream.worker.selector;
import java.util.List;
import org.skywalking.apm.collector.stream.worker.WorkerRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* @author pengys5
*/
public class ForeverFirstSelector implements WorkerSelector {
+ private final Logger logger = LoggerFactory.getLogger(ForeverFirstSelector.class);
+
@Override public WorkerRef select(List members, Object message) {
+ logger.debug("member size: {}", members.size());
return members.get(0);
}
}
diff --git a/apm-collector/apm-collector-stream/src/main/proto/RemoteCommonService.proto b/apm-collector/apm-collector-stream/src/main/proto/RemoteCommonService.proto
index bf0f2d8bc..4160d7c64 100644
--- a/apm-collector/apm-collector-stream/src/main/proto/RemoteCommonService.proto
+++ b/apm-collector/apm-collector-stream/src/main/proto/RemoteCommonService.proto
@@ -4,14 +4,24 @@ option java_multiple_files = true;
option java_package = "org.skywalking.apm.collector.remote.grpc.proto";
service RemoteCommonService {
- rpc call (Message) returns (Empty) {
+ rpc call (RemoteMessage) returns (Empty) {
}
}
-message Message {
+message RemoteMessage {
string workerRole = 1;
- int32 dataDefineId = 2;
- bytes dataBytes = 3;
+ RemoteData remoteData = 2;
+}
+
+message RemoteData {
+ int32 stringCapacity = 1;
+ int32 longCapacity = 2;
+ int32 floatCapacity = 3;
+ int32 integerCapacity = 4;
+ repeated string dataStrings = 5;
+ repeated int64 dataLongs = 6;
+ repeated float dataFloats = 7;
+ repeated int32 dataIntegers = 8;
}
message Empty {
From e507bb9e6a8c086bd15dee8778b1bb22b778c416 Mon Sep 17 00:00:00 2001
From: pengys5 <8082209@qq.com>
Date: Wed, 26 Jul 2017 23:51:14 +0800
Subject: [PATCH 13/28] 1. Application register successfully. 2. Instance
discovery table and data define finish.
---
.../application/ApplicationEsDAO.java | 31 ------
.../application/ApplicationH2DAO.java | 15 ---
...rCreate.java => ApplicationIDService.java} | 5 +-
.../application/IApplicationDAO.java | 8 --
.../ApplicationRegisterServiceHandler.java | 6 +-
.../InstanceDiscoveryServiceHandler.java | 22 ++++-
.../agentregister/instance/IInstanceDAO.java | 8 ++
.../instance/InstanceIDService.java | 44 +++++++++
.../resources/META-INF/defines/es_dao.define | 1 -
.../resources/META-INF/defines/h2_dao.define | 1 -
...icationRegisterServiceHandlerTestCase.java | 2 +-
.../application/ApplicationDataDefine.java | 18 +++-
.../ApplicationRegisterRemoteWorker.java | 1 +
.../ApplicationRegisterSerialWorker.java | 38 +++++++-
.../application/dao/ApplicationEsDAO.java | 97 +++++++++++++++++++
.../application/dao/ApplicationH2DAO.java | 28 ++++++
.../application/dao/IApplicationDAO.java | 16 +++
.../register/instance/InstanceDataDefine.java | 88 +++++++++++++++++
.../instance/InstanceEsTableDefine.java | 34 +++++++
.../instance/InstanceH2TableDefine.java | 22 +++++
.../InstanceRegisterRemoteWorker.java | 64 ++++++++++++
.../register/instance/InstanceTable.java | 13 +++
.../resources/META-INF/defines/es_dao.define | 1 +
.../resources/META-INF/defines/h2_dao.define | 1 +
.../defines/remote_worker_provider.define | 3 +-
.../resources/META-INF/defines/storage.define | 5 +-
.../elasticsearch/ElasticSearchClient.java | 4 +-
.../apm/collector/storage/dao/DAO.java | 4 +-
.../worker/AbstractLocalAsyncWorker.java | 8 --
.../AbstractLocalAsyncWorkerProvider.java | 1 +
.../stream/worker/AbstractRemoteWorker.java | 8 --
.../stream/worker/AbstractWorker.java | 18 +++-
.../LocalAsyncWorkerProviderDefineLoader.java | 1 -
33 files changed, 521 insertions(+), 95 deletions(-)
delete mode 100644 apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationEsDAO.java
delete mode 100644 apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationH2DAO.java
rename apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/{ApplicationIDGetOrCreate.java => ApplicationIDService.java} (91%)
delete mode 100644 apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/IApplicationDAO.java
create mode 100644 apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/instance/IInstanceDAO.java
create mode 100644 apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/instance/InstanceIDService.java
delete mode 100644 apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/es_dao.define
delete mode 100644 apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/h2_dao.define
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/ApplicationEsDAO.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/ApplicationH2DAO.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/IApplicationDAO.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceEsTableDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceH2TableDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceRegisterRemoteWorker.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceTable.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define
create mode 100644 apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationEsDAO.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationEsDAO.java
deleted file mode 100644
index 24a9b5c5b..000000000
--- a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationEsDAO.java
+++ /dev/null
@@ -1,31 +0,0 @@
-package org.skywalking.apm.collector.agentregister.application;
-
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationTable;
-import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient;
-import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO;
-
-/**
- * @author pengys5
- */
-public class ApplicationEsDAO extends EsDAO implements IApplicationDAO {
-
- @Override public int getApplicationId(String applicationCode) {
- ElasticSearchClient client = getClient();
-
- SearchRequestBuilder searchRequestBuilder = client.prepareSearch(ApplicationTable.TABLE);
- searchRequestBuilder.setTypes("type");
- searchRequestBuilder.setSearchType(SearchType.QUERY_THEN_FETCH);
- searchRequestBuilder.setQuery(QueryBuilders.termQuery(ApplicationTable.COLUMN_APPLICATION_CODE, applicationCode));
- searchRequestBuilder.setSize(1);
-
- SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
- if (searchResponse.getHits().totalHits > 0) {
- return searchResponse.getHits().getAt(0).getField(ApplicationTable.COLUMN_APPLICATION_ID).getValue();
- }
- return 0;
- }
-}
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationH2DAO.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationH2DAO.java
deleted file mode 100644
index 3bf09f6d8..000000000
--- a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationH2DAO.java
+++ /dev/null
@@ -1,15 +0,0 @@
-package org.skywalking.apm.collector.agentregister.application;
-
-import org.skywalking.apm.collector.client.h2.H2Client;
-import org.skywalking.apm.collector.storage.h2.dao.H2DAO;
-
-/**
- * @author pengys5
- */
-public class ApplicationH2DAO extends H2DAO implements IApplicationDAO {
-
- @Override public int getApplicationId(String applicationCode) {
- H2Client client = getClient();
- return 100;
- }
-}
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDGetOrCreate.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDService.java
similarity index 91%
rename from apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDGetOrCreate.java
rename to apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDService.java
index 1535e57a0..2cf836e90 100644
--- a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDGetOrCreate.java
+++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDService.java
@@ -2,6 +2,7 @@ package org.skywalking.apm.collector.agentregister.application;
import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine;
import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterRemoteWorker;
+import org.skywalking.apm.collector.agentstream.worker.register.application.dao.IApplicationDAO;
import org.skywalking.apm.collector.core.framework.CollectorContextHelper;
import org.skywalking.apm.collector.storage.dao.DAOContainer;
import org.skywalking.apm.collector.stream.StreamModuleContext;
@@ -14,9 +15,9 @@ import org.slf4j.LoggerFactory;
/**
* @author pengys5
*/
-public class ApplicationIDGetOrCreate {
+public class ApplicationIDService {
- private final Logger logger = LoggerFactory.getLogger(ApplicationIDGetOrCreate.class);
+ private final Logger logger = LoggerFactory.getLogger(ApplicationIDService.class);
public int getOrCreate(String applicationCode) {
IApplicationDAO dao = (IApplicationDAO)DAOContainer.INSTANCE.get(IApplicationDAO.class.getName());
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/IApplicationDAO.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/IApplicationDAO.java
deleted file mode 100644
index 38b78ab5e..000000000
--- a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/IApplicationDAO.java
+++ /dev/null
@@ -1,8 +0,0 @@
-package org.skywalking.apm.collector.agentregister.application;
-
-/**
- * @author pengys5
- */
-public interface IApplicationDAO {
- int getApplicationId(String applicationCode);
-}
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandler.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandler.java
index cd7329895..413c35552 100644
--- a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandler.java
+++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandler.java
@@ -2,7 +2,7 @@ package org.skywalking.apm.collector.agentregister.grpc.handler;
import com.google.protobuf.ProtocolStringList;
import io.grpc.stub.StreamObserver;
-import org.skywalking.apm.collector.agentregister.application.ApplicationIDGetOrCreate;
+import org.skywalking.apm.collector.agentregister.application.ApplicationIDService;
import org.skywalking.apm.collector.server.grpc.GRPCHandler;
import org.skywalking.apm.network.proto.Application;
import org.skywalking.apm.network.proto.ApplicationMapping;
@@ -18,14 +18,14 @@ public class ApplicationRegisterServiceHandler extends ApplicationRegisterServic
private final Logger logger = LoggerFactory.getLogger(ApplicationRegisterServiceHandler.class);
- private ApplicationIDGetOrCreate applicationIDGetOrCreate = new ApplicationIDGetOrCreate();
+ private ApplicationIDService applicationIDService = new ApplicationIDService();
@Override public void register(Application request, StreamObserver responseObserver) {
logger.debug("register application");
ProtocolStringList applicationCodes = request.getApplicationCodeList();
for (int i = 0; i < applicationCodes.size(); i++) {
String applicationCode = applicationCodes.get(i);
- int applicationId = applicationIDGetOrCreate.getOrCreate(applicationCode);
+ int applicationId = applicationIDService.getOrCreate(applicationCode);
KeyWithIntegerValue value = KeyWithIntegerValue.newBuilder().setKey(applicationCode).setValue(applicationId).build();
ApplicationMapping mapping = ApplicationMapping.newBuilder().addApplication(i, value).build();
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/InstanceDiscoveryServiceHandler.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/InstanceDiscoveryServiceHandler.java
index 70f64ed9b..20ead8290 100644
--- a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/InstanceDiscoveryServiceHandler.java
+++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/InstanceDiscoveryServiceHandler.java
@@ -1,6 +1,7 @@
package org.skywalking.apm.collector.agentregister.grpc.handler;
import io.grpc.stub.StreamObserver;
+import org.skywalking.apm.collector.agentregister.instance.InstanceIDService;
import org.skywalking.apm.collector.server.grpc.GRPCHandler;
import org.skywalking.apm.network.proto.ApplicationInstance;
import org.skywalking.apm.network.proto.ApplicationInstanceHeartbeat;
@@ -8,23 +9,38 @@ import org.skywalking.apm.network.proto.ApplicationInstanceMapping;
import org.skywalking.apm.network.proto.ApplicationInstanceRecover;
import org.skywalking.apm.network.proto.Downstream;
import org.skywalking.apm.network.proto.InstanceDiscoveryServiceGrpc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* @author pengys5
*/
public class InstanceDiscoveryServiceHandler extends InstanceDiscoveryServiceGrpc.InstanceDiscoveryServiceImplBase implements GRPCHandler {
+ private final Logger logger = LoggerFactory.getLogger(InstanceDiscoveryServiceHandler.class);
+
+ private InstanceIDService instanceIDService = new InstanceIDService();
+
@Override
public void register(ApplicationInstance request, StreamObserver responseObserver) {
- super.register(request, responseObserver);
+ int instanceId = instanceIDService.getOrCreate(request.getApplicationId(), request.getAgentUUID(), request.getRegisterTime());
+ ApplicationInstanceMapping.Builder builder = ApplicationInstanceMapping.newBuilder();
+ builder.setApplicationId(request.getApplicationId());
+ builder.setApplicationInstanceId(instanceId);
+ responseObserver.onNext(builder.build());
+ responseObserver.onCompleted();
}
@Override public void heartbeat(ApplicationInstanceHeartbeat request, StreamObserver responseObserver) {
- super.heartbeat(request, responseObserver);
+ instanceIDService.heartBeat(request.getApplicationInstanceId(), request.getHeartbeatTime());
+ responseObserver.onNext(Downstream.newBuilder().build());
+ responseObserver.onCompleted();
}
@Override
public void registerRecover(ApplicationInstanceRecover request, StreamObserver responseObserver) {
- super.registerRecover(request, responseObserver);
+ instanceIDService.recover(request.getApplicationInstanceId(), request.getApplicationId(), request.getRegisterTime());
+ responseObserver.onNext(Downstream.newBuilder().build());
+ responseObserver.onCompleted();
}
}
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/instance/IInstanceDAO.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/instance/IInstanceDAO.java
new file mode 100644
index 000000000..7cdb17369
--- /dev/null
+++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/instance/IInstanceDAO.java
@@ -0,0 +1,8 @@
+package org.skywalking.apm.collector.agentregister.instance;
+
+/**
+ * @author pengys5
+ */
+public interface IInstanceDAO {
+ int getInstanceId(String agentUUID);
+}
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/instance/InstanceIDService.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/instance/InstanceIDService.java
new file mode 100644
index 000000000..9a74f3118
--- /dev/null
+++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/instance/InstanceIDService.java
@@ -0,0 +1,44 @@
+package org.skywalking.apm.collector.agentregister.instance;
+
+import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterRemoteWorker;
+import org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceDataDefine;
+import org.skywalking.apm.collector.core.framework.CollectorContextHelper;
+import org.skywalking.apm.collector.storage.dao.DAOContainer;
+import org.skywalking.apm.collector.stream.StreamModuleContext;
+import org.skywalking.apm.collector.stream.StreamModuleGroupDefine;
+import org.skywalking.apm.collector.stream.worker.WorkerInvokeException;
+import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class InstanceIDService {
+
+ private final Logger logger = LoggerFactory.getLogger(InstanceIDService.class);
+
+ public int getOrCreate(int applicationId, String agentUUID, long registerTime) {
+ IInstanceDAO dao = (IInstanceDAO)DAOContainer.INSTANCE.get(IInstanceDAO.class.getName());
+ int instanceId = dao.getInstanceId(agentUUID);
+
+ if (instanceId == 0) {
+ StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME);
+ InstanceDataDefine.Instance instance = new InstanceDataDefine.Instance(agentUUID, applicationId, agentUUID, registerTime, 0);
+ try {
+ context.getClusterWorkerContext().lookup(ApplicationRegisterRemoteWorker.WorkerRole.INSTANCE).tell(instance);
+ } catch (WorkerNotFoundException | WorkerInvokeException e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+ return applicationId;
+ }
+
+ public void heartBeat(int instanceId, long heartbeatTime) {
+
+ }
+
+ public void recover(int instanceId, int applicationId, long registerTime) {
+
+ }
+}
diff --git a/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/es_dao.define b/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/es_dao.define
deleted file mode 100644
index 36d1e8c1a..000000000
--- a/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/es_dao.define
+++ /dev/null
@@ -1 +0,0 @@
-org.skywalking.apm.collector.agentregister.application.ApplicationEsDAO
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/h2_dao.define b/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/h2_dao.define
deleted file mode 100644
index c6024dcf1..000000000
--- a/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/h2_dao.define
+++ /dev/null
@@ -1 +0,0 @@
-org.skywalking.apm.collector.agentregister.application.ApplicationH2DAO
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentregister/src/test/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandlerTestCase.java b/apm-collector/apm-collector-agentregister/src/test/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandlerTestCase.java
index cb6bbb414..d71a04333 100644
--- a/apm-collector/apm-collector-agentregister/src/test/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandlerTestCase.java
+++ b/apm-collector/apm-collector-agentregister/src/test/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandlerTestCase.java
@@ -19,7 +19,7 @@ public class ApplicationRegisterServiceHandlerTestCase {
ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 11800).usePlaintext(true).build();
stub = ApplicationRegisterServiceGrpc.newBlockingStub(channel);
- Application application = Application.newBuilder().addApplicationCode("test").build();
+ Application application = Application.newBuilder().addApplicationCode("test141").build();
ApplicationMapping mapping = stub.register(application);
System.out.println(mapping.getApplication(0).getKey() + ", " + mapping.getApplication(0).getValue());
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationDataDefine.java
index 35bb52c7b..cb964a3a4 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationDataDefine.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationDataDefine.java
@@ -45,9 +45,9 @@ public class ApplicationDataDefine extends DataDefine {
}
public static class Application {
- private final String id;
- private final String applicationCode;
- private final int applicationId;
+ private String id;
+ private String applicationCode;
+ private int applicationId;
public Application(String id, String applicationCode, int applicationId) {
this.id = id;
@@ -66,5 +66,17 @@ public class ApplicationDataDefine extends DataDefine {
public int getApplicationId() {
return applicationId;
}
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public void setApplicationCode(String applicationCode) {
+ this.applicationCode = applicationCode;
+ }
+
+ public void setApplicationId(int applicationId) {
+ this.applicationId = applicationId;
+ }
}
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterRemoteWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterRemoteWorker.java
index 2ead835fa..fad05955a 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterRemoteWorker.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterRemoteWorker.java
@@ -29,6 +29,7 @@ public class ApplicationRegisterRemoteWorker extends AbstractRemoteWorker {
@Override protected void onWork(Object message) throws WorkerException {
ApplicationDataDefine.Application application = (ApplicationDataDefine.Application)message;
logger.debug("application code: {}", application.getApplicationCode());
+ getClusterContext().lookup(ApplicationRegisterSerialWorker.WorkerRole.INSTANCE).tell(application);
}
public static class Factory extends AbstractRemoteWorkerProvider {
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java
index 49b80a365..ff01be792 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java
@@ -1,5 +1,7 @@
package org.skywalking.apm.collector.agentstream.worker.register.application;
+import org.skywalking.apm.collector.agentstream.worker.register.application.dao.IApplicationDAO;
+import org.skywalking.apm.collector.storage.dao.DAOContainer;
import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorker;
import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider;
import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
@@ -7,14 +9,18 @@ import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
import org.skywalking.apm.collector.stream.worker.Role;
import org.skywalking.apm.collector.stream.worker.WorkerException;
import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
-import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector;
+import org.skywalking.apm.collector.stream.worker.selector.ForeverFirstSelector;
import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* @author pengys5
*/
public class ApplicationRegisterSerialWorker extends AbstractLocalAsyncWorker {
+ private final Logger logger = LoggerFactory.getLogger(ApplicationRegisterSerialWorker.class);
+
public ApplicationRegisterSerialWorker(Role role, ClusterWorkerContext clusterContext) {
super(role, clusterContext);
}
@@ -24,7 +30,34 @@ public class ApplicationRegisterSerialWorker extends AbstractLocalAsyncWorker {
}
@Override protected void onWork(Object message) throws WorkerException {
+ if (message instanceof ApplicationDataDefine.Application) {
+ ApplicationDataDefine.Application application = (ApplicationDataDefine.Application)message;
+ logger.debug("register application, application code: {}", application.getApplicationCode());
+ IApplicationDAO dao = (IApplicationDAO)DAOContainer.INSTANCE.get(IApplicationDAO.class.getName());
+ int min = dao.getMinApplicationId();
+ if (min == 0) {
+ application.setApplicationId(1);
+ application.setId("1");
+ } else {
+ int max = dao.getMaxApplicationId();
+ int instanceId;
+ if (min == max) {
+ instanceId = -1;
+ } else if (min + max == 0) {
+ instanceId = max + 1;
+ } else if (min + max > 0) {
+ instanceId = min - 1;
+ } else if (max < 0) {
+ instanceId = 1;
+ } else {
+ instanceId = max + 1;
+ }
+ application.setApplicationId(instanceId);
+ application.setId(String.valueOf(instanceId));
+ }
+ dao.save(application);
+ }
}
public static class Factory extends AbstractLocalAsyncWorkerProvider {
@@ -53,12 +86,11 @@ public class ApplicationRegisterSerialWorker extends AbstractLocalAsyncWorker {
@Override
public WorkerSelector workerSelector() {
- return new HashCodeSelector();
+ return new ForeverFirstSelector();
}
@Override public DataDefine dataDefine() {
return new ApplicationDataDefine();
}
-
}
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/ApplicationEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/ApplicationEsDAO.java
new file mode 100644
index 000000000..7f12f50c3
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/ApplicationEsDAO.java
@@ -0,0 +1,97 @@
+package org.skywalking.apm.collector.agentstream.worker.register.application.dao;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.metrics.max.Max;
+import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.min.Min;
+import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
+import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine;
+import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationTable;
+import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient;
+import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class ApplicationEsDAO extends EsDAO implements IApplicationDAO {
+
+ private final Logger logger = LoggerFactory.getLogger(ApplicationEsDAO.class);
+
+ @Override public int getApplicationId(String applicationCode) {
+ ElasticSearchClient client = getClient();
+
+ SearchRequestBuilder searchRequestBuilder = client.prepareSearch(ApplicationTable.TABLE);
+ searchRequestBuilder.setTypes("type");
+ searchRequestBuilder.setSearchType(SearchType.QUERY_THEN_FETCH);
+ searchRequestBuilder.setQuery(QueryBuilders.termQuery(ApplicationTable.COLUMN_APPLICATION_CODE, applicationCode));
+ searchRequestBuilder.setSize(1);
+
+ SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
+ if (searchResponse.getHits().totalHits > 0) {
+ SearchHit searchHit = searchResponse.getHits().iterator().next();
+ int applicationId = (int)searchHit.getSource().get(ApplicationTable.COLUMN_APPLICATION_ID);
+ return applicationId;
+ }
+ return 0;
+ }
+
+ @Override public int getMaxApplicationId() {
+ ElasticSearchClient client = getClient();
+ SearchRequestBuilder searchRequestBuilder = client.prepareSearch(ApplicationTable.TABLE);
+ searchRequestBuilder.setTypes("type");
+ searchRequestBuilder.setSize(0);
+ MaxAggregationBuilder aggregation = AggregationBuilders.max("agg").field(ApplicationTable.COLUMN_APPLICATION_ID);
+ searchRequestBuilder.addAggregation(aggregation);
+
+ SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
+ Max agg = searchResponse.getAggregations().get("agg");
+
+ int id = (int)agg.getValue();
+ if (id == Integer.MAX_VALUE || id == Integer.MIN_VALUE) {
+ return 0;
+ } else {
+ return id;
+ }
+ }
+
+ @Override public int getMinApplicationId() {
+ ElasticSearchClient client = getClient();
+ SearchRequestBuilder searchRequestBuilder = client.prepareSearch(ApplicationTable.TABLE);
+ searchRequestBuilder.setTypes("type");
+ searchRequestBuilder.setSize(0);
+ MinAggregationBuilder aggregation = AggregationBuilders.min("agg").field(ApplicationTable.COLUMN_APPLICATION_ID);
+ searchRequestBuilder.addAggregation(aggregation);
+
+ SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
+ Min agg = searchResponse.getAggregations().get("agg");
+
+ int id = (int)agg.getValue();
+ if (id == Integer.MAX_VALUE || id == Integer.MIN_VALUE) {
+ return 0;
+ } else {
+ return id;
+ }
+ }
+
+ @Override public void save(ApplicationDataDefine.Application application) {
+ logger.debug("save application register info, application id: {}, application code: {}", application.getApplicationId(), application.getApplicationCode());
+ ElasticSearchClient client = getClient();
+ Map source = new HashMap();
+ source.put(ApplicationTable.COLUMN_APPLICATION_CODE, application.getApplicationCode());
+ source.put(ApplicationTable.COLUMN_APPLICATION_ID, application.getApplicationId());
+
+ IndexResponse response = client.prepareIndex(ApplicationTable.TABLE, application.getId()).setSource(source).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get();
+ logger.debug("save application register info, application id: {}, application code: {}, status: {}", application.getApplicationId(), application.getApplicationCode(), response.status().name());
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/ApplicationH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/ApplicationH2DAO.java
new file mode 100644
index 000000000..3c6a03e9e
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/ApplicationH2DAO.java
@@ -0,0 +1,28 @@
+package org.skywalking.apm.collector.agentstream.worker.register.application.dao;
+
+import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine;
+import org.skywalking.apm.collector.client.h2.H2Client;
+import org.skywalking.apm.collector.storage.h2.dao.H2DAO;
+
+/**
+ * @author pengys5
+ */
+public class ApplicationH2DAO extends H2DAO implements IApplicationDAO {
+
+ @Override public int getApplicationId(String applicationCode) {
+ H2Client client = getClient();
+ return 100;
+ }
+
+ @Override public int getMaxApplicationId() {
+ return 0;
+ }
+
+ @Override public int getMinApplicationId() {
+ return 0;
+ }
+
+ @Override public void save(ApplicationDataDefine.Application application) {
+
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/IApplicationDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/IApplicationDAO.java
new file mode 100644
index 000000000..ad1083a00
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/IApplicationDAO.java
@@ -0,0 +1,16 @@
+package org.skywalking.apm.collector.agentstream.worker.register.application.dao;
+
+import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine;
+
+/**
+ * @author pengys5
+ */
+public interface IApplicationDAO {
+ int getApplicationId(String applicationCode);
+
+ int getMaxApplicationId();
+
+ int getMinApplicationId();
+
+ void save(ApplicationDataDefine.Application application);
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java
new file mode 100644
index 000000000..446a47f8e
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java
@@ -0,0 +1,88 @@
+package org.skywalking.apm.collector.agentstream.worker.register.instance;
+
+import org.skywalking.apm.collector.remote.grpc.proto.RemoteData;
+import org.skywalking.apm.collector.stream.worker.impl.data.Attribute;
+import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
+import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation;
+import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation;
+
+/**
+ * @author pengys5
+ */
+public class InstanceDataDefine extends DataDefine {
+
+ public static final int DEFINE_ID = 102;
+
+ @Override public int defineId() {
+ return DEFINE_ID;
+ }
+
+ @Override protected int initialCapacity() {
+ return 3;
+ }
+
+ @Override protected void attributeDefine() {
+ addAttribute(0, new Attribute("id", AttributeType.STRING, new NonOperation()));
+ addAttribute(1, new Attribute(InstanceTable.COLUMN_APPLICATION_ID, AttributeType.INTEGER, new CoverOperation()));
+ addAttribute(2, new Attribute(InstanceTable.COLUMN_AGENTUUID, AttributeType.STRING, new CoverOperation()));
+ addAttribute(3, new Attribute(InstanceTable.COLUMN_REGISTER_TIME, AttributeType.LONG, new CoverOperation()));
+ addAttribute(4, new Attribute(InstanceTable.COLUMN_INSTANCE_ID, AttributeType.INTEGER, new CoverOperation()));
+ addAttribute(4, new Attribute(InstanceTable.COLUMN_HEARTBEAT_TIME, AttributeType.LONG, new CoverOperation()));
+ }
+
+ @Override public Object deserialize(RemoteData remoteData) {
+ String id = remoteData.getDataStrings(0);
+ int applicationId = remoteData.getDataIntegers(0);
+ String agentUUID = remoteData.getDataStrings(1);
+ int instanceId = remoteData.getDataIntegers(1);
+ long registerTime = remoteData.getDataLongs(0);
+ return new Instance(id, applicationId, agentUUID, registerTime, instanceId);
+ }
+
+ @Override public RemoteData serialize(Object object) {
+ Instance instance = (Instance)object;
+ RemoteData.Builder builder = RemoteData.newBuilder();
+ builder.addDataStrings(instance.getId());
+ builder.addDataIntegers(instance.getApplicationId());
+ builder.addDataStrings(instance.getAgentUUID());
+ builder.addDataLongs(instance.getRegisterTime());
+ return builder.build();
+ }
+
+ public static class Instance {
+ private final String id;
+ private final int applicationId;
+ private final String agentUUID;
+ private final long registerTime;
+ private final int instanceId;
+
+ public Instance(String id, int applicationId, String agentUUID, long registerTime, int instanceId) {
+ this.id = id;
+ this.applicationId = applicationId;
+ this.agentUUID = agentUUID;
+ this.registerTime = registerTime;
+ this.instanceId = instanceId;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public int getApplicationId() {
+ return applicationId;
+ }
+
+ public String getAgentUUID() {
+ return agentUUID;
+ }
+
+ public long getRegisterTime() {
+ return registerTime;
+ }
+
+ public int getInstanceId() {
+ return instanceId;
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceEsTableDefine.java
new file mode 100644
index 000000000..dffcfa9fb
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceEsTableDefine.java
@@ -0,0 +1,34 @@
+package org.skywalking.apm.collector.agentstream.worker.register.instance;
+
+import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine;
+import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine;
+
+/**
+ * @author pengys5
+ */
+public class InstanceEsTableDefine extends ElasticSearchTableDefine {
+
+ public InstanceEsTableDefine() {
+ super(InstanceTable.TABLE);
+ }
+
+ @Override public int refreshInterval() {
+ return 0;
+ }
+
+ @Override public int numberOfShards() {
+ return 2;
+ }
+
+ @Override public int numberOfReplicas() {
+ return 0;
+ }
+
+ @Override public void initialize() {
+ addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_APPLICATION_ID, ElasticSearchColumnDefine.Type.Integer.name()));
+ addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_AGENTUUID, ElasticSearchColumnDefine.Type.Keyword.name()));
+ addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_REGISTER_TIME, ElasticSearchColumnDefine.Type.Long.name()));
+ addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_INSTANCE_ID, ElasticSearchColumnDefine.Type.Integer.name()));
+ addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_HEARTBEAT_TIME, ElasticSearchColumnDefine.Type.Long.name()));
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceH2TableDefine.java
new file mode 100644
index 000000000..58ca3e626
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceH2TableDefine.java
@@ -0,0 +1,22 @@
+package org.skywalking.apm.collector.agentstream.worker.register.instance;
+
+import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine;
+import org.skywalking.apm.collector.storage.h2.define.H2TableDefine;
+
+/**
+ * @author pengys5
+ */
+public class InstanceH2TableDefine extends H2TableDefine {
+
+ public InstanceH2TableDefine() {
+ super(InstanceTable.TABLE);
+ }
+
+ @Override public void initialize() {
+ addColumn(new H2ColumnDefine(InstanceTable.COLUMN_APPLICATION_ID, H2ColumnDefine.Type.Int.name()));
+ addColumn(new H2ColumnDefine(InstanceTable.COLUMN_AGENTUUID, H2ColumnDefine.Type.Varchar.name()));
+ addColumn(new H2ColumnDefine(InstanceTable.COLUMN_REGISTER_TIME, H2ColumnDefine.Type.Bigint.name()));
+ addColumn(new H2ColumnDefine(InstanceTable.COLUMN_INSTANCE_ID, H2ColumnDefine.Type.Int.name()));
+ addColumn(new H2ColumnDefine(InstanceTable.COLUMN_HEARTBEAT_TIME, H2ColumnDefine.Type.Bigint.name()));
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceRegisterRemoteWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceRegisterRemoteWorker.java
new file mode 100644
index 000000000..3e7cd93ec
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceRegisterRemoteWorker.java
@@ -0,0 +1,64 @@
+package org.skywalking.apm.collector.agentstream.worker.register.instance;
+
+import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine;
+import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorker;
+import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorkerProvider;
+import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
+import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
+import org.skywalking.apm.collector.stream.worker.Role;
+import org.skywalking.apm.collector.stream.worker.WorkerException;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
+import org.skywalking.apm.collector.stream.worker.selector.ForeverFirstSelector;
+import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class InstanceRegisterRemoteWorker extends AbstractRemoteWorker {
+
+ private final Logger logger = LoggerFactory.getLogger(InstanceRegisterRemoteWorker.class);
+
+ protected InstanceRegisterRemoteWorker(Role role, ClusterWorkerContext clusterContext) {
+ super(role, clusterContext);
+ }
+
+ @Override public void preStart() throws ProviderNotFoundException {
+ }
+
+ @Override protected void onWork(Object message) throws WorkerException {
+ InstanceDataDefine.Instance instance = (InstanceDataDefine.Instance)message;
+ logger.debug("application id: {}, agentUUID: {}, register time: {}", instance.getApplicationId(), instance.getAgentUUID(), instance.getRegisterTime());
+ }
+
+ public static class Factory extends AbstractRemoteWorkerProvider {
+ @Override
+ public Role role() {
+ return WorkerRole.INSTANCE;
+ }
+
+ @Override
+ public InstanceRegisterRemoteWorker workerInstance(ClusterWorkerContext clusterContext) {
+ return new InstanceRegisterRemoteWorker(role(), clusterContext);
+ }
+ }
+
+ public enum WorkerRole implements Role {
+ INSTANCE;
+
+ @Override
+ public String roleName() {
+ return InstanceRegisterRemoteWorker.class.getSimpleName();
+ }
+
+ @Override
+ public WorkerSelector workerSelector() {
+ return new ForeverFirstSelector();
+ }
+
+ @Override public DataDefine dataDefine() {
+ return new ApplicationDataDefine();
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceTable.java
new file mode 100644
index 000000000..5a26adb6e
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceTable.java
@@ -0,0 +1,13 @@
+package org.skywalking.apm.collector.agentstream.worker.register.instance;
+
+/**
+ * @author pengys5
+ */
+public class InstanceTable {
+ public static final String TABLE = "instance";
+ public static final String COLUMN_APPLICATION_ID = "application_id";
+ public static final String COLUMN_AGENTUUID = "agent_uuid";
+ public static final String COLUMN_REGISTER_TIME = "register_time";
+ public static final String COLUMN_INSTANCE_ID = "instance_id";
+ public static final String COLUMN_HEARTBEAT_TIME = "heartbeatTime";
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define
new file mode 100644
index 000000000..2a346cdbf
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define
@@ -0,0 +1 @@
+org.skywalking.apm.collector.agentstream.worker.register.application.dao.ApplicationEsDAO
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define
new file mode 100644
index 000000000..c243fe509
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define
@@ -0,0 +1 @@
+org.skywalking.apm.collector.agentstream.worker.register.application.dao.ApplicationH2DAO
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define
index 84c9c058e..8fccdab58 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define
@@ -1 +1,2 @@
-org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterRemoteWorker$Factory
\ No newline at end of file
+org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterRemoteWorker$Factory
+org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceRegisterRemoteWorker$Factory
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define
index 12f4a13a6..87b96eb9d 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define
@@ -4,4 +4,7 @@ org.skywalking.apm.collector.agentstream.worker.node.define.NodeMappingEsTableDe
org.skywalking.apm.collector.agentstream.worker.node.define.NodeMappingH2TableDefine
org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationEsTableDefine
-org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationH2TableDefine
\ No newline at end of file
+org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationH2TableDefine
+
+org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceEsTableDefine
+org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceH2TableDefine
\ No newline at end of file
diff --git a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java
index 884907803..303222e16 100644
--- a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java
+++ b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java
@@ -106,7 +106,7 @@ public class ElasticSearchClient implements Client {
return client.prepareSearch(indexName);
}
- public IndexRequestBuilder prepareIndex(String indexName) {
- return null;
+ public IndexRequestBuilder prepareIndex(String indexName, String id) {
+ return client.prepareIndex(indexName, "type", id);
}
}
diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/DAO.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/DAO.java
index 3e2d14c36..1ddeb9445 100644
--- a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/DAO.java
+++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/DAO.java
@@ -8,11 +8,11 @@ import org.skywalking.apm.collector.core.client.Client;
public abstract class DAO {
private C client;
- public C getClient() {
+ public final C getClient() {
return client;
}
- public void setClient(C client) {
+ public final void setClient(C client) {
this.client = client;
}
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorker.java
index 17fd2edfd..e43561bee 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorker.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorker.java
@@ -41,12 +41,4 @@ public abstract class AbstractLocalAsyncWorker extends AbstractLocalWorker imple
final public void allocateJob(Object message) throws WorkerException {
onWork(message);
}
-
- /**
- * The data process logic in this method.
- *
- * @param message Cast the message object to a expect subclass.
- * @throws WorkerException Don't handle the exception, throw it.
- */
- protected abstract void onWork(Object message) throws WorkerException;
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java
index 712b13a76..c7e4ff5c2 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java
@@ -22,6 +22,7 @@ public abstract class AbstractLocalAsyncWorkerProvider definitionLoader = DefinitionLoader.load(AbstractLocalAsyncWorkerProvider.class, definitionFile);
- int id = 1;
for (AbstractLocalAsyncWorkerProvider provider : definitionLoader) {
logger.info("loaded local async worker provider definition class: {}", provider.getClass().getName());
providers.add(provider);
From 54fe8e43f8a337afc396d03c4d4277fa0fd23f37 Mon Sep 17 00:00:00 2001
From: pengys5 <8082209@qq.com>
Date: Thu, 27 Jul 2017 09:45:45 +0800
Subject: [PATCH 14/28] Instance register, heartBeat, recover finish
---
.../agentregister/instance/IInstanceDAO.java | 8 --
.../instance/InstanceIDService.java | 11 ++-
.../worker/register/IdAutoIncrement.java | 24 +++++
.../ApplicationRegisterSerialWorker.java | 18 +---
.../application/dao/ApplicationEsDAO.java | 39 +--------
.../register/instance/InstanceDataDefine.java | 30 +++++--
.../InstanceRegisterRemoteWorker.java | 4 +-
.../InstanceRegisterSerialWorker.java | 87 +++++++++++++++++++
.../register/instance/dao/IInstanceDAO.java | 18 ++++
.../register/instance/dao/InstanceEsDAO.java | 84 ++++++++++++++++++
.../register/instance/dao/InstanceH2DAO.java | 25 ++++++
.../resources/META-INF/defines/es_dao.define | 3 +-
.../resources/META-INF/defines/h2_dao.define | 3 +-
.../local_async_worker_provider.define | 4 +-
.../elasticsearch/ElasticSearchClient.java | 10 +++
.../storage/elasticsearch/dao/EsDAO.java | 45 ++++++++++
16 files changed, 340 insertions(+), 73 deletions(-)
delete mode 100644 apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/instance/IInstanceDAO.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/IdAutoIncrement.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceRegisterSerialWorker.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/IInstanceDAO.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceEsDAO.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceH2DAO.java
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/instance/IInstanceDAO.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/instance/IInstanceDAO.java
deleted file mode 100644
index 7cdb17369..000000000
--- a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/instance/IInstanceDAO.java
+++ /dev/null
@@ -1,8 +0,0 @@
-package org.skywalking.apm.collector.agentregister.instance;
-
-/**
- * @author pengys5
- */
-public interface IInstanceDAO {
- int getInstanceId(String agentUUID);
-}
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/instance/InstanceIDService.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/instance/InstanceIDService.java
index 9a74f3118..4ac42d8bd 100644
--- a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/instance/InstanceIDService.java
+++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/instance/InstanceIDService.java
@@ -2,6 +2,7 @@ package org.skywalking.apm.collector.agentregister.instance;
import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterRemoteWorker;
import org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceDataDefine;
+import org.skywalking.apm.collector.agentstream.worker.register.instance.dao.IInstanceDAO;
import org.skywalking.apm.collector.core.framework.CollectorContextHelper;
import org.skywalking.apm.collector.storage.dao.DAOContainer;
import org.skywalking.apm.collector.stream.StreamModuleContext;
@@ -20,11 +21,11 @@ public class InstanceIDService {
public int getOrCreate(int applicationId, String agentUUID, long registerTime) {
IInstanceDAO dao = (IInstanceDAO)DAOContainer.INSTANCE.get(IInstanceDAO.class.getName());
- int instanceId = dao.getInstanceId(agentUUID);
+ int instanceId = dao.getInstanceId(applicationId, agentUUID);
if (instanceId == 0) {
StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME);
- InstanceDataDefine.Instance instance = new InstanceDataDefine.Instance(agentUUID, applicationId, agentUUID, registerTime, 0);
+ InstanceDataDefine.Instance instance = new InstanceDataDefine.Instance("0", applicationId, agentUUID, registerTime, 0);
try {
context.getClusterWorkerContext().lookup(ApplicationRegisterRemoteWorker.WorkerRole.INSTANCE).tell(instance);
} catch (WorkerNotFoundException | WorkerInvokeException e) {
@@ -35,10 +36,14 @@ public class InstanceIDService {
}
public void heartBeat(int instanceId, long heartbeatTime) {
-
+ IInstanceDAO dao = (IInstanceDAO)DAOContainer.INSTANCE.get(IInstanceDAO.class.getName());
+ dao.updateHeartbeatTime(instanceId, heartbeatTime);
}
public void recover(int instanceId, int applicationId, long registerTime) {
+ IInstanceDAO dao = (IInstanceDAO)DAOContainer.INSTANCE.get(IInstanceDAO.class.getName());
+ InstanceDataDefine.Instance instance = new InstanceDataDefine.Instance(String.valueOf(instanceId), applicationId, "", registerTime, instanceId);
+ dao.save(instance);
}
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/IdAutoIncrement.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/IdAutoIncrement.java
new file mode 100644
index 000000000..d9864e4e7
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/IdAutoIncrement.java
@@ -0,0 +1,24 @@
+package org.skywalking.apm.collector.agentstream.worker.register;
+
+/**
+ * @author pengys5
+ */
+public enum IdAutoIncrement {
+ INSTANCE;
+
+ public int increment(int min, int max) {
+ int instanceId;
+ if (min == max) {
+ instanceId = -1;
+ } else if (min + max == 0) {
+ instanceId = max + 1;
+ } else if (min + max > 0) {
+ instanceId = min - 1;
+ } else if (max < 0) {
+ instanceId = 1;
+ } else {
+ instanceId = max + 1;
+ }
+ return instanceId;
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java
index ff01be792..66c6c0e2a 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java
@@ -1,5 +1,6 @@
package org.skywalking.apm.collector.agentstream.worker.register.application;
+import org.skywalking.apm.collector.agentstream.worker.register.IdAutoIncrement;
import org.skywalking.apm.collector.agentstream.worker.register.application.dao.IApplicationDAO;
import org.skywalking.apm.collector.storage.dao.DAOContainer;
import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorker;
@@ -41,20 +42,9 @@ public class ApplicationRegisterSerialWorker extends AbstractLocalAsyncWorker {
application.setId("1");
} else {
int max = dao.getMaxApplicationId();
- int instanceId;
- if (min == max) {
- instanceId = -1;
- } else if (min + max == 0) {
- instanceId = max + 1;
- } else if (min + max > 0) {
- instanceId = min - 1;
- } else if (max < 0) {
- instanceId = 1;
- } else {
- instanceId = max + 1;
- }
- application.setApplicationId(instanceId);
- application.setId(String.valueOf(instanceId));
+ int applicationId = IdAutoIncrement.INSTANCE.increment(min, max);
+ application.setApplicationId(applicationId);
+ application.setId(String.valueOf(applicationId));
}
dao.save(application);
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/ApplicationEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/ApplicationEsDAO.java
index 7f12f50c3..ebe4360a8 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/ApplicationEsDAO.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/ApplicationEsDAO.java
@@ -9,11 +9,6 @@ import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.aggregations.AggregationBuilders;
-import org.elasticsearch.search.aggregations.metrics.max.Max;
-import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.min.Min;
-import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine;
import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationTable;
import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient;
@@ -47,41 +42,11 @@ public class ApplicationEsDAO extends EsDAO implements IApplicationDAO {
}
@Override public int getMaxApplicationId() {
- ElasticSearchClient client = getClient();
- SearchRequestBuilder searchRequestBuilder = client.prepareSearch(ApplicationTable.TABLE);
- searchRequestBuilder.setTypes("type");
- searchRequestBuilder.setSize(0);
- MaxAggregationBuilder aggregation = AggregationBuilders.max("agg").field(ApplicationTable.COLUMN_APPLICATION_ID);
- searchRequestBuilder.addAggregation(aggregation);
-
- SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
- Max agg = searchResponse.getAggregations().get("agg");
-
- int id = (int)agg.getValue();
- if (id == Integer.MAX_VALUE || id == Integer.MIN_VALUE) {
- return 0;
- } else {
- return id;
- }
+ return getMaxId(ApplicationTable.TABLE, ApplicationTable.COLUMN_APPLICATION_ID);
}
@Override public int getMinApplicationId() {
- ElasticSearchClient client = getClient();
- SearchRequestBuilder searchRequestBuilder = client.prepareSearch(ApplicationTable.TABLE);
- searchRequestBuilder.setTypes("type");
- searchRequestBuilder.setSize(0);
- MinAggregationBuilder aggregation = AggregationBuilders.min("agg").field(ApplicationTable.COLUMN_APPLICATION_ID);
- searchRequestBuilder.addAggregation(aggregation);
-
- SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
- Min agg = searchResponse.getAggregations().get("agg");
-
- int id = (int)agg.getValue();
- if (id == Integer.MAX_VALUE || id == Integer.MIN_VALUE) {
- return 0;
- } else {
- return id;
- }
+ return getMinId(ApplicationTable.TABLE, ApplicationTable.COLUMN_APPLICATION_ID);
}
@Override public void save(ApplicationDataDefine.Application application) {
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java
index 446a47f8e..6470d4520 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java
@@ -51,11 +51,11 @@ public class InstanceDataDefine extends DataDefine {
}
public static class Instance {
- private final String id;
- private final int applicationId;
- private final String agentUUID;
- private final long registerTime;
- private final int instanceId;
+ private String id;
+ private int applicationId;
+ private String agentUUID;
+ private long registerTime;
+ private int instanceId;
public Instance(String id, int applicationId, String agentUUID, long registerTime, int instanceId) {
this.id = id;
@@ -84,5 +84,25 @@ public class InstanceDataDefine extends DataDefine {
public int getInstanceId() {
return instanceId;
}
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public void setApplicationId(int applicationId) {
+ this.applicationId = applicationId;
+ }
+
+ public void setAgentUUID(String agentUUID) {
+ this.agentUUID = agentUUID;
+ }
+
+ public void setRegisterTime(long registerTime) {
+ this.registerTime = registerTime;
+ }
+
+ public void setInstanceId(int instanceId) {
+ this.instanceId = instanceId;
+ }
}
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceRegisterRemoteWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceRegisterRemoteWorker.java
index 3e7cd93ec..c6a85dd3e 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceRegisterRemoteWorker.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceRegisterRemoteWorker.java
@@ -1,6 +1,5 @@
package org.skywalking.apm.collector.agentstream.worker.register.instance;
-import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine;
import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorker;
import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorkerProvider;
import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
@@ -30,6 +29,7 @@ public class InstanceRegisterRemoteWorker extends AbstractRemoteWorker {
@Override protected void onWork(Object message) throws WorkerException {
InstanceDataDefine.Instance instance = (InstanceDataDefine.Instance)message;
logger.debug("application id: {}, agentUUID: {}, register time: {}", instance.getApplicationId(), instance.getAgentUUID(), instance.getRegisterTime());
+ getClusterContext().lookup(InstanceRegisterSerialWorker.WorkerRole.INSTANCE).tell(instance);
}
public static class Factory extends AbstractRemoteWorkerProvider {
@@ -58,7 +58,7 @@ public class InstanceRegisterRemoteWorker extends AbstractRemoteWorker {
}
@Override public DataDefine dataDefine() {
- return new ApplicationDataDefine();
+ return new InstanceDataDefine();
}
}
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceRegisterSerialWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceRegisterSerialWorker.java
new file mode 100644
index 000000000..9bfabe44e
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceRegisterSerialWorker.java
@@ -0,0 +1,87 @@
+package org.skywalking.apm.collector.agentstream.worker.register.instance;
+
+import org.skywalking.apm.collector.agentstream.worker.register.IdAutoIncrement;
+import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine;
+import org.skywalking.apm.collector.agentstream.worker.register.instance.dao.IInstanceDAO;
+import org.skywalking.apm.collector.storage.dao.DAOContainer;
+import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorker;
+import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider;
+import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
+import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
+import org.skywalking.apm.collector.stream.worker.Role;
+import org.skywalking.apm.collector.stream.worker.WorkerException;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
+import org.skywalking.apm.collector.stream.worker.selector.ForeverFirstSelector;
+import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class InstanceRegisterSerialWorker extends AbstractLocalAsyncWorker {
+
+ private final Logger logger = LoggerFactory.getLogger(InstanceRegisterSerialWorker.class);
+
+ public InstanceRegisterSerialWorker(Role role, ClusterWorkerContext clusterContext) {
+ super(role, clusterContext);
+ }
+
+ @Override public void preStart() throws ProviderNotFoundException {
+ super.preStart();
+ }
+
+ @Override protected void onWork(Object message) throws WorkerException {
+ if (message instanceof InstanceDataDefine.Instance) {
+ InstanceDataDefine.Instance instance = (InstanceDataDefine.Instance)message;
+ logger.debug("register instance, application id: {}, agentUUID: {}", instance.getApplicationId(), instance.getAgentUUID());
+
+ IInstanceDAO dao = (IInstanceDAO)DAOContainer.INSTANCE.get(IInstanceDAO.class.getName());
+ int min = dao.getMinInstanceId();
+ if (min == 0) {
+ instance.setId("1");
+ instance.setInstanceId(1);
+ } else {
+ int max = dao.getMaxInstanceId();
+ int instanceId = IdAutoIncrement.INSTANCE.increment(min, max);
+ instance.setId(String.valueOf(instanceId));
+ instance.setInstanceId(instanceId);
+ }
+ dao.save(instance);
+ }
+ }
+
+ public static class Factory extends AbstractLocalAsyncWorkerProvider {
+ @Override
+ public Role role() {
+ return WorkerRole.INSTANCE;
+ }
+
+ @Override
+ public InstanceRegisterSerialWorker workerInstance(ClusterWorkerContext clusterContext) {
+ return new InstanceRegisterSerialWorker(role(), clusterContext);
+ }
+
+ @Override public int queueSize() {
+ return 256;
+ }
+ }
+
+ public enum WorkerRole implements Role {
+ INSTANCE;
+
+ @Override
+ public String roleName() {
+ return InstanceRegisterSerialWorker.class.getSimpleName();
+ }
+
+ @Override
+ public WorkerSelector workerSelector() {
+ return new ForeverFirstSelector();
+ }
+
+ @Override public DataDefine dataDefine() {
+ return new ApplicationDataDefine();
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/IInstanceDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/IInstanceDAO.java
new file mode 100644
index 000000000..f0c311146
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/IInstanceDAO.java
@@ -0,0 +1,18 @@
+package org.skywalking.apm.collector.agentstream.worker.register.instance.dao;
+
+import org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceDataDefine;
+
+/**
+ * @author pengys5
+ */
+public interface IInstanceDAO {
+ int getInstanceId(int applicationId, String agentUUID);
+
+ int getMaxInstanceId();
+
+ int getMinInstanceId();
+
+ void save(InstanceDataDefine.Instance instance);
+
+ void updateHeartbeatTime(int instanceId, long heartbeatTime);
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceEsDAO.java
new file mode 100644
index 000000000..237f50009
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceEsDAO.java
@@ -0,0 +1,84 @@
+package org.skywalking.apm.collector.agentstream.worker.register.instance.dao;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHit;
+import org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceDataDefine;
+import org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceTable;
+import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient;
+import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class InstanceEsDAO extends EsDAO implements IInstanceDAO {
+
+ private final Logger logger = LoggerFactory.getLogger(InstanceEsDAO.class);
+
+ @Override public int getInstanceId(int applicationId, String agentUUID) {
+ ElasticSearchClient client = getClient();
+
+ SearchRequestBuilder searchRequestBuilder = client.prepareSearch(InstanceTable.TABLE);
+ searchRequestBuilder.setTypes("type");
+ searchRequestBuilder.setSearchType(SearchType.QUERY_THEN_FETCH);
+ BoolQueryBuilder builder = QueryBuilders.boolQuery();
+ builder.must().add(QueryBuilders.termQuery(InstanceTable.COLUMN_APPLICATION_ID, applicationId));
+ builder.must().add(QueryBuilders.termQuery(InstanceTable.COLUMN_AGENTUUID, agentUUID));
+ searchRequestBuilder.setQuery(builder);
+ searchRequestBuilder.setSize(1);
+
+ SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
+ if (searchResponse.getHits().totalHits > 0) {
+ SearchHit searchHit = searchResponse.getHits().iterator().next();
+ int instanceId = (int)searchHit.getSource().get(InstanceTable.COLUMN_INSTANCE_ID);
+ return instanceId;
+ }
+ return 0;
+ }
+
+ @Override public int getMaxInstanceId() {
+ return getMaxId(InstanceTable.TABLE, InstanceTable.COLUMN_INSTANCE_ID);
+ }
+
+ @Override public int getMinInstanceId() {
+ return getMinId(InstanceTable.TABLE, InstanceTable.COLUMN_INSTANCE_ID);
+ }
+
+ @Override public void save(InstanceDataDefine.Instance instance) {
+ logger.debug("save instance register info, application id: {}, agentUUID: {}", instance.getApplicationId(), instance.getAgentUUID());
+ ElasticSearchClient client = getClient();
+ Map source = new HashMap();
+ source.put(InstanceTable.COLUMN_INSTANCE_ID, instance.getInstanceId());
+ source.put(InstanceTable.COLUMN_APPLICATION_ID, instance.getApplicationId());
+ source.put(InstanceTable.COLUMN_AGENTUUID, instance.getAgentUUID());
+ source.put(InstanceTable.COLUMN_REGISTER_TIME, instance.getRegisterTime());
+
+ IndexResponse response = client.prepareIndex(InstanceTable.TABLE, instance.getId()).setSource(source).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get();
+ logger.debug("save instance register info, application id: {}, agentUUID: {}, status: {}", instance.getApplicationId(), instance.getAgentUUID(), response.status().name());
+ }
+
+ @Override public void updateHeartbeatTime(int instanceId, long heartbeatTime) {
+ ElasticSearchClient client = getClient();
+ UpdateRequest updateRequest = new UpdateRequest();
+ updateRequest.index(InstanceTable.TABLE);
+ updateRequest.type("type");
+ updateRequest.id(String.valueOf(instanceId));
+ updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+
+ Map source = new HashMap();
+ source.put(InstanceTable.COLUMN_HEARTBEAT_TIME, heartbeatTime);
+
+ updateRequest.doc(source);
+ client.update(updateRequest);
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceH2DAO.java
new file mode 100644
index 000000000..d3bda6437
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceH2DAO.java
@@ -0,0 +1,25 @@
+package org.skywalking.apm.collector.agentstream.worker.register.instance.dao;
+
+import org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceDataDefine;
+import org.skywalking.apm.collector.storage.h2.dao.H2DAO;
+
+/**
+ * @author pengys5
+ */
+public class InstanceH2DAO extends H2DAO implements IInstanceDAO {
+ @Override public int getInstanceId(int applicationId, String agentUUID) {
+ return 0;
+ }
+
+ @Override public int getMaxInstanceId() {
+ return 0;
+ }
+
+ @Override public int getMinInstanceId() {
+ return 0;
+ }
+
+ @Override public void save(InstanceDataDefine.Instance instance) {
+
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define
index 2a346cdbf..c4c309747 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define
@@ -1 +1,2 @@
-org.skywalking.apm.collector.agentstream.worker.register.application.dao.ApplicationEsDAO
\ No newline at end of file
+org.skywalking.apm.collector.agentstream.worker.register.application.dao.ApplicationEsDAO
+org.skywalking.apm.collector.agentstream.worker.register.instance.dao.InstanceEsDAO
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define
index c243fe509..65950d42f 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define
@@ -1 +1,2 @@
-org.skywalking.apm.collector.agentstream.worker.register.application.dao.ApplicationH2DAO
\ No newline at end of file
+org.skywalking.apm.collector.agentstream.worker.register.application.dao.ApplicationH2DAO
+org.skywalking.apm.collector.agentstream.worker.register.instance.dao.InstanceH2DAO
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define
index 418ecfce1..b272a20ce 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define
@@ -1,3 +1,3 @@
org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentAggWorker$Factory
-
-org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterSerialWorker$Factory
\ No newline at end of file
+org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterSerialWorker$Factory
+org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceRegisterSerialWorker$Factory
\ No newline at end of file
diff --git a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java
index 303222e16..27218ce35 100644
--- a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java
+++ b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java
@@ -4,11 +4,13 @@ import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.LinkedList;
import java.util.List;
+import java.util.concurrent.ExecutionException;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.IndicesAdminClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
@@ -109,4 +111,12 @@ public class ElasticSearchClient implements Client {
public IndexRequestBuilder prepareIndex(String indexName, String id) {
return client.prepareIndex(indexName, "type", id);
}
+
+ public void update(UpdateRequest updateRequest) {
+ try {
+ client.update(updateRequest).get();
+ } catch (InterruptedException | ExecutionException e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
}
diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAO.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAO.java
index a1a15ac0f..f6eb18a85 100644
--- a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAO.java
+++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAO.java
@@ -1,5 +1,12 @@
package org.skywalking.apm.collector.storage.elasticsearch.dao;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.metrics.max.Max;
+import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.min.Min;
+import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient;
import org.skywalking.apm.collector.storage.dao.DAO;
@@ -7,4 +14,42 @@ import org.skywalking.apm.collector.storage.dao.DAO;
* @author pengys5
*/
public abstract class EsDAO extends DAO {
+
+ public final int getMaxId(String indexName, String columnName) {
+ ElasticSearchClient client = getClient();
+ SearchRequestBuilder searchRequestBuilder = client.prepareSearch(indexName);
+ searchRequestBuilder.setTypes("type");
+ searchRequestBuilder.setSize(0);
+ MaxAggregationBuilder aggregation = AggregationBuilders.max("agg").field(columnName);
+ searchRequestBuilder.addAggregation(aggregation);
+
+ SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
+ Max agg = searchResponse.getAggregations().get("agg");
+
+ int id = (int)agg.getValue();
+ if (id == Integer.MAX_VALUE || id == Integer.MIN_VALUE) {
+ return 0;
+ } else {
+ return id;
+ }
+ }
+
+ public final int getMinId(String indexName, String columnName) {
+ ElasticSearchClient client = getClient();
+ SearchRequestBuilder searchRequestBuilder = client.prepareSearch(indexName);
+ searchRequestBuilder.setTypes("type");
+ searchRequestBuilder.setSize(0);
+ MinAggregationBuilder aggregation = AggregationBuilders.min("agg").field(columnName);
+ searchRequestBuilder.addAggregation(aggregation);
+
+ SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
+ Min agg = searchResponse.getAggregations().get("agg");
+
+ int id = (int)agg.getValue();
+ if (id == Integer.MAX_VALUE || id == Integer.MIN_VALUE) {
+ return 0;
+ } else {
+ return id;
+ }
+ }
}
From c1a47d4d019a81758e661cc1208f2382940f5d61 Mon Sep 17 00:00:00 2001
From: pengys5 <8082209@qq.com>
Date: Thu, 27 Jul 2017 10:32:57 +0800
Subject: [PATCH 15/28] service name register finish
---
.../ServiceNameDiscoveryServiceHandler.java | 26 ++++-
.../servicename/ServiceNameService.java | 37 ++++++++
.../servicename/ServiceNameDataDefine.java | 95 +++++++++++++++++++
.../servicename/ServiceNameEsTableDefine.java | 32 +++++++
.../servicename/ServiceNameH2TableDefine.java | 20 ++++
.../ServiceNameRegisterRemoteWorker.java | 64 +++++++++++++
.../ServiceNameRegisterSerialWorker.java | 86 +++++++++++++++++
.../servicename/ServiceNameTable.java | 11 +++
.../servicename/dao/IServiceNameDAO.java | 16 ++++
.../servicename/dao/ServiceNameEsDAO.java | 67 +++++++++++++
.../servicename/dao/ServiceNameH2DAO.java | 26 +++++
.../resources/META-INF/defines/data.define | 4 +-
.../resources/META-INF/defines/es_dao.define | 3 +-
.../resources/META-INF/defines/h2_dao.define | 3 +-
.../local_async_worker_provider.define | 3 +-
.../defines/remote_worker_provider.define | 3 +-
.../resources/META-INF/defines/storage.define | 5 +-
17 files changed, 494 insertions(+), 7 deletions(-)
create mode 100644 apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/servicename/ServiceNameService.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameDataDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameEsTableDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameH2TableDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameRegisterRemoteWorker.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameRegisterSerialWorker.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameTable.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/IServiceNameDAO.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/ServiceNameEsDAO.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/ServiceNameH2DAO.java
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ServiceNameDiscoveryServiceHandler.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ServiceNameDiscoveryServiceHandler.java
index a0e040c9f..1009658a7 100644
--- a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ServiceNameDiscoveryServiceHandler.java
+++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ServiceNameDiscoveryServiceHandler.java
@@ -1,18 +1,42 @@
package org.skywalking.apm.collector.agentregister.grpc.handler;
import io.grpc.stub.StreamObserver;
+import java.util.List;
+import org.skywalking.apm.collector.agentregister.servicename.ServiceNameService;
import org.skywalking.apm.collector.server.grpc.GRPCHandler;
import org.skywalking.apm.network.proto.ServiceNameCollection;
import org.skywalking.apm.network.proto.ServiceNameDiscoveryServiceGrpc;
+import org.skywalking.apm.network.proto.ServiceNameElement;
import org.skywalking.apm.network.proto.ServiceNameMappingCollection;
+import org.skywalking.apm.network.proto.ServiceNameMappingElement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* @author pengys5
*/
public class ServiceNameDiscoveryServiceHandler extends ServiceNameDiscoveryServiceGrpc.ServiceNameDiscoveryServiceImplBase implements GRPCHandler {
+ private final Logger logger = LoggerFactory.getLogger(ServiceNameDiscoveryServiceHandler.class);
+
+ private ServiceNameService serviceNameService = new ServiceNameService();
+
@Override public void discovery(ServiceNameCollection request,
StreamObserver responseObserver) {
- super.discovery(request, responseObserver);
+ List serviceNameElementList = request.getElementsList();
+
+ ServiceNameMappingCollection.Builder builder = ServiceNameMappingCollection.newBuilder();
+ for (ServiceNameElement serviceNameElement : serviceNameElementList) {
+ int applicationId = serviceNameElement.getApplicationId();
+ String serviceName = serviceNameElement.getServiceName();
+ int serviceId = serviceNameService.getOrCreate(applicationId, serviceName);
+ ServiceNameMappingElement.Builder mappingElement = ServiceNameMappingElement.newBuilder();
+ mappingElement.setServiceId(serviceId);
+ mappingElement.setElement(serviceNameElement);
+ builder.addElements(mappingElement);
+ }
+
+ responseObserver.onNext(builder.build());
+ responseObserver.onCompleted();
}
}
diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/servicename/ServiceNameService.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/servicename/ServiceNameService.java
new file mode 100644
index 000000000..259acc984
--- /dev/null
+++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/servicename/ServiceNameService.java
@@ -0,0 +1,37 @@
+package org.skywalking.apm.collector.agentregister.servicename;
+
+import org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameDataDefine;
+import org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameRegisterRemoteWorker;
+import org.skywalking.apm.collector.agentstream.worker.register.servicename.dao.IServiceNameDAO;
+import org.skywalking.apm.collector.core.framework.CollectorContextHelper;
+import org.skywalking.apm.collector.storage.dao.DAOContainer;
+import org.skywalking.apm.collector.stream.StreamModuleContext;
+import org.skywalking.apm.collector.stream.StreamModuleGroupDefine;
+import org.skywalking.apm.collector.stream.worker.WorkerInvokeException;
+import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class ServiceNameService {
+
+ private final Logger logger = LoggerFactory.getLogger(ServiceNameService.class);
+
+ public int getOrCreate(int applicationId, String serviceName) {
+ IServiceNameDAO dao = (IServiceNameDAO)DAOContainer.INSTANCE.get(IServiceNameDAO.class.getName());
+ int serviceId = dao.getServiceId(applicationId, serviceName);
+
+ if (serviceId == 0) {
+ StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME);
+ ServiceNameDataDefine.ServiceName service = new ServiceNameDataDefine.ServiceName("0", serviceName, applicationId, 0);
+ try {
+ context.getClusterWorkerContext().lookup(ServiceNameRegisterRemoteWorker.WorkerRole.INSTANCE).tell(service);
+ } catch (WorkerNotFoundException | WorkerInvokeException e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+ return serviceId;
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameDataDefine.java
new file mode 100644
index 000000000..d30a15240
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameDataDefine.java
@@ -0,0 +1,95 @@
+package org.skywalking.apm.collector.agentstream.worker.register.servicename;
+
+import org.skywalking.apm.collector.remote.grpc.proto.RemoteData;
+import org.skywalking.apm.collector.stream.worker.impl.data.Attribute;
+import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
+import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation;
+import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation;
+
+/**
+ * @author pengys5
+ */
+public class ServiceNameDataDefine extends DataDefine {
+
+ public static final int DEFINE_ID = 103;
+
+ @Override public int defineId() {
+ return DEFINE_ID;
+ }
+
+ @Override protected int initialCapacity() {
+ return 4;
+ }
+
+ @Override protected void attributeDefine() {
+ addAttribute(0, new Attribute("id", AttributeType.STRING, new NonOperation()));
+ addAttribute(1, new Attribute(ServiceNameTable.COLUMN_SERVICE_NAME, AttributeType.STRING, new CoverOperation()));
+ addAttribute(2, new Attribute(ServiceNameTable.COLUMN_APPLICATION_ID, AttributeType.INTEGER, new CoverOperation()));
+ addAttribute(3, new Attribute(ServiceNameTable.COLUMN_SERVICE_ID, AttributeType.INTEGER, new CoverOperation()));
+ }
+
+ @Override public Object deserialize(RemoteData remoteData) {
+ String id = remoteData.getDataStrings(0);
+ String serviceName = remoteData.getDataStrings(1);
+ int applicationId = remoteData.getDataIntegers(0);
+ int serviceId = remoteData.getDataIntegers(1);
+ return new ServiceName(id, serviceName, applicationId, serviceId);
+ }
+
+ @Override public RemoteData serialize(Object object) {
+ ServiceName serviceName = (ServiceName)object;
+ RemoteData.Builder builder = RemoteData.newBuilder();
+ builder.addDataStrings(serviceName.getId());
+ builder.addDataStrings(serviceName.getServiceName());
+ builder.addDataIntegers(serviceName.getApplicationId());
+ builder.addDataIntegers(serviceName.getServiceId());
+ return builder.build();
+ }
+
+ public static class ServiceName {
+ private String id;
+ private String serviceName;
+ private int applicationId;
+ private int serviceId;
+
+ public ServiceName(String id, String serviceName, int applicationId, int serviceId) {
+ this.id = id;
+ this.serviceName = serviceName;
+ this.applicationId = applicationId;
+ this.serviceId = serviceId;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getServiceName() {
+ return serviceName;
+ }
+
+ public void setServiceName(String serviceName) {
+ this.serviceName = serviceName;
+ }
+
+ public int getApplicationId() {
+ return applicationId;
+ }
+
+ public void setApplicationId(int applicationId) {
+ this.applicationId = applicationId;
+ }
+
+ public int getServiceId() {
+ return serviceId;
+ }
+
+ public void setServiceId(int serviceId) {
+ this.serviceId = serviceId;
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameEsTableDefine.java
new file mode 100644
index 000000000..9d3a33384
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameEsTableDefine.java
@@ -0,0 +1,32 @@
+package org.skywalking.apm.collector.agentstream.worker.register.servicename;
+
+import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine;
+import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine;
+
+/**
+ * @author pengys5
+ */
+public class ServiceNameEsTableDefine extends ElasticSearchTableDefine {
+
+ public ServiceNameEsTableDefine() {
+ super(ServiceNameTable.TABLE);
+ }
+
+ @Override public int refreshInterval() {
+ return 0;
+ }
+
+ @Override public int numberOfShards() {
+ return 2;
+ }
+
+ @Override public int numberOfReplicas() {
+ return 0;
+ }
+
+ @Override public void initialize() {
+ addColumn(new ElasticSearchColumnDefine(ServiceNameTable.COLUMN_APPLICATION_ID, ElasticSearchColumnDefine.Type.Integer.name()));
+ addColumn(new ElasticSearchColumnDefine(ServiceNameTable.COLUMN_SERVICE_NAME, ElasticSearchColumnDefine.Type.Keyword.name()));
+ addColumn(new ElasticSearchColumnDefine(ServiceNameTable.COLUMN_SERVICE_ID, ElasticSearchColumnDefine.Type.Integer.name()));
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameH2TableDefine.java
new file mode 100644
index 000000000..1ead6dc16
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameH2TableDefine.java
@@ -0,0 +1,20 @@
+package org.skywalking.apm.collector.agentstream.worker.register.servicename;
+
+import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine;
+import org.skywalking.apm.collector.storage.h2.define.H2TableDefine;
+
+/**
+ * @author pengys5
+ */
+public class ServiceNameH2TableDefine extends H2TableDefine {
+
+ public ServiceNameH2TableDefine() {
+ super(ServiceNameTable.TABLE);
+ }
+
+ @Override public void initialize() {
+ addColumn(new H2ColumnDefine(ServiceNameTable.COLUMN_APPLICATION_ID, H2ColumnDefine.Type.Int.name()));
+ addColumn(new H2ColumnDefine(ServiceNameTable.COLUMN_SERVICE_NAME, H2ColumnDefine.Type.Varchar.name()));
+ addColumn(new H2ColumnDefine(ServiceNameTable.COLUMN_SERVICE_ID, H2ColumnDefine.Type.Int.name()));
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameRegisterRemoteWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameRegisterRemoteWorker.java
new file mode 100644
index 000000000..397eb475b
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameRegisterRemoteWorker.java
@@ -0,0 +1,64 @@
+package org.skywalking.apm.collector.agentstream.worker.register.servicename;
+
+import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorker;
+import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorkerProvider;
+import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
+import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
+import org.skywalking.apm.collector.stream.worker.Role;
+import org.skywalking.apm.collector.stream.worker.WorkerException;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
+import org.skywalking.apm.collector.stream.worker.selector.ForeverFirstSelector;
+import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class ServiceNameRegisterRemoteWorker extends AbstractRemoteWorker {
+
+ private final Logger logger = LoggerFactory.getLogger(ServiceNameRegisterRemoteWorker.class);
+
+ protected ServiceNameRegisterRemoteWorker(Role role, ClusterWorkerContext clusterContext) {
+ super(role, clusterContext);
+ }
+
+ @Override public void preStart() throws ProviderNotFoundException {
+ }
+
+ @Override protected void onWork(Object message) throws WorkerException {
+ ServiceNameDataDefine.ServiceName serviceName = (ServiceNameDataDefine.ServiceName)message;
+ logger.debug("service name: {}", serviceName.getServiceName());
+ getClusterContext().lookup(ServiceNameRegisterSerialWorker.WorkerRole.INSTANCE).tell(serviceName);
+ }
+
+ public static class Factory extends AbstractRemoteWorkerProvider {
+ @Override
+ public Role role() {
+ return WorkerRole.INSTANCE;
+ }
+
+ @Override
+ public ServiceNameRegisterRemoteWorker workerInstance(ClusterWorkerContext clusterContext) {
+ return new ServiceNameRegisterRemoteWorker(role(), clusterContext);
+ }
+ }
+
+ public enum WorkerRole implements Role {
+ INSTANCE;
+
+ @Override
+ public String roleName() {
+ return ServiceNameRegisterRemoteWorker.class.getSimpleName();
+ }
+
+ @Override
+ public WorkerSelector workerSelector() {
+ return new ForeverFirstSelector();
+ }
+
+ @Override public DataDefine dataDefine() {
+ return new ServiceNameDataDefine();
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameRegisterSerialWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameRegisterSerialWorker.java
new file mode 100644
index 000000000..b0d235759
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameRegisterSerialWorker.java
@@ -0,0 +1,86 @@
+package org.skywalking.apm.collector.agentstream.worker.register.servicename;
+
+import org.skywalking.apm.collector.agentstream.worker.register.IdAutoIncrement;
+import org.skywalking.apm.collector.agentstream.worker.register.servicename.dao.IServiceNameDAO;
+import org.skywalking.apm.collector.storage.dao.DAOContainer;
+import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorker;
+import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider;
+import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
+import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
+import org.skywalking.apm.collector.stream.worker.Role;
+import org.skywalking.apm.collector.stream.worker.WorkerException;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
+import org.skywalking.apm.collector.stream.worker.selector.ForeverFirstSelector;
+import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class ServiceNameRegisterSerialWorker extends AbstractLocalAsyncWorker {
+
+ private final Logger logger = LoggerFactory.getLogger(ServiceNameRegisterSerialWorker.class);
+
+ public ServiceNameRegisterSerialWorker(Role role, ClusterWorkerContext clusterContext) {
+ super(role, clusterContext);
+ }
+
+ @Override public void preStart() throws ProviderNotFoundException {
+ super.preStart();
+ }
+
+ @Override protected void onWork(Object message) throws WorkerException {
+ if (message instanceof ServiceNameDataDefine.ServiceName) {
+ ServiceNameDataDefine.ServiceName serviceName = (ServiceNameDataDefine.ServiceName)message;
+ logger.debug("register service name: {}, application id: {}", serviceName.getServiceName(), serviceName.getApplicationId());
+
+ IServiceNameDAO dao = (IServiceNameDAO)DAOContainer.INSTANCE.get(IServiceNameDAO.class.getName());
+ int min = dao.getMinServiceId();
+ if (min == 0) {
+ serviceName.setServiceId(1);
+ serviceName.setId("1");
+ } else {
+ int max = dao.getMaxServiceId();
+ int serviceId = IdAutoIncrement.INSTANCE.increment(min, max);
+ serviceName.setApplicationId(serviceId);
+ serviceName.setId(String.valueOf(serviceId));
+ }
+ dao.save(serviceName);
+ }
+ }
+
+ public static class Factory extends AbstractLocalAsyncWorkerProvider {
+ @Override
+ public Role role() {
+ return WorkerRole.INSTANCE;
+ }
+
+ @Override
+ public ServiceNameRegisterSerialWorker workerInstance(ClusterWorkerContext clusterContext) {
+ return new ServiceNameRegisterSerialWorker(role(), clusterContext);
+ }
+
+ @Override public int queueSize() {
+ return 256;
+ }
+ }
+
+ public enum WorkerRole implements Role {
+ INSTANCE;
+
+ @Override
+ public String roleName() {
+ return ServiceNameRegisterSerialWorker.class.getSimpleName();
+ }
+
+ @Override
+ public WorkerSelector workerSelector() {
+ return new ForeverFirstSelector();
+ }
+
+ @Override public DataDefine dataDefine() {
+ return new ServiceNameDataDefine();
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameTable.java
new file mode 100644
index 000000000..f15fd6075
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameTable.java
@@ -0,0 +1,11 @@
+package org.skywalking.apm.collector.agentstream.worker.register.servicename;
+
+/**
+ * @author pengys5
+ */
+public class ServiceNameTable {
+ public static final String TABLE = "service_name";
+ public static final String COLUMN_SERVICE_NAME = "service_name";
+ public static final String COLUMN_APPLICATION_ID = "application_id";
+ public static final String COLUMN_SERVICE_ID = "service_id";
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/IServiceNameDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/IServiceNameDAO.java
new file mode 100644
index 000000000..a8752e4cc
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/IServiceNameDAO.java
@@ -0,0 +1,16 @@
+package org.skywalking.apm.collector.agentstream.worker.register.servicename.dao;
+
+import org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameDataDefine;
+
+/**
+ * @author pengys5
+ */
+public interface IServiceNameDAO {
+ int getServiceId(int applicationId, String serviceName);
+
+ int getMaxServiceId();
+
+ int getMinServiceId();
+
+ void save(ServiceNameDataDefine.ServiceName serviceName);
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/ServiceNameEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/ServiceNameEsDAO.java
new file mode 100644
index 000000000..e3cd72202
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/ServiceNameEsDAO.java
@@ -0,0 +1,67 @@
+package org.skywalking.apm.collector.agentstream.worker.register.servicename.dao;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHit;
+import org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameDataDefine;
+import org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameTable;
+import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient;
+import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class ServiceNameEsDAO extends EsDAO implements IServiceNameDAO {
+
+ private final Logger logger = LoggerFactory.getLogger(ServiceNameEsDAO.class);
+
+ @Override public int getServiceId(int applicationId, String serviceName) {
+ ElasticSearchClient client = getClient();
+
+ SearchRequestBuilder searchRequestBuilder = client.prepareSearch(ServiceNameTable.TABLE);
+ searchRequestBuilder.setTypes("type");
+ searchRequestBuilder.setSearchType(SearchType.QUERY_THEN_FETCH);
+ BoolQueryBuilder builder = QueryBuilders.boolQuery();
+ builder.must().add(QueryBuilders.termQuery(ServiceNameTable.COLUMN_APPLICATION_ID, applicationId));
+ builder.must().add(QueryBuilders.termQuery(ServiceNameTable.COLUMN_SERVICE_NAME, serviceName));
+ searchRequestBuilder.setQuery(builder);
+ searchRequestBuilder.setSize(1);
+
+ SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
+ if (searchResponse.getHits().totalHits > 0) {
+ SearchHit searchHit = searchResponse.getHits().iterator().next();
+ int serviceId = (int)searchHit.getSource().get(ServiceNameTable.COLUMN_SERVICE_ID);
+ return serviceId;
+ }
+ return 0;
+ }
+
+ @Override public int getMaxServiceId() {
+ return getMaxId(ServiceNameTable.TABLE, ServiceNameTable.COLUMN_SERVICE_ID);
+ }
+
+ @Override public int getMinServiceId() {
+ return getMinId(ServiceNameTable.TABLE, ServiceNameTable.COLUMN_SERVICE_ID);
+ }
+
+ @Override public void save(ServiceNameDataDefine.ServiceName serviceName) {
+ logger.debug("save service name register info, application id: {}, service name: {}", serviceName.getApplicationId(), serviceName.getServiceName());
+ ElasticSearchClient client = getClient();
+ Map source = new HashMap();
+ source.put(ServiceNameTable.COLUMN_SERVICE_ID, serviceName.getServiceId());
+ source.put(ServiceNameTable.COLUMN_APPLICATION_ID, serviceName.getApplicationId());
+ source.put(ServiceNameTable.COLUMN_SERVICE_NAME, serviceName.getServiceName());
+
+ IndexResponse response = client.prepareIndex(ServiceNameTable.TABLE, serviceName.getId()).setSource(source).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get();
+ logger.debug("save service name register info, application id: {}, service name: {}, status: {}", serviceName.getApplicationId(), serviceName.getServiceName(), response.status().name());
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/ServiceNameH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/ServiceNameH2DAO.java
new file mode 100644
index 000000000..f0e2c359f
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/ServiceNameH2DAO.java
@@ -0,0 +1,26 @@
+package org.skywalking.apm.collector.agentstream.worker.register.servicename.dao;
+
+import org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameDataDefine;
+import org.skywalking.apm.collector.storage.h2.dao.H2DAO;
+
+/**
+ * @author pengys5
+ */
+public class ServiceNameH2DAO extends H2DAO implements IServiceNameDAO {
+
+ @Override public int getServiceId(int applicationId, String serviceName) {
+ return 0;
+ }
+
+ @Override public int getMaxServiceId() {
+ return 0;
+ }
+
+ @Override public int getMinServiceId() {
+ return 0;
+ }
+
+ @Override public void save(ServiceNameDataDefine.ServiceName serviceName) {
+
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define
index dce35f4df..205e6c74a 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define
@@ -1,2 +1,4 @@
org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentDataDefine
-org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine
\ No newline at end of file
+org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine
+org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceDataDefine
+org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameDataDefine
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define
index c4c309747..ad40a6443 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define
@@ -1,2 +1,3 @@
org.skywalking.apm.collector.agentstream.worker.register.application.dao.ApplicationEsDAO
-org.skywalking.apm.collector.agentstream.worker.register.instance.dao.InstanceEsDAO
\ No newline at end of file
+org.skywalking.apm.collector.agentstream.worker.register.instance.dao.InstanceEsDAO
+org.skywalking.apm.collector.agentstream.worker.register.servicename.dao.ServiceNameEsDAO
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define
index 65950d42f..1366f2d92 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define
@@ -1,2 +1,3 @@
org.skywalking.apm.collector.agentstream.worker.register.application.dao.ApplicationH2DAO
-org.skywalking.apm.collector.agentstream.worker.register.instance.dao.InstanceH2DAO
\ No newline at end of file
+org.skywalking.apm.collector.agentstream.worker.register.instance.dao.InstanceH2DAO
+org.skywalking.apm.collector.agentstream.worker.register.servicename.dao.ServiceNameH2DAO
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define
index b272a20ce..7acdd7e09 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define
@@ -1,3 +1,4 @@
org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentAggWorker$Factory
org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterSerialWorker$Factory
-org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceRegisterSerialWorker$Factory
\ No newline at end of file
+org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceRegisterSerialWorker$Factory
+org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameRegisterSerialWorker$Factory
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define
index 8fccdab58..2b5f3709e 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define
@@ -1,2 +1,3 @@
org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterRemoteWorker$Factory
-org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceRegisterRemoteWorker$Factory
\ No newline at end of file
+org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceRegisterRemoteWorker$Factory
+org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameRegisterRemoteWorker$Factory
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define
index 87b96eb9d..0d0a2264d 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define
@@ -7,4 +7,7 @@ org.skywalking.apm.collector.agentstream.worker.register.application.Application
org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationH2TableDefine
org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceEsTableDefine
-org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceH2TableDefine
\ No newline at end of file
+org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceH2TableDefine
+
+org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameEsTableDefine
+org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameH2TableDefine
\ No newline at end of file
From 5ad301eadc5f44f49fe31df1ffcf309368beca5f Mon Sep 17 00:00:00 2001
From: pengys5 <8082209@qq.com>
Date: Thu, 27 Jul 2017 22:37:44 +0800
Subject: [PATCH 16/28] Segment parse finish, but not test.
---
.../handler/TraceSegmentServiceHandler.java | 4 +
.../agentstream/worker/CommonTable.java | 1 +
.../component/NodeComponentAggWorker.java | 1 +
.../component/NodeComponentDataDefine.java | 37 ----
.../component/NodeComponentSpanListener.java | 52 ++++++
.../define/NodeComponentDataDefine.java | 75 ++++++++
.../define/NodeComponentEsTableDefine.java | 6 +-
.../define/NodeComponentH2TableDefine.java | 7 +-
.../{ => define}/NodeComponentTable.java | 4 +-
.../node/mapping/NodeMappingSpanListener.java | 47 +++++
.../mapping/define/NodeMappingDataDefine.java | 75 ++++++++
.../define/NodeMappingEsTableDefine.java | 4 +-
.../define/NodeMappingH2TableDefine.java | 5 +-
.../define/NodeMappingTable.java | 4 +-
.../reference/NodeRefSpanListener.java | 68 ++++++++
.../reference/define/NodeRefDataDefine.java | 85 +++++++++
.../define/NodeRefEsTableDefine.java | 31 ++++
.../define/NodeRefH2TableDefine.java | 19 ++
.../reference/define/NodeRefTable.java | 10 ++
.../summary/define/NodeRefSumDataDefine.java | 164 ++++++++++++++++++
.../define/NodeRefSumEsTableDefine.java | 37 ++++
.../define/NodeRefSumH2TableDefine.java | 25 +++
.../summary/define/NodeRefSumTable.java | 16 ++
.../worker/segment/EntrySpanListener.java | 10 ++
.../worker/segment/ExitSpanListener.java | 10 ++
.../worker/segment/FirstSpanListener.java | 10 ++
.../worker/segment/LocalSpanListener.java | 10 ++
.../worker/segment/RefsListener.java | 10 ++
.../worker/segment/SegmentParse.java | 99 +++++++++++
.../worker/segment/SpanListener.java | 8 +
.../reference/ServiceRefSpanListener.java | 79 +++++++++
.../define/ServiceRefDataDefine.java | 98 +++++++++++
.../define/ServiceRefEsTableDefine.java | 32 ++++
.../define/ServiceRefH2TableDefine.java | 20 +++
.../reference/define/ServiceRefTable.java | 11 ++
.../worker/util/TimeBucketUtils.java | 47 +++++
.../resources/META-INF/defines/data.define | 2 +-
.../resources/META-INF/defines/storage.define | 8 +-
.../collector/stream/worker/impl/Const.java | 13 --
39 files changed, 1169 insertions(+), 75 deletions(-)
delete mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentDataDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java
rename apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/{ => component}/define/NodeComponentEsTableDefine.java (70%)
rename apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/{ => component}/define/NodeComponentH2TableDefine.java (56%)
rename apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/{ => define}/NodeComponentTable.java (70%)
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingSpanListener.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingDataDefine.java
rename apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/{ => mapping}/define/NodeMappingEsTableDefine.java (73%)
rename apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/{ => mapping}/define/NodeMappingH2TableDefine.java (67%)
rename apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/{ => mapping}/define/NodeMappingTable.java (53%)
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefSpanListener.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefDataDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefEsTableDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefH2TableDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefTable.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumDataDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumEsTableDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumH2TableDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumTable.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/EntrySpanListener.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/ExitSpanListener.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/FirstSpanListener.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/LocalSpanListener.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/RefsListener.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SpanListener.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/ServiceRefSpanListener.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefDataDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefEsTableDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefH2TableDefine.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefTable.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/util/TimeBucketUtils.java
delete mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/Const.java
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandler.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandler.java
index d778e921e..17ab6e578 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandler.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandler.java
@@ -3,6 +3,7 @@ package org.skywalking.apm.collector.agentstream.grpc.handler;
import com.google.protobuf.InvalidProtocolBufferException;
import io.grpc.stub.StreamObserver;
import java.util.List;
+import org.skywalking.apm.collector.agentstream.worker.segment.SegmentParse;
import org.skywalking.apm.collector.server.grpc.GRPCHandler;
import org.skywalking.apm.network.proto.Downstream;
import org.skywalking.apm.network.proto.TraceSegmentObject;
@@ -19,12 +20,15 @@ public class TraceSegmentServiceHandler extends TraceSegmentServiceGrpc.TraceSeg
private final Logger logger = LoggerFactory.getLogger(TraceSegmentServiceHandler.class);
+ private SegmentParse segmentParse = new SegmentParse();
+
@Override public StreamObserver collect(StreamObserver responseObserver) {
return new StreamObserver() {
@Override public void onNext(UpstreamSegment segment) {
try {
List traceIds = segment.getGlobalTraceIdsList();
TraceSegmentObject segmentObject = TraceSegmentObject.parseFrom(segment.getSegment());
+ segmentParse.parse(traceIds, segmentObject);
} catch (InvalidProtocolBufferException e) {
logger.error(e.getMessage(), e);
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/CommonTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/CommonTable.java
index 4f8b27ac4..4d5669c16 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/CommonTable.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/CommonTable.java
@@ -4,6 +4,7 @@ package org.skywalking.apm.collector.agentstream.worker;
* @author pengys5
*/
public class CommonTable {
+ public static final String COLUMN_ID = "id";
public static final String COLUMN_AGG = "agg";
public static final String COLUMN_TIME_BUCKET = "time_bucket";
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggWorker.java
index 005198b68..a3dec152e 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggWorker.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggWorker.java
@@ -1,5 +1,6 @@
package org.skywalking.apm.collector.agentstream.worker.node.component;
+import org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentDataDefine;
import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider;
import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentDataDefine.java
deleted file mode 100644
index 1d09a862e..000000000
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentDataDefine.java
+++ /dev/null
@@ -1,37 +0,0 @@
-package org.skywalking.apm.collector.agentstream.worker.node.component;
-
-import org.skywalking.apm.collector.remote.grpc.proto.RemoteData;
-import org.skywalking.apm.collector.stream.worker.impl.data.Attribute;
-import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType;
-import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
-import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation;
-import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation;
-
-/**
- * @author pengys5
- */
-public class NodeComponentDataDefine extends DataDefine {
-
- @Override public int defineId() {
- return 0;
- }
-
- @Override protected int initialCapacity() {
- return 4;
- }
-
- @Override protected void attributeDefine() {
- addAttribute(0, new Attribute("id", AttributeType.STRING, new NonOperation()));
- addAttribute(1, new Attribute("name", AttributeType.STRING, new CoverOperation()));
- addAttribute(2, new Attribute("peers", AttributeType.STRING, new CoverOperation()));
- addAttribute(3, new Attribute("aggregation", AttributeType.STRING, new CoverOperation()));
- }
-
- @Override public Object deserialize(RemoteData remoteData) {
- return null;
- }
-
- @Override public RemoteData serialize(Object object) {
- return null;
- }
-}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java
new file mode 100644
index 000000000..ee9fca213
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java
@@ -0,0 +1,52 @@
+package org.skywalking.apm.collector.agentstream.worker.node.component;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.skywalking.apm.collector.agentstream.worker.Const;
+import org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentDataDefine;
+import org.skywalking.apm.collector.agentstream.worker.segment.EntrySpanListener;
+import org.skywalking.apm.collector.agentstream.worker.segment.ExitSpanListener;
+import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener;
+import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils;
+import org.skywalking.apm.network.proto.SpanObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class NodeComponentSpanListener implements EntrySpanListener, ExitSpanListener, FirstSpanListener {
+
+ private final Logger logger = LoggerFactory.getLogger(NodeComponentSpanListener.class);
+
+ private List nodeComponents = new ArrayList<>();
+ private long timeBucket;
+
+ @Override public void parseExit(SpanObject spanObject, int applicationId, int applicationInstanceId) {
+ String peers = spanObject.getPeer();
+ if (spanObject.getPeerId() == 0) {
+ peers = String.valueOf(spanObject.getPeerId());
+ }
+ String agg = spanObject.getComponent() + Const.ID_SPLIT + peers;
+ nodeComponents.add(agg);
+ }
+
+ @Override public void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId) {
+ String peers = String.valueOf(applicationId);
+ String agg = spanObject.getComponent() + Const.ID_SPLIT + peers;
+ nodeComponents.add(agg);
+ }
+
+ @Override public void parseFirst(SpanObject spanObject, int applicationId, int applicationInstanceId) {
+ timeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(spanObject.getStartTime());
+ }
+
+ @Override public void build() {
+ for (String agg : nodeComponents) {
+ NodeComponentDataDefine.NodeComponent nodeComponent = new NodeComponentDataDefine.NodeComponent();
+ nodeComponent.setId(timeBucket + Const.ID_SPLIT + agg);
+ nodeComponent.setAgg(agg);
+ nodeComponent.setTimeBucket(timeBucket);
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java
new file mode 100644
index 000000000..61941347b
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java
@@ -0,0 +1,75 @@
+package org.skywalking.apm.collector.agentstream.worker.node.component.define;
+
+import org.skywalking.apm.collector.remote.grpc.proto.RemoteData;
+import org.skywalking.apm.collector.stream.worker.impl.data.Attribute;
+import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
+import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation;
+import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation;
+
+/**
+ * @author pengys5
+ */
+public class NodeComponentDataDefine extends DataDefine {
+
+ @Override public int defineId() {
+ return 101;
+ }
+
+ @Override protected int initialCapacity() {
+ return 3;
+ }
+
+ @Override protected void attributeDefine() {
+ addAttribute(0, new Attribute(NodeComponentTable.COLUMN_ID, AttributeType.STRING, new NonOperation()));
+ addAttribute(1, new Attribute(NodeComponentTable.COLUMN_AGG, AttributeType.STRING, new CoverOperation()));
+ addAttribute(2, new Attribute(NodeComponentTable.COLUMN_TIME_BUCKET, AttributeType.LONG, new CoverOperation()));
+ }
+
+ @Override public Object deserialize(RemoteData remoteData) {
+ return null;
+ }
+
+ @Override public RemoteData serialize(Object object) {
+ return null;
+ }
+
+ public static class NodeComponent {
+ private String id;
+ private String agg;
+ private long timeBucket;
+
+ public NodeComponent(String id, String agg, long timeBucket) {
+ this.id = id;
+ this.agg = agg;
+ this.timeBucket = timeBucket;
+ }
+
+ public NodeComponent() {
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public String getAgg() {
+ return agg;
+ }
+
+ public long getTimeBucket() {
+ return timeBucket;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public void setAgg(String agg) {
+ this.agg = agg;
+ }
+
+ public void setTimeBucket(long timeBucket) {
+ this.timeBucket = timeBucket;
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentEsTableDefine.java
similarity index 70%
rename from apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentEsTableDefine.java
rename to apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentEsTableDefine.java
index 01f76b785..700393edd 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentEsTableDefine.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentEsTableDefine.java
@@ -1,6 +1,5 @@
-package org.skywalking.apm.collector.agentstream.worker.node.define;
+package org.skywalking.apm.collector.agentstream.worker.node.component.define;
-import org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentTable;
import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine;
import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine;
@@ -26,8 +25,7 @@ public class NodeComponentEsTableDefine extends ElasticSearchTableDefine {
}
@Override public void initialize() {
- addColumn(new ElasticSearchColumnDefine(NodeComponentTable.COLUMN_NAME, ElasticSearchColumnDefine.Type.Keyword.name()));
- addColumn(new ElasticSearchColumnDefine(NodeComponentTable.COLUMN_PEERS, ElasticSearchColumnDefine.Type.Keyword.name()));
addColumn(new ElasticSearchColumnDefine(NodeComponentTable.COLUMN_AGG, ElasticSearchColumnDefine.Type.Keyword.name()));
+ addColumn(new ElasticSearchColumnDefine(NodeComponentTable.COLUMN_TIME_BUCKET, ElasticSearchColumnDefine.Type.Long.name()));
}
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentH2TableDefine.java
similarity index 56%
rename from apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentH2TableDefine.java
rename to apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentH2TableDefine.java
index 550890ba2..62b77b04e 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeComponentH2TableDefine.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentH2TableDefine.java
@@ -1,6 +1,5 @@
-package org.skywalking.apm.collector.agentstream.worker.node.define;
+package org.skywalking.apm.collector.agentstream.worker.node.component.define;
-import org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentTable;
import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine;
import org.skywalking.apm.collector.storage.h2.define.H2TableDefine;
@@ -14,8 +13,8 @@ public class NodeComponentH2TableDefine extends H2TableDefine {
}
@Override public void initialize() {
- addColumn(new H2ColumnDefine(NodeComponentTable.COLUMN_NAME, H2ColumnDefine.Type.Varchar.name()));
- addColumn(new H2ColumnDefine(NodeComponentTable.COLUMN_PEERS, H2ColumnDefine.Type.Varchar.name()));
+ addColumn(new H2ColumnDefine(NodeComponentTable.COLUMN_ID, H2ColumnDefine.Type.Varchar.name()));
addColumn(new H2ColumnDefine(NodeComponentTable.COLUMN_AGG, H2ColumnDefine.Type.Varchar.name()));
+ addColumn(new H2ColumnDefine(NodeComponentTable.COLUMN_TIME_BUCKET, H2ColumnDefine.Type.Bigint.name()));
}
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentTable.java
similarity index 70%
rename from apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentTable.java
rename to apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentTable.java
index 211c8b5e6..1bce8df83 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentTable.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentTable.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.agentstream.worker.node.component;
+package org.skywalking.apm.collector.agentstream.worker.node.component.define;
import org.skywalking.apm.collector.agentstream.worker.CommonTable;
@@ -7,6 +7,4 @@ import org.skywalking.apm.collector.agentstream.worker.CommonTable;
*/
public class NodeComponentTable extends CommonTable {
public static final String TABLE = "node_component";
- public static final String COLUMN_NAME = "name";
- public static final String COLUMN_PEERS = "peers";
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingSpanListener.java
new file mode 100644
index 000000000..6240656fb
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingSpanListener.java
@@ -0,0 +1,47 @@
+package org.skywalking.apm.collector.agentstream.worker.node.mapping;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.skywalking.apm.collector.agentstream.worker.Const;
+import org.skywalking.apm.collector.agentstream.worker.node.mapping.define.NodeMappingDataDefine;
+import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener;
+import org.skywalking.apm.collector.agentstream.worker.segment.RefsListener;
+import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils;
+import org.skywalking.apm.network.proto.SpanObject;
+import org.skywalking.apm.network.proto.TraceSegmentReference;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class NodeMappingSpanListener implements RefsListener, FirstSpanListener {
+
+ private final Logger logger = LoggerFactory.getLogger(NodeMappingSpanListener.class);
+
+ private List nodeMappings = new ArrayList<>();
+ private long timeBucket;
+
+ @Override public void parseRef(TraceSegmentReference reference, int applicationId, int applicationInstanceId) {
+ String peers = Const.PEERS_FRONT_SPLIT + reference.getNetworkAddressId() + Const.PEERS_BEHIND_SPLIT;
+ if (reference.getNetworkAddressId() == 0) {
+ peers = Const.PEERS_FRONT_SPLIT + reference.getNetworkAddress() + Const.PEERS_BEHIND_SPLIT;
+ }
+
+ String agg = applicationId + Const.ID_SPLIT + peers;
+ nodeMappings.add(agg);
+ }
+
+ @Override public void parseFirst(SpanObject spanObject, int applicationId, int applicationInstanceId) {
+ timeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(spanObject.getStartTime());
+ }
+
+ @Override public void build() {
+ for (String agg : nodeMappings) {
+ NodeMappingDataDefine.NodeMapping nodeMapping = new NodeMappingDataDefine.NodeMapping();
+ nodeMapping.setId(timeBucket + Const.ID_SPLIT + agg);
+ nodeMapping.setAgg(agg);
+ nodeMapping.setTimeBucket(timeBucket);
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingDataDefine.java
new file mode 100644
index 000000000..f89b5d85e
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingDataDefine.java
@@ -0,0 +1,75 @@
+package org.skywalking.apm.collector.agentstream.worker.node.mapping.define;
+
+import org.skywalking.apm.collector.remote.grpc.proto.RemoteData;
+import org.skywalking.apm.collector.stream.worker.impl.data.Attribute;
+import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
+import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation;
+import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation;
+
+/**
+ * @author pengys5
+ */
+public class NodeMappingDataDefine extends DataDefine {
+
+ @Override public int defineId() {
+ return 102;
+ }
+
+ @Override protected int initialCapacity() {
+ return 3;
+ }
+
+ @Override protected void attributeDefine() {
+ addAttribute(0, new Attribute(NodeMappingTable.COLUMN_ID, AttributeType.STRING, new NonOperation()));
+ addAttribute(1, new Attribute(NodeMappingTable.COLUMN_AGG, AttributeType.STRING, new CoverOperation()));
+ addAttribute(2, new Attribute(NodeMappingTable.COLUMN_TIME_BUCKET, AttributeType.LONG, new CoverOperation()));
+ }
+
+ @Override public Object deserialize(RemoteData remoteData) {
+ return null;
+ }
+
+ @Override public RemoteData serialize(Object object) {
+ return null;
+ }
+
+ public static class NodeMapping {
+ private String id;
+ private String agg;
+ private long timeBucket;
+
+ public NodeMapping(String id, String agg, long timeBucket) {
+ this.id = id;
+ this.agg = agg;
+ this.timeBucket = timeBucket;
+ }
+
+ public NodeMapping() {
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public String getAgg() {
+ return agg;
+ }
+
+ public long getTimeBucket() {
+ return timeBucket;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public void setAgg(String agg) {
+ this.agg = agg;
+ }
+
+ public void setTimeBucket(long timeBucket) {
+ this.timeBucket = timeBucket;
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeMappingEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingEsTableDefine.java
similarity index 73%
rename from apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeMappingEsTableDefine.java
rename to apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingEsTableDefine.java
index ef8a1a2ef..29223471a 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeMappingEsTableDefine.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingEsTableDefine.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.agentstream.worker.node.define;
+package org.skywalking.apm.collector.agentstream.worker.node.mapping.define;
import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine;
import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine;
@@ -25,8 +25,6 @@ public class NodeMappingEsTableDefine extends ElasticSearchTableDefine {
}
@Override public void initialize() {
- addColumn(new ElasticSearchColumnDefine(NodeMappingTable.COLUMN_NAME, ElasticSearchColumnDefine.Type.Keyword.name()));
- addColumn(new ElasticSearchColumnDefine(NodeMappingTable.COLUMN_PEERS, ElasticSearchColumnDefine.Type.Keyword.name()));
addColumn(new ElasticSearchColumnDefine(NodeMappingTable.COLUMN_AGG, ElasticSearchColumnDefine.Type.Keyword.name()));
addColumn(new ElasticSearchColumnDefine(NodeMappingTable.COLUMN_TIME_BUCKET, ElasticSearchColumnDefine.Type.Long.name()));
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeMappingH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingH2TableDefine.java
similarity index 67%
rename from apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeMappingH2TableDefine.java
rename to apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingH2TableDefine.java
index f11217ccc..3805df48c 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeMappingH2TableDefine.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingH2TableDefine.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.agentstream.worker.node.define;
+package org.skywalking.apm.collector.agentstream.worker.node.mapping.define;
import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine;
import org.skywalking.apm.collector.storage.h2.define.H2TableDefine;
@@ -13,8 +13,7 @@ public class NodeMappingH2TableDefine extends H2TableDefine {
}
@Override public void initialize() {
- addColumn(new H2ColumnDefine(NodeMappingTable.COLUMN_NAME, H2ColumnDefine.Type.Varchar.name()));
- addColumn(new H2ColumnDefine(NodeMappingTable.COLUMN_PEERS, H2ColumnDefine.Type.Varchar.name()));
+ addColumn(new H2ColumnDefine(NodeMappingTable.COLUMN_ID, H2ColumnDefine.Type.Varchar.name()));
addColumn(new H2ColumnDefine(NodeMappingTable.COLUMN_AGG, H2ColumnDefine.Type.Varchar.name()));
addColumn(new H2ColumnDefine(NodeMappingTable.COLUMN_TIME_BUCKET, H2ColumnDefine.Type.Bigint.name()));
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeMappingTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingTable.java
similarity index 53%
rename from apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeMappingTable.java
rename to apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingTable.java
index 26f728fa9..811b297a2 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/define/NodeMappingTable.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingTable.java
@@ -1,4 +1,4 @@
-package org.skywalking.apm.collector.agentstream.worker.node.define;
+package org.skywalking.apm.collector.agentstream.worker.node.mapping.define;
import org.skywalking.apm.collector.agentstream.worker.CommonTable;
@@ -7,6 +7,4 @@ import org.skywalking.apm.collector.agentstream.worker.CommonTable;
*/
public class NodeMappingTable extends CommonTable {
public static final String TABLE = "node_mapping";
- public static final String COLUMN_NAME = "name";
- public static final String COLUMN_PEERS = "peers";
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefSpanListener.java
new file mode 100644
index 000000000..19c76d606
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefSpanListener.java
@@ -0,0 +1,68 @@
+package org.skywalking.apm.collector.agentstream.worker.noderef.reference;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.skywalking.apm.collector.agentstream.worker.Const;
+import org.skywalking.apm.collector.agentstream.worker.noderef.reference.define.NodeRefDataDefine;
+import org.skywalking.apm.collector.agentstream.worker.segment.EntrySpanListener;
+import org.skywalking.apm.collector.agentstream.worker.segment.ExitSpanListener;
+import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener;
+import org.skywalking.apm.collector.agentstream.worker.segment.RefsListener;
+import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils;
+import org.skywalking.apm.network.proto.SpanObject;
+import org.skywalking.apm.network.proto.TraceSegmentReference;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class NodeRefSpanListener implements EntrySpanListener, ExitSpanListener, FirstSpanListener, RefsListener {
+
+ private final Logger logger = LoggerFactory.getLogger(NodeRefSpanListener.class);
+
+ private List nodeExitReferences = new ArrayList<>();
+ private List nodeEntryReferences = new ArrayList<>();
+ private long timeBucket;
+ private boolean hasReference = false;
+
+ @Override public void parseExit(SpanObject spanObject, int applicationId, int applicationInstanceId) {
+ String front = String.valueOf(applicationId);
+ String behind = String.valueOf(spanObject.getPeerId());
+ if (spanObject.getPeerId() == 0) {
+ behind = spanObject.getPeer();
+ }
+
+ String agg = front + Const.ID_SPLIT + behind;
+ nodeExitReferences.add(agg);
+ }
+
+ @Override public void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId) {
+ String behind = String.valueOf(applicationId);
+ String front = Const.USER_CODE;
+
+ String agg = front + Const.ID_SPLIT + behind;
+ nodeEntryReferences.add(agg);
+ }
+
+ @Override public void parseFirst(SpanObject spanObject, int applicationId, int applicationInstanceId) {
+ timeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(spanObject.getStartTime());
+ }
+
+ @Override public void parseRef(TraceSegmentReference reference, int applicationId, int applicationInstanceId) {
+ hasReference = true;
+ }
+
+ @Override public void build() {
+ if (!hasReference) {
+ nodeExitReferences.addAll(nodeEntryReferences);
+ }
+
+ for (String agg : nodeExitReferences) {
+ NodeRefDataDefine.NodeReference nodeReference = new NodeRefDataDefine.NodeReference();
+ nodeReference.setId(timeBucket + Const.ID_SPLIT + agg);
+ nodeReference.setAgg(agg);
+ nodeReference.setTimeBucket(timeBucket);
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefDataDefine.java
new file mode 100644
index 000000000..09031c8a9
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefDataDefine.java
@@ -0,0 +1,85 @@
+package org.skywalking.apm.collector.agentstream.worker.noderef.reference.define;
+
+import org.skywalking.apm.collector.remote.grpc.proto.RemoteData;
+import org.skywalking.apm.collector.stream.worker.impl.data.Attribute;
+import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
+import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation;
+import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation;
+
+/**
+ * @author pengys5
+ */
+public class NodeRefDataDefine extends DataDefine {
+
+ public static final int DEFINE_ID = 201;
+
+ @Override public int defineId() {
+ return DEFINE_ID;
+ }
+
+ @Override protected int initialCapacity() {
+ return 3;
+ }
+
+ @Override protected void attributeDefine() {
+ addAttribute(0, new Attribute(NodeRefTable.COLUMN_ID, AttributeType.STRING, new NonOperation()));
+ addAttribute(1, new Attribute(NodeRefTable.COLUMN_AGG, AttributeType.STRING, new CoverOperation()));
+ addAttribute(2, new Attribute(NodeRefTable.COLUMN_TIME_BUCKET, AttributeType.LONG, new CoverOperation()));
+ }
+
+ @Override public Object deserialize(RemoteData remoteData) {
+ String id = remoteData.getDataStrings(0);
+ String agg = remoteData.getDataStrings(1);
+ long timeBucket = remoteData.getDataLongs(0);
+ return new NodeReference(id, agg, timeBucket);
+ }
+
+ @Override public RemoteData serialize(Object object) {
+ NodeReference nodeReference = (NodeReference)object;
+ RemoteData.Builder builder = RemoteData.newBuilder();
+ builder.addDataStrings(nodeReference.getId());
+ builder.addDataStrings(nodeReference.getAgg());
+ builder.addDataLongs(nodeReference.getTimeBucket());
+ return builder.build();
+ }
+
+ public static class NodeReference {
+ private String id;
+ private String agg;
+ private long timeBucket;
+
+ public NodeReference(String id, String agg, long timeBucket) {
+ this.id = id;
+ this.agg = agg;
+ this.timeBucket = timeBucket;
+ }
+
+ public NodeReference() {
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public String getAgg() {
+ return agg;
+ }
+
+ public long getTimeBucket() {
+ return timeBucket;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public void setAgg(String agg) {
+ this.agg = agg;
+ }
+
+ public void setTimeBucket(long timeBucket) {
+ this.timeBucket = timeBucket;
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefEsTableDefine.java
new file mode 100644
index 000000000..7ba2e614f
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefEsTableDefine.java
@@ -0,0 +1,31 @@
+package org.skywalking.apm.collector.agentstream.worker.noderef.reference.define;
+
+import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine;
+import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine;
+
+/**
+ * @author pengys5
+ */
+public class NodeRefEsTableDefine extends ElasticSearchTableDefine {
+
+ public NodeRefEsTableDefine() {
+ super(NodeRefTable.TABLE);
+ }
+
+ @Override public int refreshInterval() {
+ return 0;
+ }
+
+ @Override public int numberOfShards() {
+ return 2;
+ }
+
+ @Override public int numberOfReplicas() {
+ return 0;
+ }
+
+ @Override public void initialize() {
+ addColumn(new ElasticSearchColumnDefine(NodeRefTable.COLUMN_AGG, ElasticSearchColumnDefine.Type.Keyword.name()));
+ addColumn(new ElasticSearchColumnDefine(NodeRefTable.COLUMN_TIME_BUCKET, ElasticSearchColumnDefine.Type.Long.name()));
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefH2TableDefine.java
new file mode 100644
index 000000000..c2815b1de
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefH2TableDefine.java
@@ -0,0 +1,19 @@
+package org.skywalking.apm.collector.agentstream.worker.noderef.reference.define;
+
+import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine;
+import org.skywalking.apm.collector.storage.h2.define.H2TableDefine;
+
+/**
+ * @author pengys5
+ */
+public class NodeRefH2TableDefine extends H2TableDefine {
+
+ public NodeRefH2TableDefine() {
+ super(NodeRefTable.TABLE);
+ }
+
+ @Override public void initialize() {
+ addColumn(new H2ColumnDefine(NodeRefTable.COLUMN_AGG, H2ColumnDefine.Type.Varchar.name()));
+ addColumn(new H2ColumnDefine(NodeRefTable.COLUMN_TIME_BUCKET, H2ColumnDefine.Type.Bigint.name()));
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefTable.java
new file mode 100644
index 000000000..17c058b09
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefTable.java
@@ -0,0 +1,10 @@
+package org.skywalking.apm.collector.agentstream.worker.noderef.reference.define;
+
+import org.skywalking.apm.collector.agentstream.worker.CommonTable;
+
+/**
+ * @author pengys5
+ */
+public class NodeRefTable extends CommonTable {
+ public static final String TABLE = "node_reference";
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumDataDefine.java
new file mode 100644
index 000000000..758ecc16a
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumDataDefine.java
@@ -0,0 +1,164 @@
+package org.skywalking.apm.collector.agentstream.worker.noderef.summary.define;
+
+import org.skywalking.apm.collector.remote.grpc.proto.RemoteData;
+import org.skywalking.apm.collector.stream.worker.impl.data.Attribute;
+import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
+import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation;
+import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation;
+
+/**
+ * @author pengys5
+ */
+public class NodeRefSumDataDefine extends DataDefine {
+
+ public static final int DEFINE_ID = 202;
+
+ @Override public int defineId() {
+ return DEFINE_ID;
+ }
+
+ @Override protected int initialCapacity() {
+ return 9;
+ }
+
+ @Override protected void attributeDefine() {
+ addAttribute(0, new Attribute(NodeRefSumTable.COLUMN_ID, AttributeType.STRING, new NonOperation()));
+ addAttribute(1, new Attribute(NodeRefSumTable.COLUMN_ONE_SECOND_LESS, AttributeType.LONG, new NonOperation()));
+ addAttribute(2, new Attribute(NodeRefSumTable.COLUMN_THREE_SECOND_LESS, AttributeType.LONG, new NonOperation()));
+ addAttribute(3, new Attribute(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS, AttributeType.LONG, new NonOperation()));
+ addAttribute(4, new Attribute(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER, AttributeType.LONG, new NonOperation()));
+ addAttribute(5, new Attribute(NodeRefSumTable.COLUMN_ERROR, AttributeType.LONG, new NonOperation()));
+ addAttribute(6, new Attribute(NodeRefSumTable.COLUMN_SUMMARY, AttributeType.LONG, new NonOperation()));
+ addAttribute(7, new Attribute(NodeRefSumTable.COLUMN_AGG, AttributeType.STRING, new CoverOperation()));
+ addAttribute(8, new Attribute(NodeRefSumTable.COLUMN_TIME_BUCKET, AttributeType.LONG, new CoverOperation()));
+ }
+
+ @Override public Object deserialize(RemoteData remoteData) {
+ String id = remoteData.getDataStrings(0);
+ String agg = remoteData.getDataStrings(1);
+ Long oneSecondLess = remoteData.getDataLongs(0);
+ Long threeSecondLess = remoteData.getDataLongs(1);
+ Long fiveSecondLess = remoteData.getDataLongs(2);
+ Long fiveSecondGreater = remoteData.getDataLongs(3);
+ Long error = remoteData.getDataLongs(4);
+ Long summary = remoteData.getDataLongs(5);
+ long timeBucket = remoteData.getDataLongs(6);
+ return new NodeReferenceSum(id, oneSecondLess, threeSecondLess, fiveSecondLess, fiveSecondGreater, error, summary, agg, timeBucket);
+ }
+
+ @Override public RemoteData serialize(Object object) {
+ NodeReferenceSum nodeReferenceSum = (NodeReferenceSum)object;
+ RemoteData.Builder builder = RemoteData.newBuilder();
+ builder.addDataStrings(nodeReferenceSum.getId());
+ builder.addDataStrings(nodeReferenceSum.getAgg());
+ builder.addDataLongs(nodeReferenceSum.getOneSecondLess());
+ builder.addDataLongs(nodeReferenceSum.getThreeSecondLess());
+ builder.addDataLongs(nodeReferenceSum.getFiveSecondLess());
+ builder.addDataLongs(nodeReferenceSum.getFiveSecondGreater());
+ builder.addDataLongs(nodeReferenceSum.getError());
+ builder.addDataLongs(nodeReferenceSum.getSummary());
+ builder.addDataLongs(nodeReferenceSum.getTimeBucket());
+ return builder.build();
+ }
+
+ public static class NodeReferenceSum {
+ private String id;
+ private Long oneSecondLess;
+ private Long threeSecondLess;
+ private Long fiveSecondLess;
+ private Long fiveSecondGreater;
+ private Long error;
+ private Long summary;
+ private String agg;
+ private long timeBucket;
+
+ public NodeReferenceSum(String id, Long oneSecondLess, Long threeSecondLess, Long fiveSecondLess,
+ Long fiveSecondGreater, Long error, Long summary, String agg, long timeBucket) {
+ this.id = id;
+ this.oneSecondLess = oneSecondLess;
+ this.threeSecondLess = threeSecondLess;
+ this.fiveSecondLess = fiveSecondLess;
+ this.fiveSecondGreater = fiveSecondGreater;
+ this.error = error;
+ this.summary = summary;
+ this.agg = agg;
+ this.timeBucket = timeBucket;
+ }
+
+ public NodeReferenceSum() {
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public Long getOneSecondLess() {
+ return oneSecondLess;
+ }
+
+ public void setOneSecondLess(Long oneSecondLess) {
+ this.oneSecondLess = oneSecondLess;
+ }
+
+ public Long getThreeSecondLess() {
+ return threeSecondLess;
+ }
+
+ public void setThreeSecondLess(Long threeSecondLess) {
+ this.threeSecondLess = threeSecondLess;
+ }
+
+ public Long getFiveSecondLess() {
+ return fiveSecondLess;
+ }
+
+ public void setFiveSecondLess(Long fiveSecondLess) {
+ this.fiveSecondLess = fiveSecondLess;
+ }
+
+ public Long getFiveSecondGreater() {
+ return fiveSecondGreater;
+ }
+
+ public void setFiveSecondGreater(Long fiveSecondGreater) {
+ this.fiveSecondGreater = fiveSecondGreater;
+ }
+
+ public Long getError() {
+ return error;
+ }
+
+ public void setError(Long error) {
+ this.error = error;
+ }
+
+ public Long getSummary() {
+ return summary;
+ }
+
+ public void setSummary(Long summary) {
+ this.summary = summary;
+ }
+
+ public String getAgg() {
+ return agg;
+ }
+
+ public void setAgg(String agg) {
+ this.agg = agg;
+ }
+
+ public long getTimeBucket() {
+ return timeBucket;
+ }
+
+ public void setTimeBucket(long timeBucket) {
+ this.timeBucket = timeBucket;
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumEsTableDefine.java
new file mode 100644
index 000000000..c10b0d4a6
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumEsTableDefine.java
@@ -0,0 +1,37 @@
+package org.skywalking.apm.collector.agentstream.worker.noderef.summary.define;
+
+import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine;
+import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine;
+
+/**
+ * @author pengys5
+ */
+public class NodeRefSumEsTableDefine extends ElasticSearchTableDefine {
+
+ public NodeRefSumEsTableDefine() {
+ super(NodeRefSumTable.TABLE);
+ }
+
+ @Override public int refreshInterval() {
+ return 0;
+ }
+
+ @Override public int numberOfShards() {
+ return 2;
+ }
+
+ @Override public int numberOfReplicas() {
+ return 0;
+ }
+
+ @Override public void initialize() {
+ addColumn(new ElasticSearchColumnDefine(NodeRefSumTable.COLUMN_ONE_SECOND_LESS, ElasticSearchColumnDefine.Type.Long.name()));
+ addColumn(new ElasticSearchColumnDefine(NodeRefSumTable.COLUMN_THREE_SECOND_LESS, ElasticSearchColumnDefine.Type.Long.name()));
+ addColumn(new ElasticSearchColumnDefine(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS, ElasticSearchColumnDefine.Type.Long.name()));
+ addColumn(new ElasticSearchColumnDefine(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER, ElasticSearchColumnDefine.Type.Long.name()));
+ addColumn(new ElasticSearchColumnDefine(NodeRefSumTable.COLUMN_ERROR, ElasticSearchColumnDefine.Type.Long.name()));
+ addColumn(new ElasticSearchColumnDefine(NodeRefSumTable.COLUMN_SUMMARY, ElasticSearchColumnDefine.Type.Long.name()));
+ addColumn(new ElasticSearchColumnDefine(NodeRefSumTable.COLUMN_AGG, ElasticSearchColumnDefine.Type.Keyword.name()));
+ addColumn(new ElasticSearchColumnDefine(NodeRefSumTable.COLUMN_TIME_BUCKET, ElasticSearchColumnDefine.Type.Long.name()));
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumH2TableDefine.java
new file mode 100644
index 000000000..a31257102
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumH2TableDefine.java
@@ -0,0 +1,25 @@
+package org.skywalking.apm.collector.agentstream.worker.noderef.summary.define;
+
+import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine;
+import org.skywalking.apm.collector.storage.h2.define.H2TableDefine;
+
+/**
+ * @author pengys5
+ */
+public class NodeRefSumH2TableDefine extends H2TableDefine {
+
+ public NodeRefSumH2TableDefine() {
+ super(NodeRefSumTable.TABLE);
+ }
+
+ @Override public void initialize() {
+ addColumn(new H2ColumnDefine(NodeRefSumTable.COLUMN_ONE_SECOND_LESS, H2ColumnDefine.Type.Bigint.name()));
+ addColumn(new H2ColumnDefine(NodeRefSumTable.COLUMN_THREE_SECOND_LESS, H2ColumnDefine.Type.Bigint.name()));
+ addColumn(new H2ColumnDefine(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS, H2ColumnDefine.Type.Bigint.name()));
+ addColumn(new H2ColumnDefine(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER, H2ColumnDefine.Type.Bigint.name()));
+ addColumn(new H2ColumnDefine(NodeRefSumTable.COLUMN_ERROR, H2ColumnDefine.Type.Bigint.name()));
+ addColumn(new H2ColumnDefine(NodeRefSumTable.COLUMN_SUMMARY, H2ColumnDefine.Type.Bigint.name()));
+ addColumn(new H2ColumnDefine(NodeRefSumTable.COLUMN_AGG, H2ColumnDefine.Type.Varchar.name()));
+ addColumn(new H2ColumnDefine(NodeRefSumTable.COLUMN_TIME_BUCKET, H2ColumnDefine.Type.Bigint.name()));
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumTable.java
new file mode 100644
index 000000000..0c07ae9a8
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumTable.java
@@ -0,0 +1,16 @@
+package org.skywalking.apm.collector.agentstream.worker.noderef.summary.define;
+
+import org.skywalking.apm.collector.agentstream.worker.CommonTable;
+
+/**
+ * @author pengys5
+ */
+public class NodeRefSumTable extends CommonTable {
+ public static final String TABLE = "node_reference_sum";
+ public static final String COLUMN_ONE_SECOND_LESS = "one_second_less";
+ public static final String COLUMN_THREE_SECOND_LESS = "three_second_less";
+ public static final String COLUMN_FIVE_SECOND_LESS = "five_second_less";
+ public static final String COLUMN_FIVE_SECOND_GREATER = "five_second_greater";
+ public static final String COLUMN_ERROR = "error";
+ public static final String COLUMN_SUMMARY = "summary";
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/EntrySpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/EntrySpanListener.java
new file mode 100644
index 000000000..1879681b6
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/EntrySpanListener.java
@@ -0,0 +1,10 @@
+package org.skywalking.apm.collector.agentstream.worker.segment;
+
+import org.skywalking.apm.network.proto.SpanObject;
+
+/**
+ * @author pengys5
+ */
+public interface EntrySpanListener extends SpanListener {
+ void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId);
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/ExitSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/ExitSpanListener.java
new file mode 100644
index 000000000..baace12bd
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/ExitSpanListener.java
@@ -0,0 +1,10 @@
+package org.skywalking.apm.collector.agentstream.worker.segment;
+
+import org.skywalking.apm.network.proto.SpanObject;
+
+/**
+ * @author pengys5
+ */
+public interface ExitSpanListener extends SpanListener {
+ void parseExit(SpanObject spanObject, int applicationId, int applicationInstanceId);
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/FirstSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/FirstSpanListener.java
new file mode 100644
index 000000000..b1617cfea
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/FirstSpanListener.java
@@ -0,0 +1,10 @@
+package org.skywalking.apm.collector.agentstream.worker.segment;
+
+import org.skywalking.apm.network.proto.SpanObject;
+
+/**
+ * @author pengys5
+ */
+public interface FirstSpanListener extends SpanListener {
+ void parseFirst(SpanObject spanObject, int applicationId, int applicationInstanceId);
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/LocalSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/LocalSpanListener.java
new file mode 100644
index 000000000..0486b3ffe
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/LocalSpanListener.java
@@ -0,0 +1,10 @@
+package org.skywalking.apm.collector.agentstream.worker.segment;
+
+import org.skywalking.apm.network.proto.SpanObject;
+
+/**
+ * @author pengys5
+ */
+public interface LocalSpanListener extends SpanListener {
+ void parseLocal(SpanObject spanObject, int applicationId, int applicationInstanceId);
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/RefsListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/RefsListener.java
new file mode 100644
index 000000000..a931cef5a
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/RefsListener.java
@@ -0,0 +1,10 @@
+package org.skywalking.apm.collector.agentstream.worker.segment;
+
+import org.skywalking.apm.network.proto.TraceSegmentReference;
+
+/**
+ * @author pengys5
+ */
+public interface RefsListener extends SpanListener {
+ void parseRef(TraceSegmentReference reference, int applicationId, int applicationInstanceId);
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java
index 09a90bfd9..2e6168b8e 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java
@@ -1,8 +1,107 @@
package org.skywalking.apm.collector.agentstream.worker.segment;
+import java.util.ArrayList;
+import java.util.List;
+import org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentSpanListener;
+import org.skywalking.apm.collector.agentstream.worker.node.mapping.NodeMappingSpanListener;
+import org.skywalking.apm.collector.agentstream.worker.noderef.reference.NodeRefSpanListener;
+import org.skywalking.apm.collector.core.util.CollectionUtils;
+import org.skywalking.apm.network.proto.SpanObject;
+import org.skywalking.apm.network.proto.SpanType;
+import org.skywalking.apm.network.proto.TraceSegmentObject;
+import org.skywalking.apm.network.proto.TraceSegmentReference;
+import org.skywalking.apm.network.proto.UniqueId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* @author pengys5
*/
public class SegmentParse {
+ private final Logger logger = LoggerFactory.getLogger(SegmentParse.class);
+
+ private List spanListeners;
+ private List refsListeners;
+
+ public SegmentParse() {
+ spanListeners = new ArrayList<>();
+ spanListeners.add(new NodeRefSpanListener());
+ spanListeners.add(new NodeComponentSpanListener());
+ spanListeners.add(new NodeMappingSpanListener());
+
+ refsListeners = new ArrayList<>();
+ }
+
+ public void parse(List traceIds, TraceSegmentObject segmentObject) {
+ for (UniqueId uniqueId : traceIds) {
+ uniqueId.getIdPartsList();
+ }
+
+ int applicationId = segmentObject.getApplicationId();
+ int applicationInstanceId = segmentObject.getApplicationInstanceId();
+
+ for (TraceSegmentReference reference : segmentObject.getRefsList()) {
+ notifyRefsListener(reference, applicationId, applicationInstanceId);
+ }
+
+ List spans = segmentObject.getSpansList();
+ if (CollectionUtils.isNotEmpty(spans)) {
+ for (SpanObject spanObject : spans) {
+ if (spanObject.getSpanId() == 0) {
+ notifyFirstListener(spanObject, applicationId, applicationInstanceId);
+ }
+
+ if (SpanType.Exit.equals(spanObject.getSpanType())) {
+ notifyExitListener(spanObject, applicationId, applicationInstanceId);
+ } else if (SpanType.Entry.equals(spanObject.getSpanType())) {
+ notifyEntryListener(spanObject, applicationId, applicationInstanceId);
+ } else if (SpanType.Local.equals(spanObject.getSpanType())) {
+ notifyLocalListener(spanObject, applicationId, applicationInstanceId);
+ } else {
+ logger.error("span type error, span type: {}", spanObject.getSpanType().name());
+ }
+ }
+ }
+ }
+
+ private void notifyExitListener(SpanObject spanObject, int applicationId, int applicationInstanceId) {
+ for (SpanListener listener : spanListeners) {
+ if (listener instanceof ExitSpanListener) {
+ ((ExitSpanListener)listener).parseExit(spanObject, applicationId, applicationInstanceId);
+ }
+ }
+ }
+
+ private void notifyEntryListener(SpanObject spanObject, int applicationId, int applicationInstanceId) {
+ for (SpanListener listener : spanListeners) {
+ if (listener instanceof EntrySpanListener) {
+ ((EntrySpanListener)listener).parseEntry(spanObject, applicationId, applicationInstanceId);
+ }
+ }
+ }
+
+ private void notifyLocalListener(SpanObject spanObject, int applicationId, int applicationInstanceId) {
+ for (SpanListener listener : spanListeners) {
+ if (listener instanceof LocalSpanListener) {
+ ((LocalSpanListener)listener).parseLocal(spanObject, applicationId, applicationInstanceId);
+ }
+ }
+ }
+
+ private void notifyFirstListener(SpanObject spanObject, int applicationId, int applicationInstanceId) {
+ for (SpanListener listener : spanListeners) {
+ if (listener instanceof FirstSpanListener) {
+ ((FirstSpanListener)listener).parseFirst(spanObject, applicationId, applicationInstanceId);
+ }
+ }
+ }
+
+ private void notifyRefsListener(TraceSegmentReference reference, int applicationId, int applicationInstanceId) {
+ for (SpanListener listener : refsListeners) {
+ if (listener instanceof RefsListener) {
+ ((RefsListener)listener).parseRef(reference, applicationId, applicationInstanceId);
+ }
+ }
+ }
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SpanListener.java
new file mode 100644
index 000000000..3dde04717
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SpanListener.java
@@ -0,0 +1,8 @@
+package org.skywalking.apm.collector.agentstream.worker.segment;
+
+/**
+ * @author pengys5
+ */
+public interface SpanListener {
+ void build();
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/ServiceRefSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/ServiceRefSpanListener.java
new file mode 100644
index 000000000..9c5cc6f87
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/ServiceRefSpanListener.java
@@ -0,0 +1,79 @@
+package org.skywalking.apm.collector.agentstream.worker.serviceref.reference;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.skywalking.apm.collector.agentstream.worker.Const;
+import org.skywalking.apm.collector.agentstream.worker.segment.EntrySpanListener;
+import org.skywalking.apm.collector.agentstream.worker.segment.ExitSpanListener;
+import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener;
+import org.skywalking.apm.collector.agentstream.worker.segment.RefsListener;
+import org.skywalking.apm.collector.agentstream.worker.serviceref.reference.define.ServiceRefDataDefine;
+import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils;
+import org.skywalking.apm.network.proto.SpanObject;
+import org.skywalking.apm.network.proto.TraceSegmentReference;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class ServiceRefSpanListener implements FirstSpanListener, EntrySpanListener, ExitSpanListener, RefsListener {
+
+ private final Logger logger = LoggerFactory.getLogger(ServiceRefSpanListener.class);
+
+ private String front;
+ private List behinds = new ArrayList<>();
+ private List fronts = new ArrayList<>();
+ private long timeBucket;
+
+ @Override public void parseFirst(SpanObject spanObject, int applicationId, int applicationInstanceId) {
+ timeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(spanObject.getStartTime());
+ }
+
+ @Override public void parseRef(TraceSegmentReference reference, int applicationId, int applicationInstanceId) {
+ String entryService = String.valueOf(reference.getEntryServiceId());
+ if (reference.getEntryServiceId() == 0) {
+ entryService = reference.getEntryServiceName();
+ }
+ String parentService = String.valueOf(reference.getParentServiceId());
+ if (reference.getParentServiceId() == 0) {
+ parentService = reference.getParentServiceName();
+ }
+ fronts.add(new ServiceTemp(entryService, parentService));
+ }
+
+ @Override public void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId) {
+ front = String.valueOf(spanObject.getOperationNameId());
+ if (spanObject.getOperationNameId() == 0) {
+ front = spanObject.getOperationName();
+ }
+ }
+
+ @Override public void parseExit(SpanObject spanObject, int applicationId, int applicationInstanceId) {
+ String behind = String.valueOf(spanObject.getOperationNameId());
+ if (spanObject.getOperationNameId() == 0) {
+ behind = spanObject.getOperationName();
+ }
+ behinds.add(behind);
+ }
+
+ @Override public void build() {
+ for (String behind : behinds) {
+ String agg = front + Const.ID_SPLIT + behind;
+ ServiceRefDataDefine.ServiceReference serviceReference = new ServiceRefDataDefine.ServiceReference();
+ serviceReference.setId(timeBucket + Const.ID_SPLIT + agg);
+ serviceReference.setAgg(agg);
+ serviceReference.setTimeBucket(timeBucket);
+ }
+ }
+
+ class ServiceTemp {
+ private final String entryService;
+ private final String parentService;
+
+ public ServiceTemp(String entryService, String parentService) {
+ this.entryService = entryService;
+ this.parentService = parentService;
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefDataDefine.java
new file mode 100644
index 000000000..00ff6b140
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefDataDefine.java
@@ -0,0 +1,98 @@
+package org.skywalking.apm.collector.agentstream.worker.serviceref.reference.define;
+
+import org.skywalking.apm.collector.remote.grpc.proto.RemoteData;
+import org.skywalking.apm.collector.stream.worker.impl.data.Attribute;
+import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
+import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation;
+import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation;
+
+/**
+ * @author pengys5
+ */
+public class ServiceRefDataDefine extends DataDefine {
+
+ public static final int DEFINE_ID = 501;
+
+ @Override public int defineId() {
+ return DEFINE_ID;
+ }
+
+ @Override protected int initialCapacity() {
+ return 4;
+ }
+
+ @Override protected void attributeDefine() {
+ addAttribute(0, new Attribute(ServiceRefTable.COLUMN_ID, AttributeType.STRING, new NonOperation()));
+ addAttribute(1, new Attribute(ServiceRefTable.COLUMN_ENTRY_SERVICE, AttributeType.STRING, new NonOperation()));
+ addAttribute(2, new Attribute(ServiceRefTable.COLUMN_AGG, AttributeType.STRING, new CoverOperation()));
+ addAttribute(3, new Attribute(ServiceRefTable.COLUMN_TIME_BUCKET, AttributeType.LONG, new CoverOperation()));
+ }
+
+ @Override public Object deserialize(RemoteData remoteData) {
+ String id = remoteData.getDataStrings(0);
+ String entryService = remoteData.getDataStrings(1);
+ String agg = remoteData.getDataStrings(2);
+ long timeBucket = remoteData.getDataLongs(0);
+ return new ServiceReference(id, entryService, agg, timeBucket);
+ }
+
+ @Override public RemoteData serialize(Object object) {
+ ServiceReference serviceReference = (ServiceReference)object;
+ RemoteData.Builder builder = RemoteData.newBuilder();
+ builder.addDataStrings(serviceReference.getId());
+ builder.addDataStrings(serviceReference.getEntryService());
+ builder.addDataStrings(serviceReference.getAgg());
+ builder.addDataLongs(serviceReference.getTimeBucket());
+ return builder.build();
+ }
+
+ public static class ServiceReference {
+ private String id;
+ private String entryService;
+ private String agg;
+ private long timeBucket;
+
+ public ServiceReference(String id, String entryService, String agg, long timeBucket) {
+ this.id = id;
+ this.entryService = entryService;
+ this.agg = agg;
+ this.timeBucket = timeBucket;
+ }
+
+ public ServiceReference() {
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public String getAgg() {
+ return agg;
+ }
+
+ public long getTimeBucket() {
+ return timeBucket;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public void setAgg(String agg) {
+ this.agg = agg;
+ }
+
+ public void setTimeBucket(long timeBucket) {
+ this.timeBucket = timeBucket;
+ }
+
+ public String getEntryService() {
+ return entryService;
+ }
+
+ public void setEntryService(String entryService) {
+ this.entryService = entryService;
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefEsTableDefine.java
new file mode 100644
index 000000000..067624ed5
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefEsTableDefine.java
@@ -0,0 +1,32 @@
+package org.skywalking.apm.collector.agentstream.worker.serviceref.reference.define;
+
+import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine;
+import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine;
+
+/**
+ * @author pengys5
+ */
+public class ServiceRefEsTableDefine extends ElasticSearchTableDefine {
+
+ public ServiceRefEsTableDefine() {
+ super(ServiceRefTable.TABLE);
+ }
+
+ @Override public int refreshInterval() {
+ return 0;
+ }
+
+ @Override public int numberOfShards() {
+ return 2;
+ }
+
+ @Override public int numberOfReplicas() {
+ return 0;
+ }
+
+ @Override public void initialize() {
+ addColumn(new ElasticSearchColumnDefine(ServiceRefTable.COLUMN_AGG, ElasticSearchColumnDefine.Type.Keyword.name()));
+ addColumn(new ElasticSearchColumnDefine(ServiceRefTable.COLUMN_ENTRY_SERVICE, ElasticSearchColumnDefine.Type.Keyword.name()));
+ addColumn(new ElasticSearchColumnDefine(ServiceRefTable.COLUMN_TIME_BUCKET, ElasticSearchColumnDefine.Type.Long.name()));
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefH2TableDefine.java
new file mode 100644
index 000000000..c6756cab5
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefH2TableDefine.java
@@ -0,0 +1,20 @@
+package org.skywalking.apm.collector.agentstream.worker.serviceref.reference.define;
+
+import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine;
+import org.skywalking.apm.collector.storage.h2.define.H2TableDefine;
+
+/**
+ * @author pengys5
+ */
+public class ServiceRefH2TableDefine extends H2TableDefine {
+
+ public ServiceRefH2TableDefine() {
+ super(ServiceRefTable.TABLE);
+ }
+
+ @Override public void initialize() {
+ addColumn(new H2ColumnDefine(ServiceRefTable.COLUMN_AGG, H2ColumnDefine.Type.Varchar.name()));
+ addColumn(new H2ColumnDefine(ServiceRefTable.COLUMN_ENTRY_SERVICE, H2ColumnDefine.Type.Varchar.name()));
+ addColumn(new H2ColumnDefine(ServiceRefTable.COLUMN_TIME_BUCKET, H2ColumnDefine.Type.Bigint.name()));
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefTable.java
new file mode 100644
index 000000000..d145b93f5
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefTable.java
@@ -0,0 +1,11 @@
+package org.skywalking.apm.collector.agentstream.worker.serviceref.reference.define;
+
+import org.skywalking.apm.collector.agentstream.worker.CommonTable;
+
+/**
+ * @author pengys5
+ */
+public class ServiceRefTable extends CommonTable {
+ public static final String TABLE = "service_reference";
+ public static final String COLUMN_ENTRY_SERVICE = "entry_service";
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/util/TimeBucketUtils.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/util/TimeBucketUtils.java
new file mode 100644
index 000000000..8f8b0a114
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/util/TimeBucketUtils.java
@@ -0,0 +1,47 @@
+package org.skywalking.apm.collector.agentstream.worker.util;
+
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.TimeZone;
+
+/**
+ * @author pengys5
+ */
+public enum TimeBucketUtils {
+ INSTANCE;
+
+ private final SimpleDateFormat DAY_DATE_FORMAT = new SimpleDateFormat("yyyyMMdd");
+ private final SimpleDateFormat HOUR_DATE_FORMAT = new SimpleDateFormat("yyyyMMddHH");
+ private final SimpleDateFormat MINUTE_DATE_FORMAT = new SimpleDateFormat("yyyyMMddHHmm");
+
+ public long getMinuteTimeBucket(long time) {
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(time);
+ String timeStr = MINUTE_DATE_FORMAT.format(calendar.getTime());
+ return Long.valueOf(timeStr);
+ }
+
+ public long getHourTimeBucket(long time) {
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(time);
+ String timeStr = HOUR_DATE_FORMAT.format(calendar.getTime()) + "00";
+ return Long.valueOf(timeStr);
+ }
+
+ public long getDayTimeBucket(long time) {
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(time);
+ String timeStr = DAY_DATE_FORMAT.format(calendar.getTime()) + "0000";
+ return Long.valueOf(timeStr);
+ }
+
+ public long changeToUTCTimeBucket(long timeBucket) {
+ String timeBucketStr = String.valueOf(timeBucket);
+
+ if (TimeZone.getDefault().getID().equals("GMT+08:00") || timeBucketStr.endsWith("0000")) {
+ return timeBucket;
+ } else {
+ return timeBucket - 800;
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define
index 205e6c74a..c8abe9d72 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define
@@ -1,4 +1,4 @@
-org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentDataDefine
+org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentDataDefine
org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine
org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceDataDefine
org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameDataDefine
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define
index 0d0a2264d..d5017553e 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define
@@ -1,7 +1,7 @@
-org.skywalking.apm.collector.agentstream.worker.node.define.NodeComponentEsTableDefine
-org.skywalking.apm.collector.agentstream.worker.node.define.NodeComponentH2TableDefine
-org.skywalking.apm.collector.agentstream.worker.node.define.NodeMappingEsTableDefine
-org.skywalking.apm.collector.agentstream.worker.node.define.NodeMappingH2TableDefine
+org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentEsTableDefine
+org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentH2TableDefine
+org.skywalking.apm.collector.agentstream.worker.node.mapping.define.NodeMappingEsTableDefine
+org.skywalking.apm.collector.agentstream.worker.node.mapping.define.NodeMappingH2TableDefine
org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationEsTableDefine
org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationH2TableDefine
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/Const.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/Const.java
deleted file mode 100644
index c3acf882b..000000000
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/Const.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package org.skywalking.apm.collector.stream.worker.impl;
-
-/**
- * @author pengys5
- */
-public class Const {
- public static final String ID_SPLIT = "..-..";
- public static final String IDS_SPLIT = "\\.\\.-\\.\\.";
- public static final String PEERS_FRONT_SPLIT = "[";
- public static final String PEERS_BEHIND_SPLIT = "]";
- public static final String USER_CODE = "User";
- public static final String RESULT = "result";
-}
From d80e2ff43dc3639d79e5af48d3b914ab1170ec6f Mon Sep 17 00:00:00 2001
From: pengys5 <8082209@qq.com>
Date: Sat, 29 Jul 2017 12:07:42 +0800
Subject: [PATCH 17/28] node component save to es success.
---
.../AgentStreamModuleInstaller.java | 3 +
.../handler/TraceSegmentServiceHandler.java | 5 +-
.../agentstream/worker/CommonTable.java | 1 +
.../agentstream/worker/TimeSlice.java | 28 ------
...va => NodeComponentAggregationWorker.java} | 27 +++---
.../NodeComponentPersistenceWorker.java | 70 ++++++++++++++
.../component/NodeComponentRemoteWorker.java | 35 +++++++
.../component/NodeComponentSpanListener.java | 16 +++-
.../node/component/dao/INodeComponentDAO.java | 12 +++
.../component/dao/NodeComponentEsDAO.java | 29 ++++++
.../component/dao/NodeComponentH2DAO.java | 15 +++
.../define/NodeComponentDataDefine.java | 13 ++-
.../define/NodeComponentEsTableDefine.java | 2 +-
.../register/instance/InstanceDataDefine.java | 4 +-
.../register/instance/dao/InstanceH2DAO.java | 4 +
.../worker/segment/SegmentParse.java | 7 ++
.../worker/storage/PersistenceTimer.java | 55 +++++++++++
.../resources/META-INF/defines/es_dao.define | 3 +-
.../resources/META-INF/defines/h2_dao.define | 3 +-
.../local_async_worker_provider.define | 3 +-
.../defines/remote_worker_provider.define | 4 +-
.../TraceSegmentServiceHandlerTestCase.java | 92 +++++++++++++++++++
apm-collector/apm-collector-client/pom.xml | 4 -
.../elasticsearch/ElasticSearchClient.java | 5 +
.../src/main/resources/logback.xml | 2 +-
.../apm/collector/storage/dao/IBatchDAO.java | 10 ++
.../storage/elasticsearch/dao/BatchEsDAO.java | 35 +++++++
.../collector/storage/h2/dao/BatchH2DAO.java | 14 +++
.../resources/META-INF/defines/es_dao.define | 1 +
.../resources/META-INF/defines/h2_dao.define | 1 +
.../stream/StreamModuleInstaller.java | 1 -
.../AbstractLocalAsyncWorkerProvider.java | 6 ++
.../stream/worker/WorkerContext.java | 5 +
.../stream/worker/impl/AggregationWorker.java | 32 ++++++-
.../stream/worker/impl/FlushAndSwitch.java | 7 ++
.../stream/worker/impl/PersistenceWorker.java | 82 +++++++++++++++++
.../impl/PersistenceWorkerContainer.java | 21 +++++
.../stream/worker/impl/data/Data.java | 6 +-
.../stream/worker/impl/data/DataCache.java | 4 +
.../stream/worker/impl/data/DataDefine.java | 11 +--
.../worker/impl/data/DataDefineLoader.java | 1 -
.../worker/impl/data/TransformToData.java | 8 ++
.../stream/worker/impl/data/Window.java | 18 ++++
apm-collector/pom.xml | 10 ++
44 files changed, 644 insertions(+), 71 deletions(-)
delete mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/TimeSlice.java
rename apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/{NodeComponentAggWorker.java => NodeComponentAggregationWorker.java} (57%)
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentPersistenceWorker.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/INodeComponentDAO.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentEsDAO.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentH2DAO.java
create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java
create mode 100644 apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandlerTestCase.java
create mode 100644 apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/IBatchDAO.java
create mode 100644 apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/BatchEsDAO.java
create mode 100644 apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/BatchH2DAO.java
create mode 100644 apm-collector/apm-collector-storage/src/main/resources/META-INF/defines/es_dao.define
create mode 100644 apm-collector/apm-collector-storage/src/main/resources/META-INF/defines/h2_dao.define
create mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/FlushAndSwitch.java
create mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/PersistenceWorker.java
create mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/PersistenceWorkerContainer.java
create mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/TransformToData.java
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleInstaller.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleInstaller.java
index ace5f61c0..bf5227d48 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleInstaller.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleInstaller.java
@@ -2,6 +2,7 @@ package org.skywalking.apm.collector.agentstream;
import java.util.Iterator;
import java.util.Map;
+import org.skywalking.apm.collector.agentstream.worker.storage.PersistenceTimer;
import org.skywalking.apm.collector.core.client.ClientException;
import org.skywalking.apm.collector.core.framework.CollectorContextHelper;
import org.skywalking.apm.collector.core.framework.DefineException;
@@ -32,5 +33,7 @@ public class AgentStreamModuleInstaller implements ModuleInstaller {
logger.info("module {} initialize", moduleDefine.getClass().getName());
moduleDefine.initialize((ObjectUtils.isNotEmpty(moduleConfig) && moduleConfig.containsKey(moduleDefine.name())) ? moduleConfig.get(moduleDefine.name()) : null, serverHolder);
}
+
+ new PersistenceTimer().start();
}
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandler.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandler.java
index 17ab6e578..70887d0c1 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandler.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandler.java
@@ -20,11 +20,11 @@ public class TraceSegmentServiceHandler extends TraceSegmentServiceGrpc.TraceSeg
private final Logger logger = LoggerFactory.getLogger(TraceSegmentServiceHandler.class);
- private SegmentParse segmentParse = new SegmentParse();
-
@Override public StreamObserver collect(StreamObserver responseObserver) {
return new StreamObserver() {
@Override public void onNext(UpstreamSegment segment) {
+ logger.debug("receive segment");
+ SegmentParse segmentParse = new SegmentParse();
try {
List traceIds = segment.getGlobalTraceIdsList();
TraceSegmentObject segmentObject = TraceSegmentObject.parseFrom(segment.getSegment());
@@ -39,6 +39,7 @@ public class TraceSegmentServiceHandler extends TraceSegmentServiceGrpc.TraceSeg
}
@Override public void onCompleted() {
+ responseObserver.onNext(Downstream.newBuilder().build());
responseObserver.onCompleted();
}
};
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/CommonTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/CommonTable.java
index 4d5669c16..c66f627ad 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/CommonTable.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/CommonTable.java
@@ -4,6 +4,7 @@ package org.skywalking.apm.collector.agentstream.worker;
* @author pengys5
*/
public class CommonTable {
+ public static final String TABLE_TYPE = "type";
public static final String COLUMN_ID = "id";
public static final String COLUMN_AGG = "agg";
public static final String COLUMN_TIME_BUCKET = "time_bucket";
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/TimeSlice.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/TimeSlice.java
deleted file mode 100644
index c75132a0e..000000000
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/TimeSlice.java
+++ /dev/null
@@ -1,28 +0,0 @@
-package org.skywalking.apm.collector.agentstream.worker;
-
-/**
- * @author pengys5
- */
-public abstract class TimeSlice {
- private String sliceType;
- private long startTime;
- private long endTime;
-
- public TimeSlice(String sliceType, long startTime, long endTime) {
- this.startTime = startTime;
- this.endTime = endTime;
- this.sliceType = sliceType;
- }
-
- public String getSliceType() {
- return sliceType;
- }
-
- public long getStartTime() {
- return startTime;
- }
-
- public long getEndTime() {
- return endTime;
- }
-}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggregationWorker.java
similarity index 57%
rename from apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggWorker.java
rename to apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggregationWorker.java
index a3dec152e..85349b10c 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggWorker.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggregationWorker.java
@@ -4,17 +4,20 @@ import org.skywalking.apm.collector.agentstream.worker.node.component.define.Nod
import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider;
import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
+import org.skywalking.apm.collector.stream.worker.Role;
+import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException;
+import org.skywalking.apm.collector.stream.worker.WorkerRefs;
import org.skywalking.apm.collector.stream.worker.impl.AggregationWorker;
import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
-import org.skywalking.apm.collector.stream.worker.selector.RollingSelector;
+import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector;
import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector;
/**
* @author pengys5
*/
-public class NodeComponentAggWorker extends AggregationWorker {
+public class NodeComponentAggregationWorker extends AggregationWorker {
- public NodeComponentAggWorker(Role role, ClusterWorkerContext clusterContext) {
+ public NodeComponentAggregationWorker(Role role, ClusterWorkerContext clusterContext) {
super(role, clusterContext);
}
@@ -22,19 +25,19 @@ public class NodeComponentAggWorker extends AggregationWorker {
super.preStart();
}
- @Override protected void sendToNext() {
-
+ @Override protected WorkerRefs nextWorkRef(String id) throws WorkerNotFoundException {
+ return getClusterContext().lookup(NodeComponentRemoteWorker.WorkerRole.INSTANCE);
}
- public static class Factory extends AbstractLocalAsyncWorkerProvider {
+ public static class Factory extends AbstractLocalAsyncWorkerProvider {
@Override
public Role role() {
- return Role.INSTANCE;
+ return WorkerRole.INSTANCE;
}
@Override
- public NodeComponentAggWorker workerInstance(ClusterWorkerContext clusterContext) {
- return new NodeComponentAggWorker(role(), clusterContext);
+ public NodeComponentAggregationWorker workerInstance(ClusterWorkerContext clusterContext) {
+ return new NodeComponentAggregationWorker(role(), clusterContext);
}
@Override
@@ -43,17 +46,17 @@ public class NodeComponentAggWorker extends AggregationWorker {
}
}
- public enum Role implements org.skywalking.apm.collector.stream.worker.Role {
+ public enum WorkerRole implements Role {
INSTANCE;
@Override
public String roleName() {
- return NodeComponentAggWorker.class.getSimpleName();
+ return NodeComponentAggregationWorker.class.getSimpleName();
}
@Override
public WorkerSelector workerSelector() {
- return new RollingSelector();
+ return new HashCodeSelector();
}
@Override public DataDefine dataDefine() {
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentPersistenceWorker.java
new file mode 100644
index 000000000..2e7ccf0b9
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentPersistenceWorker.java
@@ -0,0 +1,70 @@
+package org.skywalking.apm.collector.agentstream.worker.node.component;
+
+import java.util.List;
+import java.util.Map;
+import org.skywalking.apm.collector.agentstream.worker.node.component.dao.INodeComponentDAO;
+import org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentDataDefine;
+import org.skywalking.apm.collector.storage.dao.DAOContainer;
+import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider;
+import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
+import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
+import org.skywalking.apm.collector.stream.worker.Role;
+import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker;
+import org.skywalking.apm.collector.stream.worker.impl.data.Data;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
+import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector;
+import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector;
+
+/**
+ * @author pengys5
+ */
+public class NodeComponentPersistenceWorker extends PersistenceWorker {
+
+ public NodeComponentPersistenceWorker(Role role, ClusterWorkerContext clusterContext) {
+ super(role, clusterContext);
+ }
+
+ @Override public void preStart() throws ProviderNotFoundException {
+ super.preStart();
+ }
+
+ @Override protected List prepareBatch(Map dataMap) {
+ INodeComponentDAO dao = (INodeComponentDAO)DAOContainer.INSTANCE.get(INodeComponentDAO.class.getName());
+ return dao.prepareBatch(dataMap);
+ }
+
+ public static class Factory extends AbstractLocalAsyncWorkerProvider {
+ @Override
+ public Role role() {
+ return WorkerRole.INSTANCE;
+ }
+
+ @Override
+ public NodeComponentPersistenceWorker workerInstance(ClusterWorkerContext clusterContext) {
+ return new NodeComponentPersistenceWorker(role(), clusterContext);
+ }
+
+ @Override
+ public int queueSize() {
+ return 1024;
+ }
+ }
+
+ public enum WorkerRole implements Role {
+ INSTANCE;
+
+ @Override
+ public String roleName() {
+ return NodeComponentPersistenceWorker.class.getSimpleName();
+ }
+
+ @Override
+ public WorkerSelector workerSelector() {
+ return new HashCodeSelector();
+ }
+
+ @Override public DataDefine dataDefine() {
+ return new NodeComponentDataDefine();
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentRemoteWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentRemoteWorker.java
index 41b9096ce..8ed1e4114 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentRemoteWorker.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentRemoteWorker.java
@@ -1,10 +1,15 @@
package org.skywalking.apm.collector.agentstream.worker.node.component;
+import org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentDataDefine;
import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorker;
+import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorkerProvider;
import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
import org.skywalking.apm.collector.stream.worker.Role;
import org.skywalking.apm.collector.stream.worker.WorkerException;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
+import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector;
+import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector;
/**
* @author pengys5
@@ -20,6 +25,36 @@ public class NodeComponentRemoteWorker extends AbstractRemoteWorker {
}
@Override protected void onWork(Object message) throws WorkerException {
+ getClusterContext().lookup(NodeComponentPersistenceWorker.WorkerRole.INSTANCE).tell(message);
+ }
+ public static class Factory extends AbstractRemoteWorkerProvider {
+ @Override
+ public Role role() {
+ return WorkerRole.INSTANCE;
+ }
+
+ @Override
+ public NodeComponentRemoteWorker workerInstance(ClusterWorkerContext clusterContext) {
+ return new NodeComponentRemoteWorker(role(), clusterContext);
+ }
+ }
+
+ public enum WorkerRole implements Role {
+ INSTANCE;
+
+ @Override
+ public String roleName() {
+ return NodeComponentRemoteWorker.class.getSimpleName();
+ }
+
+ @Override
+ public WorkerSelector workerSelector() {
+ return new HashCodeSelector();
+ }
+
+ @Override public DataDefine dataDefine() {
+ return new NodeComponentDataDefine();
+ }
}
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java
index ee9fca213..615f1bb10 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java
@@ -8,6 +8,11 @@ import org.skywalking.apm.collector.agentstream.worker.segment.EntrySpanListener
import org.skywalking.apm.collector.agentstream.worker.segment.ExitSpanListener;
import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener;
import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils;
+import org.skywalking.apm.collector.core.framework.CollectorContextHelper;
+import org.skywalking.apm.collector.stream.StreamModuleContext;
+import org.skywalking.apm.collector.stream.StreamModuleGroupDefine;
+import org.skywalking.apm.collector.stream.worker.WorkerInvokeException;
+import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException;
import org.skywalking.apm.network.proto.SpanObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -27,13 +32,13 @@ public class NodeComponentSpanListener implements EntrySpanListener, ExitSpanLis
if (spanObject.getPeerId() == 0) {
peers = String.valueOf(spanObject.getPeerId());
}
- String agg = spanObject.getComponent() + Const.ID_SPLIT + peers;
+ String agg = spanObject.getComponentId() + Const.ID_SPLIT + peers;
nodeComponents.add(agg);
}
@Override public void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId) {
String peers = String.valueOf(applicationId);
- String agg = spanObject.getComponent() + Const.ID_SPLIT + peers;
+ String agg = spanObject.getComponentId() + Const.ID_SPLIT + peers;
nodeComponents.add(agg);
}
@@ -42,11 +47,18 @@ public class NodeComponentSpanListener implements EntrySpanListener, ExitSpanLis
}
@Override public void build() {
+ StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME);
for (String agg : nodeComponents) {
NodeComponentDataDefine.NodeComponent nodeComponent = new NodeComponentDataDefine.NodeComponent();
nodeComponent.setId(timeBucket + Const.ID_SPLIT + agg);
nodeComponent.setAgg(agg);
nodeComponent.setTimeBucket(timeBucket);
+ try {
+ logger.debug("send to node component aggregation worker, id: {}", nodeComponent.getId());
+ context.getClusterWorkerContext().lookup(NodeComponentAggregationWorker.WorkerRole.INSTANCE).tell(nodeComponent.transform());
+ } catch (WorkerInvokeException | WorkerNotFoundException e) {
+ logger.error(e.getMessage(), e);
+ }
}
}
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/INodeComponentDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/INodeComponentDAO.java
new file mode 100644
index 000000000..ffd9087fd
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/INodeComponentDAO.java
@@ -0,0 +1,12 @@
+package org.skywalking.apm.collector.agentstream.worker.node.component.dao;
+
+import java.util.List;
+import java.util.Map;
+import org.skywalking.apm.collector.stream.worker.impl.data.Data;
+
+/**
+ * @author pengys5
+ */
+public interface INodeComponentDAO {
+ List prepareBatch(Map dataMap);
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentEsDAO.java
new file mode 100644
index 000000000..97840799f
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentEsDAO.java
@@ -0,0 +1,29 @@
+package org.skywalking.apm.collector.agentstream.worker.node.component.dao;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentTable;
+import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO;
+import org.skywalking.apm.collector.stream.worker.impl.data.Data;
+
+/**
+ * @author pengys5
+ */
+public class NodeComponentEsDAO extends EsDAO implements INodeComponentDAO {
+
+ @Override public List prepareBatch(Map dataMap) {
+ List indexRequestBuilders = new ArrayList<>();
+ dataMap.forEach((id, data) -> {
+ Map source = new HashMap();
+ source.put(NodeComponentTable.COLUMN_AGG, data.getDataString(1));
+ source.put(NodeComponentTable.COLUMN_TIME_BUCKET, data.getDataLong(0));
+
+ IndexRequestBuilder builder = getClient().prepareIndex(NodeComponentTable.TABLE, id).setSource();
+ indexRequestBuilders.add(builder);
+ });
+ return indexRequestBuilders;
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentH2DAO.java
new file mode 100644
index 000000000..c581fc5ed
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentH2DAO.java
@@ -0,0 +1,15 @@
+package org.skywalking.apm.collector.agentstream.worker.node.component.dao;
+
+import java.util.List;
+import java.util.Map;
+import org.skywalking.apm.collector.storage.h2.dao.H2DAO;
+
+/**
+ * @author pengys5
+ */
+public class NodeComponentH2DAO extends H2DAO implements INodeComponentDAO {
+
+ @Override public List prepareBatch(Map map) {
+ return null;
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java
index 61941347b..caa691ce9 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java
@@ -3,7 +3,9 @@ package org.skywalking.apm.collector.agentstream.worker.node.component.define;
import org.skywalking.apm.collector.remote.grpc.proto.RemoteData;
import org.skywalking.apm.collector.stream.worker.impl.data.Attribute;
import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType;
+import org.skywalking.apm.collector.stream.worker.impl.data.Data;
import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine;
+import org.skywalking.apm.collector.stream.worker.impl.data.TransformToData;
import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation;
import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation;
@@ -34,7 +36,7 @@ public class NodeComponentDataDefine extends DataDefine {
return null;
}
- public static class NodeComponent {
+ public static class NodeComponent implements TransformToData {
private String id;
private String agg;
private long timeBucket;
@@ -48,6 +50,15 @@ public class NodeComponentDataDefine extends DataDefine {
public NodeComponent() {
}
+ @Override public Data transform() {
+ NodeComponentDataDefine define = new NodeComponentDataDefine();
+ Data data = define.build(id);
+ data.setDataString(0, this.id);
+ data.setDataString(1, this.agg);
+ data.setDataLong(0, this.timeBucket);
+ return data;
+ }
+
public String getId() {
return id;
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentEsTableDefine.java
index 700393edd..e4e8e2809 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentEsTableDefine.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentEsTableDefine.java
@@ -13,7 +13,7 @@ public class NodeComponentEsTableDefine extends ElasticSearchTableDefine {
}
@Override public int refreshInterval() {
- return 0;
+ return 2;
}
@Override public int numberOfShards() {
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java
index 6470d4520..e14afa7b2 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java
@@ -19,7 +19,7 @@ public class InstanceDataDefine extends DataDefine {
}
@Override protected int initialCapacity() {
- return 3;
+ return 6;
}
@Override protected void attributeDefine() {
@@ -28,7 +28,7 @@ public class InstanceDataDefine extends DataDefine {
addAttribute(2, new Attribute(InstanceTable.COLUMN_AGENTUUID, AttributeType.STRING, new CoverOperation()));
addAttribute(3, new Attribute(InstanceTable.COLUMN_REGISTER_TIME, AttributeType.LONG, new CoverOperation()));
addAttribute(4, new Attribute(InstanceTable.COLUMN_INSTANCE_ID, AttributeType.INTEGER, new CoverOperation()));
- addAttribute(4, new Attribute(InstanceTable.COLUMN_HEARTBEAT_TIME, AttributeType.LONG, new CoverOperation()));
+ addAttribute(5, new Attribute(InstanceTable.COLUMN_HEARTBEAT_TIME, AttributeType.LONG, new CoverOperation()));
}
@Override public Object deserialize(RemoteData remoteData) {
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceH2DAO.java
index d3bda6437..64070090e 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceH2DAO.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceH2DAO.java
@@ -22,4 +22,8 @@ public class InstanceH2DAO extends H2DAO implements IInstanceDAO {
@Override public void save(InstanceDataDefine.Instance instance) {
}
+
+ @Override public void updateHeartbeatTime(int instanceId, long heartbeatTime) {
+
+ }
}
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java
index 2e6168b8e..7ddebb9d6 100644
--- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java
@@ -63,6 +63,13 @@ public class SegmentParse {
}
}
}
+
+ notifyListenerToBuild();
+ }
+
+ private void notifyListenerToBuild() {
+ spanListeners.forEach(listener -> listener.build());
+ refsListeners.forEach(listener -> listener.build());
}
private void notifyExitListener(SpanObject spanObject, int applicationId, int applicationInstanceId) {
diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java
new file mode 100644
index 000000000..8e72282dd
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java
@@ -0,0 +1,55 @@
+package org.skywalking.apm.collector.agentstream.worker.storage;
+
+import java.util.List;
+import org.skywalking.apm.collector.core.framework.Starter;
+import org.skywalking.apm.collector.storage.dao.DAOContainer;
+import org.skywalking.apm.collector.storage.dao.IBatchDAO;
+import org.skywalking.apm.collector.stream.worker.WorkerException;
+import org.skywalking.apm.collector.stream.worker.impl.FlushAndSwitch;
+import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker;
+import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorkerContainer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class PersistenceTimer implements Starter {
+
+ private final Logger logger = LoggerFactory.getLogger(PersistenceTimer.class);
+
+ public void start() {
+ logger.info("persistence timer start");
+ //TODO timer value config
+// final long timeInterval = EsConfig.Es.Persistence.Timer.VALUE * 1000;
+ final long timeInterval = 3 * 1000;
+
+ Runnable runnable = () -> {
+ while (true) {
+ try {
+ extractDataAndSave();
+ Thread.sleep(timeInterval);
+ } catch (Throwable e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+ };
+ Thread persistenceThread = new Thread(runnable);
+ persistenceThread.setName("timerPersistence");
+ persistenceThread.start();
+ }
+
+ private void extractDataAndSave() {
+ List workers = PersistenceWorkerContainer.INSTANCE.getPersistenceWorkers();
+ workers.forEach(worker -> {
+ try {
+ worker.allocateJob(new FlushAndSwitch());
+ List batchCollection = worker.buildBatchCollection();
+ IBatchDAO dao = (IBatchDAO)DAOContainer.INSTANCE.get(IBatchDAO.class.getName());
+ dao.batchPersistence(batchCollection);
+ } catch (WorkerException e) {
+ logger.error(e.getMessage(), e);
+ }
+ });
+ }
+}
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define
index ad40a6443..c35382f07 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define
@@ -1,3 +1,4 @@
org.skywalking.apm.collector.agentstream.worker.register.application.dao.ApplicationEsDAO
org.skywalking.apm.collector.agentstream.worker.register.instance.dao.InstanceEsDAO
-org.skywalking.apm.collector.agentstream.worker.register.servicename.dao.ServiceNameEsDAO
\ No newline at end of file
+org.skywalking.apm.collector.agentstream.worker.register.servicename.dao.ServiceNameEsDAO
+org.skywalking.apm.collector.agentstream.worker.node.component.dao.NodeComponentEsDAO
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define
index 1366f2d92..1fda64fb7 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define
@@ -1,3 +1,4 @@
org.skywalking.apm.collector.agentstream.worker.register.application.dao.ApplicationH2DAO
org.skywalking.apm.collector.agentstream.worker.register.instance.dao.InstanceH2DAO
-org.skywalking.apm.collector.agentstream.worker.register.servicename.dao.ServiceNameH2DAO
\ No newline at end of file
+org.skywalking.apm.collector.agentstream.worker.register.servicename.dao.ServiceNameH2DAO
+org.skywalking.apm.collector.agentstream.worker.node.component.dao.NodeComponentH2DAO
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define
index 7acdd7e09..5e84c08ca 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define
@@ -1,4 +1,5 @@
-org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentAggWorker$Factory
+org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentAggregationWorker$Factory
+org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentPersistenceWorker$Factory
org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterSerialWorker$Factory
org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceRegisterSerialWorker$Factory
org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameRegisterSerialWorker$Factory
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define
index 2b5f3709e..9fb6fc70c 100644
--- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define
+++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define
@@ -1,3 +1,5 @@
org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterRemoteWorker$Factory
org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceRegisterRemoteWorker$Factory
-org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameRegisterRemoteWorker$Factory
\ No newline at end of file
+org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameRegisterRemoteWorker$Factory
+
+org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentRemoteWorker$Factory
\ No newline at end of file
diff --git a/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandlerTestCase.java b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandlerTestCase.java
new file mode 100644
index 000000000..4ecfbea07
--- /dev/null
+++ b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandlerTestCase.java
@@ -0,0 +1,92 @@
+package org.skywalking.apm.collector.agentstream.grpc.handler;
+
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import io.grpc.stub.StreamObserver;
+import org.junit.Test;
+import org.skywalking.apm.network.proto.Downstream;
+import org.skywalking.apm.network.proto.SpanLayer;
+import org.skywalking.apm.network.proto.SpanObject;
+import org.skywalking.apm.network.proto.SpanType;
+import org.skywalking.apm.network.proto.TraceSegmentObject;
+import org.skywalking.apm.network.proto.TraceSegmentServiceGrpc;
+import org.skywalking.apm.network.proto.UniqueId;
+import org.skywalking.apm.network.proto.UpstreamSegment;
+import org.skywalking.apm.network.trace.component.ComponentsDefine;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class TraceSegmentServiceHandlerTestCase {
+
+ private final Logger logger = LoggerFactory.getLogger(TraceSegmentServiceHandlerTestCase.class);
+
+ private TraceSegmentServiceGrpc.TraceSegmentServiceStub stub;
+
+ @Test
+ public void testCollect() {
+ ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 11800).usePlaintext(true).build();
+ stub = TraceSegmentServiceGrpc.newStub(channel);
+
+ StreamObserver streamObserver = stub.collect(new StreamObserver() {
+ @Override public void onNext(Downstream downstream) {
+ }
+
+ @Override public void onError(Throwable throwable) {
+ logger.error(throwable.getMessage(), throwable);
+ }
+
+ @Override public void onCompleted() {
+
+ }
+ });
+
+ UpstreamSegment.Builder builder = UpstreamSegment.newBuilder();
+ buildGlobalTraceIds(builder);
+ buildSegment(builder);
+
+ streamObserver.onNext(builder.build());
+ streamObserver.onCompleted();
+
+ try {
+ Thread.sleep(30000);
+ } catch (InterruptedException e) {
+ }
+ }
+
+ private void buildGlobalTraceIds(UpstreamSegment.Builder builder) {
+ UniqueId.Builder builder1 = UniqueId.newBuilder();
+ builder1.addIdParts(100);
+ builder1.addIdParts(100);
+ builder1.addIdParts(100);
+ builder.addGlobalTraceIds(builder1.build());
+ }
+
+ private void buildSegment(UpstreamSegment.Builder builder) {
+ long now = System.currentTimeMillis();
+
+ TraceSegmentObject.Builder segmentBuilder = TraceSegmentObject.newBuilder();
+ segmentBuilder.setApplicationId(1);
+ segmentBuilder.setApplicationInstanceId(1);
+ segmentBuilder.setTraceSegmentId(UniqueId.newBuilder().addIdParts(200).addIdParts(200).addIdParts(200).build());
+
+ SpanObject.Builder span_0 = SpanObject.newBuilder();
+ span_0.setSpanId(0);
+ span_0.setOperationName("/dubbox-case/case/dubbox-rest");
+ span_0.setOperationNameId(0);
+ span_0.setParentSpanId(-1);
+ span_0.setSpanLayer(SpanLayer.Http);
+ span_0.setStartTime(now);
+ span_0.setEndTime(now + 100000);
+ span_0.setComponentId(ComponentsDefine.TOMCAT.getId());
+ span_0.setIsError(false);
+ span_0.setSpanType(SpanType.Entry);
+ span_0.setPeerId(0);
+ span_0.setPeer("localhost:8080");
+ segmentBuilder.addSpans(span_0);
+
+ builder.setSegment(segmentBuilder.build().toByteString());
+ }
+}
diff --git a/apm-collector/apm-collector-client/pom.xml b/apm-collector/apm-collector-client/pom.xml
index 5fe914473..1f6b17431 100644
--- a/apm-collector/apm-collector-client/pom.xml
+++ b/apm-collector/apm-collector-client/pom.xml
@@ -52,10 +52,6 @@
slf4j-log4j12
org.slf4j
-
- log4j
- log4j
-
diff --git a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java
index 27218ce35..847f5e7cf 100644
--- a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java
+++ b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java
@@ -8,6 +8,7 @@ import java.util.concurrent.ExecutionException;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.update.UpdateRequest;
@@ -112,6 +113,10 @@ public class ElasticSearchClient implements Client {
return client.prepareIndex(indexName, "type", id);
}
+ public BulkRequestBuilder prepareBulk() {
+ return client.prepareBulk();
+ }
+
public void update(UpdateRequest updateRequest) {
try {
client.update(updateRequest).get();
diff --git a/apm-collector/apm-collector-core/src/main/resources/logback.xml b/apm-collector/apm-collector-core/src/main/resources/logback.xml
index 687182620..78debf076 100644
--- a/apm-collector/apm-collector-core/src/main/resources/logback.xml
+++ b/apm-collector/apm-collector-core/src/main/resources/logback.xml
@@ -2,7 +2,7 @@
- %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n
+ %d{HH:mm:ss.SSS} [%thread] %-5level %logger{70} - %msg%n
diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/IBatchDAO.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/IBatchDAO.java
new file mode 100644
index 000000000..afcac1bf1
--- /dev/null
+++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/IBatchDAO.java
@@ -0,0 +1,10 @@
+package org.skywalking.apm.collector.storage.dao;
+
+import java.util.List;
+
+/**
+ * @author pengys5
+ */
+public interface IBatchDAO {
+ void batchPersistence(List batchCollection);
+}
diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/BatchEsDAO.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/BatchEsDAO.java
new file mode 100644
index 000000000..74c06e634
--- /dev/null
+++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/BatchEsDAO.java
@@ -0,0 +1,35 @@
+package org.skywalking.apm.collector.storage.elasticsearch.dao;
+
+import java.util.List;
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.skywalking.apm.collector.core.util.CollectionUtils;
+import org.skywalking.apm.collector.storage.dao.IBatchDAO;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public class BatchEsDAO extends EsDAO implements IBatchDAO {
+
+ private final Logger logger = LoggerFactory.getLogger(BatchEsDAO.class);
+
+ @Override public void batchPersistence(List batchCollection) {
+ BulkRequestBuilder bulkRequest = getClient().prepareBulk();
+
+ logger.info("bulk data size: {}", batchCollection.size());
+ if (CollectionUtils.isNotEmpty(batchCollection)) {
+ for (int i = 0; i < batchCollection.size(); i++) {
+ IndexRequestBuilder builder = (IndexRequestBuilder)batchCollection.get(i);
+ bulkRequest.add(builder);
+ }
+
+ BulkResponse bulkResponse = bulkRequest.execute().actionGet();
+ if (bulkResponse.hasFailures()) {
+ logger.error(bulkResponse.buildFailureMessage());
+ }
+ }
+ }
+}
diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/BatchH2DAO.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/BatchH2DAO.java
new file mode 100644
index 000000000..27b69f6d4
--- /dev/null
+++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/BatchH2DAO.java
@@ -0,0 +1,14 @@
+package org.skywalking.apm.collector.storage.h2.dao;
+
+import java.util.List;
+import org.skywalking.apm.collector.storage.dao.IBatchDAO;
+
+/**
+ * @author pengys5
+ */
+public class BatchH2DAO extends H2DAO implements IBatchDAO {
+
+ @Override public void batchPersistence(List batchCollection) {
+
+ }
+}
diff --git a/apm-collector/apm-collector-storage/src/main/resources/META-INF/defines/es_dao.define b/apm-collector/apm-collector-storage/src/main/resources/META-INF/defines/es_dao.define
new file mode 100644
index 000000000..1fcbc9a03
--- /dev/null
+++ b/apm-collector/apm-collector-storage/src/main/resources/META-INF/defines/es_dao.define
@@ -0,0 +1 @@
+org.skywalking.apm.collector.storage.elasticsearch.dao.BatchEsDAO
\ No newline at end of file
diff --git a/apm-collector/apm-collector-storage/src/main/resources/META-INF/defines/h2_dao.define b/apm-collector/apm-collector-storage/src/main/resources/META-INF/defines/h2_dao.define
new file mode 100644
index 000000000..de143481d
--- /dev/null
+++ b/apm-collector/apm-collector-storage/src/main/resources/META-INF/defines/h2_dao.define
@@ -0,0 +1 @@
+org.skywalking.apm.collector.storage.h2.dao.BatchH2DAO
\ No newline at end of file
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java
index a714b6fdd..908eecbb0 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java
@@ -66,7 +66,6 @@ public class StreamModuleInstaller implements ModuleInstaller {
List remoteProviders = remoteProviderLoader.load();
for (AbstractRemoteWorkerProvider provider : remoteProviders) {
provider.setClusterContext(clusterWorkerContext);
-// provider.create();
clusterWorkerContext.putRole(provider.role());
clusterWorkerContext.putProvider(provider);
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java
index c7e4ff5c2..094b960cb 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java
@@ -6,6 +6,8 @@ import org.skywalking.apm.collector.core.queue.QueueEventHandler;
import org.skywalking.apm.collector.core.queue.QueueExecutor;
import org.skywalking.apm.collector.queue.QueueModuleContext;
import org.skywalking.apm.collector.queue.QueueModuleGroupDefine;
+import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker;
+import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorkerContainer;
/**
* @author pengys5
@@ -18,6 +20,10 @@ public abstract class AbstractLocalAsyncWorkerProvider remoteWorkerRefs;
private Map> roleWorkers;
private Map roles;
@@ -56,6 +60,7 @@ public abstract class WorkerContext implements Context {
}
@Override final public void put(WorkerRef workerRef) {
+ logger.debug("put worker reference into context, role name: {}", workerRef.getRole().roleName());
if (!getRoleWorkers().containsKey(workerRef.getRole().roleName())) {
getRoleWorkers().putIfAbsent(workerRef.getRole().roleName(), new ArrayList<>());
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/AggregationWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/AggregationWorker.java
index 222caa4b6..6d9dff63f 100644
--- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/AggregationWorker.java
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/AggregationWorker.java
@@ -6,23 +6,29 @@ import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
import org.skywalking.apm.collector.stream.worker.Role;
import org.skywalking.apm.collector.stream.worker.WorkerException;
+import org.skywalking.apm.collector.stream.worker.WorkerInvokeException;
+import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException;
+import org.skywalking.apm.collector.stream.worker.WorkerRefs;
import org.skywalking.apm.collector.stream.worker.impl.data.Data;
import org.skywalking.apm.collector.stream.worker.impl.data.DataCache;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* @author pengys5
*/
public abstract class AggregationWorker extends AbstractLocalAsyncWorker {
+ private final Logger logger = LoggerFactory.getLogger(AggregationWorker.class);
+
private DataCache dataCache;
+ private int messageNum;
public AggregationWorker(Role role, ClusterWorkerContext clusterContext) {
super(role, clusterContext);
dataCache = new DataCache();
}
- private int messageNum;
-
@Override public void preStart() throws ProviderNotFoundException {
super.preStart();
}
@@ -41,14 +47,34 @@ public abstract class AggregationWorker extends AbstractLocalAsyncWorker {
}
}
- protected abstract void sendToNext();
+ protected abstract WorkerRefs nextWorkRef(String id) throws WorkerNotFoundException;
+
+ private void sendToNext() throws WorkerException {
+ dataCache.switchPointer();
+ while (dataCache.getLast().isHolding()) {
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ throw new WorkerException(e.getMessage(), e);
+ }
+ }
+ dataCache.getLast().asMap().forEach((id, data) -> {
+ try {
+ nextWorkRef(id).tell(data);
+ } catch (WorkerNotFoundException | WorkerInvokeException e) {
+ logger.error(e.getMessage(), e);
+ }
+ });
+ }
protected final void aggregate(Object message) {
Data data = (Data)message;
+ dataCache.hold();
if (dataCache.containsKey(data.id())) {
getClusterContext().getDataDefine(data.getDefineId()).mergeData(data, dataCache.get(data.id()));
} else {
dataCache.put(data.id(), data);
}
+ dataCache.release();
}
}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/FlushAndSwitch.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/FlushAndSwitch.java
new file mode 100644
index 000000000..15b8ee037
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/FlushAndSwitch.java
@@ -0,0 +1,7 @@
+package org.skywalking.apm.collector.stream.worker.impl;
+
+/**
+ * @author pengys5
+ */
+public class FlushAndSwitch {
+}
diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/PersistenceWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/PersistenceWorker.java
new file mode 100644
index 000000000..c32e676de
--- /dev/null
+++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/PersistenceWorker.java
@@ -0,0 +1,82 @@
+package org.skywalking.apm.collector.stream.worker.impl;
+
+import java.util.List;
+import java.util.Map;
+import org.skywalking.apm.collector.core.queue.EndOfBatchCommand;
+import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorker;
+import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext;
+import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException;
+import org.skywalking.apm.collector.stream.worker.Role;
+import org.skywalking.apm.collector.stream.worker.WorkerException;
+import org.skywalking.apm.collector.stream.worker.impl.data.Data;
+import org.skywalking.apm.collector.stream.worker.impl.data.DataCache;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author pengys5
+ */
+public abstract class PersistenceWorker extends AbstractLocalAsyncWorker {
+
+ private final Logger logger = LoggerFactory.getLogger(PersistenceWorker.class);
+
+ private DataCache dataCache;
+
+ public PersistenceWorker(Role role, ClusterWorkerContext clusterContext) {
+ super(role, clusterContext);
+ dataCache = new DataCache();
+ }
+
+ @Override public void preStart() throws ProviderNotFoundException {
+ super.preStart();
+ }
+
+ @Override protected final void onWork(Object message) throws WorkerException {
+ if (message instanceof EndOfBatchCommand || message instanceof FlushAndSwitch) {
+ if (dataCache.trySwitchPointer()) {
+ dataCache.switchPointer();
+ }
+ } else {
+ if (dataCache.currentCollectionSize() >= 1000) {
+ if (dataCache.trySwitchPointer()) {
+ dataCache.switchPointer();
+ }
+ }
+ aggregate(message);
+ }
+ }
+
+ public List buildBatchCollection() throws WorkerException {
+ List batchCollection;
+ try {
+ while (dataCache.getLast().isHolding()) {
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ logger.warn("thread wake up");
+ }
+ }
+ batchCollection = prepareBatch(dataCache.getLast().asMap());
+ } finally {
+ dataCache.releaseLast();
+ }
+ return batchCollection;
+ }
+
+ protected abstract List prepareBatch(Map